diff src/org/tmatesoft/hg/internal/InflaterDataAccess.java @ 576:3c4db86e8c1f

Issue 43: poor performance with InflaterDataAccess. Phase 2: inflate into buffer, effective skip and readByte/readBytes()
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Tue, 16 Apr 2013 19:31:57 +0200
parents 8bf184c9d733
children ed243b668502
line wrap: on
line diff
--- a/src/org/tmatesoft/hg/internal/InflaterDataAccess.java	Tue Apr 16 16:59:59 2013 +0200
+++ b/src/org/tmatesoft/hg/internal/InflaterDataAccess.java	Tue Apr 16 19:31:57 2013 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2012 TMate Software Ltd
+ * Copyright (c) 2011-2013 TMate Software Ltd
  *  
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -16,8 +16,8 @@
  */
 package org.tmatesoft.hg.internal;
 
-import java.io.EOFException;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 import java.util.zip.ZipException;
@@ -32,9 +32,9 @@
 public class InflaterDataAccess extends FilterDataAccess {
 
 	private final Inflater inflater;
-	private final byte[] buffer;
-	private final byte[] singleByte = new byte[1];
-	private int decompressedPos = 0;
+	private final byte[] inBuffer;
+	private final ByteBuffer outBuffer;
+	private int inflaterPos = 0;
 	private int decompressedLength;
 
 	public InflaterDataAccess(DataAccess dataAccess, long offset, int compressedLength) {
@@ -52,20 +52,23 @@
 		}
 		this.inflater = inflater;
 		this.decompressedLength = actualLength;
-		buffer = buf;
+		inBuffer = buf;
+		outBuffer = ByteBuffer.allocate(inBuffer.length * 2);
+		outBuffer.limit(0); // there's nothing to read in the buffer 
 	}
 	
 	@Override
 	public InflaterDataAccess reset() throws IOException {
 		super.reset();
 		inflater.reset();
-		decompressedPos = 0;
+		inflaterPos = 0;
+		outBuffer.clear().limit(0); // or flip(), to indicate nothing to read
 		return this;
 	}
 	
 	@Override
 	protected int available() throws IOException {
-		return length() - decompressedPos;
+		return length() - decompressedPosition();
 	}
 	
 	@Override
@@ -80,29 +83,16 @@
 		if (decompressedLength != -1) {
 			return decompressedLength;
 		}
-		decompressedLength = 0; // guard to avoid endless loop in case length() would get invoked from below. 
-		int c = 0;
-		int oldPos = decompressedPos;
-		byte[] dummy = new byte[buffer.length];
-		try {
-			int toRead = -1;
-			do {
-				while (!inflater.needsInput()) {
-					c += inflater.inflate(dummy, 0, dummy.length);
-				}
-				if (inflater.needsInput() && (toRead = super.available()) > 0) {
-					// fill:
-					if (toRead > buffer.length) {
-						toRead = buffer.length;
-					}
-					super.readBytes(buffer, 0, toRead);
-					inflater.setInput(buffer, 0, toRead);
-				}
-			} while(toRead > 0);
-		} catch (DataFormatException ex) {
-			throw new IOException(ex.toString());
-		}
-		decompressedLength = c + oldPos;
+		decompressedLength = 0; // guard to avoid endless loop in case length() would get invoked from below.
+		final int oldPos = decompressedPosition();
+		final int inflatedUpTo = inflaterPos;
+		int inflatedMore = 0, c;
+		do {
+			outBuffer.limit(outBuffer.position()); // pretend the buffer is consumed
+			c = fillOutBuffer();
+			inflatedMore += c;
+		} while (c == outBuffer.capacity()); // once we unpacked less than capacity, input is over
+		decompressedLength = inflatedUpTo + inflatedMore;
 		reset();
 		seek(oldPos);
 		return decompressedLength;
@@ -113,8 +103,9 @@
 		if (localOffset < 0 /* || localOffset >= length() */) {
 			throw new IllegalArgumentException();
 		}
-		if (localOffset >= decompressedPos) {
-			skip(localOffset - decompressedPos);
+		int currentPos = decompressedPosition();
+		if (localOffset >= currentPos) {
+			skip(localOffset - currentPos);
 		} else {
 			reset();
 			skip(localOffset);
@@ -125,57 +116,122 @@
 	public void skip(final int bytesToSkip) throws IOException {
 		int bytes = bytesToSkip;
 		if (bytes < 0) {
-			bytes += decompressedPos;
+			bytes += decompressedPosition();
 			if (bytes < 0) {
-				throw new IOException(String.format("Underflow. Rewind past start of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, decompressedPos, decompressedLength, bytes));
+				throw new IOException(String.format("Underflow. Rewind past start of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, inflaterPos, decompressedLength, bytes));
 			}
 			reset();
 			// fall-through
 		}
 		while (!isEmpty() && bytes > 0) {
-			readByte();
-			bytes--;
+			int fromBuffer = outBuffer.remaining();
+			if (fromBuffer > 0) {
+				if (fromBuffer >= bytes) {
+					outBuffer.position(outBuffer.position() + bytes);
+					bytes = 0;
+					break;
+				} else {
+					bytes -= fromBuffer;
+					outBuffer.limit(outBuffer.position()); // mark consumed
+					// fall through to fill the buffer
+				}
+			}
+			fillOutBuffer();
 		}
 		if (bytes != 0) {
-			throw new IOException(String.format("Underflow. Rewind past end of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, decompressedPos, decompressedLength, bytes));
+			throw new IOException(String.format("Underflow. Rewind past end of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, inflaterPos, decompressedLength, bytes));
 		}
 	}
 
 	@Override
 	public byte readByte() throws IOException {
-		readBytes(singleByte, 0, 1);
-		return singleByte[0];
+		if (!outBuffer.hasRemaining()) {
+			fillOutBuffer();
+		}
+		return outBuffer.get();
 	}
 
 	@Override
 	public void readBytes(byte[] b, int off, int len) throws IOException {
+		do {
+			int fromBuffer = outBuffer.remaining();
+			if (fromBuffer > 0) {
+				if (fromBuffer >= len) {
+					outBuffer.get(b, off, len);
+					return;
+				} else {
+					outBuffer.get(b, off, fromBuffer);
+					off += fromBuffer;
+					len -= fromBuffer;
+					// fall-through
+				}
+			}
+			fillOutBuffer();
+		} while (len > 0);
+	}
+	
+	@Override
+	public void readBytes(ByteBuffer buf) throws IOException {
+		int len = Math.min(available(), buf.remaining());
+		while (len > 0) {
+			if (outBuffer.remaining() >= len) {
+				ByteBuffer slice = outBuffer.slice();
+				slice.limit(len);
+				buf.put(slice);
+				outBuffer.position(outBuffer.position() + len);
+				return;
+			} else { 
+				len -= outBuffer.remaining();
+				buf.put(outBuffer);
+			}
+			fillOutBuffer();
+		}
+	}
+	
+	private int decompressedPosition() {
+		assert outBuffer.remaining() <= inflaterPos; 
+		return inflaterPos - outBuffer.remaining();
+	}
+	
+	// after #fillOutBuffer(), outBuffer is ready for read
+	private int fillOutBuffer() throws IOException {
+		assert !outBuffer.hasRemaining();
 		try {
-		    int n;
-		    while (len > 0) {
-			    while ((n = inflater.inflate(b, off, len)) == 0) {
+			int inflatedBytes = 0;
+		    outBuffer.clear();
+		    int len = outBuffer.capacity();
+		    int off = 0;
+		    do {
+			    int n;
+			    while ((n = inflater.inflate(outBuffer.array(), off, len)) == 0) {
 			    	// XXX few last bytes (checksum?) may be ignored by inflater, thus inflate may return 0 in
 			    	// perfectly legal conditions (when all data already expanded, but there are still some bytes
 			    	// in the input stream)
 					int toRead = -1;
 					if (inflater.needsInput() && (toRead = super.available()) > 0) {
-						// fill:
-						if (toRead > buffer.length) {
-							toRead = buffer.length;
+						// fill
+						if (toRead > inBuffer.length) {
+							toRead = inBuffer.length;
 						}
-						super.readBytes(buffer, 0, toRead);
-						inflater.setInput(buffer, 0, toRead);
+						super.readBytes(inBuffer, 0, toRead);
+						inflater.setInput(inBuffer, 0, toRead);
 					} else {
+						// inflated nothing and doesn't want any more data (or no date available) - assume we're done 
+						assert inflater.finished();
+						assert toRead <= 0;
+						break;
 						// prevent hang up in this cycle if no more data is available, see Issue 25
-						throw new EOFException(String.format("No more compressed data is available to satisfy request for %d bytes. [finished:%b, needDict:%b, needInp:%b, available:%d", len, inflater.finished(), inflater.needsDictionary(), inflater.needsInput(), toRead));
+//						throw new EOFException(String.format("No more compressed data is available to satisfy request for %d bytes. [finished:%b, needDict:%b, needInp:%b, available:%d", len, inflater.finished(), inflater.needsDictionary(), inflater.needsInput(), toRead));
 					}
 			    }
 				off += n;
 				len -= n;
-				decompressedPos += n;
-				if (len == 0) {
-					return; // filled
-				}
-		    }
+				inflatedBytes += n;
+		    } while (len > 0 && !inflater.finished()); // either the buffer is filled or nothing more to unpack
+		    inflaterPos += inflatedBytes;
+		    outBuffer.limit(inflatedBytes);
+		    assert outBuffer.position() == 0; // didn't change since #clear() above
+		    return inflatedBytes;
 		} catch (DataFormatException e) {
 		    String s = e.getMessage();
 		    throw new ZipException(s != null ? s : "Invalid ZLIB data format");