Mercurial > hg4j

comparison src/org/tmatesoft/hg/internal/InflaterDataAccess.java @ 576:3c4db86e8c1f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue 43: poor performance with InflaterDataAccess. Phase 2: inflate into buffer, effective skip and readByte/readBytes()
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Tue, 16 Apr 2013 19:31:57 +0200
parents 8bf184c9d733
children ed243b668502
comparison
equal deleted inserted replaced
575:8bf184c9d733 576:3c4db86e8c1f
1 /* 1 /*
2 * Copyright (c) 2011-2012 TMate Software Ltd 2 * Copyright (c) 2011-2013 TMate Software Ltd
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by 5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; version 2 of the License. 6 * the Free Software Foundation; version 2 of the License.
7 * 7 *
14 * the terms of a license other than GNU General Public License 14 * the terms of a license other than GNU General Public License
15 * contact TMate Software at support@hg4j.com 15 * contact TMate Software at support@hg4j.com
16 */ 16 */
17 package org.tmatesoft.hg.internal; 17 package org.tmatesoft.hg.internal;
18 18
19 import java.io.EOFException;
20 import java.io.IOException; 19 import java.io.IOException;
20 import java.nio.ByteBuffer;
21 import java.util.zip.DataFormatException; 21 import java.util.zip.DataFormatException;
22 import java.util.zip.Inflater; 22 import java.util.zip.Inflater;
23 import java.util.zip.ZipException; 23 import java.util.zip.ZipException;
24 24
25 /** 25 /**
30 * @author TMate Software Ltd. 30 * @author TMate Software Ltd.
31 */ 31 */
32 public class InflaterDataAccess extends FilterDataAccess { 32 public class InflaterDataAccess extends FilterDataAccess {
33 33
34 private final Inflater inflater; 34 private final Inflater inflater;
35 private final byte[] buffer; 35 private final byte[] inBuffer;
36 private final byte[] singleByte = new byte[1]; 36 private final ByteBuffer outBuffer;
37 private int decompressedPos = 0; 37 private int inflaterPos = 0;
38 private int decompressedLength; 38 private int decompressedLength;
39 39
40 public InflaterDataAccess(DataAccess dataAccess, long offset, int compressedLength) { 40 public InflaterDataAccess(DataAccess dataAccess, long offset, int compressedLength) {
41 this(dataAccess, offset, compressedLength, -1, new Inflater(), new byte[512]); 41 this(dataAccess, offset, compressedLength, -1, new Inflater(), new byte[512]);
42 } 42 }
50 if (inflater == null || buf == null) { 50 if (inflater == null || buf == null) {
51 throw new IllegalArgumentException(); 51 throw new IllegalArgumentException();
52 } 52 }
53 this.inflater = inflater; 53 this.inflater = inflater;
54 this.decompressedLength = actualLength; 54 this.decompressedLength = actualLength;
55 buffer = buf; 55 inBuffer = buf;
56 outBuffer = ByteBuffer.allocate(inBuffer.length * 2);
57 outBuffer.limit(0); // there's nothing to read in the buffer
56 } 58 }
57 59
58 @Override 60 @Override
59 public InflaterDataAccess reset() throws IOException { 61 public InflaterDataAccess reset() throws IOException {
60 super.reset(); 62 super.reset();
61 inflater.reset(); 63 inflater.reset();
62 decompressedPos = 0; 64 inflaterPos = 0;
65 outBuffer.clear().limit(0); // or flip(), to indicate nothing to read
63 return this; 66 return this;
64 } 67 }
65 68
66 @Override 69 @Override
67 protected int available() throws IOException { 70 protected int available() throws IOException {
68 return length() - decompressedPos; 71 return length() - decompressedPosition();
69 } 72 }
70 73
71 @Override 74 @Override
72 public boolean isEmpty() throws IOException { 75 public boolean isEmpty() throws IOException {
73 // can't use super.available() <= 0 because even when 0 < super.count < 6(?) 76 // can't use super.available() <= 0 because even when 0 < super.count < 6(?)
78 @Override 81 @Override
79 public int length() throws IOException { 82 public int length() throws IOException {
80 if (decompressedLength != -1) { 83 if (decompressedLength != -1) {
81 return decompressedLength; 84 return decompressedLength;
82 } 85 }
83 decompressedLength = 0; // guard to avoid endless loop in case length() would get invoked from below. 86 decompressedLength = 0; // guard to avoid endless loop in case length() would get invoked from below.
84 int c = 0; 87 final int oldPos = decompressedPosition();
85 int oldPos = decompressedPos; 88 final int inflatedUpTo = inflaterPos;
86 byte[] dummy = new byte[buffer.length]; 89 int inflatedMore = 0, c;
87 try { 90 do {
88 int toRead = -1; 91 outBuffer.limit(outBuffer.position()); // pretend the buffer is consumed
89 do { 92 c = fillOutBuffer();
90 while (!inflater.needsInput()) { 93 inflatedMore += c;
91 c += inflater.inflate(dummy, 0, dummy.length); 94 } while (c == outBuffer.capacity()); // once we unpacked less than capacity, input is over
92 } 95 decompressedLength = inflatedUpTo + inflatedMore;
93 if (inflater.needsInput() && (toRead = super.available()) > 0) {
94 // fill:
95 if (toRead > buffer.length) {
96 toRead = buffer.length;
97 }
98 super.readBytes(buffer, 0, toRead);
99 inflater.setInput(buffer, 0, toRead);
100 }
101 } while(toRead > 0);
102 } catch (DataFormatException ex) {
103 throw new IOException(ex.toString());
104 }
105 decompressedLength = c + oldPos;
106 reset(); 96 reset();
107 seek(oldPos); 97 seek(oldPos);
108 return decompressedLength; 98 return decompressedLength;
109 } 99 }
110 100
111 @Override 101 @Override
112 public void seek(int localOffset) throws IOException { 102 public void seek(int localOffset) throws IOException {
113 if (localOffset < 0 /* || localOffset >= length() */) { 103 if (localOffset < 0 /* || localOffset >= length() */) {
114 throw new IllegalArgumentException(); 104 throw new IllegalArgumentException();
115 } 105 }
116 if (localOffset >= decompressedPos) { 106 int currentPos = decompressedPosition();
117 skip(localOffset - decompressedPos); 107 if (localOffset >= currentPos) {
108 skip(localOffset - currentPos);
118 } else { 109 } else {
119 reset(); 110 reset();
120 skip(localOffset); 111 skip(localOffset);
121 } 112 }
122 } 113 }
123 114
124 @Override 115 @Override
125 public void skip(final int bytesToSkip) throws IOException { 116 public void skip(final int bytesToSkip) throws IOException {
126 int bytes = bytesToSkip; 117 int bytes = bytesToSkip;
127 if (bytes < 0) { 118 if (bytes < 0) {
128 bytes += decompressedPos; 119 bytes += decompressedPosition();
129 if (bytes < 0) { 120 if (bytes < 0) {
130 throw new IOException(String.format("Underflow. Rewind past start of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, decompressedPos, decompressedLength, bytes)); 121 throw new IOException(String.format("Underflow. Rewind past start of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, inflaterPos, decompressedLength, bytes));
131 } 122 }
132 reset(); 123 reset();
133 // fall-through 124 // fall-through
134 } 125 }
135 while (!isEmpty() && bytes > 0) { 126 while (!isEmpty() && bytes > 0) {
136 readByte(); 127 int fromBuffer = outBuffer.remaining();
137 bytes--; 128 if (fromBuffer > 0) {
129 if (fromBuffer >= bytes) {
130 outBuffer.position(outBuffer.position() + bytes);
131 bytes = 0;
132 break;
133 } else {
134 bytes -= fromBuffer;
135 outBuffer.limit(outBuffer.position()); // mark consumed
136 // fall through to fill the buffer
137 }
138 }
139 fillOutBuffer();
138 } 140 }
139 if (bytes != 0) { 141 if (bytes != 0) {
140 throw new IOException(String.format("Underflow. Rewind past end of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, decompressedPos, decompressedLength, bytes)); 142 throw new IOException(String.format("Underflow. Rewind past end of the slice. To skip:%d, decPos:%d, decLen:%d. Left:%d", bytesToSkip, inflaterPos, decompressedLength, bytes));
141 } 143 }
142 } 144 }
143 145
144 @Override 146 @Override
145 public byte readByte() throws IOException { 147 public byte readByte() throws IOException {
146 readBytes(singleByte, 0, 1); 148 if (!outBuffer.hasRemaining()) {
147 return singleByte[0]; 149 fillOutBuffer();
150 }
151 return outBuffer.get();
148 } 152 }
149 153
150 @Override 154 @Override
151 public void readBytes(byte[] b, int off, int len) throws IOException { 155 public void readBytes(byte[] b, int off, int len) throws IOException {
156 do {
157 int fromBuffer = outBuffer.remaining();
158 if (fromBuffer > 0) {
159 if (fromBuffer >= len) {
160 outBuffer.get(b, off, len);
161 return;
162 } else {
163 outBuffer.get(b, off, fromBuffer);
164 off += fromBuffer;
165 len -= fromBuffer;
166 // fall-through
167 }
168 }
169 fillOutBuffer();
170 } while (len > 0);
171 }
172
173 @Override
174 public void readBytes(ByteBuffer buf) throws IOException {
175 int len = Math.min(available(), buf.remaining());
176 while (len > 0) {
177 if (outBuffer.remaining() >= len) {
178 ByteBuffer slice = outBuffer.slice();
179 slice.limit(len);
180 buf.put(slice);
181 outBuffer.position(outBuffer.position() + len);
182 return;
183 } else {
184 len -= outBuffer.remaining();
185 buf.put(outBuffer);
186 }
187 fillOutBuffer();
188 }
189 }
190
191 private int decompressedPosition() {
192 assert outBuffer.remaining() <= inflaterPos;
193 return inflaterPos - outBuffer.remaining();
194 }
195
196 // after #fillOutBuffer(), outBuffer is ready for read
197 private int fillOutBuffer() throws IOException {
198 assert !outBuffer.hasRemaining();
152 try { 199 try {
153 int n; 200 int inflatedBytes = 0;
154 while (len > 0) { 201 outBuffer.clear();
155 while ((n = inflater.inflate(b, off, len)) == 0) { 202 int len = outBuffer.capacity();
203 int off = 0;
204 do {
205 int n;
206 while ((n = inflater.inflate(outBuffer.array(), off, len)) == 0) {
156 // XXX few last bytes (checksum?) may be ignored by inflater, thus inflate may return 0 in 207 // XXX few last bytes (checksum?) may be ignored by inflater, thus inflate may return 0 in
157 // perfectly legal conditions (when all data already expanded, but there are still some bytes 208 // perfectly legal conditions (when all data already expanded, but there are still some bytes
158 // in the input stream) 209 // in the input stream)
159 int toRead = -1; 210 int toRead = -1;
160 if (inflater.needsInput() && (toRead = super.available()) > 0) { 211 if (inflater.needsInput() && (toRead = super.available()) > 0) {
161 // fill: 212 // fill
162 if (toRead > buffer.length) { 213 if (toRead > inBuffer.length) {
163 toRead = buffer.length; 214 toRead = inBuffer.length;
164 } 215 }
165 super.readBytes(buffer, 0, toRead); 216 super.readBytes(inBuffer, 0, toRead);
166 inflater.setInput(buffer, 0, toRead); 217 inflater.setInput(inBuffer, 0, toRead);
167 } else { 218 } else {
219 // inflated nothing and doesn't want any more data (or no date available) - assume we're done
220 assert inflater.finished();
221 assert toRead <= 0;
222 break;
168 // prevent hang up in this cycle if no more data is available, see Issue 25 223 // prevent hang up in this cycle if no more data is available, see Issue 25
169 throw new EOFException(String.format("No more compressed data is available to satisfy request for %d bytes. [finished:%b, needDict:%b, needInp:%b, available:%d", len, inflater.finished(), inflater.needsDictionary(), inflater.needsInput(), toRead)); 224 // throw new EOFException(String.format("No more compressed data is available to satisfy request for %d bytes. [finished:%b, needDict:%b, needInp:%b, available:%d", len, inflater.finished(), inflater.needsDictionary(), inflater.needsInput(), toRead));
170 } 225 }
171 } 226 }
172 off += n; 227 off += n;
173 len -= n; 228 len -= n;
174 decompressedPos += n; 229 inflatedBytes += n;
175 if (len == 0) { 230 } while (len > 0 && !inflater.finished()); // either the buffer is filled or nothing more to unpack
176 return; // filled 231 inflaterPos += inflatedBytes;
177 } 232 outBuffer.limit(inflatedBytes);
178 } 233 assert outBuffer.position() == 0; // didn't change since #clear() above
234 return inflatedBytes;
179 } catch (DataFormatException e) { 235 } catch (DataFormatException e) {
180 String s = e.getMessage(); 236 String s = e.getMessage();
181 throw new ZipException(s != null ? s : "Invalid ZLIB data format"); 237 throw new ZipException(s != null ? s : "Invalid ZLIB data format");
182 } 238 }
183 } 239 }