Mercurial > jhg

comparison src/org/tmatesoft/hg/repo/RevlogStream.java @ 74:6f1b88693d48

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Complete refactoring to org.tmatesoft
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Mon, 24 Jan 2011 03:14:45 +0100
parents src/com/tmate/hgkit/ll/RevlogStream.java@30bd38978846
children
comparison
equal deleted inserted replaced
73:0d279bcc4442 74:6f1b88693d48
1 /*
2 * Copyright (c) 2010-2011 TMate Software Ltd
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; version 2 of the License.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * For information on how to redistribute this software under
14 * the terms of a license other than GNU General Public License
15 * contact TMate Software at support@svnkit.com
16 */
17 package org.tmatesoft.hg.repo;
18
19 import static org.tmatesoft.hg.repo.HgRepository.TIP;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.LinkedList;
26 import java.util.List;
27 import java.util.zip.DataFormatException;
28 import java.util.zip.Inflater;
29
30 import org.tmatesoft.hg.core.Nodeid;
31 import org.tmatesoft.hg.internal.DataAccess;
32 import org.tmatesoft.hg.internal.DataAccessProvider;
33
34
35 /**
36 * XXX move to .internal?
37 * ? Single RevlogStream per file per repository with accessor to record access session (e.g. with back/forward operations),
38 * or numerous RevlogStream with separate representation of the underlaying data (cached, lazy ChunkStream)?
39 *
40 * @see http://mercurial.selenic.com/wiki/Revlog
41 * @see http://mercurial.selenic.com/wiki/RevlogNG
42 *
43 * @author Artem Tikhomirov
44 * @author TMate Software Ltd.
45 */
46 public class RevlogStream {
47
48 private List<IndexEntry> index; // indexed access highly needed
49 private boolean inline = false;
50 private final File indexFile;
51 private final DataAccessProvider dataAccess;
52
53 // if we need anything else from HgRepo, might replace DAP parameter with HgRepo and query it for DAP.
54 RevlogStream(DataAccessProvider dap, File indexFile) {
55 this.dataAccess = dap;
56 this.indexFile = indexFile;
57 }
58
59 /*package*/ DataAccess getIndexStream() {
60 return dataAccess.create(indexFile);
61 }
62
63 /*package*/ DataAccess getDataStream() {
64 final String indexName = indexFile.getName();
65 File dataFile = new File(indexFile.getParentFile(), indexName.substring(0, indexName.length() - 1) + "d");
66 return dataAccess.create(dataFile);
67 }
68
69 public int revisionCount() {
70 initOutline();
71 return index.size();
72 }
73
74 public int dataLength(int revision) {
75 // XXX in fact, use of iterate() instead of this implementation may be quite reasonable.
76 //
77 final int indexSize = revisionCount();
78 DataAccess daIndex = getIndexStream(); // XXX may supply a hint that I'll need really few bytes of data (although at some offset)
79 if (revision == TIP) {
80 revision = indexSize - 1;
81 }
82 try {
83 int recordOffset = inline ? (int) index.get(revision).offset : revision * REVLOGV1_RECORD_SIZE;
84 daIndex.seek(recordOffset + 12); // 6+2+4
85 int actualLen = daIndex.readInt();
86 return actualLen;
87 } catch (IOException ex) {
88 ex.printStackTrace(); // log error. FIXME better handling
89 throw new IllegalStateException(ex);
90 } finally {
91 daIndex.done();
92 }
93 }
94
95 // Perhaps, RevlogStream should be limited to use of plain int revisions for access,
96 // while Nodeids should be kept on the level up, in Revlog. Guess, Revlog better keep
97 // map of nodeids, and once this comes true, we may get rid of this method.
98 // Unlike its counterpart, Revlog#getLocalRevisionNumber, doesn't fail with exception if node not found,
99 // returns a predefined constant instead
100 /*package-local*/ int findLocalRevisionNumber(Nodeid nodeid) {
101 // XXX this one may be implemented with iterate() once there's mechanism to stop iterations
102 final int indexSize = revisionCount();
103 DataAccess daIndex = getIndexStream();
104 try {
105 byte[] nodeidBuf = new byte[20];
106 for (int i = 0; i < indexSize; i++) {
107 daIndex.skip(8);
108 int compressedLen = daIndex.readInt();
109 daIndex.skip(20);
110 daIndex.readBytes(nodeidBuf, 0, 20);
111 if (nodeid.equalsTo(nodeidBuf)) {
112 return i;
113 }
114 daIndex.skip(inline ? 12 + compressedLen : 12);
115 }
116 } catch (IOException ex) {
117 ex.printStackTrace(); // log error. FIXME better handling
118 throw new IllegalStateException(ex);
119 } finally {
120 daIndex.done();
121 }
122 return Integer.MIN_VALUE;
123 }
124
125
126 private final int REVLOGV1_RECORD_SIZE = 64;
127
128 // should be possible to use TIP, ALL, or -1, -2, -n notation of Hg
129 // ? boolean needsNodeid
130 public void iterate(int start, int end, boolean needData, Revlog.Inspector inspector) {
131 initOutline();
132 final int indexSize = index.size();
133 if (indexSize == 0) {
134 return;
135 }
136 if (end == TIP) {
137 end = indexSize - 1;
138 }
139 if (start == TIP) {
140 start = indexSize - 1;
141 }
142 if (start < 0 || start >= indexSize) {
143 throw new IllegalArgumentException("Bad left range boundary " + start);
144 }
145 if (end < start || end >= indexSize) {
146 throw new IllegalArgumentException("Bad right range boundary " + end);
147 }
148 // XXX may cache [start .. end] from index with a single read (pre-read)
149
150 DataAccess daIndex = null, daData = null;
151 daIndex = getIndexStream();
152 if (needData && !inline) {
153 daData = getDataStream();
154 }
155 try {
156 byte[] nodeidBuf = new byte[20];
157 byte[] lastData = null;
158 int i;
159 boolean extraReadsToBaseRev = false;
160 if (needData && index.get(start).baseRevision < start) {
161 i = index.get(start).baseRevision;
162 extraReadsToBaseRev = true;
163 } else {
164 i = start;
165 }
166
167 daIndex.seek(inline ? (int) index.get(i).offset : i * REVLOGV1_RECORD_SIZE);
168 for (; i <= end; i++ ) {
169 long l = daIndex.readLong();
170 @SuppressWarnings("unused")
171 long offset = l >>> 16;
172 @SuppressWarnings("unused")
173 int flags = (int) (l & 0X0FFFF);
174 int compressedLen = daIndex.readInt();
175 int actualLen = daIndex.readInt();
176 int baseRevision = daIndex.readInt();
177 int linkRevision = daIndex.readInt();
178 int parent1Revision = daIndex.readInt();
179 int parent2Revision = daIndex.readInt();
180 // Hg has 32 bytes here, uses 20 for nodeid, and keeps 12 last bytes empty
181 daIndex.readBytes(nodeidBuf, 0, 20);
182 daIndex.skip(12);
183 byte[] data = null;
184 if (needData) {
185 byte[] dataBuf = new byte[compressedLen];
186 if (inline) {
187 daIndex.readBytes(dataBuf, 0, compressedLen);
188 } else {
189 daData.seek(index.get(i).offset);
190 daData.readBytes(dataBuf, 0, compressedLen);
191 }
192 if (dataBuf[0] == 0x78 /* 'x' */) {
193 try {
194 Inflater zlib = new Inflater(); // XXX Consider reuse of Inflater, and/or stream alternative
195 zlib.setInput(dataBuf, 0, compressedLen);
196 byte[] result = new byte[actualLen*2]; // FIXME need to use zlib.finished() instead
197 int resultLen = zlib.inflate(result);
198 zlib.end();
199 data = new byte[resultLen];
200 System.arraycopy(result, 0, data, 0, resultLen);
201 } catch (DataFormatException ex) {
202 ex.printStackTrace();
203 data = new byte[0]; // FIXME need better failure strategy
204 }
205 } else if (dataBuf[0] == 0x75 /* 'u' */) {
206 data = new byte[dataBuf.length - 1];
207 System.arraycopy(dataBuf, 1, data, 0, data.length);
208 } else {
209 // XXX Python impl in fact throws exception when there's not 'x', 'u' or '0'
210 // but I don't see reason not to return data as is
211 data = dataBuf;
212 }
213 // XXX
214 if (baseRevision != i) { // XXX not sure if this is the right way to detect a patch
215 // this is a patch
216 LinkedList<PatchRecord> patches = new LinkedList<PatchRecord>();
217 int patchElementIndex = 0;
218 do {
219 PatchRecord pr = PatchRecord.read(data, patchElementIndex);
220 patches.add(pr);
221 patchElementIndex += 12 + pr.len;
222 } while (patchElementIndex < data.length);
223 //
224 byte[] baseRevContent = lastData;
225 data = apply(baseRevContent, actualLen, patches);
226 }
227 } else {
228 if (inline) {
229 daIndex.skip(compressedLen);
230 }
231 }
232 if (!extraReadsToBaseRev || i >= start) {
233 inspector.next(i, actualLen, baseRevision, linkRevision, parent1Revision, parent2Revision, nodeidBuf, data);
234 }
235 lastData = data;
236 }
237 } catch (IOException ex) {
238 throw new IllegalStateException(ex); // FIXME need better handling
239 } finally {
240 daIndex.done();
241 if (daData != null) {
242 daData.done();
243 }
244 }
245 }
246
247 private void initOutline() {
248 if (index != null && !index.isEmpty()) {
249 return;
250 }
251 ArrayList<IndexEntry> res = new ArrayList<IndexEntry>();
252 DataAccess da = getIndexStream();
253 try {
254 int versionField = da.readInt();
255 da.readInt(); // just to skip next 2 bytes of offset + flags
256 final int INLINEDATA = 1 << 16;
257 inline = (versionField & INLINEDATA) != 0;
258 long offset = 0; // first offset is always 0, thus Hg uses it for other purposes
259 while(true) {
260 int compressedLen = da.readInt();
261 // 8+4 = 12 bytes total read here
262 @SuppressWarnings("unused")
263 int actualLen = da.readInt();
264 int baseRevision = da.readInt();
265 // 12 + 8 = 20 bytes read here
266 // int linkRevision = di.readInt();
267 // int parent1Revision = di.readInt();
268 // int parent2Revision = di.readInt();
269 // byte[] nodeid = new byte[32];
270 if (inline) {
271 res.add(new IndexEntry(offset + REVLOGV1_RECORD_SIZE * res.size(), baseRevision));
272 da.skip(3*4 + 32 + compressedLen); // Check: 44 (skip) + 20 (read) = 64 (total RevlogNG record size)
273 } else {
274 res.add(new IndexEntry(offset, baseRevision));
275 da.skip(3*4 + 32);
276 }
277 if (da.isEmpty()) {
278 // fine, done then
279 res.trimToSize();
280 index = res;
281 break;
282 } else {
283 // start reading next record
284 long l = da.readLong();
285 offset = l >>> 16;
286 }
287 }
288 } catch (IOException ex) {
289 ex.printStackTrace(); // log error
290 // too bad, no outline then.
291 index = Collections.emptyList();
292 } finally {
293 da.done();
294 }
295
296 }
297
298
299 // perhaps, package-local or protected, if anyone else from low-level needs them
300 // XXX think over if we should keep offset in case of separate data file - we read the field anyway. Perhaps, distinct entry classes for Inline and non-inline indexes?
301 private static class IndexEntry {
302 public final long offset; // for separate .i and .d - copy of index record entry, for inline index - actual offset of the record in the .i file (record entry + revision * record size))
303 //public final int length; // data past fixed record (need to decide whether including header size or not), and whether length is of compressed data or not
304 public final int baseRevision;
305
306 public IndexEntry(long o, int baseRev) {
307 offset = o;
308 baseRevision = baseRev;
309 }
310 }
311
312 // mpatch.c : apply()
313 // FIXME need to implement patch merge (fold, combine, gather and discard from aforementioned mpatch.[c|py]), also see Revlog and Mercurial PDF
314 /*package-local for HgBundle; until moved to better place*/static byte[] apply(byte[] baseRevisionContent, int outcomeLen, List<PatchRecord> patch) {
315 int last = 0, destIndex = 0;
316 if (outcomeLen == -1) {
317 outcomeLen = baseRevisionContent.length;
318 for (PatchRecord pr : patch) {
319 outcomeLen += pr.start - last + pr.len;
320 last = pr.end;
321 }
322 outcomeLen -= last;
323 last = 0;
324 }
325 byte[] rv = new byte[outcomeLen];
326 for (PatchRecord pr : patch) {
327 System.arraycopy(baseRevisionContent, last, rv, destIndex, pr.start-last);
328 destIndex += pr.start - last;
329 System.arraycopy(pr.data, 0, rv, destIndex, pr.data.length);
330 destIndex += pr.data.length;
331 last = pr.end;
332 }
333 System.arraycopy(baseRevisionContent, last, rv, destIndex, baseRevisionContent.length - last);
334 return rv;
335 }
336
337 // @see http://mercurial.selenic.com/wiki/BundleFormat, in Changelog group description
338 /*package-local*/ static class PatchRecord { // copy of struct frag from mpatch.c
339 /*
340 Given there are pr1 and pr2:
341 pr1.start to pr1.end will be replaced with pr's data (of pr1.len)
342 pr1.end to pr2.start gets copied from base
343 */
344 int start, end, len;
345 byte[] data;
346
347 // TODO consider PatchRecord that only records data position (absolute in data source), and acquires data as needed
348 private PatchRecord(int p1, int p2, int length, byte[] src) {
349 start = p1;
350 end = p2;
351 len = length;
352 data = src;
353 }
354
355 /*package-local*/ static PatchRecord read(byte[] data, int offset) {
356 final int x = offset; // shorthand
357 int p1 = ((data[x] & 0xFF)<< 24) | ((data[x+1] & 0xFF) << 16) | ((data[x+2] & 0xFF) << 8) | (data[x+3] & 0xFF);
358 int p2 = ((data[x+4] & 0xFF) << 24) | ((data[x+5] & 0xFF) << 16) | ((data[x+6] & 0xFF) << 8) | (data[x+7] & 0xFF);
359 int len = ((data[x+8] & 0xFF) << 24) | ((data[x+9] & 0xFF) << 16) | ((data[x+10] & 0xFF) << 8) | (data[x+11] & 0xFF);
360 byte[] dataCopy = new byte[len];
361 System.arraycopy(data, x+12, dataCopy, 0, len);
362 return new PatchRecord(p1, p2, len, dataCopy);
363 }
364
365 /*package-local*/ static PatchRecord read(DataAccess da) throws IOException {
366 int p1 = da.readInt();
367 int p2 = da.readInt();
368 int len = da.readInt();
369 byte[] src = new byte[len];
370 da.readBytes(src, 0, len);
371 return new PatchRecord(p1, p2, len, src);
372 }
373
374
375 }
376 }