# HG changeset patch # User Artem Tikhomirov # Date 1293310212 -3600 # Node ID 382cfe9463db0484a14136e4b38407419525f0c0 # Parent d6d2a630f4a6d670c90a5ca909150f2b426ec88f Dirstate parsing. DataAccess refactored to allow reuse and control over constants diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/console/Status.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/com/tmate/hgkit/console/Status.java Sat Dec 25 21:50:12 2010 +0100 @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2010 Artem Tikhomirov + */ +package com.tmate.hgkit.console; + +import com.tmate.hgkit.fs.RepositoryLookup; +import com.tmate.hgkit.ll.HgRepository; +import com.tmate.hgkit.ll.LocalHgRepo; + +/** + * + * @author artem + */ +public class Status { + + public static void main(String[] args) throws Exception { + RepositoryLookup repoLookup = new RepositoryLookup(); + RepositoryLookup.Options cmdLineOpts = RepositoryLookup.Options.parse(args); + HgRepository hgRepo = repoLookup.detect(cmdLineOpts); + if (hgRepo.isInvalid()) { + System.err.printf("Can't find repository in: %s\n", hgRepo.getLocation()); + return; + } + System.out.println(hgRepo.getLocation()); + ((LocalHgRepo) hgRepo).loadDirstate().dump(); + } +} diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/fs/DataAccess.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/com/tmate/hgkit/fs/DataAccess.java Sat Dec 25 21:50:12 2010 +0100 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2010 Artem Tikhomirov + */ +package com.tmate.hgkit.fs; + +import java.io.IOException; + +/** + * relevant parts of DataInput, non-stream nature (seek operation), explicit check for end of data. + * convenient skip (+/- bytes) + * Primary goal - effective file read, so that clients don't need to care whether to call few + * distinct getInt() or readBytes(totalForFewInts) and parse themselves instead in an attempt to optimize. + */ +public class DataAccess { + public boolean isEmpty() { + return true; + } + // absolute positioning + public void seek(long offset) throws IOException { + throw new UnsupportedOperationException(); + } + // relative positioning + public void skip(int bytes) throws IOException { + throw new UnsupportedOperationException(); + } + // shall be called once this object no longer needed + public void done() { + // no-op in this empty implementation + } + public int readInt() throws IOException { + byte[] b = new byte[4]; + readBytes(b, 0, 4); + return b[0] << 24 | (b[1] & 0xFF) << 16 | (b[2] & 0xFF) << 8 | (b[3] & 0xFF); + } + public long readLong() throws IOException { + byte[] b = new byte[8]; + readBytes(b, 0, 8); + int i1 = b[0] << 24 | (b[1] & 0xFF) << 16 | (b[2] & 0xFF) << 8 | (b[3] & 0xFF); + int i2 = b[4] << 24 | (b[5] & 0xFF) << 16 | (b[6] & 0xFF) << 8 | (b[7] & 0xFF); + return ((long) i1) << 32 | ((long) i2 & 0xFFFFFFFF); + } + public void readBytes(byte[] buf, int offset, int length) throws IOException { + throw new UnsupportedOperationException(); + } + public byte readByte() throws IOException { + throw new UnsupportedOperationException(); + } +} \ No newline at end of file diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/fs/DataAccessProvider.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/com/tmate/hgkit/fs/DataAccessProvider.java Sat Dec 25 21:50:12 2010 +0100 @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2010 Artem Tikhomirov + */ +package com.tmate.hgkit.fs; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; + +/** + * + * @author artem + */ +public class DataAccessProvider { + + private final int mapioMagicBoundary; + private final int bufferSize; + + public DataAccessProvider() { + mapioMagicBoundary = 100 * 1024; + bufferSize = 8 * 1024; + } + + public DataAccess create(File f) { + if (!f.exists()) { + return new DataAccess(); + } + try { + FileChannel fc = new FileInputStream(f).getChannel(); + if (fc.size() > mapioMagicBoundary) { + return new MemoryMapFileAccess(fc, fc.size()); + } else { + // XXX once implementation is more or less stable, + // may want to try ByteBuffer.allocateDirect() to see + // if there's any performance gain. + boolean useDirectBuffer = false; + return new FileAccess(fc, fc.size(), bufferSize, useDirectBuffer); + } + } catch (IOException ex) { + // unlikely to happen, we've made sure file exists. + ex.printStackTrace(); // FIXME log error + } + return new DataAccess(); // non-null, empty. + } + + // DOESN'T WORK YET + private static class MemoryMapFileAccess extends DataAccess { + private FileChannel fileChannel; + private final long size; + private long position = 0; + + public MemoryMapFileAccess(FileChannel fc, long channelSize) { + fileChannel = fc; + size = channelSize; + } + + @Override + public void seek(long offset) { + position = offset; + } + + @Override + public void skip(int bytes) throws IOException { + position += bytes; + } + + private boolean fill() throws IOException { + final int BUFFER_SIZE = 8 * 1024; + long left = size - position; + MappedByteBuffer rv = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < BUFFER_SIZE ? left : BUFFER_SIZE); + position += rv.capacity(); + return rv.hasRemaining(); + } + + @Override + public void done() { + if (fileChannel != null) { + try { + fileChannel.close(); + } catch (IOException ex) { + ex.printStackTrace(); // log debug + } + fileChannel = null; + } + } + } + + // (almost) regular file access - FileChannel and buffers. + private static class FileAccess extends DataAccess { + private FileChannel fileChannel; + private final long size; + private ByteBuffer buffer; + private long bufferStartInFile = 0; // offset of this.buffer in the file. + + public FileAccess(FileChannel fc, long channelSize, int bufferSizeHint, boolean useDirect) { + fileChannel = fc; + size = channelSize; + final int capacity = size < bufferSizeHint ? (int) size : bufferSizeHint; + buffer = useDirect ? ByteBuffer.allocateDirect(capacity) : ByteBuffer.allocate(capacity); + buffer.flip(); // or .limit(0) to indicate it's empty + } + + @Override + public boolean isEmpty() { + return bufferStartInFile + buffer.position() >= size; + } + + @Override + public void seek(long offset) throws IOException { + if (offset < bufferStartInFile + buffer.limit() && offset >= bufferStartInFile) { + buffer.position((int) (offset - bufferStartInFile)); + } else { + // out of current buffer, invalidate it (force re-read) + // XXX or ever re-read it right away? + bufferStartInFile = offset; + buffer.clear(); + buffer.limit(0); // or .flip() to indicate we switch to reading + fileChannel.position(offset); + } + } + + @Override + public void skip(int bytes) throws IOException { + final int newPos = buffer.position() + bytes; + if (newPos >= 0 && newPos < buffer.limit()) { + // no need to move file pointer, just rewind/seek buffer + buffer.position(newPos); + } else { + // + seek(fileChannel.position()+ bytes); + } + } + + private boolean fill() throws IOException { + if (!buffer.hasRemaining()) { + bufferStartInFile += buffer.limit(); + buffer.clear(); + if (bufferStartInFile < size) { // just in case there'd be any exception on EOF, not -1 + fileChannel.read(buffer); + // may return -1 when EOF, but empty will reflect this, hence no explicit support here + } + buffer.flip(); + } + return buffer.hasRemaining(); + } + + @Override + public void readBytes(byte[] buf, int offset, int length) throws IOException { + final int tail = buffer.remaining(); + if (tail >= length) { + buffer.get(buf, offset, length); + } else { + buffer.get(buf, offset, tail); + if (fill()) { + buffer.get(buf, offset + tail, length - tail); + } else { + throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past nonEmpty() == false are made. + } + } + } + + @Override + public byte readByte() throws IOException { + if (buffer.hasRemaining()) { + return buffer.get(); + } + if (fill()) { + return buffer.get(); + } + throw new IOException(); + } + + @Override + public void done() { + if (buffer != null) { + buffer = null; + } + if (fileChannel != null) { + try { + fileChannel.close(); + } catch (IOException ex) { + ex.printStackTrace(); // log debug + } + fileChannel = null; + } + } + } +} diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/ll/HgDirstate.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/com/tmate/hgkit/ll/HgDirstate.java Sat Dec 25 21:50:12 2010 +0100 @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2010 Artem Tikhomirov + */ +package com.tmate.hgkit.ll; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +import com.tmate.hgkit.fs.DataAccess; +import com.tmate.hgkit.fs.DataAccessProvider; + +/** + * @see http://mercurial.selenic.com/wiki/DirState + * @see http://mercurial.selenic.com/wiki/FileFormats#dirstate + * @author artem + */ +public class HgDirstate { + + private final LocalHgRepo repo; + private final File dirstateFile; + private List normal; + private List added; + private List removed; + private List merged; + + public HgDirstate(LocalHgRepo hgRepo, File dirstate) { + this.repo = hgRepo; + this.dirstateFile = dirstate; + } + + private void read() { + normal = added = removed = merged = Collections.emptyList(); + if (!dirstateFile.exists()) { + return; + } + DataAccessProvider dap = repo.getDataAccess(); + DataAccess da = dap.create(dirstateFile); + if (da.isEmpty()) { + return; + } + normal = new LinkedList(); + added = new LinkedList(); + removed = new LinkedList(); + merged = new LinkedList(); + try { + // XXX skip(40) if we don't need these? + byte[] parents = new byte[40]; + da.readBytes(parents, 0, 40); + parents = null; + do { + final byte state = da.readByte(); + final int fmode = da.readInt(); + final int size = da.readInt(); + final int time = da.readInt(); + final int nameLen = da.readInt(); + String fn1 = null, fn2 = null; + byte[] name = new byte[nameLen]; + da.readBytes(name, 0, nameLen); + for (int i = 0; i < nameLen; i++) { + if (name[i] == 0) { + fn1 = new String(name, 0, i); + fn2 = new String(name, i+1, nameLen); + break; + } + } + if (fn1 == null) { + fn1 = new String(name); + } + Record r = new Record(fmode, size, time, fn1, fn2); + if (state == 'n') { + normal.add(r); + } else if (state == 'a') { + added.add(r); + } else if (state == 'r') { + removed.add(r); + } else if (state == 'm') { + merged.add(r); + } else { + // FIXME log error? + } + } while (!da.isEmpty()); + } catch (IOException ex) { + ex.printStackTrace(); // FIXME log error, clean dirstate? + } finally { + da.done(); + } + } + + public void dump() { + read(); + @SuppressWarnings("unchecked") + List[] all = new List[] { normal, added, removed, merged }; + char[] x = new char[] {'n', 'a', 'r', 'm' }; + for (int i = 0; i < all.length; i++) { + for (Record r : all[i]) { + System.out.printf("%c %3o%6d %30tc\t\t%s", x[i], r.mode, r.size, (long) r.time, r.name1); + if (r.name2 != null) { + System.out.printf(" --> %s", r.name2); + } + System.out.println(); + } + System.out.println(); + } + } + + private static class Record { + final int mode; + final int size; + final int time; + final String name1; + final String name2; + + public Record(int fmode, int fsize, int ftime, String name1, String name2) { + mode = fmode; + size = fsize; + time = ftime; + this.name1 = name1; + this.name2 = name2; + + } + } +} diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/ll/HgRepository.java --- a/src/com/tmate/hgkit/ll/HgRepository.java Sat Dec 25 04:45:59 2010 +0100 +++ b/src/com/tmate/hgkit/ll/HgRepository.java Sat Dec 25 21:50:12 2010 +0100 @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2010 Artem Tikhomirov */ package com.tmate.hgkit.ll; @@ -45,10 +45,6 @@ } return this.manifest; } - - public final Object/*HgDirstate*/ getDirstate() { - throw notImplemented(); - } public abstract HgDataFile getFileNode(String path); diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/ll/LocalHgRepo.java --- a/src/com/tmate/hgkit/ll/LocalHgRepo.java Sat Dec 25 04:45:59 2010 +0100 +++ b/src/com/tmate/hgkit/ll/LocalHgRepo.java Sat Dec 25 21:50:12 2010 +0100 @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2010 Artem Tikhomirov */ package com.tmate.hgkit.ll; @@ -13,6 +13,8 @@ import java.util.HashMap; import java.util.TreeSet; +import com.tmate.hgkit.fs.DataAccessProvider; + /** * @author artem */ @@ -20,10 +22,12 @@ private File repoDir; // .hg folder private final String repoLocation; + private final DataAccessProvider dataAccess; public LocalHgRepo(String repositoryPath) { setInvalid(true); repoLocation = repositoryPath; + dataAccess = null; } public LocalHgRepo(File repositoryRoot) throws IOException { @@ -31,6 +35,7 @@ setInvalid(false); repoDir = repositoryRoot; repoLocation = repositoryRoot.getParentFile().getCanonicalPath(); + dataAccess = new DataAccessProvider(); parseRequires(); } @@ -39,6 +44,16 @@ return repoLocation; } + // XXX package-local, unless there are cases when required from outside (guess, working dir/revision walkers may hide dirstate access and no public visibility needed) + public final HgDirstate loadDirstate() { + // XXX may cache in SoftReference if creation is expensive + return new HgDirstate(this, new File(repoDir, "dirstate")); + } + + /*package-local*/ DataAccessProvider getDataAccess() { + return dataAccess; + } + private final HashMap> streamsCache = new HashMap>(); /** @@ -53,7 +68,7 @@ } File f = new File(repoDir, path); if (f.exists()) { - RevlogStream s = new RevlogStream(f); + RevlogStream s = new RevlogStream(dataAccess, f); streamsCache.put(path, new SoftReference(s)); return s; } diff -r d6d2a630f4a6 -r 382cfe9463db src/com/tmate/hgkit/ll/RevlogStream.java --- a/src/com/tmate/hgkit/ll/RevlogStream.java Sat Dec 25 04:45:59 2010 +0100 +++ b/src/com/tmate/hgkit/ll/RevlogStream.java Sat Dec 25 21:50:12 2010 +0100 @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2010 Artem Tikhomirov */ package com.tmate.hgkit.ll; @@ -6,11 +6,7 @@ import static com.tmate.hgkit.ll.HgRepository.TIP; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; @@ -18,6 +14,9 @@ import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import com.tmate.hgkit.fs.DataAccess; +import com.tmate.hgkit.fs.DataAccessProvider; + /** * ? Single RevlogStream per file per repository with accessor to record access session (e.g. with back/forward operations), * or numerous RevlogStream with separate representation of the underlaying data (cached, lazy ChunkStream)? @@ -30,40 +29,24 @@ private List index; // indexed access highly needed private boolean inline = false; private final File indexFile; + private final DataAccessProvider dataAccess; - RevlogStream(File indexFile) { + // if we need anything else from HgRepo, might replace DAP parameter with HgRepo and query it for DAP. + RevlogStream(DataAccessProvider dap, File indexFile) { + this.dataAccess = dap; this.indexFile = indexFile; } /*package*/ DataAccess getIndexStream() { - return create(indexFile); + return dataAccess.create(indexFile); } /*package*/ DataAccess getDataStream() { final String indexName = indexFile.getName(); File dataFile = new File(indexFile.getParentFile(), indexName.substring(0, indexName.length() - 1) + "d"); - return create(dataFile); + return dataAccess.create(dataFile); } - private DataAccess create(File f) { - if (!f.exists()) { - return new DataAccess(); - } - try { - FileChannel fc = new FileInputStream(f).getChannel(); - final int MAPIO_MAGIC_BOUNDARY = 100 * 1024; - if (fc.size() > MAPIO_MAGIC_BOUNDARY) { - return new MemoryMapFileAccess(fc, fc.size()); - } else { - return new FileAccess(fc, fc.size()); - } - } catch (IOException ex) { - // unlikely to happen, we've made sure file exists. - ex.printStackTrace(); // FIXME log error - } - return new DataAccess(); // non-null, empty. - } - public int revisionCount() { initOutline(); return index.size(); @@ -220,13 +203,14 @@ res.add(new IndexEntry(offset, baseRevision)); da.skip(3*4 + 32); } - if (da.nonEmpty()) { - long l = da.readLong(); - offset = l >>> 16; - } else { + if (da.isEmpty()) { // fine, done then index = res; break; + } else { + // start reading next record + long l = da.readLong(); + offset = l >>> 16; } } } catch (IOException ex) { @@ -284,171 +268,4 @@ System.arraycopy(src, srcOffset, data, 0, len); } } - - /*package-local*/ class DataAccess { - public boolean nonEmpty() { - return false; - } - // absolute positioning - public void seek(long offset) throws IOException { - throw new UnsupportedOperationException(); - } - // relative positioning - public void skip(int bytes) throws IOException { - throw new UnsupportedOperationException(); - } - // shall be called once this object no longer needed - public void done() { - // no-op in this empty implementation - } - public int readInt() throws IOException { - byte[] b = new byte[4]; - readBytes(b, 0, 4); - return b[0] << 24 | (b[1] & 0xFF) << 16 | (b[2] & 0xFF) << 8 | (b[3] & 0xFF); - } - public long readLong() throws IOException { - byte[] b = new byte[8]; - readBytes(b, 0, 8); - int i1 = b[0] << 24 | (b[1] & 0xFF) << 16 | (b[2] & 0xFF) << 8 | (b[3] & 0xFF); - int i2 = b[4] << 24 | (b[5] & 0xFF) << 16 | (b[6] & 0xFF) << 8 | (b[7] & 0xFF); - return ((long) i1) << 32 | ((long) i2 & 0xFFFFFFFF); - } - public void readBytes(byte[] buf, int offset, int length) throws IOException { - throw new UnsupportedOperationException(); - } - } - - // DOESN'T WORK YET - private class MemoryMapFileAccess extends DataAccess { - private FileChannel fileChannel; - private final long size; - private long position = 0; - - public MemoryMapFileAccess(FileChannel fc, long channelSize) { - fileChannel = fc; - size = channelSize; - } - - @Override - public void seek(long offset) { - position = offset; - } - - @Override - public void skip(int bytes) throws IOException { - position += bytes; - } - - private boolean fill() throws IOException { - final int BUFFER_SIZE = 8 * 1024; - long left = size - position; - MappedByteBuffer rv = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < BUFFER_SIZE ? left : BUFFER_SIZE); - position += rv.capacity(); - return rv.hasRemaining(); - } - - @Override - public void done() { - if (fileChannel != null) { - try { - fileChannel.close(); - } catch (IOException ex) { - ex.printStackTrace(); // log debug - } - fileChannel = null; - } - } - } - - private class FileAccess extends DataAccess { - private FileChannel fileChannel; - private final long size; - private ByteBuffer buffer; - private long bufferStartInFile = 0; // offset of this.buffer in the file. - - public FileAccess(FileChannel fc, long channelSize) { - fileChannel = fc; - size = channelSize; - final int BUFFER_SIZE = 8 * 1024; - // XXX once implementation is more or less stable, - // may want to try ByteBuffer.allocateDirect() to see - // if there's any performance gain. - buffer = ByteBuffer.allocate(size < BUFFER_SIZE ? (int) size : BUFFER_SIZE); - buffer.flip(); // or .limit(0) to indicate it's empty - } - - @Override - public boolean nonEmpty() { - return bufferStartInFile + buffer.position() < size; - } - - @Override - public void seek(long offset) throws IOException { - if (offset < bufferStartInFile + buffer.limit() && offset >= bufferStartInFile) { - buffer.position((int) (offset - bufferStartInFile)); - } else { - // out of current buffer, invalidate it (force re-read) - // XXX or ever re-read it right away? - bufferStartInFile = offset; - buffer.clear(); - buffer.limit(0); // or .flip() to indicate we switch to reading - fileChannel.position(offset); - } - } - - @Override - public void skip(int bytes) throws IOException { - final int newPos = buffer.position() + bytes; - if (newPos >= 0 && newPos < buffer.limit()) { - // no need to move file pointer, just rewind/seek buffer - buffer.position(newPos); - } else { - // - seek(fileChannel.position()+ bytes); - } - } - - private boolean fill() throws IOException { - if (!buffer.hasRemaining()) { - bufferStartInFile += buffer.limit(); - buffer.clear(); - if (bufferStartInFile < size) { // just in case there'd be any exception on EOF, not -1 - fileChannel.read(buffer); - // may return -1 when EOF, but empty will reflect this, hence no explicit support here - } - buffer.flip(); - } - return buffer.hasRemaining(); - } - - @Override - public void readBytes(byte[] buf, int offset, int length) throws IOException { - final int tail = buffer.remaining(); - if (tail >= length) { - buffer.get(buf, offset, length); - } else { - buffer.get(buf, offset, tail); - if (fill()) { - buffer.get(buf, offset + tail, length - tail); - } else { - throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past nonEmpty() == false are made. - } - } - } - - @Override - public void done() { - if (buffer != null) { - buffer = null; - } - if (fileChannel != null) { - try { - fileChannel.close(); - } catch (IOException ex) { - ex.printStackTrace(); // log debug - } - fileChannel = null; - } - } - } }