# HG changeset patch # User Artem Tikhomirov # Date 1300909560 -3600 # Node ID 71ddbf8603e8e09d54ac9c5fe4bb5ae824589f1d # Parent 8c8e3f372fa1fbfcf92b004b6f2ada2dbaf60028 Initial clone: populate given directory from a bundle. Everything but remote server access is there, albeit prototype code style diff -r 8c8e3f372fa1 -r 71ddbf8603e8 cmdline/org/tmatesoft/hg/console/Clone.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cmdline/org/tmatesoft/hg/console/Clone.java Wed Mar 23 20:46:00 2011 +0100 @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2011 TMate Software Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * For information on how to redistribute this software under + * the terms of a license other than GNU General Public License + * contact TMate Software at support@hg4j.com + */ +package org.tmatesoft.hg.console; + +import static org.tmatesoft.hg.core.Nodeid.NULL; +import static org.tmatesoft.hg.internal.RequiresFile.*; +import static org.tmatesoft.hg.internal.RequiresFile.DOTENCODE; +import static org.tmatesoft.hg.internal.RequiresFile.FNCACHE; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.TreeMap; +import java.util.zip.DeflaterOutputStream; + +import org.tmatesoft.hg.core.HgBadStateException; +import org.tmatesoft.hg.core.HgRepoFacade; +import org.tmatesoft.hg.core.Nodeid; +import org.tmatesoft.hg.internal.ByteArrayDataAccess; +import org.tmatesoft.hg.internal.DataAccess; +import org.tmatesoft.hg.internal.DigestHelper; +import org.tmatesoft.hg.internal.Internals; +import org.tmatesoft.hg.internal.RequiresFile; +import org.tmatesoft.hg.internal.RevlogStream; +import org.tmatesoft.hg.repo.HgBundle; +import org.tmatesoft.hg.repo.HgLookup; +import org.tmatesoft.hg.repo.HgRemoteRepository; +import org.tmatesoft.hg.repo.HgBundle.GroupElement; +import org.tmatesoft.hg.repo.HgRepository; +import org.tmatesoft.hg.util.PathRewrite; + +/** + * WORK IN PROGRESS, DO NOT USE + * + * @author Artem Tikhomirov + * @author TMate Software Ltd. + */ +public class Clone { +/* + * Changegroup: + * http://mercurial.selenic.com/wiki/Merge + * http://mercurial.selenic.com/wiki/WireProtocol + * + * according to latter, bundleformat data is sent through zlib + * (there's no header like HG10?? with the server output, though, + * as one may expect according to http://mercurial.selenic.com/wiki/BundleFormat) + */ + public static void main(String[] args) throws Exception { + Options cmdLineOpts = Options.parse(args); + HgRepoFacade hgRepo = new HgRepoFacade(); + if (!hgRepo.init(cmdLineOpts.findRepository())) { + System.err.printf("Can't find repository in: %s\n", hgRepo.getRepository().getLocation()); + return; + } + File destDir = new File("/temp/hg/clone-01/"); + if (destDir.exists()) { + if (!destDir.isDirectory()) { + throw new IllegalArgumentException(); + } else if (destDir.list().length > 0) { + throw new IllegalArgumentException(); + } + } else { + destDir.mkdirs(); + } + // if cloning remote repo, which can stream and no revision is specified - + // can use 'stream_out' wireproto + // + // //////// 1. from Remote.java take code that asks changegroup from remote server and write it down to temp file + // //////// 2. then, read the file with HgBundle + // //////// 3. process changelog, memorize nodeids to index + // //////// 4. process manifest, using map from step 3, collect manifest nodeids + // //////// 5. process every file, using map from 3, and consult set from step 4 to ensure repo is correct + // access source + HgRemoteRepository remoteRepo = new HgRemoteRepository();// new HgLookup().detect(new URL("https://asd/hg/")); + // discover changes + HgBundle completeChanges = remoteRepo.getChanges(Collections.singletonList(NULL)); + WriteDownMate mate = new WriteDownMate(destDir); + // instantiate new repo in the destdir + mate.initEmptyRepository(); + // pull changes + completeChanges.inspectAll(mate); + mate.complete(); + // completeChanges.unlink(); + } + + private static class WriteDownMate implements HgBundle.Inspector { + private final File hgDir; + private FileOutputStream indexFile; + private final PathRewrite storagePathHelper; + + private final TreeMap changelogIndexes = new TreeMap(); + private boolean collectChangelogIndexes = false; + + private int base = -1; + private long offset = 0; + private DataAccess prevRevContent; + private final DigestHelper dh = new DigestHelper(); + private final ArrayList revisionSequence = new ArrayList(); // last visited nodes first + + private final LinkedList fncacheFiles = new LinkedList(); + + public WriteDownMate(File destDir) { + hgDir = new File(destDir, ".hg"); + Internals i = new Internals(); + i.setStorageConfig(1, STORE | FNCACHE | DOTENCODE); + storagePathHelper = i.buildDataFilesHelper(); + } + + public void initEmptyRepository() throws IOException { + hgDir.mkdir(); + FileOutputStream requiresFile = new FileOutputStream(new File(hgDir, "requires")); + requiresFile.write("revlogv1\nstore\nfncache\ndotencode\n".getBytes()); + requiresFile.close(); + new File(hgDir, "store").mkdir(); // with that, hg verify says ok. + } + + public void complete() throws IOException { + FileOutputStream fncacheFile = new FileOutputStream(new File(hgDir, "store/fncache")); + for (String s : fncacheFiles) { + fncacheFile.write(s.getBytes()); + fncacheFile.write(0x0A); // http://mercurial.selenic.com/wiki/fncacheRepoFormat + } + fncacheFile.close(); + } + + public void changelogStart() { + try { + base = -1; + offset = 0; + revisionSequence.clear(); + indexFile = new FileOutputStream(new File(hgDir, "store/00changelog.i")); + collectChangelogIndexes = true; + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + public void changelogEnd() { + try { + if (prevRevContent != null) { + prevRevContent.done(); + prevRevContent = null; + } + collectChangelogIndexes = false; + indexFile.close(); + indexFile = null; + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + public void manifestStart() { + try { + base = -1; + offset = 0; + revisionSequence.clear(); + indexFile = new FileOutputStream(new File(hgDir, "store/00manifest.i")); + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + public void manifestEnd() { + try { + if (prevRevContent != null) { + prevRevContent.done(); + prevRevContent = null; + } + indexFile.close(); + indexFile = null; + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + public void fileStart(String name) { + try { + base = -1; + offset = 0; + revisionSequence.clear(); + fncacheFiles.add("data/" + name + ".i"); // FIXME this is pure guess, + // need to investigate more how filenames are kept in fncache + File file = new File(hgDir, storagePathHelper.rewrite(name)); + file.getParentFile().mkdirs(); + indexFile = new FileOutputStream(file); + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + public void fileEnd(String name) { + try { + if (prevRevContent != null) { + prevRevContent.done(); + prevRevContent = null; + } + indexFile.close(); + indexFile = null; + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + } + + private int knownRevision(Nodeid p) { + if (NULL.equals(p)) { + return -1; + } else { + for (int i = revisionSequence.size() - 1; i >= 0; i--) { + if (revisionSequence.get(i).equals(p)) { + return i; + } + } + } + throw new HgBadStateException(String.format("Can't find index of %s", p.shortNotation())); + } + + public boolean element(GroupElement ge) { + try { + assert indexFile != null; + boolean writeComplete = false; + Nodeid p1 = ge.firstParent(); + Nodeid p2 = ge.secondParent(); + if (NULL.equals(p1) && NULL.equals(p2) /* or forced flag, does REVIDX_PUNCHED_FLAG indicate that? */) { + prevRevContent = new ByteArrayDataAccess(new byte[0]); + writeComplete = true; + } + byte[] content = ge.apply(prevRevContent); + byte[] calculated = dh.sha1(p1, p2, content).asBinary(); + final Nodeid node = ge.node(); + if (!node.equalsTo(calculated)) { + throw new HgBadStateException("Checksum failed"); + } + final int link; + if (collectChangelogIndexes) { + changelogIndexes.put(node, revisionSequence.size()); + link = revisionSequence.size(); + } else { + Integer csRev = changelogIndexes.get(ge.cset()); + if (csRev == null) { + throw new HgBadStateException(String.format("Changelog doesn't contain revision %s", ge.cset().shortNotation())); + } + link = csRev.intValue(); + } + final int p1Rev = knownRevision(p1), p2Rev = knownRevision(p2); + DataAccess patchContent = ge.rawData(); + writeComplete = writeComplete || patchContent.length() >= (/* 3/4 of actual */content.length - (content.length >>> 2)); + if (writeComplete) { + base = revisionSequence.size(); + } + final byte[] sourceData = writeComplete ? content : patchContent.byteArray(); + final byte[] data; + ByteArrayOutputStream bos = new ByteArrayOutputStream(content.length); + DeflaterOutputStream dos = new DeflaterOutputStream(bos); + dos.write(sourceData); + dos.close(); + final byte[] compressedData = bos.toByteArray(); + dos = null; + bos = null; + final Byte dataPrefix; + if (compressedData.length >= (sourceData.length - (sourceData.length >>> 2))) { + // compression wasn't too effective, + data = sourceData; + dataPrefix = 'u'; + } else { + data = compressedData; + dataPrefix = null; + } + + ByteBuffer header = ByteBuffer.allocate(64 /* REVLOGV1_RECORD_SIZE */); + if (offset == 0) { + final int INLINEDATA = 1 << 16; + header.putInt(1 /* RevlogNG */ | INLINEDATA); + header.putInt(0); + } else { + header.putLong(offset << 16); + } + final int compressedLen = data.length + (dataPrefix == null ? 0 : 1); + header.putInt(compressedLen); + header.putInt(content.length); + header.putInt(base); + header.putInt(link); + header.putInt(p1Rev); + header.putInt(p2Rev); + header.put(node.toByteArray()); + // assume 12 bytes left are zeros + indexFile.write(header.array()); + if (dataPrefix != null) { + indexFile.write(dataPrefix.byteValue()); + } + indexFile.write(data); + // + offset += compressedLen; + revisionSequence.add(node); + prevRevContent.done(); + prevRevContent = new ByteArrayDataAccess(content); + } catch (IOException ex) { + throw new HgBadStateException(ex); + } + return true; + } + } +} diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/core/Nodeid.java --- a/src/org/tmatesoft/hg/core/Nodeid.java Wed Mar 23 14:13:11 2011 +0100 +++ b/src/org/tmatesoft/hg/core/Nodeid.java Wed Mar 23 20:46:00 2011 +0100 @@ -30,7 +30,7 @@ * @author TMate Software Ltd. * */ -public final class Nodeid { +public final class Nodeid implements Comparable { /** * nullid, empty root revision. @@ -72,6 +72,18 @@ return Arrays.equals(this.binaryData, buf); } + public int compareTo(Nodeid o) { + if (this == o) { + return 0; + } + for (int i = 0; i < 20; i++) { + if (binaryData[i] != o.binaryData[i]) { + return binaryData[i] < o.binaryData[i] ? -1 : 1; + } + } + return 0; + } + @Override public String toString() { // XXX may want to output just single 0 for the NULL id? diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/internal/RequiresFile.java --- a/src/org/tmatesoft/hg/internal/RequiresFile.java Wed Mar 23 14:13:11 2011 +0100 +++ b/src/org/tmatesoft/hg/internal/RequiresFile.java Wed Mar 23 20:46:00 2011 +0100 @@ -53,9 +53,9 @@ dotencode |= "dotencode".equals(line); } int flags = 0; - flags += store ? 1 : 0; - flags += fncache ? 2 : 0; - flags += dotencode ? 4 : 0; + flags += store ? STORE : 0; + flags += fncache ? FNCACHE : 0; + flags += dotencode ? DOTENCODE : 0; repoImpl.setStorageConfig(revlogv1 ? 1 : 0, flags); } catch (IOException ex) { ex.printStackTrace(); // FIXME log diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/internal/RevlogStream.java --- a/src/org/tmatesoft/hg/internal/RevlogStream.java Wed Mar 23 14:13:11 2011 +0100 +++ b/src/org/tmatesoft/hg/internal/RevlogStream.java Wed Mar 23 20:46:00 2011 +0100 @@ -241,7 +241,7 @@ streamDataAccess = daData; daData.seek(streamOffset); } - final boolean patchToPrevious = baseRevision != i; // XXX not sure if this is the right way to detect a patch + final boolean patchToPrevious = baseRevision != i; // the only way I found to tell if it's a patch firstByte = streamDataAccess.readByte(); if (firstByte == 0x78 /* 'x' */) { userDataAccess = new InflaterDataAccess(streamDataAccess, streamOffset, compressedLen, patchToPrevious ? -1 : actualLen); @@ -300,7 +300,7 @@ DataAccess da = getIndexStream(); try { int versionField = da.readInt(); - da.readInt(); // just to skip next 2 bytes of offset + flags + da.readInt(); // just to skip next 4 bytes of offset + flags final int INLINEDATA = 1 << 16; inline = (versionField & INLINEDATA) != 0; long offset = 0; // first offset is always 0, thus Hg uses it for other purposes diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/repo/HgBundle.java --- a/src/org/tmatesoft/hg/repo/HgBundle.java Wed Mar 23 14:13:11 2011 +0100 +++ b/src/org/tmatesoft/hg/repo/HgBundle.java Wed Mar 23 20:46:00 2011 +0100 @@ -148,6 +148,7 @@ } RawChangeset cs = RawChangeset.parse(csetDataAccess); System.out.println(cs.toString()); + prevRevContent.done(); prevRevContent = csetDataAccess.reset(); } catch (CancelledException ex) { return false; @@ -190,8 +191,8 @@ void fileEnd(String name); /** - * @param element - * data element, instance might be reused + * XXX desperately need exceptions here + * @param element data element, instance might be reused, don't keep a reference to it or its raw data * @return true to continue */ boolean element(GroupElement element); @@ -349,6 +350,9 @@ // regardless whether that slice has read it or not. GroupElement ge = new GroupElement(nb, slice); good2go = inspector.element(ge); + slice.done(); // BADA doesn't implement done(), but it could (e.g. free array) + /// and we'd better tell it we are not going to use it any more. However, it's important to ensure Inspector + // implementations out there do not retain GroupElement.rawData() len = da.isEmpty() ? 0 : da.readInt(); } // need to skip up to group end if inspector told he don't want to continue with the group, diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/repo/HgLookup.java --- a/src/org/tmatesoft/hg/repo/HgLookup.java Wed Mar 23 14:13:11 2011 +0100 +++ b/src/org/tmatesoft/hg/repo/HgLookup.java Wed Mar 23 20:46:00 2011 +0100 @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import org.tmatesoft.hg.core.HgException; import org.tmatesoft.hg.internal.DataAccessProvider; @@ -69,4 +70,11 @@ } return new HgBundle(new DataAccessProvider(), location); } + + public HgRemoteRepository detect(URL url) throws HgException { + if (Boolean.FALSE.booleanValue()) { + throw HgRepository.notImplemented(); + } + return null; + } } diff -r 8c8e3f372fa1 -r 71ddbf8603e8 src/org/tmatesoft/hg/repo/HgRemoteRepository.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/org/tmatesoft/hg/repo/HgRemoteRepository.java Wed Mar 23 20:46:00 2011 +0100 @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 TMate Software Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * For information on how to redistribute this software under + * the terms of a license other than GNU General Public License + * contact TMate Software at support@hg4j.com + */ +package org.tmatesoft.hg.repo; + +import java.io.File; +import java.util.List; + +import org.tmatesoft.hg.core.HgException; +import org.tmatesoft.hg.core.Nodeid; + +/** + * WORK IN PROGRESS, DO NOT USE + * + * @see http://mercurial.selenic.com/wiki/WireProtocol + * + * @author Artem Tikhomirov + * @author TMate Software Ltd. + */ +public class HgRemoteRepository { + + // WireProtocol wiki: roots = a list of the latest nodes on every service side changeset branch that both the client and server know about. + public HgBundle getChanges(List roots) throws HgException { + return new HgLookup().loadBundle(new File("/temp/hg/hg-bundle-000000000000-gz.tmp")); + } +}