Mercurial > jhg

changeset 170:71ddbf8603e8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial clone: populate given directory from a bundle. Everything but remote server access is there, albeit prototype code style
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Wed, 23 Mar 2011 20:46:00 +0100
parents 8c8e3f372fa1
children 2c3e96674e2a
files cmdline/org/tmatesoft/hg/console/Clone.java src/org/tmatesoft/hg/core/Nodeid.java src/org/tmatesoft/hg/internal/RequiresFile.java src/org/tmatesoft/hg/internal/RevlogStream.java src/org/tmatesoft/hg/repo/HgBundle.java src/org/tmatesoft/hg/repo/HgLookup.java src/org/tmatesoft/hg/repo/HgRemoteRepository.java
diffstat 7 files changed, 394 insertions(+), 8 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cmdline/org/tmatesoft/hg/console/Clone.java	Wed Mar 23 20:46:00 2011 +0100
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2011 TMate Software Ltd
+ *  
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * For information on how to redistribute this software under
+ * the terms of a license other than GNU General Public License
+ * contact TMate Software at support@hg4j.com
+ */
+package org.tmatesoft.hg.console;
+
+import static org.tmatesoft.hg.core.Nodeid.NULL;
+import static org.tmatesoft.hg.internal.RequiresFile.*;
+import static org.tmatesoft.hg.internal.RequiresFile.DOTENCODE;
+import static org.tmatesoft.hg.internal.RequiresFile.FNCACHE;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.TreeMap;
+import java.util.zip.DeflaterOutputStream;
+
+import org.tmatesoft.hg.core.HgBadStateException;
+import org.tmatesoft.hg.core.HgRepoFacade;
+import org.tmatesoft.hg.core.Nodeid;
+import org.tmatesoft.hg.internal.ByteArrayDataAccess;
+import org.tmatesoft.hg.internal.DataAccess;
+import org.tmatesoft.hg.internal.DigestHelper;
+import org.tmatesoft.hg.internal.Internals;
+import org.tmatesoft.hg.internal.RequiresFile;
+import org.tmatesoft.hg.internal.RevlogStream;
+import org.tmatesoft.hg.repo.HgBundle;
+import org.tmatesoft.hg.repo.HgLookup;
+import org.tmatesoft.hg.repo.HgRemoteRepository;
+import org.tmatesoft.hg.repo.HgBundle.GroupElement;
+import org.tmatesoft.hg.repo.HgRepository;
+import org.tmatesoft.hg.util.PathRewrite;
+
+/**
+ * WORK IN PROGRESS, DO NOT USE
+ * 
+ * @author Artem Tikhomirov
+ * @author TMate Software Ltd.
+ */
+public class Clone {
+/*
+ * Changegroup: 
+ * http://mercurial.selenic.com/wiki/Merge 
+ * http://mercurial.selenic.com/wiki/WireProtocol 
+ * 
+ * according to latter, bundleformat data is sent through zlib
+ * (there's no header like HG10?? with the server output, though, 
+ * as one may expect according to http://mercurial.selenic.com/wiki/BundleFormat)
+ */
+	public static void main(String[] args) throws Exception {
+		Options cmdLineOpts = Options.parse(args);
+		HgRepoFacade hgRepo = new HgRepoFacade();
+		if (!hgRepo.init(cmdLineOpts.findRepository())) {
+			System.err.printf("Can't find repository in: %s\n", hgRepo.getRepository().getLocation());
+			return;
+		}
+		File destDir = new File("/temp/hg/clone-01/");
+		if (destDir.exists()) {
+			if (!destDir.isDirectory()) {
+				throw new IllegalArgumentException();
+			} else if (destDir.list().length > 0) {
+				throw new IllegalArgumentException();
+			}
+		} else {
+			destDir.mkdirs();
+		}
+		// if cloning remote repo, which can stream and no revision is specified -
+		// can use 'stream_out' wireproto
+		//
+		// //////// 1. from Remote.java take code that asks changegroup from remote server and write it down to temp file
+		// //////// 2. then, read the file with HgBundle
+		// //////// 3. process changelog, memorize nodeids to index
+		// //////// 4. process manifest, using map from step 3, collect manifest nodeids
+		// //////// 5. process every file, using map from 3, and consult set from step 4 to ensure repo is correct
+		// access source
+		HgRemoteRepository remoteRepo = new HgRemoteRepository();// new HgLookup().detect(new URL("https://asd/hg/"));
+		// discover changes
+		HgBundle completeChanges = remoteRepo.getChanges(Collections.singletonList(NULL));
+		WriteDownMate mate = new WriteDownMate(destDir);
+		// instantiate new repo in the destdir
+		mate.initEmptyRepository();
+		// pull changes
+		completeChanges.inspectAll(mate);
+		mate.complete();
+		// completeChanges.unlink();
+	}
+
+	private static class WriteDownMate implements HgBundle.Inspector {
+		private final File hgDir;
+		private FileOutputStream indexFile;
+		private final PathRewrite storagePathHelper;
+
+		private final TreeMap<Nodeid, Integer> changelogIndexes = new TreeMap<Nodeid, Integer>();
+		private boolean collectChangelogIndexes = false;
+
+		private int base = -1;
+		private long offset = 0;
+		private DataAccess prevRevContent;
+		private final DigestHelper dh = new DigestHelper();
+		private final ArrayList<Nodeid> revisionSequence = new ArrayList<Nodeid>(); // last visited nodes first
+
+		private final LinkedList<String> fncacheFiles = new LinkedList<String>();
+
+		public WriteDownMate(File destDir) {
+			hgDir = new File(destDir, ".hg");
+			Internals i = new Internals();
+			i.setStorageConfig(1, STORE | FNCACHE | DOTENCODE);
+			storagePathHelper = i.buildDataFilesHelper();
+		}
+
+		public void initEmptyRepository() throws IOException {
+			hgDir.mkdir();
+			FileOutputStream requiresFile = new FileOutputStream(new File(hgDir, "requires"));
+			requiresFile.write("revlogv1\nstore\nfncache\ndotencode\n".getBytes());
+			requiresFile.close();
+			new File(hgDir, "store").mkdir(); // with that, hg verify says ok.
+		}
+
+		public void complete() throws IOException {
+			FileOutputStream fncacheFile = new FileOutputStream(new File(hgDir, "store/fncache"));
+			for (String s : fncacheFiles) {
+				fncacheFile.write(s.getBytes());
+				fncacheFile.write(0x0A); // http://mercurial.selenic.com/wiki/fncacheRepoFormat
+			}
+			fncacheFile.close();
+		}
+
+		public void changelogStart() {
+			try {
+				base = -1;
+				offset = 0;
+				revisionSequence.clear();
+				indexFile = new FileOutputStream(new File(hgDir, "store/00changelog.i"));
+				collectChangelogIndexes = true;
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+
+		public void changelogEnd() {
+			try {
+				if (prevRevContent != null) {
+					prevRevContent.done();
+					prevRevContent = null;
+				}
+				collectChangelogIndexes = false;
+				indexFile.close();
+				indexFile = null;
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+
+		public void manifestStart() {
+			try {
+				base = -1;
+				offset = 0;
+				revisionSequence.clear();
+				indexFile = new FileOutputStream(new File(hgDir, "store/00manifest.i"));
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+
+		public void manifestEnd() {
+			try {
+				if (prevRevContent != null) {
+					prevRevContent.done();
+					prevRevContent = null;
+				}
+				indexFile.close();
+				indexFile = null;
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+		
+		public void fileStart(String name) {
+			try {
+				base = -1;
+				offset = 0;
+				revisionSequence.clear();
+				fncacheFiles.add("data/" + name + ".i"); // FIXME this is pure guess, 
+				// need to investigate more how filenames are kept in fncache
+				File file = new File(hgDir, storagePathHelper.rewrite(name));
+				file.getParentFile().mkdirs();
+				indexFile = new FileOutputStream(file);
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+
+		public void fileEnd(String name) {
+			try {
+				if (prevRevContent != null) {
+					prevRevContent.done();
+					prevRevContent = null;
+				}
+				indexFile.close();
+				indexFile = null;
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+		}
+
+		private int knownRevision(Nodeid p) {
+			if (NULL.equals(p)) {
+				return -1;
+			} else {
+				for (int i = revisionSequence.size() - 1; i >= 0; i--) {
+					if (revisionSequence.get(i).equals(p)) {
+						return i;
+					}
+				}
+			}
+			throw new HgBadStateException(String.format("Can't find index of %s", p.shortNotation()));
+		}
+
+		public boolean element(GroupElement ge) {
+			try {
+				assert indexFile != null;
+				boolean writeComplete = false;
+				Nodeid p1 = ge.firstParent();
+				Nodeid p2 = ge.secondParent();
+				if (NULL.equals(p1) && NULL.equals(p2) /* or forced flag, does REVIDX_PUNCHED_FLAG indicate that? */) {
+					prevRevContent = new ByteArrayDataAccess(new byte[0]);
+					writeComplete = true;
+				}
+				byte[] content = ge.apply(prevRevContent);
+				byte[] calculated = dh.sha1(p1, p2, content).asBinary();
+				final Nodeid node = ge.node();
+				if (!node.equalsTo(calculated)) {
+					throw new HgBadStateException("Checksum failed");
+				}
+				final int link;
+				if (collectChangelogIndexes) {
+					changelogIndexes.put(node, revisionSequence.size());
+					link = revisionSequence.size();
+				} else {
+					Integer csRev = changelogIndexes.get(ge.cset());
+					if (csRev == null) {
+						throw new HgBadStateException(String.format("Changelog doesn't contain revision %s", ge.cset().shortNotation()));
+					}
+					link = csRev.intValue();
+				}
+				final int p1Rev = knownRevision(p1), p2Rev = knownRevision(p2);
+				DataAccess patchContent = ge.rawData();
+				writeComplete = writeComplete || patchContent.length() >= (/* 3/4 of actual */content.length - (content.length >>> 2));
+				if (writeComplete) {
+					base = revisionSequence.size();
+				}
+				final byte[] sourceData = writeComplete ? content : patchContent.byteArray();
+				final byte[] data;
+				ByteArrayOutputStream bos = new ByteArrayOutputStream(content.length);
+				DeflaterOutputStream dos = new DeflaterOutputStream(bos);
+				dos.write(sourceData);
+				dos.close();
+				final byte[] compressedData = bos.toByteArray();
+				dos = null;
+				bos = null;
+				final Byte dataPrefix;
+				if (compressedData.length >= (sourceData.length - (sourceData.length >>> 2))) {
+					// compression wasn't too effective,
+					data = sourceData;
+					dataPrefix = 'u';
+				} else {
+					data = compressedData;
+					dataPrefix = null;
+				}
+
+				ByteBuffer header = ByteBuffer.allocate(64 /* REVLOGV1_RECORD_SIZE */);
+				if (offset == 0) {
+					final int INLINEDATA = 1 << 16;
+					header.putInt(1 /* RevlogNG */ | INLINEDATA);
+					header.putInt(0);
+				} else {
+					header.putLong(offset << 16);
+				}
+				final int compressedLen = data.length + (dataPrefix == null ? 0 : 1);
+				header.putInt(compressedLen);
+				header.putInt(content.length);
+				header.putInt(base);
+				header.putInt(link);
+				header.putInt(p1Rev);
+				header.putInt(p2Rev);
+				header.put(node.toByteArray());
+				// assume 12 bytes left are zeros
+				indexFile.write(header.array());
+				if (dataPrefix != null) {
+					indexFile.write(dataPrefix.byteValue());
+				}
+				indexFile.write(data);
+				//
+				offset += compressedLen;
+				revisionSequence.add(node);
+				prevRevContent.done();
+				prevRevContent = new ByteArrayDataAccess(content);
+			} catch (IOException ex) {
+				throw new HgBadStateException(ex);
+			}
+			return true;
+		}
+	}
+}
--- a/src/org/tmatesoft/hg/core/Nodeid.java	Wed Mar 23 14:13:11 2011 +0100
+++ b/src/org/tmatesoft/hg/core/Nodeid.java	Wed Mar 23 20:46:00 2011 +0100
@@ -30,7 +30,7 @@
  * @author TMate Software Ltd.
  *
  */
-public final class Nodeid {
+public final class Nodeid implements Comparable<Nodeid> {
 	
 	/**
 	 * <b>nullid</b>, empty root revision.
@@ -72,6 +72,18 @@
 		return Arrays.equals(this.binaryData, buf);
 	}
 	
+	public int compareTo(Nodeid o) {
+		if (this == o) {
+			return 0;
+		}
+		for (int i = 0; i < 20; i++) {
+			if (binaryData[i] != o.binaryData[i]) {
+				return binaryData[i] < o.binaryData[i] ? -1 : 1;
+			}
+		}
+		return 0;
+	}
+	
 	@Override
 	public String toString() {
 		// XXX may want to output just single 0 for the NULL id?
--- a/src/org/tmatesoft/hg/internal/RequiresFile.java	Wed Mar 23 14:13:11 2011 +0100
+++ b/src/org/tmatesoft/hg/internal/RequiresFile.java	Wed Mar 23 20:46:00 2011 +0100
@@ -53,9 +53,9 @@
 				dotencode |= "dotencode".equals(line);
 			}
 			int flags = 0;
-			flags += store ? 1 : 0;
-			flags += fncache ? 2 : 0;
-			flags += dotencode ? 4 : 0;
+			flags += store ? STORE : 0;
+			flags += fncache ? FNCACHE : 0;
+			flags += dotencode ? DOTENCODE : 0;
 			repoImpl.setStorageConfig(revlogv1 ? 1 : 0, flags);
 		} catch (IOException ex) {
 			ex.printStackTrace(); // FIXME log
--- a/src/org/tmatesoft/hg/internal/RevlogStream.java	Wed Mar 23 14:13:11 2011 +0100
+++ b/src/org/tmatesoft/hg/internal/RevlogStream.java	Wed Mar 23 20:46:00 2011 +0100
@@ -241,7 +241,7 @@
 						streamDataAccess = daData;
 						daData.seek(streamOffset);
 					}
-					final boolean patchToPrevious = baseRevision != i; // XXX not sure if this is the right way to detect a patch
+					final boolean patchToPrevious = baseRevision != i; // the only way I found to tell if it's a patch
 					firstByte = streamDataAccess.readByte();
 					if (firstByte == 0x78 /* 'x' */) {
 						userDataAccess = new InflaterDataAccess(streamDataAccess, streamOffset, compressedLen, patchToPrevious ? -1 : actualLen);
@@ -300,7 +300,7 @@
 		DataAccess da = getIndexStream();
 		try {
 			int versionField = da.readInt();
-			da.readInt(); // just to skip next 2 bytes of offset + flags
+			da.readInt(); // just to skip next 4 bytes of offset + flags
 			final int INLINEDATA = 1 << 16;
 			inline = (versionField & INLINEDATA) != 0;
 			long offset = 0; // first offset is always 0, thus Hg uses it for other purposes
--- a/src/org/tmatesoft/hg/repo/HgBundle.java	Wed Mar 23 14:13:11 2011 +0100
+++ b/src/org/tmatesoft/hg/repo/HgBundle.java	Wed Mar 23 20:46:00 2011 +0100
@@ -148,6 +148,7 @@
 					}
 					RawChangeset cs = RawChangeset.parse(csetDataAccess);
 					System.out.println(cs.toString());
+					prevRevContent.done();
 					prevRevContent = csetDataAccess.reset();
 				} catch (CancelledException ex) {
 					return false;
@@ -190,8 +191,8 @@
 		void fileEnd(String name);
 
 		/**
-		 * @param element
-		 *            data element, instance might be reused
+		 * XXX desperately need exceptions here
+		 * @param element data element, instance might be reused, don't keep a reference to it or its raw data
 		 * @return <code>true</code> to continue
 		 */
 		boolean element(GroupElement element);
@@ -349,6 +350,9 @@
 			// regardless whether that slice has read it or not.
 			GroupElement ge = new GroupElement(nb, slice);
 			good2go = inspector.element(ge);
+			slice.done(); // BADA doesn't implement done(), but it could (e.g. free array) 
+			/// and we'd better tell it we are not going to use it any more. However, it's important to ensure Inspector
+			// implementations out there do not retain GroupElement.rawData()
 			len = da.isEmpty() ? 0 : da.readInt();
 		}
 		// need to skip up to group end if inspector told he don't want to continue with the group, 
--- a/src/org/tmatesoft/hg/repo/HgLookup.java	Wed Mar 23 14:13:11 2011 +0100
+++ b/src/org/tmatesoft/hg/repo/HgLookup.java	Wed Mar 23 20:46:00 2011 +0100
@@ -18,6 +18,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URL;
 
 import org.tmatesoft.hg.core.HgException;
 import org.tmatesoft.hg.internal.DataAccessProvider;
@@ -69,4 +70,11 @@
 		}
 		return new HgBundle(new DataAccessProvider(), location);
 	}
+
+	public HgRemoteRepository detect(URL url) throws HgException {
+		if (Boolean.FALSE.booleanValue()) {
+			throw HgRepository.notImplemented();
+		}
+		return null;
+	}
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/org/tmatesoft/hg/repo/HgRemoteRepository.java	Wed Mar 23 20:46:00 2011 +0100
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2011 TMate Software Ltd
+ *  
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * For information on how to redistribute this software under
+ * the terms of a license other than GNU General Public License
+ * contact TMate Software at support@hg4j.com
+ */
+package org.tmatesoft.hg.repo;
+
+import java.io.File;
+import java.util.List;
+
+import org.tmatesoft.hg.core.HgException;
+import org.tmatesoft.hg.core.Nodeid;
+
+/**
+ * WORK IN PROGRESS, DO NOT USE
+ * 
+ * @see http://mercurial.selenic.com/wiki/WireProtocol
+ * 
+ * @author Artem Tikhomirov
+ * @author TMate Software Ltd.
+ */
+public class HgRemoteRepository {
+
+	// WireProtocol wiki: roots = a list of the latest nodes on every service side changeset branch that both the client and server know about. 
+	public HgBundle getChanges(List<Nodeid> roots) throws HgException {
+		return new HgLookup().loadBundle(new File("/temp/hg/hg-bundle-000000000000-gz.tmp"));
+	}
+}