Mercurial > hg4j

changeset 574:88afffd39899

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Improve memory consumption of HgManifest#getFileRevision(): avoid extra byte[] instances
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Tue, 16 Apr 2013 14:44:57 +0200
parents e49f9d9513fa
children 8bf184c9d733
files cmdline/org/tmatesoft/hg/console/Main.java src/org/tmatesoft/hg/internal/ByteVector.java src/org/tmatesoft/hg/internal/IntVector.java src/org/tmatesoft/hg/repo/HgManifest.java test/org/tmatesoft/hg/test/TestAuxUtilities.java
diffstat 5 files changed, 187 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/cmdline/org/tmatesoft/hg/console/Main.java	Fri Apr 12 19:50:21 2013 +0200
+++ b/cmdline/org/tmatesoft/hg/console/Main.java	Tue Apr 16 14:44:57 2013 +0200
@@ -28,6 +28,7 @@
 import java.util.Map;
 
 import org.tmatesoft.hg.core.HgChangeset;
+import org.tmatesoft.hg.core.HgChangesetFileSneaker;
 import org.tmatesoft.hg.core.HgChangesetTreeHandler;
 import org.tmatesoft.hg.core.HgCheckoutCommand;
 import org.tmatesoft.hg.core.HgException;
@@ -42,6 +43,7 @@
 import org.tmatesoft.hg.internal.ByteArrayChannel;
 import org.tmatesoft.hg.internal.ConfigFile;
 import org.tmatesoft.hg.internal.DigestHelper;
+import org.tmatesoft.hg.internal.ManifestRevision;
 import org.tmatesoft.hg.internal.PathGlobMatcher;
 import org.tmatesoft.hg.internal.RelativePathRewrite;
 import org.tmatesoft.hg.internal.StreamLogFacility;
@@ -100,7 +102,8 @@
 
 	public static void main(String[] args) throws Exception {
 		Main m = new Main(args);
-		m.testRevert();
+		m.checkFileSneakerPerformance2();
+//		m.testRevert();
 //		m.testCheckout();
 //		m.tryExtensions();
 //		m.dumpBookmarks();
@@ -127,6 +130,45 @@
 //		m.bunchOfTests();
 	}
 	
+	private void checkFileSneakerPerformance() throws Exception {
+		HgChangesetFileSneaker fs1 = new HgChangesetFileSneaker(hgRepo);
+		HgChangesetFileSneaker fs2 = new HgChangesetFileSneaker(hgRepo);
+		fs1.followRenames(true);
+		fs2.followRenames(true);
+		Nodeid cset = hgRepo.getChangelog().getRevision(TIP);
+		Path fname = Path.create("dir3/file8");
+		fs1.changeset(cset);
+		fs2.changeset(cset);
+//		hgRepo.getManifest().getFileRevision(TIP, fname);
+		final long start1 = System.nanoTime();
+		boolean e1 = fs1.checkExists(fname);
+		final long end1 = System.nanoTime();
+		boolean e2 = fs2.checkExists(fname);
+		final long end2 = System.nanoTime();
+		Nodeid fr = hgRepo.getManifest().getFileRevision(TIP, fname);
+		final long end3 = System.nanoTime();
+		System.out.printf("\t1st run: %d ms, %b\n\t2nd run: %d ms, %b\n\tfile only: %d ms", (end1 - start1) / 1000000, e1, (end2 - end1) / 1000000, e2, (end3-end2)/1000000);
+		if (!fr.equals(fs1.revision()) || !fr.equals(fs2.revision())) {
+			throw new AssertionError();
+		}
+		ManifestRevision mr = new ManifestRevision(null, null);
+		final long _s1 = System.nanoTime();
+		hgRepo.getManifest().walk(2, 2, mr);
+		final long _e1 = System.nanoTime();
+		hgRepo.getManifest().getFileRevision(2, fname);
+		final long _e2 = System.nanoTime();
+		System.out.printf("\n\tManifestRevision:%d ms, getFileRevision:%d ms\n", (_e1-_s1)/1000000, (_e2-_e1)/1000000);
+	}
+	
+	//  -agentlib:hprof=cpu=times,heap=sites,depth=10
+	private void checkFileSneakerPerformance2() throws Exception {
+		Path fname = Path.create("dir3/file8");
+		hgRepo.getManifest().getFileRevision(2, fname);
+//		ManifestRevision mr = new ManifestRevision(null, null);
+//		hgRepo.getManifest().walk(2, 2, mr);
+	}
+	
+	
 	private void testRevert() throws Exception {
 		HgRevertCommand cmd = new HgRevertCommand(hgRepo);
 		cmd.file(Path.create("a.txt")).execute();
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/org/tmatesoft/hg/internal/ByteVector.java	Tue Apr 16 14:44:57 2013 +0200
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2013 TMate Software Ltd
+ *  
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * For information on how to redistribute this software under
+ * the terms of a license other than GNU General Public License
+ * contact TMate Software at support@hg4j.com
+ */
+package org.tmatesoft.hg.internal;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * Alternative to {@link ByteArrayOutputStream}, with extra operation that prevent extra byte[] instances
+ * 
+ * @author Artem Tikhomirov
+ * @author TMate Software Ltd.
+ */
+public class ByteVector {
+	private byte[] data;
+	private int count;
+	private final int increment;
+	
+	
+	public ByteVector(int initialSize, int increment) {
+		data = new byte[initialSize];
+		this.increment = increment;
+	}
+
+	public void add(int b) {
+		if (count == data.length) {
+			byte[] newData = new byte[count + increment];
+			System.arraycopy(data, 0, newData, 0, count);
+			data = newData;
+		}
+		data[count++] = (byte) b;
+	}
+
+	public int size() {
+		return count;
+	}
+
+	public void clear() {
+		count = 0;
+	}
+	
+	public boolean equalsTo(byte[] array) {
+		if (array == null || array.length != count) {
+			return false;
+		}
+		for (int i = 0; i < count; i++) {
+			if (data[i] != array[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+	
+	/**
+	 * Copies content of this vector into destination array.
+	 * @param destination array, greater or equal to {@link #size()} of the vector
+	 */
+	public void copyTo(byte[] destination) {
+		if (destination == null || destination.length < count) {
+			throw new IllegalArgumentException();
+		}
+		System.arraycopy(data, 0, destination, 0, count);
+	}
+
+	public byte[] toByteArray() {
+		byte[] rv = new byte[count];
+		copyTo(rv);
+		return rv;
+	}
+}
--- a/src/org/tmatesoft/hg/internal/IntVector.java	Fri Apr 12 19:50:21 2013 +0200
+++ b/src/org/tmatesoft/hg/internal/IntVector.java	Tue Apr 16 14:44:57 2013 +0200
@@ -142,7 +142,7 @@
 		if (increment == 0) {
 			throw new UnsupportedOperationException("This vector is not allowed to expand");
 		}
-		int newCapacity = increment < 0 ? data.length << 1 : data.length + increment;
+		int newCapacity = increment < 0 ? data.length << 1 : (data.length + increment);
 		if (newCapacityHint > 0 && newCapacity < newCapacityHint) {
 			newCapacity = newCapacityHint;
 		}
--- a/src/org/tmatesoft/hg/repo/HgManifest.java	Fri Apr 12 19:50:21 2013 +0200
+++ b/src/org/tmatesoft/hg/repo/HgManifest.java	Tue Apr 16 14:44:57 2013 +0200
@@ -20,26 +20,26 @@
 import static org.tmatesoft.hg.repo.HgRepository.*;
 import static org.tmatesoft.hg.util.LogFacility.Severity.Info;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 
 import org.tmatesoft.hg.core.HgChangesetFileSneaker;
 import org.tmatesoft.hg.core.Nodeid;
+import org.tmatesoft.hg.internal.ByteVector;
 import org.tmatesoft.hg.internal.Callback;
 import org.tmatesoft.hg.internal.DataAccess;
 import org.tmatesoft.hg.internal.DigestHelper;
 import org.tmatesoft.hg.internal.EncodingHelper;
+import org.tmatesoft.hg.internal.IdentityPool;
 import org.tmatesoft.hg.internal.IntMap;
 import org.tmatesoft.hg.internal.IterateControlMediator;
 import org.tmatesoft.hg.internal.Lifecycle;
-import org.tmatesoft.hg.internal.IdentityPool;
 import org.tmatesoft.hg.internal.RevlogStream;
 import org.tmatesoft.hg.util.CancelSupport;
+import org.tmatesoft.hg.util.LogFacility.Severity;
 import org.tmatesoft.hg.util.Path;
 import org.tmatesoft.hg.util.ProgressSupport;
-import org.tmatesoft.hg.util.LogFacility.Severity;
 
 
 /**
@@ -51,7 +51,8 @@
  */
 public final class HgManifest extends Revlog {
 	private RevisionMapper revisionMap;
-	private EncodingHelper encodingHelper;
+	private final EncodingHelper encodingHelper;
+	private final Path.Source pathFactory; 
 	
 	/**
 	 * File flags recorded in manifest
@@ -111,11 +112,19 @@
 			}
 			throw new IllegalStateException(toString());
 		}
+		
+		public int fsMode() {
+			if (this == Exec) {
+				return 0755;
+			}
+			return 0644;
+		}
 	}
 
 	/*package-local*/ HgManifest(HgRepository hgRepo, RevlogStream content, EncodingHelper eh) {
 		super(hgRepo, content);
 		encodingHelper = eh;
+		pathFactory = hgRepo.getSessionContext().getPathFactory();
 	}
 
 	/**
@@ -446,8 +455,8 @@
 		
 		public Path freeze() {
 			if (result == null) {
-				Path.Source pathFactory = HgManifest.this.getRepo().getSessionContext().getPathFactory();
-				result = pathFactory.path(HgManifest.this.encodingHelper.fromManifest(data, start, length));
+				Path.Source pf = HgManifest.this.pathFactory;
+				result = pf.path(HgManifest.this.encodingHelper.fromManifest(data, start, length));
 				// release reference to bigger data array, make a copy of relevant part only
 				// use original bytes, not those from String above to avoid cache misses due to different encodings 
 				byte[] d = new byte[length];
@@ -689,16 +698,14 @@
 		}
 		
 		public void next(int revisionNumber, int actualLen, int baseRevision, int linkRevision, int parent1Revision, int parent2Revision, byte[] nodeid, DataAccess data) {
-			ByteArrayOutputStream bos = new ByteArrayOutputStream();
+			ByteVector byteVector = new ByteVector(256, 128); // allocate for long paths right away
 			try {
 				byte b;
 				while (!data.isEmpty() && (b = data.readByte()) != '\n') {
 					if (b != 0) {
-						bos.write(b);
+						byteVector.add(b);
 					} else {
-						byte[] byteArray = bos.toByteArray();
-						bos.reset();
-						if (Arrays.equals(filenameAsBytes, byteArray)) {
+						if (byteVector.equalsTo(filenameAsBytes)) {
 							Nodeid fileRev = null;
 							Flags flags = null;
 							if (csetIndex2FileRev != null || delegate != null) {
@@ -709,15 +716,15 @@
 								data.skip(40);
 							}
 							if (csetIndex2Flags != null || delegate != null) {
+								byteVector.clear();
 								while (!data.isEmpty() && (b = data.readByte()) != '\n') {
-									bos.write(b);
+									byteVector.add(b);
 								}
-								if (bos.size() == 0) {
+								if (byteVector.size() == 0) {
 									flags = Flags.RegularFile;
 								} else {
-									flags = Flags.parse(bos.toByteArray(), 0, bos.size());
+									flags = Flags.parse(byteVector.toByteArray(), 0, byteVector.size());
 								}
-								
 							}
 							if (delegate != null) {
 								assert flags != null;
@@ -741,6 +748,8 @@
 						// else skip to the end of line
 						while (!data.isEmpty() && (b = data.readByte()) != '\n')
 							;
+
+						byteVector.clear();
 					}
 				}
 			} catch (IOException ex) {
--- a/test/org/tmatesoft/hg/test/TestAuxUtilities.java	Fri Apr 12 19:50:21 2013 +0200
+++ b/test/org/tmatesoft/hg/test/TestAuxUtilities.java	Tue Apr 16 14:44:57 2013 +0200
@@ -31,6 +31,7 @@
 import org.tmatesoft.hg.core.HgCatCommand;
 import org.tmatesoft.hg.core.Nodeid;
 import org.tmatesoft.hg.internal.ArrayHelper;
+import org.tmatesoft.hg.internal.ByteVector;
 import org.tmatesoft.hg.internal.IntVector;
 import org.tmatesoft.hg.internal.PathScope;
 import org.tmatesoft.hg.internal.RangeSeq;
@@ -543,8 +544,41 @@
 		errorCollector.assertFalse(rs.includesTargetLine(12));
 	}
 	
+	@Test
+	public void testByteVector() {
+		ByteVector v = new ByteVector(4, 2);
+		v.add(7);
+		v.add(9);
+		errorCollector.assertEquals(2, v.size());
+		v.clear();
+		errorCollector.assertEquals(0, v.size());
+		v.add(10);
+		v.add(9);
+		v.add(8);
+		v.add(7);
+		v.add(6);
+		errorCollector.assertEquals(5, v.size());
+		v.add(5);
+		v.add(4);
+		errorCollector.assertEquals(7, v.size());
+		byte x = 10;
+		for (byte d : v.toByteArray()) {
+			errorCollector.assertEquals(x, d);
+			x--;
+		}
+		x = 10;
+		byte[] dd = new byte[10];
+		v.copyTo(dd);
+		for (int i = 0; i < v.size(); i++) {
+			errorCollector.assertEquals(x, dd[i]);
+			x--;
+		}
+		errorCollector.assertTrue(v.equalsTo(new byte[] { 10,9,8,7,6,5,4 }));
+	}
 	
-	public static void main(String[] args) throws Exception {
-		new TestAuxUtilities().testRepositoryConfig();
+	public static void main(String[] args) throws Throwable {
+		TestAuxUtilities t = new TestAuxUtilities();
+		t.testByteVector();
+		t.errorCollector.verify();
 	}
 }