# HG changeset patch # User Artem Tikhomirov # Date 1366116297 -7200 # Node ID 88afffd39899feb9aa1be8ad4780fd311c9cfe9c # Parent e49f9d9513fa2e92c9e92b07f7d6ba168777ee46 Improve memory consumption of HgManifest#getFileRevision(): avoid extra byte[] instances diff -r e49f9d9513fa -r 88afffd39899 cmdline/org/tmatesoft/hg/console/Main.java --- a/cmdline/org/tmatesoft/hg/console/Main.java Fri Apr 12 19:50:21 2013 +0200 +++ b/cmdline/org/tmatesoft/hg/console/Main.java Tue Apr 16 14:44:57 2013 +0200 @@ -28,6 +28,7 @@ import java.util.Map; import org.tmatesoft.hg.core.HgChangeset; +import org.tmatesoft.hg.core.HgChangesetFileSneaker; import org.tmatesoft.hg.core.HgChangesetTreeHandler; import org.tmatesoft.hg.core.HgCheckoutCommand; import org.tmatesoft.hg.core.HgException; @@ -42,6 +43,7 @@ import org.tmatesoft.hg.internal.ByteArrayChannel; import org.tmatesoft.hg.internal.ConfigFile; import org.tmatesoft.hg.internal.DigestHelper; +import org.tmatesoft.hg.internal.ManifestRevision; import org.tmatesoft.hg.internal.PathGlobMatcher; import org.tmatesoft.hg.internal.RelativePathRewrite; import org.tmatesoft.hg.internal.StreamLogFacility; @@ -100,7 +102,8 @@ public static void main(String[] args) throws Exception { Main m = new Main(args); - m.testRevert(); + m.checkFileSneakerPerformance2(); +// m.testRevert(); // m.testCheckout(); // m.tryExtensions(); // m.dumpBookmarks(); @@ -127,6 +130,45 @@ // m.bunchOfTests(); } + private void checkFileSneakerPerformance() throws Exception { + HgChangesetFileSneaker fs1 = new HgChangesetFileSneaker(hgRepo); + HgChangesetFileSneaker fs2 = new HgChangesetFileSneaker(hgRepo); + fs1.followRenames(true); + fs2.followRenames(true); + Nodeid cset = hgRepo.getChangelog().getRevision(TIP); + Path fname = Path.create("dir3/file8"); + fs1.changeset(cset); + fs2.changeset(cset); +// hgRepo.getManifest().getFileRevision(TIP, fname); + final long start1 = System.nanoTime(); + boolean e1 = fs1.checkExists(fname); + final long end1 = System.nanoTime(); + boolean e2 = fs2.checkExists(fname); + final long end2 = System.nanoTime(); + Nodeid fr = hgRepo.getManifest().getFileRevision(TIP, fname); + final long end3 = System.nanoTime(); + System.out.printf("\t1st run: %d ms, %b\n\t2nd run: %d ms, %b\n\tfile only: %d ms", (end1 - start1) / 1000000, e1, (end2 - end1) / 1000000, e2, (end3-end2)/1000000); + if (!fr.equals(fs1.revision()) || !fr.equals(fs2.revision())) { + throw new AssertionError(); + } + ManifestRevision mr = new ManifestRevision(null, null); + final long _s1 = System.nanoTime(); + hgRepo.getManifest().walk(2, 2, mr); + final long _e1 = System.nanoTime(); + hgRepo.getManifest().getFileRevision(2, fname); + final long _e2 = System.nanoTime(); + System.out.printf("\n\tManifestRevision:%d ms, getFileRevision:%d ms\n", (_e1-_s1)/1000000, (_e2-_e1)/1000000); + } + + // -agentlib:hprof=cpu=times,heap=sites,depth=10 + private void checkFileSneakerPerformance2() throws Exception { + Path fname = Path.create("dir3/file8"); + hgRepo.getManifest().getFileRevision(2, fname); +// ManifestRevision mr = new ManifestRevision(null, null); +// hgRepo.getManifest().walk(2, 2, mr); + } + + private void testRevert() throws Exception { HgRevertCommand cmd = new HgRevertCommand(hgRepo); cmd.file(Path.create("a.txt")).execute(); diff -r e49f9d9513fa -r 88afffd39899 src/org/tmatesoft/hg/internal/ByteVector.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/org/tmatesoft/hg/internal/ByteVector.java Tue Apr 16 14:44:57 2013 +0200 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2013 TMate Software Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * For information on how to redistribute this software under + * the terms of a license other than GNU General Public License + * contact TMate Software at support@hg4j.com + */ +package org.tmatesoft.hg.internal; + +import java.io.ByteArrayOutputStream; + +/** + * Alternative to {@link ByteArrayOutputStream}, with extra operation that prevent extra byte[] instances + * + * @author Artem Tikhomirov + * @author TMate Software Ltd. + */ +public class ByteVector { + private byte[] data; + private int count; + private final int increment; + + + public ByteVector(int initialSize, int increment) { + data = new byte[initialSize]; + this.increment = increment; + } + + public void add(int b) { + if (count == data.length) { + byte[] newData = new byte[count + increment]; + System.arraycopy(data, 0, newData, 0, count); + data = newData; + } + data[count++] = (byte) b; + } + + public int size() { + return count; + } + + public void clear() { + count = 0; + } + + public boolean equalsTo(byte[] array) { + if (array == null || array.length != count) { + return false; + } + for (int i = 0; i < count; i++) { + if (data[i] != array[i]) { + return false; + } + } + return true; + } + + /** + * Copies content of this vector into destination array. + * @param destination array, greater or equal to {@link #size()} of the vector + */ + public void copyTo(byte[] destination) { + if (destination == null || destination.length < count) { + throw new IllegalArgumentException(); + } + System.arraycopy(data, 0, destination, 0, count); + } + + public byte[] toByteArray() { + byte[] rv = new byte[count]; + copyTo(rv); + return rv; + } +} diff -r e49f9d9513fa -r 88afffd39899 src/org/tmatesoft/hg/internal/IntVector.java --- a/src/org/tmatesoft/hg/internal/IntVector.java Fri Apr 12 19:50:21 2013 +0200 +++ b/src/org/tmatesoft/hg/internal/IntVector.java Tue Apr 16 14:44:57 2013 +0200 @@ -142,7 +142,7 @@ if (increment == 0) { throw new UnsupportedOperationException("This vector is not allowed to expand"); } - int newCapacity = increment < 0 ? data.length << 1 : data.length + increment; + int newCapacity = increment < 0 ? data.length << 1 : (data.length + increment); if (newCapacityHint > 0 && newCapacity < newCapacityHint) { newCapacity = newCapacityHint; } diff -r e49f9d9513fa -r 88afffd39899 src/org/tmatesoft/hg/repo/HgManifest.java --- a/src/org/tmatesoft/hg/repo/HgManifest.java Fri Apr 12 19:50:21 2013 +0200 +++ b/src/org/tmatesoft/hg/repo/HgManifest.java Tue Apr 16 14:44:57 2013 +0200 @@ -20,26 +20,26 @@ import static org.tmatesoft.hg.repo.HgRepository.*; import static org.tmatesoft.hg.util.LogFacility.Severity.Info; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import org.tmatesoft.hg.core.HgChangesetFileSneaker; import org.tmatesoft.hg.core.Nodeid; +import org.tmatesoft.hg.internal.ByteVector; import org.tmatesoft.hg.internal.Callback; import org.tmatesoft.hg.internal.DataAccess; import org.tmatesoft.hg.internal.DigestHelper; import org.tmatesoft.hg.internal.EncodingHelper; +import org.tmatesoft.hg.internal.IdentityPool; import org.tmatesoft.hg.internal.IntMap; import org.tmatesoft.hg.internal.IterateControlMediator; import org.tmatesoft.hg.internal.Lifecycle; -import org.tmatesoft.hg.internal.IdentityPool; import org.tmatesoft.hg.internal.RevlogStream; import org.tmatesoft.hg.util.CancelSupport; +import org.tmatesoft.hg.util.LogFacility.Severity; import org.tmatesoft.hg.util.Path; import org.tmatesoft.hg.util.ProgressSupport; -import org.tmatesoft.hg.util.LogFacility.Severity; /** @@ -51,7 +51,8 @@ */ public final class HgManifest extends Revlog { private RevisionMapper revisionMap; - private EncodingHelper encodingHelper; + private final EncodingHelper encodingHelper; + private final Path.Source pathFactory; /** * File flags recorded in manifest @@ -111,11 +112,19 @@ } throw new IllegalStateException(toString()); } + + public int fsMode() { + if (this == Exec) { + return 0755; + } + return 0644; + } } /*package-local*/ HgManifest(HgRepository hgRepo, RevlogStream content, EncodingHelper eh) { super(hgRepo, content); encodingHelper = eh; + pathFactory = hgRepo.getSessionContext().getPathFactory(); } /** @@ -446,8 +455,8 @@ public Path freeze() { if (result == null) { - Path.Source pathFactory = HgManifest.this.getRepo().getSessionContext().getPathFactory(); - result = pathFactory.path(HgManifest.this.encodingHelper.fromManifest(data, start, length)); + Path.Source pf = HgManifest.this.pathFactory; + result = pf.path(HgManifest.this.encodingHelper.fromManifest(data, start, length)); // release reference to bigger data array, make a copy of relevant part only // use original bytes, not those from String above to avoid cache misses due to different encodings byte[] d = new byte[length]; @@ -689,16 +698,14 @@ } public void next(int revisionNumber, int actualLen, int baseRevision, int linkRevision, int parent1Revision, int parent2Revision, byte[] nodeid, DataAccess data) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteVector byteVector = new ByteVector(256, 128); // allocate for long paths right away try { byte b; while (!data.isEmpty() && (b = data.readByte()) != '\n') { if (b != 0) { - bos.write(b); + byteVector.add(b); } else { - byte[] byteArray = bos.toByteArray(); - bos.reset(); - if (Arrays.equals(filenameAsBytes, byteArray)) { + if (byteVector.equalsTo(filenameAsBytes)) { Nodeid fileRev = null; Flags flags = null; if (csetIndex2FileRev != null || delegate != null) { @@ -709,15 +716,15 @@ data.skip(40); } if (csetIndex2Flags != null || delegate != null) { + byteVector.clear(); while (!data.isEmpty() && (b = data.readByte()) != '\n') { - bos.write(b); + byteVector.add(b); } - if (bos.size() == 0) { + if (byteVector.size() == 0) { flags = Flags.RegularFile; } else { - flags = Flags.parse(bos.toByteArray(), 0, bos.size()); + flags = Flags.parse(byteVector.toByteArray(), 0, byteVector.size()); } - } if (delegate != null) { assert flags != null; @@ -741,6 +748,8 @@ // else skip to the end of line while (!data.isEmpty() && (b = data.readByte()) != '\n') ; + + byteVector.clear(); } } } catch (IOException ex) { diff -r e49f9d9513fa -r 88afffd39899 test/org/tmatesoft/hg/test/TestAuxUtilities.java --- a/test/org/tmatesoft/hg/test/TestAuxUtilities.java Fri Apr 12 19:50:21 2013 +0200 +++ b/test/org/tmatesoft/hg/test/TestAuxUtilities.java Tue Apr 16 14:44:57 2013 +0200 @@ -31,6 +31,7 @@ import org.tmatesoft.hg.core.HgCatCommand; import org.tmatesoft.hg.core.Nodeid; import org.tmatesoft.hg.internal.ArrayHelper; +import org.tmatesoft.hg.internal.ByteVector; import org.tmatesoft.hg.internal.IntVector; import org.tmatesoft.hg.internal.PathScope; import org.tmatesoft.hg.internal.RangeSeq; @@ -543,8 +544,41 @@ errorCollector.assertFalse(rs.includesTargetLine(12)); } + @Test + public void testByteVector() { + ByteVector v = new ByteVector(4, 2); + v.add(7); + v.add(9); + errorCollector.assertEquals(2, v.size()); + v.clear(); + errorCollector.assertEquals(0, v.size()); + v.add(10); + v.add(9); + v.add(8); + v.add(7); + v.add(6); + errorCollector.assertEquals(5, v.size()); + v.add(5); + v.add(4); + errorCollector.assertEquals(7, v.size()); + byte x = 10; + for (byte d : v.toByteArray()) { + errorCollector.assertEquals(x, d); + x--; + } + x = 10; + byte[] dd = new byte[10]; + v.copyTo(dd); + for (int i = 0; i < v.size(); i++) { + errorCollector.assertEquals(x, dd[i]); + x--; + } + errorCollector.assertTrue(v.equalsTo(new byte[] { 10,9,8,7,6,5,4 })); + } - public static void main(String[] args) throws Exception { - new TestAuxUtilities().testRepositoryConfig(); + public static void main(String[] args) throws Throwable { + TestAuxUtilities t = new TestAuxUtilities(); + t.testByteVector(); + t.errorCollector.verify(); } }