changeset 607:66f1cc23b906

Refresh revlogs if a change to a file has been detected; do not force reload of the whole repository
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Tue, 07 May 2013 16:52:46 +0200
parents 5daa42067e7c
children e1b29756f901
files src/org/tmatesoft/hg/internal/CommitFacility.java src/org/tmatesoft/hg/internal/RevlogChangeMonitor.java src/org/tmatesoft/hg/internal/RevlogStream.java src/org/tmatesoft/hg/repo/HgManifest.java src/org/tmatesoft/hg/repo/Revlog.java test/org/tmatesoft/hg/test/TestCommit.java
diffstat 6 files changed, 195 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/src/org/tmatesoft/hg/internal/CommitFacility.java	Tue May 07 14:16:35 2013 +0200
+++ b/src/org/tmatesoft/hg/internal/CommitFacility.java	Tue May 07 16:52:46 2013 +0200
@@ -98,6 +98,8 @@
 		final int clogRevisionIndex = clog.getRevisionCount();
 		ManifestRevision c1Manifest = new ManifestRevision(null, null);
 		ManifestRevision c2Manifest = new ManifestRevision(null, null);
+		final Nodeid p1Cset = p1Commit == NO_REVISION ? null : clog.getRevision(p1Commit);
+		final Nodeid p2Cset = p2Commit == NO_REVISION ? null : clog.getRevision(p2Commit);
 		if (p1Commit != NO_REVISION) {
 			repo.getRepo().getManifest().walk(p1Commit, p1Commit, c1Manifest);
 		}
@@ -205,8 +207,6 @@
 		dirstateBuilder.parents(changesetRev, Nodeid.NULL);
 		dirstateBuilder.serialize();
 		// update bookmarks
-		Nodeid p1Cset = p1Commit == NO_REVISION ? null : clog.getRevision(p1Commit);
-		Nodeid p2Cset = p2Commit == NO_REVISION ? null : clog.getRevision(p2Commit);
 		if (p1Commit != NO_REVISION || p2Commit != NO_REVISION) {
 			repo.getRepo().getBookmarks().updateActive(p1Cset, p2Cset, changesetRev);
 		}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/org/tmatesoft/hg/internal/RevlogChangeMonitor.java	Tue May 07 16:52:46 2013 +0200
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2013 TMate Software Ltd
+ *  
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * For information on how to redistribute this software under
+ * the terms of a license other than GNU General Public License
+ * contact TMate Software at support@hg4j.com
+ */
+package org.tmatesoft.hg.internal;
+
+import java.io.File;
+import java.util.WeakHashMap;
+
+/**
+ * Detect changes to revlog files. Not a general file change monitoring as we utilize the fact revlogs are append-only (and even in case
+ * of stripped-off tail revisions, with e.g. mq, detection approach is still valid).
+ * 
+ * @author Artem Tikhomirov
+ * @author TMate Software Ltd.
+ */
+public class RevlogChangeMonitor {
+	
+	private final WeakHashMap<File, Long> lastKnownSize;
+	private final File soleFile;
+	private long soleFileLength = -1;
+	
+	// use single for multiple files. TODO repository/session context shall provide
+	// alternative (configurable) implementations, so that Java7 users may supply better one
+	public RevlogChangeMonitor() {
+		lastKnownSize = new WeakHashMap<File, Long>();
+		soleFile = null;
+	}
+	
+	public RevlogChangeMonitor(File f) {
+		assert f != null;
+		lastKnownSize = null;
+		soleFile = f;
+	}
+	
+	public void touch(File f) {
+		assert f != null;
+		if (lastKnownSize == null) {
+			assert f == soleFile;
+			soleFileLength = f.length();
+		} else {
+			lastKnownSize.put(f, f.length());
+		}
+	}
+	
+	public boolean hasChanged(File f) {
+		assert f != null;
+		if (lastKnownSize == null) {
+			assert f == soleFile;
+			return soleFileLength != f.length();
+		} else {
+			Long lastSize = lastKnownSize.get(f);
+			if (lastSize == null) {
+				return true;
+			}
+			return f.length() != lastSize;
+		}
+	}
+}
--- a/src/org/tmatesoft/hg/internal/RevlogStream.java	Tue May 07 14:16:35 2013 +0200
+++ b/src/org/tmatesoft/hg/internal/RevlogStream.java	Tue May 07 16:52:46 2013 +0200
@@ -25,6 +25,8 @@
 import java.lang.ref.Reference;
 import java.lang.ref.ReferenceQueue;
 import java.lang.ref.SoftReference;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.zip.Inflater;
 
 import org.tmatesoft.hg.core.Nodeid;
@@ -70,11 +72,19 @@
 	// parents/children are analyzed.
 	private SoftReference<CachedRevision> lastRevisionRead;
 	private final ReferenceQueue<CachedRevision> lastRevisionQueue = new ReferenceQueue<CachedRevision>();
+	//
+	private final RevlogChangeMonitor changeTracker;
+	private List<Observer> observers;
+	private boolean shallDropDerivedCaches = false;
 
 	// if we need anything else from HgRepo, might replace DAP parameter with HgRepo and query it for DAP.
 	public RevlogStream(DataAccessProvider dap, File indexFile) {
 		this.dataAccess = dap;
 		this.indexFile = indexFile;
+		// TODO in fact, shall ask Internals for an instance (there we'll decide whether to use
+		// one monitor per multiple files or an instance per file; and let SessionContext pass
+		// alternative implementation)
+		changeTracker = new RevlogChangeMonitor(indexFile);
 	}
 
 	/**
@@ -214,7 +224,6 @@
 	 * @throws HgInvalidRevisionException if revisionIndex argument doesn't represent a valid record in the revlog
 	 */
 	public int baseRevision(int revisionIndex) throws HgInvalidControlFileException, HgInvalidRevisionException {
-		initOutline();
 		revisionIndex = checkRevisionIndex(revisionIndex);
 		return getBaseRevision(revisionIndex);
 	}
@@ -354,8 +363,41 @@
 			setLastRevisionRead(cr);
 		}
 	}
+	
+	public void attach(Observer listener) {
+		assert listener != null;
+		if (observers == null) {
+			observers = new ArrayList<Observer>(3);
+		}
+		observers.add(listener);
+	}
+	
+	public void detach(Observer listener) {
+		assert listener != null;
+		if (observers != null) {
+			observers.remove(listener);
+		}
+	}
+	
+	/*
+	 * Note, this method IS NOT a replacement for Observer. It has to be invoked when the validity of any
+	 * cache built using revision information is in doubt, but it provides reasonable value only till the
+	 * first initOutline() to be invoked, i.e. in [change..revlog read operation] time frame. If your code
+	 * accesses cached information without any prior explicit read operation, you shall consult this method
+	 * if next read operation would in fact bring changed content.
+	 * Observer is needed in addition to this method because any revlog read operation (e.g. Revlog#getLastRevision)
+	 * would clear shallDropDerivedCaches(), and if code relies only on this method to clear its derived caches,
+	 * it would miss the update.
+	 */
+	public boolean shallDropDerivedCaches() {
+		if (shallDropDerivedCaches) {
+			return shallDropDerivedCaches;
+		}
+		return shallDropDerivedCaches = changeTracker.hasChanged(indexFile);
+	}
 
 	void revisionAdded(int revisionIndex, Nodeid revision, int baseRevisionIndex, long revisionOffset) throws HgInvalidControlFileException {
+		shallDropDerivedCaches = true;
 		if (!outlineCached()) {
 			return;
 		}
@@ -421,10 +463,20 @@
 		return o + REVLOGV1_RECORD_SIZE * recordIndex;
 	}
 
+	// every access to index revlog goes after this method only.
 	private void initOutline() throws HgInvalidControlFileException {
+		// true to send out 'drop-your-caches' event after outline has been built
+		final boolean notifyReload;
 		if (outlineCached()) {
-			return;
+			if (!changeTracker.hasChanged(indexFile)) {
+				return;
+			}
+			notifyReload = true;
+		} else {
+			// no cached outline - inital read, do not send any reload/invalidate notifications
+			notifyReload = false;
 		}
+		changeTracker.touch(indexFile);
 		DataAccess da = getIndexStream(false);
 		try {
 			if (da.isEmpty()) {
@@ -483,6 +535,12 @@
 			throw new HgInvalidControlFileException("Failed to analyze revlog index", ex, indexFile);
 		} finally {
 			da.done();
+			if (notifyReload && observers != null) {
+				for (Observer l : observers) {
+					l.reloaded(this);
+				}
+				shallDropDerivedCaches = false;
+			}
 		}
 	}
 	
@@ -777,4 +835,8 @@
 		void next(int revisionIndex, int actualLen, int baseRevision, int linkRevision, int parent1Revision, int parent2Revision, byte[/*20*/] nodeid, DataAccess data);
 	}
 
+	public interface Observer {
+		// notify observer of invalidate/reload event in the stream
+		public void reloaded(RevlogStream src);
+	}
 }
--- a/src/org/tmatesoft/hg/repo/HgManifest.java	Tue May 07 14:16:35 2013 +0200
+++ b/src/org/tmatesoft/hg/repo/HgManifest.java	Tue May 07 16:52:46 2013 +0200
@@ -55,6 +55,14 @@
 	private RevisionMapper revisionMap;
 	private final EncodingHelper encodingHelper;
 	private final Path.Source pathFactory; 
+	private final RevlogStream.Observer revisionMapCleaner = new RevlogStream.Observer() {
+		public void reloaded(RevlogStream src) {
+			revisionMap = null;
+			// TODO RevlogDerivedCache<T> class, to wrap revisionMap and super.revisionLookup
+			// and their respective cleanup observers, or any other all-in-one alternative
+			// not to keep both field and it's cleaner
+		}
+	};
 	
 	/**
 	 * File flags recorded in manifest
@@ -244,15 +252,24 @@
 			throw new HgInvalidRevisionException("Can't use constants like WORKING_COPY or BAD_REVISION", null, changesetRevisionIndex);
 		}
 		// revisionNumber == TIP is processed by RevisionMapper 
-		if (revisionMap == null) {
-			revisionMap = new RevisionMapper(super.revisionLookup == null);
-			content.iterate(0, TIP, false, revisionMap);
-			revisionMap.fixReusedManifests();
-			if (super.useRevisionLookup && super.revisionLookup == null) {
+		if (revisionMap == null || content.shallDropDerivedCaches()) {
+			content.detach(revisionMapCleaner);
+			final boolean buildOwnLookup = super.revisionLookup == null;
+			RevisionMapper rmap = new RevisionMapper(buildOwnLookup);
+			content.iterate(0, TIP, false, rmap);
+			rmap.fixReusedManifests();
+			if (buildOwnLookup && super.useRevisionLookup) {
 				// reuse RevisionLookup if there's none yet
-				super.revisionLookup = revisionMap.manifestNodeids;
+				super.setRevisionLookup(rmap.manifestNodeids);
 			}
-			revisionMap.manifestNodeids = null;
+			rmap.manifestNodeids = null;
+			revisionMap = rmap;
+			// although in most cases modified manifest is accessed through one of the methods in this class
+			// and hence won't have a chance till this moment to be reloaded via revisionMapCleaner
+			// (RevlogStream sends events on attempt to read revlog, and so far we haven't tried to read anything,
+			// it's still reasonable to have this cleaner attached, just in case any method from Revlog base class
+			// has been called (e.g. getLastRevision())
+			content.attach(revisionMapCleaner);
 		}
 		return revisionMap.at(changesetRevisionIndex);
 	}
@@ -333,6 +350,15 @@
 	}
 
 
+	/*package-local*/ void dropCachesOnChangelogChange() {
+		// sort of a hack as it may happen that #fromChangelog()
+		// is invoked for modified repository where revisionMap still points to an old state
+		// Since there's no access to RevlogStream in #fromChangelog() if there's revisionMap 
+		// in place, there's no chance for RevlogStream to detect the change and to dispatch 
+		// change notification so that revisionMap got cleared.
+		revisionMap = null;
+	}
+
 	/**
 	 * @param changelogRevisionIndexes non-null
 	 * @param inspector may be null if reporting of missing manifests is not needed
--- a/src/org/tmatesoft/hg/repo/Revlog.java	Tue May 07 14:16:35 2013 +0200
+++ b/src/org/tmatesoft/hg/repo/Revlog.java	Tue May 07 16:52:46 2013 +0200
@@ -56,6 +56,7 @@
 	protected final RevlogStream content;
 	protected final boolean useRevisionLookup;
 	protected RevisionLookup revisionLookup;
+	private final RevlogStream.Observer revisionLookupCleaner;
 
 	protected Revlog(HgRepository hgRepo, RevlogStream contentStream, boolean needRevisionLookup) {
 		if (hgRepo == null) {
@@ -67,6 +68,16 @@
 		repo = hgRepo;
 		content = contentStream;
 		useRevisionLookup = needRevisionLookup;
+		if (needRevisionLookup) {
+			revisionLookupCleaner = new RevlogStream.Observer() {
+				
+				public void reloaded(RevlogStream src) {
+					revisionLookup = null;
+				}
+			};
+		} else {
+			revisionLookupCleaner = null;
+		}
 	}
 	
 	// invalid Revlog
@@ -74,6 +85,7 @@
 		repo = hgRepo;
 		content = null;
 		useRevisionLookup = false;
+		revisionLookupCleaner = null;
 	}
 
 	public final HgRepository getRepo() {
@@ -162,8 +174,9 @@
 	
 	private int doFindWithCache(Nodeid nid) {
 		if (useRevisionLookup) {
-			if (revisionLookup == null) {
-				revisionLookup = RevisionLookup.createFor(content);
+			if (revisionLookup == null || content.shallDropDerivedCaches()) {
+				content.detach(revisionLookupCleaner);
+				setRevisionLookup(RevisionLookup.createFor(content));
 			}
 			return revisionLookup.findIndex(nid);
 		} else {
@@ -172,6 +185,16 @@
 	}
 	
 	/**
+	 * use selected helper for revision lookup, register appropriate listeners to clear cache on revlog changes
+	 * @param rl not <code>null</code>
+	 */
+	protected void setRevisionLookup(RevisionLookup rl) {
+		assert rl != null;
+		revisionLookup = rl;
+		content.attach(revisionLookupCleaner);
+	}
+	
+	/**
 	 * Note, {@link Nodeid#NULL} nodeid is not reported as known in any revlog.
 	 * 
 	 * @param nodeid
--- a/test/org/tmatesoft/hg/test/TestCommit.java	Tue May 07 14:16:35 2013 +0200
+++ b/test/org/tmatesoft/hg/test/TestCommit.java	Tue May 07 16:52:46 2013 +0200
@@ -133,8 +133,6 @@
 		Nodeid commitRev1 = cf.commit("FIRST");
 		contentProvider.done();
 		//
-		// FIXME requirement to reload repository is disgusting 
-		hgRepo = new HgLookup().detect(repoLoc);
 		List<HgChangeset> commits = new HgLogCommand(hgRepo).range(parentCsetRevIndex+1, TIP).execute();
 		assertEquals(1, commits.size());
 		HgChangeset c1 = commits.get(0);
@@ -164,8 +162,6 @@
 		contentProvider.done();
 		// Note, working directory still points to original revision, CommitFacility doesn't update dirstate
 		//
-		// FIXME requirement to reload repository is disgusting 
-		hgRepo = new HgLookup().detect(repoLoc);
 		List<HgChangeset> commits = new HgLogCommand(hgRepo).changeset(commitRev).execute();
 		HgChangeset cmt = commits.get(0);
 		errorCollector.assertEquals(1, cmt.getAddedFiles().size());
@@ -211,8 +207,6 @@
 		Nodeid commitRev3 = cf.commit("THIRD");
 		contentProvider.done();
 		//
-		// FIXME requirement to reload repository is disgusting 
-		hgRepo = new HgLookup().detect(repoLoc);
 		List<HgChangeset> commits = new HgLogCommand(hgRepo).range(parentCsetRevIndex+1, TIP).execute();
 		assertEquals(3, commits.size());
 		HgChangeset c1 = commits.get(0);
@@ -247,7 +241,6 @@
 		Nodeid c1 = cmd.getCommittedRevision();
 		
 		// check that modified files are no longer reported as such
-		hgRepo = new HgLookup().detect(repoLoc);
 		TestStatus.StatusCollector status = new TestStatus.StatusCollector();
 		new HgStatusCommand(hgRepo).all().execute(status);
 		errorCollector.assertTrue(status.getErrors().isEmpty());
@@ -267,7 +260,6 @@
 		errorCollector.assertTrue(r.isOk());
 		Nodeid c2 = cmd.getCommittedRevision();
 		//
-		hgRepo = new HgLookup().detect(repoLoc);
 		int lastRev = hgRepo.getChangelog().getLastRevision();
 		List<HgChangeset> csets = new HgLogCommand(hgRepo).range(lastRev-1, lastRev).execute();
 		errorCollector.assertEquals(csets.get(0).getNodeid(), c1);