Mercurial > jhg

changeset 177:e10225daface

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use POST for long between queries. Batch between queries (pass multiple pairs to a server) to minimize number thereof
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Sat, 02 Apr 2011 23:05:28 +0200
parents a8df7162ec75
children 62665d8f0686
files cmdline/org/tmatesoft/hg/console/Incoming.java cmdline/org/tmatesoft/hg/console/Remote.java src/org/tmatesoft/hg/repo/HgRemoteRepository.java
diffstat 3 files changed, 136 insertions(+), 40 deletions(-) [+]
line wrap: on
line diff
--- a/cmdline/org/tmatesoft/hg/console/Incoming.java	Sat Apr 02 03:01:14 2011 +0200
+++ b/cmdline/org/tmatesoft/hg/console/Incoming.java	Sat Apr 02 23:05:28 2011 +0200
@@ -23,11 +23,13 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.tmatesoft.hg.core.HgBadStateException;
@@ -38,6 +40,7 @@
 import org.tmatesoft.hg.repo.HgChangelog;
 import org.tmatesoft.hg.repo.HgLookup;
 import org.tmatesoft.hg.repo.HgRemoteRepository;
+import org.tmatesoft.hg.repo.HgRemoteRepository.Range;
 import org.tmatesoft.hg.repo.HgRemoteRepository.RemoteBranch;
 import org.tmatesoft.hg.repo.HgRepository;
 
@@ -133,6 +136,9 @@
 		int totalQueries = 1;
 		HashSet<Nodeid> queried = new HashSet<Nodeid>();
 		while(!datas.isEmpty()) {
+			// keep record of those planned to be queried next time we call between()
+			// although may keep these in queried, if really don't want separate collection
+			HashSet<Nodeid> scheduled = new HashSet<Nodeid>();  
 			do {
 				DataEntry de = datas.removeFirst();
 				// populate result with discovered elements between de.qiueryRoot and branch's head
@@ -149,9 +155,10 @@
 					for (int i =1, j = 0; j < de.entries.size(); i = i<<1, j++) {
 						int idx = de.headIndex + i;
 						Nodeid x = de.entries.get(j);
-						if (!queried.contains(x) && (rootIndex == -1 || rootIndex - de.headIndex > 1)) {
+						if (!queried.contains(x) && !scheduled.contains(x) && (rootIndex == -1 || rootIndex - de.headIndex > 1)) {
 							/*queries for elements right before head is senseless, but unless we know head's index, do it anyway*/
 							toQuery.add(new DataEntry(x, idx, null));
+							scheduled.add(x);
 						}
 					}
 				}
@@ -159,19 +166,31 @@
 			if (!toQuery.isEmpty()) {
 				totalQueries++;
 			}
+			// for each query, create an between request range, keep record Range->DataEntry to know range's start index  
+			LinkedList<HgRemoteRepository.Range> betweenBatch = new LinkedList<HgRemoteRepository.Range>();
+			HashMap<HgRemoteRepository.Range, DataEntry> rangeToEntry = new HashMap<HgRemoteRepository.Range, DataEntry>();
 			for (DataEntry de : toQuery) {
-				if (!queried.contains(de.queryHead)) {
-					queried.add(de.queryHead);
-					List<Nodeid> between = hgRemote.between(de.queryHead, rb.root);
-					if (rootIndex == -1 && between.size() == 1) {
+				queried.add(de.queryHead);
+				HgRemoteRepository.Range r = new HgRemoteRepository.Range(rb.root, de.queryHead);
+				betweenBatch.add(r);
+				rangeToEntry.put(r, de);
+			}
+			if (!betweenBatch.isEmpty()) {
+				Map<Range, List<Nodeid>> between = hgRemote.between(betweenBatch);
+				for (Entry<Range, List<Nodeid>> e : between.entrySet()) {
+					DataEntry de = rangeToEntry.get(e.getKey());
+					assert de != null;
+					de.entries = e.getValue();
+					if (rootIndex == -1 && de.entries.size() == 1) {
 						// returned sequence of length 1 means we used element from [head-2] as root
 						int numberOfElementsExcludingRootAndHead = de.headIndex + 1;
 						rootIndex = numberOfElementsExcludingRootAndHead + 1;
 						System.out.printf("On query %d found out exact number of missing elements: %d\n", totalQueries, numberOfElementsExcludingRootAndHead);
 					}
-					de.entries = between;
 					datas.add(de); // queue up to record result and construct further requests
 				}
+				betweenBatch.clear();
+				rangeToEntry.clear();
 			}
 			toQuery.clear();
 		}
@@ -189,6 +208,7 @@
 			}
 			fromRootToHead.addFirst(n); // reverse order
 		}
+		System.out.println("Total queries:" + totalQueries);
 		if (!resultOk) {
 			throw new HgBadStateException("See console for details"); // FIXME
 		}
--- a/cmdline/org/tmatesoft/hg/console/Remote.java	Sat Apr 02 03:01:14 2011 +0200
+++ b/cmdline/org/tmatesoft/hg/console/Remote.java	Sat Apr 02 23:05:28 2011 +0200
@@ -20,6 +20,8 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
@@ -93,9 +95,9 @@
 		cfg.addLocation(new File(System.getProperty("user.home"), ".hgrc"));
 		String svnkitServer = cfg.getSection("paths").get("svnkit");
 //		URL url = new URL(svnkitServer + "?cmd=branches&nodes=30bd389788464287cee22ccff54c330a4b715de5");
-		URL url = new URL(svnkitServer + "?cmd=between&pairs=71ddbf8603e8e09d54ac9c5fe4bb5ae824589f1d-8c8e3f372fa1fbfcf92b004b6f2ada2dbaf60028"); 
+		URL url = new URL(svnkitServer + "?cmd=between"); 
 //		URL url = new URL(svnkitServer + "?cmd=changegroup&roots=" + Nodeid.NULL.toString());
-//		URL url = new URL("http://localhost:8000/" + "?cmd=stream_out");
+//		URL url = new URL("http://localhost:8000/" + "?cmd=between");
 //		URL url = new URL(svnkitServer + "?cmd=stream_out");
 	
 		SSLContext sslContext = SSLContext.getInstance("SSL");
@@ -118,11 +120,22 @@
 		//
 		sslContext.init(null, new TrustManager[] { new TrustEveryone() }, null);
 		HttpsURLConnection urlConnection = (HttpsURLConnection) url.openConnection();
-		urlConnection.addRequestProperty("User-Agent", "jhg/0.1.0");
-		urlConnection.addRequestProperty("Accept", "application/mercurial-0.1");
-		urlConnection.addRequestProperty("Authorization", "Basic " + authInfo);
+//		HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
+		urlConnection.setRequestProperty("User-Agent", "jhg/0.1.0");
+		urlConnection.setRequestProperty("Accept", "application/mercurial-0.1");
+		urlConnection.setRequestProperty("Authorization", "Basic " + authInfo);
 		urlConnection.setSSLSocketFactory(sslContext.getSocketFactory());
-		urlConnection.connect();
+		byte[] body = "pairs=30bd389788464287cee22ccff54c330a4b715de5-dbd663faec1f0175619cf7668bddc6350548b8d6".getBytes();
+		urlConnection.setRequestMethod("POST");
+		urlConnection.setRequestProperty("Content-Length", String.valueOf(body.length));
+		urlConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+		urlConnection.setDoOutput(true);
+		urlConnection.setDoInput(true);
+//		urlConnection.connect();
+		OutputStream os = urlConnection.getOutputStream();
+		os.write(body);
+		os.flush();
+		os.close();
 		System.out.println("Query:" + url.getQuery());
 		System.out.println("Response headers:");
 		final Map<String, List<String>> headerFields = urlConnection.getHeaderFields();
--- a/src/org/tmatesoft/hg/repo/HgRemoteRepository.java	Sat Apr 02 03:01:14 2011 +0200
+++ b/src/org/tmatesoft/hg/repo/HgRemoteRepository.java	Sat Apr 02 23:05:28 2011 +0200
@@ -18,8 +18,10 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.io.StreamTokenizer;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -27,6 +29,7 @@
 import java.security.cert.X509Certificate;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -39,6 +42,7 @@
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
 
+import org.tmatesoft.hg.core.HgBadStateException;
 import org.tmatesoft.hg.core.HgException;
 import org.tmatesoft.hg.core.Nodeid;
 
@@ -106,25 +110,99 @@
 	}
 	
 	public List<Nodeid> between(Nodeid tip, Nodeid base) throws HgException {
+		Range r = new Range(base, tip);
+		// XXX shall handle errors like no range key in the returned map, not sure how.
+		return between(Collections.singletonList(r)).get(r);
+	}
+
+	/**
+	 * @param ranges
+	 * @return map, where keys are input instances, values are corresponding server reply
+	 * @throws HgException 
+	 */
+	public Map<Range, List<Nodeid>> between(Collection<Range> ranges) throws HgException {
+		if (ranges.isEmpty()) {
+			return Collections.emptyMap();
+		}
+		// if fact, shall do other way round, this method shall send 
+		LinkedHashMap<Range, List<Nodeid>> rv = new LinkedHashMap<HgRemoteRepository.Range, List<Nodeid>>(ranges.size() * 4 / 3);
+		StringBuilder sb = new StringBuilder(20 + ranges.size() * 82);
+		sb.append("pairs=");
+		for (Range r : ranges) {
+			sb.append(r.end.toString());
+			sb.append('-');
+			sb.append(r.start.toString());
+			sb.append('+');
+		}
+		if (sb.charAt(sb.length() - 1) == '+') {
+			// strip last space 
+			sb.setLength(sb.length() - 1);
+		}
 		try {
-			LinkedList<Nodeid> rv = new LinkedList<Nodeid>();
-			URL u = new URL(url, url.getPath() + "?cmd=between&pairs=" + tip.toString() + '-' + base.toString());
-			URLConnection c = setupConnection(u.openConnection());
-			c.connect();
-			System.out.println("Query:" + u.getQuery());
+			boolean usePOST = ranges.size() > 3;
+			URL u = new URL(url, url.getPath() + "?cmd=between" + (usePOST ? "" : '&' + sb.toString()));
+			HttpURLConnection c = setupConnection(u.openConnection());
+			if (usePOST) {
+				c.setRequestMethod("POST");
+				c.setRequestProperty("Content-Length", String.valueOf(sb.length()/*nodeids are ASCII, bytes == characters */));
+				c.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+				c.setDoOutput(true);
+				c.connect();
+				OutputStream os = c.getOutputStream();
+				os.write(sb.toString().getBytes());
+				os.flush();
+				os.close();
+			} else {
+				c.connect();
+			}
+			System.out.printf("%d ranges, method:%s \n", ranges.size(), c.getRequestMethod());
+			System.out.printf("Query (%d bytes):%s\n", u.getQuery().length(), u.getQuery());
 			System.out.println("Response headers:");
 			final Map<String, List<String>> headerFields = c.getHeaderFields();
 			for (String s : headerFields.keySet()) {
 				System.out.printf("%s: %s\n", s, c.getHeaderField(s));
 			}
-			InputStream is = c.getInputStream();
+			InputStreamReader is = new InputStreamReader(c.getInputStream(), "US-ASCII");
 			StreamTokenizer st = new StreamTokenizer(is);
 			st.ordinaryChars('0', '9');
 			st.wordChars('0', '9');
+			st.eolIsSignificant(true);
+			Iterator<Range> rangeItr = ranges.iterator();
+			LinkedList<Nodeid> currRangeList = null;
+			Range currRange = null;
+			boolean possiblyEmptyNextLine = true;
 			while (st.nextToken() != StreamTokenizer.TT_EOF) {
-				System.out.println(st.sval);
-				Nodeid nid = Nodeid.fromAscii(st.sval);
-				rv.addLast(nid);
+				if (st.ttype == StreamTokenizer.TT_EOL) {
+					if (possiblyEmptyNextLine) {
+						// newline follows newline;
+						assert currRange == null;
+						assert currRangeList == null;
+						if (!rangeItr.hasNext()) {
+							throw new HgBadStateException();
+						}
+						rv.put(rangeItr.next(), Collections.<Nodeid>emptyList());
+					} else {
+						if (currRange == null || currRangeList == null) {
+							throw new HgBadStateException();
+						}
+						// indicate next range value is needed
+						currRange = null;
+						currRangeList = null;
+						possiblyEmptyNextLine = true;
+					}
+				} else {
+					possiblyEmptyNextLine = false;
+					if (currRange == null) {
+						if (!rangeItr.hasNext()) {
+							throw new HgBadStateException();
+						}
+						currRange = rangeItr.next();
+						currRangeList = new LinkedList<Nodeid>();
+						rv.put(currRange, currRangeList);
+					}
+					Nodeid nid = Nodeid.fromAscii(st.sval);
+					currRangeList.addLast(nid);
+				}
 			}
 			is.close();
 			return rv;
@@ -135,21 +213,6 @@
 		}
 	}
 
-	/**
-	 * @param ranges
-	 * @return map, where keys are input instances, values are corresponding server reply
-	 * @throws HgException 
-	 */
-	public Map<Range, List<Nodeid>> between(Collection<Range> ranges) throws HgException {
-		// if fact, shall do other way round, this method shall send 
-		LinkedHashMap<Range, List<Nodeid>> rv = new LinkedHashMap<HgRemoteRepository.Range, List<Nodeid>>(ranges.size() * 4 / 3);
-		for (Range r : ranges) {
-			List<Nodeid> between = between(r.end, r.start);
-			rv.put(r, between);
-		}
-		return rv;
-	}
-
 	public List<RemoteBranch> branches(List<Nodeid> nodes) {
 		return Collections.emptyList();
 	}
@@ -159,8 +222,8 @@
 		return new HgLookup().loadBundle(new File("/temp/hg/hg-bundle-000000000000-gz.tmp"));
 	}
 	
-	private URLConnection setupConnection(URLConnection urlConnection) {
-		urlConnection.addRequestProperty("User-Agent", "hg4j/0.5.0");
+	private HttpURLConnection setupConnection(URLConnection urlConnection) {
+		urlConnection.setRequestProperty("User-Agent", "hg4j/0.5.0");
 		urlConnection.addRequestProperty("Accept", "application/mercurial-0.1");
 		if (authInfo != null) {
 			urlConnection.addRequestProperty("Authorization", "Basic " + authInfo);
@@ -168,7 +231,7 @@
 		if (sslContext != null) {
 			((HttpsURLConnection) urlConnection).setSSLSocketFactory(sslContext.getSocketFactory());
 		}
-		return urlConnection;
+		return (HttpURLConnection) urlConnection;
 	}
 
 	public static final class Range {