# HG changeset patch # User Artem Tikhomirov # Date 1301778328 -7200 # Node ID e10225dafacede3af3759a0b39daa10626ae80ea # Parent a8df7162ec7513bb3807da91d1b4006c8dad69f3 Use POST for long between queries. Batch between queries (pass multiple pairs to a server) to minimize number thereof diff -r a8df7162ec75 -r e10225daface cmdline/org/tmatesoft/hg/console/Incoming.java --- a/cmdline/org/tmatesoft/hg/console/Incoming.java Sat Apr 02 03:01:14 2011 +0200 +++ b/cmdline/org/tmatesoft/hg/console/Incoming.java Sat Apr 02 23:05:28 2011 +0200 @@ -23,11 +23,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import org.tmatesoft.hg.core.HgBadStateException; @@ -38,6 +40,7 @@ import org.tmatesoft.hg.repo.HgChangelog; import org.tmatesoft.hg.repo.HgLookup; import org.tmatesoft.hg.repo.HgRemoteRepository; +import org.tmatesoft.hg.repo.HgRemoteRepository.Range; import org.tmatesoft.hg.repo.HgRemoteRepository.RemoteBranch; import org.tmatesoft.hg.repo.HgRepository; @@ -133,6 +136,9 @@ int totalQueries = 1; HashSet queried = new HashSet(); while(!datas.isEmpty()) { + // keep record of those planned to be queried next time we call between() + // although may keep these in queried, if really don't want separate collection + HashSet scheduled = new HashSet(); do { DataEntry de = datas.removeFirst(); // populate result with discovered elements between de.qiueryRoot and branch's head @@ -149,9 +155,10 @@ for (int i =1, j = 0; j < de.entries.size(); i = i<<1, j++) { int idx = de.headIndex + i; Nodeid x = de.entries.get(j); - if (!queried.contains(x) && (rootIndex == -1 || rootIndex - de.headIndex > 1)) { + if (!queried.contains(x) && !scheduled.contains(x) && (rootIndex == -1 || rootIndex - de.headIndex > 1)) { /*queries for elements right before head is senseless, but unless we know head's index, do it anyway*/ toQuery.add(new DataEntry(x, idx, null)); + scheduled.add(x); } } } @@ -159,19 +166,31 @@ if (!toQuery.isEmpty()) { totalQueries++; } + // for each query, create an between request range, keep record Range->DataEntry to know range's start index + LinkedList betweenBatch = new LinkedList(); + HashMap rangeToEntry = new HashMap(); for (DataEntry de : toQuery) { - if (!queried.contains(de.queryHead)) { - queried.add(de.queryHead); - List between = hgRemote.between(de.queryHead, rb.root); - if (rootIndex == -1 && between.size() == 1) { + queried.add(de.queryHead); + HgRemoteRepository.Range r = new HgRemoteRepository.Range(rb.root, de.queryHead); + betweenBatch.add(r); + rangeToEntry.put(r, de); + } + if (!betweenBatch.isEmpty()) { + Map> between = hgRemote.between(betweenBatch); + for (Entry> e : between.entrySet()) { + DataEntry de = rangeToEntry.get(e.getKey()); + assert de != null; + de.entries = e.getValue(); + if (rootIndex == -1 && de.entries.size() == 1) { // returned sequence of length 1 means we used element from [head-2] as root int numberOfElementsExcludingRootAndHead = de.headIndex + 1; rootIndex = numberOfElementsExcludingRootAndHead + 1; System.out.printf("On query %d found out exact number of missing elements: %d\n", totalQueries, numberOfElementsExcludingRootAndHead); } - de.entries = between; datas.add(de); // queue up to record result and construct further requests } + betweenBatch.clear(); + rangeToEntry.clear(); } toQuery.clear(); } @@ -189,6 +208,7 @@ } fromRootToHead.addFirst(n); // reverse order } + System.out.println("Total queries:" + totalQueries); if (!resultOk) { throw new HgBadStateException("See console for details"); // FIXME } diff -r a8df7162ec75 -r e10225daface cmdline/org/tmatesoft/hg/console/Remote.java --- a/cmdline/org/tmatesoft/hg/console/Remote.java Sat Apr 02 03:01:14 2011 +0200 +++ b/cmdline/org/tmatesoft/hg/console/Remote.java Sat Apr 02 23:05:28 2011 +0200 @@ -20,6 +20,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -93,9 +95,9 @@ cfg.addLocation(new File(System.getProperty("user.home"), ".hgrc")); String svnkitServer = cfg.getSection("paths").get("svnkit"); // URL url = new URL(svnkitServer + "?cmd=branches&nodes=30bd389788464287cee22ccff54c330a4b715de5"); - URL url = new URL(svnkitServer + "?cmd=between&pairs=71ddbf8603e8e09d54ac9c5fe4bb5ae824589f1d-8c8e3f372fa1fbfcf92b004b6f2ada2dbaf60028"); + URL url = new URL(svnkitServer + "?cmd=between"); // URL url = new URL(svnkitServer + "?cmd=changegroup&roots=" + Nodeid.NULL.toString()); -// URL url = new URL("http://localhost:8000/" + "?cmd=stream_out"); +// URL url = new URL("http://localhost:8000/" + "?cmd=between"); // URL url = new URL(svnkitServer + "?cmd=stream_out"); SSLContext sslContext = SSLContext.getInstance("SSL"); @@ -118,11 +120,22 @@ // sslContext.init(null, new TrustManager[] { new TrustEveryone() }, null); HttpsURLConnection urlConnection = (HttpsURLConnection) url.openConnection(); - urlConnection.addRequestProperty("User-Agent", "jhg/0.1.0"); - urlConnection.addRequestProperty("Accept", "application/mercurial-0.1"); - urlConnection.addRequestProperty("Authorization", "Basic " + authInfo); +// HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestProperty("User-Agent", "jhg/0.1.0"); + urlConnection.setRequestProperty("Accept", "application/mercurial-0.1"); + urlConnection.setRequestProperty("Authorization", "Basic " + authInfo); urlConnection.setSSLSocketFactory(sslContext.getSocketFactory()); - urlConnection.connect(); + byte[] body = "pairs=30bd389788464287cee22ccff54c330a4b715de5-dbd663faec1f0175619cf7668bddc6350548b8d6".getBytes(); + urlConnection.setRequestMethod("POST"); + urlConnection.setRequestProperty("Content-Length", String.valueOf(body.length)); + urlConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); +// urlConnection.connect(); + OutputStream os = urlConnection.getOutputStream(); + os.write(body); + os.flush(); + os.close(); System.out.println("Query:" + url.getQuery()); System.out.println("Response headers:"); final Map> headerFields = urlConnection.getHeaderFields(); diff -r a8df7162ec75 -r e10225daface src/org/tmatesoft/hg/repo/HgRemoteRepository.java --- a/src/org/tmatesoft/hg/repo/HgRemoteRepository.java Sat Apr 02 03:01:14 2011 +0200 +++ b/src/org/tmatesoft/hg/repo/HgRemoteRepository.java Sat Apr 02 23:05:28 2011 +0200 @@ -18,8 +18,10 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; import java.io.StreamTokenizer; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; @@ -27,6 +29,7 @@ import java.security.cert.X509Certificate; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -39,6 +42,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import org.tmatesoft.hg.core.HgBadStateException; import org.tmatesoft.hg.core.HgException; import org.tmatesoft.hg.core.Nodeid; @@ -106,25 +110,99 @@ } public List between(Nodeid tip, Nodeid base) throws HgException { + Range r = new Range(base, tip); + // XXX shall handle errors like no range key in the returned map, not sure how. + return between(Collections.singletonList(r)).get(r); + } + + /** + * @param ranges + * @return map, where keys are input instances, values are corresponding server reply + * @throws HgException + */ + public Map> between(Collection ranges) throws HgException { + if (ranges.isEmpty()) { + return Collections.emptyMap(); + } + // if fact, shall do other way round, this method shall send + LinkedHashMap> rv = new LinkedHashMap>(ranges.size() * 4 / 3); + StringBuilder sb = new StringBuilder(20 + ranges.size() * 82); + sb.append("pairs="); + for (Range r : ranges) { + sb.append(r.end.toString()); + sb.append('-'); + sb.append(r.start.toString()); + sb.append('+'); + } + if (sb.charAt(sb.length() - 1) == '+') { + // strip last space + sb.setLength(sb.length() - 1); + } try { - LinkedList rv = new LinkedList(); - URL u = new URL(url, url.getPath() + "?cmd=between&pairs=" + tip.toString() + '-' + base.toString()); - URLConnection c = setupConnection(u.openConnection()); - c.connect(); - System.out.println("Query:" + u.getQuery()); + boolean usePOST = ranges.size() > 3; + URL u = new URL(url, url.getPath() + "?cmd=between" + (usePOST ? "" : '&' + sb.toString())); + HttpURLConnection c = setupConnection(u.openConnection()); + if (usePOST) { + c.setRequestMethod("POST"); + c.setRequestProperty("Content-Length", String.valueOf(sb.length()/*nodeids are ASCII, bytes == characters */)); + c.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + c.setDoOutput(true); + c.connect(); + OutputStream os = c.getOutputStream(); + os.write(sb.toString().getBytes()); + os.flush(); + os.close(); + } else { + c.connect(); + } + System.out.printf("%d ranges, method:%s \n", ranges.size(), c.getRequestMethod()); + System.out.printf("Query (%d bytes):%s\n", u.getQuery().length(), u.getQuery()); System.out.println("Response headers:"); final Map> headerFields = c.getHeaderFields(); for (String s : headerFields.keySet()) { System.out.printf("%s: %s\n", s, c.getHeaderField(s)); } - InputStream is = c.getInputStream(); + InputStreamReader is = new InputStreamReader(c.getInputStream(), "US-ASCII"); StreamTokenizer st = new StreamTokenizer(is); st.ordinaryChars('0', '9'); st.wordChars('0', '9'); + st.eolIsSignificant(true); + Iterator rangeItr = ranges.iterator(); + LinkedList currRangeList = null; + Range currRange = null; + boolean possiblyEmptyNextLine = true; while (st.nextToken() != StreamTokenizer.TT_EOF) { - System.out.println(st.sval); - Nodeid nid = Nodeid.fromAscii(st.sval); - rv.addLast(nid); + if (st.ttype == StreamTokenizer.TT_EOL) { + if (possiblyEmptyNextLine) { + // newline follows newline; + assert currRange == null; + assert currRangeList == null; + if (!rangeItr.hasNext()) { + throw new HgBadStateException(); + } + rv.put(rangeItr.next(), Collections.emptyList()); + } else { + if (currRange == null || currRangeList == null) { + throw new HgBadStateException(); + } + // indicate next range value is needed + currRange = null; + currRangeList = null; + possiblyEmptyNextLine = true; + } + } else { + possiblyEmptyNextLine = false; + if (currRange == null) { + if (!rangeItr.hasNext()) { + throw new HgBadStateException(); + } + currRange = rangeItr.next(); + currRangeList = new LinkedList(); + rv.put(currRange, currRangeList); + } + Nodeid nid = Nodeid.fromAscii(st.sval); + currRangeList.addLast(nid); + } } is.close(); return rv; @@ -135,21 +213,6 @@ } } - /** - * @param ranges - * @return map, where keys are input instances, values are corresponding server reply - * @throws HgException - */ - public Map> between(Collection ranges) throws HgException { - // if fact, shall do other way round, this method shall send - LinkedHashMap> rv = new LinkedHashMap>(ranges.size() * 4 / 3); - for (Range r : ranges) { - List between = between(r.end, r.start); - rv.put(r, between); - } - return rv; - } - public List branches(List nodes) { return Collections.emptyList(); } @@ -159,8 +222,8 @@ return new HgLookup().loadBundle(new File("/temp/hg/hg-bundle-000000000000-gz.tmp")); } - private URLConnection setupConnection(URLConnection urlConnection) { - urlConnection.addRequestProperty("User-Agent", "hg4j/0.5.0"); + private HttpURLConnection setupConnection(URLConnection urlConnection) { + urlConnection.setRequestProperty("User-Agent", "hg4j/0.5.0"); urlConnection.addRequestProperty("Accept", "application/mercurial-0.1"); if (authInfo != null) { urlConnection.addRequestProperty("Authorization", "Basic " + authInfo); @@ -168,7 +231,7 @@ if (sslContext != null) { ((HttpsURLConnection) urlConnection).setSSLSocketFactory(sslContext.getSocketFactory()); } - return urlConnection; + return (HttpURLConnection) urlConnection; } public static final class Range {