ReceiveCommits: Split the history negotation hook out
Advertising history to support finding a common merge base isn't
entirely part of ReceiveCommits. Its just a feature hacked into the
code to support Gerrit Code Review's unique workflow where clients
are frequently behind the server.
Pull this code into its own hook, fired after the ReceiveCommits
specific hook that hides refs/changes and shows open changes as
additional objects.
This refactoring is mostly selfish; the server environment at Google
can benefit from these two unrelated concepts being separate hooks.
Change-Id: I91536f79d7605daf6136eaf8c5417729bfe56c15
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/HackPushNegotiateHook.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/HackPushNegotiateHook.java
new file mode 100644
index 0000000..5cd7dc3
--- /dev/null
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/HackPushNegotiateHook.java
@@ -0,0 +1,155 @@
+// Copyright (C) 2010 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static org.eclipse.jgit.lib.RefDatabase.ALL;
+
+import com.google.common.collect.Sets;
+
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.AdvertiseRefsHook;
+import org.eclipse.jgit.transport.BaseReceivePack;
+import org.eclipse.jgit.transport.ServiceMayNotContinueException;
+import org.eclipse.jgit.transport.UploadPack;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Advertises part of history to git push clients.
+ * <p>
+ * This is a hack to work around the lack of negotiation in the
+ * send-pack/receive-pack wire protocol.
+ * <p>
+ * When the server is frequently advancing master by creating merge commits, the
+ * client may not be able to discover a common ancestor during push. Attempting
+ * to push will re-upload a very large amount of history. This hook hacks in a
+ * fake negotiation replacement by walking history and sending recent commits as
+ * {@code ".have"} lines in the wire protocol, allowing the client to find a
+ * common ancestor.
+ */
+public class HackPushNegotiateHook implements AdvertiseRefsHook {
+ private static final Logger log = LoggerFactory
+ .getLogger(HackPushNegotiateHook.class);
+
+ /** Size of an additional ".have" line. */
+ private static final int HAVE_LINE_LEN = 4
+ + Constants.OBJECT_ID_STRING_LENGTH
+ + 1 + 5 + 1;
+
+ /**
+ * Maximum number of bytes to "waste" in the advertisement with a peek at this
+ * repository's current reachable history.
+ */
+ private static final int MAX_EXTRA_BYTES = 8192;
+
+ /**
+ * Number of recent commits to advertise immediately, hoping to show a client
+ * a nearby merge base.
+ */
+ private static final int BASE_COMMITS = 64;
+
+ /** Number of commits to skip once base has already been shown. */
+ private static final int STEP_COMMITS = 16;
+
+ /** Total number of commits to extract from the history. */
+ private static final int MAX_HISTORY = MAX_EXTRA_BYTES / HAVE_LINE_LEN;
+
+ @Override
+ public void advertiseRefs(UploadPack us) {
+ throw new UnsupportedOperationException(
+ "HackPushNegotiateHook cannot be used for UploadPack");
+ }
+
+ @Override
+ public void advertiseRefs(BaseReceivePack rp)
+ throws ServiceMayNotContinueException {
+ Map<String, Ref> r = rp.getAdvertisedRefs();
+ if (r == null) {
+ try {
+ r = rp.getRepository().getRefDatabase().getRefs(ALL);
+ } catch (ServiceMayNotContinueException e) {
+ throw e;
+ } catch (IOException e) {
+ ServiceMayNotContinueException ex = new ServiceMayNotContinueException();
+ ex.initCause(e);
+ throw ex;
+ }
+ }
+ rp.setAdvertisedRefs(r, history(r.values(), rp));
+ }
+
+ private Set<ObjectId> history(Collection<Ref> refs, BaseReceivePack rp) {
+ Set<ObjectId> alreadySending = rp.getAdvertisedObjects();
+ if (alreadySending.isEmpty()) {
+ alreadySending = idsOf(refs);
+ }
+
+ int max = MAX_HISTORY - Math.max(0, alreadySending.size() - refs.size());
+ if (max <= 0) {
+ return Collections.emptySet();
+ }
+
+ // Scan history until the advertisement is full.
+ RevWalk rw = rp.getRevWalk();
+ try {
+ for (Ref ref : refs) {
+ try {
+ if (ref.getObjectId() != null) {
+ rw.markStart(rw.parseCommit(ref.getObjectId()));
+ }
+ } catch (IOException badCommit) {
+ continue;
+ }
+ }
+
+ Set<ObjectId> history = Sets.newHashSetWithExpectedSize(max);
+ try {
+ int stepCnt = 0;
+ for (RevCommit c; history.size() < max && (c = rw.next()) != null;) {
+ if (c.getParentCount() <= 1
+ && !alreadySending.contains(c)
+ && (history.size() < BASE_COMMITS || (++stepCnt % STEP_COMMITS) == 0)) {
+ history.add(c);
+ }
+ }
+ } catch (IOException err) {
+ log.error("error trying to advertise history", err);
+ }
+ return history;
+ } finally {
+ rw.reset();
+ }
+ }
+
+ private static Set<ObjectId> idsOf(Collection<Ref> refs) {
+ Set<ObjectId> r = Sets.newHashSetWithExpectedSize(refs.size());
+ for (Ref ref : refs) {
+ if (ref.getObjectId() != null) {
+ r.add(ref.getObjectId());
+ }
+ }
+ return r;
+ }
+}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java
index 9e5353a..02c6d13 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java
@@ -485,6 +485,7 @@
advHooks.add(rp.getAdvertiseRefsHook());
advHooks.add(new ReceiveCommitsAdvertiseRefsHook(
db, queryProvider, projectControl.getProject().getNameKey()));
+ advHooks.add(new HackPushNegotiateHook());
rp.setAdvertiseRefsHook(AdvertiseRefsHookChain.newChain(advHooks));
}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommitsAdvertiseRefsHook.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommitsAdvertiseRefsHook.java
index b2d3632..43a4d4b 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommitsAdvertiseRefsHook.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommitsAdvertiseRefsHook.java
@@ -28,11 +28,8 @@
import com.google.gwtorm.server.OrmException;
import com.google.inject.Provider;
-import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.AdvertiseRefsHook;
import org.eclipse.jgit.transport.BaseReceivePack;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
@@ -41,6 +38,7 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collections;
import java.util.Map;
import java.util.Set;
@@ -89,15 +87,11 @@
r.put(name, e.getValue());
}
}
- rp.setAdvertisedRefs(r, advertiseHistory(r.values(), rp));
+ rp.setAdvertisedRefs(r, advertiseOpenChanges());
}
- private Set<ObjectId> advertiseHistory(
- Iterable<Ref> sending,
- BaseReceivePack rp) {
- Set<ObjectId> toInclude = Sets.newHashSet();
-
- // Advertise some recent open changes, in case a commit is based one.
+ private Set<ObjectId> advertiseOpenChanges() {
+ // Advertise some recent open changes, in case a commit is based on one.
final int limit = 32;
try {
Set<PatchSet.Id> toGet = Sets.newHashSetWithExpectedSize(limit);
@@ -110,68 +104,18 @@
toGet.add(id);
}
}
+
+ Set<ObjectId> r = Sets.newHashSetWithExpectedSize(toGet.size());
for (PatchSet ps : db.patchSets().get(toGet)) {
if (ps.getRevision() != null && ps.getRevision().get() != null) {
- toInclude.add(ObjectId.fromString(ps.getRevision().get()));
+ r.add(ObjectId.fromString(ps.getRevision().get()));
}
}
+ return r;
} catch (OrmException err) {
log.error("Cannot list open changes of " + projectName, err);
+ return Collections.emptySet();
}
-
- // Size of an additional ".have" line.
- final int haveLineLen = 4 + Constants.OBJECT_ID_STRING_LENGTH + 1 + 5 + 1;
-
- // Maximum number of bytes to "waste" in the advertisement with
- // a peek at this repository's current reachable history.
- final int maxExtraSize = 8192;
-
- // Number of recent commits to advertise immediately, hoping to
- // show a client a nearby merge base.
- final int base = 64;
-
- // Number of commits to skip once base has already been shown.
- final int step = 16;
-
- // Total number of commits to extract from the history.
- final int max = maxExtraSize / haveLineLen;
-
- // Scan history until the advertisement is full.
- Set<ObjectId> alreadySending = Sets.newHashSet();
- RevWalk rw = rp.getRevWalk();
- for (Ref ref : sending) {
- try {
- if (ref.getObjectId() != null) {
- alreadySending.add(ref.getObjectId());
- rw.markStart(rw.parseCommit(ref.getObjectId()));
- }
- } catch (IOException badCommit) {
- continue;
- }
- }
-
- int stepCnt = 0;
- RevCommit c;
- try {
- while ((c = rw.next()) != null && toInclude.size() < max) {
- if (alreadySending.contains(c)
- || toInclude.contains(c)
- || c.getParentCount() > 1) {
- // Do nothing
- } else if (toInclude.size() < base) {
- toInclude.add(c);
- } else {
- stepCnt = ++stepCnt % step;
- if (stepCnt == 0) {
- toInclude.add(c);
- }
- }
- }
- } catch (IOException err) {
- log.error("Error trying to advertise history on " + projectName, err);
- }
- rw.reset();
- return toInclude;
}
private static boolean skip(String name) {
