UploadPack: support custom packfile-to-URI mapping
Teach UploadPack to take a provider of URIs corresponding to cached
packs. When fetching, if the client supports the packfile-uri feature,
and if such a cached pack were to be streamed, instead send the
corresponding URI.
This packfile-uri feature is implemented in the jt/fetch-cdn-offload
branch of Git. There is interest in this feature [1], but it is not yet
merged.
[1] https://public-inbox.org/git/cover.1552073690.git.jonathantanmy@google.com/
Change-Id: I9a32dae131c9c56ad2ff4a8a9638ae3b5e44dc15
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
index 22c67c1..528a63f 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
@@ -35,6 +35,8 @@
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.internal.storage.file.PackLock;
+import org.eclipse.jgit.internal.storage.pack.CachedPack;
+import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
@@ -2150,6 +2152,58 @@
}
@Test
+ public void testV2FetchPackfileUris() throws Exception {
+ // Inside the pack
+ RevCommit commit = remote.commit().message("x").create();
+ remote.update("master", commit);
+ generateBitmaps(server);
+
+ // Outside the pack
+ RevCommit commit2 = remote.commit().message("x").parent(commit).create();
+ remote.update("master", commit2);
+
+ server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true);
+
+ ByteArrayInputStream recvStream = uploadPackV2(
+ (UploadPack up) -> {
+ up.setCachedPackUriProvider(new CachedPackUriProvider() {
+ @Override
+ public PackInfo getInfo(CachedPack pack,
+ Collection<String> protocolsSupported)
+ throws IOException {
+ assertThat(protocolsSupported, hasItems("https"));
+ if (!protocolsSupported.contains("https"))
+ return null;
+ return new PackInfo("myhash", "myuri");
+ }
+
+ });
+ },
+ "command=fetch\n",
+ PacketLineIn.DELIM,
+ "want " + commit2.getName() + "\n",
+ "sideband-all\n",
+ "packfile-uris https\n",
+ "done\n",
+ PacketLineIn.END);
+ PacketLineIn pckIn = new PacketLineIn(recvStream);
+
+ String s;
+ // skip all \002 strings
+ for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) {
+ // do nothing
+ }
+ assertThat(s, is("\001packfile-uris"));
+ assertThat(pckIn.readString(), is("\001myhash myuri"));
+ assertTrue(PacketLineIn.isDelimiter(pckIn.readString()));
+ assertThat(pckIn.readString(), is("\001packfile"));
+ parsePack(recvStream);
+
+ assertFalse(client.getObjectDatabase().has(commit.toObjectId()));
+ assertTrue(client.getObjectDatabase().has(commit2.toObjectId()));
+ }
+
+ @Test
public void testGetPeerAgentProtocolV0() throws Exception {
RevCommit one = remote.commit().message("1").create();
remote.update("one", one);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java
index 72699b0..43bd9d4 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java
@@ -313,6 +313,14 @@
int matchLimit) throws IOException;
/**
+ * @return the checksum of the pack; caller must not modify it
+ * @since 5.5
+ */
+ public byte[] getChecksum() {
+ return packChecksum;
+ }
+
+ /**
* Represent mutable entry of pack index consisting of object id and offset
* in pack (both mutable).
*
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java
new file mode 100644
index 0000000..5cbc2ba
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2019, Google LLC.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.internal.storage.pack;
+
+import java.io.IOException;
+import java.util.Collection;
+import org.eclipse.jgit.annotations.Nullable;
+
+/**
+ * Provider of URIs corresponding to cached packs. For use with the
+ * "packfile-uris" feature.
+ * @since 5.5
+ */
+public interface CachedPackUriProvider {
+
+ /**
+ * @param pack the cached pack for which to check if a corresponding URI
+ * exists
+ * @param protocolsSupported the protocols that the client has declared
+ * support for; if a URI is returned, it must be of one of these
+ * protocols
+ * @throws IOException implementations may throw this
+ * @return if a URI corresponds to the cached pack, an object
+ * containing the URI and some other information; null otherwise
+ * @since 5.5
+ */
+ @Nullable
+ PackInfo getInfo(CachedPack pack, Collection<String> protocolsSupported)
+ throws IOException;
+
+ /**
+ * Information about a packfile.
+ * @since 5.5
+ */
+ public static class PackInfo {
+ private final String hash;
+ private final String uri;
+
+ /**
+ * Constructs an object containing information about a packfile.
+ * @param hash the hash of the packfile as a hexadecimal string
+ * @param uri the URI corresponding to the packfile
+ */
+ public PackInfo(String hash, String uri) {
+ this.hash = hash;
+ this.uri = uri;
+ }
+
+ /**
+ * @return the hash of the packfile as a hexadecimal string
+ */
+ public String getHash() {
+ return hash;
+ }
+
+ /**
+ * @return the URI corresponding to the packfile
+ */
+ public String getUri() {
+ return uri;
+ }
+ }
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 6506789..02cfe90 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -122,6 +122,7 @@
import org.eclipse.jgit.storage.pack.PackStatistics;
import org.eclipse.jgit.transport.FilterSpec;
import org.eclipse.jgit.transport.ObjectCountCallback;
+import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.transport.WriteAbortedException;
import org.eclipse.jgit.util.BlockList;
import org.eclipse.jgit.util.TemporaryBuffer;
@@ -307,6 +308,8 @@
private FilterSpec filterSpec = FilterSpec.NO_FILTER;
+ private PackfileUriConfig packfileUriConfig;
+
/**
* Create writer for specified repository.
* <p>
@@ -651,6 +654,14 @@
}
/**
+ * @param config configuration related to packfile URIs
+ * @since 5.5
+ */
+ public void setPackfileUriConfig(PackfileUriConfig config) {
+ packfileUriConfig = config;
+ }
+
+ /**
* Returns objects number in a pack file that was created by this writer.
*
* @return number of objects in pack.
@@ -673,6 +684,26 @@
return stats.totalObjects;
}
+ private long getUnoffloadedObjectCount() throws IOException {
+ long objCnt = 0;
+
+ objCnt += objectsLists[OBJ_COMMIT].size();
+ objCnt += objectsLists[OBJ_TREE].size();
+ objCnt += objectsLists[OBJ_BLOB].size();
+ objCnt += objectsLists[OBJ_TAG].size();
+
+ for (CachedPack pack : cachedPacks) {
+ CachedPackUriProvider.PackInfo packInfo =
+ packfileUriConfig.cachedPackUriProvider.getInfo(
+ pack, packfileUriConfig.protocolsSupported);
+ if (packInfo == null) {
+ objCnt += pack.getObjectCount();
+ }
+ }
+
+ return objCnt;
+ }
+
/**
* Returns the object ids in the pack file that was created by this writer.
* <p>
@@ -1177,13 +1208,38 @@
: new CheckedOutputStream(packStream, crc32),
this);
- long objCnt = getObjectCount();
+ long objCnt = packfileUriConfig == null ? getObjectCount() :
+ getUnoffloadedObjectCount();
stats.totalObjects = objCnt;
if (callback != null)
callback.setObjectCount(objCnt);
beginPhase(PackingPhase.WRITING, writeMonitor, objCnt);
long writeStart = System.currentTimeMillis();
try {
+ List<CachedPack> unwrittenCachedPacks;
+
+ if (packfileUriConfig != null) {
+ unwrittenCachedPacks = new ArrayList<>();
+ CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider;
+ PacketLineOut o = packfileUriConfig.pckOut;
+
+ o.writeString("packfile-uris\n");
+ for (CachedPack pack : cachedPacks) {
+ CachedPackUriProvider.PackInfo packInfo = p.getInfo(
+ pack, packfileUriConfig.protocolsSupported);
+ if (packInfo != null) {
+ o.writeString(packInfo.getHash() + ' ' +
+ packInfo.getUri() + '\n');
+ } else {
+ unwrittenCachedPacks.add(pack);
+ }
+ }
+ packfileUriConfig.pckOut.writeDelim();
+ packfileUriConfig.pckOut.writeString("packfile\n");
+ } else {
+ unwrittenCachedPacks = cachedPacks;
+ }
+
out.writeFileHeader(PACK_VERSION_GENERATED, objCnt);
out.flush();
@@ -1197,7 +1253,7 @@
}
stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
- for (CachedPack pack : cachedPacks) {
+ for (CachedPack pack : unwrittenCachedPacks) {
long deltaCnt = pack.getDeltaCount();
stats.reusedObjects += pack.getObjectCount();
stats.reusedDeltas += deltaCnt;
@@ -2426,4 +2482,37 @@
return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
}
}
+
+ /**
+ * Configuration related to the packfile URI feature.
+ *
+ * @since 5.5
+ */
+ public static class PackfileUriConfig {
+ @NonNull
+ private final PacketLineOut pckOut;
+
+ @NonNull
+ private final Collection<String> protocolsSupported;
+
+ @NonNull
+ private final CachedPackUriProvider cachedPackUriProvider;
+
+ /**
+ * @param pckOut where to write "packfile-uri" lines to (should
+ * output to the same stream as the one passed to
+ * PackWriter#writePack)
+ * @param protocolsSupported list of protocols supported (e.g. "https")
+ * @param cachedPackUriProvider provider of URIs corresponding
+ * to cached packs
+ * @since 5.5
+ */
+ public PackfileUriConfig(@NonNull PacketLineOut pckOut,
+ @NonNull Collection<String> protocolsSupported,
+ @NonNull CachedPackUriProvider cachedPackUriProvider) {
+ this.pckOut = pckOut;
+ this.protocolsSupported = protocolsSupported;
+ this.cachedPackUriProvider = cachedPackUriProvider;
+ }
+ }
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java
index 86574c1..fe1b697 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java
@@ -74,6 +74,9 @@
private final boolean sidebandAll;
+ @NonNull
+ private final List<String> packfileUriProtocols;
+
FetchV2Request(@NonNull List<ObjectId> peerHas,
@NonNull List<String> wantedRefs,
@NonNull Set<ObjectId> wantIds,
@@ -82,7 +85,7 @@
@NonNull FilterSpec filterSpec,
boolean doneReceived, @NonNull Set<String> clientCapabilities,
@Nullable String agent, @NonNull List<String> serverOptions,
- boolean sidebandAll) {
+ boolean sidebandAll, @NonNull List<String> packfileUriProtocols) {
super(wantIds, depth, clientShallowCommits, filterSpec,
clientCapabilities, deepenSince,
deepenNotRefs, agent);
@@ -91,6 +94,7 @@
this.doneReceived = doneReceived;
this.serverOptions = requireNonNull(serverOptions);
this.sidebandAll = sidebandAll;
+ this.packfileUriProtocols = packfileUriProtocols;
}
/**
@@ -138,6 +142,11 @@
return sidebandAll;
}
+ @NonNull
+ List<String> getPackfileUriProtocols() {
+ return packfileUriProtocols;
+ }
+
/** @return A builder of {@link FetchV2Request}. */
static Builder builder() {
return new Builder();
@@ -172,6 +181,8 @@
boolean sidebandAll;
+ final List<String> packfileUriProtocols = new ArrayList<>();
+
private Builder() {
}
@@ -339,6 +350,11 @@
return this;
}
+ Builder addPackfileUriProtocol(@NonNull String value) {
+ packfileUriProtocols.add(value);
+ return this;
+ }
+
/**
* @return Initialized fetch request
*/
@@ -347,7 +363,8 @@
clientShallowCommits, deepenSince, deepenNotRefs,
depth, filterSpec, doneReceived, clientCapabilities,
agent, Collections.unmodifiableList(serverOptions),
- sidebandAll);
+ sidebandAll,
+ Collections.unmodifiableList(packfileUriProtocols));
}
}
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java
index 453be7f..14ccddf 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java
@@ -214,6 +214,10 @@
} else if (transferConfig.isAllowSidebandAll()
&& line2.equals(OPTION_SIDEBAND_ALL)) {
reqBuilder.setSidebandAll(true);
+ } else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$
+ for (String s : line2.substring(14).split(",")) {
+ reqBuilder.addPackfileUriProtocol(s);
+ }
} else {
throw new PackProtocolException(MessageFormat
.format(JGitText.get().unexpectedPacketLine, line2));
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
index 2194f2f..1e49c7b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
@@ -93,6 +93,7 @@
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
import org.eclipse.jgit.internal.storage.pack.PackWriter;
import org.eclipse.jgit.internal.transport.parser.FirstWant;
import org.eclipse.jgit.lib.BitmapIndex;
@@ -359,6 +360,8 @@
*/
private FetchRequest currentRequest;
+ private CachedPackUriProvider cachedPackUriProvider;
+
/**
* Create a new pack upload for an open repository.
*
@@ -736,6 +739,15 @@
this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$
}
+ /**
+ * @param p provider of URIs corresponding to cached packs (to support
+ * the packfile URIs feature)
+ * @since 5.5
+ */
+ public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) {
+ cachedPackUriProvider = p;
+ }
+
private boolean useProtocolV2() {
return ProtocolVersion.V2.equals(transferConfig.protocolVersion)
&& clientRequestedV2;
@@ -1285,6 +1297,7 @@
(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$
(advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$
(transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$
+ (cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$
OPTION_SHALLOW);
caps.add(CAPABILITY_SERVER_OPTION);
return caps;
@@ -2298,7 +2311,21 @@
}
if (pckOut.isUsingSideband()) {
- pckOut.writeString("packfile\n"); //$NON-NLS-1$
+ if (req instanceof FetchV2Request &&
+ cachedPackUriProvider != null &&
+ !((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) {
+ FetchV2Request reqV2 = (FetchV2Request) req;
+ pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig(
+ pckOut,
+ reqV2.getPackfileUriProtocols(),
+ cachedPackUriProvider));
+ } else {
+ // PackWriter will write "packfile-uris\n" and "packfile\n"
+ // for us if provided a PackfileUriConfig. In this case, we
+ // are not providing a PackfileUriConfig, so we have to
+ // write this line ourselves.
+ pckOut.writeString("packfile\n"); //$NON-NLS-1$
+ }
}
pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);
