[pgm] Fetch-CLI: add support for shallow
This adds support for shallow cloning. The CloneCommand and the
FetchCommand now have the new options --depth, --shallow-since and
--shallow-exclude to tell the server that the client doesn't want to
download the complete history.
Bug: https://bugs.eclipse.org/bugs/show_bug.cgi?id=475615
Change-Id: I8f113bed25dd6df64f2f95de6a59d4675ab8a903
diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java
index 4cbd61c..cbb5bbb 100644
--- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java
+++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java
@@ -17,14 +17,20 @@
import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.time.Instant;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.junit.JGitTestUtil;
import org.eclipse.jgit.junit.MockSystemReader;
+import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.CLIRepositoryTestCase;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
@@ -41,10 +47,14 @@ public class CloneTest extends CLIRepositoryTestCase {
private Git git;
+ private TestRepository<Repository> tr;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
+
+ tr = new TestRepository<>(db);
git = new Git(db);
}
@@ -112,6 +122,22 @@ private RevCommit createInitialCommit() throws Exception {
return git.commit().setMessage("Initial commit").call();
}
+ private RevCommit createSecondCommit() throws Exception {
+ JGitTestUtil.writeTrashFile(db, "Test.txt", "Some change");
+ git.add().addFilepattern("Test.txt").call();
+ return git.commit()
+ .setCommitter(new PersonIdent(this.committer, tr.getDate()))
+ .setMessage("Second commit").call();
+ }
+
+ private RevCommit createThirdCommit() throws Exception {
+ JGitTestUtil.writeTrashFile(db, "change.txt", "another change");
+ git.add().addFilepattern("change.txt").call();
+ return git.commit()
+ .setCommitter(new PersonIdent(this.committer, tr.getDate()))
+ .setMessage("Third commit").call();
+ }
+
@Test
public void testCloneEmpty() throws Exception {
File gitDir = db.getDirectory();
@@ -203,4 +229,117 @@ public void testCloneMirror() throws Exception {
assertEquals("refs/*", fetchRefSpec.getDestination());
assertNotNull(git2.getRepository().exactRef("refs/meta/foo/bar"));
}
+
+ @Test
+ public void testDepth() throws Exception {
+ createInitialCommit();
+ createSecondCommit();
+ createThirdCommit();
+
+ File gitDir = db.getDirectory();
+ String sourceURI = gitDir.toURI().toString();
+ File target = createTempDirectory("target");
+ String cmd = "git clone --depth 1 " + sourceURI + " "
+ + shellQuote(target.getPath());
+ String[] result = execute(cmd);
+ assertArrayEquals(new String[] {
+ "Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+ Git git2 = Git.open(target);
+ addRepoToClose(git2.getRepository());
+
+ List<RevCommit> log = StreamSupport
+ .stream(git2.log().all().call().spliterator(), false)
+ .collect(Collectors.toList());
+ assertEquals(1, log.size());
+ RevCommit commit = log.get(0);
+ assertEquals(Set.of(commit.getId()),
+ git2.getRepository().getObjectDatabase().getShallowCommits());
+ assertEquals("Third commit", commit.getFullMessage());
+ assertEquals(0, commit.getParentCount());
+ }
+
+ @Test
+ public void testDepth2() throws Exception {
+ createInitialCommit();
+ createSecondCommit();
+ createThirdCommit();
+
+ File gitDir = db.getDirectory();
+ String sourceURI = gitDir.toURI().toString();
+ File target = createTempDirectory("target");
+ String cmd = "git clone --depth 2 " + sourceURI + " "
+ + shellQuote(target.getPath());
+ String[] result = execute(cmd);
+ assertArrayEquals(new String[] {
+ "Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+ Git git2 = Git.open(target);
+ addRepoToClose(git2.getRepository());
+
+ List<RevCommit> log = StreamSupport
+ .stream(git2.log().all().call().spliterator(), false)
+ .collect(Collectors.toList());
+ assertEquals(2, log.size());
+ assertEquals(List.of("Third commit", "Second commit"), log.stream()
+ .map(RevCommit::getFullMessage).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testCloneRepositoryWithShallowSince() throws Exception {
+ createInitialCommit();
+ tr.tick(30);
+ RevCommit secondCommit = createSecondCommit();
+ tr.tick(45);
+ createThirdCommit();
+
+ File gitDir = db.getDirectory();
+ String sourceURI = gitDir.toURI().toString();
+ File target = createTempDirectory("target");
+ String cmd = "git clone --shallow-since="
+ + Instant.ofEpochSecond(secondCommit.getCommitTime()).toString()
+ + " " + sourceURI + " " + shellQuote(target.getPath());
+ String[] result = execute(cmd);
+ assertArrayEquals(new String[] {
+ "Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+ Git git2 = Git.open(target);
+ addRepoToClose(git2.getRepository());
+
+ List<RevCommit> log = StreamSupport
+ .stream(git2.log().all().call().spliterator(), false)
+ .collect(Collectors.toList());
+ assertEquals(2, log.size());
+ assertEquals(List.of("Third commit", "Second commit"), log.stream()
+ .map(RevCommit::getFullMessage).collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testCloneRepositoryWithShallowExclude() throws Exception {
+ final RevCommit firstCommit = createInitialCommit();
+ final RevCommit secondCommit = createSecondCommit();
+ createThirdCommit();
+
+ File gitDir = db.getDirectory();
+ String sourceURI = gitDir.toURI().toString();
+ File target = createTempDirectory("target");
+ String cmd = "git clone --shallow-exclude="
+ + firstCommit.getId().getName() + " --shallow-exclude="
+ + secondCommit.getId().getName() + " " + sourceURI + " "
+ + shellQuote(target.getPath());
+ String[] result = execute(cmd);
+ assertArrayEquals(new String[] {
+ "Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+ Git git2 = Git.open(target);
+ addRepoToClose(git2.getRepository());
+
+ List<RevCommit> log = StreamSupport
+ .stream(git2.log().all().call().spliterator(), false)
+ .collect(Collectors.toList());
+ assertEquals(1, log.size());
+ assertEquals(List.of("Third commit"), log.stream()
+ .map(RevCommit::getFullMessage).collect(Collectors.toList()));
+ }
+
}
diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties
index 48f4e85..98d711d 100644
--- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties
+++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties
@@ -137,6 +137,7 @@
metaVar_commitPaths=paths
metaVar_configFile=FILE
metaVar_connProp=conn.prop
+metaVar_depth=<depth>
metaVar_diffAlg=ALGORITHM
metaVar_directory=DIRECTORY
metaVar_extraArgument=ours|theirs
@@ -144,6 +145,7 @@
metaVar_filepattern=filepattern
metaVar_gitDir=GIT_DIR
metaVar_hostName=HOSTNAME
+metaVar_instant=<instant>
metaVar_lfsStorage=STORAGE
metaVar_linesOfContext=lines
metaVar_message=message
@@ -168,6 +170,8 @@
metaVar_s3StorageClass=STORAGE-CLASS
metaVar_seconds=SECONDS
metaVar_service=SERVICE
+metaVar_shallowExclude=<revision>
+metaVar_shallowSince=<date>
metaVar_tagLocalUser=<GPG key ID>
metaVar_tool=TOOL
metaVar_treeish=tree-ish
@@ -374,6 +378,7 @@
usage_diffAlgorithm=the diff algorithm to use. Currently supported are: 'myers', 'histogram'
usage_DiffTool=git difftool is a Git command that allows you to compare and edit files between revisions using common diff tools.\ngit difftool is a frontend to git diff and accepts the same options and arguments.
usage_MergeTool=git-mergetool - Run merge conflict resolution tools to resolve merge conflicts.\nUse git mergetool to run one of several merge utilities to resolve merge conflicts. It is typically run after git merge.
+usage_depth=Limit fetching to the specified number of commits from the tip of each remote branch history.
usage_directoriesToExport=directories to export
usage_disableTheServiceInAllRepositories=disable the service in all repositories
usage_displayAListOfAllRegisteredJgitCommands=Display a list of all registered jgit commands
@@ -447,6 +452,8 @@
usage_runLfsStore=Run LFS Store in a given directory
usage_S3NoSslVerify=Skip verification of Amazon server certificate and hostname
usage_setTheGitRepositoryToOperateOn=set the git repository to operate on
+usage_shallowExclude=Deepen or shorten the history of a shallow repository to exclude commits reachable from a specified remote branch or tag.
+usage_shallowSince=Deepen or shorten the history of a shallow repository to include all reachable commits after <date>.
usage_show=Display one commit
usage_showRefNamesMatchingCommits=Show ref names matching commits
usage_showPatch=display patch
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java
index f28915d..9f9fa8f 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java
@@ -13,7 +13,10 @@
import java.io.File;
import java.io.IOException;
import java.text.MessageFormat;
+import java.time.Instant;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
@@ -48,6 +51,15 @@ class Clone extends AbstractFetchCommand implements CloneCommand.Callback {
@Option(name = "--quiet", usage = "usage_quiet")
private Boolean quiet;
+ @Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth")
+ private Integer depth = null;
+
+ @Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince")
+ private Instant shallowSince = null;
+
+ @Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude")
+ private List<String> shallowExcludes = new ArrayList<>();
+
@Option(name = "--recurse-submodules", usage = "usage_recurseSubmodules")
private boolean cloneSubmodules;
@@ -97,6 +109,16 @@ protected void run() throws Exception {
.setMirror(isMirror).setNoCheckout(noCheckout).setBranch(branch)
.setCloneSubmodules(cloneSubmodules).setTimeout(timeout);
+ if (depth != null) {
+ command.setDepth(depth.intValue());
+ }
+ if (shallowSince != null) {
+ command.setShallowSince(shallowSince);
+ }
+ for (String shallowExclude : shallowExcludes) {
+ command.addShallowExclude(shallowExclude);
+ }
+
command.setGitDir(gitdir == null ? null : new File(gitdir));
command.setDirectory(localNameF);
boolean msgs = quiet == null || !quiet.booleanValue();
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java
index fbce4a5..2e0c36b 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java
@@ -14,6 +14,8 @@
import java.io.IOException;
import java.text.MessageFormat;
+import java.time.Instant;
+import java.util.ArrayList;
import java.util.List;
import org.eclipse.jgit.api.FetchCommand;
@@ -62,6 +64,15 @@ void nothin(@SuppressWarnings("unused") final boolean ignored) {
@Option(name = "--tags", usage="usage_tags", aliases = { "-t" })
private Boolean tags;
+ @Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth")
+ private Integer depth = null;
+
+ @Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince")
+ private Instant shallowSince = null;
+
+ @Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude")
+ private List<String> shallowExcludes = new ArrayList<>();
+
@Option(name = "--no-tags", usage = "usage_notags", aliases = { "-n" })
void notags(@SuppressWarnings("unused")
final boolean ignored) {
@@ -120,6 +131,15 @@ protected void run() {
fetch.setTagOpt(tags.booleanValue() ? TagOpt.FETCH_TAGS
: TagOpt.NO_TAGS);
}
+ if (depth != null) {
+ fetch.setDepth(depth.intValue());
+ }
+ if (shallowSince != null) {
+ fetch.setShallowSince(shallowSince);
+ }
+ for (String shallowExclude : shallowExcludes) {
+ fetch.addShallowExclude(shallowExclude);
+ }
if (0 <= timeout) {
fetch.setTimeout(timeout);
}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
index 490f800..d07268b 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
@@ -214,6 +214,7 @@ public static String fatalError(String message) {
/***/ public String metaVar_filepattern;
/***/ public String metaVar_gitDir;
/***/ public String metaVar_hostName;
+ /***/ public String metaVar_instant;
/***/ public String metaVar_lfsStorage;
/***/ public String metaVar_linesOfContext;
/***/ public String metaVar_message;
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java
index 5d32e65..df0b39b 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java
@@ -13,6 +13,7 @@
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Field;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -55,6 +56,7 @@ public class CmdLineParser extends org.kohsuke.args4j.CmdLineParser {
registry.registerHandler(RevCommit.class, RevCommitHandler.class);
registry.registerHandler(RevTree.class, RevTreeHandler.class);
registry.registerHandler(List.class, OptionWithValuesListHandler.class);
+ registry.registerHandler(Instant.class, InstantHandler.class);
}
private final Repository db;
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java
new file mode 100644
index 0000000..feee78e
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2022, Harald Weiner and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.pgm.opt;
+
+import java.time.Instant;
+
+import org.eclipse.jgit.pgm.internal.CLIText;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.OptionDef;
+import org.kohsuke.args4j.spi.OptionHandler;
+import org.kohsuke.args4j.spi.Parameters;
+import org.kohsuke.args4j.spi.Setter;
+
+/**
+ * Custom argument handler {@link java.time.Instant} from string values.
+ * <p>
+ * Assumes the parser has been initialized with a Repository.
+ *
+ * @since 6.5
+ */
+public class InstantHandler extends OptionHandler<Instant> {
+ /**
+ * Create a new handler for the command name.
+ * <p>
+ * This constructor is used only by args4j.
+ *
+ * @param parser
+ * a {@link org.kohsuke.args4j.CmdLineParser} object.
+ * @param option
+ * a {@link org.kohsuke.args4j.OptionDef} object.
+ * @param setter
+ * a {@link org.kohsuke.args4j.spi.Setter} object.
+ */
+ public InstantHandler(CmdLineParser parser, OptionDef option,
+ Setter<? super Instant> setter) {
+ super(parser, option, setter);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int parseArguments(Parameters params) throws CmdLineException {
+ Instant instant = Instant.parse(params.getParameter(0));
+ setter.addValue(instant);
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getDefaultMetaVariable() {
+ return CLIText.get().metaVar_instant;
+ }
+}
