Merge "Update Orbit to S20210223232630"
diff --git a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF
index b4c892e..419c4ca 100644
--- a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF
@@ -7,7 +7,9 @@
Bundle-Vendor: %Bundle-Vendor
Bundle-Localization: plugin
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
-Import-Package: org.eclipse.jgit.internal.storage.dfs;version="[5.11.0,5.12.0)",
+Import-Package: org.eclipse.jgit.api;version="[5.11.0,5.12.0)",
+ org.eclipse.jgit.attributes;version="[5.11.0,5.12.0)",
+ org.eclipse.jgit.internal.storage.dfs;version="[5.11.0,5.12.0)",
org.eclipse.jgit.junit;version="[5.11.0,5.12.0)",
org.eclipse.jgit.lfs;version="[5.11.0,5.12.0)",
org.eclipse.jgit.lfs.errors;version="[5.11.0,5.12.0)",
diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java
new file mode 100644
index 0000000..8964310
--- /dev/null
+++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.lfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.ResetCommand.ResetType;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
+import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class LfsGitTest extends RepositoryTestCase {
+
+ private static final String SMUDGE_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+ + Constants.ATTR_FILTER_DRIVER_PREFIX
+ + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE;
+
+ private static final String CLEAN_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+ + Constants.ATTR_FILTER_DRIVER_PREFIX
+ + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN;
+
+ @BeforeClass
+ public static void installLfs() {
+ FilterCommandRegistry.register(SMUDGE_NAME, SmudgeFilter.FACTORY);
+ FilterCommandRegistry.register(CLEAN_NAME, CleanFilter.FACTORY);
+ }
+
+ @AfterClass
+ public static void removeLfs() {
+ FilterCommandRegistry.unregister(SMUDGE_NAME);
+ FilterCommandRegistry.unregister(CLEAN_NAME);
+ }
+
+ private Git git;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ git = new Git(db);
+ // commit something
+ writeTrashFile("Test.txt", "Hello world");
+ git.add().addFilepattern("Test.txt").call();
+ git.commit().setMessage("Initial commit").call();
+ // prepare the config for LFS
+ StoredConfig config = git.getRepository().getConfig();
+ config.setString("filter", "lfs", "clean", CLEAN_NAME);
+ config.setString("filter", "lfs", "smudge", SMUDGE_NAME);
+ config.save();
+ }
+
+ @Test
+ public void checkoutNonLfsPointer() throws Exception {
+ String content = "size_t\nsome_function(void* ptr);\n";
+ File smallFile = writeTrashFile("Test.txt", content);
+ StringBuilder largeContent = new StringBuilder(
+ LfsPointer.SIZE_THRESHOLD * 4);
+ while (largeContent.length() < LfsPointer.SIZE_THRESHOLD * 4) {
+ largeContent.append(content);
+ }
+ File largeFile = writeTrashFile("large.txt", largeContent.toString());
+ fsTick(largeFile);
+ git.add().addFilepattern("Test.txt").addFilepattern("large.txt").call();
+ git.commit().setMessage("Text files").call();
+ writeTrashFile(".gitattributes", "*.txt filter=lfs");
+ git.add().addFilepattern(".gitattributes").call();
+ git.commit().setMessage("attributes").call();
+ assertTrue(smallFile.delete());
+ assertTrue(largeFile.delete());
+ // This reset will run the two text files through the smudge filter
+ git.reset().setMode(ResetType.HARD).call();
+ assertTrue(smallFile.exists());
+ assertTrue(largeFile.exists());
+ checkFile(smallFile, content);
+ checkFile(largeFile, largeContent.toString());
+ // Modify the large file
+ largeContent.append(content);
+ writeTrashFile("large.txt", largeContent.toString());
+ // This should convert largeFile to an LFS pointer
+ git.add().addFilepattern("large.txt").call();
+ git.commit().setMessage("Large modified").call();
+ String lfsPtr = "version https://git-lfs.github.com/spec/v1\n"
+ + "oid sha256:d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614\n"
+ + "size 858\n";
+ assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]"
+ + "[Test.txt, mode:100644, content:" + content + ']'
+ + "[large.txt, mode:100644, content:" + lfsPtr + ']',
+ indexState(CONTENT));
+ // Verify the file has been saved
+ File savedFile = new File(db.getDirectory(), "lfs");
+ savedFile = new File(savedFile, "objects");
+ savedFile = new File(savedFile, "d0");
+ savedFile = new File(savedFile, "41");
+ savedFile = new File(savedFile,
+ "d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614");
+ String saved = new String(Files.readAllBytes(savedFile.toPath()),
+ StandardCharsets.UTF_8);
+ assertEquals(saved, largeContent.toString());
+
+ assertTrue(smallFile.delete());
+ assertTrue(largeFile.delete());
+ git.reset().setMode(ResetType.HARD).call();
+ assertTrue(smallFile.exists());
+ assertTrue(largeFile.exists());
+ checkFile(smallFile, content);
+ checkFile(largeFile, largeContent.toString());
+ assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]"
+ + "[Test.txt, mode:100644, content:" + content + ']'
+ + "[large.txt, mode:100644, content:" + lfsPtr + ']',
+ indexState(CONTENT));
+ git.add().addFilepattern("Test.txt").call();
+ git.commit().setMessage("Small committed again").call();
+ String lfsPtrSmall = "version https://git-lfs.github.com/spec/v1\n"
+ + "oid sha256:9110463275fb0e2f0e9fdeaf84e598e62915666161145cf08927079119cc7814\n"
+ + "size 33\n";
+ assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]"
+ + "[Test.txt, mode:100644, content:" + lfsPtrSmall + ']'
+ + "[large.txt, mode:100644, content:" + lfsPtr + ']',
+ indexState(CONTENT));
+
+ assertTrue(git.status().call().isClean());
+ }
+}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java
index 52c3001..032a19b 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others
+ * Copyright (C) 2017, 2021 Markus Duft <markus.duft@ssi-schaefer.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -45,7 +45,7 @@
*/
public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader)
throws IOException {
- if (loader.getSize() > LfsPointer.SIZE_THRESHOLD) {
+ if (loader.getSize() > LfsPointer.FULL_SIZE_THRESHOLD) {
return loader;
}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java
index aef4416..0a8a3fa 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java
@@ -11,7 +11,9 @@
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.io.BufferedInputStream;
import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -25,6 +27,7 @@
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.util.IO;
/**
* Represents an LFS pointer file
@@ -57,6 +60,12 @@
public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION
.toLowerCase(Locale.ROOT).replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$
+ /**
+ * {@link #SIZE_THRESHOLD} is too low; with lfs extensions a LFS pointer can
+ * be larger. But 8kB should be more than enough.
+ */
+ static final int FULL_SIZE_THRESHOLD = 8 * 1024;
+
private final AnyLongObjectId oid;
private final long size;
@@ -115,64 +124,113 @@
/**
* Try to parse the data provided by an InputStream to the format defined by
- * {@link #VERSION}
+ * {@link #VERSION}. If the given stream supports mark and reset as
+ * indicated by {@link InputStream#markSupported()}, its input position will
+ * be reset if the stream content is not actually a LFS pointer (i.e., when
+ * {@code null} is returned). If the stream content is an invalid LFS
+ * pointer or the given stream does not support mark/reset, the input
+ * position may not be reset.
*
* @param in
* the {@link java.io.InputStream} from where to read the data
- * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or <code>null</code>
- * if the stream was not parseable as LfsPointer
+ * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or {@code null} if the
+ * stream was not parseable as LfsPointer
* @throws java.io.IOException
*/
@Nullable
public static LfsPointer parseLfsPointer(InputStream in)
throws IOException {
+ if (in.markSupported()) {
+ return parse(in);
+ }
+ // Fallback; note that while parse() resets its input stream, that won't
+ // reset "in".
+ return parse(new BufferedInputStream(in));
+ }
+
+ @Nullable
+ private static LfsPointer parse(InputStream in)
+ throws IOException {
+ if (!in.markSupported()) {
+ // No translation; internal error
+ throw new IllegalArgumentException(
+ "LFS pointer parsing needs InputStream.markSupported() == true"); //$NON-NLS-1$
+ }
+ // Try reading only a short block first.
+ in.mark(SIZE_THRESHOLD);
+ byte[] preamble = new byte[SIZE_THRESHOLD];
+ int length = IO.readFully(in, preamble, 0);
+ if (length < preamble.length || in.read() < 0) {
+ // We have the whole file. Try to parse a pointer from it.
+ try (BufferedReader r = new BufferedReader(new InputStreamReader(
+ new ByteArrayInputStream(preamble, 0, length), UTF_8))) {
+ LfsPointer ptr = parse(r);
+ if (ptr == null) {
+ in.reset();
+ }
+ return ptr;
+ }
+ }
+ // Longer than SIZE_THRESHOLD: expect "version" to be the first line.
+ boolean hasVersion = checkVersion(preamble);
+ in.reset();
+ if (!hasVersion) {
+ return null;
+ }
+ in.mark(FULL_SIZE_THRESHOLD);
+ byte[] fullPointer = new byte[FULL_SIZE_THRESHOLD];
+ length = IO.readFully(in, fullPointer, 0);
+ if (length == fullPointer.length && in.read() >= 0) {
+ in.reset();
+ return null; // Too long.
+ }
+ try (BufferedReader r = new BufferedReader(new InputStreamReader(
+ new ByteArrayInputStream(fullPointer, 0, length), UTF_8))) {
+ LfsPointer ptr = parse(r);
+ if (ptr == null) {
+ in.reset();
+ }
+ return ptr;
+ }
+ }
+
+ private static LfsPointer parse(BufferedReader r) throws IOException {
boolean versionLine = false;
LongObjectId id = null;
long sz = -1;
-
// This parsing is a bit too general if we go by the spec at
// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
- // Comment lines are not mentioned in the spec, and the "version" line
- // MUST be the first.
- try (BufferedReader br = new BufferedReader(
- new InputStreamReader(in, UTF_8))) {
- for (String s = br.readLine(); s != null; s = br.readLine()) {
- if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$
- continue;
- } else if (s.startsWith("version")) { //$NON-NLS-1$
- if (versionLine || s.length() < 8 || s.charAt(7) != ' ') {
- return null; // Not a LFS pointer
- }
- String rest = s.substring(8).trim();
- versionLine = VERSION.equals(rest)
- || VERSION_LEGACY.equals(rest);
- if (!versionLine) {
- return null; // Not a LFS pointer
- }
- } else {
- try {
- if (s.startsWith("oid sha256:")) { //$NON-NLS-1$
- if (id != null) {
- return null; // Not a LFS pointer
- }
- id = LongObjectId
- .fromString(s.substring(11).trim());
- } else if (s.startsWith("size")) { //$NON-NLS-1$
- if (sz > 0 || s.length() < 5
- || s.charAt(4) != ' ') {
- return null; // Not a LFS pointer
- }
- sz = Long.parseLong(s.substring(5).trim());
+ // Comment lines are not mentioned in the spec, the "version" line
+ // MUST be the first, and keys are ordered alphabetically.
+ for (String s = r.readLine(); s != null; s = r.readLine()) {
+ if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$
+ continue;
+ } else if (s.startsWith("version")) { //$NON-NLS-1$
+ if (versionLine || !checkVersionLine(s)) {
+ return null; // Not a LFS pointer
+ }
+ versionLine = true;
+ } else {
+ try {
+ if (s.startsWith("oid sha256:")) { //$NON-NLS-1$
+ if (id != null) {
+ return null; // Not a LFS pointer
}
- } catch (RuntimeException e) {
- // We could not parse the line. If we have a version
- // already, this is a corrupt LFS pointer. Otherwise it
- // is just not an LFS pointer.
- if (versionLine) {
- throw e;
+ id = LongObjectId.fromString(s.substring(11).trim());
+ } else if (s.startsWith("size")) { //$NON-NLS-1$
+ if (sz > 0 || s.length() < 5 || s.charAt(4) != ' ') {
+ return null; // Not a LFS pointer
}
- return null;
+ sz = Long.parseLong(s.substring(5).trim());
}
+ } catch (RuntimeException e) {
+ // We could not parse the line. If we have a version
+ // already, this is a corrupt LFS pointer. Otherwise it
+ // is just not an LFS pointer.
+ if (versionLine) {
+ throw e;
+ }
+ return null;
}
}
if (versionLine && id != null && sz > -1) {
@@ -182,6 +240,30 @@
return null;
}
+ private static boolean checkVersion(byte[] data) {
+ // According to the spec at
+ // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+ // it MUST always be the first line.
+ try (BufferedReader r = new BufferedReader(
+ new InputStreamReader(new ByteArrayInputStream(data), UTF_8))) {
+ String s = r.readLine();
+ if (s != null && s.startsWith("version")) { //$NON-NLS-1$
+ return checkVersionLine(s);
+ }
+ } catch (IOException e) {
+ // Doesn't occur, we're reading from a byte array!
+ }
+ return false;
+ }
+
+ private static boolean checkVersionLine(String s) {
+ if (s.length() < 8 || s.charAt(7) != ' ') {
+ return false; // Not a valid LFS pointer version line
+ }
+ String rest = s.substring(8).trim();
+ return VERSION.equals(rest) || VERSION_LEGACY.equals(rest);
+ }
+
/** {@inheritDoc} */
@Override
public String toString() {
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
index 2f80d5b..3411887 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others
+ * Copyright (C) 2016, 2021 Christian Halstrick <christian.halstrick@sap.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -11,6 +11,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -87,20 +88,31 @@
*/
public SmudgeFilter(Repository db, InputStream in, OutputStream out)
throws IOException {
+ this(in.markSupported() ? in : new BufferedInputStream(in), out, db);
+ }
+
+ private SmudgeFilter(InputStream in, OutputStream out, Repository db)
+ throws IOException {
super(in, out);
+ InputStream from = in;
try {
- Lfs lfs = new Lfs(db);
- LfsPointer res = LfsPointer.parseLfsPointer(in);
+ LfsPointer res = LfsPointer.parseLfsPointer(from);
if (res != null) {
AnyLongObjectId oid = res.getOid();
+ Lfs lfs = new Lfs(db);
Path mediaFile = lfs.getMediaFile(oid);
if (!Files.exists(mediaFile)) {
downloadLfsResource(lfs, db, res);
}
this.in = Files.newInputStream(mediaFile);
+ } else {
+ // Not swapped; stream was reset, don't close!
+ from = null;
}
} finally {
- in.close(); // make sure the swapped stream is closed properly.
+ if (from != null) {
+ from.close(); // Close the swapped-out stream
+ }
}
}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java
index d84eebd..99bae49 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2015, 2017, Dariusz Luksza <dariusz@luksza.org> and others
+ * Copyright (C) 2015, 2021 Dariusz Luksza <dariusz@luksza.org> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -58,6 +58,8 @@
try (ObjectStream stream = object.openStream()) {
pointer = LfsPointer.parseLfsPointer(stream);
return pointer != null;
+ } catch (RuntimeException e) {
+ return false;
}
}
