Merge branch 'master' into stable-7.7 * master: RefDirectory#refreshPathToLooseRef: also refresh loose ref itself ReceivePack: Add missing @since tag for new API method TreeWalkConnectivityChecker: Support non-commit objects in refs ReceivePack: make getClientShallowCommits protected DfsObjDatabase: Make createDfsPackFile public DfsPackCompactor: add a pre-commit hook to plug midx calculation Update bouncycastle to 1.84 [ssh] Reject host certificates if the certified key is revoked Add new TreeWalkConnectivityChecker Make the PackParser.needNewObjectIds method public orbit: update org.apache.ant to 1.10.17.v20260410-1000 MidxMetadataReader: add private constructor and adjust method visibility MidxWriter: Write to tmp files and use atomic rename to commit Instrument ReceivePack to track time spent in pre-receive hooks MidxWriter: Do not accept midx to build new midx PackDirectory: Integrate Multi-Pack Index (MIDX) support PackMidx: A Pack subclass that uses midx Change-Id: I5fd4c34673abaecf0db805cfd9383a4feb5028f9
diff --git a/MODULE.bazel b/MODULE.bazel index 05a9306..0d46628 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -23,7 +23,7 @@ remote = "https://gerrit.googlesource.com/bazlets", ) -BOUNCYCASTLE_VERSION = "1.83" +BOUNCYCASTLE_VERSION = "1.84" BYTE_BUDDY_VERSION = "1.18.8"
diff --git a/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF index 1b2793a..d9d1b0f 100644 --- a/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF
@@ -8,12 +8,12 @@ Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-17 Bundle-SCM: url=https://github.com/eclipse-jgit/jgit, connection=scm:git:https://eclipse.gerrithub.io/eclipse-jgit/jgit.git, developerConnection=scm:git:https://eclipse.gerrithub.io/a/eclipse-jgit/jgit.git -Import-Package: org.bouncycastle.asn1.cryptlib;version="[1.83.0,2.0.0)", - org.bouncycastle.jce.provider;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp.operator;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp.operator.jcajce;version="[1.83.0,2.0.0)", - org.bouncycastle.util.encoders;version="[1.83.0,2.0.0)", +Import-Package: org.bouncycastle.asn1.cryptlib;version="[1.84.0,2.0.0)", + org.bouncycastle.jce.provider;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp.operator;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp.operator.jcajce;version="[1.84.0,2.0.0)", + org.bouncycastle.util.encoders;version="[1.84.0,2.0.0)", org.eclipse.jgit.gpg.bc.internal;version="[7.7.0,7.8.0)", org.eclipse.jgit.gpg.bc.internal.keys;version="[7.7.0,7.8.0)", org.eclipse.jgit.util.sha1;version="[7.7.0,7.8.0)",
diff --git a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF index c039403..d6d826c 100644 --- a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF
@@ -9,23 +9,23 @@ Bundle-Version: 7.7.0.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-17 Bundle-SCM: url=https://github.com/eclipse-jgit/jgit, connection=scm:git:https://eclipse.gerrithub.io/eclipse-jgit/jgit.git, developerConnection=scm:git:https://eclipse.gerrithub.io/a/eclipse-jgit/jgit.git -Import-Package: org.bouncycastle.asn1;version="[1.83.0,2.0.0)", - org.bouncycastle.asn1.x9;version="[1.83.0,2.0.0)", - org.bouncycastle.bcpg;version="[1.83.0,2.0.0)", - org.bouncycastle.bcpg.sig;version="[1.83.0,2.0.0)", - org.bouncycastle.crypto.ec;version="[1.83.0,2.0.0)", - org.bouncycastle.gpg;version="[1.83.0,2.0.0)", - org.bouncycastle.gpg.keybox;version="[1.83.0,2.0.0)", - org.bouncycastle.gpg.keybox.jcajce;version="[1.83.0,2.0.0)", - org.bouncycastle.jcajce.interfaces;version="[1.83.0,2.0.0)", - org.bouncycastle.jcajce.util;version="[1.83.0,2.0.0)", - org.bouncycastle.math.ec;version="[1.83.0,2.0.0)", - org.bouncycastle.math.field;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp.jcajce;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp.operator;version="[1.83.0,2.0.0)", - org.bouncycastle.openpgp.operator.jcajce;version="[1.83.0,2.0.0)", - org.bouncycastle.util.encoders;version="[1.83.0,2.0.0)", +Import-Package: org.bouncycastle.asn1;version="[1.84.0,2.0.0)", + org.bouncycastle.asn1.x9;version="[1.84.0,2.0.0)", + org.bouncycastle.bcpg;version="[1.84.0,2.0.0)", + org.bouncycastle.bcpg.sig;version="[1.84.0,2.0.0)", + org.bouncycastle.crypto.ec;version="[1.84.0,2.0.0)", + org.bouncycastle.gpg;version="[1.84.0,2.0.0)", + org.bouncycastle.gpg.keybox;version="[1.84.0,2.0.0)", + org.bouncycastle.gpg.keybox.jcajce;version="[1.84.0,2.0.0)", + org.bouncycastle.jcajce.interfaces;version="[1.84.0,2.0.0)", + org.bouncycastle.jcajce.util;version="[1.84.0,2.0.0)", + org.bouncycastle.math.ec;version="[1.84.0,2.0.0)", + org.bouncycastle.math.field;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp.jcajce;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp.operator;version="[1.84.0,2.0.0)", + org.bouncycastle.openpgp.operator.jcajce;version="[1.84.0,2.0.0)", + org.bouncycastle.util.encoders;version="[1.84.0,2.0.0)", org.slf4j;version="[1.7.0,3.0.0)" Export-Package: org.eclipse.jgit.gpg.bc.internal;version="7.7.0";x-friends:="org.eclipse.jgit.gpg.bc.test", org.eclipse.jgit.gpg.bc.internal.keys;version="7.7.0";x-friends:="org.eclipse.jgit.gpg.bc.test"
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target index 9a89f02..9f231d1 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.34" sequenceNumber="1775482489"> +<target name="jgit-4.34" sequenceNumber="1776265462"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target index f4db06e..345a39c 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.35" sequenceNumber="1775482489"> +<target name="jgit-4.35" sequenceNumber="1776265462"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target index c78cd61..9d1aa8a 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.36" sequenceNumber="1775482489"> +<target name="jgit-4.36" sequenceNumber="1776265463"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target index 39d7588..d89a1ba 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.37" sequenceNumber="1775482489"> +<target name="jgit-4.37" sequenceNumber="1776265462"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target index e88003a..4d7ff17 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.38" sequenceNumber="1775482489"> +<target name="jgit-4.38" sequenceNumber="1776265464"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target index d2e4728..1155365 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.39" sequenceNumber="1775482489"> +<target name="jgit-4.39" sequenceNumber="1776265464"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.40.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.40.target index 3c81647..d494040 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.40.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.40.target
@@ -1,15 +1,15 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.40" sequenceNumber="1775482489"> +<target name="jgit-4.40" sequenceNumber="1776265463"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> <unit id="com.jcraft.jsch.source" version="0.1.55.v20230916-1400"/> <unit id="com.jcraft.jzlib" version="1.1.3.v20230916-1400"/> <unit id="com.jcraft.jzlib.source" version="1.1.3.v20230916-1400"/> - <unit id="org.apache.ant" version="1.10.15.v20240901-1000"/> - <unit id="org.apache.ant.source" version="1.10.15.v20240901-1000"/> + <unit id="org.apache.ant" version="1.10.17.v20260410-1000"/> + <unit id="org.apache.ant.source" version="1.10.17.v20260410-1000"/> <unit id="org.apache.aries.spifly.dynamic.bundle" version="1.3.7"/> <unit id="org.apache.httpcomponents.httpclient" version="4.5.14"/> <unit id="org.apache.httpcomponents.httpclient.source" version="4.5.14"/> @@ -217,25 +217,25 @@ <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpg-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcutil-jdk18on</artifactId> - <version>1.83</version> + <version>1.84</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd index ed24f94..3ada4c5 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd
@@ -69,22 +69,22 @@ dependency { groupId = "org.bouncycastle" artifactId = "bcpg-jdk18on" - version = "1.83" + version = "1.84" } dependency { groupId = "org.bouncycastle" artifactId = "bcprov-jdk18on" - version = "1.83" + version = "1.84" } dependency { groupId = "org.bouncycastle" artifactId = "bcpkix-jdk18on" - version = "1.83" + version = "1.84" } dependency { groupId = "org.bouncycastle" artifactId = "bcutil-jdk18on" - version = "1.83" + version = "1.84" } }
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.40.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.40.tpd index 4dbf8f3..4ce74f7 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.40.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.40.tpd
@@ -6,8 +6,8 @@ com.jcraft.jsch.source [0.1.55.v20230916-1400,0.1.55.v20230916-1400] com.jcraft.jzlib [1.1.3.v20230916-1400,1.1.3.v20230916-1400] com.jcraft.jzlib.source [1.1.3.v20230916-1400,1.1.3.v20230916-1400] - org.apache.ant [1.10.15.v20240901-1000,1.10.15.v20240901-1000] - org.apache.ant.source [1.10.15.v20240901-1000,1.10.15.v20240901-1000] + org.apache.ant [1.10.17.v20260410-1000,1.10.17.v20260410-1000] + org.apache.ant.source [1.10.17.v20260410-1000,1.10.17.v20260410-1000] org.apache.aries.spifly.dynamic.bundle [1.3.7,1.3.7] org.apache.httpcomponents.httpclient [4.5.14,4.5.14] org.apache.httpcomponents.httpclient.source [4.5.14,4.5.14]
diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabaseTest.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabaseTest.java index 6b61821..ff6eb2b 100644 --- a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabaseTest.java +++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabaseTest.java
@@ -28,6 +28,7 @@ import org.apache.sshd.common.SshConstants; import org.apache.sshd.common.cipher.ECCurves; import org.apache.sshd.common.config.keys.KeyUtils; +import org.apache.sshd.common.config.keys.OpenSshCertificate; import org.apache.sshd.common.config.keys.PublicKeyEntry; import org.apache.sshd.common.util.security.SecurityUtils; import org.eclipse.jgit.annotations.NonNull; @@ -58,7 +59,8 @@ public class OpenSshServerKeyDatabaseTest { private static PublicKey ec256; private static PublicKey ec384; private static PublicKey caKey; - private static PublicKey certificate; + + private static OpenSshCertificate certificate; @BeforeClass public static void initKeys() throws Exception { @@ -179,7 +181,7 @@ public void testCaKeyNotConsidered() throws Exception { } @Test - public void testkeyPlainAndCa() throws Exception { + public void testKeyPlainAndCa() throws Exception { Files.write(knownHosts, List.of( "localhost,127.0.0.1 " + PublicKeyEntry.toString(rsa1024), "some.other.com " + PublicKeyEntry.toString(ec384), @@ -192,7 +194,46 @@ public void testkeyPlainAndCa() throws Exception { } @Test + public void testCaIsPlain() throws Exception { + Files.write(knownHosts, List.of( + "localhost,127.0.0.1 " + PublicKeyEntry.toString(rsa1024), + "some.other.com " + PublicKeyEntry.toString(ec384), + "localhost,127.0.0.1 " + PublicKeyEntry.toString(caKey))); + // caKey not marked as cert-authorithy + assertFalse(database.accept("localhost", LOCAL, certificate, + new KnownHostsConfig(), null)); + } + + @Test + public void testRevokedCa() throws Exception { + Files.write(knownHosts, + List.of("localhost,127.0.0.1 " + + PublicKeyEntry.toString(rsa1024), + "some.other.com " + PublicKeyEntry.toString(ec384), + "@revoked localhost,127.0.0.1 " + + PublicKeyEntry.toString(caKey))); + assertFalse(database.accept("localhost", LOCAL, certificate, + new KnownHostsConfig(), null)); + } + + @Test + public void testRevokedCertifiedKey() throws Exception { + Files.write(knownHosts, + List.of("localhost,127.0.0.1 " + + PublicKeyEntry.toString(rsa1024), + "some.other.com " + PublicKeyEntry.toString(ec384), + "@cert-authority localhost,127.0.0.1 " + + PublicKeyEntry.toString(caKey), + "@revoked localhost,127.0.0.1 " + PublicKeyEntry + .toString(certificate.getCertPubKey()))); + assertFalse(database.accept("localhost", LOCAL, certificate, + new KnownHostsConfig(), null)); + } + + @Test public void testLookupCertificate() throws Exception { + Files.write(knownHosts, List.of("@cert-authority localhost,127.0.0.1 " + + PublicKeyEntry.toString(caKey))); List<PublicKey> keys = database.lookup("localhost", LOCAL, new KnownHostsConfig()); // Certificates or CA keys are not reported via lookup.
diff --git a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF index 2647126..5cff8a8 100644 --- a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF
@@ -38,7 +38,7 @@ org.apache.sshd.client.session, org.apache.sshd.client.keyverifier", org.eclipse.jgit.transport.sshd.agent;version="7.7.0" -Import-Package: org.bouncycastle.jce.provider;version="[1.83.0,2.0.0)", +Import-Package: org.bouncycastle.jce.provider;version="[1.84.0,2.0.0)", org.apache.sshd.agent;version="[2.17.1,2.18.0)", org.apache.sshd.client;version="[2.17.1,2.18.0)", org.apache.sshd.client.auth;version="[2.17.1,2.18.0)",
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java index acb77c5..7300a44 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java
@@ -210,10 +210,26 @@ public boolean accept(@NonNull String connectAddress, @NonNull Configuration config, CredentialsProvider provider) { List<HostKeyFile> filesToUse = getFilesToUse(config); AskUser ask = new AskUser(config, provider); - HostEntryPair[] modified = { null }; - Path path = null; Collection<SshdSocketAddress> candidates = getCandidates(connectAddress, remoteAddress); + if (acceptKey(remoteAddress, candidates, serverKey, filesToUse, config, + ask)) { + if (serverKey instanceof OpenSshCertificate certificate) { + if (isRevoked(remoteAddress, candidates, filesToUse, + certificate.getCertPubKey(), ask)) { + return false; + } + } + return true; + } + return false; + } + + private boolean acceptKey(InetSocketAddress remoteAddress, + Collection<SshdSocketAddress> candidates, PublicKey serverKey, + List<HostKeyFile> filesToUse, Configuration config, AskUser ask) { + HostEntryPair[] modified = { null }; + Path path = null; for (HostKeyFile file : filesToUse) { HostEntryPair lastModified = modified[0]; try { @@ -285,7 +301,26 @@ private static boolean isCertificateAuthority(KnownHostEntry entry) { return MARKER_CA.equals(entry.getMarker()); } - private boolean find(Collection<SshdSocketAddress> candidates, + private static boolean isRevoked(InetSocketAddress remoteAddress, + Collection<SshdSocketAddress> candidates, + List<HostKeyFile> filesToUse, PublicKey serverKey, AskUser ask) { + HostEntryPair[] dummy = { null }; + for (HostKeyFile file : filesToUse) { + try { + if (find(candidates, serverKey, file.get(), dummy)) { + // It's definitely not revoked. + return false; + } + } catch (RevokedKeyException e) { + ask.revokedKey(remoteAddress, serverKey, file.getPath()); + return true; + } + } + // Not found, so it is not revoked + return false; + } + + private static boolean find(Collection<SshdSocketAddress> candidates, PublicKey serverKey, List<HostEntryPair> entries, HostEntryPair[] modified) throws RevokedKeyException { PublicKey keyToCheck = serverKey;
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/MidxWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/MidxWriterTest.java new file mode 100644 index 0000000..bb45d0a --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/MidxWriterTest.java
@@ -0,0 +1,91 @@ +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.lib.Constants.MIDX_FILE; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; + +import org.eclipse.jgit.internal.storage.midx.MultiPackIndex; +import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader; +import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Before; +import org.junit.Test; + +public class MidxWriterTest extends LocalDiskRepositoryTestCase { + private int streamThreshold = 16 * 1024; + + private final static ObjectId UNKNOWN_OBJ = ObjectId + .fromString("678668bdd3a609c6e1d7fea516e9fc1ed4f50ad2"); + + private FileRepository repo; + + private TestRepository<Repository> tr; + +// private WindowCursor wc; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + WindowCacheConfig cfg = new WindowCacheConfig(); + cfg.setStreamFileThreshold(streamThreshold); + cfg.install(); + + repo = createBareRepository(); + tr = new TestRepository<>(repo); +// wc = (WindowCursor) repo.newObjectReader(); + } + + @Test + public void midx_write() throws Exception { + TestRepository<Repository>.BranchBuilder branch = tr + .branch("refs/heads/main"); + RevBlob contentsOfA = tr.blob("contents of a"); + RevCommit commitA = branch.commit().add("a.txt", contentsOfA).create(); + tr.packAndPrune(); + + RevBlob contentsOfB = tr.blob("contents of b"); + RevCommit commitB = branch.commit().add("b.txt", contentsOfB).create(); + tr.packAndPrune(); + + RevCommit commitC = branch.commit().add("c.txt", "contents of c commit") + .create(); + tr.packAndPrune(); + + File midxFile = new File(repo.getObjectDatabase().getPackDirectory(), + MIDX_FILE); + assertFalse(midxFile.exists()); + MidxWriter.writeMidx( + new TextProgressMonitor(new OutputStreamWriter(System.out, + StandardCharsets.UTF_8)), + repo, repo.getObjectDatabase().getPacks(), midxFile, + new PackConfig()); + assertTrue(midxFile.exists()); + + MultiPackIndex midx = MultiPackIndexLoader.open(midxFile); + assertTrue(midx.hasObject(commitA)); + assertTrue(midx.hasObject(commitB)); + assertTrue(midx.hasObject(commitC)); + assertFalse(midx.hasObject(UNKNOWN_OBJ)); + + File midxBitmaps = new File(repo.getObjectDatabase().getPackDirectory(), + String.format("%s-%s.%s", MIDX_FILE, + ObjectId.fromRaw(midx.getChecksum()).name(), + BITMAP_INDEX.getExtension())); + assertTrue(midxBitmaps.exists()); + + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackDirectoryTest.java new file mode 100644 index 0000000..b04d31a --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackDirectoryTest.java
@@ -0,0 +1,112 @@ +/* + * Copyright (C) 2026, Google LLC. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.lib.Constants.MIDX_FILE; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Before; +import org.junit.Test; + +public class PackDirectoryTest extends LocalDiskRepositoryTestCase { + private int streamThreshold = 16 * 1024; + + private FileRepository repo; + + private TestRepository<FileRepository> tr; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + WindowCacheConfig cfg = new WindowCacheConfig(); + cfg.setStreamFileThreshold(streamThreshold); + cfg.install(); + + mockSystemReader.getSystemConfig().setBoolean( + ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_MULTIPACKINDEX, true); + + repo = createBareRepository(); + tr = new TestRepository<>(repo); + } + + @Test + public void getPacks_midxReplacesCoveredPacks() throws Exception { + createThreePacks(); + Collection<Pack> packs = repo.getObjectDatabase().getPacks(); + assertEquals(3, packs.size()); + writeMidxOverAllPacks(); + + // Reading now should return the midx + packs = repo.getObjectDatabase().getPacks(); + assertEquals(1, packs.size()); + + tr.branch("refs/heads/main").commit().add("more.txt", "booooo") + .create(); + tr.packAndPrune(); + + packs = repo.getObjectDatabase().getPacks(); + assertEquals(2, packs.size()); + } + + private TestRepoObjects createThreePacks() throws Exception { + TestRepository<FileRepository>.BranchBuilder branch = tr + .branch("refs/heads/main"); + RevBlob contentsOfA = tr.blob("contents of a"); + RevCommit commitA = branch.commit().add("a.txt", contentsOfA).create(); + tr.packAndPrune(); + + RevBlob contentsOfB = tr.blob("contents of b"); + RevCommit commitB = branch.commit().add("b.txt", contentsOfB).create(); + tr.packAndPrune(); + + RevCommit commitC = branch.commit().add("c.txt", "contents of c commit") + .create(); + tr.packAndPrune(); + + Map<RevObject, Integer> midxOffsets = Map.of(commitA, 163, commitB, 12, + commitC, 694, contentsOfA, 399, contentsOfB, 421); + + return new TestRepoObjects(commitA, commitB, commitC, contentsOfA, + contentsOfB, midxOffsets); + } + + private record TestRepoObjects(RevCommit commitA, RevCommit commitB, + RevCommit commitC, RevBlob contentsOfA, RevBlob contentsOfB, + Map<RevObject, Integer> midxOffsets) { + } + + private void writeMidxOverAllPacks() throws IOException { + Collection<Pack> packs = tr.getRepository().getObjectDatabase() + .getPacks(); + assertEquals(3, packs.size()); + + File midxDest = new File(repo.getObjectDatabase().getPackDirectory(), + MIDX_FILE); + MidxWriter.writeMidx(new TextProgressMonitor(), repo, packs, midxDest, + new PackConfig(repo)); + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackMidxTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackMidxTest.java new file mode 100644 index 0000000..7a497d6 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackMidxTest.java
@@ -0,0 +1,380 @@ +/* + * Copyright (C) 2026, Google LLC + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Before; +import org.junit.Test; + +public class PackMidxTest extends LocalDiskRepositoryTestCase { + private int streamThreshold = 16 * 1024; + + private final static ObjectId UNKNOWN_OBJ = ObjectId + .fromString("678668bdd3a609c6e1d7fea516e9fc1ed4f50ad2"); + + private FileRepository repo; + + private TestRepository<Repository> tr; + + private WindowCursor wc; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + WindowCacheConfig cfg = new WindowCacheConfig(); + cfg.setStreamFileThreshold(streamThreshold); + cfg.install(); + + repo = createBareRepository(); + tr = new TestRepository<>(repo); + wc = (WindowCursor) repo.newObjectReader(); + } + + @Test + public void midx_hasObject() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + assertTrue(midx.hasObject(objs.commitA)); + assertTrue(midx.hasObject(objs.commitB)); + assertTrue(midx.hasObject(objs.commitC)); + assertTrue(midx.hasObject(objs.contentsOfB)); + assertTrue(midx.hasObject(objs.contentsOfA)); + assertFalse(midx.hasObject(UNKNOWN_OBJ)); + } + + @Test + public void midx_getObjectCount() throws Exception { + createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + assertEquals(9, midx.getObjectCount()); + } + + @Test + public void midx_getObjectSize_byId() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + assertEquals(getObjectSizeFromObjectDb(objs.commitA), + midx.getObjectSize(wc, objs.commitA)); + assertEquals(getObjectSizeFromObjectDb(objs.commitB), + midx.getObjectSize(wc, objs.commitB)); + assertEquals(getObjectSizeFromObjectDb(objs.commitC), + midx.getObjectSize(wc, objs.commitC)); + assertEquals(getObjectSizeFromObjectDb(objs.contentsOfA), + midx.getObjectSize(wc, objs.contentsOfA)); + assertEquals(getObjectSizeFromObjectDb(objs.contentsOfB), + midx.getObjectSize(wc, objs.contentsOfB)); + assertEquals(-1, midx.getObjectSize(wc, UNKNOWN_OBJ)); + } + + @Test + public void midx_getObjectSize_byOffset() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> midxOffsets = objs.midxOffsets; + Integer offsetCommitA = midxOffsets.get(objs.commitA); + assertEquals(getObjectSizeFromObjectDb(objs.commitA), + midx.getObjectSize(wc, offsetCommitA)); + + Integer offsetCommitB = midxOffsets.get(objs.commitB); + assertEquals(getObjectSizeFromObjectDb(objs.commitB), + midx.getObjectSize(wc, offsetCommitB)); + + Integer offsetCommitC = midxOffsets.get(objs.commitC); + assertEquals(getObjectSizeFromObjectDb(objs.commitC), + midx.getObjectSize(wc, offsetCommitC)); + + Integer offsetContentsOfA = midxOffsets.get(objs.contentsOfA); + assertEquals(getObjectSizeFromObjectDb(objs.contentsOfA), + midx.getObjectSize(wc, offsetContentsOfA)); + + Integer offsetContentsOfB = midxOffsets.get(objs.contentsOfB); + assertEquals(midx.getObjectSize(wc, objs.contentsOfB), + midx.getObjectSize(wc, offsetContentsOfB)); + } + + private long getObjectSizeFromObjectDb(AnyObjectId oid) throws IOException { + return repo.getObjectDatabase().getObjectSize(wc, oid); + } + + @Test + public void midx_get_byOid() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + assertEquals("contents of a", midxGet(midx, objs.contentsOfA)); + assertEquals("contents of b", midxGet(midx, objs.contentsOfB)); + String commitContent = midxGet(midx, objs.commitB); + assertTrue(commitContent + .startsWith("tree 4ad3577dd1af1fe281a6e55d48cee2d397d570d5")); + assertTrue(commitContent + .contains("author J. Author <jauthor@example.com>")); + + assertNull(midx.get(wc, UNKNOWN_OBJ)); + } + + private String midxGet(PackMidx midx, AnyObjectId oid) throws IOException { + ObjectLoader ol = midx.get(wc, oid); + return new String(ol.getCachedBytes(), UTF_8); + } + + @Test + public void midx_load_byOffset() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> midxOffsets = objs.midxOffsets; + assertEquals("contents of a", + midxLoad(midx, midxOffsets.get(objs.contentsOfA))); + assertEquals("contents of b", + midxLoad(midx, midxOffsets.get(objs.contentsOfB))); + String commitContent = midxLoad(midx, midxOffsets.get(objs.commitB)); + assertTrue(commitContent + .startsWith("tree 4ad3577dd1af1fe281a6e55d48cee2d397d570d5")); + assertTrue(commitContent + .contains("author J. Author <jauthor@example.com>")); + } + + private String midxLoad(PackMidx midx, int pos) throws IOException { + ObjectLoader ol = midx.load(wc, pos); + return new String(ol.getCachedBytes(), UTF_8); + } + + @Test + public void midx_findObject_byOffset() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> midxOffsets = objs.midxOffsets; + assertEquals(objs.commitA, + midx.findObjectForOffset(midxOffsets.get(objs.commitA))); + assertEquals(objs.commitB, + midx.findObjectForOffset(midxOffsets.get(objs.commitB))); + assertEquals(objs.commitC, + midx.findObjectForOffset(midxOffsets.get(objs.commitC))); + assertEquals(objs.contentsOfA, + midx.findObjectForOffset(midxOffsets.get(objs.contentsOfA))); + assertEquals(objs.contentsOfB, + midx.findObjectForOffset(midxOffsets.get(objs.contentsOfB))); + } + + @Test + public void midx_getObjectType() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> midxOffsets = objs.midxOffsets; + assertEquals(Constants.OBJ_COMMIT, + midx.getObjectType(wc, midxOffsets.get(objs.commitA))); + assertEquals(Constants.OBJ_COMMIT, + midx.getObjectType(wc, midxOffsets.get(objs.commitB))); + assertEquals(Constants.OBJ_COMMIT, + midx.getObjectType(wc, midxOffsets.get(objs.commitC))); + assertEquals(Constants.OBJ_BLOB, + midx.getObjectType(wc, midxOffsets.get(objs.contentsOfA))); + assertEquals(Constants.OBJ_BLOB, + midx.getObjectType(wc, midxOffsets.get(objs.contentsOfB))); + } + + @Test + public void midx_iterator() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> midxOffsets = objs.midxOffsets; + Iterator<PackIndex.MutableEntry> it = midx.iterator(); + assertIteratorEntry(it, objs.contentsOfB, + midxOffsets.get(objs.contentsOfB)); + it.next(); + it.next(); + assertIteratorEntry(it, objs.commitB, midxOffsets.get(objs.commitB)); + assertIteratorEntry(it, objs.commitA, midxOffsets.get(objs.commitA)); + it.next(); + it.next(); + assertIteratorEntry(it, objs.contentsOfA, + midxOffsets.get(objs.contentsOfA)); + assertIteratorEntry(it, objs.commitC, midxOffsets.get(objs.commitC)); + assertFalse(it.hasNext()); + } + + private void assertIteratorEntry(Iterator<PackIndex.MutableEntry> it, + ObjectId oid, int offset) { + assertTrue(it.hasNext()); + PackIndex.MutableEntry next = it.next(); + assertEquals(oid, next.toObjectId()); + assertEquals(offset, next.getOffset()); + } + + @Test + public void midx_packIndex_hasObject() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + PackIndex index = midx.getIndex(); + assertTrue(index.hasObject(objs.commitA)); + assertTrue(index.hasObject(objs.commitB)); + assertTrue(index.hasObject(objs.commitC)); + assertTrue(index.hasObject(objs.contentsOfA)); + assertTrue(index.hasObject(objs.contentsOfB)); + assertFalse(index.hasObject(UNKNOWN_OBJ)); + } + + @Test + public void midx_packIndex_findPosition_getByPosition() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + PackIndex index = midx.getIndex(); + assertEquals(objs.commitA, + index.getObjectId(index.findPosition(objs.commitA))); + assertEquals(objs.commitB, + index.getObjectId(index.findPosition(objs.commitB))); + assertEquals(objs.commitC, + index.getObjectId(index.findPosition(objs.commitC))); + assertEquals(objs.contentsOfA, + index.getObjectId(index.findPosition(objs.contentsOfA))); + assertEquals(objs.contentsOfB, + index.getObjectId(index.findPosition(objs.contentsOfB))); + assertEquals(-1, index.findPosition(UNKNOWN_OBJ)); + } + + @Test + public void midx_packIndex_findOffset() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + Map<RevObject, Integer> offsets = objs.midxOffsets; + PackIndex index = midx.getIndex(); + assertEquals((long) offsets.get(objs.commitA), + index.findOffset(objs.commitA)); + assertEquals((long) offsets.get(objs.commitB), + index.findOffset(objs.commitB)); + assertEquals((long) offsets.get(objs.commitC), + index.findOffset(objs.commitC)); + assertEquals((long) offsets.get(objs.contentsOfA), + index.findOffset(objs.contentsOfA)); + assertEquals((long) offsets.get(objs.contentsOfB), + index.findOffset(objs.contentsOfB)); + assertEquals(-1, index.findOffset(UNKNOWN_OBJ)); + } + + @Test + public void midx_reversePackIndex_findObject() throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + List<ObjectOffset> offsetOrdered = inRidxOrder(objs.midxOffsets); + PackReverseIndex ridx = midx.getReverseIdx(); + + assertFalse(offsetOrdered.isEmpty()); + for (int i = 0; i < offsetOrdered.size(); i++) { + ObjectOffset entry = offsetOrdered.get(i); + assertEquals(entry.obj().getId(), ridx.findObject(entry.offset())); + } + } + + @Test + public void midx_reversePackIndex_findPosition_findObjectByPosition() + throws Exception { + TestRepoObjects objs = createThreePacks(); + PackMidx midx = writeAndOpenMidxAllPacks(); + + List<ObjectOffset> offsetOrdered = inRidxOrder(objs.midxOffsets); + PackReverseIndex ridx = midx.getReverseIdx(); + + assertFalse(offsetOrdered.isEmpty()); + for (int i = 0; i < offsetOrdered.size(); i++) { + ObjectOffset entry = offsetOrdered.get(i); + assertEquals(entry.obj(), ridx + .findObjectByPosition(ridx.findPosition(entry.offset()))); + } + } + + private PackMidx writeAndOpenMidxAllPacks() throws IOException { + Collection<Pack> packs = ((ObjectDirectory) tr.getRepository() + .getObjectDatabase()).getPacks(); + assertEquals(3, packs.size()); + + File midxDest = new File(repo.getObjectDatabase().getPackDirectory(), + Constants.MIDX_FILE); + MidxWriter.writeMidx(new TextProgressMonitor(), repo, packs, midxDest, + new PackConfig(repo)); + + return new PackMidx(new Config(), midxDest, packs.stream().toList()); + + } + + private TestRepoObjects createThreePacks() throws Exception { + TestRepository<Repository>.BranchBuilder branch = tr + .branch("refs/heads/main"); + RevBlob contentsOfA = tr.blob("contents of a"); + RevCommit commitA = branch.commit().add("a.txt", contentsOfA).create(); + tr.packAndPrune(); + + RevBlob contentsOfB = tr.blob("contents of b"); + RevCommit commitB = branch.commit().add("b.txt", contentsOfB).create(); + tr.packAndPrune(); + + RevCommit commitC = branch.commit().add("c.txt", "contents of c commit") + .create(); + tr.packAndPrune(); + + Map<RevObject, Integer> midxOffsets = Map.of(commitA, 163, commitB, 12, + commitC, 694, contentsOfA, 399, contentsOfB, 421); + + return new TestRepoObjects(commitA, commitB, commitC, contentsOfA, + contentsOfB, midxOffsets); + } + + private record TestRepoObjects(RevCommit commitA, RevCommit commitB, + RevCommit commitC, RevBlob contentsOfA, RevBlob contentsOfB, + Map<RevObject, Integer> midxOffsets) { + } + + private List<ObjectOffset> inRidxOrder(Map<RevObject, Integer> objs) { + return objs.entrySet().stream() + .map(e -> new ObjectOffset(e.getKey(), e.getValue())) + .sorted(Comparator.comparing(ObjectOffset::offset)).toList(); + } + + private record ObjectOffset(RevObject obj, Integer offset) { + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java index 1ca8aaf..1743fb0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java
@@ -174,6 +174,18 @@ public void jgit_emptyMidx() throws IOException { assertEquals(5, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); } + @Test + public void jgit_noPacks() throws IOException { + PackIndexMerger packs = PackIndexMerger.builder().build(); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MultiPackIndexWriter.Result result = writer + .write(NullProgressMonitor.INSTANCE, out, packs); + assertEquals(0, result.packNames().size()); + MidxHeader header = readHeader(out); + assertEquals(0, header.packCount()); + } + private List<Integer> readChunkIds(ByteArrayOutputStream out) { List<Integer> chunkIds = new ArrayList<>(); byte[] raw = out.toByteArray(); @@ -193,4 +205,14 @@ private static PackIndex indexOf(IndexObject... objs) { private static IndexObject object(String name, long offset) { return new IndexObject(name, offset); } + + private record MidxHeader(int packCount) { + } + + private MidxHeader readHeader(ByteArrayOutputStream out) { + byte[] midx = out.toByteArray(); + + int packCount = NB.decodeInt32(midx, 8); + return new MidxHeader(packCount); + } }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityCheckerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityCheckerTest.java new file mode 100644 index 0000000..371f876 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityCheckerTest.java
@@ -0,0 +1,1129 @@ +/* + * Copyright (C) 2026, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.transport.connectivity; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; +import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdSubclassMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.TreeFormatter; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.ConnectivityChecker.ConnectivityCheckInfo; +import org.eclipse.jgit.transport.PackParser; +import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import jdk.jfr.Recording; +import jdk.jfr.consumer.RecordingFile; + +/** Tests for {@link TreeWalkConnectivityChecker}. */ +public class TreeWalkConnectivityCheckerTest { + @Rule + public MockitoRule rule = MockitoJUnit.rule(); + + private TestRepository<InMemoryRepository> tr; + + private TreeWalkConnectivityChecker checker; + + private ConnectivityCheckInfo info; + + private Set<ObjectId> haves; + + private CountingProgressMonitor pm = new CountingProgressMonitor(); + + private static class CountingProgressMonitor implements ProgressMonitor { + private long objectsCheckedCount = 0; + + @Override + public void start(int totalTasks) { + // noop + } + + @Override + public void beginTask(String title, int totalWork) { + // noop + } + + @Override + public void update(int completed) { + objectsCheckedCount += completed; + } + + @Override + public void endTask() { + // noop + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public void showDuration(boolean show) { + // noop + } + + public long getObjectsCheckedCount() { + return objectsCheckedCount; + } + + public void reset() { + objectsCheckedCount = 0; + } + } + + @Mock + private PackParser parser; + + @Before + @SuppressWarnings("boxing") + public void setUp() throws Exception { + tr = new TestRepository<>( + new InMemoryRepository(new DfsRepositoryDescription("test"))); + checker = new TreeWalkConnectivityChecker(); + info = new ConnectivityCheckInfo(); + info.setRepository(tr.getRepository()); + info.setWalk(tr.getRevWalk()); + info.setParser(parser); + when(parser.needNewObjectIds()).thenReturn(true); + haves = new HashSet<>(); + pm.reset(); + } + + @Test + public void testFailureMissingParent() throws Exception { + ObjectId missingParentId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567"); + + RevCommit newCommit = createCommitWithParent(missingParentId); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit); + + // MOE thrown within RevWalk.next() when emitting newCommit, count isn't + // updated + runCheckAndExpectMissingObject(missingParentId, 0); + } + + @Test + public void testFailureMissingParentChain() throws Exception { + ObjectId missingParentId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567"); + + RevCommit commit1 = createCommitWithParent(missingParentId); + RevCommit commit2 = createCommitWithParent(commit1); + + setupSingleReceiveCommand(ObjectId.zeroId(), commit2.getId()); + mockNewPackObjects(commit1, commit2); + + runCheckAndExpectMissingObject(missingParentId, 1); + } + + @Test + public void testFailureMissingParentMultipleBranches() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + ObjectId missingParentId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567"); + + RevCommit newCommit1 = tr.commit().parent(base).create(); + RevCommit newCommit2 = createCommitWithParent(missingParentId); + + ReceiveCommand cmd1 = new ReceiveCommand(base, newCommit1, + "refs/heads/branch1"); + ReceiveCommand cmd2 = new ReceiveCommand(ObjectId.zeroId(), newCommit2, + "refs/heads/branch2"); + + info.setCommands(Arrays.asList(cmd1, cmd2)); + + mockNewPackObjects(newCommit1, newCommit2); + + runCheckAndExpectMissingObject(missingParentId, 1); + } + + @Test + public void testFailureMissingBlob() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + // Create a missing blob ID that is not in DB or pack + ObjectId missingBlobId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567"); + RevBlob missingBlob = tr.getRevWalk().lookupBlob(missingBlobId); + + RevCommit newCommit = tr.commit().parent(base).add("foo", missingBlob) + .create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndExpectMissingObject(missingBlobId, 4); + } + + @Test + public void testFailureMissingSubtree() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + ObjectId missingTreeId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234568"); + + RevBlob blob = tr.blob("content"); + + TreeFormatter formatter = new TreeFormatter(); + formatter.append("file", FileMode.REGULAR_FILE, blob); + formatter.append("dir", FileMode.TREE, missingTreeId); + + try (ObjectInserter inserter = tr.getRepository().newObjectInserter()) { + ObjectId rootTreeId = inserter.insert(formatter); + + CommitBuilder cb = new CommitBuilder(); + cb.setTreeId(rootTreeId); + cb.setAuthor(new PersonIdent("Author", "author@example.com")); + cb.setCommitter( + new PersonIdent("Committer", "committer@example.com")); + cb.setMessage("Commit with missing subtree"); + cb.setParentIds(base.getId()); + + ObjectId commitId = inserter.insert(cb); + inserter.flush(); + + RevCommit newCommit = tr.getRevWalk().parseCommit(commitId); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree(), blob); + + runCheckAndExpectMissingObject(missingTreeId, 4); + } + } + + @Test + public void testFailureUnreachableParentCommit() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevCommit secretCommit = tr.commit().create(); + RevCommit newCommit = tr.commit().parent(secretCommit).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndExpectMissingObject(secretCommit.getId(), 3); + } + + @Test + public void testMultipleBranchesUnreachableParent() throws Exception { + // base2 & new2 have newer timestamps + RevCommit base1 = tr.commit().create(); + RevCommit new1 = tr.commit().parent(base1).create(); + RevCommit base2 = tr.commit().create(); + RevCommit new2 = tr.commit().parent(base2).create(); + + tr.branch("refs/heads/base1").update(base1); + haves.add(base1); + + ReceiveCommand cmd1 = new ReceiveCommand(base1, new1, "refs/heads/b1"); + ReceiveCommand cmd2 = new ReceiveCommand(base2, new2, "refs/heads/b2"); + + info.setCommands(Arrays.asList(cmd1, cmd2)); + mockNewPackObjects(new1, new1.getTree(), new2, new2.getTree()); + + runCheckAndExpectMissingObject(base2, 5); + + // Verify order doesn't matter - reset state and swap order + setUp(); + + // base2 & new2 have newer timestamps + base1 = tr.commit().create(); + new1 = tr.commit().parent(base1).create(); + base2 = tr.commit().create(); + new2 = tr.commit().parent(base2).create(); + + tr.branch("refs/heads/base2").update(base2); + haves.add(base2); + + cmd1 = new ReceiveCommand(base1, new1, "refs/heads/b1"); + cmd2 = new ReceiveCommand(base2, new2, "refs/heads/b2"); + + info.setCommands(Arrays.asList(cmd2, cmd1)); + mockNewPackObjects(new1, new1.getTree(), new2, new2.getTree()); + + runCheckAndExpectMissingObject(base1, 5); + } + + @Test + public void testThinPackFailureUnreachableDeltaBase() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob deltafiledBlob = tr.blob("simulated delta blob"); + RevBlob missingBase = tr.getRevWalk().lookupBlob(ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567")); + RevCommit newCommit = tr.commit().parent(base) + .add("foo", deltafiledBlob).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit); + mockNewPackObjects(newCommit, newCommit.getTree(), deltafiledBlob); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(missingBase); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndExpectMissingObject(missingBase.getId(), 2); + } + + @Test + public void testSuccessMatchingParent() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + // Create a new commit in the pack that explicitly matches the base tree + ObjectId newCommitId = tr.unparsedCommit(1, base.getTree(), + base.getId()); + RevCommit newCommit = tr.getRevWalk().parseCommit(newCommitId); + + setupSingleReceiveCommand(base.getId(), newCommitId); + mockNewPackObjects(newCommit); + runCheckAndAssertCount(4); + } + + @Test + public void testSuccessWithDifferentBlob() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob blob = tr.blob("hello"); + RevCommit newCommit = tr.commit().parent(base).add("foo", blob) + .create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree(), blob); + + runCheckAndAssertCount(4); + } + + @Test + public void testSuccessNoReachabilityCheckWhenParentsInHaves() + throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevCommit currentHaves = base; + for (int i = 0; i < 250; i++) { + currentHaves = tr.commit().parent(currentHaves).create(); + haves.add(currentHaves.getId()); + } + + RevBlob blob = tr.blob("hello"); + RevCommit newCommit = tr.commit().parent(base).add("foo", blob) + .create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit); + + runCheckAndAssertCount(4); + } + + /** + * Test with a large flat tree to measure objects checked. + * + * <pre>{@code + * [Existing DB] [Received Pack] + * base <------------------- newCommit + * | | + * root_tree root_tree (differs) + * | | + * +- file0 +- file0 (differs) + * +- file1 +- file1 (differs) + * +- file2 +- file2 (differs) + * +- file3 +- file3 (differs) + * +- file4 +- file4 (differs) + * +- file5 +- file5 (shared) + * +- ... +- ... + * `- file99 `- file99 (shared) + * }</pre> + */ + @Test + public void testLargeFlatTree() throws Exception { + TestRepository.CommitBuilder cb = tr.commit(); + for (int i = 0; i < 100; i++) { + cb.add("file" + i, tr.blob("content" + i)); + } + RevCommit base = cb.create(); + haves.add(base.getId()); + + TestRepository.CommitBuilder cbNew = tr.commit().parent(base); + List<ObjectId> allNewObjects = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + RevBlob b = tr.blob("new content " + i); + cbNew.add("file" + i, b); + allNewObjects.add(b); + } + RevCommit newCommit = cbNew.create(); + allNewObjects.add(newCommit); + allNewObjects.add(newCommit.getTree()); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(allNewObjects); + + runCheckAndAssertCount(4); + } + + @Test + public void testGitlink() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + ObjectId submoduleCommitId = ObjectId + .fromString("0123456789abcdef0123456789abcdef01234567"); + + TreeFormatter formatter = new TreeFormatter(); + formatter.append("submodule", FileMode.GITLINK, submoduleCommitId); + + try (ObjectInserter inserter = tr.getRepository().newObjectInserter()) { + ObjectId rootTreeId = inserter.insert(formatter); + + CommitBuilder cb = new CommitBuilder(); + cb.setTreeId(rootTreeId); + cb.setAuthor(new PersonIdent("Author", "author@example.com")); + cb.setCommitter( + new PersonIdent("Committer", "committer@example.com")); + cb.setMessage("Commit with gitlink"); + cb.setParentIds(base.getId()); + + ObjectId commitId = inserter.insert(cb); + inserter.flush(); + + RevCommit newCommit = tr.getRevWalk().parseCommit(commitId); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + // Should not fail gitlink identifier not being in the local + // database + runCheckAndAssertCount(4); + } + } + + @Test + public void testZeroParents() throws Exception { + RevBlob blob = tr.blob("content"); + RevCommit rootCommit = tr.commit().add("file1", blob).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), rootCommit.getId()); + mockNewPackObjects(rootCommit, rootCommit.getTree(), blob); + + runCheckAndAssertCount(2); + } + + @Test + public void testMultipleParents() throws Exception { + RevCommit parent1 = tr.commit().add("file1", tr.blob("content1")) + .create(); + RevCommit parent2 = tr.commit().add("file2", tr.blob("content2")) + .create(); + haves.add(parent1.getId()); + haves.add(parent2.getId()); + + RevCommit mergeCommit = tr.commit().parent(parent1).parent(parent2) + .add("file3", tr.blob("content3")).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), mergeCommit.getId()); + mockNewPackObjects(mergeCommit, mergeCommit.getTree()); + + runCheckAndAssertCount(6); + } + + @Test + public void testChainedCommitsInPack() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevCommit c1 = tr.commit().parent(base).create(); + RevCommit c2 = tr.commit().parent(c1).create(); + + setupSingleReceiveCommand(base.getId(), c2.getId()); + mockNewPackObjects(c1, c2, c1.getTree(), c2.getTree()); + + runCheckAndAssertCount(7); + } + + @Test + public void testSuccessReachableParentCommit() throws Exception { + RevCommit base = tr.commit().create(); + RevCommit reachableCommit = tr.commit().parent(base).create(); + haves.add(reachableCommit.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(5); + } + + @Test + public void testSuccessFallbackToFullRefDatabase() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevCommit newRootCommit = tr.commit().create(); + tr.branch("refs/heads/unconnected").update(newRootCommit); + + RevCommit newCommit = tr.commit().parent(newRootCommit).create(); + + info.setCommands(Collections.singletonList(new ReceiveCommand( + ObjectId.zeroId(), newCommit.getId(), "refs/heads/master"))); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(5); + } + + @Test + public void testCheckReachabilityWithBlobInHaves() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob blob = tr.blob("blob content"); + haves.add(blob.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(4); + } + + @Test + public void testCheckReachabilityWithAnnotatedTagInHaves() + throws Exception { + RevCommit base = tr.commit().create(); + + RevTag tag = tr.tag("my-tag", base); + haves.add(tag.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(5); + } + + @Test + public void testCheckReachabilityWithBlobInRefs() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob blob = tr.blob("blob content"); + tr.update("refs/tags/my-blob", blob); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(4); + } + + @Test + public void testCheckReachabilityWithAnnotatedTagInRefs() throws Exception { + RevCommit base = tr.commit().create(); + + RevTag tag = tr.tag("my-tag", base); + tr.update("refs/tags/my-tag", tag); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(ObjectId.zeroId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(4); + } + + @Test + public void testSuccessWithDeletion() throws Exception { + RevBlob blob = tr.blob("hello"); + RevCommit base = tr.commit().add("foo", blob).create(); + haves.add(base.getId()); + + // Create a new commit that deletes "foo" + RevCommit newCommit = tr.commit().parent(base).rm("foo").create(); + + info.setCommands(Collections.singletonList(new ReceiveCommand( + base.getId(), newCommit.getId(), "refs/heads/master"))); + + mockNewPackObjects(newCommit, newCommit.getTree()); + + runCheckAndAssertCount(4); + } + + @Test + public void testDeleteCommandIgnored() throws Exception { + RevCommit base = tr.commit().create(); + tr.branch("refs/heads/master").update(base); + + ReceiveCommand cmd = new ReceiveCommand(base.getId(), ObjectId.zeroId(), + "refs/heads/master", ReceiveCommand.Type.DELETE); + + info.setCommands(Collections.singletonList(cmd)); + + runCheckAndAssertCount(0); + } + + @Test + public void testMultipleCommands() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob blob1 = tr.blob("content1"); + RevCommit newCommit1 = tr.commit().parent(base).add("foo1", blob1) + .create(); + + RevBlob blob2 = tr.blob("content2"); + RevCommit newCommit2 = tr.commit().parent(base).add("foo2", blob2) + .create(); + + ReceiveCommand cmd1 = new ReceiveCommand(base.getId(), + newCommit1.getId(), "refs/heads/branch1"); + ReceiveCommand cmd2 = new ReceiveCommand(base.getId(), + newCommit2.getId(), "refs/heads/branch2"); + + info.setCommands(Arrays.asList(cmd1, cmd2)); + + mockNewPackObjects(newCommit1, newCommit1.getTree(), blob1, newCommit2, + newCommit2.getTree(), blob2); + + runCheckAndAssertCount(7); + } + + @Test + public void testThinPackSuccess() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob baseBlob = tr.blob("base blob content"); + haves.add(baseBlob.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(baseBlob.getId()); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndAssertCount(4); + } + + @Test + public void testThinPackWithBlobInHaves() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob baseBlob = tr.blob("base blob content"); + + RevBlob unrelatedBlob = tr.blob("unrelated blob content"); + haves.add(unrelatedBlob.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(baseBlob.getId()); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndAssertCount(4); + } + + @Test + public void testThinPackWithSignedTagInHaves() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob baseBlob = tr.blob("base blob content"); + + RevTag tag = tr.tag("my-tag", base); + haves.add(tag.getId()); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(baseBlob.getId()); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndAssertCount(4); + } + + @Test + public void testThinPackWithBlobInRefs() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob baseBlob = tr.blob("base blob content"); + + RevBlob unrelatedBlob = tr.blob("unrelated blob content"); + tr.update("refs/tags/my-blob", unrelatedBlob); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(baseBlob.getId()); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndAssertCount(4); + } + + @Test + public void testThinPackWithAnnotatedTagInRefs() throws Exception { + RevCommit base = tr.commit().create(); + haves.add(base.getId()); + + RevBlob baseBlob = tr.blob("base blob content"); + + RevTag tag = tr.tag("my-tag", base); + tr.update("refs/tags/my-tag", tag); + + RevCommit newCommit = tr.commit().parent(base).create(); + + setupSingleReceiveCommand(base.getId(), newCommit.getId()); + mockNewPackObjects(newCommit, newCommit.getTree()); + + ObjectIdSubclassMap<ObjectId> baseObjectIds = new ObjectIdSubclassMap<>(); + baseObjectIds.add(baseBlob.getId()); + when(parser.getBaseObjectIds()).thenReturn(baseObjectIds); + info.setCheckObjects(true); + + runCheckAndAssertCount(4); + } + + @Test + public void testMultipleCommandsDifferentParents() throws Exception { + RevCommit base1 = tr.commit().create(); + RevCommit base2 = tr.commit().create(); + tr.branch("refs/heads/base1").update(base1); + tr.branch("refs/heads/base2").update(base2); + haves.add(base1.getId()); + haves.add(base2.getId()); + + RevBlob blob1 = tr.blob("content1"); + RevCommit newCommit1 = tr.commit().parent(base1).add("foo1", blob1) + .create(); + + RevBlob blob2 = tr.blob("content2"); + RevCommit newCommit2 = tr.commit().parent(base2).add("foo2", blob2) + .create(); + + ReceiveCommand cmd1 = new ReceiveCommand(base1.getId(), + newCommit1.getId(), "refs/heads/branch1"); + ReceiveCommand cmd2 = new ReceiveCommand(base2.getId(), + newCommit2.getId(), "refs/heads/branch2"); + + info.setCommands(Arrays.asList(cmd1, cmd2)); + + mockNewPackObjects(newCommit1, newCommit1.getTree(), blob1, newCommit2, + newCommit2.getTree(), blob2); + + runCheckAndAssertCount(8); + } + + /** + * Test case with a deep and wide tree structure. Modifying a file 5 levels + * deep should only check the trees along that path. + * + * <pre>{@code + * [Existing DB] [Received Pack] + * base <------------------- newCommit + * | | + * root_tree root_tree (differs) + * | | + * +- A +- A (differs) + * | +- C | +- C (differs) + * | | +- I | | +- I (differs) + * | | | +- J | | | +- J (differs) + * | | | | +- K | | | | +- K (differs) + * | | | | | +- L | | | | | +- L (differs) + * | | | | | | +- M | | | | | | +- M (differs) + * | | | | | | | `- 1 | | | | | | | `- 1 (differs) + * | +- D | +- D (shared) + * | `- 2 | `- 2 (shared) + * +- B +- B (shared) + * +- E | +- E (shared) + * | `- 3 | `- 3 (shared) + * +- F | +- F (shared) + * | `- 4 | `- 4 (shared) + * +- G | +- G (shared) + * | `- 5 | `- 5 (shared) + * +- H | +- H (shared) + * `- 6 | `- 6 (shared) + * }</pre> + */ + @Test + public void testDeepTreeSuccess() throws Exception { + // Create base with deep and wide structure + TestRepository.CommitBuilder cb = tr.commit(); + cb.add("A/C/I/J/K/L/M/1", tr.blob("content1")); + cb.add("A/D/2", tr.blob("content2")); + cb.add("B/E/3", tr.blob("content3")); + cb.add("B/F/4", tr.blob("content4")); + cb.add("B/G/5", tr.blob("content5")); + cb.add("B/H/6", tr.blob("content6")); + RevCommit base = cb.create(); + haves.add(base.getId()); + + RevCommit currentHaves = base; + for (int i = 0; i < 250; i++) { + currentHaves = tr.commit().parent(currentHaves).create(); + haves.add(currentHaves.getId()); + } + + // Create new commit modifying file at end of deep path + RevBlob newBlob = tr.blob("new content"); + RevCommit newCommit = tr.commit().parent(base) + .add("A/C/I/J/K/L/M/000", newBlob).create(); + + info.setCommands(Collections.singletonList(new ReceiveCommand( + base.getId(), newCommit.getId(), "refs/heads/master"))); + + List<ObjectId> allNewObjects = new ArrayList<>(); + allNewObjects.add(newCommit); + allNewObjects.add(newCommit.getTree()); + allNewObjects.add(newBlob); + try (TreeWalk tw = new TreeWalk(tr.getRepository())) { + tw.setRecursive(false); + tw.addTree(newCommit.getTree()); + String[] pathSegments = { "A", "C", "I", "J", "K", "L", "M" }; + for (String segment : pathSegments) { + while (tw.next()) { + if (tw.getNameString().equals(segment)) { + allNewObjects.add(tw.getObjectId(0)); + tw.enterSubtree(); + break; + } + } + } + } + mockNewPackObjects(allNewObjects); + + runCheckAndAssertCount(18); + } + + /** + * Compare the efficiency of the TreeWalk checker vs. iterative vs. full + * checkers. Initial JFR results on this test: + * <ul> + * <li>TreeWalk Checker: 22 checked objects, 25 JFR allocations</li> + * <li>Iterative Checker: 14 checked objects, 41 JFR allocations</li> + * <li>Full Checker: 264 checked objects, 147 JFR allocations</li> + * </ul> + * + * <pre>{@code + * [Existing DB] [Received Pack] + * base <------------------- newCommit + * | | + * root_tree root_tree (differs) + * | | + * +- A +- A (differs) + * | +- C | +- C (differs) + * | | +- I | | +- I (differs) + * | | | +- J | | | +- J (differs) + * | | | | +- K | | | | +- K (differs) + * | | | | | +- L | | | | | +- L (differs) + * | | | | | | +- M | | | | | | +- M (differs) + * | | | | | | | `- 1 | | | | | | | `- 1 (differs) + * | +- D | +- D (shared) + * | `- 2 | `- 2 (shared) + * +- B +- B (differs) + * +- E | +- E (shared) + * | `- 3 | `- 3 (shared) + * +- F | +- F (shared) + * | `- 4 | `- 4 (shared) + * +- G | +- G (shared) + * | `- 5 | `- 5 (shared) + * +- H | +- H (differs) + * `- 6 | `- 6 (differs) + * }</pre> + */ + @Test + public void testDeepTreePerformance() throws Exception { + // Create base with deep and wide structure + TestRepository.CommitBuilder cb = tr.commit(); + cb.add("A/C/I/J/K/L/M/1", tr.blob("content1")); + cb.add("A/D/2", tr.blob("content2")); + cb.add("B/E/3", tr.blob("content3")); + cb.add("B/F/4", tr.blob("content4")); + cb.add("B/G/5", tr.blob("content5")); + cb.add("B/H/6", tr.blob("content6")); + + RevCommit base = cb.create(); + haves.add(base.getId()); + RevCommit currentHaves = base; + for (int i = 0; i < 250; i++) { + currentHaves = tr.commit().parent(currentHaves).create(); + haves.add(currentHaves.getId()); + } + + // Create new commit modifying file at end of deep path + RevBlob newBlob = tr.blob("new content"); + RevCommit newCommit = tr.commit().parent(base) + .add("A/C/I/J/K/L/M/1", newBlob).add("B/H/6", newBlob).create(); + + info.setCommands(Collections.singletonList(new ReceiveCommand( + base.getId(), newCommit.getId(), "refs/heads/new_change"))); + + ObjectIdSubclassMap<ObjectId> newObjectIds = new ObjectIdSubclassMap<>(); + newObjectIds.add(newCommit); + newObjectIds.add(newCommit.getTree()); + newObjectIds.add(newBlob); + try (TreeWalk tw = new TreeWalk(tr.getRepository())) { + tw.setRecursive(false); + tw.addTree(newCommit.getTree()); + String[] pathSegments = { "A", "C", "I", "J", "K", "L", "M" }; + for (String segment : pathSegments) { + while (tw.next()) { + if (tw.getNameString().equals(segment)) { + newObjectIds.add(tw.getObjectId(0)); + tw.enterSubtree(); + break; + } + } + } + } + try (TreeWalk tw = new TreeWalk(tr.getRepository())) { + tw.setRecursive(false); + tw.addTree(newCommit.getTree()); + while (tw.next()) { + if (tw.getNameString().equals("B")) { + newObjectIds.add(tw.getObjectId(0)); + tw.enterSubtree(); + while (tw.next()) { + if (tw.getNameString().equals("H")) { + newObjectIds.add(tw.getObjectId(0)); + break; + } + } + break; + } + } + } + when(parser.getNewObjectIds()).thenReturn(newObjectIds); + + TreeWalkConnectivityChecker scChecker = new TreeWalkConnectivityChecker(); + + long scJfrCount = 0; + try (Recording r = new Recording()) { + r.enable("jdk.ObjectAllocationInNewTLAB"); + r.enable("jdk.ObjectAllocationOutsideTLAB"); + r.start(); + + for (int i = 0; i < 1000; i++) { + try (RevWalk rw = new RevWalk(info.getRepository())) { + info.setWalk(rw); + pm.reset(); + scChecker.checkConnectivity(info, haves, pm); + } + } + + r.stop(); + Path p = Files.createTempFile("sc_jfr", ".jfr"); + r.dump(p); + try (RecordingFile file = new RecordingFile(p)) { + while (file.hasMoreEvents()) { + file.readEvent(); + scJfrCount++; + } + } + Files.delete(p); + } + long scCount = pm.getObjectsCheckedCount(); + + // Run IterativeConnectivityChecker + IterativeConnectivityChecker icChecker = new IterativeConnectivityChecker( + new FullConnectivityChecker()); + + long icJfrCount = 0; + try (jdk.jfr.Recording r = new jdk.jfr.Recording()) { + r.enable("jdk.ObjectAllocationInNewTLAB"); + r.enable("jdk.ObjectAllocationOutsideTLAB"); + r.start(); + + for (int i = 0; i < 1000; i++) { + try (RevWalk rw = new RevWalk(info.getRepository())) { + info.setWalk(rw); + pm.reset(); + icChecker.checkConnectivity(info, haves, pm); + } + } + + r.stop(); + Path p = Files.createTempFile("ic_jfr_deep", ".jfr"); + r.dump(p); + try (jdk.jfr.consumer.RecordingFile file = new jdk.jfr.consumer.RecordingFile( + p)) { + while (file.hasMoreEvents()) { + jdk.jfr.consumer.RecordedEvent event = file.readEvent(); + if (event.getEventType().getName() + .startsWith("jdk.ObjectAllocation")) { + icJfrCount++; + } + } + } + Files.delete(p); + } + long icCount = pm.getObjectsCheckedCount(); + + // Run FullConnectivityChecker + FullConnectivityChecker fcChecker = new FullConnectivityChecker(); + + long fcJfrCount = 0; + try (Recording r = new Recording()) { + r.enable("jdk.ObjectAllocationInNewTLAB"); + r.enable("jdk.ObjectAllocationOutsideTLAB"); + r.start(); + + for (int i = 0; i < 1000; i++) { + try (RevWalk rw = new RevWalk(info.getRepository())) { + info.setWalk(rw); + pm.reset(); + fcChecker.checkConnectivity(info, haves, pm); + } + } + + r.stop(); + Path p = Files.createTempFile("fc_jfr", ".jfr"); + r.dump(p); + try (RecordingFile file = new RecordingFile(p)) { + while (file.hasMoreEvents()) { + file.readEvent(); + fcJfrCount++; + } + } + Files.delete(p); + } + long fcCount = pm.getObjectsCheckedCount(); + + System.out.println("TreeWalk Checked Objects: " + scCount); + System.out.println("TreeWalk JFR allocations: " + scJfrCount); + System.out.println("Iterative Checked Objects: " + icCount); + System.out.println("Iterative JFR allocations: " + icJfrCount); + System.out.println("Full Checked Objects: " + fcCount); + System.out.println("Full JFR allocations: " + fcJfrCount); + + assertEquals(22, scCount); + assertEquals(14, icCount); + assertEquals(264, fcCount); + + assertTrue( + "TreeWalkConnectivityChecker should produce fewer JFR allocation events. Expected " + + scJfrCount + " < " + fcJfrCount, + scJfrCount < fcJfrCount); + } + + private void setupSingleReceiveCommand(ObjectId oldId, ObjectId newId) { + info.setCommands(Collections.singletonList( + new ReceiveCommand(oldId, newId, "refs/heads/master"))); + } + + private void mockNewPackObjects(ObjectId... ids) { + mockNewPackObjects(Arrays.asList(ids)); + } + + private void mockNewPackObjects(Collection<? extends ObjectId> ids) { + ObjectIdSubclassMap<ObjectId> map = new ObjectIdSubclassMap<>(); + for (ObjectId id : ids) { + map.add(id); + } + when(parser.getNewObjectIds()).thenReturn(map); + } + + private void runCheckAndAssertCount(int expectedCount) throws IOException { + checker.checkConnectivity(info, haves, pm); + assertEquals(expectedCount, pm.getObjectsCheckedCount()); + } + + private void runCheckAndExpectMissingObject(ObjectId expectedMissingId, + int expectedCount) throws IOException { + try { + checker.checkConnectivity(info, haves, pm); + fail("Expected MissingObjectException"); + } catch (MissingObjectException e) { + assertEquals(expectedMissingId, e.getObjectId()); + assertEquals(expectedCount, pm.getObjectsCheckedCount()); + } + } + + private RevCommit createCommitWithParent(ObjectId parentId) + throws Exception { + CommitBuilder cb = new CommitBuilder(); + cb.setTreeId(tr.tree().getId()); + cb.setParentId(parentId); + cb.setAuthor(new PersonIdent("Author", "author@example.com", + tr.getInstant(), tr.getTimeZoneId())); + cb.setCommitter(new PersonIdent("Committer", "committer@example.com", + tr.getInstant(), tr.getTimeZoneId())); + cb.setMessage("Commit with specific parent"); + + try (ObjectInserter ins = tr.getRepository().newObjectInserter()) { + ObjectId id = ins.insert(cb); + ins.flush(); + return tr.getRevWalk().parseCommit(id); + } + } +}
diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 5dedccd..dca75e2 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -117,6 +117,7 @@ cannotMoveIndexTo=Cannot move index to {0} cannotMovePackTo=Cannot move pack to {0} cannotOpenService=cannot open {0} +cannotOpenMidx=Cannot open midx at {0}: {1} cannotParseDate=The date specification "{0}" could not be parsed with the following formats: {1} cannotParseGitURIish=Cannot parse Git URI-ish cannotPullOnARepoWithState=Cannot pull into a repository with state: {0}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index db4a4e2..4251f4c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -147,6 +147,7 @@ public static JGitText get() { /***/ public String cannotMoveIndexTo; /***/ public String cannotMovePackTo; /***/ public String cannotOpenService; + /***/ public String cannotOpenMidx; /***/ public String cannotParseDate; /***/ public String cannotParseGitURIish; /***/ public String cannotPullOnARepoWithState;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index fc6596e..3f457f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
@@ -677,7 +677,7 @@ private PackList scanPacksImpl(PackList old) throws IOException { * pack description * @return the dfs packfile */ - protected DfsPackFile createDfsPackFile(DfsBlockCache cache, + public DfsPackFile createDfsPackFile(DfsBlockCache cache, DfsPackDescription dsc) { return new DfsPackFile(cache, dsc); }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 8860cce..fbe641e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
@@ -26,6 +26,7 @@ import java.util.List; import java.util.Set; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackIndex; @@ -79,6 +80,62 @@ public class DfsPackCompactor { private RevFlag isBase; /** + * Hook invoked after the compact calculation is done, but before committing + * the resulting packs. + * <p> + * The hook implementations must not modify the incoming collections. + * Instead, they should return any extra packs to commit or remove in a + * {@link Packs} record, which the caller will combine with the calculated + * results. + */ + public interface PreCommitHook { + /** + * Extra packs to commit and remove as result of the hook execution. + * + * @param toCommit + * packs that this hook wants to add to the compactor + * transaction + * @param toRemove + * packs that this hook wants to remove in the compactor + * transaction + */ + record Packs(List<DfsPackDescription> toCommit, + List<DfsPackDescription> toRemove) { + } + + /** + * Apply the hook. + * + * @param newPacks + * new packs created by this compaction. Read only. + * @param removedPacks + * packs that are being replaced by this compaction. Read + * only. + * @return Packs containing extra packs to commit and remove, or null if + * none. + * @throws IOException + * an error occurred in the hook. + */ + @Nullable + Packs apply(List<DfsPackDescription> newPacks, + Set<DfsPackDescription> removedPacks) throws IOException; + } + + private PreCommitHook preCommitHook; + + /** + * Set the pre-commit hook. + * + * @param hook + * pre-commit hook. + * @return {@code this} + */ + public DfsPackCompactor setPreCommitHook(PreCommitHook hook) { + this.preCommitHook = hook; + return this; + } + + /** * Initialize a pack compactor. * * @param repository @@ -196,8 +253,15 @@ public void compact(ProgressMonitor pm) throws IOException { } compactPacks(ctx, pm); - List<DfsPackDescription> commit = getNewPacks(); - Collection<DfsPackDescription> remove = toPrune(); + List<DfsPackDescription> commit = new ArrayList<>(getNewPacks()); + Set<DfsPackDescription> remove = toPrune(); + if (preCommitHook != null) { + PreCommitHook.Packs extra = preCommitHook.apply(commit, remove); + if (extra != null) { + commit.addAll(extra.toCommit()); + remove.addAll(extra.toRemove()); + } + } if (!commit.isEmpty() || !remove.isEmpty()) { objdb.commitPack(commit, remove); } @@ -318,7 +382,7 @@ public List<PackStatistics> getNewPackStatistics() { : Collections.emptyList(); } - private Collection<DfsPackDescription> toPrune() { + private Set<DfsPackDescription> toPrune() { Set<DfsPackDescription> packs = new HashSet<>(); for (DfsPackFile pack : srcPacks) { packs.add(pack.getPackDescription());
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/MidxWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/MidxWriter.java index 543ceeb..8143bbb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/MidxWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/MidxWriter.java
@@ -9,11 +9,15 @@ */ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -21,6 +25,7 @@ import java.util.Set; import org.eclipse.jgit.internal.revwalk.RefAdvancerWalk; +import org.eclipse.jgit.internal.storage.midx.MidxMetadataReader; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndexWriter; import org.eclipse.jgit.internal.storage.midx.PackIndexMerger; @@ -37,6 +42,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.Base64; +import org.eclipse.jgit.util.FileUtils; /** * Helper to write multipack indexes. @@ -44,6 +50,11 @@ public class MidxWriter { /** + * Do not build a midx if there are less than this amount of packs to cover. + */ + private static final int MIN_PACKS_FOR_MIDX = 2; + + /** * Write a mdix over the packs * * @param pm @@ -62,34 +73,76 @@ public class MidxWriter { public static void writeMidx(ProgressMonitor pm, Repository repo, Collection<Pack> packs, File midxOut, PackConfig packConfig) throws IOException { + + Collection<Pack> packList = flattenMidxPackList(packs).stream() + .sorted(Comparator.comparing(Pack::getPackName)).toList(); + if (packList.size() < MIN_PACKS_FOR_MIDX) { + return; + } PackIndexMerger.Builder builder = PackIndexMerger.builder(); builder.setProgressMonitor(pm); - - Collection<Pack> packList = packs.stream() - .sorted(Comparator.comparing(Pack::getPackName)).toList(); - pm.beginTask("Adding packs to midx", packList.size()); + pm.beginTask("Adding packs to midx", packList.size()); //$NON-NLS-1$ for (Pack pack : packList) { + if (pack instanceof PackMidx) { + throw new IllegalArgumentException( + "Building midx from other midx not supported yet"); + } PackFile packFile = pack.getPackFile().create(PackExt.INDEX); builder.addPack(packFile.getName(), pack.getIndex()); + pm.update(1); } PackIndexMerger data = builder.build(); pm.endTask(); + File oldMidxBitmaps = null; + if (midxOut.exists()) { + MidxMetadataReader.MidxMetadata midxMetadata = MidxMetadataReader + .read(midxOut); + byte[] checksum = midxMetadata.checksum(); + String midxBitmapsPath = midxOut.getAbsoluteFile() + "-" //$NON-NLS-1$ + + ObjectId.fromRaw(checksum).name() + "." //$NON-NLS-1$ + + BITMAP_INDEX.getExtension(); + File previousBitmaps = new File(midxBitmapsPath); + if (previousBitmaps.exists()) { + oldMidxBitmaps = previousBitmaps; + } + } + + String midxFilename = midxOut.getAbsolutePath(); + File midxOutTmp = new File( + midxFilename + BITMAP_INDEX.getTmpExtension()); MultiPackIndexWriter writer = new MultiPackIndexWriter(); MultiPackIndexWriter.Result result; try (FileOutputStream out = new FileOutputStream( - midxOut.getAbsolutePath())) { + midxOutTmp.getAbsolutePath())) { result = writer.write(pm, out, data); } + File midxOutBitmaps = new File(midxOut.getAbsoluteFile() + "-" //$NON-NLS-1$ + + ObjectId.fromRaw(Base64.decode(result.checksum())).name() + + "." + BITMAP_INDEX.getExtension()); + File midxOutBitmapsTmp = null; if (packConfig != null) { - File midxOutBitmaps = new File( - midxOut.getAbsolutePath() + ".bitmaps"); - createAndAttachBitmaps(pm, repo, midxOutBitmaps, + midxOutBitmapsTmp = new File(midxOut.getAbsolutePath() + "-" //$NON-NLS-1$ + + ObjectId.fromRaw(Base64.decode(result.checksum())).name() + + BITMAP_INDEX.getTmpExtension()); + createAndAttachBitmaps(pm, repo, midxOutBitmapsTmp, Base64.decode(result.checksum()), data, packList, new PackConfig(repo)); } + + FileUtils.rename(midxOutTmp.getAbsoluteFile(), + midxOut.getAbsoluteFile(), StandardCopyOption.ATOMIC_MOVE); + if (midxOutBitmapsTmp != null) { + FileUtils.rename(midxOutBitmapsTmp.getAbsoluteFile(), + midxOutBitmaps.getAbsoluteFile(), + StandardCopyOption.ATOMIC_MOVE); + } + + if (oldMidxBitmaps != null && !oldMidxBitmaps.equals(midxOutBitmaps)) { + FileUtils.delete(oldMidxBitmaps); + } } private static void createAndAttachBitmaps(ProgressMonitor pm, @@ -158,4 +211,18 @@ private static List<ObjectToPack> asObjectsToPack(ProgressMonitor pm, pm.endTask(); return result; } + + static List<Pack> flattenMidxPackList(Collection<Pack> packs) { + List<Pack> output = new ArrayList<>(); + for (Pack p : packs) { + List<Pack> coveredPacks = new ArrayList<>(p.getCoveredPacks()); + if (coveredPacks.isEmpty()) { + output.add(p); + } else { + Collections.reverse(coveredPacks); + output.addAll(coveredPacks); + } + } + return Collections.unmodifiableList(output); + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index f38e02b..4e299e6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -171,6 +171,28 @@ public Pack(Config cfg, File packFile, @Nullable PackFile bitmapIdxFile) { length = Long.MAX_VALUE; } + /** + * Set the checksum of this pack + * + * @param checksum + * the checksum + */ + protected void setPackChecksum(byte[] checksum) { + this.packChecksum = checksum; + } + + /** + * Packs covered by this pack + * <p> + * For regular packs, this list is empty. For midx packs, this list has the + * names of packs indexed by the midx. + * + * @return list of packs covered by this pack. + */ + protected List<Pack> getCoveredPacks() { + return Collections.emptyList(); + } + private PackIndex idx() throws IOException { Optional<PackIndex> optional = loadedIdx.getOptional(); if (optional.isPresent()) { @@ -200,10 +222,9 @@ private synchronized PackIndex memoizeIdxIfNeeded() throws IOException { Long.valueOf(System.currentTimeMillis() - start))); } - if (packChecksum == null) { + if (packChecksum == null) { packChecksum = idx.getChecksum(); - fileSnapshot.setChecksum( - ObjectId.fromRaw(packChecksum)); + fileSnapshot.setChecksum(ObjectId.fromRaw(packChecksum)); } else if (!Arrays.equals(packChecksum, idx.getChecksum())) { throw new PackMismatchException(MessageFormat @@ -418,6 +439,7 @@ void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit) * packs to close */ public static void close(Set<Pack> packs) { + // TODO(ifrade): purge also nested packs in midx WindowCache.purge(packs); packs.forEach(p -> p.closeIndices()); } @@ -430,7 +452,12 @@ public void close() { closeIndices(); } - private synchronized void closeIndices() { + /** + * Clear the indexes referenced in this pack. + * <p> + * Subclasses override this method to clear references to any index they add + */ + protected synchronized void closeIndices() { loadedIdx = Optionally.empty(); reverseIdx = Optionally.empty(); bitmapIdx = Optionally.empty(); @@ -1318,8 +1345,8 @@ private synchronized PackBitmapIndex memoizeBitmapIndexIfNeeded() throws IOExcep return optional.get(); } try { - PackBitmapIndex idx = PackBitmapIndex.open(bitmapIdxFile, idx(), - getReverseIdx()); + PackBitmapIndex idx = PackBitmapIndex.open(bitmapIdxFile, + getIndex(), getReverseIdx()); // At this point, idx() will have set packChecksum. if (Arrays.equals(packChecksum, idx.getPackChecksum())) { bitmapIdx = optionally(idx); @@ -1364,7 +1391,14 @@ protected List<Pack> fullyIncludedIn(BitmapIndex.BitmapBuilder needBitmaps) return Collections.emptyList(); } - private PackReverseIndex getReverseIdx() throws IOException { + /** + * Get the reverse index of this pack + * + * @return a reverse index + * @throws IOException + * an error loading or calculating the reverse index. + */ + protected PackReverseIndex getReverseIdx() throws IOException { Optional<PackReverseIndex> optional = reverseIdx.getOptional(); if (optional.isPresent()) { return optional.get(); @@ -1421,7 +1455,16 @@ public String toString() { + ObjectId.fromRaw(packChecksum).name() + "]"; } - private <T> Optionally<T> optionally(T element) { + /** + * Wrap the reference in an Optionally + * + * @param element + * reference + * @return Optionally with the reference inside + * @param <T> + * the type + */ + protected <T> Optionally<T> optionally(T element) { return useStrongRefs ? new Optionally.Hard<>(element) : new Optionally.Soft<>(element); } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 872bb9e..2e06f3c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java
@@ -13,6 +13,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.lib.Constants.MIDX_FILE; import java.io.File; import java.io.FileNotFoundException; @@ -20,10 +21,12 @@ import java.io.InputStream; import java.nio.file.Files; import java.text.MessageFormat; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Deque; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -32,6 +35,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; @@ -45,6 +49,7 @@ import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.CoreConfig; import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; @@ -79,6 +84,8 @@ class PackDirectory { private final TrustStat trustPackStat; + private final boolean useMidx; + /** * Initialize a reference to an on-disk 'pack' directory. * @@ -92,6 +99,8 @@ class PackDirectory { this.directory = directory; packList = new AtomicReference<>(NO_PACKS); trustPackStat = config.get(CoreConfig.KEY).getTrustPackStat(); + useMidx = config.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_MULTIPACKINDEX, true); } /** @@ -122,8 +131,7 @@ Collection<Pack> getPacks() { list = scanPacks(list); } } while (searchPacksAgain(list)); - Pack[] packs = list.packs; - return Collections.unmodifiableCollection(Arrays.asList(packs)); + return Collections.unmodifiableCollection(Arrays.asList(list.packs)); } @Override @@ -136,7 +144,8 @@ public String toString() { * * @param objectId * identity of the object to test for existence of. - * @return {@code true} if the specified object is stored in this PackDirectory. + * @return {@code true} if the specified object is stored in this + * PackDirectory. */ boolean has(AnyObjectId objectId) { return getPack(objectId) != null; @@ -379,7 +388,8 @@ boolean searchPacksAgain(PackList old) { case AFTER_OPEN: try (InputStream stream = Files .newInputStream(directory.toPath())) { - // open the pack directory to refresh attributes (on some NFS clients) + // open the pack directory to refresh attributes (on some NFS + // clients) } catch (IOException e) { // ignore } @@ -470,6 +480,7 @@ private PackList scanPacks(PackList original) { private PackList scanPacksImpl(PackList old) { final Map<String, Pack> forReuse = reuseMap(old); final FileSnapshot snapshot = FileSnapshot.save(directory); + Map<String, Map<PackExt, PackFile>> packFilesByExtById = getPackFilesByExtById(); List<Pack> list = new ArrayList<>(packFilesByExtById.size()); boolean foundNew = false; @@ -496,10 +507,32 @@ private PackList scanPacksImpl(PackList old) { continue; } - list.add(new Pack(config, packFile, packFilesByExt.get(BITMAP_INDEX))); + list.add(new Pack(config, packFile, + packFilesByExt.get(BITMAP_INDEX))); foundNew = true; } + PackMidx theMidx = null; + File midx = new File(directory, MIDX_FILE); + if (useMidx && midx.exists()) { + Pack oldMidx = forReuse.get(midx.getName()); + if (oldMidx != null + && !oldMidx.getFileSnapshot().isModified(midx)) { + // Reuse the previous instance + forReuse.remove(midx.getName()); + theMidx = (PackMidx) oldMidx; + } else { + try { + theMidx = new PackMidx(config, midx, list); + foundNew = true; + } catch (IOException e) { + // pass + LOG.warn(MessageFormat.format(JGitText.get().cannotOpenMidx, + midx.getAbsolutePath(), e.getMessage())); + } + } + } + // If we did not discover any new files, the modification time was not // changed, and we did not remove any files, then the set of files is // the same as the set we were given. Instead of building a new object @@ -516,6 +549,22 @@ private PackList scanPacksImpl(PackList old) { return new PackList(snapshot, NO_PACKS.packs); } + if (useMidx && theMidx != null) { + // Replace the covered packs with the midx in the list + Set<String> coveredPackNames = theMidx.getCoveredPacks().stream() + .map(p -> p.getPackName()) + .collect(Collectors.toUnmodifiableSet()); + int packsBefore = list.size(); + list = list.stream() + .filter(p -> !coveredPackNames.contains(p.getPackName())) + .collect(Collectors.toCollection(ArrayList::new)); + int packsAfter = list.size(); + LOG.debug(String.format( + "Mangling packlist: midx replaces %d packs (list went from %d to %d packs)", //$NON-NLS-1$ + coveredPackNames.size(), packsBefore, packsAfter)); + list.add(theMidx); + } + final Pack[] r = list.toArray(new Pack[0]); Arrays.sort(r, Pack.SORT); return new PackList(snapshot, r); @@ -523,7 +572,9 @@ private PackList scanPacksImpl(PackList old) { private static Map<String, Pack> reuseMap(PackList old) { final Map<String, Pack> forReuse = new HashMap<>(); - for (Pack p : old.packs) { + Deque<Pack> queue = new ArrayDeque<>(List.of(old.packs)); + while (!queue.isEmpty()) { + Pack p = queue.removeFirst(); if (p.invalid()) { // The pack instance is corrupted, and cannot be safely used // again. Do not include it in our reuse map. @@ -532,6 +583,8 @@ private static Map<String, Pack> reuseMap(PackList old) { continue; } + queue.addAll(p.getCoveredPacks()); + final Pack prior = forReuse.put(p.getPackFile().getName(), p); if (prior != null) { // This should never occur. It should be impossible for us
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackMidx.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackMidx.java new file mode 100644 index 0000000..672d0bc --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackMidx.java
@@ -0,0 +1,544 @@ +/* + * Copyright (C) 2026, Google LLC + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static java.util.stream.Collectors.toMap; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.DataFormatException; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.PackInvalidException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.internal.storage.midx.MidxMetadataReader; +import org.eclipse.jgit.internal.storage.midx.MultiPackIndex; +import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackOutputStream; +import org.eclipse.jgit.internal.util.Optionally; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.BitmapIndex; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; + +/** + * A pack that uses internally a midx + * + * This pack answers what it can directly from the midx (e.g. hasObject) and + * sends to the right pack other calls (e.g. reading the actual object). + */ +public class PackMidx extends Pack { + private final List<Pack> packsInIdOrder; + + private final OffsetCalculator offsetCalculator; + + private Optionally<MultiPackIndex> midx = Optionally.empty(); + + private volatile PackIndex idx; + + private volatile PackReverseIndex ridx; + + // TODO(ifrade): Encapsulate invalid/invalidatingCause in Pack to reuse here + private IOException invalidatingCause; + + /** + * Construct a reader for existing, pre-indexed packfiles. + * + * @param cfg + * configuration this directory consults for write settings. + * @param midxFile + * path of the <code>.midx</code> file holding the data. + * @param knownPacks + * packs in the repo to find the packs covered by this midx + * @throws IOException + * an error reading any of the files involved + */ + public PackMidx(Config cfg, File midxFile, List<Pack> knownPacks) + throws IOException { + super(cfg, midxFile, null); + MidxMetadataReader.MidxMetadata midxMeta = MidxMetadataReader + .read(midxFile); + // Maybe we could load only the packnames chunk at this point + setPackChecksum(midxMeta.checksum()); + String[] packNames = midxMeta.packNames().toArray(new String[0]); + + Map<String, Pack> knownPacksByName = knownPacks.stream() + .collect(toMap( + p -> p.getPackFile().create(PackExt.INDEX).getName(), + Function.identity())); + packsInIdOrder = Arrays.stream(packNames).map(knownPacksByName::get) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableList()); + if (packsInIdOrder.size() != packNames.length) { + throw new IOException("Midx refers to packs not in the pack list"); //$NON-NLS-1$ + } + offsetCalculator = new OffsetCalculator(packsInIdOrder.stream() + .mapToLong(p -> p.getPackFile().length()).toArray()); + + ObjectId checksumHex = ObjectId.fromRaw(midxMeta.checksum()); + File midxBitmaps = new File(midxFile.getParentFile(), String.format( + "%s-%s.bitmap", Constants.MIDX_FILE, checksumHex.name())); + if (midxBitmaps.exists()) { + setBitmapIndexFile(new PackFile(midxBitmaps)); + } + } + + /** + * Return the packs covered by this midx (in midx order) + * + * @return packs covered by this midx + */ + @Override + public List<Pack> getCoveredPacks() { + return packsInIdOrder; + } + + @Override + public PackFile getPackFile() { + return super.getPackFile(); + } + + @Override + protected List<Pack> fullyIncludedIn(BitmapIndex.BitmapBuilder needBitmaps) + throws IOException { + PackBitmapIndex bitmapIndex = getBitmapIndex(); + if (needBitmaps.removeAllOrNone(bitmapIndex)) { + return getCoveredPacks(); + } + return Collections.emptyList(); + } + + @Override + public PackIndex getIndex() { + if (idx == null) { + synchronized (this) { + if (idx == null) { + try { + idx = new MidxPackIndex(getMidx(), offsetCalculator); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + } + return idx; + } + + @Override + public PackReverseIndex getReverseIdx() { + if (ridx == null) { + synchronized (this) { + if (ridx == null) { + try { + ridx = new MidxPackReverseIndex(getMidx(), + offsetCalculator); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + } + return ridx; + } + + @Override + public boolean hasObjectSizeIndex() throws IOException { + return false; + } + + @Override + public long getObjectSizeIndexCount() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long getIndexedObjectSize(AnyObjectId id) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasObject(AnyObjectId id) throws IOException { + return getMidx().hasObject(id); + } + + @Override + public boolean shouldBeKept() { + return false; + } + + @Override + ObjectLoader get(WindowCursor curs, AnyObjectId id) throws IOException { + MultiPackIndex.PackOffset packOffset = getMidx().find(id); + if (packOffset == null) { + return null; + } + return packsInIdOrder.get(packOffset.getPackId()).get(curs, id); + } + + @Override + void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit) + throws IOException { + getMidx().resolve(matches, id, matchLimit); + } + + @Override + protected synchronized void closeIndices() { + for (Pack p : getCoveredPacks()) { + p.closeIndices(); + } + midx = Optionally.empty(); + super.closeIndices(); + } + + @Override + public Iterator<PackIndex.MutableEntry> iterator() { + return getIndex().iterator(); + } + + @Override + long getObjectCount() throws IOException { + return getMidx().getObjectCount(); + } + + @Override + ObjectId findObjectForOffset(long offset) throws IOException { + MultiPackIndex.PackOffset decode = offsetCalculator.decode(offset); + if (decode == null) { + return null; + } + MultiPackIndex theMidx = getMidx(); + int bitmapPosition = theMidx.findBitmapPosition(decode); + if (bitmapPosition < 0) { + throw new IllegalStateException( + "Object in midx without ridx position"); + } + + return theMidx.getObjectAtBitmapPosition(bitmapPosition); + } + + @Override + ObjectId getPackChecksum() { + try { + return ObjectId.fromRaw(getMidx().getChecksum()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + void copyPackAsIs(PackOutputStream out, WindowCursor curs) + throws IOException { + for (Pack p : packsInIdOrder) { + p.copyPackAsIs(out, curs); + } + } + + @Override + boolean beginWindowCache() throws IOException { + boolean startedWindow = false; + for (Pack p : packsInIdOrder) { + startedWindow |= p.beginWindowCache(); + } + return startedWindow; + } + + @Override + boolean endWindowCache() { + boolean lastWindow = false; + for (Pack p : packsInIdOrder) { + lastWindow |= p.endWindowCache(); + } + return lastWindow; + } + + @Override + ByteArrayWindow read(long pos, int size) throws IOException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).read(po.getOffset(), size); + } + + @Override + ByteWindow mmap(long pos, int size) throws IOException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).mmap(po.getOffset(), size); + } + + @Override + ObjectLoader load(WindowCursor curs, long pos) + throws IOException, LargeObjectException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).load(curs, po.getOffset()); + } + + @Override + byte[] getDeltaHeader(WindowCursor wc, long pos) + throws IOException, DataFormatException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).getDeltaHeader(wc, + po.getOffset()); + } + + @Override + int getObjectType(WindowCursor curs, long pos) throws IOException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).getObjectType(curs, + po.getOffset()); + } + + @Override + long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { + MultiPackIndex.PackOffset po = getMidx().find(id); + if (po == null) { + return -1; + } + return packsInIdOrder.get(po.getPackId()).getObjectSize(curs, + po.getOffset()); + } + + @Override + long getObjectSize(WindowCursor curs, long pos) throws IOException { + MultiPackIndex.PackOffset po = offsetCalculator.decode(pos); + return packsInIdOrder.get(po.getPackId()).getObjectSize(curs, + po.getOffset()); + } + + @Override + LocalObjectRepresentation representation(WindowCursor curs, + AnyObjectId objectId) throws IOException { + MultiPackIndex.PackOffset po = getMidx().find(objectId); + if (po == null) { + return null; + } + return packsInIdOrder.get(po.getPackId()).representation(curs, + objectId); + } + + private MultiPackIndex getMidx() throws IOException { + Optional<MultiPackIndex> optional = midx.getOptional(); + if (optional.isPresent()) { + return optional.get(); + } + return memoizeMidxIfNeeded(); + } + + private synchronized MultiPackIndex memoizeMidxIfNeeded() + throws IOException { + if (invalid()) { + throw new PackInvalidException(getPackFile(), invalidatingCause); + } + Optional<MultiPackIndex> optional = midx.getOptional(); + if (optional.isPresent()) { + return optional.get(); + } + + try { + MultiPackIndex loadedMidx = MultiPackIndexLoader + .open(getPackFile()); + midx = optionally(loadedMidx); + return loadedMidx; + } catch (IOException e) { + setInvalid(); + invalidatingCause = e; + throw e; + } + } + + private static class OffsetCalculator { + private final MultiPackIndex.PackOffset mutablePo = new MultiPackIndex.PackOffset(); + + private final long[] accSizes; + + OffsetCalculator(long[] packSizes) { + accSizes = new long[packSizes.length]; + accSizes[0] = 0; + for (int i = 1; i < packSizes.length; i++) { + accSizes[i] = accSizes[i - 1] + packSizes[i - 1]; + } + } + + long encode(MultiPackIndex.PackOffset po) { + if (po == null) { + return -1; + } + return accSizes[po.getPackId()] + po.getOffset(); + } + + long encode(int packId, long offset) { + return accSizes[packId] + offset; + } + + MultiPackIndex.PackOffset decode(long totalOffset) { + if (totalOffset < 0) { + return null; + } + + for (int i = accSizes.length - 1; i >= 0; i--) { + if (totalOffset >= accSizes[i]) { + return mutablePo.setValues(i, totalOffset - accSizes[i]); + } + } + return null; + } + } + + private static class MidxPackIndex implements PackIndex { + + private final MultiPackIndex midx; + + private final OffsetCalculator offsetCalculator; + + MidxPackIndex(MultiPackIndex midx, OffsetCalculator offsetCalculator) { + this.midx = midx; + this.offsetCalculator = offsetCalculator; + } + + @Override + public Iterator<MutableEntry> iterator() { + MultiPackIndex.MidxIterator it = midx.iterator(); + return new Iterator<MutableEntry>() { + + private final MutableEntry me = new MutableEntry(); + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public MutableEntry next() { + MultiPackIndex.MutableEntry entry = it.next(); + me.idBuffer.fromObjectId(entry.getObjectId()); + me.offset = offsetCalculator.encode(entry.getPackId(), + entry.getOffset()); + return me; + } + }; + } + + @Override + public long getObjectCount() { + return midx.getObjectCount(); + } + + @Override + public long getOffset64Count() { + return 0; + } + + @Override + public ObjectId getObjectId(long nthPosition) { + if (nthPosition < 0) { + throw new IllegalArgumentException(); + } + return midx.getObjectAt((int) nthPosition); + } + + @Override + public long getOffset(long nthPosition) { + ObjectId objectAt = midx.getObjectAt((int) nthPosition); + MultiPackIndex.PackOffset packOffset = midx.find(objectAt); + return offsetCalculator.encode(packOffset); + } + + @Override + public long findOffset(AnyObjectId objId) { + MultiPackIndex.PackOffset packOffset = midx.find(objId); + return offsetCalculator.encode(packOffset); + } + + @Override + public int findPosition(AnyObjectId objId) { + return midx.findPosition(objId); + } + + @Override + public long findCRC32(AnyObjectId objId) + throws MissingObjectException, UnsupportedOperationException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasCRC32Support() { + return false; + } + + @Override + public void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) throws IOException { + midx.resolve(matches, id, matchLimit); + } + + @Override + public byte[] getChecksum() { + return midx.getChecksum(); + } + } + + private static final class MidxPackReverseIndex + implements PackReverseIndex { + + private final MultiPackIndex midx; + + private final OffsetCalculator offsetCalculator; + + MidxPackReverseIndex(MultiPackIndex midx, + OffsetCalculator offsetCalculator) { + this.midx = midx; + this.offsetCalculator = offsetCalculator; + } + + @Override + public void verifyPackChecksum(String packFilePath) + throws PackMismatchException { + + } + + @Override + public ObjectId findObject(long offset) { + MultiPackIndex.PackOffset po = offsetCalculator.decode(offset); + int revPos = midx.findBitmapPosition(po); + return midx.getObjectAtBitmapPosition(revPos); + } + + @Override + public long findNextOffset(long offset, long maxOffset) + throws CorruptObjectException { + throw new UnsupportedOperationException(); + } + + @Override + public int findPosition(long offset) { + MultiPackIndex.PackOffset po = offsetCalculator.decode(offset); + return midx.findBitmapPosition(po); + } + + @Override + public ObjectId findObjectByPosition(int nthPosition) { + return midx.getObjectAtBitmapPosition(nthPosition); + } + } +}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 9fa3ff3..cbf3566 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java
@@ -1320,6 +1320,7 @@ class LooseItems { * path of a loose ref relative to the repository root */ void refreshPathToLooseRef(Path refPath) { + boolean failed = false; for (int i = 1; i < refPath.getNameCount(); i++) { File dir = fileFor(refPath.subpath(0, i).toString()); // Use Files.newInputStream(Path) as it is consistent with other @@ -1328,7 +1329,16 @@ void refreshPathToLooseRef(Path refPath) { try (InputStream stream = Files.newInputStream(dir.toPath())) { // open the dir to refresh attributes (on some NFS clients) } catch (IOException e) { - break; // loose ref may not exist + failed = true; + break; // directory may not exist + } + } + if (!failed) { + try (InputStream stream = Files.newInputStream(refPath)) { + // open the loose ref to refresh attributes (on some NFS + // clients) + } catch (IOException e) { + // loose ref may not exist } } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MidxMetadataReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MidxMetadataReader.java index 0d8dfe4..26c0bb4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MidxMetadataReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MidxMetadataReader.java
@@ -42,6 +42,9 @@ */ public class MidxMetadataReader { + private MidxMetadataReader() { + } + /** * Metadata useful before loading the full midx * @@ -220,7 +223,7 @@ long getStartOffset(int chunkId) { throw new IllegalArgumentException("Asking for unknown chunk"); //$NON-NLS-1$ } - public long getEndOffset(int chunkId) { + long getEndOffset(int chunkId) { int pos = -1; for (int i = 0; i < chunks.size(); i++) { if (chunks.get(i).id() == chunkId) { @@ -235,7 +238,7 @@ public long getEndOffset(int chunkId) { return chunks.get(pos + 1).offset(); } - public long getLastOffset() { + long getLastOffset() { return chunks.get(chunks.size() - 1).offset(); } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityChecker.java new file mode 100644 index 0000000..cfe6bcf --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/TreeWalkConnectivityChecker.java
@@ -0,0 +1,447 @@ +/* + * Copyright (C) 2026, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.transport.connectivity; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdSubclassMap; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.ConnectivityChecker; +import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.treewalk.TreeWalk; + +/** + * A connectivity checker that avoids the object allocations that occur when + * doing standard graph coloring via {@code ObjectWalk}. + * + * {@code TreeWalkConnectivityChecker} requires the {@code PackParser} to report + * new objects in the pack, and will fail immediately if that is not configured. + * + * The {@code ObjectWalk}-based algorithm marks objects interesting and + * uninteresting and parses new subtrees and blobs to propagate those states. + * Each propagation of state to a child node requires a Java object allocation. + * This algorithm is closer to the diff algorithm. It parses tree objects at + * common paths and compares hash values in those trees. Objects only need to be + * created for subtrees that differ. + * + * The tree connectivity part of this algorithm always creates O(commits + tree + * objects in the new pack) Java objects, independent of where the parent commit + * is in the graph, and independent of the number of references. (Caveat, it + * does a standard reachability check if a parent commit is not in advertised + * "haves", and a standard object reachability check for any base objects + * referenced in a thin pack.) The {@code ObjectWalk}-based algorithm is + * equivalent to this algorithm when the parents of new commits are all in the + * advertised "haves", creating O(commit + tree objects in the new pack) Java + * objects. It is much less efficient otherwise, creating either O(tree objects + * in checkout) or O(all objects in checkout) Java objects. + * + * This algorithm first validates that the commits in the commands (new branch + * tips) are connected. Starting with those commits, it walks back until a + * commit not in the pack is found, or until a commit with no parents is found. + * If a parent commit id is not in the database, connectivity fails. If the + * parent commit is in the database and was present in the "haves" advertised + * for the client, it moves on to verifying tree connectivity. Otherwise, it + * performs a reachability check to make sure the client has access to the + * unadvertised parent commit, and proceeds to verifying tree connectivity if + * that succeeds. + * + * Tree connectivity is verified for every commit in the receive pack that was + * visited when checking commit connectivity. For each commit, get its root tree + * and the root trees of its parents. For each path segment in the child + * commit's root tree, do the following (includes recursing into differing + * subtrees, where the same actions are applied): + * <ul> + * <li>if a blob or subtree's id is identical to one of the parent's blob or + * subtree's ids, continue/li> + * <li>if a new blob id is not present in the database, connectivity fails</li> + * <li>if a new subtree id is present in the pack, traverse into the subtree to + * continue the check, performing the same actions in this list</li> + * <li>if the new subtree id is not in the pack but is present in the database, + * continue/li> + * <li>if the new subtree id is not in either the pack or the database, + * connectivity fails</li> + * </ul> + */ +public class TreeWalkConnectivityChecker implements ConnectivityChecker { + private ObjectIdSubclassMap<ObjectId> objectsInPack; + + private Set<ObjectId> advertisedHaves; + + @Override + public void checkConnectivity(ConnectivityCheckInfo connectivityCheckInfo, + Set<ObjectId> haves, ProgressMonitor pm) throws IOException { + + if (!connectivityCheckInfo.getParser().needNewObjectIds()) { + throw new IllegalStateException( + "PackParser.setNeedNewObjectIds(true) must be set"); //$NON-NLS-1$ + } + + Repository repo = connectivityCheckInfo.getRepository(); + RevWalk rw = connectivityCheckInfo.getWalk(); + this.objectsInPack = connectivityCheckInfo.getParser() + .getNewObjectIds(); + this.advertisedHaves = haves; + + Set<RevCommit> nonAdvertisedParentCommitsOutsidePack = new HashSet<>(); + List<RevCommit> newCommitsToVerify = extractCommitsToVerify(rw, repo, + connectivityCheckInfo.getCommands(), + nonAdvertisedParentCommitsOutsidePack, pm); + + /* + * This class underreports the full set of objects parsed and does not + * advance the progress monitor in the checkReachability() and + * checkThinPackBases() methods. These methods invoke {@code + * ReachabilityChecker} and @{code ObjectReachabilityChecker}, neither + * of which take a progress monitor. Pushes are generally based on + * commits near branch tips, so the checkers normally do minimal walks. + * Setup for these checkers involves at a minimum marking all "haves" as + * starts, so we capture that in the progress monitor. + */ + if (!nonAdvertisedParentCommitsOutsidePack.isEmpty()) { + pm.update(advertisedHaves.size()); + checkReachability(rw, repo, nonAdvertisedParentCommitsOutsidePack); + } + + checkThinPackBases(rw, connectivityCheckInfo); + + verifyTreeConnectivity(rw, repo, newCommitsToVerify, + connectivityCheckInfo.isCheckObjects(), pm); + } + + /** + * Extracts commits that need to be verified and identifies parent commits + * outside the pack. + * + * @param rw + * the RevWalk to use + * @param repo + * the repository + * @param commands + * the receive commands + * @param nonAdvertisedParentCommitsOutsidePack + * set to populate with parent commits outside the pack and not + * in the advertised set (haves) + * @param pm + * progress monitor + * @return list of new commits in the pack that require verification + * @throws IOException + * if an I/O error occurs + */ + private List<RevCommit> extractCommitsToVerify(RevWalk rw, Repository repo, + List<ReceiveCommand> commands, + Set<RevCommit> nonAdvertisedParentCommitsOutsidePack, + ProgressMonitor pm) throws IOException { + + List<RevCommit> newCommitsToVerify = new ArrayList<>(); + + // Gather new commits from the receive commands + for (ReceiveCommand cmd : commands) { + if (cmd.getType() != ReceiveCommand.Type.DELETE + && !advertisedHaves.contains(cmd.getNewId())) { + rw.markStart(rw.parseCommit(cmd.getNewId())); + } + } + + // Walk commits in the receive pack, recording both commits in the pack + // and parent commits outside the pack + RevCommit commit; + while ((commit = rw.next()) != null) { + pm.update(1); + if (objectsInPack.contains(commit)) { + newCommitsToVerify.add(commit); + } else { + // Commit not in pack, check object database + if (!repo.getObjectDatabase().has(commit)) { + throw new MissingObjectException(commit.getId(), + Constants.TYPE_COMMIT); + } + // Advertised parents are reachable, only track non-advertised + if (!advertisedHaves.contains(commit.getId())) { + nonAdvertisedParentCommitsOutsidePack.add(commit); + } + rw.markUninteresting(commit); + } + } + + return newCommitsToVerify; + } + + /** + * Performs reachability checks for parent commits outside the pack. + * + * @param rw + * the RevWalk to use + * @param repo + * the repository + * @param nonAdvertisedParentCommitsOutsidePack + * parent commits to check visibility for + * @throws IOException + * if an I/O error occurs + */ + private void checkReachability(RevWalk rw, Repository repo, + Set<RevCommit> nonAdvertisedParentCommitsOutsidePack) + throws IOException { + + try { + // First try with the smaller set of advertised haves + ReachabilityChecker checker = rw.getObjectReader() + .createReachabilityChecker(rw); + Stream<RevCommit> starterCommits = advertisedHaves.stream() + .map(id -> parseCommitOrNull(rw, id)).filter(Objects::nonNull); + + Optional<RevCommit> unreachable = checker.areAllReachable( + nonAdvertisedParentCommitsOutsidePack, starterCommits); + + if (unreachable.isPresent()) { + // Fallback to check against full ref database + Stream<RevCommit> allRefCommits = repo.getRefDatabase() + .getRefs().stream() + .map(ref -> parseCommitOrNull(rw, ref.getObjectId())) + .filter(Objects::nonNull); + unreachable = checker.areAllReachable( + nonAdvertisedParentCommitsOutsidePack, allRefCommits); + if (unreachable.isPresent()) { + throw new MissingObjectException(unreachable.get().getId(), + Constants.TYPE_COMMIT); + } + } + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + /** + * Validates that base objects referenced in a thin pack are reachable. + * + * @param rw + * the RevWalk to use + * @param connectivityCheckInfo + * the connectivity check info + * @throws IOException + * if an I/O error occurs + */ + private void checkThinPackBases(RevWalk rw, + ConnectivityCheckInfo connectivityCheckInfo) throws IOException { + + if (!connectivityCheckInfo.isCheckObjects()) { + return; + } + ObjectIdSubclassMap<ObjectId> baseObjectIds = connectivityCheckInfo + .getParser().getBaseObjectIds(); + + try (ObjectWalk ow = new ObjectWalk(rw.getObjectReader())) { + ObjectReachabilityChecker checker = rw.getObjectReader() + .createObjectReachabilityChecker(ow); + + List<RevObject> targetObjs = StreamSupport + .stream(baseObjectIds.spliterator(), false) + .map(id -> parseAnyUnchecked(ow, id)) + .collect(Collectors.toList()); + + // First try with the smaller set of advertised haves + Stream<RevObject> starterObjs = advertisedHaves.stream() + .map(id -> parseAnyUnchecked(ow, id)) + .filter(Objects::nonNull); + + Optional<RevObject> unreachable = checker + .areAllReachable(targetObjs, starterObjs); + if (unreachable.isPresent()) { + // Fallback to check against full ref database + Stream<RevObject> allRefCommits = connectivityCheckInfo + .getRepository().getRefDatabase().getRefs().stream() + .map(ref -> parseAnyUnchecked(ow, ref.getObjectId())) + .filter(Objects::nonNull); + unreachable = checker.areAllReachable(targetObjs, + allRefCommits); + if (unreachable.isPresent()) { + throw new MissingObjectException(unreachable.get().getId(), + Constants.TYPE_COMMIT); + } + } + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + /** + * Verifies tree connectivity for the new commits. + * + * @param rw + * the RevWalk to use + * @param repo + * the repository + * @param newCommitsToVerify + * commits to verify + * @param checkObjects + * whether to check objects + * @param pm + * progress monitor + * @throws IOException + * if an I/O error occurs + */ + private void verifyTreeConnectivity(RevWalk rw, Repository repo, + List<RevCommit> newCommitsToVerify, boolean checkObjects, + ProgressMonitor pm) throws IOException { + + for (RevCommit c : newCommitsToVerify) { + verifyTreeConnectivityForCommit(c, rw, repo, checkObjects, pm); + } + } + + /** + * Verifies tree connectivity for a single commit. + * + * @param c + * the commit + * @param rw + * the RevWalk to use + * @param repo + * the repository + * @param checkObjects + * whether to check objects + * @param pm + * progress monitor + * @throws IOException + * if an I/O error occurs + */ + private void verifyTreeConnectivityForCommit(RevCommit c, RevWalk rw, + Repository repo, boolean checkObjects, ProgressMonitor pm) + throws IOException { + try (TreeWalk tw = new TreeWalk(rw.getObjectReader())) { + tw.setRecursive(false); + tw.addTree(c.getTree()); + pm.update(1); + for (RevCommit p : c.getParents()) { + tw.addTree(p.getTree()); + pm.update(1); + } + + while (tw.next()) { + if (tw.getFileMode(0) == FileMode.MISSING) { + continue; // Object deleted or moved + } + + if (tw.getFileMode(0) == FileMode.GITLINK) { + continue; // Skip submodule entries + } + + ObjectId newObjId = tw.getObjectId(0); + if (matchesAnyParent(tw, newObjId)) { + continue; // matched via object ids + } + + if (!tw.isSubtree()) { + /* + * Blob case. objectsInPack are already in the object + * database, but the map lookup is faster than the database + * search, so try the map lookup first. + */ + if (!objectsInPack.contains(newObjId) && (checkObjects + || !repo.getObjectDatabase().has(newObjId))) { + throw new MissingObjectException(newObjId, + tw.getFileMode(0).getObjectType()); + } + } else { + // Subtree case + if (objectsInPack.contains(newObjId)) { + tw.enterSubtree(); + pm.update(1 + c.getParentCount()); + } else if (checkObjects + || !repo.getObjectDatabase().has(newObjId)) { + throw new MissingObjectException(newObjId, + FileMode.TREE.getObjectType()); + } + } // Else object is in the database + } + } + } + + /** + * Parses an object, throwing UncheckedIOException on failure. For use with + * streams. + * + * @param rw + * the RevWalk to use + * @param id + * the object ID + * @return the parsed object + */ + private static RevObject parseAnyUnchecked(RevWalk rw, ObjectId id) { + try { + return rw.parseAny(id); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Parses an object as a commit, peeling tags if necessary. Returns null if + * the object is not a commit or cannot be peeled to a commit. Throws + * UncheckedIOException on other I/O failures. For use with streams. + * + * @param rw + * the RevWalk to use + * @param id + * the object ID + * @return the parsed commit, or null + */ + private static RevCommit parseCommitOrNull(RevWalk rw, ObjectId id) { + try { + return rw.parseCommit(id); + } catch (IncorrectObjectTypeException e) { + return null; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Checks if the new object ID matches any parent tree's object ID at the + * current path. + * + * @param tw + * the TreeWalk + * @param newObjId + * the new object ID + * @return true if matches any parent + */ + private static boolean matchesAnyParent(TreeWalk tw, ObjectId newObjId) { + for (int i = 1; i < tw.getTreeCount(); i++) { + if (tw.getFileMode(i) != FileMode.MISSING + && newObjId.equals(tw.getObjectId(i))) { + return true; + } + } + return false; + } + +}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 9de8392..5265e76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
@@ -839,6 +839,13 @@ public static int decodeTypeString(final AnyObjectId id, */ public static final int COMMIT_GENERATION_NOT_COMPUTED = 0; + /** + * Name of the .git/objects/packs/multi-pack-index file. + * + * @since 7.7 + */ + public static final String MIDX_FILE = "multi-pack-index"; + private Constants() { // Hide the default constructor }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index e1f2b19..e842f47 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java
@@ -254,7 +254,13 @@ public void setNeedNewObjectIds(boolean b) { newObjectIds = null; } - private boolean needNewObjectIds() { + /** + * Whether the parser is configured to keep track of new objects. + * + * @return {@code true} if new objects are tracked, {@code false} otherwise. + * @since 7.7 + */ + public boolean needNewObjectIds() { return newObjectIds != null; }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index 0ee0ad5..8b05d41 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java
@@ -1107,8 +1107,9 @@ public long getPackSize() { * @return if the client is a shallow repository, the list of edge commits * that define the client's shallow boundary. Empty set if the * client is earlier than Git 1.9, or is a full clone. + * @since 7.7 */ - private Set<ObjectId> getClientShallowCommits() { + protected Set<ObjectId> getClientShallowCommits() { return clientShallowCommits; } @@ -2262,6 +2263,7 @@ private void service() throws IOException { } Instant startProcessing = Instant.now(); + long timePreReceiveHooks = 0; try { setAtomic(isCapabilityEnabled(CAPABILITY_ATOMIC)); @@ -2270,8 +2272,10 @@ private void service() throws IOException { failPendingCommands(); } + Instant startPreReceive = Instant.now(); preReceive.onPreReceive( this, filterCommands(Result.NOT_ATTEMPTED)); + timePreReceiveHooks = Duration.between(startPreReceive, Instant.now()).toMillis(); if (atomic && anyRejects()) { failPendingCommands(); } @@ -2280,7 +2284,7 @@ private void service() throws IOException { unlockPack(); } long timeProcessingCommands = Duration - .between(startProcessing, Instant.now()).toMillis(); + .between(startProcessing, Instant.now()).toMillis() - timePreReceiveHooks; ReceivedPackStatistics.Builder statsBuilder = stats != null ? ReceivedPackStatistics.Builder.toBuilder(stats) @@ -2288,6 +2292,7 @@ private void service() throws IOException { stats = statsBuilder.setTimeNegotiating(timeNegotiating) .setTimeReceiving(timeReceiving) .setTimeCheckingConnectivity(timeCheckingConnectivity) + .setTimePreReceiveHooks(timePreReceiveHooks) .setTimeProcessingCommands(timeProcessingCommands) .build();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java index f804c31..39fadd6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java
@@ -37,6 +37,7 @@ public class ReceivedPackStatistics { private long timeNegotiating; private long timeReceiving; private long timeCheckingConnectivity; + private long timePreReceiveHooks; private long timeProcessingCommands; /** @@ -193,7 +194,17 @@ public long getTimeCheckingConnectivity() { } /** - * Get time in milliseconds spent processing commands (validation, hooks, + * Get time in milliseconds spent in pre-receive hooks + * + * @return time in milliseconds spent in pre-receive hooks + * @since 7.7 + */ + public long getTimePreReceiveHooks() { + return timePreReceiveHooks; + } + + /** + * Get time in milliseconds spent processing commands (validation * and ref updates) * * @return time in milliseconds spent processing commands @@ -224,6 +235,7 @@ public static class Builder { private long timeNegotiating; private long timeReceiving; private long timeCheckingConnectivity; + private long timePreReceiveHooks; private long timeProcessingCommands; /** @@ -250,6 +262,7 @@ public static Builder toBuilder(ReceivedPackStatistics s) { b.timeNegotiating = s.getTimeNegotiating(); b.timeReceiving = s.getTimeReceiving(); b.timeCheckingConnectivity = s.getTimeCheckingConnectivity(); + b.timePreReceiveHooks = s.getTimePreReceiveHooks(); b.timeProcessingCommands = s.getTimeProcessingCommands(); return b; } @@ -342,6 +355,17 @@ public Builder setTimeCheckingConnectivity( } /** + * @param timePreReceiveHooks + * time in milliseconds spent in pre-receive hooks + * @return this + * @since 7.7 + */ + public Builder setTimePreReceiveHooks(long timePreReceiveHooks) { + this.timePreReceiveHooks = timePreReceiveHooks; + return this; + } + + /** * @param timeProcessingCommands * time in milliseconds spent processing commands * @return this @@ -429,6 +453,7 @@ ReceivedPackStatistics build() { s.timeNegotiating = timeNegotiating; s.timeReceiving = timeReceiving; s.timeCheckingConnectivity = timeCheckingConnectivity; + s.timePreReceiveHooks = timePreReceiveHooks; s.timeProcessingCommands = timeProcessingCommands; return s; }
diff --git a/pom.xml b/pom.xml index 75c906b..a9dd3de 100644 --- a/pom.xml +++ b/pom.xml
@@ -137,7 +137,7 @@ <slf4j-version>2.0.17</slf4j-version> <maven-javadoc-plugin-version>3.12.0</maven-javadoc-plugin-version> <gson-version>2.13.2</gson-version> - <bouncycastle-version>1.83</bouncycastle-version> + <bouncycastle-version>1.84</bouncycastle-version> <spotbugs-maven-plugin-version>4.9.8.2</spotbugs-maven-plugin-version> <maven-project-info-reports-plugin-version>3.9.0</maven-project-info-reports-plugin-version> <maven-jxr-plugin-version>3.6.0</maven-jxr-plugin-version>