Merge branch 'master' into stable-7.6 * master: Update org.assertj:assertj-core to 3.27.7 DfsPackFileMidxNPacks: getBitmapIndex tries to load midx bitmaps PackBitmapIndexBuilderTest: add test to the bitmap builder storage Update bytebuddy to 1.18.4 Update org.objectweb.asm to 9.9.1 PackBitmapCalculator: Move code to calculate bitmaps out of PackWriter DfsPackFileMidx: expose the checksum of the midx DfsPackFile: Use getPackIndex while loading bitmaps DfsMidxWriter: set file size for the midx extension DfsPackFileMidx*Test: Use MidxTestUtils when possible DfsPackFileMidx: Implement PackReverseIndex over midx DfsPackFileMidx: Implement PackIndex over midx MultiPackIndex: add the checksum to the midx DfsPackFileMidx: Add method to translate midx position to objectId MultiPackIndex: Add #getObjectAt to translate position to id MultiPackIndex: implement methods for the reverse index Update python dependencies of download_release.py Change-Id: I8056c7ffab3c8608cbe94af9aea803959605f615
diff --git a/MODULE.bazel b/MODULE.bazel index f98cf57..ac7d934 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -5,8 +5,8 @@ bazel_dep(name = "rbe_autoconfig") git_override( module_name = "rbe_autoconfig", - remote = "https://github.com/davido/rbe_autoconfig.git", commit = "71f39817c028949bd6d25f1806502bcd33028d14", + remote = "https://github.com/davido/rbe_autoconfig.git", ) register_toolchains("//tools:error_prone_warnings_toolchain_java17_definition") @@ -23,7 +23,7 @@ BOUNCYCASTLE_VERSION = "1.83" -BYTE_BUDDY_VERSION = "1.18.2" +BYTE_BUDDY_VERSION = "1.18.4" JETTY_VERSION = "12.1.5" @@ -62,7 +62,7 @@ "org.apache.httpcomponents:httpcore:4.4.16", "org.apache.sshd:sshd-osgi:" + SSHD_VERSION, "org.apache.sshd:sshd-sftp:" + SSHD_VERSION, - "org.assertj:assertj-core:3.27.6", + "org.assertj:assertj-core:3.27.7", "org.bouncycastle:bcpg-jdk18on:" + BOUNCYCASTLE_VERSION, "org.bouncycastle:bcpkix-jdk18on:" + BOUNCYCASTLE_VERSION, "org.bouncycastle:bcprov-jdk18on:" + BOUNCYCASTLE_VERSION,
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target index 957e6aa..2b01f35 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.34.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.34" sequenceNumber="1765895853"> +<target name="jgit-4.34" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target index d503246..b838cfc 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.35.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.35" sequenceNumber="1765895853"> +<target name="jgit-4.35" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target index ce5da93..278511b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.36.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.36" sequenceNumber="1765895853"> +<target name="jgit-4.36" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target index ddfd6a1..3df478a 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.37.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.37" sequenceNumber="1765895853"> +<target name="jgit-4.37" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target index 8d89bf2..3fb1dbe 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.38.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.38" sequenceNumber="1766134741"> +<target name="jgit-4.38" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target index e21a5b9..7742ddb 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.39.target
@@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?pde?> <!-- generated with https://github.com/eclipse-cbi/targetplatform-dsl --> -<target name="jgit-4.39" sequenceNumber="1766135179"> +<target name="jgit-4.39" sequenceNumber="1769375519"> <locations> <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit"> <unit id="com.jcraft.jsch" version="0.1.55.v20230916-1400"/> @@ -19,11 +19,11 @@ <unit id="org.hamcrest.source" version="3.0.0"/> <unit id="org.junit" version="4.13.2.v20240929-1000"/> <unit id="org.junit.source" version="4.13.2.v20240929-1000"/> - <unit id="org.objectweb.asm" version="9.9.0"/> - <unit id="org.objectweb.asm.commons" version="9.9.0"/> - <unit id="org.objectweb.asm.util" version="9.9.0"/> - <unit id="org.objectweb.asm.tree" version="9.9.0"/> - <unit id="org.objectweb.asm.tree.analysis" version="9.9.0"/> + <unit id="org.objectweb.asm" version="9.9.1"/> + <unit id="org.objectweb.asm.commons" version="9.9.1"/> + <unit id="org.objectweb.asm.util" version="9.9.1"/> + <unit id="org.objectweb.asm.tree" version="9.9.1"/> + <unit id="org.objectweb.asm.tree.analysis" version="9.9.1"/> <unit id="org.objenesis" version="3.4.0"/> <unit id="org.objenesis.source" version="3.4.0"/> <unit id="org.osgi.service.cm" version="1.6.1.202109301733"/> @@ -195,13 +195,13 @@ <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> <dependency> <groupId>net.bytebuddy</groupId> <artifactId>byte-buddy-agent</artifactId> - <version>1.18.2</version> + <version>1.18.4</version> <type>jar</type> </dependency> </dependencies> @@ -239,7 +239,7 @@ <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> - <version>3.27.6</version> + <version>3.27.7</version> <type>jar</type> </dependency> </dependencies>
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd index 55f40d1..5368a95 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/maven/dependencies.tpd
@@ -56,7 +56,7 @@ dependency { groupId = "org.assertj" artifactId = "assertj-core" - version = "3.27.6" + version = "3.27.7" } } @@ -97,12 +97,12 @@ dependency { groupId = "net.bytebuddy" artifactId = "byte-buddy" - version = "1.18.2" + version = "1.18.4" } dependency { groupId = "net.bytebuddy" artifactId = "byte-buddy-agent" - version = "1.18.2" + version = "1.18.4" } }
diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.39.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.39.tpd index 299d7d0..48725ca 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.39.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/orbit-4.39.tpd
@@ -17,11 +17,11 @@ org.hamcrest.source [3.0.0,3.0.0] org.junit [4.13.2.v20240929-1000,4.13.2.v20240929-1000] org.junit.source [4.13.2.v20240929-1000,4.13.2.v20240929-1000] - org.objectweb.asm [9.9.0,9.9.0] - org.objectweb.asm.commons [9.9.0,9.9.0] - org.objectweb.asm.util [9.9.0,9.9.0] - org.objectweb.asm.tree [9.9.0,9.9.0] - org.objectweb.asm.tree.analysis [9.9.0,9.9.0] + org.objectweb.asm [9.9.1,9.9.1] + org.objectweb.asm.commons [9.9.1,9.9.1] + org.objectweb.asm.util [9.9.1,9.9.1] + org.objectweb.asm.tree [9.9.1,9.9.1] + org.objectweb.asm.tree.analysis [9.9.1,9.9.1] org.objenesis [3.4,3.4] org.objenesis.source [3.4,3.4] org.osgi.service.cm [1.6.1.202109301733,1.6.1.202109301733]
diff --git a/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java new file mode 100644 index 0000000..b34b822 --- /dev/null +++ b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java
@@ -0,0 +1,147 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.zip.Deflater; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; + +/** + * Helpers to write multipack indexes + */ +public class MidxTestUtils { + private MidxTestUtils() { + } + + /** + * Write a single pack into the repo with the blob as contents + * + * @param db + * repository + * @param blob + * blob to write into the pack + * @return object id of the blob written in the pack + * @throws IOException + * a problem writing in the repo + */ + static ObjectId writePackWithBlob(DfsRepository db, String blob) + throws IOException { + return writePackWithBlobs(db, blob)[0]; + } + + /** + * Write multiple blobs into a single pack in the repo + * + * @param db + * repository + * @param blobs + * blobs to write into the pack + * @return object ids of the blobs written in the pack, in the same order as + * the input parameters + * @throws IOException + * a problem writing in the repo + */ + static ObjectId[] writePackWithBlobs(DfsRepository db, String... blobs) + throws IOException { + ObjectId[] oids = new ObjectId[blobs.length]; + + DfsInserter ins = (DfsInserter) db.newObjectInserter(); + ins.setCompressionLevel(Deflater.NO_COMPRESSION); + for (int i = 0; i < blobs.length; i++) { + oids[i] = ins.insert(OBJ_BLOB, blobs[i].getBytes(UTF_8)); + } + ins.flush(); + return oids; + } + + /** + * Write a midx covering the only pack in the repo + * + * @param db + * a repository with a single pack + * @return a midx covering that single pack + * @throws IOException + * a problem writing in the repo + */ + static DfsPackFileMidx writeSinglePackMidx(DfsRepository db) + throws IOException { + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + assertEquals("More than one pack in db", 1, packs.length); + return writeSinglePackMidx(db, packs[0]); + } + + /** + * Write a midx covering a single pack + * + * @param db + * a repository to write the midx covering the pack + * @param pack + * a pack in the repository that will be covered by a new midx + * @return a midx covering that single pack + * @throws IOException + * a problem writing in the repo + */ + static DfsPackFileMidx writeSinglePackMidx(DfsRepository db, + DfsPackFile pack) throws IOException { + return writeMultipackIndex(db, new DfsPackFile[] { pack }, null); + } + + /** + * Write a midx covering a single pack + * + * @param db + * a repository to write the midx covering the pack + * @param pack + * a pack in the repository that will be covered by a new midx + * @param base + * base of this midx (can be null) + * @return a midx covering that single pack + * @throws IOException + * a problem writing in the repo + */ + static DfsPackFileMidx writeSinglePackMidx(DfsRepository db, + DfsPackFile pack, @Nullable DfsPackFileMidx base) + throws IOException { + return writeMultipackIndex(db, new DfsPackFile[] { pack }, base); + } + + /** + * Write a midx in the repository + * + * @param db + * the repository + * @param packs + * packs to be covered by this midx + * @param base + * base of the newly created midx + * @return the new midx instance + * @throws IOException + * a problem writing in the repo + */ + static DfsPackFileMidx writeMultipackIndex(DfsRepository db, + DfsPackFile[] packs, DfsPackFileMidx base) throws IOException { + DfsPackDescription desc = DfsMidxWriter.writeMidx( + NullProgressMonitor.INSTANCE, db.getObjectDatabase(), + Arrays.asList(packs), + base != null ? base.getPackDescription() : null); + db.getObjectDatabase().commitPack(List.of(desc), null); + return DfsPackFileMidx.create(DfsBlockCache.getInstance(), desc, + Arrays.asList(packs), base); + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java new file mode 100644 index 0000000..8574688 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java
@@ -0,0 +1,241 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeMultipackIndex; +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writePackWithBlobs; +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeSinglePackMidx; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.internal.storage.file.PackReverseIndex; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.ObjectId; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class DfsPackFileMidxIndexTest { + + private static final ObjectId NOT_IN_PACK = ObjectId + .fromString("3f306cb3fcd5116919fecad615524bd6e6ea4ba7"); + + private static final List<String> BLOBS = List.of("blob one", "blob two", + "blob three", "blob four", "blob five", "blob six"); + + @Parameters(name = "{0}") + public static Iterable<TestInput> data() throws IOException { + return List.of(setupOneMidxOverOnePack(), setupOneMidxOverNPacks(), + setupMidxChainEachOverNPacks(), + setupMidxChainSingleAndNPacks()); + } + + private record TestInput(String testDesc, DfsRepository db, + DfsPackFileMidx midx, ObjectId[] oids) { + @Override + public String toString() { + return testDesc; + } + + } + + private TestInput ti; + + public DfsPackFileMidxIndexTest(TestInput ti) { + this.ti = ti; + } + + @Test + public void getPackIndex_getObjectCount() { + try (DfsReader ctx = ti.db().getObjectDatabase().newReader()) { + assertEquals(ti.oids().length, + ti.midx().getPackIndex(ctx).getObjectCount()); + } + } + + @Test + public void getPackIndex_position_findPosition_getObjectId() { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + for (int i = 0; i < ti.oids().length; i++) { + ObjectId expected = ti.oids()[i]; + int position = idx.findPosition(expected); + assertNotEquals(-1, position); + ObjectId actual = idx.getObjectId(position); + assertEquals(expected, actual); + } + assertEquals(-1, idx.findPosition(NOT_IN_PACK)); + } + } + + @Test + public void getPackIndex_offset_findOffset_getOffset() { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + for (int i = 0; i < ti.oids().length; i++) { + ObjectId oid = ti.oids()[i]; + int oidPosition = idx.findPosition(oid); + + long offsetById = idx.findOffset(oid); + long offsetByPos = idx.getOffset(oidPosition); + assertEquals(offsetById, offsetByPos); + } + assertEquals(-1, idx.findOffset(NOT_IN_PACK)); + } + } + + @Test + public void getPackIndex_objects_contains_hasObjects() { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + for (int i = 0; i < ti.oids().length; i++) { + ObjectId oid = ti.oids()[i]; + assertTrue(idx.contains(oid)); + assertTrue(idx.hasObject(oid)); + } + assertFalse(idx.contains(NOT_IN_PACK)); + assertFalse(idx.hasObject(NOT_IN_PACK)); + } + } + + @Test + public void getPackIndex_resolve() throws IOException { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + Set<ObjectId> matches = new HashSet<>(); + // Sha1 of "blob two" = ae4116e0972d85cd751b458fea94ca9eb84dd692 + idx.resolve(matches, AbbreviatedObjectId.fromString("ae411"), 100); + assertEquals(1, matches.size()); + } + } + + @Test + public void getReverseIndex_findObject() throws IOException { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + PackReverseIndex ridx = ti.midx().getReverseIdx(ctx); + for (ObjectId oid : ti.oids()) { + long offset = idx.findOffset(oid); + assertEquals(oid, ridx.findObject(offset)); + } + } + } + + @Test + public void getReverseIndex_findObjectByPosition() throws IOException { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + ObjectId[] offsetOrder = ti.oids().clone(); + Arrays.sort(offsetOrder, Comparator.comparingLong(idx::findOffset)); + + PackReverseIndex ridx = ti.midx().getReverseIdx(ctx); + for (int i = 0; i < offsetOrder.length; i++) { + assertEquals(offsetOrder[i], ridx.findObjectByPosition(i)); + } + } + } + + @Test + public void getReverseIndex_findPosition() throws IOException { + try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) { + PackIndex idx = ti.midx().getPackIndex(ctx); + ObjectId[] offsetOrder = ti.oids().clone(); + Arrays.sort(offsetOrder, Comparator.comparingLong(idx::findOffset)); + + PackReverseIndex ridx = ti.midx().getReverseIdx(ctx); + for (int i = 0; i < offsetOrder.length; i++) { + long offset = idx.findOffset(offsetOrder[i]); + int position = ridx.findPosition(offset); + assertEquals(i, position); + } + } + } + static TestInput setupOneMidxOverOnePack() throws IOException { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("one_midx_one_pack")); + ObjectId[] objectIds = writePackWithBlobs(db, + BLOBS.toArray(String[]::new)); + DfsPackFileMidx midx1 = writeSinglePackMidx(db); + return new TestInput("one midx - one pack", db, midx1, objectIds); + } + + static TestInput setupOneMidxOverNPacks() throws IOException { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("one_midx_n_packs")); + + ObjectId[] objectIds = BLOBS.stream().map(s -> { + try { + return MidxTestUtils.writePackWithBlob(db, s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).toArray(ObjectId[]::new); + DfsPackFileMidx midx1 = writeMultipackIndex(db, + db.getObjectDatabase().getPacks(), null); + return new TestInput("one midx - n packs", db, midx1, objectIds); + } + + static TestInput setupMidxChainEachOverNPacks() throws IOException { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("two_midx_3_packs_each")); + + ObjectId[] objectIds = BLOBS.stream().map(s -> { + try { + return MidxTestUtils.writePackWithBlob(db, s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).toArray(ObjectId[]::new); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + // If the amount of blobs (i.e. packs), adjust the ranges covered by + // midx. + assertEquals(6, BLOBS.size()); + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 3, 6), null); + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 0, 3), midxBase); + return new TestInput("two midx - 3 packs each", db, midxTip, objectIds); + } + + static TestInput setupMidxChainSingleAndNPacks() throws IOException { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("two_midx_3_packs_each")); + + ObjectId[] objectIds = BLOBS.stream().map(s -> { + try { + return MidxTestUtils.writePackWithBlob(db, s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).toArray(ObjectId[]::new); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + // If the amount of blobs (i.e. packs), adjust the ranges covered by + // midx. + assertEquals(6, BLOBS.size()); + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 1, 6), null); + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 0, 1), midxBase); + return new TestInput("two midx - 1 pack, 5 packs", db, midxTip, + objectIds); + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java index 758ac1f..706bc91 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java
@@ -18,6 +18,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -27,11 +28,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.zip.Deflater; @@ -40,11 +38,8 @@ import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidx.VOffsetCalculator; import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidxNPacks.VOffsetCalculatorNPacks; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; -import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset; -import org.eclipse.jgit.internal.storage.midx.MultiPackIndexWriter; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.JGitTestUtil; @@ -101,11 +96,11 @@ public void midx_findIdxPosition_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 4, 6), null); - DfsPackFileMidx midxMid = writeMultipackIndex( + DfsPackFileMidx midxMid = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 2, 4), midxBase); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 2), midxMid); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -120,6 +115,49 @@ public void midx_findIdxPosition_withBase() throws IOException { } @Test + public void midx_getObjectAt() throws IOException { + ObjectId o1 = writePackWithBlob("something".getBytes(UTF_8)); + ObjectId o2 = writePackWithBlob("something else".getBytes(UTF_8)); + ObjectId o3 = writePackWithBlob("and more".getBytes(UTF_8)); + DfsPackFileMidx midx = writeMultipackIndex(); + + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertEquals(o1, midx.getObjectAt(ctx, 2)); + assertEquals(o2, midx.getObjectAt(ctx, 0)); + assertEquals(o3, midx.getObjectAt(ctx, 1)); + } + } + + @Test + public void midx_getObjectAt_withBase() throws IOException { + ObjectId o1 = writePackWithBlob("o1".getBytes(UTF_8)); + ObjectId o2 = writePackWithBlob("o2".getBytes(UTF_8)); + ObjectId o3 = writePackWithBlob("o3".getBytes(UTF_8)); + ObjectId o4 = writePackWithBlob("o4".getBytes(UTF_8)); + ObjectId o5 = writePackWithBlob("o5".getBytes(UTF_8)); + ObjectId o6 = writePackWithBlob("o6".getBytes(UTF_8)); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + + // Packs are in reverse insertion order + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 4, 6), null); + DfsPackFileMidx midxMid = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 2, 4), midxBase); + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 0, 2), midxMid); + + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertEquals(o1, midxTip.getObjectAt(ctx, 0)); + assertEquals(o2, midxTip.getObjectAt(ctx, 1)); + assertEquals(o3, midxTip.getObjectAt(ctx, 2)); + assertEquals(o4, midxTip.getObjectAt(ctx, 3)); + // In sha1 order + assertEquals(o5, midxTip.getObjectAt(ctx, 5)); + assertEquals(o6, midxTip.getObjectAt(ctx, 4)); + } + } + + @Test public void midx_hasObject() throws IOException { ObjectId o1 = writePackWithRandomBlob(100); ObjectId o2 = writePackWithRandomBlob(200); @@ -146,9 +184,9 @@ public void midx_hasObject_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -210,9 +248,9 @@ public void midx_get_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -277,9 +315,9 @@ public void midx_load_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -376,9 +414,9 @@ public void midx_resolve_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -446,9 +484,9 @@ public void midx_findAllFromPack_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order (o6 -> o1) - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); List<ObjectToPack> otps = List.of(new DfsObjectToPack(o1, OBJ_BLOB), @@ -524,9 +562,9 @@ public void midx_copyPackAsIs_withBase() throws Exception { .map(size -> size - 12 - 20) // remove header + CRC .sum(); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader(); @@ -561,9 +599,9 @@ public void midx_copyAsIs_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); assertEquals(213, copyAsIs(midxTip, baseObject).length); @@ -633,9 +671,9 @@ public void midx_getObjectType_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 4, 7), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 4), midxBase); try (RevWalk rw = new RevWalk(db); @@ -692,9 +730,9 @@ public void midx_getObjectSize_byId_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 5), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (RevWalk rw = new RevWalk(db); @@ -744,9 +782,9 @@ public void midx_getObjectSize_byOffset_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 5), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (RevWalk rw = new RevWalk(db); @@ -815,9 +853,9 @@ public void midx_fillRepresentation_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(6, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -872,7 +910,8 @@ public void midx_getAllCoveredPacks() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(4, packs.length); - DfsPackFileMidx midx = writeMultipackIndex(packs, null); + DfsPackFileMidx midx = MidxTestUtils.writeMultipackIndex(db, packs, + null); assertEquals(4, midx.getAllCoveredPacks().size()); List<DfsPackDescription> expected = Arrays.stream(packs) @@ -892,11 +931,11 @@ public void midx_getAllCoveredPacks_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(6, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 4, 6), null); - DfsPackFileMidx midxMiddle = writeMultipackIndex( + DfsPackFileMidx midxMiddle = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 2, 4), midxBase); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 2), midxMiddle); assertEquals(6, midxTip.getAllCoveredPacks().size()); @@ -917,9 +956,9 @@ public void midx_getCoveredPacks_withBase_onlyTopMidx() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(6, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 3), midxBase); assertEquals(3, midxTip.getCoveredPacks().size()); @@ -977,6 +1016,24 @@ public void packwriter_via_midx() throws Exception { } @Test + public void getChecksum() throws Exception { + MidxTestUtils.writePackWithBlob(db, "something"); + MidxTestUtils.writePackWithBlob(db, "something else"); + MidxTestUtils.writePackWithBlob(db, "and more"); + DfsPackFileMidx midx = writeMultipackIndex(); + + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + byte[] checksum = midx.getChecksum(ctx); + assertNotNull(checksum); + assertEquals(20, checksum.length); + assertNotEquals('M', checksum[0]); + assertNotEquals('I', checksum[1]); + assertNotEquals('D', checksum[2]); + assertNotEquals('X', checksum[3]); + } + } + + @Test public void voffsetcalculator_encode() { DfsPackFile one = createDfsPackFile(800); DfsPackFile two = createDfsPackFile(1200); @@ -1154,7 +1211,8 @@ private static DfsPackFile createDfsPackFile(int size) { } private DfsPackFileMidx writeMultipackIndex() throws IOException { - return writeMultipackIndex(db.getObjectDatabase().getPacks(), null); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + return MidxTestUtils.writeMultipackIndex(db, packs, null); } private void gcWithBitmaps() throws IOException { @@ -1162,35 +1220,6 @@ private void gcWithBitmaps() throws IOException { garbageCollector.pack(NullProgressMonitor.INSTANCE); } - private DfsPackFileMidx writeMultipackIndex(DfsPackFile[] packs, - DfsPackFileMidx base) throws IOException { - LinkedHashMap<String, PackIndex> forMidx = new LinkedHashMap<>( - packs.length); - Map<String, DfsPackDescription> descByName = new HashMap<>( - packs.length); - try (DfsReader ctx = db.getObjectDatabase().newReader()) { - for (DfsPackFile pack : packs) { - forMidx.put(pack.getPackDescription().getPackName(), - pack.getPackIndex(ctx)); - descByName.put(pack.getPackDescription().getPackName(), - pack.getPackDescription()); - } - } - MultiPackIndexWriter w = new MultiPackIndexWriter(); - DfsPackDescription desc = db.getObjectDatabase().newPack(GC); - try (DfsOutputStream out = db.getObjectDatabase().writeFile(desc, - PackExt.MULTI_PACK_INDEX)) { - MultiPackIndexWriter.Result midxStats = w - .write(NullProgressMonitor.INSTANCE, out, forMidx); - desc.setCoveredPacks(midxStats.packNames().stream() - .map(descByName::get).toList()); - desc.addFileExt(PackExt.MULTI_PACK_INDEX); - } - db.getObjectDatabase().commitPack(List.of(desc), null); - return DfsPackFileMidx.create(DfsBlockCache.getInstance(), desc, - Arrays.asList(packs), base); - } - private RevCommit writePackWithCommit() throws Exception { try (TestRepository<InMemoryRepository> repository = new TestRepository<>( db)) {
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingleTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingleTest.java index 9f698db..aa2ddca 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingleTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingleTest.java
@@ -11,6 +11,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writePackWithBlobs; +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeSinglePackMidx; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; @@ -18,6 +20,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -27,23 +30,16 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.zip.Deflater; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidx.DfsPackOffset; import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidxSingle.SingleVOffsetCalculator; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; -import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset; -import org.eclipse.jgit.internal.storage.midx.MultiPackIndexWriter; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.JGitTestUtil; @@ -76,10 +72,10 @@ public void setUp() { @Test public void findIdxPosition() throws IOException { - ObjectId[] oids = writePackWithBlobs("something", "something else", - "and more"); + ObjectId[] oids = writePackWithBlobs(db, "something", + "something else", "and more"); // oids = [a4..., 33..., 64...] - DfsPackFileMidx midx = writeSinglePackMidx(); + DfsPackFileMidx midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { assertEquals(2, midx.findIdxPosition(ctx, oids[0])); @@ -91,19 +87,20 @@ public void findIdxPosition() throws IOException { @Test public void findIdxPosition_withBase() throws IOException { - ObjectId o1 = writePackWithBlob("o1".getBytes(UTF_8)); // 38 - ObjectId o2 = writePackWithBlob("o2".getBytes(UTF_8)); // 4a - ObjectId o3 = writePackWithBlob("o3".getBytes(UTF_8)); // 45 - ObjectId o4 = writePackWithBlob("o4".getBytes(UTF_8)); // 4b - ObjectId o5 = writePackWithBlob("o5".getBytes(UTF_8)); // 68 - ObjectId o6 = writePackWithBlob("o6".getBytes(UTF_8)); // 4d + ObjectId o1 = MidxTestUtils.writePackWithBlob(db, "o1"); // 38 + ObjectId o2 = MidxTestUtils.writePackWithBlob(db, "o2"); // 4a + ObjectId o3 = MidxTestUtils.writePackWithBlob(db, "o3"); // 45 + ObjectId o4 = MidxTestUtils.writePackWithBlob(db, "o4"); // 4b + ObjectId o5 = MidxTestUtils.writePackWithBlob(db, "o5"); // 68 + ObjectId o6 = MidxTestUtils.writePackWithBlob(db, "o6"); // 4d DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 3, 6), null); - DfsPackFileMidx midxMid = writeSinglePackMidx(packs[2], midxBase); - DfsPackFileMidx midxTip = writeMultipackIndex( + DfsPackFileMidx midxMid = writeSinglePackMidx(db, + packs[2], midxBase); + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 0, 2), midxMid); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -117,9 +114,53 @@ public void findIdxPosition_withBase() throws IOException { } @Test + public void getObjectAt() throws IOException { + ObjectId[] oids = writePackWithBlobs(db, "something", + "something else", "and more"); + // oids = [a4..., 33..., 64...] + DfsPackFileMidx midx = writeSinglePackMidx(db); + + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertEquals(oids[0], midx.getObjectAt(ctx, 2)); + assertEquals(oids[1], midx.getObjectAt(ctx, 0)); + assertEquals(oids[2], midx.getObjectAt(ctx, 1)); + } + } + + @Test + public void getObjectAt_withBase() throws IOException { + ObjectId o1 = MidxTestUtils.writePackWithBlob(db, "o1"); + ObjectId o2 = MidxTestUtils.writePackWithBlob(db, "o2"); + ObjectId o3 = MidxTestUtils.writePackWithBlob(db, "o3"); + ObjectId o4 = MidxTestUtils.writePackWithBlob(db, "o4"); + ObjectId o5 = MidxTestUtils.writePackWithBlob(db, "o5"); + ObjectId o6 = MidxTestUtils.writePackWithBlob(db, "o6"); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + + // Packs are in reverse insertion order + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 3, 6), null); + DfsPackFileMidx midxMid = writeSinglePackMidx(db, + packs[2], midxBase); + DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db, + Arrays.copyOfRange(packs, 0, 2), midxMid); + + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertEquals(o1, midxTip.getObjectAt(ctx, 0)); + assertEquals(o3, midxTip.getObjectAt(ctx, 1)); + assertEquals(o2, midxTip.getObjectAt(ctx, 2)); + assertEquals(o4, midxTip.getObjectAt(ctx, 3)); + // In sha1 order + assertEquals(o6, midxTip.getObjectAt(ctx, 4)); + assertEquals(o5, midxTip.getObjectAt(ctx, 5)); + } + } + + @Test public void hasObject() throws IOException { - ObjectId[] oids = writePackWithBlobs("aaaa", "bbbb", "cccc"); - DfsPackFile midx = writeSinglePackMidx(); + ObjectId[] oids = writePackWithBlobs(db, "aaaa", "bbbb", + "cccc"); + DfsPackFile midx = writeSinglePackMidx(db); // DfsPackFile midx = readDfsPackFileMidx(); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -141,9 +182,10 @@ public void hasObject_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { assertTrue(midxBase.hasObject(ctx, o1)); @@ -172,8 +214,9 @@ public void get() throws IOException { byte[] contentTwo = "TWO".getBytes(UTF_8); byte[] contentThree = "THREE".getBytes(UTF_8); - ObjectId[] oids = writePackWithBlobs("ONE", "TWO", "THREE"); - DfsPackFile midx = writeSinglePackMidx(); + ObjectId[] oids = writePackWithBlobs(db, "ONE", "TWO", + "THREE"); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { ObjectLoader objectLoader = midx.get(ctx, oids[0]); @@ -204,9 +247,10 @@ public void get_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { ObjectLoader objectLoader = midxTip.get(ctx, o1); @@ -232,8 +276,9 @@ public void load() throws IOException { byte[] contentTwo = "TWO".getBytes(UTF_8); byte[] contentThree = "THREE".getBytes(UTF_8); - ObjectId[] oids = writePackWithBlobs("ONE", "TWO", "THREE"); - DfsPackFile midx = writeSinglePackMidx(); + ObjectId[] oids = writePackWithBlobs(db, "ONE", "TWO", + "THREE"); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { ObjectLoader objectLoader = midx.load(ctx, @@ -268,9 +313,10 @@ public void load_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { ObjectLoader objectLoader = midxTip.load(ctx, @@ -293,8 +339,9 @@ public void load_withBase() throws IOException { @Test public void findOffset() throws IOException { - ObjectId[] oids = writePackWithBlobs("ONE", "TWO", "THREE"); - DfsPackFileMidx midx = writeSinglePackMidx(); + ObjectId[] oids = writePackWithBlobs(db, "ONE", "TWO", + "THREE"); + DfsPackFileMidx midx = writeSinglePackMidx(db); DfsPackFile realPack = findPack(oids[0]); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -328,9 +375,10 @@ public void findOffset_withBase() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); DfsPackFile coveredPack = findPack(o6); try (DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -363,8 +411,9 @@ public void findOffset_withBase() throws IOException { @Test public void resolve() throws Exception { // These oids do NOT have same prefix - ObjectId[] oids = writePackWithBlobs("AAAAAA", "BBBBBB", "CCCCCCC"); - DfsPackFile midx = writeSinglePackMidx(); + ObjectId[] oids = writePackWithBlobs(db, "AAAAAA", + "BBBBBB", "CCCCCCC"); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { Set<ObjectId> matches = new HashSet<>(); @@ -392,9 +441,10 @@ public void resolve_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { Set<ObjectId> matches = new HashSet<>(); @@ -423,8 +473,8 @@ public void resolve_withBase() throws Exception { @Test public void findAllFromPack() throws Exception { - ObjectId[] objectIds = writePackWithBlobs("aaaaaaaa", "bbbbbbbbb", - "cccccccccc"); + ObjectId[] objectIds = writePackWithBlobs(db, "aaaaaaaa", + "bbbbbbbbb", "cccccccccc"); DfsPackFile midx = writeMultipackIndex(); List<ObjectToPack> otps = List.of( @@ -463,9 +513,10 @@ public void findAllFromPack_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order (o6 -> o1) - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); List<ObjectToPack> otps = List.of(new DfsObjectToPack(o1, OBJ_BLOB), new DfsObjectToPack(o4, OBJ_BLOB), @@ -504,9 +555,9 @@ public void findAllFromPack_withBase() throws Exception { @Test public void copyPackAsIs() throws Exception { - ObjectId[] objectIds = writePackWithBlobs("aaaaaa", "bbbbbbbb", - "ccccccccc"); - DfsPackFileMidx midx = writeSinglePackMidx(); + ObjectId[] objectIds = writePackWithBlobs(db, "aaaaaa", + "bbbbbbbb", "ccccccccc"); + DfsPackFileMidx midx = writeSinglePackMidx(db); DfsPackFile pack = findPack(objectIds[0]); assertArrayEquals(copyPackAsIs(pack), copyPackAsIs(midx)); } @@ -526,9 +577,10 @@ public void copyPackAsIs_withBase() throws Exception { .map(size -> size - 12 - 20) // remove header + CRC .sum(); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader(); PackWriter pw = new PackWriter(new PackConfig(), ctx); @@ -556,7 +608,7 @@ public void copyPackAsIs_withBase() throws Exception { @Test public void copyAsIs() throws Exception { ObjectId blob = writePackWithRandomBlob(200); - DfsPackFileMidx midx = writeSinglePackMidx(); + DfsPackFileMidx midx = writeSinglePackMidx(db); assertEquals(213, copyAsIs(midx, blob).length); } @@ -567,14 +619,15 @@ public void copyAsIs_withBase() throws Exception { ObjectId baseObject = writePackWithRandomBlob(200); writePackWithRandomBlob(150); writePackWithRandomBlob(400); - writePackWithBlob("woohooABCxxxxx11111".getBytes(UTF_8)); + MidxTestUtils.writePackWithBlob(db, "woohooABCxxxxx11111"); ObjectId tipObject = writePackWithRandomBlob(600); DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); assertEquals(213, copyAsIs(midxTip, baseObject).length); @@ -613,7 +666,7 @@ public void getDeltaHeader() { public void getObjectType() throws Exception { CommitObjects commitObjects = writePackWithOneCommit(); gcWithBitmaps(); - DfsPackFile midx = writeSinglePackMidx(); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { long commitPos = midx.findOffset(ctx, commitObjects.commit()); @@ -642,9 +695,10 @@ public void getObjectType_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 7), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { long commitPos = midxTip.findOffset(ctx, objs.commit()); @@ -668,7 +722,7 @@ public void getObjectType_withBase() throws Exception { public void getObjectSize_byId() throws Exception { CommitObjects objs = writePackWithOneCommit(); gcWithBitmaps(); - DfsPackFile midx = writeSinglePackMidx(); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { assertEquals(168, midx.getObjectSize(ctx, objs.commit())); @@ -689,9 +743,10 @@ public void getObjectSize_byId_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 5), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (RevWalk rw = new RevWalk(db); DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -711,7 +766,7 @@ public void getObjectSize_byId_withBase() throws Exception { public void getObjectSize_byOffset() throws Exception { ObjectId commit = writePackWithCommit(); gcWithBitmaps(); - DfsPackFile midx = writeSinglePackMidx(); + DfsPackFile midx = writeSinglePackMidx(db); try (RevWalk rw = new RevWalk(db); DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -737,9 +792,10 @@ public void getObjectSize_byOffset_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 5), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (RevWalk rw = new RevWalk(db); DfsReader ctx = db.getObjectDatabase().newReader()) { @@ -764,7 +820,7 @@ public void getObjectSize_byOffset_withBase() throws Exception { @Test public void objectSizeIndex_disabled() throws Exception { writePackWithRandomBlob(200); - DfsPackFile midx = writeSinglePackMidx(); + DfsPackFile midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { assertFalse(midx.hasObjectSizeIndex(ctx)); @@ -775,7 +831,7 @@ public void objectSizeIndex_disabled() throws Exception { public void fillRepresentation() throws Exception { RevCommit commit = writePackWithCommit(); gcWithBitmaps(); - DfsPackFile midx = writeSinglePackMidx(); + DfsPackFile midx = writeSinglePackMidx(db); DfsObjectRepresentation rep = fillRepresentation(midx, commit, OBJ_COMMIT); @@ -806,9 +862,10 @@ public void fillRepresentation_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(6, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 6), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); try (DfsReader ctx = db.getObjectDatabase().newReader()) { // Commit in tip midx @@ -841,7 +898,7 @@ public void getBitmapIndex() throws Exception { RevCommit c2 = writePackWithCommit(); gcWithBitmaps(); - DfsPackFileMidx midx = writeSinglePackMidx(); + DfsPackFileMidx midx = writeSinglePackMidx(db); try (DfsReader ctx = db.getObjectDatabase().newReader()) { PackBitmapIndex bitmapIndex = midx.getBitmapIndex(ctx); assertNotNull(bitmapIndex); @@ -853,9 +910,9 @@ public void getBitmapIndex() throws Exception { @Test public void getAllCoveredPacks() throws Exception { - ObjectId[] objectIds = writePackWithBlobs("aaaaaaaa", "bbbbbbb", - "ccccccc"); - DfsPackFileMidx midx = writeSinglePackMidx(); + ObjectId[] objectIds = writePackWithBlobs(db, "aaaaaaaa", + "bbbbbbb", "ccccccc"); + DfsPackFileMidx midx = writeSinglePackMidx(db); DfsPackFile realPack = findPack(objectIds[0]); assertEquals(1, midx.getAllCoveredPacks().size()); @@ -872,9 +929,10 @@ public void getAllCoveredPacks_withBase() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(5, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 5), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); assertEquals(5, midxTip.getAllCoveredPacks().size()); List<DfsPackDescription> expected = Arrays.stream(packs) @@ -893,9 +951,10 @@ public void getCoveredPacks_withBase_onlyTopMidx() throws Exception { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); assertEquals(5, packs.length); - DfsPackFileMidx midxBase = writeMultipackIndex( + DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db, Arrays.copyOfRange(packs, 1, 5), null); - DfsPackFileMidx midxTip = writeSinglePackMidx(packs[0], midxBase); + DfsPackFileMidx midxTip = writeSinglePackMidx(db, + packs[0], midxBase); assertEquals(1, midxTip.getCoveredPacks().size()); assertEquals(packs[0].getPackDescription(), @@ -905,7 +964,8 @@ public void getCoveredPacks_withBase_onlyTopMidx() throws Exception { @Test public void corrupt() throws Exception { RevCommit commit = writePackWithCommit(); - DfsPackFileMidx midx = writeSinglePackMidx(findPack(commit)); + DfsPackFile pack = findPack(commit); + DfsPackFileMidx midx = writeSinglePackMidx(db, pack); try (DfsReader ctx = db.getObjectDatabase().newReader()) { assertFalse(midx.isCorrupt(midx.findOffset(ctx, commit))); } @@ -913,13 +973,13 @@ public void corrupt() throws Exception { @Test public void packwriter_via_midx() throws Exception { - ObjectId[] oids = writePackWithBlobs("xxxxxxxyyyy", "booooohooooo", - "baaaaahaaaaaa"); + ObjectId[] oids = writePackWithBlobs(db, "xxxxxxxyyyy", + "booooohooooo", "baaaaahaaaaaa"); ObjectId blob = oids[0]; ObjectId blobTwo = oids[1]; ObjectId notPacked = oids[2]; - writeSinglePackMidx(); + writeSinglePackMidx(db); db.getObjectDatabase().setUseMultipackIndex(true); byte[] writtenPack; @@ -951,6 +1011,26 @@ public void packwriter_via_midx() throws Exception { } @Test + public void getChecksum() throws Exception { + writePackWithBlobs(db, "something", "something else", "and more"); + DfsPackFileMidx midx = writeSinglePackMidx(db); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + // The checksum includes packnames and can + // change between runs (e.g. running this test alone, or locally vs + // jenkins). + // Check at least that we don't get an empty checksum or the first + // bytes of the midx. + byte[] checksum = midx.getChecksum(ctx); + assertNotNull(checksum); + assertEquals(20, checksum.length); + assertNotEquals('M', checksum[0]); + assertNotEquals('I', checksum[1]); + assertNotEquals('D', checksum[2]); + assertNotEquals('X', checksum[3]); + } + } + + @Test public void voffsetcalculator_encode() { DfsPackFile pack = createDfsPackFile(900); SingleVOffsetCalculator calc = new SingleVOffsetCalculator(pack, null); @@ -1061,23 +1141,8 @@ private static DfsPackFile createDfsPackFile(int size) { } private DfsPackFileMidx writeMultipackIndex() throws IOException { - return writeMultipackIndex(db.getObjectDatabase().getPacks(), null); - } - - private DfsPackFileMidx writeSinglePackMidx() throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); - assertEquals("More than one pack in db", 1, packs.length); - return writeSinglePackMidx(packs[0]); - } - - private DfsPackFileMidx writeSinglePackMidx(DfsPackFile pack) - throws IOException { - return writeMultipackIndex(new DfsPackFile[] { pack }, null); - } - - private DfsPackFileMidx writeSinglePackMidx(DfsPackFile pack, - DfsPackFileMidx base) throws IOException { - return writeMultipackIndex(new DfsPackFile[] { pack }, base); + return MidxTestUtils.writeMultipackIndex(db, packs, null); } private void gcWithBitmaps() throws IOException { @@ -1085,35 +1150,6 @@ private void gcWithBitmaps() throws IOException { garbageCollector.pack(NullProgressMonitor.INSTANCE); } - private DfsPackFileMidx writeMultipackIndex(DfsPackFile[] packs, - DfsPackFileMidx base) throws IOException { - LinkedHashMap<String, PackIndex> forMidx = new LinkedHashMap<>( - packs.length); - Map<String, DfsPackDescription> descByName = new HashMap<>( - packs.length); - try (DfsReader ctx = db.getObjectDatabase().newReader()) { - for (DfsPackFile pack : packs) { - forMidx.put(pack.getPackDescription().getPackName(), - pack.getPackIndex(ctx)); - descByName.put(pack.getPackDescription().getPackName(), - pack.getPackDescription()); - } - } - MultiPackIndexWriter w = new MultiPackIndexWriter(); - DfsPackDescription desc = db.getObjectDatabase().newPack(GC); - try (DfsOutputStream out = db.getObjectDatabase().writeFile(desc, - PackExt.MULTI_PACK_INDEX)) { - MultiPackIndexWriter.Result midxStats = w - .write(NullProgressMonitor.INSTANCE, out, forMidx); - desc.setCoveredPacks(midxStats.packNames().stream() - .map(descByName::get).toList()); - desc.addFileExt(PackExt.MULTI_PACK_INDEX); - } - db.getObjectDatabase().commitPack(List.of(desc), null); - return DfsPackFileMidx.create(DfsBlockCache.getInstance(), desc, - Arrays.asList(packs), base); - } - private RevCommit writePackWithCommit() throws Exception { try (TestRepository<InMemoryRepository> repository = new TestRepository<>( db)) { @@ -1153,18 +1189,6 @@ private ObjectId writePackWithBlob(byte[] data) throws IOException { return blobId; } - private ObjectId[] writePackWithBlobs(String... blobs) throws IOException { - ObjectId[] oids = new ObjectId[blobs.length]; - - DfsInserter ins = (DfsInserter) db.newObjectInserter(); - ins.setCompressionLevel(Deflater.NO_COMPRESSION); - for (int i = 0; i < blobs.length; i++) { - oids[i] = ins.insert(OBJ_BLOB, blobs[i].getBytes(UTF_8)); - } - ins.flush(); - return oids; - } - private DfsPackFile findPack(ObjectId oid) throws IOException { DfsPackFile[] packs = db.getObjectDatabase().getPacks(); try (DfsReader ctx = db.getObjectDatabase().newReader()) {
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilderTest.java new file mode 100644 index 0000000..10d9c2e --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilderTest.java
@@ -0,0 +1,318 @@ +/* + * Copyright (C) 2026, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.internal.storage.pack.BitmapCommit; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.lib.BitmapIndex; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import com.googlecode.javaewah.EWAHCompressedBitmap; + +@RunWith(Parameterized.class) +public class PackBitmapIndexBuilderTest { + + /** + * Numbers are oids + * + * <pre> + * tip + * 10 (commit) -> 9 (tree) -> 8 (blob) + * | + * 7 (commit) -> 6 (tree) -> 5,4 (blobs) + * | + * 3 (commit) -> 2 (tree) -> 1 (blob) + * </pre> + */ + private static final List<ObjectToPack> opts = List.of( + otp(10, 400, Constants.OBJ_COMMIT), otp(9, 900, Constants.OBJ_TREE), + otp(8, 280, Constants.OBJ_BLOB), otp(7, 200, Constants.OBJ_COMMIT), + otp(6, 410, Constants.OBJ_TREE), otp(5, 500, Constants.OBJ_BLOB), + otp(4, 450, Constants.OBJ_BLOB), otp(3, 470, Constants.OBJ_COMMIT), + otp(2, 700, Constants.OBJ_TREE), otp(1, 240, Constants.OBJ_BLOB)); + + public enum TestSetup { + SINGLE_BUILDER + } + + @Parameterized.Parameters(name = "{0}") + public static Iterable<TestSetup> data() { + return Arrays.stream(TestSetup.values()).toList(); + } + + private final TestSetup currentSetup; + + public PackBitmapIndexBuilderTest(TestSetup ts) { + this.currentSetup = ts; + } + + @Test + public void getObjectSet() { + ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> objectSet = createBuilder() + .getObjectSet(); + for (int i = 1; i <= 10; i++) { + assertTrue(objectSet.contains(oid(i))); + } + } + + @Test + public void addBitmap_getBitmap() { + EWAHCompressedBitmap added = EWAHCompressedBitmap.bitmapOf(1, 3, 5); + PackBitmapIndexBuilder pbi = createBuilder(); + pbi.addBitmap(oid(11), added, 10); + + EWAHCompressedBitmap retrieved = pbi.getBitmap(oid(11)); + assertEquals(added, retrieved); + } + + @Test + public void ofObjectType() { + PackBitmapIndexBuilder pbi = createBuilder(); + BitmapIndexImpl bii = new BitmapIndexImpl(pbi); + BitmapIndex.BitmapBuilder bb = bii.newBitmapBuilder(); + bb.addObject(oid(10), Constants.OBJ_COMMIT); + bb.addObject(oid(9), Constants.OBJ_TREE); + bb.addObject(oid(8), Constants.OBJ_BLOB); + BitmapIndex.Bitmap b = bb.build(); + + EWAHCompressedBitmap commits = pbi.ofObjectType(b.retrieveCompressed(), + Constants.OBJ_COMMIT); + assertInBitmap(pbi, commits, 10); + assertNotInBitmap(pbi, commits, 9); + assertNotInBitmap(pbi, commits, 8); + + EWAHCompressedBitmap trees = pbi.ofObjectType(b.retrieveCompressed(), + Constants.OBJ_TREE); + assertNotInBitmap(pbi, trees, 10); + assertInBitmap(pbi, trees, 9); + assertNotInBitmap(pbi, trees, 8); + + EWAHCompressedBitmap blobs = pbi.ofObjectType(b.retrieveCompressed(), + Constants.OBJ_BLOB); + assertNotInBitmap(pbi, blobs, 10); + assertNotInBitmap(pbi, blobs, 9); + assertInBitmap(pbi, blobs, 8); + } + + @Test + public void findPosition() { + PackBitmapIndexBuilder pbi = createBuilder(); + // All valid, non-repeating positions + Set<Integer> seen = new HashSet<>(); + for (int oidAsInt = 1; oidAsInt <= 10; oidAsInt++) { + int p = pbi.findPosition(oid(oidAsInt)); + assertTrue(p >= 0); + assertTrue(p < 10); + assertFalse(seen.contains(p)); + seen.add(p); + } + } + + @Test + public void getObject() { + PackBitmapIndexBuilder pbi = createBuilder(); + for (int oidAsInt = 1; oidAsInt <= 10; oidAsInt++) { + ObjectId expected = oid(oidAsInt); + int p = pbi.findPosition(expected); + ObjectId actual = pbi.getObject(p); + assertEquals(expected, actual); + } + } + + @Test + public void commits() { + PackBitmapIndexBuilder pbi = createBuilder(); + EWAHCompressedBitmap commits = pbi.getCommits(); + assertInBitmap(pbi, commits, 10); + assertInBitmap(pbi, commits, 7); + assertInBitmap(pbi, commits, 3); + + assertNotInBitmap(pbi, commits, 9); + assertNotInBitmap(pbi, commits, 8); + } + + @Test + public void trees() { + PackBitmapIndexBuilder pbi = createBuilder(); + EWAHCompressedBitmap trees = pbi.getTrees(); + assertInBitmap(pbi, trees, 9); + assertInBitmap(pbi, trees, 6); + assertInBitmap(pbi, trees, 2); + + assertNotInBitmap(pbi, trees, 10); + assertNotInBitmap(pbi, trees, 8); + } + + @Test + public void blobs() { + PackBitmapIndexBuilder pbi = createBuilder(); + EWAHCompressedBitmap blobs = pbi.getBlobs(); + assertInBitmap(pbi, blobs, 8); + assertInBitmap(pbi, blobs, 5); + assertInBitmap(pbi, blobs, 4); + assertInBitmap(pbi, blobs, 1); + + assertNotInBitmap(pbi, blobs, 10); + assertNotInBitmap(pbi, blobs, 9); + } + + @Test + public void getBitmapCount() { + // The builder only counts bitmaps "in progress" (in xor queue or ready + // to write) + assertEquals(0, createBuilder().getBitmapCount()); + } + + @Test + public void getBitmapCount_newBitmaps() { + + } + + @Test + public void reachableObjects_setInBitmap() { + PackBitmapIndexBuilder builder = createBuilder(); + EWAHCompressedBitmap bitmapTen = builder.getBitmap(oid(10)); + assertNotNull(bitmapTen); + for (int oidAsInt = 10; oidAsInt > 0; oidAsInt--) { + assertInBitmap(builder, bitmapTen, oidAsInt); + } + + EWAHCompressedBitmap bitmapSeven = builder.getBitmap(oid(7)); + assertNotNull(bitmapSeven); + assertNotInBitmap(builder, bitmapSeven, 10); + assertNotInBitmap(builder, bitmapSeven, 9); + assertNotInBitmap(builder, bitmapSeven, 8); + for (int i = 7; i > 0; i--) { + assertInBitmap(builder, bitmapSeven, i); + } + } + + @Test + public void processBitmapForWrite() { + // processBitmapForWrite keeps the bitmaps in the builder instead of in + // the superclass, to calculate the XORs + PackBitmapIndexBuilder builder = createBuilder(); + BitmapIndex bb = new BitmapIndexImpl(builder); + BitmapIndex.BitmapBuilder oneBitmap = bb.newBitmapBuilder(); + oneBitmap.addObject(oid(10), Constants.OBJ_COMMIT); + oneBitmap.addObject(oid(9), Constants.OBJ_TREE); + oneBitmap.addObject(oid(8), Constants.OBJ_BLOB); + oneBitmap.addObject(oid(7), Constants.OBJ_COMMIT); + oneBitmap.addObject(oid(6), Constants.OBJ_TREE); + oneBitmap.addObject(oid(5), Constants.OBJ_BLOB); + + BitmapIndex.BitmapBuilder anotherBitmap = bb.newBitmapBuilder(); + anotherBitmap.addObject(oid(7), Constants.OBJ_COMMIT); + anotherBitmap.addObject(oid(6), Constants.OBJ_TREE); + anotherBitmap.addObject(oid(5), Constants.OBJ_BLOB); + + builder.processBitmapForWrite( + new BitmapCommit(oid(10), false, 0, false), oneBitmap, 0); + builder.processBitmapForWrite(new BitmapCommit(oid(7), false, 0, false), + oneBitmap, 0); + assertEquals(2, builder.getBitmapCount()); + + List<PackBitmapIndexBuilder.StoredEntry> ses = builder + .getCompressedBitmaps(); + assertEquals(2, ses.size()); + assertEquals(oid(7), ses.get(0).getObjectId()); + assertEquals(6, ses.get(0).getIdxPosition()); + + assertEquals(oid(10), ses.get(1).getObjectId()); + assertEquals(9, ses.get(1).getIdxPosition()); + } + + @Test + public void getObjectCount() { + assertEquals(10, createBuilder().getObjectCount()); + } + + private static void assertInBitmap(PackBitmapIndexBuilder builder, + EWAHCompressedBitmap bitmap, int oidAsInt) { + ObjectId oid = oid(oidAsInt); + int pos = builder.findPosition(oid); + assertTrue(pos >= 0); + assertTrue(bitmap.get(pos)); + } + + private static void assertNotInBitmap(PackBitmapIndexBuilder builder, + EWAHCompressedBitmap bitmap, int oidAsInt) { + ObjectId oid = oid(oidAsInt); + int pos = builder.findPosition(oid); + assertTrue(pos >= 0); + assertFalse(bitmap.get(pos)); + } + + private PackBitmapIndexBuilder createBuilder() { + ArrayList<ObjectToPack> objectsToPack = new ArrayList<>(opts); + Collections.reverse(objectsToPack); + return switch (currentSetup) { + case SINGLE_BUILDER -> setupSinglePackBitmapIndex(objectsToPack); + }; + } + + static PackBitmapIndexBuilder setupSinglePackBitmapIndex( + List<ObjectToPack> objectsToPack) { + PackBitmapIndexBuilder b = new PackBitmapIndexBuilder(objectsToPack); + + BitmapIndex bi = new BitmapIndexImpl(b); + // Base commit (3, 2, 1) + BitmapIndex.BitmapBuilder bitmapOne = bi.newBitmapBuilder(); + bitmapOne.addObject(oid(3), Constants.OBJ_COMMIT); + bitmapOne.addObject(oid(2), Constants.OBJ_TREE); + bitmapOne.addObject(oid(1), Constants.OBJ_BLOB); + b.addBitmap(oid(3), bitmapOne.build(), 0); + + // Middle commit (7, 6, 5, 4) with base commit as parent + BitmapIndex.BitmapBuilder middleBitmap = bi.newBitmapBuilder(); + middleBitmap.addObject(oid(7), Constants.OBJ_COMMIT); + middleBitmap.addObject(oid(6), Constants.OBJ_TREE); + middleBitmap.addObject(oid(5), Constants.OBJ_BLOB); + middleBitmap.addObject(oid(4), Constants.OBJ_BLOB); + middleBitmap.or(bitmapOne); + b.addBitmap(oid(7), middleBitmap, 0); + + BitmapIndex.BitmapBuilder tipBitmap = bi.newBitmapBuilder(); + tipBitmap.addObject(oid(10), Constants.OBJ_COMMIT); + tipBitmap.addObject(oid(9), Constants.OBJ_TREE); + tipBitmap.addObject(oid(8), Constants.OBJ_BLOB); + tipBitmap.or(middleBitmap); + b.addBitmap(oid(10), tipBitmap, 0); + + return b; + } + + private static ObjectToPack otp(int oid, long offset, int type) { + ObjectToPack otp = new ObjectToPack(oid(oid), type); + otp.setOffset(offset); + return otp; + } + + private static ObjectId oid(int n) { + return ObjectId.fromRaw(new int[] { 0, 0, 0, 0, n }); + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java index 6436e31..3a90e21 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java
@@ -126,6 +126,7 @@ public void basicMidx() throws IOException { 1502); assertNull(midx.find(ObjectId.zeroId())); + assertNotNull(midx.getChecksum()); } @Test @@ -387,24 +388,24 @@ public void jgit_findPosition() throws IOException { MultiPackIndex midx = MultiPackIndexLoader .read(new ByteArrayInputStream(out.toByteArray())); assertEquals(3, midx.getPackNames().length); - assertEquals(0, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000001"))); - assertEquals(1, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000002"))); - assertEquals(2, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000003"))); - assertEquals(3, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000004"))); - assertEquals(4, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000005"))); - assertEquals(5, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000007"))); - assertEquals(6, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000010"))); - assertEquals(7, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000012"))); - assertEquals(8, midx.findPosition(ObjectId - .fromString("0000000000000000000000000000000000000015"))); + assertEquals(0, midx.findPosition(oid("001"))); + assertEquals(oid("001"), midx.getObjectAt(0)); + assertEquals(1, midx.findPosition(oid("002"))); + assertEquals(oid("002"), midx.getObjectAt(1)); + assertEquals(2, midx.findPosition(oid("003"))); + assertEquals(oid("003"), midx.getObjectAt(2)); + assertEquals(3, midx.findPosition(oid("004"))); + assertEquals(oid("004"), midx.getObjectAt(3)); + assertEquals(4, midx.findPosition(oid("005"))); + assertEquals(oid("005"), midx.getObjectAt(4)); + assertEquals(5, midx.findPosition(oid("007"))); + assertEquals(oid("007"), midx.getObjectAt(5)); + assertEquals(6, midx.findPosition(oid("010"))); + assertEquals(oid("010"), midx.getObjectAt(6)); + assertEquals(7, midx.findPosition(oid("012"))); + assertEquals(oid("012"), midx.getObjectAt(7)); + assertEquals(8, midx.findPosition(oid("015"))); + assertEquals(oid("015"), midx.getObjectAt(8)); assertNull(midx.find(ObjectId.zeroId())); } @@ -460,6 +461,129 @@ public void jgit_getObjectCount_emtpy() throws IOException { assertEquals(0, midx.getObjectCount()); } + @Test + public void jgit_findBitmapPosition() throws IOException { + PackIndex idxOne = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000001", 500), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000005", 12), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000010", 1500))); + PackIndex idxTwo = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000002", 501), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000003", 13), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000015", 1501))); + PackIndex idxThree = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000004", 502), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000007", 14), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000012", 1502))); + + LinkedHashMap<String, PackIndex> packs = orderedMapOf("p1", idxOne, + "p2", idxTwo, "p3", idxThree); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + MultiPackIndex.PackOffset packOffset; + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000005")); + assertEquals(0, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000001")); + assertEquals(1, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000010")); + assertEquals(2, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000003")); + assertEquals(3, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000002")); + assertEquals(4, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000015")); + assertEquals(5, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000007")); + assertEquals(6, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000004")); + assertEquals(7, midx.findBitmapPosition(packOffset)); + packOffset = midx.find(ObjectId + .fromString("0000000000000000000000000000000000000012")); + assertEquals(8, midx.findBitmapPosition(packOffset)); + } + + @Test + public void jgit_getObjectAtBitmapPosition() throws IOException { + PackIndex idxOne = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000001", 500), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000005", 12), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000010", 1500))); + PackIndex idxTwo = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000002", 501), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000003", 13), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000015", 1501))); + PackIndex idxThree = FakeIndexFactory.indexOf(List.of( + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000004", 502), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000007", 14), + new FakeIndexFactory.IndexObject( + "0000000000000000000000000000000000000012", 1502))); + + LinkedHashMap<String, PackIndex> packs = orderedMapOf("p1", idxOne, + "p2", idxTwo, "p3", idxThree); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000005"), + midx.getObjectAtBitmapPosition(0)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000001"), + midx.getObjectAtBitmapPosition(1)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000010"), + midx.getObjectAtBitmapPosition(2)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000003"), + midx.getObjectAtBitmapPosition(3)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000002"), + midx.getObjectAtBitmapPosition(4)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000015"), + midx.getObjectAtBitmapPosition(5)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000007"), + midx.getObjectAtBitmapPosition(6)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000004"), + midx.getObjectAtBitmapPosition(7)); + assertEquals( + ObjectId.fromString("0000000000000000000000000000000000000012"), + midx.getObjectAtBitmapPosition(8)); + } + private static PackIndex indexWith(String... oids) { List<FakeIndexFactory.IndexObject> idxObjs = new ArrayList<>( oids.length); @@ -497,4 +621,10 @@ private static LinkedHashMap<String, PackIndex> orderedMapOf(String s1, map.put(s3, pi3); return map; } + + private static ObjectId oid(String last3chars) { + assertEquals(3, last3chars.length()); + return ObjectId.fromString( + "0000000000000000000000000000000000000" + last3chars); + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java index 28ef98d..8302694 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java
@@ -68,6 +68,7 @@ public static DfsPackDescription writeMidx(ProgressMonitor pm, MultiPackIndexWriter w = new MultiPackIndexWriter(); MultiPackIndexWriter.Result result = w.write(pm, out, inputs); midxPackDesc.addFileExt(MULTI_PACK_INDEX); + midxPackDesc.setFileSize(MULTI_PACK_INDEX, result.bytesWritten()); midxPackDesc.setObjectCount(result.objectCount()); Map<String, DfsPackDescription> byName = packs.stream()
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index ecc97e9..4997b90 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
@@ -241,20 +241,6 @@ void setPackIndex(PackIndex idx) { } /** - * Get the PackIndex for this PackFile. - * - * @param ctx - * reader context to support reading from the backing store if - * the index is not already loaded in memory. - * @return the PackIndex. - * @throws java.io.IOException - * the pack index is not available, or is corrupt. - */ - public PackIndex getPackIndex(DfsReader ctx) throws IOException { - return idx(ctx); - } - - /** * Get a view of this packfile as a set of objects * <p> * To use when the caller only needs to check inclusion (without specific @@ -267,10 +253,20 @@ public PackIndex getPackIndex(DfsReader ctx) throws IOException { * cannot load the backing data from storage */ public ObjectIdSet asObjectIdSet(DfsReader ctx) throws IOException { - return idx(ctx); + return getPackIndex(ctx); } - private PackIndex idx(DfsReader ctx) throws IOException { + /** + * Get the PackIndex for this PackFile. + * + * @param ctx + * reader context to support reading from the backing store if + * the index is not already loaded in memory. + * @return the PackIndex. + * @throws java.io.IOException + * the pack index is not available, or is corrupt. + */ + public PackIndex getPackIndex(DfsReader ctx) throws IOException { if (index != null) { return index; } @@ -399,7 +395,8 @@ public PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException { return reverseIndex; } - reverseIndex = indexFactory.getPackIndexes().reverseIndex(ctx, idx(ctx)); + reverseIndex = indexFactory.getPackIndexes().reverseIndex(ctx, + getPackIndex(ctx)); if (reverseIndex == null) { throw new IOException( "Couldn't get a reference to the reverse index"); //$NON-NLS-1$ @@ -460,12 +457,12 @@ private PackObjectSizeIndex getObjectSizeIndex(DfsReader ctx) * the pack index is not available, or is corrupt. */ public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException { - final long offset = idx(ctx).findOffset(id); + final long offset = getPackIndex(ctx).findOffset(id); return 0 < offset && !isCorrupt(offset); } int findIdxPosition(DfsReader ctx, AnyObjectId id) throws IOException { - return idx(ctx).findPosition(id); + return getPackIndex(ctx).findPosition(id); } /** @@ -482,12 +479,12 @@ int findIdxPosition(DfsReader ctx, AnyObjectId id) throws IOException { */ ObjectLoader get(DfsReader ctx, AnyObjectId id) throws IOException { - long offset = idx(ctx).findOffset(id); + long offset = getPackIndex(ctx).findOffset(id); return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null; } long findOffset(DfsReader ctx, AnyObjectId id) throws IOException { - return idx(ctx).findOffset(id); + return getPackIndex(ctx).findOffset(id); } /** @@ -513,7 +510,7 @@ List<DfsObjectToPack> findAllFromPack(DfsReader ctx, if (skipFound && otp.isFound()) { continue; } - long p = idx(ctx).findOffset(otp); + long p = getPackIndex(ctx).findOffset(otp); if (p <= 0 || isCorrupt(p)) { continue; } @@ -526,7 +523,7 @@ List<DfsObjectToPack> findAllFromPack(DfsReader ctx, void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit) throws IOException { - idx(ctx).resolve(matches, id, matchLimit); + getPackIndex(ctx).resolve(matches, id, matchLimit); } private byte[] decompress(long position, int sz, DfsReader ctx) @@ -703,11 +700,11 @@ void copyAsIs(PackOutputStream out, DfsObjectToPack src, try { quickCopy = ctx.quickCopy(this, dataOffset, dataLength); - if (validate && idx(ctx).hasCRC32Support()) { + if (validate && getPackIndex(ctx).hasCRC32Support()) { assert(crc1 != null); // Index has the CRC32 code cached, validate the object. // - expectedCRC = idx(ctx).findCRC32(src); + expectedCRC = getPackIndex(ctx).findCRC32(src); if (quickCopy != null) { quickCopy.crc32(crc1, dataOffset, (int) dataLength); } else { @@ -1000,7 +997,7 @@ ObjectLoader load(DfsReader ctx, long pos) private long findDeltaBase(DfsReader ctx, ObjectId baseId) throws IOException, MissingObjectException { - long ofs = idx(ctx).findOffset(baseId); + long ofs = getPackIndex(ctx).findOffset(baseId); if (ofs < 0) { throw new MissingObjectException(baseId, JGitText.get().missingDeltaBase); @@ -1094,7 +1091,7 @@ int getObjectType(DfsReader ctx, long pos) throws IOException { } long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException { - final long offset = idx(ctx).findOffset(id); + final long offset = getPackIndex(ctx).findOffset(id); return 0 < offset ? getObjectSize(ctx, offset) : -1; } @@ -1545,7 +1542,8 @@ public LoadResult loadPackBitmapIndex(DfsReader ctx, DfsPackFile pack) PackBitmapIndex bmidx; try { bmidx = PackBitmapIndex.read(alignTo8kBlocks(rc), - () -> pack.idx(ctx), () -> pack.getReverseIdx(ctx), + () -> pack.getPackIndex(ctx), + () -> pack.getReverseIdx(ctx), ctx.getOptions().shouldLoadRevIndexInParallel()); } finally { size = rc.position();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java index 0f96593..ba4d5f6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
@@ -11,7 +11,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.zip.DataFormatException; import org.eclipse.jgit.annotations.Nullable; @@ -19,6 +21,9 @@ import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; /** @@ -104,6 +109,24 @@ public List<DfsPackFile> getAllCoveredPacks() { } /** + * Get the objectId at the corresponding position in the midx chain up to + * this point + * <p> + * In a chain with midx-tip (100 objects) and midx-base (50 objects), + * positions 0-49 belong to the base midx and 50-149 to the tip midx. + * + * @param ctx + * a reader for the midx data + * @param nthPosition + * position in midx chain + * @return the objectId + * @throws IOException + * a problem reading midx bytes + */ + abstract ObjectId getObjectAt(DfsReader ctx, long nthPosition) + throws IOException; + + /** * Count of objects in this <b>pack</b> (i.e. including, recursively, its * base) * @@ -117,17 +140,25 @@ protected int getObjectCount(DfsReader ctx) throws IOException { return (int) getPackDescription().getObjectCount(); } + /** + * Return checksum of the midx + * + * @param ctx + * a reader + * @return checksum of the midx + * @throws IOException + * an error reading the file + */ + protected abstract byte[] getChecksum(DfsReader ctx) throws IOException; + @Override - public PackIndex getPackIndex(DfsReader ctx) { - throw new IllegalStateException( - "Shouldn't use multipack index if the primary index is needed"); //$NON-NLS-1$ + public final PackIndex getPackIndex(DfsReader ctx) { + return new MidxPackIndex(this, ctx); } @Override - public PackReverseIndex getReverseIdx(DfsReader ctx) { - throw new IllegalStateException( - "Shouldn't use multipack index if the reverse index is needed"); //$NON-NLS-1$ - } + public abstract PackReverseIndex getReverseIdx(DfsReader ctx) + throws IOException; @Override ObjectLoader load(DfsReader ctx, long midxOffset) throws IOException { @@ -335,4 +366,101 @@ long getPackOffset() { return midxOffset - packStart; } } + + private static class MidxPackIndex implements PackIndex { + + private final DfsPackFileMidx pack; + + private final DfsReader ctx; + + MidxPackIndex(DfsPackFileMidx pack, DfsReader ctx) { + this.pack = pack; + this.ctx = ctx; + } + + @Override + public Iterator<MutableEntry> iterator() { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public long getObjectCount() { + try { + return pack.getObjectCount(ctx); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long getOffset64Count() { + // TODO(ifrade): This method seems to be used only for stats. + // Maybe we can just remove it. + return 0; + } + + @Override + public ObjectId getObjectId(long nthPosition) { + try { + return pack.getObjectAt(ctx, nthPosition); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long getOffset(long nthPosition) { + ObjectId objectAt; + try { + objectAt = pack.getObjectAt(ctx, nthPosition); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (objectAt == null) { + return -1; + } + + return findOffset(objectAt); + } + + @Override + public long findOffset(AnyObjectId objId) { + try { + return pack.findOffset(ctx, objId); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int findPosition(AnyObjectId objId) { + try { + return pack.findIdxPosition(ctx, objId); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long findCRC32(AnyObjectId objId) + throws UnsupportedOperationException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasCRC32Support() { + return false; + } + + @Override + public void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) throws IOException { + pack.resolve(ctx, matches, id, matchLimit); + } + + @Override + public byte[] getChecksum() { + throw new UnsupportedOperationException(); + } + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java index 6ff078a..9bc3cb1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java
@@ -9,6 +9,7 @@ */ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.MULTI_PACK_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; @@ -23,8 +24,11 @@ import java.util.stream.Collectors; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; +import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset; import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader; @@ -130,7 +134,7 @@ private record RefWithSize(MultiPackIndex idx, long size) { // Visible for testing @Override - protected VOffsetCalculator getOffsetCalculator() { + protected VOffsetCalculatorNPacks getOffsetCalculator() { return offsetCalculator; } @@ -140,16 +144,19 @@ public ObjectIdSet asObjectIdSet(DfsReader ctx) throws IOException { return multiPackIndex::hasObject; } - @Override public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException { - // TODO(ifrade): at some point we will have bitmaps over the multipack - // index - // At the moment bitmap is in GC, at the end of the chain + // We have bitmaps only at the bottom of the midx or pack stack if (base != null) { return base.getBitmapIndex(ctx); } + if (ctx.getOptions().shouldUseMidxBitmaps() + && getPackDescription().hasFileExt(BITMAP_INDEX)) { + // Return our own bitmaps + return super.getBitmapIndex(ctx); + } + for (DfsPackFile pack : packsInIdOrder) { PackBitmapIndex bitmapIndex = pack.getBitmapIndex(ctx); if (bitmapIndex != null) { @@ -188,6 +195,14 @@ public CommitGraph getCommitGraph(DfsReader ctx) throws IOException { return null; } + @Override + public PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException { + return new MidxReverseIndex(ctx, this, + base == null ? 0 : base.getObjectCount(ctx), + getOffsetCalculator().baseMaxOffset, + base == null ? null : base.getReverseIdx(ctx)); + } + /** * Count of objects in this <b>pack</b> (i.e. including, recursively, its * base) @@ -204,6 +219,11 @@ protected int getObjectCount(DfsReader ctx) throws IOException { return midx(ctx).getObjectCount() + baseObjectCount; } + @Override + protected byte[] getChecksum(DfsReader ctx) throws IOException { + return midx(ctx).getChecksum(); + } + /** * Packs indexed by this multipack index (base NOT included) * @@ -227,6 +247,17 @@ public DfsPackFileMidx getMultipackIndexBase() { } @Override + ObjectId getObjectAt(DfsReader ctx, long nthPosition) throws IOException { + int baseObjectCount = base == null ? 0 : base.getObjectCount(ctx); + if (nthPosition >= baseObjectCount) { + long localPosition = nthPosition - baseObjectCount; + return midx(ctx).getObjectAt((int) localPosition); + } + + return base.getObjectAt(ctx, nthPosition); + } + + @Override public int findIdxPosition(DfsReader ctx, AnyObjectId id) throws IOException { int p = midx(ctx).findPosition(id); @@ -363,7 +394,6 @@ List<DfsObjectToPack> findAllFromPack(DfsReader ctx, return tmp; } - // Visible for testing static class VOffsetCalculatorNPacks implements VOffsetCalculator { private final DfsPackFile[] packs; @@ -376,6 +406,8 @@ static class VOffsetCalculatorNPacks implements VOffsetCalculator { private final DfsPackOffset poBuffer = new DfsPackOffset(); + private final PackOffset localPoBuffer = new PackOffset(); + static VOffsetCalculatorNPacks fromPacks(DfsPackFile[] packsInIdOrder, VOffsetCalculator baseOffsetCalculator) { long[] accSizes = new long[packsInIdOrder.length + 1]; @@ -406,6 +438,22 @@ long encode(MultiPackIndex.PackOffset location) { + baseMaxOffset; } + MultiPackIndex.PackOffset decodeLocal(long voffset) { + if (voffset == -1 || voffset < baseMaxOffset + || voffset > getMaxOffset()) { + return null; + } + + long localOffset = voffset - baseMaxOffset; + for (int i = 1; i < accSizes.length; i++) { + if (localOffset <= accSizes[i]) { + return localPoBuffer.setValues(i - 1, + localOffset - accSizes[i - 1]); + } + } + return null; + } + @Override public DfsPackOffset decode(long voffset) { if (voffset == -1) { @@ -431,4 +479,94 @@ public long getMaxOffset() { return accSizes[accSizes.length - 1] + baseMaxOffset; } } + + private static final class MidxReverseIndex implements PackReverseIndex { + private final PackReverseIndex parentRidx; + + private final DfsPackFileMidxNPacks localMidx; + + private final DfsReader ctx; + + private final long baseObjectCount; + + private final long baseMaxOffset; + + MidxReverseIndex(DfsReader ctx, DfsPackFileMidxNPacks localMidx, + long baseObjectCount, long baseMaxOffset, + PackReverseIndex parentRidx) { + this.ctx = ctx; + this.parentRidx = parentRidx; + this.localMidx = localMidx; + this.baseObjectCount = baseObjectCount; + this.baseMaxOffset = baseMaxOffset; + } + + @Override + public void verifyPackChecksum(String packFilePath) + throws PackMismatchException { + + } + + private MultiPackIndex loadLocalMidx() { + try { + return localMidx.midx(ctx); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ObjectId findObject(long offset) { + if (offset < baseMaxOffset) { + return parentRidx.findObject(offset); + } + + PackOffset localPo = localMidx.getOffsetCalculator() + .decodeLocal(offset); + if (localPo == null) { + return null; + } + + int p = loadLocalMidx().findBitmapPosition(localPo); + if (p == -1) { + // If we found the local + // position, this should NOT + // happen + throw new IllegalStateException(); + } + + return loadLocalMidx().getObjectAtBitmapPosition(p); + } + + @Override + public long findNextOffset(long offset, long maxOffset) + throws CorruptObjectException { + throw new UnsupportedOperationException(); + } + + @Override + public int findPosition(long offset) { + if (offset < baseMaxOffset) { + return parentRidx.findPosition(offset); + } + + PackOffset localPo = localMidx.getOffsetCalculator() + .decodeLocal(offset); + if (localPo == null) { + return -1; + } + + return loadLocalMidx().findBitmapPosition(localPo) + + (int) baseObjectCount; + } + + @Override + public ObjectId findObjectByPosition(int nthPosition) { + if (nthPosition < baseObjectCount) { + return parentRidx.findObjectByPosition(nthPosition); + } + return loadLocalMidx().getObjectAtBitmapPosition( + nthPosition - (int) baseObjectCount); + } + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java index 60fd7d5..72d6998 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java
@@ -9,14 +9,21 @@ */ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.pack.PackExt.MULTI_PACK_INDEX; + import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; import java.util.ArrayList; import java.util.List; import java.util.Set; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; +import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -28,6 +35,7 @@ import org.eclipse.jgit.lib.ObjectIdSet; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.BlockList; +import org.eclipse.jgit.util.IO; /** * A single pack that looks like a midx, to be used in the midx chain @@ -39,6 +47,8 @@ public final class DfsPackFileMidxSingle extends DfsPackFileMidx { private final DfsPackFileMidx base; + private byte[] checksum; + private final LocalPackOffset poBuffer = new LocalPackOffset(); DfsPackFileMidxSingle(DfsBlockCache cache, DfsPackDescription midxDesc, @@ -63,6 +73,14 @@ public ObjectIdSet asObjectIdSet(DfsReader ctx) throws IOException { } @Override + public PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException { + return new MidxReverseIndex(pack.getReverseIdx(ctx), + offsetCalculator.baseMaxOffset, + base == null ? null : base.getReverseIdx(ctx), + base == null ? 0 : base.getObjectCount(ctx)); + } + + @Override public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException { // TODO(ifrade): at some point we will have bitmaps over the multipack // index @@ -112,6 +130,24 @@ protected int getObjectCount(DfsReader ctx) throws IOException { + baseObjectCount; } + @Override + protected byte[] getChecksum(DfsReader ctx) throws IOException { + if (checksum == null) { + long checksumPos = desc.getFileSize(MULTI_PACK_INDEX) - 20; + if (checksumPos <= 0) { + throw new IllegalStateException("Midx stream too short"); + } + + try (ReadableChannel rc = ctx.db.openFile(desc, MULTI_PACK_INDEX); + InputStream in = Channels.newInputStream(rc)) { + checksum = new byte[20]; + in.skip(checksumPos); + IO.readFully(in, checksum, 0, 20); + } + } + return checksum; + } + /** * Packs indexed by this multipack index (base NOT included) * @@ -151,6 +187,17 @@ public int findIdxPosition(DfsReader ctx, AnyObjectId id) } @Override + ObjectId getObjectAt(DfsReader ctx, long nthPosition) throws IOException { + int baseObjects = base == null ? 0 : base.getObjectCount(ctx); + if (nthPosition >= baseObjects) { + long localPosition = nthPosition - baseObjects; + return pack.getPackIndex(ctx).getObjectId(localPosition); + } + + return base.getObjectAt(ctx, nthPosition); + } + + @Override public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException { if (pack.hasObject(ctx, id)) { return true; @@ -351,4 +398,66 @@ void setOffset(long offset) { super.setValues(0, offset); } } + + private static class MidxReverseIndex implements PackReverseIndex { + private final long baseMaxOffset; + + private final long baseObjectCount; + + private final PackReverseIndex baseRidx; + + private final PackReverseIndex ridx; + + MidxReverseIndex(PackReverseIndex ridx, long baseMaxOffset, + PackReverseIndex baseRidx, long baseObjectCount) { + this.ridx = ridx; + this.baseMaxOffset = baseMaxOffset; + this.baseRidx = baseRidx; + this.baseObjectCount = baseObjectCount; + + } + + @Override + public void verifyPackChecksum(String packFilePath) + throws PackMismatchException { + + } + + @Override + public ObjectId findObject(long offset) { + if (offset < baseMaxOffset) { + return baseRidx.findObject(offset); + } + + long localOffset = offset - baseMaxOffset; + return ridx.findObject(localOffset); + } + + @Override + public long findNextOffset(long offset, long maxOffset) + throws CorruptObjectException { + // TODO(ifrade): In this single-pack midx we can actually implement + // this + throw new UnsupportedOperationException(); + } + + @Override + public int findPosition(long offset) { + if (offset < baseMaxOffset) { + return baseRidx.findPosition(offset); + } + long localOffset = offset - baseMaxOffset; + return ridx.findPosition(localOffset) + (int) baseObjectCount; + } + + @Override + public ObjectId findObjectByPosition(int nthPosition) { + if (nthPosition < baseObjectCount) { + return baseRidx.findObjectByPosition(nthPosition); + } + long localPosition = nthPosition - baseObjectCount; + // TODO(ifrade): Check downcasting + return ridx.findObjectByPosition((int) localPosition); + } + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index c397469..e1d7047 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java
@@ -40,6 +40,8 @@ public class DfsReaderOptions { private boolean useObjectSizeIndex; + private boolean useMidxBitmaps; + /** * Create a default reader configuration. */ @@ -163,6 +165,29 @@ public DfsReaderOptions setUseObjectSizeIndex(boolean useObjectSizeIndex) { } /** + * Use bitmaps in the midx if available. + * + * @return true if a midx pack should use its own bitmaps. false to fallback + * to GC bitmaps. + */ + public boolean shouldUseMidxBitmaps() { + return useMidxBitmaps; + } + + /** + * Set if the midx packs should use their bitmaps or fallback to bitmaps in + * GC pack. + * + * @param useMidxBitmaps + * useMidxBitmaps to set + * @return {@code this} + */ + public DfsReaderOptions setUseMidxBitmaps(boolean useMidxBitmaps) { + this.useMidxBitmaps = useMidxBitmaps; + return this; + } + + /** * Update properties by setting fields from the configuration. * <p> * If a property is not defined in the configuration, then it is left
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java index edaa221..31fea79 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java
@@ -26,7 +26,7 @@ public interface MultiPackIndex { * <p> * The pack ids correspond to positions in this list. * - * @return array of packnames refered in this multipak index + * @return array of packnames refered in this multipack index */ String[] getPackNames(); @@ -66,6 +66,35 @@ public interface MultiPackIndex { int findPosition(AnyObjectId objectId); /** + * Return the position in offset order (i.e. ridx or bitmap position) for + * the (packId, offset) pair. + * + * @param po + * a location in the midx (packId, offset) + * @return the position in the midx, in offset order + */ + int findBitmapPosition(PackOffset po); + + /** + * Object id at the specified position in offset order (i.e. position in the + * ridx or bitmap) + * + * @param bitmapPosition + * position in the bitmap + * @return object id at that position. + */ + ObjectId getObjectAtBitmapPosition(int bitmapPosition); + + /** + * ObjectId at this position in the midx + * + * @param position + * position inside this midx in sha1 order + * @return the object id at that position + */ + ObjectId getObjectAt(int position); + + /** * Number of objects in this midx * <p> * This number doesn't match with the sum of objects in each covered pack @@ -90,6 +119,13 @@ public interface MultiPackIndex { void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit); /** + * Index checksum of the contents of this midx file + * + * @return checksum of the contents of this midx file + */ + byte[] getChecksum(); + + /** * Memory size of this multipack index * * @return size of this multipack index in memory, in bytes @@ -102,7 +138,7 @@ public interface MultiPackIndex { * Mutable object to avoid creating many instances while looking for objects * in the pack. Use #copy() to get a new instance with the data. */ - class PackOffset { + class PackOffset implements Comparable<PackOffset> { private int packId; @@ -123,7 +159,7 @@ public static PackOffset create(int packId, long offset) { return new PackOffset().setValues(packId, offset); } - protected PackOffset setValues(int packId, long offset) { + public PackOffset setValues(int packId, long offset) { this.packId = packId; this.offset = offset; return this; @@ -141,5 +177,15 @@ public PackOffset copy() { PackOffset copy = new PackOffset(); return copy.setValues(this.packId, this.offset); } + + @Override + public int compareTo(PackOffset packOffset) { + int cmp = this.packId - packOffset.packId; + if (cmp != 0) { + return cmp; + } + + return Long.compare(this.offset, packOffset.offset); + } } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java index 8501b63..4b70c2c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java
@@ -195,6 +195,9 @@ public static MultiPackIndex read(InputStream fd) Integer.toHexString(chunkId))); } } + byte[] checksum = new byte[20]; + IO.readFully(fd, checksum, 0, 20); + builder.addChecksum(checksum); return builder.build(); } @@ -226,6 +229,8 @@ static class MultiPackIndexBuilder { // Optional private byte[] bitmapPackOrder; + private byte[] checksum; + private MultiPackIndexBuilder(int hashLength) { this.hashLength = hashLength; } @@ -304,7 +309,7 @@ MultiPackIndex build() throws MultiPackIndexFormatException { assertPackCounts(packCount, packNames.length); return new MultiPackIndexV1(hashLength, oidFanout, oidLookup, packNames, bitmappedPackfiles, objectOffsets, - largeObjectOffsets, bitmapPackOrder); + largeObjectOffsets, bitmapPackOrder, checksum); } private static void assertChunkNotNull(Object object, int chunkId) @@ -334,6 +339,10 @@ private static void assertPackCounts(int headerCount, Integer.valueOf(packfileNamesCount))); } } + + public void addChecksum(byte[] checksum) { + this.checksum = checksum; + } } /**
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java index d1eb11c..08e4bfc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java
@@ -34,24 +34,25 @@ class MultiPackIndexV1 implements MultiPackIndex { private final String[] packNames; - private final byte[] bitmappedPackfiles; - - private final byte[] bitmapPackOrder; - private final OffsetLookup offsets; private final PackOffset result = new PackOffset(); + private final ReverseIndex ridx; + + private final byte[] checksum; + MultiPackIndexV1(int hashLength, @NonNull byte[] oidFanout, @NonNull byte[] oidLookup, String[] packNames, byte[] bitmappedPackfiles, byte[] objectOffsets, - byte[] largeObjectOffsets, byte[] bitmapPackOrder) + byte[] largeObjectOffsets, byte[] bitmapPackOrder, byte[] checksum) throws MultiPackIndexFormatException { - this.bitmappedPackfiles = bitmappedPackfiles; - this.bitmapPackOrder = bitmapPackOrder; this.idx = new OidLookup(hashLength, oidFanout, oidLookup); this.offsets = new OffsetLookup(objectOffsets, largeObjectOffsets); this.packNames = packNames; + this.ridx = new ReverseIndex(bitmapPackOrder, idx, offsets, + bitmappedPackfiles); + this.checksum = checksum; } @Override @@ -70,6 +71,21 @@ public int findPosition(AnyObjectId oid) { } @Override + public int findBitmapPosition(PackOffset po) { + return ridx.findBitmapPosition(po); + } + + @Override + public ObjectId getObjectAtBitmapPosition(int bitmapPosition) { + return ridx.getObjectId(bitmapPosition); + } + + @Override + public ObjectId getObjectAt(int position) { + return idx.getObjectAt(position); + } + + @Override public int getObjectCount() { return idx.getObjectCount(); } @@ -92,26 +108,26 @@ public void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, } @Override + public byte[] getChecksum() { + return checksum; + } + + @Override public long getMemorySize() { int packNamesSize = Arrays.stream(packNames) .mapToInt(s -> s.getBytes(StandardCharsets.UTF_8).length).sum(); - return packNamesSize + byteArrayLengh(bitmappedPackfiles) - + byteArrayLengh(bitmapPackOrder) - + idx.getMemorySize() + offsets.getMemorySize(); + return packNamesSize + ridx.getMemorySize() + idx.getMemorySize() + + offsets.getMemorySize() + checksum.length; } @Override public String toString() { return "MultiPackIndexV1 {idx=" + idx + ", packfileNames=" //$NON-NLS-1$ //$NON-NLS-2$ - + Arrays.toString(packNames) + ", bitmappedPackfiles=" //$NON-NLS-1$ - + byteArrayToString(bitmappedPackfiles) + ", objectOffsets=" //$NON-NLS-1$ + + Arrays.toString(packNames) + ", ridx=" //$NON-NLS-1$ + + ridx + ", objectOffsets=" //$NON-NLS-1$ + offsets + '}'; } - private static String byteArrayToString(byte[] array) { - return array == null ? "null" : new String(array); //$NON-NLS-1$ - } - private static int byteArrayLengh(byte[] array) { return array == null ? 0 : array.length; } @@ -251,6 +267,11 @@ int findMultiPackIndexPosition(AnyObjectId id) { return -1; } + ObjectId getObjectAt(int midxPosition) { + int rawOffset = hashLength * midxPosition; + return ObjectId.fromRaw(oidLookup, rawOffset); + } + void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit) { if (matches.size() >= matchLimit) { @@ -313,4 +334,77 @@ int getObjectCount() { return fanoutTable[FANOUT - 1]; } } + + private static class ReverseIndex { + private final byte[] bitmappedPackfiles; + + private final byte[] bitmapPackOrder; + + private final OffsetLookup offsets; + + private final OidLookup oids; + + ReverseIndex(byte[] bitmapPackOrder, OidLookup oids, + OffsetLookup offsets, byte[] bitmappedPackfiles) { + this.bitmappedPackfiles = bitmappedPackfiles; + this.bitmapPackOrder = bitmapPackOrder; + this.oids = oids; + this.offsets = offsets; + } + + public int findBitmapPosition(PackOffset po) { + if (bitmapPackOrder == null || bitmappedPackfiles == null) { + return -1; + } + + /* + * Bitmapped Packfiles (ID: {'B', 'T', 'M', 'P'}) Stores a table of + * two 4-byte unsigned integers in network order. Each table entry + * corresponds to a single pack (in the order that they appear above + * in the `PNAM` chunk). The values for each table entry are as + * follows: - The first bit position (in pseudo-pack order, see + * below) to contain an object from that pack. - The number of bits + * whose objects are selected from that pack. + */ + int packStartBit = NB.decodeInt32(bitmappedPackfiles, + po.getPackId() * 8); + int packBitLength = NB.decodeInt32(bitmappedPackfiles, + (po.getPackId() * 8) + 4); + return binarySearch(packStartBit, packStartBit + packBitLength, po); + } + + private int binarySearch(int start, int end, PackOffset needle) { + PackOffset result = new PackOffset(); + int low = start; + int high = end; + while (low < high) { + int mid = (low + high) >>> 1; + int midxPosition = NB.decodeInt32(bitmapPackOrder, mid * 4); + offsets.getObjectOffset(midxPosition, result); + int cmp = needle.compareTo(result); + if (cmp < 0) { + high = mid; + } else if (cmp == 0) { + return mid; + } else { + low = mid + 1; + } + } + return -1; + } + + public ObjectId getObjectId(int bitmapPosition) { + if (bitmapPackOrder == null) { + return null; + } + int midxPosition = NB.decodeInt32(bitmapPackOrder, + bitmapPosition * 4); + return oids.getObjectAt(midxPosition); + } + + public long getMemorySize() { + return (long) byteArrayLengh(bitmapPackOrder) + + byteArrayLengh(bitmappedPackfiles); + } + } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java index 33c478e..41cbe4b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java
@@ -30,8 +30,21 @@ public final class BitmapCommit extends ObjectId { this.addToIndex = false; } - BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags, - boolean addToIndex) { + // Visible for testing + /** + * Commit that should get a bitmap + * + * @param objectId + * objectId of the commit + * @param reuseWalker + * if the bitmap walker can be reused + * @param flags + * flags + * @param addToIndex + * if the bitmap should be added to the index + */ + public BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags, + boolean addToIndex) { super(objectId); this.reuseWalker = reuseWalker; this.flags = flags;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackBitmapCalculator.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackBitmapCalculator.java new file mode 100644 index 0000000..7fa635f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackBitmapCalculator.java
@@ -0,0 +1,101 @@ +/* + * Copyright (C) 2026, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.BitmapIndex; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.revwalk.BitmapWalker; +import org.eclipse.jgit.storage.pack.PackConfig; + +/** + * Helper to find out interesting commits, create their bitmaps and write them + * into the PackBitmapIndexBuilder. + */ +public class PackBitmapCalculator { + + private final PackConfig config; + + /** + * Constructor + * + * @param config + * configuration + */ + public PackBitmapCalculator(PackConfig config) { + this.config = config; + } + + /** + * Choose commits, create bitmaps and send them to the builder + * + * @param reader + * an object reader + * @param pm + * a progress monitor + * @param wants + * where to start walking looking for commits to bitmap + * @param numCommits + * expected number of new commits to include in the bitmap + * @param excludeFromBitmapSelection + * commits to ignore + * @param writeBitmaps + * store of newly created bitmaps + * @throws IOException + * an error + */ + public void calculate(ObjectReader reader, ProgressMonitor pm, + int numCommits, Set<? extends ObjectId> wants, + Set<? extends ObjectId> excludeFromBitmapSelection, + PackBitmapIndexBuilder writeBitmaps) throws IOException { + PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( + reader, writeBitmaps, pm, wants, config); + + Collection<BitmapCommit> selectedCommits = bitmapPreparer + .selectCommits(numCommits, excludeFromBitmapSelection); + pm.beginTask(JGitText.get().buildingBitmaps, selectedCommits.size()); + + BitmapWalker walker = bitmapPreparer.newBitmapWalker(); + AnyObjectId last = null; + for (BitmapCommit cmit : selectedCommits) { + if (!cmit.isReuseWalker()) { + walker = bitmapPreparer.newBitmapWalker(); + } + BitmapIndex.BitmapBuilder bitmap = walker + .findObjects(Collections.singleton(cmit), null, false); + + if (last != null && cmit.isReuseWalker() && !bitmap.contains(last)) + throw new IllegalStateException( + MessageFormat.format(JGitText.get().bitmapMissingObject, + cmit.name(), last.name())); + last = BitmapCommit.copyFrom(cmit).build(); + writeBitmaps.processBitmapForWrite(cmit, bitmap.build(), + cmit.getFlags()); + + // The bitmap walker should stop when the walk hits the previous + // commit, which saves time. + walker.setPrevCommit(last); + walker.setPrevBitmap(bitmap); + + pm.update(1); + } + pm.endTask(); + } +}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 27fb814..c2d7577 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -2534,40 +2534,10 @@ public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException { // Allow byName to be GC'd if JVM GC runs before the end of the method. byName = null; - PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( - reader, writeBitmaps, pm, stats.interestingObjects, config); + PackBitmapCalculator bitmapWrite = new PackBitmapCalculator(config); + bitmapWrite.calculate(reader, pm, numCommits, stats.interestingObjects, + excludeFromBitmapSelection, writeBitmaps); - Collection<BitmapCommit> selectedCommits = bitmapPreparer - .selectCommits(numCommits, excludeFromBitmapSelection); - - beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size()); - - BitmapWalker walker = bitmapPreparer.newBitmapWalker(); - AnyObjectId last = null; - for (BitmapCommit cmit : selectedCommits) { - if (!cmit.isReuseWalker()) { - walker = bitmapPreparer.newBitmapWalker(); - } - BitmapBuilder bitmap = walker.findObjects( - Collections.singleton(cmit), null, false); - - if (last != null && cmit.isReuseWalker() && !bitmap.contains(last)) - throw new IllegalStateException(MessageFormat.format( - JGitText.get().bitmapMissingObject, cmit.name(), - last.name())); - last = BitmapCommit.copyFrom(cmit).build(); - writeBitmaps.processBitmapForWrite(cmit, bitmap.build(), - cmit.getFlags()); - - // The bitmap walker should stop when the walk hits the previous - // commit, which saves time. - walker.setPrevCommit(last); - walker.setPrevBitmap(bitmap); - - pm.update(1); - } - - endPhase(pm); return true; }
diff --git a/pom.xml b/pom.xml index 0469980..d8a0b3e 100644 --- a/pom.xml +++ b/pom.xml
@@ -146,9 +146,9 @@ <maven-compiler-plugin-version>3.14.1</maven-compiler-plugin-version> <plexus-compiler-version>2.16.1</plexus-compiler-version> <hamcrest-version>3.0</hamcrest-version> - <assertj-version>3.27.6</assertj-version> + <assertj-version>3.27.7</assertj-version> <jna-version>5.18.1</jna-version> - <byte-buddy-version>1.18.2</byte-buddy-version> + <byte-buddy-version>1.18.4</byte-buddy-version> <!-- Properties to enable jacoco code coverage analysis --> <sonar.core.codeCoveragePlugin>jacoco</sonar.core.codeCoveragePlugin>
diff --git a/tools/maven-central/Pipfile.lock b/tools/maven-central/Pipfile.lock index 40358c9..e566398 100644 --- a/tools/maven-central/Pipfile.lock +++ b/tools/maven-central/Pipfile.lock
@@ -27,104 +27,138 @@ }, "certifi": { "hashes": [ - "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", - "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5" + "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", + "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" ], "markers": "python_version >= '3.7'", - "version": "==2025.8.3" + "version": "==2026.1.4" }, "charset-normalizer": { "hashes": [ - "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", - "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", - "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", - "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", - "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", - "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", - "sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c", - "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", - "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", - "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", - "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", - "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", - "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", - "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", - "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", - "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", - "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", - "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", - "sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4", - "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", - "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", - "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", - "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", - "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", - "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", - "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", - "sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b", - "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", - "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", - "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", - "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", - "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", - "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", - "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", - "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", - "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", - "sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a", - "sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40", - "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", - "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", - "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", - "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", - "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", - "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", - "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", - "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", - "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", - "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", - "sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9", - "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", - "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", - "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", - "sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b", - "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", - "sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942", - "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", - "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", - "sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b", - "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", - "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", - "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", - "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", - "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", - "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", - "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", - "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", - "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", - "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", - "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", - "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", - "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", - "sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb", - "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", - "sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557", - "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", - "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", - "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", - "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", - "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9" + "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad", + "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93", + "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", + "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", + "sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc", + "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", + "sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63", + "sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d", + "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", + "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", + "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0", + "sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505", + "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", + "sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af", + "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", + "sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318", + "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", + "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", + "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", + "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", + "sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576", + "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", + "sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1", + "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8", + "sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1", + "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", + "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", + "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", + "sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88", + "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", + "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", + "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", + "sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a", + "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", + "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", + "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84", + "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db", + "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", + "sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7", + "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", + "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", + "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", + "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", + "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", + "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", + "sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2", + "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", + "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d", + "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", + "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", + "sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf", + "sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6", + "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", + "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", + "sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa", + "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", + "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", + "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", + "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", + "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", + "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", + "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", + "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", + "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e", + "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313", + "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", + "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", + "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d", + "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", + "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", + "sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3", + "sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9", + "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", + "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", + "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", + "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", + "sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50", + "sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf", + "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", + "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", + "sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac", + "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", + "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", + "sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c", + "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", + "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6", + "sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e", + "sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4", + "sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84", + "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69", + "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", + "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", + "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", + "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", + "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", + "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", + "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", + "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d", + "sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074", + "sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3", + "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", + "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", + "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", + "sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d", + "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", + "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f", + "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8", + "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", + "sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966", + "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", + "sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3", + "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e", + "sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608" ], "markers": "python_version >= '3.7'", - "version": "==3.4.3" + "version": "==3.4.4" }, "idna": { "hashes": [ - "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", + "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" ], - "markers": "python_version >= '3.6'", - "version": "==3.10" + "markers": "python_version >= '3.8'", + "version": "==3.11" }, "pyyaml": { "hashes": [ @@ -217,11 +251,11 @@ }, "urllib3": { "hashes": [ - "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", - "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" + "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", + "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" ], "markers": "python_version >= '3.9'", - "version": "==2.5.0" + "version": "==2.6.3" } }, "develop": {