DfsPackFileMidx: Implement PackIndex over midx
Change-Id: Iccc59fd16be50f813cdad7d7318bc5146a6a6964
diff --git a/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java
new file mode 100644
index 0000000..e0674eb
--- /dev/null
+++ b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2025, Google LLC.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.dfs;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.Deflater;
+
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+
+/**
+ * Helpers to write multipack indexes
+ */
+public class MidxTestUtils {
+ private MidxTestUtils() {
+ }
+
+ /**
+ * Write a single pack into the repo with the blob as contents
+ *
+ * @param db
+ * repository
+ * @param blob
+ * blob to write into the pack
+ * @return object id of the blob written in the pack
+ * @throws IOException
+ * a problem writing in the repo
+ */
+ static ObjectId writePackWithBlob(DfsRepository db, String blob)
+ throws IOException {
+ return writePackWithBlobs(db, blob)[0];
+ }
+
+ /**
+ * Write multiple blobs into a single pack in the repo
+ *
+ * @param db
+ * repository
+ * @param blobs
+ * blobs to write into the pack
+ * @return object ids of the blobs written in the pack, in the same order as
+ * the input parameters
+ * @throws IOException
+ * a problem writing in the repo
+ */
+ static ObjectId[] writePackWithBlobs(DfsRepository db, String... blobs)
+ throws IOException {
+ ObjectId[] oids = new ObjectId[blobs.length];
+
+ DfsInserter ins = (DfsInserter) db.newObjectInserter();
+ ins.setCompressionLevel(Deflater.NO_COMPRESSION);
+ for (int i = 0; i < blobs.length; i++) {
+ oids[i] = ins.insert(OBJ_BLOB, blobs[i].getBytes(UTF_8));
+ }
+ ins.flush();
+ return oids;
+ }
+
+ /**
+ * Write a midx covering the only pack in the repo
+ *
+ * @param db
+ * a repository with a single pack
+ * @return a midx covering that single pack
+ * @throws IOException
+ * a problem writing in the repo
+ */
+ static DfsPackFileMidx writeSinglePackMidx(DfsRepository db)
+ throws IOException {
+ DfsPackFile[] packs = db.getObjectDatabase().getPacks();
+ assertEquals("More than one pack in db", 1, packs.length);
+ return writeSinglePackMidx(db, packs[0]);
+ }
+
+ /**
+ * Write a midx covering a single pack
+ *
+ * @param db
+ * a repository to write the midx covering the pack
+ * @param pack
+ * a pack in the repository that will be covered by a new midx
+ * @return a midx covering that single pack
+ * @throws IOException
+ * a problem writing in the repo
+ */
+ static DfsPackFileMidx writeSinglePackMidx(DfsRepository db,
+ DfsPackFile pack) throws IOException {
+ return writeMultipackIndex(db, new DfsPackFile[] { pack }, null);
+ }
+
+ /**
+ * Write a midx in the repository
+ *
+ * @param db
+ * the repository
+ * @param packs
+ * packs to be covered by this midx
+ * @param base
+ * base of the newly created midx
+ * @return the new midx instance
+ * @throws IOException
+ * a problem writing in the repo
+ */
+ static DfsPackFileMidx writeMultipackIndex(DfsRepository db,
+ DfsPackFile[] packs, DfsPackFileMidx base) throws IOException {
+ DfsPackDescription desc = DfsMidxWriter.writeMidx(
+ NullProgressMonitor.INSTANCE, db.getObjectDatabase(),
+ Arrays.asList(packs),
+ base != null ? base.getPackDescription() : null);
+ db.getObjectDatabase().commitPack(List.of(desc), null);
+ return DfsPackFileMidx.create(DfsBlockCache.getInstance(), desc,
+ Arrays.asList(packs), base);
+ }
+}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java
new file mode 100644
index 0000000..847d58f
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxIndexTest.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2025, Google LLC.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.dfs;
+
+import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeMultipackIndex;
+import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writePackWithBlobs;
+import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeSinglePackMidx;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.eclipse.jgit.internal.storage.file.PackIndex;
+import org.eclipse.jgit.lib.AbbreviatedObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class DfsPackFileMidxIndexTest {
+
+ private static final ObjectId NOT_IN_PACK = ObjectId
+ .fromString("3f306cb3fcd5116919fecad615524bd6e6ea4ba7");
+
+ private static final List<String> BLOBS = List.of("blob one", "blob two",
+ "blob three", "blob four", "blob five", "blob six");
+
+ @Parameters(name = "{0}")
+ public static Iterable<TestInput> data() throws IOException {
+ return List.of(setupOneMidxOverOnePack(), setupOneMidxOverNPacks(),
+ setupMidxChainEachOverNPacks());
+ }
+
+ private record TestInput(String testDesc, DfsRepository db,
+ DfsPackFileMidx midx, ObjectId[] oids) {
+ @Override
+ public String toString() {
+ return testDesc;
+ }
+
+ }
+
+ private TestInput ti;
+
+ public DfsPackFileMidxIndexTest(TestInput ti) {
+ this.ti = ti;
+ }
+
+ @Test
+ public void getPackIndex_getObjectCount() {
+ try (DfsReader ctx = ti.db().getObjectDatabase().newReader()) {
+ assertEquals(ti.oids().length,
+ ti.midx().getPackIndex(ctx).getObjectCount());
+ }
+ }
+
+ @Test
+ public void getPackIndex_position_findPosition_getObjectId() {
+ try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) {
+ PackIndex idx = ti.midx().getPackIndex(ctx);
+ for (int i = 0; i < ti.oids().length; i++) {
+ ObjectId expected = ti.oids()[i];
+ int position = idx.findPosition(expected);
+ assertNotEquals(-1, position);
+ ObjectId actual = idx.getObjectId(position);
+ assertEquals(expected, actual);
+ }
+ assertEquals(-1, idx.findPosition(NOT_IN_PACK));
+ }
+ }
+
+ @Test
+ public void getPackIndex_offset_findOffset_getOffset() {
+ try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) {
+ PackIndex idx = ti.midx().getPackIndex(ctx);
+ for (int i = 0; i < ti.oids().length; i++) {
+ ObjectId oid = ti.oids()[i];
+ int oidPosition = idx.findPosition(oid);
+
+ long offsetById = idx.findOffset(oid);
+ long offsetByPos = idx.getOffset(oidPosition);
+ assertEquals(offsetById, offsetByPos);
+ }
+ assertEquals(-1, idx.findOffset(NOT_IN_PACK));
+ }
+ }
+
+ @Test
+ public void getPackIndex_objects_contains_hasObjects() {
+ try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) {
+ PackIndex idx = ti.midx().getPackIndex(ctx);
+ for (int i = 0; i < ti.oids().length; i++) {
+ ObjectId oid = ti.oids()[i];
+ assertTrue(idx.contains(oid));
+ assertTrue(idx.hasObject(oid));
+ }
+ assertFalse(idx.contains(NOT_IN_PACK));
+ assertFalse(idx.hasObject(NOT_IN_PACK));
+ }
+ }
+
+ @Test
+ public void getPackIndex_resolve() throws IOException {
+ try (DfsReader ctx = ti.db.getObjectDatabase().newReader()) {
+ PackIndex idx = ti.midx().getPackIndex(ctx);
+ Set<ObjectId> matches = new HashSet<>();
+ // Sha1 of "blob two" = ae4116e0972d85cd751b458fea94ca9eb84dd692
+ idx.resolve(matches, AbbreviatedObjectId.fromString("ae411"), 100);
+ assertEquals(1, matches.size());
+ }
+ }
+
+ static TestInput setupOneMidxOverOnePack() throws IOException {
+ InMemoryRepository db = new InMemoryRepository(
+ new DfsRepositoryDescription("one_midx_one_pack"));
+ ObjectId[] objectIds = writePackWithBlobs(db,
+ BLOBS.toArray(String[]::new));
+ DfsPackFileMidx midx1 = writeSinglePackMidx(db);
+ return new TestInput("one midx - one pack", db, midx1, objectIds);
+ }
+
+ static TestInput setupOneMidxOverNPacks() throws IOException {
+ InMemoryRepository db = new InMemoryRepository(
+ new DfsRepositoryDescription("one_midx_n_packs"));
+
+ ObjectId[] objectIds = BLOBS.stream().map(s -> {
+ try {
+ return MidxTestUtils.writePackWithBlob(db, s);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).toArray(ObjectId[]::new);
+ DfsPackFileMidx midx1 = writeMultipackIndex(db,
+ db.getObjectDatabase().getPacks(), null);
+ return new TestInput("one midx - n packs", db, midx1, objectIds);
+ }
+
+ static TestInput setupMidxChainEachOverNPacks() throws IOException {
+ InMemoryRepository db = new InMemoryRepository(
+ new DfsRepositoryDescription("two_midx_3_packs_each"));
+
+ ObjectId[] objectIds = BLOBS.stream().map(s -> {
+ try {
+ return MidxTestUtils.writePackWithBlob(db, s);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).toArray(ObjectId[]::new);
+ DfsPackFile[] packs = db.getObjectDatabase().getPacks();
+ // If the amount of blobs (i.e. packs), adjust the ranges covered by
+ // midx.
+ assertEquals(6, BLOBS.size());
+ DfsPackFileMidx midxBase = MidxTestUtils.writeMultipackIndex(db,
+ Arrays.copyOfRange(packs, 3, 6), null);
+ DfsPackFileMidx midxTip = MidxTestUtils.writeMultipackIndex(db,
+ Arrays.copyOfRange(packs, 0, 3), midxBase);
+ return new TestInput("two midx - 3 packs each", db, midxTip, objectIds);
+ }
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
index 5f08444..ba73283 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
@@ -11,7 +11,9 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.zip.DataFormatException;
import org.eclipse.jgit.annotations.Nullable;
@@ -19,6 +21,8 @@
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
+import org.eclipse.jgit.lib.AbbreviatedObjectId;
+import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
@@ -137,9 +141,8 @@ protected int getObjectCount(DfsReader ctx) throws IOException {
}
@Override
- public PackIndex getPackIndex(DfsReader ctx) {
- throw new IllegalStateException(
- "Shouldn't use multipack index if the primary index is needed"); //$NON-NLS-1$
+ public final PackIndex getPackIndex(DfsReader ctx) {
+ return new MidxPackIndex(this, ctx);
}
@Override
@@ -354,4 +357,101 @@ long getPackOffset() {
return midxOffset - packStart;
}
}
+
+ private static class MidxPackIndex implements PackIndex {
+
+ private final DfsPackFileMidx pack;
+
+ private final DfsReader ctx;
+
+ MidxPackIndex(DfsPackFileMidx pack, DfsReader ctx) {
+ this.pack = pack;
+ this.ctx = ctx;
+ }
+
+ @Override
+ public Iterator<MutableEntry> iterator() {
+ throw new UnsupportedOperationException("Not implemented yet");
+ }
+
+ @Override
+ public long getObjectCount() {
+ try {
+ return pack.getObjectCount(ctx);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public long getOffset64Count() {
+ // TODO(ifrade): This method seems to be used only for stats.
+ // Maybe we can just remove it.
+ return 0;
+ }
+
+ @Override
+ public ObjectId getObjectId(long nthPosition) {
+ try {
+ return pack.getObjectAt(ctx, nthPosition);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public long getOffset(long nthPosition) {
+ ObjectId objectAt;
+ try {
+ objectAt = pack.getObjectAt(ctx, nthPosition);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (objectAt == null) {
+ return -1;
+ }
+
+ return findOffset(objectAt);
+ }
+
+ @Override
+ public long findOffset(AnyObjectId objId) {
+ try {
+ return pack.findOffset(ctx, objId);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public int findPosition(AnyObjectId objId) {
+ try {
+ return pack.findIdxPosition(ctx, objId);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public long findCRC32(AnyObjectId objId)
+ throws UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasCRC32Support() {
+ return false;
+ }
+
+ @Override
+ public void resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
+ int matchLimit) throws IOException {
+ pack.resolve(ctx, matches, id, matchLimit);
+ }
+
+ @Override
+ public byte[] getChecksum() {
+ throw new UnsupportedOperationException();
+ }
+ }
}
