DfsPackFileMidx: Move implementation to subclass
We expect to have midxs covering a single pack (GC) in the midx
chain. With the current implementation, this would load twice the same
information, in the midx and in the pack indexes.
As a first step, move the DfsPackFileMidx implementation to a
subclass, so later we can add the single-pack implementation.
We keep DfsPackFileMidx as the superclass with a factory method, as it
reads better for callers; they invoke "DfsPackFileMidx.create" and get the
right instance.
Change-Id: I2ee77a28db35f152a6124c19af91c0906a6a6964
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java
similarity index 97%
rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxTest.java
rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java
index 48611da..758ac1f 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java
@@ -36,7 +36,9 @@
import java.util.zip.Deflater;
import org.eclipse.jgit.annotations.Nullable;
+import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidx.DfsPackOffset;
import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidx.VOffsetCalculator;
+import org.eclipse.jgit.internal.storage.dfs.DfsPackFileMidxNPacks.VOffsetCalculatorNPacks;
import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset;
@@ -61,7 +63,7 @@
import org.junit.Before;
import org.junit.Test;
-public class DfsPackFileMidxTest {
+public class DfsPackFileMidxNPacksTest {
private static final ObjectId NOT_IN_PACK = ObjectId
.fromString("3f306cb3fcd5116919fecad615524bd6e6ea4ba7");
@@ -315,7 +317,7 @@ public void midx_findOffset() throws IOException {
try (DfsReader ctx = db.getObjectDatabase().newReader()) {
long posOne = midx.findOffset(ctx, o1);
- DfsPackFileMidx.DfsPackOffset po = midx.getOffsetCalculator()
+ DfsPackOffset po = midx.getOffsetCalculator()
.decode(posOne);
assertEquals(12, po.getPackOffset());
assertEquals(packThreeSize + packTwoSize, po.getPackStart());
@@ -980,7 +982,7 @@ public void voffsetcalculator_encode() {
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculatorNPacks calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
PackOffset po = PackOffset.create(0, 12);
@@ -997,11 +999,11 @@ public void voffsetcalculator_decode() {
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculator calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
// In first pack
- DfsPackFileMidx.DfsPackOffset decoded = calc.decode(130);
+ DfsPackOffset decoded = calc.decode(130);
assertEquals(one.getPackDescription(),
decoded.getPack().getPackDescription());
assertEquals(130, decoded.getPackOffset());
@@ -1028,7 +1030,7 @@ public void voffsetcalculator_notFound() {
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculatorNPacks calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
assertEquals(-1, calc.encode(null));
@@ -1045,7 +1047,7 @@ public void voffsetcalculator_maxOffset() {
+ two.getPackDescription().getFileSize(PACK)
+ three.getPackDescription().getFileSize(PACK);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculator calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
assertEquals(totalSize, calc.getMaxOffset());
@@ -1057,14 +1059,14 @@ public void voffsetcalculator_withBase_encode() {
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator baseCalc = VOffsetCalculator
+ VOffsetCalculator baseCalc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
DfsPackFile four = createDfsPackFile(900);
DfsPackFile five = createDfsPackFile(1300);
DfsPackFile six = createDfsPackFile(1000);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculatorNPacks calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { four, five, six }, baseCalc);
// These packIds are now from the second top midx
@@ -1082,18 +1084,18 @@ public void voffsetcalculator_withBase_decode() {
DfsPackFile one = createDfsPackFile(800);
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator baseCalc = VOffsetCalculator
+ VOffsetCalculator baseCalc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
DfsPackFile four = createDfsPackFile(900);
DfsPackFile five = createDfsPackFile(1300);
DfsPackFile six = createDfsPackFile(1000);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculator calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { four, five, six }, baseCalc);
// In pack 1
- DfsPackFileMidx.DfsPackOffset decoded = calc.decode(130);
+ DfsPackOffset decoded = calc.decode(130);
assertEquals(one.getPackDescription(),
decoded.getPack().getPackDescription());
assertEquals(130, decoded.getPackOffset());
@@ -1128,14 +1130,14 @@ public void voffsetcalculator_withBase_maxOffset() {
DfsPackFile two = createDfsPackFile(1200);
DfsPackFile three = createDfsPackFile(900);
- VOffsetCalculator baseCalc = VOffsetCalculator
+ VOffsetCalculator baseCalc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { one, two, three }, null);
DfsPackFile four = createDfsPackFile(900);
DfsPackFile five = createDfsPackFile(1300);
DfsPackFile six = createDfsPackFile(1000);
- VOffsetCalculator calc = VOffsetCalculator
+ VOffsetCalculator calc = VOffsetCalculatorNPacks
.fromPacks(new DfsPackFile[] { four, five, six }, baseCalc);
int expectedMaxOffset = 800 + 1200 + 900 + 900 + 1300 + 1000;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
index 742fe6c..8755073 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidx.java
@@ -9,202 +9,94 @@
*/
package org.eclipse.jgit.internal.storage.dfs;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.MULTI_PACK_INDEX;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
-
import java.io.IOException;
-import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
-import java.util.stream.Collectors;
import java.util.zip.DataFormatException;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
-import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
-import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
-import org.eclipse.jgit.internal.storage.midx.MultiPackIndex;
-import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset;
-import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader;
-import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
-import org.eclipse.jgit.lib.AbbreviatedObjectId;
-import org.eclipse.jgit.lib.AnyObjectId;
-import org.eclipse.jgit.lib.BitmapIndex;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectIdSet;
import org.eclipse.jgit.lib.ObjectLoader;
-import org.eclipse.jgit.util.BlockList;
/**
- * Implementation of a DfsPackfile that tries to solve the queries in a
- * multipack index before resorting to the real packs.
+ * DfsPackFile with the extra methods to support midx.
* <p>
- * It uses the position in the multipack index of the objects as their "offset".
+ * This is an abstract class to keep the inheritance from DfsPackFile and allow
+ * a dummy implementation for single packs.
+ * <p>
+ * We implement at this level methods that just translate midx to pack offsets
+ * and forward to the pack.
*/
-public final class DfsPackFileMidx extends DfsPackFile {
+public abstract sealed class DfsPackFileMidx extends DfsPackFile
+ permits DfsPackFileMidxNPacks {
- private static final int REF_POSITION = 0;
-
- private final List<DfsPackFile> packs;
-
- // The required packs, in the order specified in the multipack index
- // Initialized lazily, when the midx is loaded
- private final DfsPackFile[] packsInIdOrder;
-
- private MultiPackIndex midx;
-
- private final DfsPackFileMidx base;
-
- private final VOffsetCalculator offsetCalculator;
-
- static DfsPackFileMidx create(DfsBlockCache cache, DfsPackDescription desc,
- List<DfsPackFile> reqPacks, DfsPackFileMidx base) {
- return new DfsPackFileMidx(cache, desc, reqPacks, base);
+ /**
+ * Create a midx pack
+ *
+ * @param cache
+ * dfs block cache
+ * @param desc
+ * description of the pack, covering at least one other pack
+ * @param requiredPacks
+ * DfsPackFile instances of the covered packs
+ * @param base
+ * midx acting a base of this
+ * @return a midx pack
+ */
+ public static DfsPackFileMidx create(DfsBlockCache cache,
+ DfsPackDescription desc, List<DfsPackFile> requiredPacks,
+ @Nullable DfsPackFileMidx base) {
+ return new DfsPackFileMidxNPacks(cache, desc, requiredPacks, base);
}
- private DfsPackFileMidx(DfsBlockCache cache, DfsPackDescription desc,
- List<DfsPackFile> requiredPacks, @Nullable DfsPackFileMidx base) {
+ /**
+ * Default constructor
+ *
+ * @param cache
+ * dfs block cache
+ * @param desc
+ * midx pack description
+ */
+ protected DfsPackFileMidx(DfsBlockCache cache, DfsPackDescription desc) {
super(cache, desc);
- this.base = base;
- this.packs = requiredPacks;
- String[] coveredPackNames = desc.getCoveredPacks().stream()
- .map(DfsPackDescription::getPackName).toArray(String[]::new);
- packsInIdOrder = getPacksInMidxIdOrder(coveredPackNames);
- offsetCalculator = VOffsetCalculator.fromPacks(packsInIdOrder,
- base != null ? base.getOffsetCalculator() : null);
- this.length = offsetCalculator.getMaxOffset();
}
- private MultiPackIndex midx(DfsReader ctx) throws IOException {
- if (midx != null) {
- return midx;
+ /**
+ * Base of this multipack index
+ * <p>
+ * If this midx is part of a chain, this is its parent
+ *
+ * @return the base of this multipack index
+ */
+ public abstract DfsPackFileMidx getMultipackIndexBase();
+
+ /**
+ * Packs indexed by this multipack index (base NOT included)
+ *
+ * @return packs indexed by this multipack index
+ */
+ public abstract List<DfsPackFile> getCoveredPacks();
+
+ /**
+ * All packs indexed by this multipack index and its chain
+ * <p>
+ * This does not include the inner multipack indexes themselves, only their
+ * covered packs.
+ *
+ * @return packs indexed by this multipack index and its parents.
+ */
+ public List<DfsPackFile> getAllCoveredPacks() {
+ List<DfsPackFile> coveredPacks = new ArrayList<>(getCoveredPacks());
+ DfsPackFileMidx base = getMultipackIndexBase();
+ while (base != null) {
+ coveredPacks.addAll(base.getCoveredPacks());
+ base = base.getMultipackIndexBase();
}
- DfsStreamKey revKey = desc.getStreamKey(MULTI_PACK_INDEX);
- // Keep the value parsed in the loader, in case the Ref<> is
- // nullified in ClockBlockCacheTable#reserveSpace
- // before we read its value.
- AtomicReference<MultiPackIndex> loadedRef = new AtomicReference<>(null);
- DfsBlockCache.Ref<MultiPackIndex> cachedRef = cache.getOrLoadRef(revKey,
- REF_POSITION, () -> {
- RefWithSize midx1 = loadMultiPackIndex(ctx, desc);
- loadedRef.set(midx1.idx);
- return new DfsBlockCache.Ref<>(revKey, REF_POSITION,
- midx1.size, midx1.idx);
- });
- // if (loadedRef.get() == null) {
- // ctx.stats.ridxCacheHit;
- // }
- midx = cachedRef.get() != null ? cachedRef.get() : loadedRef.get();
- return midx;
- }
-
- private static RefWithSize loadMultiPackIndex(DfsReader ctx,
- DfsPackDescription desc) throws IOException {
- try (ReadableChannel rc = ctx.db.openFile(desc, MULTI_PACK_INDEX)) {
- MultiPackIndex midx = MultiPackIndexLoader
- .read(Channels.newInputStream(rc));
- // ctx.stats.readIdxBytes += rc.position();
- return new RefWithSize(midx, midx.getMemorySize());
- }
- }
-
- private record RefWithSize(MultiPackIndex idx, long size) {
- }
-
- private DfsPackFile[] getPacksInMidxIdOrder(String[] packNames) {
- Map<String, DfsPackFile> byName = packs.stream()
- .collect(Collectors.toUnmodifiableMap(
- p -> p.getPackDescription().getPackName(),
- Function.identity()));
- DfsPackFile[] result = new DfsPackFile[desc.getCoveredPacks().size()];
- for (int i = 0; i < packNames.length; i++) {
- DfsPackFile pack = byName.get(packNames[i]);
- if (pack == null) {
- // This should have been checked in the object db
- // when the pack description was loaded
- throw new IllegalStateException("Required pack missing"); //$NON-NLS-1$
- }
- result[i] = pack;
- }
- return result;
- }
-
- // Visible for testing
- VOffsetCalculator getOffsetCalculator() {
- return offsetCalculator;
- }
-
- @Override
- public PackIndex getPackIndex(DfsReader ctx) {
- throw new IllegalStateException(
- "Shouldn't use multipack index if the primary index is needed"); //$NON-NLS-1$
- }
-
- @Override
- public ObjectIdSet asObjectIdSet(DfsReader ctx) throws IOException {
- MultiPackIndex multiPackIndex = midx(ctx);
- return objectId -> multiPackIndex.hasObject(objectId);
- }
-
- @Override
- public PackReverseIndex getReverseIdx(DfsReader ctx) {
- throw new IllegalStateException(
- "Shouldn't use multipack index if the reverse index is needed"); //$NON-NLS-1$
- }
-
- @Override
- public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
- // TODO(ifrade): at some point we will have bitmaps over the multipack
- // index
- // At the moment bitmap is in GC, at the end of the chain
- if (base != null) {
- return base.getBitmapIndex(ctx);
- }
-
- for (DfsPackFile pack : packsInIdOrder) {
- PackBitmapIndex bitmapIndex = pack.getBitmapIndex(ctx);
- if (bitmapIndex != null) {
- return bitmapIndex;
- }
- }
- return null;
- }
-
- @Override
- List<DfsPackFile> fullyIncludedIn(DfsReader ctx,
- BitmapIndex.BitmapBuilder need) throws IOException {
- List<DfsPackFile> fullyIncluded = new ArrayList<>();
- for (DfsPackFile pack : packs) {
- List<DfsPackFile> includedPacks = pack.fullyIncludedIn(ctx, need);
- if (!includedPacks.isEmpty()) {
- fullyIncluded.addAll(includedPacks);
- }
- }
-
- if (base != null) {
- fullyIncluded.addAll(base.fullyIncludedIn(ctx, need));
- }
-
- return fullyIncluded;
- }
-
- @Override
- public CommitGraph getCommitGraph(DfsReader ctx) throws IOException {
- for (DfsPackFile pack : packs) {
- CommitGraph cg = pack.getCommitGraph(ctx);
- if (cg != null) {
- return cg;
- }
- }
- return null;
+ return coveredPacks;
}
/**
@@ -217,96 +109,25 @@ public CommitGraph getCommitGraph(DfsReader ctx) throws IOException {
* @throws IOException
* an error reading a midx in the chain
*/
- private int getObjectCount(DfsReader ctx) throws IOException {
- int baseObjectCount = base == null ? 0 : base.getObjectCount(ctx);
- return midx(ctx).getObjectCount() + baseObjectCount;
- }
-
- /**
- * Packs indexed by this multipack index (base NOT included)
- *
- * @return packs indexed by this multipack index
- */
- public List<DfsPackFile> getCoveredPacks() {
- return packs;
- }
-
- /**
- * All packs indexed by this multipack index and its chain
- * <p>
- * This does not include the inner multipack indexes themselves, only their
- * covered packs.
- *
- * @return packs indexed by this multipack index and its parents.
- */
- public List<DfsPackFile> getAllCoveredPacks() {
- List<DfsPackFile> coveredPacks = new ArrayList<>(packs);
- DfsPackFileMidx base = getMultipackIndexBase();
- while (base != null) {
- coveredPacks.addAll(base.getCoveredPacks());
- base = base.getMultipackIndexBase();
- }
-
- return coveredPacks;
- }
-
- /**
- * Base of this multipack index
- * <p>
- * If this midx is part of a chain, this is its parent
- *
- * @return the base of this multipack index
- */
- public DfsPackFileMidx getMultipackIndexBase() {
- return base;
+ protected int getObjectCount(DfsReader ctx) throws IOException {
+ return (int) getPackDescription().getObjectCount();
}
@Override
- public int findIdxPosition(DfsReader ctx, AnyObjectId id)
- throws IOException {
- int p = midx(ctx).findPosition(id);
- if (p >= 0) {
- int baseObjects = base == null ? 0 : base.getObjectCount(ctx);
- return p + baseObjects;
- }
-
- if (base == null) {
- return -1;
- }
-
- return base.findIdxPosition(ctx, id);
+ public PackIndex getPackIndex(DfsReader ctx) {
+ throw new IllegalStateException(
+ "Shouldn't use multipack index if the primary index is needed"); //$NON-NLS-1$
}
@Override
- public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
- if (midx(ctx).hasObject(id)) {
- return true;
- }
-
- if (base == null) {
- return false;
- }
-
- return base.hasObject(ctx, id);
- }
-
- @Override
- ObjectLoader get(DfsReader ctx, AnyObjectId id) throws IOException {
- PackOffset location = midx(ctx).find(id);
- if (location != null) {
- return packsInIdOrder[location.getPackId()].get(ctx, id);
- }
-
- if (base == null) {
- return null;
- }
-
- return base.get(ctx, id);
+ public PackReverseIndex getReverseIdx(DfsReader ctx) {
+ throw new IllegalStateException(
+ "Shouldn't use multipack index if the reverse index is needed"); //$NON-NLS-1$
}
@Override
ObjectLoader load(DfsReader ctx, long midxOffset) throws IOException {
- DfsPackOffset location = offsetCalculator.decode(midxOffset);
+ DfsPackOffset location = getOffsetCalculator().decode(midxOffset);
if (location == null) {
return null;
}
@@ -314,41 +135,6 @@ ObjectLoader load(DfsReader ctx, long midxOffset) throws IOException {
}
@Override
- long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
- PackOffset location = midx(ctx).find(id);
- if (location != null) {
- return offsetCalculator.encode(location);
- }
-
- if (base == null) {
- return -1;
- }
-
- return base.findOffset(ctx, id);
- }
-
- @Override
- void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
- int matchLimit) throws IOException {
- midx(ctx).resolve(matches, id, matchLimit);
- if (matches.size() < matchLimit && base != null) {
- base.resolve(ctx, matches, id, matchLimit);
- }
- }
-
- @Override
- void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
- // Assumming the order of the packs does not really matter
- for (DfsPackFile pack : packs) {
- pack.copyPackAsIs(out, ctx);
- }
-
- if (base != null) {
- base.copyPackAsIs(out, ctx);
- }
- }
-
- @Override
void copyAsIs(PackOutputStream out, DfsObjectToPack src, boolean validate,
DfsReader ctx) throws IOException,
StoredObjectRepresentationNotAvailableException {
@@ -357,7 +143,7 @@ void copyAsIs(PackOutputStream out, DfsObjectToPack src, boolean validate,
"pack mismatch in object description"); //$NON-NLS-1$
}
- DfsPackOffset location = offsetCalculator.decode(src.offset);
+ DfsPackOffset location = getOffsetCalculator().decode(src.offset);
// The real pack requires the real offset
src.offset = location.getPackOffset();
location.getPack().copyAsIs(out, src, validate, ctx);
@@ -366,90 +152,31 @@ void copyAsIs(PackOutputStream out, DfsObjectToPack src, boolean validate,
}
@Override
- byte[] getDeltaHeader(DfsReader ctx, long pos)
+ final byte[] getDeltaHeader(DfsReader ctx, long pos)
throws IOException, DataFormatException {
- DfsPackOffset location = offsetCalculator.decode(pos);
+ DfsPackOffset location = getOffsetCalculator().decode(pos);
return location.getPack().getDeltaHeader(ctx, location.getPackOffset());
}
@Override
- int getObjectType(DfsReader ctx, long pos) throws IOException {
- DfsPackOffset location = offsetCalculator.decode(pos);
+ final int getObjectType(DfsReader ctx, long pos) throws IOException {
+ DfsPackOffset location = getOffsetCalculator().decode(pos);
return location.getPack().getObjectType(ctx, location.getPackOffset());
}
@Override
- long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
- PackOffset local = midx(ctx).find(id);
- if (local != null) {
- return packsInIdOrder[local.getPackId()].getObjectSize(ctx, id);
- }
-
- if (base == null) {
- return -1;
- }
-
- return base.getObjectSize(ctx, id);
- }
-
- @Override
- long getObjectSize(DfsReader ctx, long pos) throws IOException {
+ final long getObjectSize(DfsReader ctx, long pos) throws IOException {
if (pos < 0) {
return -1;
}
- DfsPackOffset location = offsetCalculator.decode(pos);
+ DfsPackOffset location = getOffsetCalculator().decode(pos);
return location.getPack().getObjectSize(ctx, location.getPackOffset());
}
@Override
- boolean hasObjectSizeIndex(DfsReader ctx) {
- return false;
- }
-
- @Override
- int getObjectSizeIndexThreshold(DfsReader ctx) {
- return Integer.MAX_VALUE;
- }
-
- @Override
- long getIndexedObjectSize(DfsReader ctx, int idxPosition) {
- // TODO(ifrade): if we forward to the pack, it reads its primary index
- return -1;
- }
-
- @Override
- List<DfsObjectToPack> findAllFromPack(DfsReader ctx,
- Iterable<ObjectToPack> objects, boolean skipFound)
- throws IOException {
- List<DfsObjectToPack> tmp = new BlockList<>();
- List<ObjectToPack> notFoundHere = new BlockList<>();
- for (ObjectToPack obj : objects) {
- DfsObjectToPack otp = (DfsObjectToPack) obj;
- if (skipFound && otp.isFound()) {
- continue;
- }
- long p = offsetCalculator.encode(midx(ctx).find(otp));
- if (p < 0) {
- notFoundHere.add(otp);
- continue;
- }
- otp.setOffset(p);
- tmp.add(otp);
- }
-
- if (base != null && !notFoundHere.isEmpty()) {
- List<DfsObjectToPack> inChain = base.findAllFromPack(ctx,
- notFoundHere, skipFound);
- tmp.addAll(inChain);
- }
- tmp.sort(OFFSET_SORT);
- return tmp;
- }
-
- @Override
- void fillRepresentation(DfsObjectRepresentation r, long offset,
+ final void fillRepresentation(DfsObjectRepresentation r, long offset,
DfsReader ctx) throws IOException {
- DfsPackOffset location = offsetCalculator.decode(offset);
+ DfsPackOffset location = getOffsetCalculator().decode(offset);
if (location == null) {
throw new IllegalArgumentException("Invalid offset in midx"); //$NON-NLS-1$
}
@@ -462,16 +189,16 @@ void fillRepresentation(DfsObjectRepresentation r, long offset,
}
@Override
- void fillRepresentation(DfsObjectRepresentation r, long offset,
+ final void fillRepresentation(DfsObjectRepresentation r, long offset,
DfsReader ctx, PackReverseIndex rev) {
// This method shouldn't be called on the midx pack
throw new UnsupportedOperationException();
}
@Override
- boolean isCorrupt(long offset) {
+ final boolean isCorrupt(long offset) {
// The index must have been loaded before to have this offset
- DfsPackOffset location = offsetCalculator.decode(offset);
+ DfsPackOffset location = getOffsetCalculator().decode(offset);
if (location == null) {
throw new IllegalArgumentException("Invalid offset in midx"); //$NON-NLS-1$
}
@@ -479,105 +206,102 @@ boolean isCorrupt(long offset) {
}
@Override
- DfsBlock readOneBlock(long pos, DfsReader ctx, ReadableChannel rc)
+ final DfsBlock readOneBlock(long pos, DfsReader ctx, ReadableChannel rc)
throws IOException {
// The index must have been loaded before to have this offset
- DfsPackOffset location = offsetCalculator.decode(pos);
+ DfsPackOffset location = getOffsetCalculator().decode(pos);
return new DfsBlockMidx(location.getPack().readOneBlock(
location.getPackOffset(), ctx, rc), location.getPackStart());
}
@Override
- DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
+ final DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
// The index must have been loaded before to have this offset
- DfsPackOffset location = offsetCalculator.decode(pos);
+ DfsPackOffset location = getOffsetCalculator().decode(pos);
return new DfsBlockMidx(location.getPack().getOrLoadBlock(
location.getPackOffset(), ctx), location.getPackStart());
}
- // Visible for testing
- static class VOffsetCalculator {
- private final DfsPackFile[] packs;
+ /**
+ * Get the object calculator of this midx
+ *
+ * @return an offset calculator for this midx (including its chain)
+ */
+ protected abstract VOffsetCalculator getOffsetCalculator();
- private final long[] accSizes;
+ /**
+ * Translates from midx-offset (considering all packs concatenated in midx
+ * order) to (pack, offset) pair. This covers the whole midx chain.
+ *
+ * @implNote implementations take care of the encoding and chaining offset
+ * calculators.
+ */
+ protected interface VOffsetCalculator {
+ /**
+ * Return the pair of pack and offset from a midx offset
+ *
+ * @param voffset
+ * an offset in the midx chain
+ * @return the corresponding pack and offset pair
+ */
+ DfsPackOffset decode(long voffset);
- private final long baseMaxOffset;
-
- private final VOffsetCalculator baseOffsetCalculator;
-
- private final DfsPackOffset poBuffer = new DfsPackOffset();
-
- static VOffsetCalculator fromPacks(DfsPackFile[] packsInIdOrder,
- VOffsetCalculator baseOffsetCalculator) {
- long[] accSizes = new long[packsInIdOrder.length + 1];
- accSizes[0] = 0;
- for (int i = 0; i < packsInIdOrder.length; i++) {
- accSizes[i + 1] = accSizes[i] + packsInIdOrder[i]
- .getPackDescription().getFileSize(PACK);
- }
- return new VOffsetCalculator(packsInIdOrder, accSizes,
- baseOffsetCalculator);
- }
-
- VOffsetCalculator(DfsPackFile[] packs, long[] packSizes,
- VOffsetCalculator baseOffsetCalculator) {
- this.packs = packs;
- this.baseOffsetCalculator = baseOffsetCalculator;
- this.baseMaxOffset = baseOffsetCalculator != null
- ? baseOffsetCalculator.getMaxOffset()
- : 0;
- accSizes = packSizes;
- }
-
- long encode(PackOffset location) {
- if (location == null) {
- return -1;
- }
- return location.getOffset() + accSizes[location.getPackId()]
- + baseMaxOffset;
- }
-
- DfsPackOffset decode(long voffset) {
- if (voffset == -1) {
- return null;
- }
-
- if (voffset < baseMaxOffset) {
- return baseOffsetCalculator.decode(voffset);
- }
-
- long localOffset = voffset - baseMaxOffset;
- for (int i = 0; i < accSizes.length; i++) {
- if (localOffset <= accSizes[i]) {
- return poBuffer.setValues(packs[i - 1],
- accSizes[i - 1] + baseMaxOffset, voffset);
- }
- }
- throw new IllegalArgumentException("Asking offset beyond limits"); //$NON-NLS-1$
- }
-
- long getMaxOffset() {
- return accSizes[accSizes.length - 1] + baseMaxOffset;
- }
+ /**
+ * Max offset for this DfsPackFileMidx
+ *
+ * @return max offset for this pack (including its parents)
+ */
+ long getMaxOffset();
}
- static class DfsPackOffset {
+ /**
+ * Data object that keeps a location readable as midx-offset or as
+ * (pack/offset).
+ * <p>
+ * midx-offset is the offset considering the concatenation of all covered
+ * packs in midx order. Only in the first pack of the base of the midx
+ * chain, the pack offsets match the midx offsets.
+ */
+ protected static final class DfsPackOffset {
private DfsPackFile pack;
private long packStart;
- private long offset;
+ private long midxOffset;
- private DfsPackOffset setValues(DfsPackFile pack, long packStart,
- long globalOffset) {
+ /**
+ * Set a location in this instance
+ *
+ * @param pack
+ * the pack that contains the object
+ * @param packStart
+ * midx-offset where the pack starts
+ * @param midxOffset
+ * midx-offset
+ * @return an instance with this data
+ */
+ DfsPackOffset setValues(DfsPackFile pack, long packStart,
+ long midxOffset) {
this.pack = pack;
this.packStart = packStart;
- this.offset = globalOffset;
+ this.midxOffset = midxOffset;
return this;
}
/**
- * The pack
+ * Set only the midx-offset
+ *
+ * @param midxOffset
+ * offset in the midx
+ * @return and updated DfsPackOffset instance
+ */
+ DfsPackOffset setMidxOffset(long midxOffset) {
+ this.midxOffset = midxOffset;
+ return this;
+ }
+
+ /**
+ * The pack containing the object
*
* @return the pack
*/
@@ -601,10 +325,10 @@ long getPackStart() {
/**
* Offset inside the pack (regular offset)
*
- * @return offset inside this pack
+ * @return offset inside the pack
*/
long getPackOffset() {
- return offset - packStart;
+ return midxOffset - packStart;
}
}
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java
new file mode 100644
index 0000000..edcfafa
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacks.java
@@ -0,0 +1,434 @@
+/*
+ * Copyright (C) 2025, Google LLC.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.dfs;
+
+import static org.eclipse.jgit.internal.storage.pack.PackExt.MULTI_PACK_INDEX;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.eclipse.jgit.annotations.Nullable;
+import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
+import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
+import org.eclipse.jgit.internal.storage.midx.MultiPackIndex;
+import org.eclipse.jgit.internal.storage.midx.MultiPackIndex.PackOffset;
+import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader;
+import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
+import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
+import org.eclipse.jgit.lib.AbbreviatedObjectId;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BitmapIndex;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdSet;
+import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.util.BlockList;
+
+/**
+ * Implementation of a DfsPackfile that tries to solve the queries in a
+ * multipack index before resorting to the real packs.
+ * <p>
+ * It uses the position in the multipack index of the objects as their "offset".
+ */
+public final class DfsPackFileMidxNPacks extends DfsPackFileMidx {
+
+ private static final int REF_POSITION = 0;
+
+ private final List<DfsPackFile> packs;
+
+ // The required packs, in the order specified in the multipack index
+ // Initialized lazily, when the midx is loaded
+ private final DfsPackFile[] packsInIdOrder;
+
+ private MultiPackIndex midx;
+
+ private final DfsPackFileMidx base;
+
+ private final VOffsetCalculatorNPacks offsetCalculator;
+
+ DfsPackFileMidxNPacks(DfsBlockCache cache, DfsPackDescription desc,
+ List<DfsPackFile> requiredPacks, @Nullable DfsPackFileMidx base) {
+ super(cache, desc);
+ this.base = base;
+ this.packs = requiredPacks;
+ String[] coveredPackNames = desc.getCoveredPacks().stream()
+ .map(DfsPackDescription::getPackName).toArray(String[]::new);
+ packsInIdOrder = getPacksInMidxIdOrder(coveredPackNames);
+ offsetCalculator = VOffsetCalculatorNPacks.fromPacks(packsInIdOrder,
+ base != null ? base.getOffsetCalculator() : null);
+ this.length = offsetCalculator.getMaxOffset();
+ }
+
+ private MultiPackIndex midx(DfsReader ctx) throws IOException {
+ if (midx != null) {
+ return midx;
+ }
+
+ DfsStreamKey revKey = desc.getStreamKey(MULTI_PACK_INDEX);
+ // Keep the value parsed in the loader, in case the Ref<> is
+ // nullified in ClockBlockCacheTable#reserveSpace
+ // before we read its value.
+ AtomicReference<MultiPackIndex> loadedRef = new AtomicReference<>(null);
+ DfsBlockCache.Ref<MultiPackIndex> cachedRef = cache.getOrLoadRef(revKey,
+ REF_POSITION, () -> {
+ RefWithSize midx1 = loadMultiPackIndex(ctx, desc);
+ loadedRef.set(midx1.idx);
+ return new DfsBlockCache.Ref<>(revKey, REF_POSITION,
+ midx1.size, midx1.idx);
+ });
+ // if (loadedRef.get() == null) {
+ // ctx.stats.ridxCacheHit;
+ // }
+ midx = cachedRef.get() != null ? cachedRef.get() : loadedRef.get();
+ return midx;
+ }
+
+ private static RefWithSize loadMultiPackIndex(DfsReader ctx,
+ DfsPackDescription desc) throws IOException {
+ try (ReadableChannel rc = ctx.db.openFile(desc, MULTI_PACK_INDEX)) {
+ MultiPackIndex midx = MultiPackIndexLoader
+ .read(Channels.newInputStream(rc));
+ // ctx.stats.readIdxBytes += rc.position();
+ return new RefWithSize(midx, midx.getMemorySize());
+ }
+ }
+
+ private record RefWithSize(MultiPackIndex idx, long size) {
+ }
+
+ private DfsPackFile[] getPacksInMidxIdOrder(String[] packNames) {
+ Map<String, DfsPackFile> byName = packs.stream()
+ .collect(Collectors.toUnmodifiableMap(
+ p -> p.getPackDescription().getPackName(),
+ Function.identity()));
+ DfsPackFile[] result = new DfsPackFile[desc.getCoveredPacks().size()];
+ for (int i = 0; i < packNames.length; i++) {
+ DfsPackFile pack = byName.get(packNames[i]);
+ if (pack == null) {
+ // This should have been checked in the object db
+ // when the pack description was loaded
+ throw new IllegalStateException("Required pack missing"); //$NON-NLS-1$
+ }
+ result[i] = pack;
+ }
+ return result;
+ }
+
+ // Visible for testing
+ @Override
+ protected VOffsetCalculator getOffsetCalculator() {
+ return offsetCalculator;
+ }
+
+ @Override
+ public ObjectIdSet asObjectIdSet(DfsReader ctx) throws IOException {
+ MultiPackIndex multiPackIndex = midx(ctx);
+ return multiPackIndex::hasObject;
+ }
+
+
+ @Override
+ public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
+ // TODO(ifrade): at some point we will have bitmaps over the multipack
+ // index
+ // At the moment bitmap is in GC, at the end of the chain
+ if (base != null) {
+ return base.getBitmapIndex(ctx);
+ }
+
+ for (DfsPackFile pack : packsInIdOrder) {
+ PackBitmapIndex bitmapIndex = pack.getBitmapIndex(ctx);
+ if (bitmapIndex != null) {
+ return bitmapIndex;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ List<DfsPackFile> fullyIncludedIn(DfsReader ctx,
+ BitmapIndex.BitmapBuilder need) throws IOException {
+ List<DfsPackFile> fullyIncluded = new ArrayList<>();
+ for (DfsPackFile pack : packs) {
+ List<DfsPackFile> includedPacks = pack.fullyIncludedIn(ctx, need);
+ if (!includedPacks.isEmpty()) {
+ fullyIncluded.addAll(includedPacks);
+ }
+ }
+
+ if (base != null) {
+ fullyIncluded.addAll(base.fullyIncludedIn(ctx, need));
+ }
+
+ return fullyIncluded;
+ }
+
+ @Override
+ public CommitGraph getCommitGraph(DfsReader ctx) throws IOException {
+ for (DfsPackFile pack : packs) {
+ CommitGraph cg = pack.getCommitGraph(ctx);
+ if (cg != null) {
+ return cg;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Count of objects in this <b>pack</> (i.e. including, recursively, its
+ * base)
+ *
+ * @param ctx
+ * a reader
+ * @return count of objects in this pack, including its bases
+ * @throws IOException
+ * an error reading a midx in the chain
+ */
+ @Override
+ protected int getObjectCount(DfsReader ctx) throws IOException {
+ int baseObjectCount = base == null ? 0 : base.getObjectCount(ctx);
+ return midx(ctx).getObjectCount() + baseObjectCount;
+ }
+
+ /**
+ * Packs indexed by this multipack index (base NOT included)
+ *
+ * @return packs indexed by this multipack index
+ */
+ @Override
+ public List<DfsPackFile> getCoveredPacks() {
+ return packs;
+ }
+
+ /**
+ * Base of this multipack index
+ * <p>
+ * If this midx is part of a chain, this is its parent
+ *
+ * @return the base of this multipack index
+ */
+ @Override
+ public DfsPackFileMidx getMultipackIndexBase() {
+ return base;
+ }
+
+ @Override
+ public int findIdxPosition(DfsReader ctx, AnyObjectId id)
+ throws IOException {
+ int p = midx(ctx).findPosition(id);
+ if (p >= 0) {
+ int baseObjects = base == null ? 0 : base.getObjectCount(ctx);
+ return p + baseObjects;
+ }
+
+ if (base == null) {
+ return -1;
+ }
+
+ return base.findIdxPosition(ctx, id);
+ }
+
+ @Override
+ public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
+ if (midx(ctx).hasObject(id)) {
+ return true;
+ }
+
+ if (base == null) {
+ return false;
+ }
+
+ return base.hasObject(ctx, id);
+ }
+
+ @Override
+ ObjectLoader get(DfsReader ctx, AnyObjectId id) throws IOException {
+ PackOffset location = midx(ctx).find(id);
+ if (location != null) {
+ return packsInIdOrder[location.getPackId()].get(ctx, id);
+ }
+
+ if (base == null) {
+ return null;
+ }
+
+ return base.get(ctx, id);
+ }
+
+ @Override
+ long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
+ PackOffset location = midx(ctx).find(id);
+ if (location != null) {
+ return offsetCalculator.encode(location);
+ }
+
+ if (base == null) {
+ return -1;
+ }
+
+ return base.findOffset(ctx, id);
+ }
+
+ @Override
+ void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
+ int matchLimit) throws IOException {
+ midx(ctx).resolve(matches, id, matchLimit);
+ if (matches.size() < matchLimit && base != null) {
+ base.resolve(ctx, matches, id, matchLimit);
+ }
+ }
+
+ @Override
+ void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
+ // Assumming the order of the packs does not really matter
+ for (DfsPackFile pack : packs) {
+ pack.copyPackAsIs(out, ctx);
+ }
+
+ if (base != null) {
+ base.copyPackAsIs(out, ctx);
+ }
+ }
+
+ @Override
+ long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
+ PackOffset local = midx(ctx).find(id);
+ if (local != null) {
+ return packsInIdOrder[local.getPackId()].getObjectSize(ctx, id);
+ }
+
+ if (base == null) {
+ return -1;
+ }
+
+ return base.getObjectSize(ctx, id);
+ }
+
+ @Override
+ boolean hasObjectSizeIndex(DfsReader ctx) {
+ return false;
+ }
+
+ @Override
+ int getObjectSizeIndexThreshold(DfsReader ctx) {
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ long getIndexedObjectSize(DfsReader ctx, int idxPosition) {
+ // TODO(ifrade): if we forward to the pack, it reads its primary index
+ return -1;
+ }
+
+ @Override
+ List<DfsObjectToPack> findAllFromPack(DfsReader ctx,
+ Iterable<ObjectToPack> objects, boolean skipFound)
+ throws IOException {
+ List<DfsObjectToPack> tmp = new BlockList<>();
+ List<ObjectToPack> notFoundHere = new BlockList<>();
+ for (ObjectToPack obj : objects) {
+ DfsObjectToPack otp = (DfsObjectToPack) obj;
+ if (skipFound && otp.isFound()) {
+ continue;
+ }
+ long p = offsetCalculator.encode(midx(ctx).find(otp));
+ if (p < 0) {
+ notFoundHere.add(otp);
+ continue;
+ }
+ otp.setOffset(p);
+ tmp.add(otp);
+ }
+
+ if (base != null && !notFoundHere.isEmpty()) {
+ List<DfsObjectToPack> inChain = base.findAllFromPack(ctx,
+ notFoundHere, skipFound);
+ tmp.addAll(inChain);
+ }
+ tmp.sort(OFFSET_SORT);
+ return tmp;
+ }
+
+
+ // Visible for testing
+ static class VOffsetCalculatorNPacks implements VOffsetCalculator {
+ private final DfsPackFile[] packs;
+
+ private final long[] accSizes;
+
+ private final long baseMaxOffset;
+
+ private final VOffsetCalculator baseOffsetCalculator;
+
+ private final DfsPackOffset poBuffer = new DfsPackOffset();
+
+ static VOffsetCalculatorNPacks fromPacks(DfsPackFile[] packsInIdOrder,
+ VOffsetCalculator baseOffsetCalculator) {
+ long[] accSizes = new long[packsInIdOrder.length + 1];
+ accSizes[0] = 0;
+ for (int i = 0; i < packsInIdOrder.length; i++) {
+ accSizes[i + 1] = accSizes[i] + packsInIdOrder[i]
+ .getPackDescription().getFileSize(PACK);
+ }
+ return new VOffsetCalculatorNPacks(packsInIdOrder, accSizes,
+ baseOffsetCalculator);
+ }
+
+ VOffsetCalculatorNPacks(DfsPackFile[] packs, long[] packSizes,
+ VOffsetCalculator baseOffsetCalculator) {
+ this.packs = packs;
+ this.baseOffsetCalculator = baseOffsetCalculator;
+ this.baseMaxOffset = baseOffsetCalculator != null
+ ? baseOffsetCalculator.getMaxOffset()
+ : 0;
+ accSizes = packSizes;
+ }
+
+ long encode(MultiPackIndex.PackOffset location) {
+ if (location == null) {
+ return -1;
+ }
+ return location.getOffset() + accSizes[location.getPackId()]
+ + baseMaxOffset;
+ }
+
+ @Override
+ public DfsPackOffset decode(long voffset) {
+ if (voffset == -1) {
+ return null;
+ }
+
+ if (voffset < baseMaxOffset) {
+ return baseOffsetCalculator.decode(voffset);
+ }
+
+ long localOffset = voffset - baseMaxOffset;
+ for (int i = 0; i < accSizes.length; i++) {
+ if (localOffset <= accSizes[i]) {
+ return poBuffer.setValues(packs[i - 1],
+ accSizes[i - 1] + baseMaxOffset, voffset);
+ }
+ }
+ throw new IllegalArgumentException("Asking offset beyond limits"); //$NON-NLS-1$
+ }
+
+ @Override
+ public long getMaxOffset() {
+ return accSizes[accSizes.length - 1] + baseMaxOffset;
+ }
+ }
+}
