Merge changes I39783eee,I874503ec,Ic942a8e4,I6ec2c3e8,I62cb5030, ...
* changes:
PackBitmapIndex: Set distance threshold
PackBitmapIndex: Not buffer inflated bitmap in BasePackBitmapIndex
PackBitmapIndex: Remove convertedBitmaps in the Remapper
PackBitmapIndex: Reduce memory usage in GC
PackBitmapIndex: Add AddToBitmapWithCacheFilter class
PackBitmapIndex: Add util methods and builder to BitmapCommit
PackBitmapIndex: Move BitmapCommit to a top-level class
Refactor: Make retriveCompressed an method of the Bitmap class
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
index f2876b7..cc826c3 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
@@ -23,7 +23,6 @@
import org.eclipse.jgit.internal.storage.file.GcTestCase;
import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder;
-import org.eclipse.jgit.internal.storage.pack.PackWriterBitmapPreparer.BitmapCommit;
import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
import org.eclipse.jgit.junit.TestRepository.CommitBuilder;
import org.eclipse.jgit.lib.Constants;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java
new file mode 100644
index 0000000..d7ccadf
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2020, Google LLC and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.revwalk;
+
+import org.eclipse.jgit.lib.BitmapIndex.Bitmap;
+import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.revwalk.filter.RevFilter;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevFlag;
+
+/**
+ * A RevFilter that adds the visited commits to {@code bitmap} as a side effect.
+ * <p>
+ * When the walk hits a commit that is the same as {@code cachedCommit} or is
+ * part of {@code bitmap}'s BitmapIndex, that entire bitmap is ORed into
+ * {@code bitmap} and the commit and its parents are marked as SEEN so that the
+ * walk does not have to visit its ancestors. This ensures the walk is very
+ * short if there is good bitmap coverage.
+ */
+public class AddToBitmapWithCacheFilter extends RevFilter {
+ private final AnyObjectId cachedCommit;
+
+ private final Bitmap cachedBitmap;
+
+ private final BitmapBuilder bitmap;
+
+ /**
+ * Create a filter with a cached BitmapCommit that adds visited commits to
+ * the given bitmap.
+ *
+ * @param cachedCommit
+ * the cached commit
+ * @param cachedBitmap
+ * the bitmap corresponds to {@code cachedCommit}}
+ * @param bitmap
+ * bitmap to write visited commits to
+ */
+ public AddToBitmapWithCacheFilter(AnyObjectId cachedCommit,
+ Bitmap cachedBitmap,
+ BitmapBuilder bitmap) {
+ this.cachedCommit = cachedCommit;
+ this.cachedBitmap = cachedBitmap;
+ this.bitmap = bitmap;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public final boolean include(RevWalk rw, RevCommit c) {
+ Bitmap visitedBitmap;
+
+ if (bitmap.contains(c)) {
+ // already included
+ } else if ((visitedBitmap = bitmap.getBitmapIndex()
+ .getBitmap(c)) != null) {
+ bitmap.or(visitedBitmap);
+ } else if (cachedCommit.equals(c)) {
+ bitmap.or(cachedBitmap);
+ } else {
+ bitmap.addObject(c, Constants.OBJ_COMMIT);
+ return true;
+ }
+
+ for (RevCommit p : c.getParents()) {
+ p.add(RevFlag.SEEN);
+ }
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public final RevFilter clone() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public final boolean requiresCommitBody() {
+ return false;
+ }
+}
+
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java
index c9bb167..74b46bc 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java
@@ -72,7 +72,6 @@
if (r instanceof EWAHCompressedBitmap) {
out = out.xor((EWAHCompressedBitmap) r);
out.trim();
- bitmapContainer = out;
return out;
}
xb = (XorCompressedBitmap) r;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java
index 6aa1a0e..0d3a2b4 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java
@@ -252,6 +252,11 @@
return bitmapIndex;
}
+ @Override
+ public EWAHCompressedBitmap retrieveCompressed() {
+ return build().retrieveCompressed();
+ }
+
private EWAHCompressedBitmap ewahBitmap(Bitmap other) {
if (other instanceof CompressedBitmap) {
CompressedBitmap b = (CompressedBitmap) other;
@@ -372,7 +377,8 @@
};
}
- EWAHCompressedBitmap getEwahCompressedBitmap() {
+ @Override
+ public EWAHCompressedBitmap retrieveCompressed() {
return bitmap;
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java
index 9538cc5..5666b57 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java
@@ -11,17 +11,16 @@
package org.eclipse.jgit.internal.storage.file;
import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
-import java.util.NoSuchElementException;
import org.eclipse.jgit.internal.JGitText;
-import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap;
+import org.eclipse.jgit.internal.storage.pack.BitmapCommit;
import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.BitmapIndex.Bitmap;
-import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdOwnerMap;
@@ -41,8 +40,12 @@
private final EWAHCompressedBitmap blobs;
private final EWAHCompressedBitmap tags;
private final BlockList<PositionEntry> byOffset;
- final BlockList<StoredBitmap>
- byAddOrder = new BlockList<>();
+
+ private final LinkedList<StoredBitmap>
+ bitmapsToWriteXorBuffer = new LinkedList<>();
+
+ private List<StoredEntry> bitmapsToWrite = new ArrayList<>();
+
final ObjectIdOwnerMap<PositionEntry>
positionEntries = new ObjectIdOwnerMap<>();
@@ -134,16 +137,64 @@
* the flags to be stored with the bitmap
*/
public void addBitmap(AnyObjectId objectId, Bitmap bitmap, int flags) {
- if (bitmap instanceof BitmapBuilder)
- bitmap = ((BitmapBuilder) bitmap).build();
+ addBitmap(objectId, bitmap.retrieveCompressed(), flags);
+ }
- EWAHCompressedBitmap compressed;
- if (bitmap instanceof CompressedBitmap)
- compressed = ((CompressedBitmap) bitmap).getEwahCompressedBitmap();
- else
- throw new IllegalArgumentException(bitmap.getClass().toString());
+ /**
+ * Processes a commit and prepares its bitmap to write to the bitmap index
+ * file.
+ *
+ * @param c
+ * the commit corresponds to the bitmap.
+ * @param bitmap
+ * the bitmap to be written.
+ * @param flags
+ * the flags of the commit.
+ */
+ public void processBitmapForWrite(BitmapCommit c, Bitmap bitmap,
+ int flags) {
+ EWAHCompressedBitmap compressed = bitmap.retrieveCompressed();
+ compressed.trim();
+ StoredBitmap newest = new StoredBitmap(c, compressed, null, flags);
- addBitmap(objectId, compressed, flags);
+ bitmapsToWriteXorBuffer.add(newest);
+ if (bitmapsToWriteXorBuffer.size() > MAX_XOR_OFFSET_SEARCH) {
+ bitmapsToWrite.add(
+ generateStoredEntry(bitmapsToWriteXorBuffer.pollFirst()));
+ }
+
+ if (c.isAddToIndex()) {
+ // The Bitmap map in the base class is used to make revwalks
+ // efficient, so only add bitmaps that keep it efficient without
+ // bloating memory.
+ addBitmap(c, bitmap, flags);
+ }
+ }
+
+ private StoredEntry generateStoredEntry(StoredBitmap bitmapToWrite) {
+ int bestXorOffset = 0;
+ EWAHCompressedBitmap bestBitmap = bitmapToWrite.getBitmap();
+
+ int offset = 1;
+ for (StoredBitmap curr : bitmapsToWriteXorBuffer) {
+ EWAHCompressedBitmap bitmap = curr.getBitmap()
+ .xor(bitmapToWrite.getBitmap());
+ if (bitmap.sizeInBytes() < bestBitmap.sizeInBytes()) {
+ bestBitmap = bitmap;
+ bestXorOffset = offset;
+ }
+ offset++;
+ }
+
+ PositionEntry entry = positionEntries.get(bitmapToWrite);
+ if (entry == null) {
+ throw new IllegalStateException();
+ }
+ bestBitmap.trim();
+ StoredEntry result = new StoredEntry(entry.namePosition, bestBitmap,
+ bestXorOffset, bitmapToWrite.getFlags());
+
+ return result;
}
/**
@@ -161,7 +212,6 @@
bitmap.trim();
StoredBitmap result = new StoredBitmap(objectId, bitmap, null, flags);
getBitmaps().add(result);
- byAddOrder.add(result);
}
/** {@inheritDoc} */
@@ -247,15 +297,18 @@
/** {@inheritDoc} */
@Override
public int getBitmapCount() {
- return getBitmaps().size();
+ return bitmapsToWriteXorBuffer.size() + bitmapsToWrite.size();
}
/**
* Remove all the bitmaps entries added.
+ *
+ * @param size
+ * the expected number of bitmap entries to be written.
*/
- public void clearBitmaps() {
- byAddOrder.clear();
+ public void resetBitmaps(int size) {
getBitmaps().clear();
+ bitmapsToWrite = new ArrayList<>(size);
}
/** {@inheritDoc} */
@@ -265,64 +318,18 @@
}
/**
- * Get an iterator over the xor compressed entries.
+ * Get list of xor compressed entries that need to be written.
*
- * @return an iterator over the xor compressed entries.
+ * @return a list of the xor compressed entries.
*/
- public Iterable<StoredEntry> getCompressedBitmaps() {
- // Add order is from oldest to newest. The reverse add order is the
- // output order.
- return () -> new Iterator<StoredEntry>() {
+ public List<StoredEntry> getCompressedBitmaps() {
+ while (!bitmapsToWriteXorBuffer.isEmpty()) {
+ bitmapsToWrite.add(
+ generateStoredEntry(bitmapsToWriteXorBuffer.pollFirst()));
+ }
- private int index = byAddOrder.size() - 1;
-
- @Override
- public boolean hasNext() {
- return index >= 0;
- }
-
- @Override
- public StoredEntry next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- StoredBitmap item = byAddOrder.get(index);
- int bestXorOffset = 0;
- EWAHCompressedBitmap bestBitmap = item.getBitmap();
-
- // Attempt to compress the bitmap with an XOR of the
- // previously written entries.
- for (int i = 1; i <= MAX_XOR_OFFSET_SEARCH; i++) {
- int curr = i + index;
- if (curr >= byAddOrder.size()) {
- break;
- }
-
- StoredBitmap other = byAddOrder.get(curr);
- EWAHCompressedBitmap bitmap = other.getBitmap()
- .xor(item.getBitmap());
-
- if (bitmap.sizeInBytes() < bestBitmap.sizeInBytes()) {
- bestBitmap = bitmap;
- bestXorOffset = i;
- }
- }
- index--;
-
- PositionEntry entry = positionEntries.get(item);
- if (entry == null) {
- throw new IllegalStateException();
- }
- bestBitmap.trim();
- return new StoredEntry(entry.namePosition, bestBitmap,
- bestXorOffset, item.getFlags());
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ Collections.reverse(bitmapsToWrite);
+ return bitmapsToWrite;
}
/** Data object for the on disk representation of a bitmap entry. */
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java
index 273eeef..4b25284 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java
@@ -18,7 +18,6 @@
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.BitmapIndex;
import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectIdOwnerMap;
import com.googlecode.javaewah.EWAHCompressedBitmap;
import com.googlecode.javaewah.IntIterator;
@@ -34,7 +33,6 @@
private final BasePackBitmapIndex oldPackIndex;
final PackBitmapIndex newPackIndex;
- private final ObjectIdOwnerMap<StoredBitmap> convertedBitmaps;
private final BitSet inflated;
private final int[] prevToNewMapping;
@@ -65,7 +63,6 @@
private PackBitmapIndexRemapper(PackBitmapIndex newPackIndex) {
this.oldPackIndex = null;
this.newPackIndex = newPackIndex;
- this.convertedBitmaps = null;
this.inflated = null;
this.prevToNewMapping = null;
}
@@ -74,7 +71,6 @@
BasePackBitmapIndex oldPackIndex, PackBitmapIndex newPackIndex) {
this.oldPackIndex = oldPackIndex;
this.newPackIndex = newPackIndex;
- convertedBitmaps = new ObjectIdOwnerMap<>();
inflated = new BitSet(newPackIndex.getObjectCount());
prevToNewMapping = new int[oldPackIndex.getObjectCount()];
@@ -152,10 +148,6 @@
if (bitmap != null || oldPackIndex == null)
return bitmap;
- StoredBitmap stored = convertedBitmaps.get(objectId);
- if (stored != null)
- return stored.getBitmap();
-
StoredBitmap oldBitmap = oldPackIndex.getBitmaps().get(objectId);
if (oldBitmap == null)
return null;
@@ -168,8 +160,6 @@
inflated.set(prevToNewMapping[i.next()]);
bitmap = inflated.toEWAHCompressedBitmap();
bitmap.trim();
- convertedBitmaps.add(
- new StoredBitmap(objectId, bitmap, null, oldBitmap.getFlags()));
return bitmap;
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java
new file mode 100644
index 0000000..33c478e
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2020, Google LLC and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.pack;
+
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+
+/**
+ * A commit object for which a bitmap index should be built.
+ */
+public final class BitmapCommit extends ObjectId {
+
+ private final boolean reuseWalker;
+
+ private final int flags;
+
+ private final boolean addToIndex;
+
+ BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags) {
+ super(objectId);
+ this.reuseWalker = reuseWalker;
+ this.flags = flags;
+ this.addToIndex = false;
+ }
+
+ BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags,
+ boolean addToIndex) {
+ super(objectId);
+ this.reuseWalker = reuseWalker;
+ this.flags = flags;
+ this.addToIndex = addToIndex;
+ }
+
+ boolean isReuseWalker() {
+ return reuseWalker;
+ }
+
+ int getFlags() {
+ return flags;
+ }
+
+ /**
+ * Whether corresponding bitmap should be added to PackBitmapIndexBuilder.
+ *
+ * @return true if the corresponding bitmap should be added to
+ * PackBitmapIndexBuilder.
+ */
+ public boolean isAddToIndex() {
+ return addToIndex;
+ }
+
+ /**
+ * Get a builder of BitmapCommit whose object id is {@code objId}.
+ *
+ * @param objId
+ * the object id of the BitmapCommit
+ * @return a BitmapCommit builder with object id set.
+ */
+ public static Builder newBuilder(AnyObjectId objId) {
+ return new Builder().setId(objId);
+ }
+
+ /**
+ * Get a builder of BitmapCommit whose fields are copied from
+ * {@code commit}.
+ *
+ * @param commit
+ * the bitmap commit the builder is copying from
+ * @return a BitmapCommit build with fields copied from an existing bitmap
+ * commit.
+ */
+ public static Builder copyFrom(BitmapCommit commit) {
+ return new Builder().setId(commit)
+ .setReuseWalker(commit.isReuseWalker())
+ .setFlags(commit.getFlags())
+ .setAddToIndex(commit.isAddToIndex());
+ }
+
+ /**
+ * Builder of BitmapCommit.
+ */
+ public static class Builder {
+ private AnyObjectId objectId;
+
+ private boolean reuseWalker;
+
+ private int flags;
+
+ private boolean addToIndex;
+
+ // Prevent default constructor.
+ private Builder() {
+ }
+
+ /**
+ * Set objectId of the builder.
+ *
+ * @param objectId
+ * the object id of the BitmapCommit
+ * @return the builder itself
+ */
+ public Builder setId(AnyObjectId objectId) {
+ this.objectId = objectId;
+ return this;
+ }
+
+ /**
+ * Set reuseWalker of the builder.
+ *
+ * @param reuseWalker
+ * whether the BitmapCommit should reuse bitmap walker when
+ * walking objects
+ * @return the builder itself
+ */
+ public Builder setReuseWalker(boolean reuseWalker) {
+ this.reuseWalker = reuseWalker;
+ return this;
+ }
+
+ /**
+ * Set flags of the builder.
+ *
+ * @param flags
+ * the flags of the BitmapCommit
+ * @return the builder itself
+ */
+ public Builder setFlags(int flags) {
+ this.flags = flags;
+ return this;
+ }
+
+ /**
+ * Set whether whether the bitmap of the BitmapCommit should be added to
+ * PackBitmapIndexBuilder when building bitmap index file.
+ *
+ * @param addToIndex
+ * whether the bitmap of the BitmapCommit should be added to
+ * PackBitmapIndexBuilder when building bitmap index file
+ * @return the builder itself
+ */
+ public Builder setAddToIndex(boolean addToIndex) {
+ this.addToIndex = addToIndex;
+ return this;
+ }
+
+ /**
+ * Builds BitmapCommit from the builder.
+ *
+ * @return the new BitmapCommit.
+ */
+ public BitmapCommit build() {
+ return new BitmapCommit(objectId, reuseWalker, flags,
+ addToIndex);
+ }
+ }
+}
\ No newline at end of file
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 75dd345..824c62a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -2313,14 +2313,14 @@
PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
reader, writeBitmaps, pm, stats.interestingObjects, config);
- Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer
+ Collection<BitmapCommit> selectedCommits = bitmapPreparer
.selectCommits(numCommits, excludeFromBitmapSelection);
beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size());
BitmapWalker walker = bitmapPreparer.newBitmapWalker();
AnyObjectId last = null;
- for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) {
+ for (BitmapCommit cmit : selectedCommits) {
if (!cmit.isReuseWalker()) {
walker = bitmapPreparer.newBitmapWalker();
}
@@ -2331,8 +2331,14 @@
throw new IllegalStateException(MessageFormat.format(
JGitText.get().bitmapMissingObject, cmit.name(),
last.name()));
- last = cmit;
- writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags());
+ last = BitmapCommit.copyFrom(cmit).build();
+ writeBitmaps.processBitmapForWrite(cmit, bitmap.build(),
+ cmit.getFlags());
+
+ // The bitmap walker should stop when the walk hits the previous
+ // commit, which saves time.
+ walker.setPrevCommit(last);
+ walker.setPrevBitmap(bitmap);
pm.update(1);
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
index 51b4993..f1ede2a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
@@ -58,6 +58,8 @@
private static final int DAY_IN_SECONDS = 24 * 60 * 60;
+ private static final int DISTANCE_THRESHOLD = 2000;
+
private static final Comparator<RevCommit> ORDER_BY_REVERSE_TIMESTAMP = (
RevCommit a, RevCommit b) -> Integer
.signum(b.getCommitTime() - a.getCommitTime());
@@ -244,6 +246,7 @@
// This commit is selected.
// Calculate where to look for the next one.
int flags = nextFlg;
+ int currDist = distanceFromTip;
nextIn = nextSpan(distanceFromTip);
nextFlg = nextIn == distantCommitSpan
? PackBitmapIndex.FLAG_REUSE
@@ -279,8 +282,17 @@
longestAncestorChain = new ArrayList<>();
chains.add(longestAncestorChain);
}
- longestAncestorChain.add(new BitmapCommit(c,
- !longestAncestorChain.isEmpty(), flags));
+
+ // The commit bc should reuse bitmap walker from its close
+ // ancestor. And the bitmap of bc should only be added to
+ // PackBitmapIndexBuilder when it's an old enough
+ // commit,i.e. distance from tip should be greater than
+ // DISTANCE_THRESHOLD, to save memory.
+ BitmapCommit bc = BitmapCommit.newBuilder(c).setFlags(flags)
+ .setAddToIndex(currDist >= DISTANCE_THRESHOLD)
+ .setReuseWalker(!longestAncestorChain.isEmpty())
+ .build();
+ longestAncestorChain.add(bc);
writeBitmaps.addBitmap(c, bitmap, 0);
}
@@ -288,12 +300,12 @@
selections.addAll(chain);
}
}
- writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps.
// Add the remaining peeledWant
for (AnyObjectId remainingWant : selectionHelper.newWants) {
selections.add(new BitmapCommit(remainingWant, false, 0));
}
+ writeBitmaps.resetBitmaps(selections.size()); // Remove the temporary commit bitmaps.
pm.endTask();
return selections;
@@ -468,28 +480,6 @@
}
/**
- * A commit object for which a bitmap index should be built.
- */
- static final class BitmapCommit extends ObjectId {
- private final boolean reuseWalker;
- private final int flags;
-
- BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags) {
- super(objectId);
- this.reuseWalker = reuseWalker;
- this.flags = flags;
- }
-
- boolean isReuseWalker() {
- return reuseWalker;
- }
-
- int getFlags() {
- return flags;
- }
- }
-
- /**
* Container for state used in the first phase of selecting commits, which
* walks all of the reachable commits via the branch tips that are not
* covered by a previous pack's bitmaps ({@code newWants}) and stores them
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java
index f61286d..f6695bd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java
@@ -14,6 +14,8 @@
import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
+import com.googlecode.javaewah.EWAHCompressedBitmap;
+
/**
* A compressed bitmap representation of the entire object graph.
*
@@ -81,6 +83,14 @@
*/
@Override
Iterator<BitmapObject> iterator();
+
+ /**
+ * Returns the corresponding raw compressed EWAH bitmap of the bitmap.
+ *
+ * @return the corresponding {@code EWAHCompressedBitmap}
+ * @since 5.8
+ */
+ EWAHCompressedBitmap retrieveCompressed();
}
/**
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java
index 023962e..aaecd23 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java
@@ -16,6 +16,7 @@
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.internal.revwalk.AddToBitmapFilter;
+import org.eclipse.jgit.internal.revwalk.AddToBitmapWithCacheFilter;
import org.eclipse.jgit.internal.revwalk.AddUnseenToBitmapFilter;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.BitmapIndex;
@@ -41,6 +42,11 @@
private long countOfBitmapIndexMisses;
+ // Cached bitmap and commit to save walk time.
+ private AnyObjectId prevCommit;
+
+ private Bitmap prevBitmap;
+
/**
* Create a BitmapWalker.
*
@@ -56,6 +62,28 @@
}
/**
+ * Set the cached commit for the walker.
+ *
+ * @param prevCommit
+ * the cached commit.
+ * @since 5.7
+ */
+ public void setPrevCommit(AnyObjectId prevCommit) {
+ this.prevCommit = prevCommit;
+ }
+
+ /**
+ * Set the bitmap associated with the cached commit for the walker.
+ *
+ * @param prevBitmap
+ * the bitmap associated with the cached commit.
+ * @since 5.7
+ */
+ public void setPrevBitmap(Bitmap prevBitmap) {
+ this.prevBitmap = prevBitmap;
+ }
+
+ /**
* Return the number of objects that had to be walked because they were not covered by a
* bitmap.
*
@@ -169,7 +197,10 @@
}
if (marked) {
- if (seen == null) {
+ if (prevCommit != null) {
+ walker.setRevFilter(new AddToBitmapWithCacheFilter(prevCommit,
+ prevBitmap, bitmapResult));
+ } else if (seen == null) {
walker.setRevFilter(new AddToBitmapFilter(bitmapResult));
} else {
walker.setRevFilter(
