Merge "Merge branch 'stable-7.3'"
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java
new file mode 100644
index 0000000..1a05d88
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright (C) 2025, Google LLC. and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import org.eclipse.jgit.internal.storage.pack.PackExt;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.pack.PackConfig;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GcObjectSizeIndexTest extends GcTestCase {
+
+ @Test
+ public void gc_2commits_noSizeLimit_blobsInIndex() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(4, stats.numberOfSizeIndexedObjects);
+
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ Pack pack = getOnlyPack(repo);
+ assertEquals(7, pack.getIndexedObjectSize(blobA1));
+ assertEquals(11, pack.getIndexedObjectSize(blobA2));
+ assertEquals(1, pack.getIndexedObjectSize(blobB1));
+ assertEquals(2, pack.getIndexedObjectSize(blobB2));
+ }
+
+ @Test
+ public void gc_2commits_sizeLimit_biggerBlobsInIndex() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 5);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ Pack pack = getOnlyPack(repo);
+ assertEquals(7, pack.getIndexedObjectSize(blobA1));
+ assertEquals(11, pack.getIndexedObjectSize(blobA2));
+ assertEquals(-1, pack.getIndexedObjectSize(blobB1));
+ assertEquals(-1, pack.getIndexedObjectSize(blobB2));
+ }
+
+ @Test
+ public void gc_2commits_disableSizeIdx_noIdx() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, -1);
+ gc.gc().get();
+
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(0, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_alreadyPacked_noChanges()
+ throws Exception {
+ tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B")
+ .create();
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+
+ // Do the gc again and check that it hasn't changed anything
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_twoReachableCommits_oneUnreachable_twoPacks()
+ throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevCommit first = bb.commit().add("A", "A").add("B", "B").create();
+ bb.commit().add("A", "A2").add("B", "B2").create();
+ tr.update("refs/heads/master", first);
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(8, stats.numberOfPackedObjects);
+ assertEquals(2, stats.numberOfPackFiles);
+ assertEquals(4, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_preserved_objSizeIdxIsPreserved() throws Exception {
+ Collection<Pack> oldPacks = preserveOldPacks();
+ assertEquals(1, oldPacks.size());
+ PackFile preserved = oldPacks.iterator().next().getPackFile()
+ .create(PackExt.OBJECT_SIZE_INDEX)
+ .createPreservedForDirectory(
+ repo.getObjectDatabase().getPreservedDirectory());
+ assertTrue(preserved.exists());
+ }
+
+ @Test
+ public void gc_preserved_prune_noPreserves() throws Exception {
+ preserveOldPacks();
+ configureGc(gc, 0).setPrunePreserved(true);
+ gc.gc().get();
+
+ assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists());
+ }
+
+ private Collection<Pack> preserveOldPacks() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ bb.commit().message("P").add("P", "P").create();
+
+ // pack loose object into packfile
+ configureGc(gc, 0);
+ gc.setExpireAgeMillis(0);
+ gc.gc().get();
+ Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase()
+ .getPacks();
+ PackFile oldPackfile = oldPacks.iterator().next().getPackFile();
+ assertTrue(oldPackfile.exists());
+
+ fsTick();
+ bb.commit().message("B").add("B", "Q").create();
+
+ // repack again but now without a grace period for packfiles. We should
+ // end up with a new packfile and the old one should be placed in the
+ // preserved directory
+ gc.setPackExpireAgeMillis(0);
+ configureGc(gc, 0).setPreserveOldPacks(true);
+ gc.gc().get();
+
+ File preservedPackFile = oldPackfile.createPreservedForDirectory(
+ repo.getObjectDatabase().getPreservedDirectory());
+ assertTrue(preservedPackFile.exists());
+ return oldPacks;
+ }
+
+ @Ignore
+ public void testPruneAndRestoreOldPacks() throws Exception {
+ String tempRef = "refs/heads/soon-to-be-unreferenced";
+ TestRepository<FileRepository>.BranchBuilder bb = tr.branch(tempRef);
+ bb.commit().add("A", "A").add("B", "B").create();
+
+ // Verify setup conditions
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+
+ // Force all referenced objects into packs (to avoid having loose objects)
+ configureGc(gc, 0);
+ gc.setExpireAgeMillis(0);
+ gc.setPackExpireAgeMillis(0);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+
+ // Delete the temp ref, orphaning its commit
+ RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
+ update.setForceUpdate(true);
+ ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it!
+ RefUpdate.Result result = update.delete();
+ assertEquals(RefUpdate.Result.FORCED, result);
+
+ fsTick();
+
+ // Repack with only orphaned commit, so packfile will be pruned
+ configureGc(gc, 0).setPreserveOldPacks(true);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ assertEquals(0, stats.numberOfPackFiles);
+
+ // Restore the temp ref to the deleted commit, should restore old-packs!
+ update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
+ update.setNewObjectId(objectId);
+ update.setExpectedOldObjectId(null);
+ result = update.update();
+ assertEquals(RefUpdate.Result.NEW, result);
+
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ }
+
+ private PackConfig configureGc(GC myGc, int minSize) {
+ PackConfig pconfig = new PackConfig(repo);
+ pconfig.setMinBytesForObjSizeIndex(minSize);
+ myGc.setPackConfig(pconfig);
+ return pconfig;
+ }
+
+ private Pack getOnlyPack(FileRepository fileRepo)
+ throws IOException {
+ Collection<Pack> packs = fileRepo.getObjectDatabase().getPacks();
+ if (packs.size() != 1) {
+ throw new IOException("More than one pack");
+ }
+
+ return packs.iterator().next();
+ }
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index c08a92e..97473bb 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -14,6 +14,7 @@
import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
@@ -131,7 +132,7 @@ public class GC {
private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP);
private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX,
- REVERSE_INDEX);
+ REVERSE_INDEX, OBJECT_SIZE_INDEX);
private static final int DEFAULT_AUTOPACKLIMIT = 50;
@@ -412,6 +413,10 @@ private void deleteOldPacks(Collection<Pack> oldPacks,
*/
private void removeOldPack(PackFile packFile, int deleteOptions)
throws IOException {
+ if (!packFile.exists()) {
+ return;
+ }
+
if (pconfig.isPreserveOldPacks()) {
File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
FileUtils.mkdir(oldPackDir, true);
@@ -1340,6 +1345,7 @@ private Pack writePack(@NonNull Set<? extends ObjectId> want,
idxChannel.force(true);
}
+
if (pw.isReverseIndexEnabled()) {
File tmpReverseIndexFile = new File(packdir,
tmpBase + REVERSE_INDEX.getTmpExtension());
@@ -1356,6 +1362,19 @@ private Pack writePack(@NonNull Set<? extends ObjectId> want,
.newOutputStream(channel)) {
pw.writeReverseIndex(stream);
channel.force(true);
+ }
+ }
+
+ // write the object size
+ if (pconfig.isWriteObjSizeIndex()) {
+ File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$
+ tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx);
+ try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx);
+ FileChannel idxChannel = fos.getChannel();
+ OutputStream idxStream = Channels
+ .newOutputStream(idxChannel)) {
+ pw.writeObjectSizeIndex(idxStream);
+ idxChannel.force(true);
}
}
@@ -1525,6 +1544,11 @@ public static class RepoStatistics {
*/
public long numberOfBitmaps;
+ /**
+ * The number of objects in the size-index of the packs
+ */
+ public long numberOfSizeIndexedObjects;
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
@@ -1540,6 +1564,8 @@ public String toString() {
b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
+ b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$
+ .append(numberOfSizeIndexedObjects);
return b.toString();
}
}
@@ -1548,7 +1574,7 @@ public String toString() {
* Returns information about objects and pack files for a FileRepository.
*
* @return information about objects and pack files for a FileRepository
- * @throws java.io.IOException
+ * @throws java.io.IOException
* if an IO error occurred
*/
public RepoStatistics getStatistics() throws IOException {
@@ -1560,16 +1586,16 @@ public RepoStatistics getStatistics() throws IOException {
ret.numberOfPackedObjects += packedObjects;
ret.numberOfPackFiles++;
ret.sizeOfPackedObjects += p.getPackFile().length();
+ ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount();
if (p.getBitmapIndex() != null) {
ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount();
if (latestBitmapTime == 0L) {
latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli();
}
- }
- else if (latestBitmapTime == 0L) {
- ret.numberOfPackFilesSinceBitmap++;
- ret.numberOfObjectsSinceBitmap += packedObjects;
- }
+ } else if (latestBitmapTime == 0L) {
+ ret.numberOfPackFilesSinceBitmap++;
+ ret.numberOfObjectsSinceBitmap += packedObjects;
+ }
}
File objDir = repo.getObjectsDirectory();
String[] fanout = objDir.list();
