Pack: getter for the indexed object size (when available) In some cases (e.g. filtering objects in a partial clone), the caller prefers the faster (but optional) indexed size rather than the slower read from storage. Expose the indexed size in Pack. The caller must check that the pack has index (with #hasObjectSizeIndex()) and ask only for objects in that pack. Change-Id: Ia04e40250a8fb4890ae556ace516944d191f24ef
diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index 2d00a85..c546ae9 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
@@ -73,6 +73,7 @@ import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; import org.eclipse.jgit.util.ChangeIdUtil; @@ -987,7 +988,7 @@ public void packAndPrune() throws Exception { ObjectDirectory odb = (ObjectDirectory) db.getObjectDatabase(); NullProgressMonitor m = NullProgressMonitor.INSTANCE; - final PackFile pack, idx; + PackFile pack; try (PackWriter pw = new PackWriter(db)) { Set<ObjectId> all = new HashSet<>(); for (Ref r : db.getRefDatabase().getRefs()) @@ -1002,12 +1003,22 @@ public void packAndPrune() throws Exception { } pack.setReadOnly(); - idx = pack.create(PackExt.INDEX); + PackFile idx = pack.create(PackExt.INDEX); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(idx))) { pw.writeIndex(out); } idx.setReadOnly(); + + PackConfig pc = new PackConfig(db); + if (pc.getMinBytesForObjSizeIndex() >= 0) { + PackFile oidx = pack.create(PackExt.OBJECT_SIZE_INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(oidx))) { + pw.writeObjectSizeIndex(out); + } + oidx.setReadOnly(); + } } odb.openPack(pack);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index e150945..016a6af 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
@@ -10,6 +10,7 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -29,6 +30,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import java.util.zip.Deflater; import org.eclipse.jgit.errors.LargeObjectException; @@ -39,6 +41,7 @@ import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRng; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -47,6 +50,7 @@ import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.transport.PackParser; import org.eclipse.jgit.transport.PackedObjectInfo; @@ -295,6 +299,29 @@ public void testConfigurableStreamFileThreshold() throws Exception { } } + @Test + public void testObjectSize() throws Exception { + byte[] data = getRng().nextBytes(300); + RevBlob aBlob = tr.blob(data); + RevCommit aCommit = tr.branch("master").commit().add("A", aBlob).create(); + repo.getConfig().setInt(CONFIG_PACK_SECTION, null, ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0); + tr.packAndPrune(); + + List<Pack> packs = repo.getObjectDatabase().getPacks().stream().collect(Collectors.toList()); + assertEquals(1, packs.size()); + // Indexed object + assertEquals(300, packs.get(0).getIndexedObjectSize(aBlob)); + assertEquals(300, packs.get(0).getObjectSize(wc, aBlob)); + // Non indexed object + assertEquals(-1, packs.get(0).getIndexedObjectSize(aCommit)); + assertEquals(168, packs.get(0).getObjectSize(wc, aCommit)); + // Object not in pack + assertEquals(-1, packs.get(0).getObjectSize(wc, + ObjectId.fromString("1111111111111111111111111111111111111111"))); + assertEquals(-1, packs.get(0).getIndexedObjectSize( + ObjectId.fromString("1111111111111111111111111111111111111111"))); + } + private static byte[] clone(int first, byte[] base) { byte[] r = new byte[base.length]; System.arraycopy(base, 1, r, 1, r.length - 1);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 5813d39..f2f5494 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -126,6 +128,10 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); + private volatile PackObjectSizeIndex loadedObjSizeIdx; + + private volatile boolean attemptLoadObjSizeIdx; + private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty(); /** @@ -210,6 +216,52 @@ private synchronized PackIndex idx() throws IOException { } } + private PackObjectSizeIndex objectSizeIndex() throws IOException { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + if (attemptLoadObjSizeIdx) { + return null; + } + + synchronized (this) { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + PackObjectSizeIndex sizeIdx; + try { + long start = System.currentTimeMillis(); + PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX); + if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) { + attemptLoadObjSizeIdx = true; + return null; + } + sizeIdx = PackObjectSizeIndexLoader.load( + new FileInputStream(sizeIdxFile.getAbsoluteFile())); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + sizeIdxFile.getAbsolutePath(), + Float.valueOf( + sizeIdxFile.length() / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() - start))); + } + + loadedObjSizeIdx = sizeIdx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + return null; + } finally { + attemptLoadObjSizeIdx = true; + } + } + + return loadedObjSizeIdx; + } + /** * Get the File object which locates this pack on disk. * @@ -231,6 +283,62 @@ public PackIndex getIndex() throws IOException { } /** + * Get the object size index for this pack file + * + * @return the object size index for this pack file if it exists (null + * otherwise) + * @throws IOException + * problem reading the index + */ + public boolean hasObjectSizeIndex() throws IOException { + return objectSizeIndex() != null; + } + + /** + * Number of objects in the object-size index of this pack + * + * @return number of objects in the index (0 if either the index is empty or + * it doesn't exist) + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getObjectSizeIndexCount() throws IOException { + if (!hasObjectSizeIndex()) { + return 0; + } + + return objectSizeIndex().getObjectCount(); + } + + /** + * Return the size of the object from the object-size index. + * + * Caller MUST check that the pack has object-size index + * ({@link #hasObjectSizeIndex()}) and that the pack contains the object. + * + * @param id + * object id of an object in the pack + * @return size of the object from the index. Negative if the object is not + * in the index. + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getIndexedObjectSize(AnyObjectId id) throws IOException { + int idxPos = idx().findPosition(id); + if (idxPos < 0) { + return -1; + } + + PackObjectSizeIndex sizeIdx = objectSizeIndex(); + if (sizeIdx == null) { + throw new IllegalStateException( + "Asking indexed size from a pack without object size index"); //$NON-NLS-1$ + } + + return sizeIdx.getSize(idxPos); + } + + /** * Get name extracted from {@code pack-*.pack} pattern. * * @return name extracted from {@code pack-*.pack} pattern.