PackInserter: write object-size index with the pack Create an object-index when writing a pack with PackInserter if config says so. Change-Id: Ic7a3a92b8f8afc4d89e37b9bf8091f438fa9b8c1
diff --git a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF index 92371ae..01df135 100644 --- a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
@@ -20,6 +20,7 @@ org.eclipse.jgit.merge;version="[7.4.0,7.5.0)", org.eclipse.jgit.revwalk;version="[7.4.0,7.5.0)", org.eclipse.jgit.storage.file;version="[7.4.0,7.5.0)", + org.eclipse.jgit.storage.pack;version="[7.4.0,7.5.0)", org.eclipse.jgit.transport;version="7.4.0", org.eclipse.jgit.treewalk;version="[7.4.0,7.5.0)", org.eclipse.jgit.treewalk.filter;version="[7.4.0,7.5.0)",
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 8504303..cc43d3c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
@@ -53,6 +53,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; @@ -77,12 +78,14 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.util.IO; @@ -489,6 +492,38 @@ public void readBackSmallObjectBeforeLargeObject() throws Exception { } } + @Test + public void createsObjectSizeIndex() throws Exception { + FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig(); + jGitConfig.setInt( + ConfigConstants.CONFIG_PACK_SECTION, + null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 10); + jGitConfig.save(); + byte[] oneBlob = Constants.encode("a blob with some content"); + byte[] anotherBlob = Constants.encode("some more contents"); + byte[] streamMeBlob = Constants.encode("some more content to write"); + + ObjectId oneBlobOid, anotherBlobOid, streamMeBlobOid; + try (PackInserter ins = newInserter()) { + oneBlobOid = ins.insert(OBJ_BLOB, oneBlob); + anotherBlobOid = ins.insert(OBJ_BLOB, anotherBlob); + streamMeBlobOid = ins.insert(OBJ_BLOB, streamMeBlob.length, + new ByteArrayInputStream(streamMeBlob)); + ins.flush(); + } + + List<Pack> listPacks = listPacks(db); + assertEquals(1, listPacks.size()); + Pack thePack = listPacks.get(0); + assertTrue(thePack.hasObjectSizeIndex()); + assertEquals(oneBlob.length, thePack.getIndexedObjectSize(oneBlobOid)); + assertEquals(anotherBlob.length, + thePack.getIndexedObjectSize(anotherBlobOid)); + assertEquals(streamMeBlob.length, + thePack.getIndexedObjectSize(streamMeBlobOid)); + } + private List<Pack> listPacks() throws Exception { List<Pack> fromOpenDb = listPacks(db); List<Pack> reopened; @@ -549,7 +584,8 @@ private static void assertBlob(ObjectReader reader, ObjectId id, } private void assertPacksOnly() throws Exception { - new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")) + new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx") + && !f.endsWith(".objsize")) .assertNoBadFiles(db.getObjectDatabase().getDirectory()); }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 55e047b..97a854b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
@@ -166,7 +166,7 @@ public ObjectId insert(int type, byte[] data, int off, int len) long offset = beginObject(type, len); packOut.compress.write(data, off, len); packOut.compress.finish(); - return endObject(id, offset); + return endObject(id, offset, len, type); } @Override @@ -195,7 +195,7 @@ public ObjectId insert(int type, long len, InputStream in) len -= n; } packOut.compress.finish(); - return endObject(md.toObjectId(), offset); + return endObject(md.toObjectId(), offset, len, type); } private long beginObject(int type, long len) throws IOException { @@ -207,10 +207,12 @@ private long beginObject(int type, long len) throws IOException { return offset; } - private ObjectId endObject(ObjectId id, long offset) { + private ObjectId endObject(ObjectId id, long offset, long len, int type) { PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setType(type); obj.setOffset(offset); obj.setCRC((int) packOut.crc32.getValue()); + obj.setFullSize(len); objectList.add(obj); objectMap.addIfAbsent(obj); return id; @@ -223,6 +225,12 @@ private static File idxFor(File packFile) { p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ } + private static File getFileFor(File packFile, PackExt ext) { + String p = packFile.getName(); + return new File(packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ext.getExtension()); + } + private void beginPack() throws IOException { objectList = new BlockList<>(); objectMap = new ObjectIdOwnerMap<>(); @@ -272,7 +280,11 @@ public void flush() throws IOException { Collections.sort(objectList); File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - + File tmpObjSizeIdx = null; + if (pconfig.isWriteObjSizeIndex()) { + tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX); + writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig); + } PackFile realPack = new PackFile(db.getPackDirectory(), computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); @@ -297,6 +309,13 @@ public void flush() throws IOException { realIdx), e); } + if (tmpObjSizeIdx != null) { + PackFile realObjSizeIdx = realPack + .create(PackExt.OBJECT_SIZE_INDEX); + tmpObjSizeIdx.setReadOnly(); + FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE); + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(realPack); @@ -327,6 +346,15 @@ private static void writePackIndex(File idx, byte[] packHash, } } + private static void writeObjectSizeIndex(File objIdx, + List<PackedObjectInfo> list, PackConfig cfg) throws IOException { + try (OutputStream os = new FileOutputStream(objIdx)) { + PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter + .createWriter(os, cfg.getMinBytesForObjSizeIndex()); + w.write(list); + } + } + private ObjectId computeName(List<PackedObjectInfo> list) { SHA1 md = digest().reset(); byte[] buf = buffer();