| /* |
| * Copyright (C) 2017, Google Inc. |
| * and other copyright owners as documented in the project's IP log. |
| * |
| * This program and the accompanying materials are made available |
| * under the terms of the Eclipse Distribution License v1.0 which |
| * accompanies this distribution, is reproduced below, and is |
| * available at http://www.eclipse.org/org/documents/edl-v10.php |
| * |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * - Neither the name of the Eclipse Foundation, Inc. nor the |
| * names of its contributors may be used to endorse or promote |
| * products derived from this software without specific prior |
| * written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
| * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
| * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| package org.eclipse.jgit.internal.storage.file; |
| |
| import static java.util.Comparator.comparing; |
| import static java.util.stream.Collectors.toList; |
| import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; |
| import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; |
| import static org.hamcrest.MatcherAssert.assertThat; |
| import static org.hamcrest.Matchers.greaterThan; |
| import static org.hamcrest.Matchers.lessThan; |
| import static org.junit.Assert.assertArrayEquals; |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertNotEquals; |
| import static org.junit.Assert.fail; |
| |
| import java.io.ByteArrayInputStream; |
| import java.io.File; |
| import java.io.IOException; |
| import java.nio.file.FileVisitResult; |
| import java.nio.file.Files; |
| import java.nio.file.Path; |
| import java.nio.file.SimpleFileVisitor; |
| import java.nio.file.attribute.BasicFileAttributes; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.List; |
| import java.util.Random; |
| import java.util.function.Predicate; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| |
| import org.eclipse.jgit.dircache.DirCache; |
| import org.eclipse.jgit.dircache.DirCacheBuilder; |
| import org.eclipse.jgit.dircache.DirCacheEntry; |
| import org.eclipse.jgit.errors.MissingObjectException; |
| import org.eclipse.jgit.junit.RepositoryTestCase; |
| import org.eclipse.jgit.lib.CommitBuilder; |
| import org.eclipse.jgit.lib.Constants; |
| import org.eclipse.jgit.lib.FileMode; |
| import org.eclipse.jgit.lib.ObjectId; |
| import org.eclipse.jgit.lib.ObjectLoader; |
| import org.eclipse.jgit.lib.ObjectReader; |
| import org.eclipse.jgit.lib.ObjectStream; |
| import org.eclipse.jgit.storage.file.WindowCacheConfig; |
| import org.eclipse.jgit.treewalk.CanonicalTreeParser; |
| import org.eclipse.jgit.util.IO; |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| @SuppressWarnings("boxing") |
| public class PackInserterTest extends RepositoryTestCase { |
| private WindowCacheConfig origWindowCacheConfig; |
| |
| private static final Random random = new Random(0); |
| |
| @Before |
| public void setWindowCacheConfig() { |
| origWindowCacheConfig = new WindowCacheConfig(); |
| origWindowCacheConfig.install(); |
| } |
| |
| @After |
| public void resetWindowCacheConfig() { |
| origWindowCacheConfig.install(); |
| } |
| |
| @Before |
| public void emptyAtSetUp() throws Exception { |
| assertEquals(0, listPacks().size()); |
| assertNoObjects(); |
| } |
| |
| @Test |
| public void noFlush() throws Exception { |
| try (PackInserter ins = newInserter()) { |
| ins.insert(OBJ_BLOB, Constants.encode("foo contents")); |
| // No flush. |
| } |
| assertNoObjects(); |
| } |
| |
| @Test |
| public void flushEmptyPack() throws Exception { |
| try (PackInserter ins = newInserter()) { |
| ins.flush(); |
| } |
| assertNoObjects(); |
| } |
| |
| @Test |
| public void singlePack() throws Exception { |
| ObjectId blobId; |
| byte[] blob = Constants.encode("foo contents"); |
| ObjectId treeId; |
| ObjectId commitId; |
| byte[] commit; |
| try (PackInserter ins = newInserter()) { |
| blobId = ins.insert(OBJ_BLOB, blob); |
| |
| DirCache dc = DirCache.newInCore(); |
| DirCacheBuilder b = dc.builder(); |
| DirCacheEntry dce = new DirCacheEntry("foo"); |
| dce.setFileMode(FileMode.REGULAR_FILE); |
| dce.setObjectId(blobId); |
| b.add(dce); |
| b.finish(); |
| treeId = dc.writeTree(ins); |
| |
| CommitBuilder cb = new CommitBuilder(); |
| cb.setTreeId(treeId); |
| cb.setAuthor(author); |
| cb.setCommitter(committer); |
| cb.setMessage("Commit message"); |
| commit = cb.toByteArray(); |
| commitId = ins.insert(cb); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| List<Pack> packs = listPacks(); |
| assertEquals(1, packs.size()); |
| assertEquals(3, packs.get(0).getObjectCount()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId, blob); |
| |
| CanonicalTreeParser treeParser = |
| new CanonicalTreeParser(null, reader, treeId); |
| assertEquals("foo", treeParser.getEntryPathString()); |
| assertEquals(blobId, treeParser.getEntryObjectId()); |
| |
| ObjectLoader commitLoader = reader.open(commitId); |
| assertEquals(OBJ_COMMIT, commitLoader.getType()); |
| assertArrayEquals(commit, commitLoader.getBytes()); |
| } |
| } |
| |
| @Test |
| public void multiplePacks() throws Exception { |
| ObjectId blobId1; |
| ObjectId blobId2; |
| byte[] blob1 = Constants.encode("blob1"); |
| byte[] blob2 = Constants.encode("blob2"); |
| |
| try (PackInserter ins = newInserter()) { |
| blobId1 = ins.insert(OBJ_BLOB, blob1); |
| ins.flush(); |
| blobId2 = ins.insert(OBJ_BLOB, blob2); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| List<Pack> packs = listPacks(); |
| assertEquals(2, packs.size()); |
| assertEquals(1, packs.get(0).getObjectCount()); |
| assertEquals(1, packs.get(1).getObjectCount()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId1, blob1); |
| assertBlob(reader, blobId2, blob2); |
| } |
| } |
| |
| @Test |
| public void largeBlob() throws Exception { |
| ObjectId blobId; |
| byte[] blob = newLargeBlob(); |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob.length, greaterThan(ins.getBufferSize())); |
| blobId = |
| ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| Collection<Pack> packs = listPacks(); |
| assertEquals(1, packs.size()); |
| Pack p = packs.iterator().next(); |
| assertEquals(1, p.getObjectCount()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId, blob); |
| } |
| } |
| |
| @Test |
| public void overwriteExistingPack() throws Exception { |
| ObjectId blobId; |
| byte[] blob = Constants.encode("foo contents"); |
| |
| try (PackInserter ins = newInserter()) { |
| blobId = ins.insert(OBJ_BLOB, blob); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| List<Pack> packs = listPacks(); |
| assertEquals(1, packs.size()); |
| Pack pack = packs.get(0); |
| assertEquals(1, pack.getObjectCount()); |
| |
| String inode = getInode(pack.getPackFile()); |
| |
| try (PackInserter ins = newInserter()) { |
| ins.checkExisting(false); |
| assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| packs = listPacks(); |
| assertEquals(1, packs.size()); |
| pack = packs.get(0); |
| assertEquals(1, pack.getObjectCount()); |
| |
| if (inode != null) { |
| // Old file was overwritten with new file, although objects were |
| // equivalent. |
| assertNotEquals(inode, getInode(pack.getPackFile())); |
| } |
| } |
| |
| @Test |
| public void checkExisting() throws Exception { |
| ObjectId blobId; |
| byte[] blob = Constants.encode("foo contents"); |
| |
| try (PackInserter ins = newInserter()) { |
| blobId = ins.insert(OBJ_BLOB, blob); |
| ins.insert(OBJ_BLOB, Constants.encode("another blob")); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(1, listPacks().size()); |
| |
| try (PackInserter ins = newInserter()) { |
| assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(1, listPacks().size()); |
| |
| try (PackInserter ins = newInserter()) { |
| ins.checkExisting(false); |
| assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(2, listPacks().size()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId, blob); |
| } |
| } |
| |
| @Test |
| public void insertSmallInputStreamRespectsCheckExisting() throws Exception { |
| ObjectId blobId; |
| byte[] blob = Constants.encode("foo contents"); |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob.length, lessThan(ins.getBufferSize())); |
| blobId = ins.insert(OBJ_BLOB, blob); |
| ins.insert(OBJ_BLOB, Constants.encode("another blob")); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(1, listPacks().size()); |
| |
| try (PackInserter ins = newInserter()) { |
| assertEquals(blobId, |
| ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(1, listPacks().size()); |
| } |
| |
| @Test |
| public void insertLargeInputStreamBypassesCheckExisting() throws Exception { |
| ObjectId blobId; |
| byte[] blob = newLargeBlob(); |
| |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob.length, greaterThan(ins.getBufferSize())); |
| blobId = ins.insert(OBJ_BLOB, blob); |
| ins.insert(OBJ_BLOB, Constants.encode("another blob")); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(1, listPacks().size()); |
| |
| try (PackInserter ins = newInserter()) { |
| assertEquals(blobId, |
| ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); |
| ins.flush(); |
| } |
| |
| assertPacksOnly(); |
| assertEquals(2, listPacks().size()); |
| } |
| |
| @Test |
| public void readBackSmallFiles() throws Exception { |
| ObjectId blobId1; |
| ObjectId blobId2; |
| ObjectId blobId3; |
| byte[] blob1 = Constants.encode("blob1"); |
| byte[] blob2 = Constants.encode("blob2"); |
| byte[] blob3 = Constants.encode("blob3"); |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob1.length, lessThan(ins.getBufferSize())); |
| blobId1 = ins.insert(OBJ_BLOB, blob1); |
| |
| try (ObjectReader reader = ins.newReader()) { |
| assertBlob(reader, blobId1, blob1); |
| } |
| |
| // Read-back should not mess up the file pointer. |
| blobId2 = ins.insert(OBJ_BLOB, blob2); |
| ins.flush(); |
| |
| blobId3 = ins.insert(OBJ_BLOB, blob3); |
| } |
| |
| assertPacksOnly(); |
| List<Pack> packs = listPacks(); |
| assertEquals(1, packs.size()); |
| assertEquals(2, packs.get(0).getObjectCount()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId1, blob1); |
| assertBlob(reader, blobId2, blob2); |
| |
| try { |
| reader.open(blobId3); |
| fail("Expected MissingObjectException"); |
| } catch (MissingObjectException expected) { |
| // Expected. |
| } |
| } |
| } |
| |
| @Test |
| public void readBackLargeFile() throws Exception { |
| ObjectId blobId; |
| byte[] blob = newLargeBlob(); |
| |
| WindowCacheConfig wcc = new WindowCacheConfig(); |
| wcc.setStreamFileThreshold(1024); |
| wcc.install(); |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); |
| } |
| |
| try (PackInserter ins = newInserter()) { |
| blobId = ins.insert(OBJ_BLOB, blob); |
| |
| try (ObjectReader reader = ins.newReader()) { |
| // Double-check threshold is propagated. |
| assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); |
| assertBlob(reader, blobId, blob); |
| } |
| } |
| |
| assertPacksOnly(); |
| // Pack was streamed out to disk and read back from the temp file, but |
| // ultimately rolled back and deleted. |
| assertEquals(0, listPacks().size()); |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| try { |
| reader.open(blobId); |
| fail("Expected MissingObjectException"); |
| } catch (MissingObjectException expected) { |
| // Expected. |
| } |
| } |
| } |
| |
| @Test |
| public void readBackFallsBackToRepo() throws Exception { |
| ObjectId blobId; |
| byte[] blob = Constants.encode("foo contents"); |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob.length, lessThan(ins.getBufferSize())); |
| blobId = ins.insert(OBJ_BLOB, blob); |
| ins.flush(); |
| } |
| |
| try (PackInserter ins = newInserter(); |
| ObjectReader reader = ins.newReader()) { |
| assertBlob(reader, blobId, blob); |
| } |
| } |
| |
| @Test |
| public void readBackSmallObjectBeforeLargeObject() throws Exception { |
| WindowCacheConfig wcc = new WindowCacheConfig(); |
| wcc.setStreamFileThreshold(1024); |
| wcc.install(); |
| |
| ObjectId blobId1; |
| ObjectId blobId2; |
| ObjectId largeId; |
| byte[] blob1 = Constants.encode("blob1"); |
| byte[] blob2 = Constants.encode("blob2"); |
| byte[] largeBlob = newLargeBlob(); |
| try (PackInserter ins = newInserter()) { |
| assertThat(blob1.length, lessThan(ins.getBufferSize())); |
| assertThat(largeBlob.length, greaterThan(ins.getBufferSize())); |
| |
| blobId1 = ins.insert(OBJ_BLOB, blob1); |
| largeId = ins.insert(OBJ_BLOB, largeBlob); |
| |
| try (ObjectReader reader = ins.newReader()) { |
| // A previous bug did not reset the file pointer to EOF after reading |
| // back. We need to seek to something further back than a full buffer, |
| // since the read-back code eagerly reads a full buffer's worth of data |
| // from the file to pass to the inflater. If we seeked back just a small |
| // amount, this step would consume the rest of the file, so the file |
| // pointer would coincidentally end up back at EOF, hiding the bug. |
| assertBlob(reader, blobId1, blob1); |
| } |
| |
| blobId2 = ins.insert(OBJ_BLOB, blob2); |
| |
| try (ObjectReader reader = ins.newReader()) { |
| assertBlob(reader, blobId1, blob1); |
| assertBlob(reader, blobId2, blob2); |
| assertBlob(reader, largeId, largeBlob); |
| } |
| |
| ins.flush(); |
| } |
| |
| try (ObjectReader reader = db.newObjectReader()) { |
| assertBlob(reader, blobId1, blob1); |
| assertBlob(reader, blobId2, blob2); |
| assertBlob(reader, largeId, largeBlob); |
| } |
| } |
| |
| private List<Pack> listPacks() throws Exception { |
| List<Pack> fromOpenDb = listPacks(db); |
| List<Pack> reopened; |
| try (FileRepository db2 = new FileRepository(db.getDirectory())) { |
| reopened = listPacks(db2); |
| } |
| assertEquals(fromOpenDb.size(), reopened.size()); |
| for (int i = 0 ; i < fromOpenDb.size(); i++) { |
| Pack a = fromOpenDb.get(i); |
| Pack b = reopened.get(i); |
| assertEquals(a.getPackName(), b.getPackName()); |
| assertEquals( |
| a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); |
| assertEquals(a.getObjectCount(), b.getObjectCount()); |
| a.getObjectCount(); |
| } |
| return fromOpenDb; |
| } |
| |
| private static List<Pack> listPacks(FileRepository db) throws Exception { |
| return db.getObjectDatabase().getPacks().stream() |
| .sorted(comparing(Pack::getPackName)).collect(toList()); |
| } |
| |
| private PackInserter newInserter() { |
| return db.getObjectDatabase().newPackInserter(); |
| } |
| |
| private static byte[] newLargeBlob() { |
| byte[] blob = new byte[10240]; |
| random.nextBytes(blob); |
| return blob; |
| } |
| |
| private static String getInode(File f) throws Exception { |
| BasicFileAttributes attrs = Files.readAttributes( |
| f.toPath(), BasicFileAttributes.class); |
| Object k = attrs.fileKey(); |
| if (k == null) { |
| return null; |
| } |
| Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$"); |
| Matcher m = p.matcher(k.toString()); |
| return m.matches() ? m.group(1) : null; |
| } |
| |
| private static void assertBlob(ObjectReader reader, ObjectId id, |
| byte[] expected) throws Exception { |
| ObjectLoader loader = reader.open(id); |
| assertEquals(OBJ_BLOB, loader.getType()); |
| assertEquals(expected.length, loader.getSize()); |
| try (ObjectStream s = loader.openStream()) { |
| int n = (int) s.getSize(); |
| byte[] actual = new byte[n]; |
| assertEquals(n, IO.readFully(s, actual, 0)); |
| assertArrayEquals(expected, actual); |
| } |
| } |
| |
| private void assertPacksOnly() throws Exception { |
| new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")) |
| .assertNoBadFiles(db.getObjectDatabase().getDirectory()); |
| } |
| |
| private void assertNoObjects() throws Exception { |
| new BadFileCollector(f -> true) |
| .assertNoBadFiles(db.getObjectDatabase().getDirectory()); |
| } |
| |
| private static class BadFileCollector extends SimpleFileVisitor<Path> { |
| private final Predicate<String> badName; |
| private List<String> bad; |
| |
| BadFileCollector(Predicate<String> badName) { |
| this.badName = badName; |
| } |
| |
| void assertNoBadFiles(File f) throws IOException { |
| bad = new ArrayList<>(); |
| Files.walkFileTree(f.toPath(), this); |
| if (!bad.isEmpty()) { |
| fail("unexpected files in object directory: " + bad); |
| } |
| } |
| |
| @Override |
| public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { |
| Path fileName = file.getFileName(); |
| if (fileName != null) { |
| String name = fileName.toString(); |
| if (!attrs.isDirectory() && badName.test(name)) { |
| bad.add(name); |
| } |
| } |
| return FileVisitResult.CONTINUE; |
| } |
| } |
| } |