Merge branch 'stable-6.10' into stable-7.0 * stable-6.10: PackDirectory: Filter out tmp GC pack files Test advertised capabilities with protocol V0 and allow*Sha1InWant Align request policies with CGit Pack.java: Recover more often in Pack.copyAsIs2() Change-Id: Ib301efa54aaf2196d764a0fc1f91f652b4d68396
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java index def73ac..017104c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
@@ -11,6 +11,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -564,6 +565,47 @@ private void checkUnadvertisedIfUnallowed(String configSection, } @Test + public void testV0CapabilitiesAllowAnySha1InWant() throws Exception { + checkAvertisedCapabilityProtocolV0IfAllowed("uploadpack", + "allowanysha1inwant", "allow-reachable-sha1-in-want", + "allow-tip-sha1-in-want"); + } + + @Test + public void testV0CapabilitiesAllowReachableSha1InWant() throws Exception { + checkAvertisedCapabilityProtocolV0IfAllowed("uploadpack", + "allowreachablesha1inwant", "allow-reachable-sha1-in-want"); + } + + @Test + public void testV0CapabilitiesAllowTipSha1InWant() throws Exception { + checkAvertisedCapabilityProtocolV0IfAllowed("uploadpack", + "allowtipsha1inwant", "allow-tip-sha1-in-want"); + } + + private void checkAvertisedCapabilityProtocolV0IfAllowed( + String configSection, String configName, String... capabilities) + throws Exception { + server.getConfig().setBoolean(configSection, null, configName, true); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V0.version(), null, + PacketLineIn.end()); + PacketLineIn pckIn = new PacketLineIn(recvStream); + + String line; + while (!PacketLineIn.isEnd((line = pckIn.readString()))) { + if (line.contains("capabilities")) { + List<String> linesCapabilities = Arrays.asList(line.substring( + line.indexOf(" ", line.indexOf("capabilities")) + 1) + .split(" ")); + assertThat(linesCapabilities, hasItems(capabilities)); + return; + } + } + fail("Server side protocol did not contain any capabilities'"); + } + + @Test public void testV2CapabilitiesAllowFilter() throws Exception { checkAdvertisedIfAllowed("uploadpack", "allowfilter", "filter"); checkUnadvertisedIfUnallowed("uploadpack", "allowfilter", "filter");
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index be45764..3518342 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -416,185 +416,196 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src, final CRC32 crc2 = validate ? new CRC32() : null; final byte[] buf = out.getCopyBuffer(); + boolean isHeaderWritten = false; // Rip apart the header so we can discover the size. // - readFully(src.offset, buf, 0, 20, curs); - int c = buf[0] & 0xff; - final int typeCode = (c >> 4) & 7; - long inflatedLength = c & 15; - int shift = 4; - int headerCnt = 1; - while ((c & 0x80) != 0) { - c = buf[headerCnt++] & 0xff; - inflatedLength += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - if (typeCode == Constants.OBJ_OFS_DELTA) { - do { - c = buf[headerCnt++] & 0xff; - } while ((c & 128) != 0); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - } else if (typeCode == Constants.OBJ_REF_DELTA) { - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - - readFully(src.offset + headerCnt, buf, 0, 20, curs); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, 20); - crc2.update(buf, 0, 20); - } - headerCnt += 20; - } else if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - - final long dataOffset = src.offset + headerCnt; - final long dataLength = src.length; - final long expectedCRC; - final ByteArrayWindow quickCopy; - - // Verify the object isn't corrupt before sending. If it is, - // we report it missing instead. - // try { - quickCopy = curs.quickCopy(this, dataOffset, dataLength); + readFully(src.offset, buf, 0, 20, curs); - if (validate && idx().hasCRC32Support()) { - assert(crc1 != null); - // Index has the CRC32 code cached, validate the object. - // - expectedCRC = idx().findCRC32(src); - if (quickCopy != null) { - quickCopy.crc32(crc1, dataOffset, (int) dataLength); - } else { - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - pos += n; - cnt -= n; - } - } - if (crc1.getValue() != expectedCRC) { - setCorrupt(src.offset); - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } - } else if (validate) { - // We don't have a CRC32 code in the index, so compute it - // now while inflating the raw data to get zlib to tell us - // whether or not the data is safe. - // - Inflater inf = curs.inflater(); - byte[] tmp = new byte[1024]; - if (quickCopy != null) { - quickCopy.check(inf, tmp, dataOffset, (int) dataLength); - } else { - assert(crc1 != null); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - inf.setInput(buf, 0, n); - while (inf.inflate(tmp, 0, tmp.length) > 0) - continue; - pos += n; - cnt -= n; - } - } - if (!inf.finished() || inf.getBytesRead() != dataLength) { - setCorrupt(src.offset); - throw new EOFException(MessageFormat.format( - JGitText.get().shortCompressedStreamAt, - Long.valueOf(src.offset))); - } - assert(crc1 != null); - expectedCRC = crc1.getValue(); - } else { - expectedCRC = -1; + int c = buf[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long inflatedLength = c & 15; + int shift = 4; + int headerCnt = 1; + while ((c & 0x80) != 0) { + c = buf[headerCnt++] & 0xff; + inflatedLength += ((long) (c & 0x7f)) << shift; + shift += 7; } - } catch (DataFormatException dataFormat) { - setCorrupt(src.offset); - CorruptObjectException corruptObject = new CorruptObjectException( - MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile()), - dataFormat); + if (typeCode == Constants.OBJ_OFS_DELTA) { + do { + c = buf[headerCnt++] & 0xff; + } while ((c & 128) != 0); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + } else if (typeCode == Constants.OBJ_REF_DELTA) { + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } - throw new StoredObjectRepresentationNotAvailableException( - corruptObject); + readFully(src.offset + headerCnt, buf, 0, 20, curs); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, 20); + crc2.update(buf, 0, 20); + } + headerCnt += 20; + } else if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } - } catch (IOException ioError) { - throw new StoredObjectRepresentationNotAvailableException(ioError); - } + final long dataOffset = src.offset + headerCnt; + final long dataLength = src.length; + final long expectedCRC; + final ByteArrayWindow quickCopy; - if (quickCopy != null) { - // The entire object fits into a single byte array window slice, - // and we have it pinned. Write this out without copying. + // Verify the object isn't corrupt before sending. If it is, + // we report it missing instead. // - out.writeHeader(src, inflatedLength); - quickCopy.write(out, dataOffset, (int) dataLength); + try { + quickCopy = curs.quickCopy(this, dataOffset, dataLength); - } else if (dataLength <= buf.length) { - // Tiny optimization: Lots of objects are very small deltas or - // deflated commits that are likely to fit in the copy buffer. - // - if (!validate) { + if (validate && idx().hasCRC32Support()) { + assert(crc1 != null); + // Index has the CRC32 code cached, validate the object. + // + expectedCRC = idx().findCRC32(src); + if (quickCopy != null) { + quickCopy.crc32(crc1, dataOffset, (int) dataLength); + } else { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + pos += n; + cnt -= n; + } + } + if (crc1.getValue() != expectedCRC) { + setCorrupt(src.offset); + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } + } else if (validate) { + // We don't have a CRC32 code in the index, so compute it + // now while inflating the raw data to get zlib to tell us + // whether or not the data is safe. + // + Inflater inf = curs.inflater(); + byte[] tmp = new byte[1024]; + if (quickCopy != null) { + quickCopy.check(inf, tmp, dataOffset, (int) dataLength); + } else { + assert(crc1 != null); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + inf.setInput(buf, 0, n); + while (inf.inflate(tmp, 0, tmp.length) > 0) + continue; + pos += n; + cnt -= n; + } + } + if (!inf.finished() || inf.getBytesRead() != dataLength) { + setCorrupt(src.offset); + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + Long.valueOf(src.offset))); + } + assert(crc1 != null); + expectedCRC = crc1.getValue(); + } else { + expectedCRC = -1; + } + } catch (DataFormatException dataFormat) { + setCorrupt(src.offset); + + CorruptObjectException corruptObject = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile()), + dataFormat); + + throw new StoredObjectRepresentationNotAvailableException( + corruptObject); + } + + if (quickCopy != null) { + // The entire object fits into a single byte array window slice, + // and we have it pinned. Write this out without copying. + // + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + quickCopy.write(out, dataOffset, (int) dataLength); + + } else if (dataLength <= buf.length) { + // Tiny optimization: Lots of objects are very small deltas or + // deflated commits that are likely to fit in the copy buffer. + // + if (!validate) { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + pos += n; + cnt -= n; + } + } + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + out.write(buf, 0, (int) dataLength); + } else { + // Now we are committed to sending the object. As we spool it out, + // check its CRC32 code to make sure there wasn't corruption between + // the verification we did above, and us actually outputting it. + // long pos = dataOffset; long cnt = dataLength; while (cnt > 0) { final int n = (int) Math.min(cnt, buf.length); readFully(pos, buf, 0, n, curs); + if (validate) { + assert(crc2 != null); + crc2.update(buf, 0, n); + } + if (!isHeaderWritten) { + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + } + out.write(buf, 0, n); pos += n; cnt -= n; } - } - out.writeHeader(src, inflatedLength); - out.write(buf, 0, (int) dataLength); - } else { - // Now we are committed to sending the object. As we spool it out, - // check its CRC32 code to make sure there wasn't corruption between - // the verification we did above, and us actually outputting it. - // - out.writeHeader(src, inflatedLength); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); if (validate) { assert(crc2 != null); - crc2.update(buf, 0, n); - } - out.write(buf, 0, n); - pos += n; - cnt -= n; - } - if (validate) { - assert(crc2 != null); - if (crc2.getValue() != expectedCRC) { - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); + if (crc2.getValue() != expectedCRC) { + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } } } + } catch (IOException ioError) { + if (!isHeaderWritten) { + throw new StoredObjectRepresentationNotAvailableException(ioError); + } + throw ioError; } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 8221cff..51a8148 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java
@@ -545,7 +545,7 @@ private Map<String, Map<PackExt, PackFile>> getPackFilesByExtById() { for (String name : nameList) { try { PackFile pack = new PackFile(directory, name); - if (pack.getPackExt() != null) { + if (pack.getPackExt() != null && !pack.isTmpGCFile()) { Map<PackExt, PackFile> packByExt = packFilesByExtById .get(pack.getId()); if (packByExt == null) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index 19979d0..c9b05ad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
@@ -27,6 +27,7 @@ public class PackFile extends File { private static final long serialVersionUID = 1L; private static final String PREFIX = "pack-"; //$NON-NLS-1$ + private static final String TMP_GC_PREFIX = ".tmp-"; //$NON-NLS-1$ private final String base; // PREFIX + id i.e. // pack-0123456789012345678901234567890123456789 @@ -126,6 +127,13 @@ public PackExt getPackExt() { } /** + * @return whether the file is a temporary GC file + */ + public boolean isTmpGCFile() { + return id.startsWith(TMP_GC_PREFIX); + } + + /** * Create a new similar PackFile with the given extension instead. * * @param ext
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 3772c00..62da98b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
@@ -117,13 +117,13 @@ public class UploadPack implements Closeable { /** Policy the server uses to validate client requests */ public enum RequestPolicy { /** Client may only ask for objects the server advertised a reference for. */ - ADVERTISED, + ADVERTISED(0x08), /** * Client may ask for any commit reachable from a reference advertised by * the server. */ - REACHABLE_COMMIT, + REACHABLE_COMMIT(0x02), /** * Client may ask for objects that are the tip of any reference, even if not @@ -133,18 +133,34 @@ public enum RequestPolicy { * * @since 3.1 */ - TIP, + TIP(0x01), /** * Client may ask for any commit reachable from any reference, even if that - * reference wasn't advertised. + * reference wasn't advertised, implies REACHABLE_COMMIT and TIP. * * @since 3.1 */ - REACHABLE_COMMIT_TIP, + REACHABLE_COMMIT_TIP(0x03), - /** Client may ask for any SHA-1 in the repository. */ - ANY; + /** Client may ask for any SHA-1 in the repository, implies REACHABLE_COMMIT_TIP. */ + ANY(0x07); + + private final int bitmask; + + RequestPolicy(int bitmask) { + this.bitmask = bitmask; + } + + /** + * Check if the current policy implies another, based on its bitmask. + * + * @param implied the implied policy based on its bitmask. + * @return true if the policy is implied. + */ + public boolean implies(RequestPolicy implied) { + return (bitmask & implied.bitmask) != 0; + } } /** @@ -1582,13 +1598,9 @@ public void sendAdvertisedRefs(RefAdvertiser adv, if (!biDirectionalPipe) adv.advertiseCapability(OPTION_NO_DONE); RequestPolicy policy = getRequestPolicy(); - if (policy == RequestPolicy.TIP - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.TIP)) adv.advertiseCapability(OPTION_ALLOW_TIP_SHA1_IN_WANT); - if (policy == RequestPolicy.REACHABLE_COMMIT - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.REACHABLE_COMMIT)) adv.advertiseCapability(OPTION_ALLOW_REACHABLE_SHA1_IN_WANT); adv.advertiseCapability(OPTION_AGENT, UserAgent.get()); if (transferConfig.isAllowFilter()) {