Merge branch 'stable-5.1' into stable-5.2 * stable-5.1: Prepare 5.1.8-SNAPSHOT builds JGit v5.1.7.201904200442-r ObjectUploadListener: Add callback interface Prepare 4.11.9-SNAPSHOT builds JGit v4.11.8.201904181247-r Prepare 4.9.11-SNAPSHOT builds JGit v4.9.10.201904181027-r Prepare 4.7.10-SNAPSHOT builds JGit v4.7.9.201904161809-r Prepare 4.5.8-SNAPSHOT builds JGit v4.5.7.201904151645-r Remember the cause for invalidating a packfile Fix API problem filters Fix pack files scan when filesnapshot isn't modified Change-Id: I76761002eedf360e93d0559942ebc927a40428d6 Signed-off-by: David Pursehouse <david.pursehouse@gmail.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
diff --git a/org.eclipse.jgit.lfs.server/.settings/.api_filters b/org.eclipse.jgit.lfs.server/.settings/.api_filters new file mode 100644 index 0000000..a8583ce --- /dev/null +++ b/org.eclipse.jgit.lfs.server/.settings/.api_filters
@@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<component id="org.eclipse.jgit.lfs.server" version="2"> + <resource path="META-INF/MANIFEST.MF"> + <filter id="924844039"> + <message_arguments> + <message_argument value="5.2.2"/> + <message_argument value="5.2.0"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener"> + <filter id="1142947843"> + <message_arguments> + <message_argument value="5.1.7"/> + <message_argument value="setCallback(ObjectUploadListener.Callback)"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener$Callback"> + <filter id="1142947843"> + <message_arguments> + <message_argument value="5.1.7"/> + <message_argument value="Callback"/> + </message_arguments> + </filter> + </resource> +</component>
diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java index c5b6a67..3bb2899 100644 --- a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java
@@ -48,6 +48,7 @@ import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; +import java.nio.file.Path; import java.util.logging.Level; import java.util.logging.Logger; @@ -87,6 +88,29 @@ public class ObjectUploadListener implements ReadListener { private final ByteBuffer buffer = ByteBuffer.allocateDirect(8192); + private final Path path; + + private long uploaded; + + private Callback callback; + + /** + * Callback invoked after object upload completed. + * + * @since 5.1.7 + */ + public interface Callback { + /** + * Notified after object upload completed. + * + * @param path + * path to the object on the backend + * @param size + * uploaded size in bytes + */ + void uploadCompleted(String path, long size); + } + /** * Constructor for ObjectUploadListener. * @@ -113,10 +137,25 @@ public ObjectUploadListener(FileLfsRepository repository, this.inChannel = Channels.newChannel(in); this.out = repository.getOutputStream(id); this.channel = Channels.newChannel(out); + this.path = repository.getPath(id); + this.uploaded = 0L; response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON); } /** + * Set the callback to invoke after upload completed. + * + * @param callback + * the callback + * @return {@code this}. + * @since 5.1.7 + */ + public ObjectUploadListener setCallback(Callback callback) { + this.callback = callback; + return this; + } + + /** * {@inheritDoc} * * Writes all the received data to the output channel @@ -126,12 +165,13 @@ public void onDataAvailable() throws IOException { while (in.isReady()) { if (inChannel.read(buffer) > 0) { buffer.flip(); - channel.write(buffer); + uploaded += Integer.valueOf(channel.write(buffer)).longValue(); buffer.compact(); } else { buffer.flip(); while (buffer.hasRemaining()) { - channel.write(buffer); + uploaded += Integer.valueOf(channel.write(buffer)) + .longValue(); } close(); return; @@ -159,6 +199,9 @@ protected void close() throws IOException { if (!response.isCommitted()) { response.setStatus(HttpServletResponse.SC_OK); } + if (callback != null) { + callback.uploadCompleted(path.toString(), uploaded); + } } finally { context.complete(); }
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 39cf2da..9b0526c 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters
@@ -8,6 +8,20 @@ </message_arguments> </filter> </resource> + <resource path="src/org/eclipse/jgit/errors/PackInvalidException.java" type="org.eclipse.jgit.errors.PackInvalidException"> + <filter id="1142947843"> + <message_arguments> + <message_argument value="4.5.7"/> + <message_argument value="PackInvalidException(File, Throwable)"/> + </message_arguments> + </filter> + <filter id="1142947843"> + <message_arguments> + <message_argument value="4.5.7"/> + <message_argument value="PackInvalidException(String, Throwable)"/> + </message_arguments> + </filter> + </resource> <resource path="src/org/eclipse/jgit/transport/TransferConfig.java" type="org.eclipse.jgit.transport.TransferConfig"> <filter id="1159725059"> <message_arguments>
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index a83de06..b238ab4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java
@@ -60,9 +60,24 @@ public class PackInvalidException extends IOException { * * @param path * path of the invalid pack file. + * @deprecated Use {@link #PackInvalidException(File, Throwable)}. */ + @Deprecated public PackInvalidException(File path) { - this(path.getAbsolutePath()); + this(path, null); + } + + /** + * Construct a pack invalid error with cause. + * + * @param path + * path of the invalid pack file. + * @param cause + * cause of the pack file becoming invalid. + * @since 4.5.7 + */ + public PackInvalidException(File path, Throwable cause) { + this(path.getAbsolutePath(), cause); } /** @@ -70,8 +85,23 @@ public PackInvalidException(File path) { * * @param path * path of the invalid pack file. + * @deprecated Use {@link #PackInvalidException(String, Throwable)}. */ + @Deprecated public PackInvalidException(String path) { - super(MessageFormat.format(JGitText.get().packFileInvalid, path)); + this(path, null); + } + + /** + * Construct a pack invalid error with cause. + * + * @param path + * path of the invalid pack file. + * @param cause + * cause of the pack file becoming invalid. + * @since 4.5.7 + */ + public PackInvalidException(String path, Throwable cause) { + super(MessageFormat.format(JGitText.get().packFileInvalid, path), cause); } }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java index b9758bd..58d12ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
@@ -84,6 +84,9 @@ abstract class BlockBasedFile { /** True once corruption has been detected that cannot be worked around. */ volatile boolean invalid; + /** Exception that caused the packfile to be flagged as invalid */ + protected volatile Exception invalidatingCause; + BlockBasedFile(DfsBlockCache cache, DfsPackDescription desc, PackExt ext) { this.cache = cache; this.key = desc.getStreamKey(ext); @@ -135,8 +138,9 @@ DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException { DfsBlock readOneBlock(long pos, DfsReader ctx, @Nullable ReadableChannel fileChannel) throws IOException { - if (invalid) - throw new PackInvalidException(getFileName()); + if (invalid) { + throw new PackInvalidException(getFileName(), invalidatingCause); + } ctx.stats.readBlock++; long start = System.nanoTime();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 05b8f61..c131f94 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
@@ -181,8 +181,9 @@ private PackIndex idx(DfsReader ctx) throws IOException { return idx; } - if (invalid) - throw new PackInvalidException(getFileName()); + if (invalid) { + throw new PackInvalidException(getFileName(), invalidatingCause); + } Repository.getGlobalListenerList() .dispatch(new BeforeDfsPackIndexLoadedEvent(this)); @@ -224,11 +225,13 @@ else if (bs <= 0) } } catch (EOFException e) { invalid = true; + invalidatingCause = e; throw new IOException(MessageFormat.format( DfsText.get().shortReadOfIndex, desc.getFileName(INDEX)), e); } catch (IOException e) { invalid = true; + invalidatingCause = e; throw new IOException(MessageFormat.format( DfsText.get().cannotReadIndex, desc.getFileName(INDEX)), e); @@ -720,8 +723,10 @@ void copyAsIs(PackOutputStream out, DfsObjectToPack src, private IOException packfileIsTruncated() { invalid = true; - return new IOException(MessageFormat.format( + IOException exc = new IOException(MessageFormat.format( JGitText.get().packfileIsTruncated, getFileName())); + invalidatingCause = exc; + return exc; } private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 869fd02..b3af545 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
@@ -132,8 +132,6 @@ public class ObjectDirectory extends FileObjectDatabase { private final File alternatesFile; - private final AtomicReference<PackList> packList; - private final FS fs; private final AtomicReference<AlternateHandle[]> alternates; @@ -146,6 +144,8 @@ public class ObjectDirectory extends FileObjectDatabase { private Set<ObjectId> shallowCommitsIds; + final AtomicReference<PackList> packList; + /** * Initialize a reference to an on-disk object directory. * @@ -674,13 +674,8 @@ private void handlePackError(IOException e, PackFile p) { transientErrorCount = p.incrementTransientErrorCount(); } if (warnTmpl != null) { - if (LOG.isDebugEnabled()) { - LOG.debug(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - } else { - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath())); - } + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); } else { if (doLogExponentialBackoff(transientErrorCount)) { // Don't remove the pack from the list, as the error may be @@ -767,7 +762,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, return InsertLooseObjectResult.FAILURE; } - private boolean searchPacksAgain(PackList old) { + boolean searchPacksAgain(PackList old) { // Whether to trust the pack folder's modification time. If set // to false we will always scan the .git/objects/pack folder to // check for new pack files. If set to true (default) we use the @@ -917,7 +912,8 @@ private PackList scanPacksImpl(PackList old) { final String packName = base + PACK.getExtension(); final File packFile = new File(packDirectory, packName); final PackFile oldPack = forReuse.remove(packName); - if (oldPack != null && oldPack.getFileSnapshot().isModified(packFile)) { + if (oldPack != null + && !oldPack.getFileSnapshot().isModified(packFile)) { list.add(oldPack); continue; } @@ -1074,7 +1070,7 @@ public File fileFor(AnyObjectId objectId) { return new File(new File(getDirectory(), d), f); } - private static final class PackList { + static final class PackList { /** State just before reading the pack directory. */ final FileSnapshot snapshot;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index ee3c0a8..d834b1a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
@@ -135,6 +135,8 @@ public int compare(PackFile a, PackFile b) { private volatile boolean invalid; + private volatile Exception invalidatingCause; + private boolean invalidBitmap; private AtomicInteger transientErrorCount = new AtomicInteger(); @@ -184,7 +186,7 @@ private PackIndex idx() throws IOException { idx = loadedIdx; if (idx == null) { if (invalid) { - throw new PackInvalidException(packFile); + throw new PackInvalidException(packFile, invalidatingCause); } try { idx = PackIndex.open(extFile(INDEX)); @@ -208,6 +210,7 @@ private PackIndex idx() throws IOException { throw e; } catch (IOException e) { invalid = true; + invalidatingCause = e; throw e; } } @@ -661,7 +664,7 @@ synchronized boolean endWindowCache() { private void doOpen() throws IOException { if (invalid) { - throw new PackInvalidException(packFile); + throw new PackInvalidException(packFile, invalidatingCause); } try { synchronized (readLock) { @@ -671,13 +674,13 @@ private void doOpen() throws IOException { } } catch (InterruptedIOException e) { // don't invalidate the pack, we are interrupted from another thread - openFail(false); + openFail(false, e); throw e; } catch (FileNotFoundException fn) { // don't invalidate the pack if opening an existing file failed // since it may be related to a temporary lack of resources (e.g. // max open files) - openFail(!packFile.exists()); + openFail(!packFile.exists(), fn); throw fn; } catch (EOFException | AccessDeniedException | NoSuchFileException | CorruptObjectException | NoPackSignatureException @@ -685,20 +688,21 @@ private void doOpen() throws IOException { | UnsupportedPackIndexVersionException | UnsupportedPackVersionException pe) { // exceptions signaling permanent problems with a pack - openFail(true); + openFail(true, pe); throw pe; } catch (IOException | RuntimeException ge) { // generic exceptions could be transient so we should not mark the // pack invalid to avoid false MissingObjectExceptions - openFail(false); + openFail(false, ge); throw ge; } } - private void openFail(boolean invalidate) { + private void openFail(boolean invalidate, Exception cause) { activeWindows = 0; activeCopyRawData = 0; invalid = invalidate; + invalidatingCause = cause; doClose(); } @@ -725,7 +729,7 @@ ByteArrayWindow read(long pos, int size) throws IOException { // Detect the situation and throw a proper exception so that can be properly // managed by the main packfile search loop and the Git client won't receive // any failures. - throw new PackInvalidException(packFile); + throw new PackInvalidException(packFile, invalidatingCause); } if (length < pos + size) size = (int) (length - pos);