ProjectRepairer: Sweep stale snapshots before repairing

Delete snapshots older than a day at the start of each repair. A
snapshot directory is deleted in a finally block once its transfer
finishes, which covers every normal path, including failures and
interrupts. It cannot cover an abnormal JVM exit, such as a kill or an
out of memory kill, between creating the hardlinks and deleting them.
The links that survive keep the objects they point at alive, so gc
cannot reclaim the space until someone removes the directory by hand.

Change-Id: Ia49e874ab39cc1eb86d0ca3fa898f2e7b31fa91d

src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java
index b70e873..d2e0edf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java
@@ -26,10 +26,13 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.OutputStream;
+import java.nio.file.DirectoryIteratorException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -77,6 +80,7 @@
     if (objectsDir == null) {
       return false;
     }
+    Snapshot.sweepStale(objectsDir.getParent());
 
     boolean isRepaired = true;
     for (Action action : actions) {
@@ -280,6 +284,7 @@
 
   private static final class Snapshot implements AutoCloseable {
     private static final String SNAPSHOT_PREFIX = "replication-repair-snapshot-";
+    private static final Duration MAX_AGE = Duration.ofDays(1);
 
     private final Path dir;
 
@@ -287,6 +292,25 @@
       this.dir = Files.createDirectory(parentDir.resolve(SNAPSHOT_PREFIX + UUID.randomUUID()));
     }
 
+    private static void sweepStale(Path parentDir) {
+      Instant cutoff = Instant.now().minus(MAX_AGE);
+      try (DirectoryStream<Path> snapshotDirs =
+          Files.newDirectoryStream(parentDir, SNAPSHOT_PREFIX + "*")) {
+        for (Path snapshotDir : snapshotDirs) {
+          try {
+            if (Files.getLastModifiedTime(snapshotDir).toInstant().isBefore(cutoff)) {
+              repLog.atWarning().log("Deleting stale repair snapshot %s", snapshotDir);
+              delete(snapshotDir);
+            }
+          } catch (IOException e) {
+            repLog.atWarning().withCause(e).log("Cannot check repair snapshot %s", snapshotDir);
+          }
+        }
+      } catch (IOException | DirectoryIteratorException e) {
+        repLog.atWarning().withCause(e).log("Cannot sweep repair snapshots in %s", parentDir);
+      }
+    }
+
     private void createSubdir(String name) throws IOException {
       Files.createDirectory(dir.resolve(name));
     }
@@ -311,11 +335,15 @@
 
     @Override
     public void close() {
+      delete(dir);
+    }
+
+    private static void delete(Path snapshotDir) {
       try {
         FileUtils.delete(
-            dir.toFile(), FileUtils.RECURSIVE | FileUtils.SKIP_MISSING | FileUtils.RETRY);
+            snapshotDir.toFile(), FileUtils.RECURSIVE | FileUtils.SKIP_MISSING | FileUtils.RETRY);
       } catch (IOException e) {
-        repLog.atSevere().withCause(e).log("Cannot delete repair snapshot %s", dir);
+        repLog.atSevere().withCause(e).log("Cannot delete repair snapshot %s", snapshotDir);
       }
     }
   }