diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 49cb059..5af0983 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -292,12 +292,12 @@
     }
 
     public void release() {
-      if (disableDeleteForTesting) {
-        logger.atFine().log("DELETE %s %s DISABLED", runningDir, updateLog());
-        return;
-      }
-
       try {
+        if (disableDeleteForTesting && Files.list(runningDir).findFirst().isPresent()) {
+          logger.atFine().log("DELETE %s %s DISABLED", runningDir, updateLog());
+          return;
+        }
+
         logger.atFine().log("DELETE %s %s", runningDir, updateLog());
         Files.delete(runningDir);
       } catch (IOException e) {