diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index e7b53c6..c36b42c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -446,8 +446,12 @@
           "Replication to %s completed in %dms, %dms delay, %d retries",
           uri, elapsed, delay, retryCount);
     } catch (RepositoryNotFoundException e) {
+      retryDone();
       stateLog.error(
-          "Cannot replicate " + projectName + "; Local repository error: " + e.getMessage(),
+          "Cannot replicate "
+              + projectName
+              + "; Local repository does not exist: "
+              + e.getMessage(),
           getStatesAsArray());
 
     } catch (RemoteRepositoryException e) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 4dd6060..3f92983 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -144,9 +144,12 @@
   private final Path runningUpdates;
   private final Path waitingUpdates;
 
+  private boolean isMultiPrimary;
+
   @Inject
   ReplicationTasksStorage(ReplicationConfig config) {
     this(config.getEventsDirectory().resolve("ref-updates"));
+    isMultiPrimary = config.getDistributionInterval() != 0;
   }
 
   @VisibleForTesting
@@ -160,6 +163,10 @@
             .create();
   }
 
+  private boolean isMultiPrimary() {
+    return isMultiPrimary;
+  }
+
   public synchronized String create(ReplicateRefUpdate r) {
     return new Task(r).create();
   }
@@ -212,13 +219,19 @@
         .map(Optional::get);
   }
 
-  private static Stream<Path> walkNonDirs(Path path) {
+  private Stream<Path> walkNonDirs(Path path) {
     try {
       return Files.list(path).flatMap(sub -> walkNonDirs(sub));
     } catch (NotDirectoryException e) {
       return Stream.of(path);
     } catch (Exception e) {
-      logger.atSevere().withCause(e).log("Error while walking directory %s", path);
+      String message = "Error while walking directory %s";
+      if (isMultiPrimary() && e instanceof NoSuchFileException) {
+        logger.atFine().log(
+            message + " (expected regularly with multi-primaries and distributor enabled)", path);
+      } else {
+        logger.atSevere().withCause(e).log(message, path);
+      }
       return Stream.empty();
     }
   }
@@ -398,7 +411,14 @@
         Files.move(from, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
         return true;
       } catch (IOException e) {
-        logger.atSevere().withCause(e).log("Error while renaming task %s", taskKey);
+        String message = "Error while renaming task %s";
+        if (isMultiPrimary() && e instanceof NoSuchFileException) {
+          logger.atFine().log(
+              message + " (expected regularly with multi-primaries and distributor enabled)",
+              taskKey);
+        } else {
+          logger.atSevere().withCause(e).log(message, taskKey);
+        }
         return false;
       }
     }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
index dc29612..fd08b7b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
@@ -14,6 +14,7 @@
 
 package com.googlesource.gerrit.plugins.replication;
 
+import static com.google.common.truth.Truth.assertThat;
 import static org.eclipse.jgit.lib.Ref.Storage.NEW;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -51,6 +52,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import org.eclipse.jgit.errors.NotSupportedException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.errors.TransportException;
 import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.ObjectId;
@@ -313,6 +315,17 @@
     verify(transportMock, times(1)).push(any(), any());
   }
 
+  @Test
+  public void shouldNotKeepRetryingWhenRepositoryNotFound() throws Exception {
+    when(gitRepositoryManagerMock.openRepository(projectNameKey))
+        .thenThrow(new RepositoryNotFoundException("not found"));
+    PushOne pushOne = createPushOne(null);
+    pushOne.addRef(PushOne.ALL_REFS);
+    pushOne.setToRetry();
+    pushOne.run();
+    assertThat(pushOne.isRetrying()).isFalse();
+  }
+
   private void replicateTwoRefs(PushOne pushOne) throws InterruptedException {
     ObjectIdRef barLocalRef =
         new ObjectIdRef.Unpeeled(