Merge branch 'stable-2.15' into stable-2.16 * stable-2.15: Cancel pending replications upon shutdown Remove replication event from pending when runway is allowed Change-Id: I81209708a42832cbe2be818e2dd50f625362b392

diff --git a/BUILD b/BUILD
index 59f8856..b6494b5 100644
--- a/BUILD
+++ b/BUILD

@@ -31,7 +31,7 @@
 
 java_library(
     name = "replication_util",
-    testonly = 1,
+    testonly = True,
     srcs = glob(
         ["src/test/java/**/*.java"],
         exclude = ["src/test/java/**/*Test.java"],

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
new file mode 100644
index 0000000..ef7e353
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java

@@ -0,0 +1,25 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.reviewdb.client.Project;
+
+public interface AdminApi {
+  public void createProject(Project.NameKey project, String head);
+
+  public void deleteProject(Project.NameKey project);
+
+  public void updateHead(Project.NameKey project, String newHead);
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
new file mode 100644
index 0000000..528aff2
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java

@@ -0,0 +1,61 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+@Singleton
+public class AdminApiFactory {
+
+  private final SshHelper sshHelper;
+
+  @Inject
+  AdminApiFactory(SshHelper sshHelper) {
+    this.sshHelper = sshHelper;
+  }
+
+  public Optional<AdminApi> create(URIish uri) {
+    if (isGerrit(uri)) {
+      return Optional.of(new GerritSshApi(sshHelper, uri));
+    } else if (!uri.isRemote()) {
+      return Optional.of(new LocalFS(uri));
+    } else if (isSSH(uri)) {
+      return Optional.of(new RemoteSsh(sshHelper, uri));
+    }
+    return Optional.empty();
+  }
+
+  public static boolean isGerrit(URIish uri) {
+    String scheme = uri.getScheme();
+    return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+  }
+
+  public static boolean isSSH(URIish uri) {
+    if (!uri.isRemote()) {
+      return false;
+    }
+    String scheme = uri.getScheme();
+    if (scheme != null && scheme.toLowerCase().contains("ssh")) {
+      return true;
+    }
+    if (scheme == null && uri.getHost() != null && uri.getPath() != null) {
+      return true;
+    }
+    return false;
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index 8b6b8fc..ad6e67d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java

@@ -13,36 +13,47 @@
 // limitations under the License.
 package com.googlesource.gerrit.plugins.replication;
 
+import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.common.FileUtil;
+import com.google.gerrit.extensions.annotations.PluginData;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.inject.Inject;
+import com.google.inject.Provider;
 import com.google.inject.Singleton;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.List;
 import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 @Singleton
 public class AutoReloadConfigDecorator implements ReplicationConfig {
-  private static final Logger log = LoggerFactory.getLogger(AutoReloadConfigDecorator.class);
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   private ReplicationFileBasedConfig currentConfig;
   private long currentConfigTs;
+  private long lastFailedConfigTs;
 
   private final SitePaths site;
-  private final WorkQueue workQueue;
-  private final DestinationFactory destinationFactory;
+  private final Destination.Factory destinationFactory;
+  private final Path pluginDataDir;
+  // Use Provider<> instead of injecting the ReplicationQueue because of circular dependency with
+  // ReplicationConfig
+  private final Provider<ReplicationQueue> replicationQueue;
 
   @Inject
   public AutoReloadConfigDecorator(
-      SitePaths site, WorkQueue workQueue, DestinationFactory destinationFactory)
+      SitePaths site,
+      Destination.Factory destinationFactory,
+      Provider<ReplicationQueue> replicationQueue,
+      @PluginData Path pluginDataDir)
       throws ConfigInvalidException, IOException {
     this.site = site;
     this.destinationFactory = destinationFactory;
+    this.pluginDataDir = pluginDataDir;
     this.currentConfig = loadConfig();
     this.currentConfigTs = getLastModified(currentConfig);
-    this.workQueue = workQueue;
+    this.replicationQueue = replicationQueue;
   }
 
   private static long getLastModified(ReplicationFileBasedConfig cfg) {
@@ -50,7 +61,7 @@
   }
 
   private ReplicationFileBasedConfig loadConfig() throws ConfigInvalidException, IOException {
-    return new ReplicationFileBasedConfig(site, destinationFactory);
+    return new ReplicationFileBasedConfig(site, destinationFactory, pluginDataDir);
   }
 
   private synchronized boolean isAutoReload() {
@@ -64,25 +75,27 @@
   }
 
   private void reloadIfNeeded() {
-    try {
-      if (isAutoReload()) {
-        long lastModified = getLastModified(currentConfig);
-        if (lastModified > currentConfigTs) {
-          ReplicationFileBasedConfig newConfig = loadConfig();
-          newConfig.startup(workQueue);
-          int discarded = currentConfig.shutdown();
-
-          this.currentConfig = newConfig;
-          this.currentConfigTs = lastModified;
-          log.info(
-              "Configuration reloaded: {} destinations, {} replication events discarded",
-              currentConfig.getDestinations(FilterType.ALL).size(),
-              discarded);
+    if (isAutoReload()) {
+      ReplicationQueue queue = replicationQueue.get();
+      long lastModified = getLastModified(currentConfig);
+      try {
+        if (lastModified > currentConfigTs && lastModified > lastFailedConfigTs) {
+          queue.stop();
+          currentConfig = loadConfig();
+          currentConfigTs = lastModified;
+          lastFailedConfigTs = 0;
+          logger.atInfo().log(
+              "Configuration reloaded: %d destinations",
+              currentConfig.getDestinations(FilterType.ALL).size());
         }
+      } catch (Exception e) {
+        logger.atSevere().withCause(e).log(
+            "Cannot reload replication configuration: keeping existing settings");
+        lastFailedConfigTs = lastModified;
+        return;
+      } finally {
+        queue.start();
       }
-    } catch (Exception e) {
-      log.error("Cannot reload replication configuration: keeping existing settings", e);
-      return;
     }
   }
 
@@ -102,6 +115,11 @@
   }
 
   @Override
+  public Path getEventsDirectory() {
+    return currentConfig.getEventsDirectory();
+  }
+
+  @Override
   public synchronized int shutdown() {
     return currentConfig.shutdown();
   }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index f8737b6..29a7ee6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java

@@ -16,18 +16,16 @@
 
 import static com.google.gerrit.common.FileUtil.lastModified;
 
+import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.inject.Inject;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.concurrent.atomic.AtomicReference;
 import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory {
-  private static final Logger log =
-      LoggerFactory.getLogger(AutoReloadSecureCredentialsFactoryDecorator.class);
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   private final AtomicReference<SecureCredentialsFactory> secureCredentialsFactory;
   private volatile long secureCredentialsFactoryLoadTs;
@@ -58,13 +56,12 @@
         secureCredentialsFactory.compareAndSet(
             secureCredentialsFactory.get(), new SecureCredentialsFactory(site));
         secureCredentialsFactoryLoadTs = getSecureConfigLastEditTs();
-        log.info("secure.config reloaded as it was updated on the file system");
+        logger.atInfo().log("secure.config reloaded as it was updated on the file system");
       }
     } catch (Exception e) {
-      log.error(
+      logger.atSevere().withCause(e).log(
           "Unexpected error while trying to reload "
-              + "secure.config: keeping existing credentials",
-          e);
+              + "secure.config: keeping existing credentials");
     }
 
     return secureCredentialsFactory.get().create(remoteName);

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index e8929d5..02304f3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java

@@ -24,9 +24,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
 import com.google.common.collect.Lists;
-import com.google.gerrit.common.EventDispatcher;
 import com.google.gerrit.common.data.GroupReference;
-import com.google.gerrit.extensions.client.ProjectState;
 import com.google.gerrit.extensions.config.FactoryModule;
 import com.google.gerrit.extensions.registration.DynamicItem;
 import com.google.gerrit.extensions.restapi.AuthException;
@@ -42,6 +40,7 @@
 import com.google.gerrit.server.account.GroupIncludeCache;
 import com.google.gerrit.server.account.ListGroupMembership;
 import com.google.gerrit.server.config.RequestScopedReviewDbProvider;
+import com.google.gerrit.server.events.EventDispatcher;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.PerThreadRequestScope;
 import com.google.gerrit.server.git.WorkQueue;
@@ -50,12 +49,14 @@
 import com.google.gerrit.server.permissions.ProjectPermission;
 import com.google.gerrit.server.permissions.RefPermission;
 import com.google.gerrit.server.project.NoSuchProjectException;
-import com.google.gerrit.server.project.PerRequestProjectControlCache;
-import com.google.gerrit.server.project.ProjectControl;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.gerrit.server.project.ProjectState;
 import com.google.gerrit.server.util.RequestContext;
+import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.Provider;
 import com.google.inject.Provides;
+import com.google.inject.assistedinject.Assisted;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.servlet.RequestScoped;
 import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
@@ -82,14 +83,20 @@
 
 public class Destination {
   private static final Logger repLog = ReplicationQueue.repLog;
+
+  public interface Factory {
+    Destination create(DestinationConfiguration config);
+  }
+
   private final ReplicationStateListener stateLog;
   private final Object stateLock = new Object();
   private final Map<URIish, PushOne> pending = new HashMap<>();
   private final Map<URIish, PushOne> inFlight = new HashMap<>();
   private final PushOne.Factory opFactory;
-  private final ProjectControl.Factory projectControlFactory;
   private final GitRepositoryManager gitManager;
   private final PermissionBackend permissionBackend;
+  private final Provider<CurrentUser> userProvider;
+  private final ProjectCache projectCache;
   private volatile ScheduledExecutorService pool;
   private final PerThreadRequestScope.Scoper threadScoper;
   private final DestinationConfiguration config;
@@ -111,23 +118,26 @@
     }
   }
 
+  @Inject
   protected Destination(
       Injector injector,
-      DestinationConfiguration cfg,
-      RemoteSiteUser.Factory replicationUserFactory,
       PluginUser pluginUser,
       GitRepositoryManager gitRepositoryManager,
       PermissionBackend permissionBackend,
+      Provider<CurrentUser> userProvider,
+      ProjectCache projectCache,
       GroupBackend groupBackend,
-      ReplicationStateListener stateLog,
+      ReplicationStateListeners stateLog,
       GroupIncludeCache groupIncludeCache,
-      DynamicItem<EventDispatcher> eventDispatcher) {
-    config = cfg;
+      DynamicItem<EventDispatcher> eventDispatcher,
+      @Assisted DestinationConfiguration cfg) {
     this.eventDispatcher = eventDispatcher;
     gitManager = gitRepositoryManager;
     this.permissionBackend = permissionBackend;
+    this.userProvider = userProvider;
+    this.projectCache = projectCache;
     this.stateLog = stateLog;
-
+    config = cfg;
     CurrentUser remoteUser;
     if (!cfg.getAuthGroupNames().isEmpty()) {
       ImmutableSet.Builder<AccountGroup.UUID> builder = ImmutableSet.builder();
@@ -140,7 +150,7 @@
           repLog.warn("Group \"{}\" not recognized, removing from authGroup", name);
         }
       }
-      remoteUser = replicationUserFactory.create(new ListGroupMembership(builder.build()));
+      remoteUser = new RemoteSiteUser(new ListGroupMembership(builder.build()));
     } else {
       remoteUser = pluginUser;
     }
@@ -152,7 +162,6 @@
               protected void configure() {
                 bindScope(RequestScoped.class, PerThreadRequestScope.REQUEST);
                 bind(PerThreadRequestScope.Propagator.class);
-                bind(PerRequestProjectControlCache.class).in(RequestScoped.class);
 
                 bind(Destination.class).toInstance(Destination.this);
                 bind(RemoteConfig.class).toInstance(config.getRemoteConfig());
@@ -184,7 +193,6 @@
               }
             });
 
-    projectControlFactory = child.getInstance(ProjectControl.Factory.class);
     opFactory = child.getInstance(PushOne.Factory.class);
     threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class);
   }
@@ -244,15 +252,21 @@
     }
   }
 
-  private boolean shouldReplicate(ProjectControl ctl) throws PermissionBackendException {
-    if (!config.replicateHiddenProjects() && ctl.getProject().getState() == ProjectState.HIDDEN) {
+  private boolean shouldReplicate(ProjectState state, CurrentUser user)
+      throws PermissionBackendException {
+    if (!config.replicateHiddenProjects()
+        && state.getProject().getState()
+            == com.google.gerrit.extensions.client.ProjectState.HIDDEN) {
       return false;
     }
+
+    // Hidden projects(permitsRead = false) should only be accessible by the project owners.
+    // READ_CONFIG is checked here because it's only allowed to project owners(ACCESS may also
+    // be allowed for other users).
+    ProjectPermission permissionToCheck =
+        state.statePermitsRead() ? ProjectPermission.ACCESS : ProjectPermission.READ_CONFIG;
     try {
-      permissionBackend
-          .user(ctl.getUser())
-          .project(ctl.getProject().getNameKey())
-          .check(ProjectPermission.ACCESS);
+      permissionBackend.user(user).project(state.getNameKey()).check(permissionToCheck);
       return true;
     } catch (AuthException e) {
       return false;
@@ -267,8 +281,19 @@
               new Callable<Boolean>() {
                 @Override
                 public Boolean call() throws NoSuchProjectException, PermissionBackendException {
-                  ProjectControl projectControl = controlFor(project);
-                  if (!shouldReplicate(projectControl)) {
+                  ProjectState projectState;
+                  try {
+                    projectState = projectCache.checkedGet(project);
+                  } catch (IOException e) {
+                    return false;
+                  }
+                  if (projectState == null) {
+                    throw new NoSuchProjectException(project);
+                  }
+                  if (!projectState.statePermitsRead()) {
+                    return false;
+                  }
+                  if (!shouldReplicate(projectState, userProvider.get())) {
                     return false;
                   }
                   if (PushOne.ALL_REFS.equals(ref)) {
@@ -276,7 +301,7 @@
                   }
                   try {
                     permissionBackend
-                        .user(projectControl.getUser())
+                        .user(userProvider.get())
                         .project(project)
                         .ref(ref)
                         .check(RefPermission.READ);
@@ -303,7 +328,16 @@
               new Callable<Boolean>() {
                 @Override
                 public Boolean call() throws NoSuchProjectException, PermissionBackendException {
-                  return shouldReplicate(controlFor(project));
+                  ProjectState projectState;
+                  try {
+                    projectState = projectCache.checkedGet(project);
+                  } catch (IOException e) {
+                    return false;
+                  }
+                  if (projectState == null) {
+                    throw new NoSuchProjectException(project);
+                  }
+                  return shouldReplicate(projectState, userProvider.get());
                 }
               })
           .call();
@@ -485,10 +519,6 @@
     }
   }
 
-  ProjectControl controlFor(Project.NameKey project) throws NoSuchProjectException {
-    return projectControlFactory.controlFor(project);
-  }
-
   RunwayStatus requestRunway(PushOne op) {
     synchronized (stateLock) {
       if (op.wasCanceled()) {

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
deleted file mode 100644
index 83eab86..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
+++ /dev/null

@@ -1,75 +0,0 @@
-// Copyright (C) 2016 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import com.google.gerrit.common.EventDispatcher;
-import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.gerrit.server.PluginUser;
-import com.google.gerrit.server.account.GroupBackend;
-import com.google.gerrit.server.account.GroupIncludeCache;
-import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.permissions.PermissionBackend;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
-
-@Singleton
-public class DestinationFactory {
-  private final Injector injector;
-  private final RemoteSiteUser.Factory replicationUserFactory;
-  private final PluginUser pluginUser;
-  private final GitRepositoryManager gitRepositoryManager;
-  private final PermissionBackend permissionBackend;
-  private final GroupBackend groupBackend;
-  private final ReplicationStateListener stateLog;
-  private final GroupIncludeCache groupIncludeCache;
-  private final DynamicItem<EventDispatcher> eventDispatcher;
-
-  @Inject
-  public DestinationFactory(
-      Injector injector,
-      RemoteSiteUser.Factory replicationUserFactory,
-      PluginUser pluginUser,
-      GitRepositoryManager gitRepositoryManager,
-      PermissionBackend permissionBackend,
-      GroupBackend groupBackend,
-      ReplicationStateListener stateLog,
-      GroupIncludeCache groupIncludeCache,
-      DynamicItem<EventDispatcher> eventDispatcher) {
-    this.injector = injector;
-    this.replicationUserFactory = replicationUserFactory;
-    this.pluginUser = pluginUser;
-    this.gitRepositoryManager = gitRepositoryManager;
-    this.permissionBackend = permissionBackend;
-    this.groupBackend = groupBackend;
-    this.stateLog = stateLog;
-    this.groupIncludeCache = groupIncludeCache;
-    this.eventDispatcher = eventDispatcher;
-  }
-
-  Destination create(DestinationConfiguration config) {
-    return new Destination(
-        injector,
-        config,
-        replicationUserFactory,
-        pluginUser,
-        gitRepositoryManager,
-        permissionBackend,
-        groupBackend,
-        stateLog,
-        groupIncludeCache,
-        eventDispatcher);
-  }
-}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java
new file mode 100644
index 0000000..0efa726
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java

@@ -0,0 +1,115 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.common.hash.Hashing;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+import com.google.inject.Singleton;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.eclipse.jgit.lib.ObjectId;
+
+@Singleton
+public class EventsStorage {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  public static class ReplicateRefUpdate {
+    public String project;
+    public String ref;
+
+    @Override
+    public String toString() {
+      return "ref-update " + project + ":" + ref;
+    }
+  }
+
+  private static Gson GSON = new Gson();
+
+  private final Path refUpdates;
+
+  @Inject
+  EventsStorage(ReplicationConfig config) {
+    refUpdates = config.getEventsDirectory().resolve("ref-updates");
+  }
+
+  public String persist(String project, String ref) {
+    ReplicateRefUpdate r = new ReplicateRefUpdate();
+    r.project = project;
+    r.ref = ref;
+
+    String json = GSON.toJson(r) + "\n";
+    String eventKey = sha1(json).name();
+    Path file = refUpdates().resolve(eventKey);
+
+    if (Files.exists(file)) {
+      return eventKey;
+    }
+
+    try {
+      Files.write(file, json.getBytes(UTF_8));
+    } catch (IOException e) {
+      logger.atWarning().log("Couldn't persist event %s", json);
+    }
+    return eventKey;
+  }
+
+  public void delete(String eventKey) {
+    if (eventKey != null) {
+      try {
+        Files.delete(refUpdates().resolve(eventKey));
+      } catch (IOException e) {
+        logger.atSevere().withCause(e).log("Error while deleting event %s", eventKey);
+      }
+    }
+  }
+
+  public List<ReplicateRefUpdate> list() {
+    ArrayList<ReplicateRefUpdate> result = new ArrayList<>();
+    try (DirectoryStream<Path> events = Files.newDirectoryStream(refUpdates())) {
+      for (Path e : events) {
+        if (Files.isRegularFile(e)) {
+          String json = new String(Files.readAllBytes(e), UTF_8);
+          result.add(GSON.fromJson(json, ReplicateRefUpdate.class));
+          Files.delete(e);
+        }
+      }
+    } catch (IOException e) {
+      logger.atSevere().withCause(e).log("Error when firing pending events");
+    }
+    return result;
+  }
+
+  @SuppressWarnings("deprecation")
+  private ObjectId sha1(String s) {
+    return ObjectId.fromRaw(Hashing.sha1().hashString(s, UTF_8).asBytes());
+  }
+
+  private Path refUpdates() {
+    try {
+      return Files.createDirectories(refUpdates);
+    } catch (IOException e) {
+      throw new ProvisionException(String.format("Couldn't create %s", refUpdates), e);
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
index b46a0d9..85b17d0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java

@@ -14,45 +14,45 @@
 
 package com.googlesource.gerrit.plugins.replication;
 
+import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.server.ssh.SshAddressesModule;
-import com.google.inject.Inject;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.HashSet;
 import java.util.Set;
 import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-public class GerritSshApi {
+public class GerritSshApi implements AdminApi {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   static int SSH_COMMAND_FAILED = -1;
-  private static final Logger log = LoggerFactory.getLogger(GerritSshApi.class);
   private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
 
   private final SshHelper sshHelper;
+  private final URIish uri;
 
   private final Set<URIish> withoutDeleteProjectPlugin = new HashSet<>();
 
-  @Inject
-  protected GerritSshApi(SshHelper sshHelper) {
+  protected GerritSshApi(SshHelper sshHelper, URIish uri) {
     this.sshHelper = sshHelper;
+    this.uri = uri;
   }
 
-  protected boolean createProject(URIish uri, Project.NameKey projectName, String head) {
+  @Override
+  public void createProject(Project.NameKey projectName, String head) {
     OutputStream errStream = sshHelper.newErrorBufferStream();
     String cmd = "gerrit create-project --branch " + head + " " + projectName.get();
     try {
       execute(uri, cmd, errStream);
     } catch (IOException e) {
       logError("creating", uri, errStream, cmd, e);
-      return false;
     }
-    return true;
   }
 
-  protected boolean deleteProject(URIish uri, Project.NameKey projectName) {
+  @Override
+  public void deleteProject(Project.NameKey projectName) {
     if (!withoutDeleteProjectPlugin.contains(uri)) {
       OutputStream errStream = sshHelper.newErrorBufferStream();
       String cmd = "deleteproject delete --yes-really-delete --force " + projectName.get();
@@ -61,36 +61,28 @@
         exitCode = execute(uri, cmd, errStream);
       } catch (IOException e) {
         logError("deleting", uri, errStream, cmd, e);
-        return false;
       }
       if (exitCode == 1) {
-        log.info(
-            "DeleteProject plugin is not installed on {}; will not try to forward this operation to that host");
+        logger.atInfo().log(
+            "DeleteProject plugin is not installed on %s;"
+                + " will not try to forward this operation to that host");
         withoutDeleteProjectPlugin.add(uri);
-        return true;
       }
     }
-    return true;
   }
 
-  protected boolean updateHead(URIish uri, Project.NameKey projectName, String newHead) {
+  @Override
+  public void updateHead(Project.NameKey projectName, String newHead) {
     OutputStream errStream = sshHelper.newErrorBufferStream();
     String cmd = "gerrit set-head " + projectName.get() + " --new-head " + newHead;
     try {
       execute(uri, cmd, errStream);
     } catch (IOException e) {
-      log.error(
-          "Error updating HEAD of remote repository at {} to {}:\n"
-              + "  Exception: {}\n  Command: {}\n  Output: {}",
-          uri,
-          newHead,
-          e,
-          cmd,
-          errStream,
-          e);
-      return false;
+      logger.atSevere().withCause(e).log(
+          "Error updating HEAD of remote repository at %s to %s:\n"
+              + "  Exception: %s\n  Command: %s\n  Output: %s",
+          uri, newHead, e, cmd, errStream);
     }
-    return true;
   }
 
   private URIish toSshUri(URIish uri) throws URISyntaxException {
@@ -114,19 +106,14 @@
       URIish sshUri = toSshUri(uri);
       return sshHelper.executeRemoteSsh(sshUri, cmd, errStream);
     } catch (URISyntaxException e) {
-      log.error("Cannot convert {} to SSH uri", uri, e);
+      logger.atSevere().withCause(e).log("Cannot convert %s to SSH uri", uri);
     }
     return SSH_COMMAND_FAILED;
   }
 
   public void logError(String msg, URIish uri, OutputStream errStream, String cmd, IOException e) {
-    log.error(
-        "Error {} remote repository at {}:\n  Exception: {}\n  Command: {}\n  Output: {}",
-        msg,
-        uri,
-        e,
-        cmd,
-        errStream,
-        e);
+    logger.atSevere().withCause(e).log(
+        "Error %s remote repository at %s:\n  Exception: %s\n  Command: %s\n  Output: %s",
+        msg, uri, e, cmd, errStream);
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
new file mode 100644
index 0000000..1012cd7
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java

@@ -0,0 +1,91 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import java.io.File;
+import java.io.IOException;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.URIish;
+
+public class LocalFS implements AdminApi {
+
+  private final URIish uri;
+
+  public LocalFS(URIish uri) {
+    this.uri = uri;
+  }
+
+  @Override
+  public void createProject(Project.NameKey project, String head) {
+    try (Repository repo = new FileRepository(uri.getPath())) {
+      repo.create(true /* bare */);
+
+      if (head != null && head.startsWith(Constants.R_REFS)) {
+        RefUpdate u = repo.updateRef(Constants.HEAD);
+        u.disableRefLog();
+        u.link(head);
+      }
+      repLog.info("Created local repository: {}", uri);
+    } catch (IOException e) {
+      repLog.error("Error creating local repository {}", uri.getPath(), e);
+    }
+  }
+
+  @Override
+  public void deleteProject(Project.NameKey project) {
+    try {
+      recursivelyDelete(new File(uri.getPath()));
+      repLog.info("Deleted local repository: {}", uri);
+    } catch (IOException e) {
+      repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
+    }
+  }
+
+  @Override
+  public void updateHead(Project.NameKey project, String newHead) {
+    try (Repository repo = new FileRepository(uri.getPath())) {
+      if (newHead != null) {
+        RefUpdate u = repo.updateRef(Constants.HEAD);
+        u.link(newHead);
+      }
+    } catch (IOException e) {
+      repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
+    }
+  }
+
+  private static void recursivelyDelete(File dir) throws IOException {
+    File[] contents = dir.listFiles();
+    if (contents != null) {
+      for (File d : contents) {
+        if (d.isDirectory()) {
+          recursivelyDelete(d);
+        } else {
+          if (!d.delete()) {
+            throw new IOException("Failed to delete: " + d.getAbsolutePath());
+          }
+        }
+      }
+    }
+    if (!dir.delete()) {
+      throw new IOException("Failed to delete: " + dir.getAbsolutePath());
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
index 227804d..ac0262d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java

@@ -15,12 +15,13 @@
 package com.googlesource.gerrit.plugins.replication;
 
 import com.google.common.util.concurrent.Atomics;
-import com.google.gerrit.common.EventDispatcher;
 import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.extensions.registration.DynamicItem;
 import com.google.gerrit.extensions.systemstatus.ServerInformation;
+import com.google.gerrit.server.events.EventDispatcher;
 import com.google.inject.Inject;
 import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
+import com.googlesource.gerrit.plugins.replication.ReplicationState.Factory;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -29,32 +30,31 @@
   private final AtomicReference<Future<?>> pushAllFuture;
   private final ServerInformation srvInfo;
   private final PushAll.Factory pushAll;
-  private final ReplicationQueue queue;
   private final ReplicationConfig config;
   private final DynamicItem<EventDispatcher> eventDispatcher;
+  private final Factory replicationStateFactory;
 
   @Inject
   protected OnStartStop(
       ServerInformation srvInfo,
       PushAll.Factory pushAll,
-      ReplicationQueue queue,
       ReplicationConfig config,
-      DynamicItem<EventDispatcher> eventDispatcher) {
+      DynamicItem<EventDispatcher> eventDispatcher,
+      ReplicationState.Factory replicationStateFactory) {
     this.srvInfo = srvInfo;
     this.pushAll = pushAll;
-    this.queue = queue;
     this.config = config;
     this.eventDispatcher = eventDispatcher;
+    this.replicationStateFactory = replicationStateFactory;
     this.pushAllFuture = Atomics.newReference();
   }
 
   @Override
   public void start() {
-    queue.start();
-
     if (srvInfo.getState() == ServerInformation.State.STARTUP
         && config.isReplicateAllOnPluginStart()) {
-      ReplicationState state = new ReplicationState(new GitUpdateProcessing(eventDispatcher.get()));
+      ReplicationState state =
+          replicationStateFactory.create(new GitUpdateProcessing(eventDispatcher.get()));
       pushAllFuture.set(
           pushAll
               .create(null, ReplicationFilter.all(), state, false)
@@ -68,6 +68,5 @@
     if (f != null) {
       f.cancel(true);
     }
-    queue.stop();
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
index db067e2..833b02b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java

@@ -43,7 +43,7 @@
       WorkQueue wq,
       ProjectCache projectCache,
       ReplicationQueue rq,
-      ReplicationStateListener stateLog,
+      ReplicationStateListeners stateLog,
       @Assisted @Nullable String urlMatch,
       @Assisted ReplicationFilter filter,
       @Assisted ReplicationState state,

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 9fe497b..88a301b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java

@@ -23,19 +23,21 @@
 import com.google.common.collect.Sets;
 import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
 import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.ResourceConflictException;
 import com.google.gerrit.metrics.Timer1;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.reviewdb.client.RefNames;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.PerThreadRequestScope;
 import com.google.gerrit.server.git.ProjectRunnable;
-import com.google.gerrit.server.git.VisibleRefFilter;
 import com.google.gerrit.server.git.WorkQueue.CanceledWhileRunning;
+import com.google.gerrit.server.ioutil.HexFormat;
 import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.permissions.PermissionBackend.RefFilterOptions;
 import com.google.gerrit.server.permissions.PermissionBackendException;
 import com.google.gerrit.server.permissions.ProjectPermission;
-import com.google.gerrit.server.project.NoSuchProjectException;
-import com.google.gerrit.server.project.ProjectControl;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.gerrit.server.project.ProjectState;
 import com.google.gerrit.server.util.IdGenerator;
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
@@ -92,7 +94,6 @@
   private final RemoteConfig config;
   private final CredentialsProvider credentialsProvider;
   private final PerThreadRequestScope.Scoper threadScoper;
-  private final VisibleRefFilter.Factory refFilterFactory;
   private final ReplicationQueue replicationQueue;
 
   private final Project.NameKey projectName;
@@ -110,6 +111,7 @@
   private final int id;
   private final long createdAt;
   private final ReplicationMetrics metrics;
+  private final ProjectCache projectCache;
   private final AtomicBoolean canceledWhileRunning;
 
   @Inject
@@ -118,20 +120,19 @@
       PermissionBackend permissionBackend,
       Destination p,
       RemoteConfig c,
-      VisibleRefFilter.Factory rff,
       CredentialsFactory cpFactory,
       PerThreadRequestScope.Scoper ts,
       ReplicationQueue rq,
       IdGenerator ig,
-      ReplicationStateListener sl,
+      ReplicationStateListeners sl,
       ReplicationMetrics m,
+      ProjectCache pc,
       @Assisted Project.NameKey d,
       @Assisted URIish u) {
     gitManager = grm;
     this.permissionBackend = permissionBackend;
     pool = p;
     config = c;
-    refFilterFactory = rff;
     credentialsProvider = cpFactory.create(c.getName());
     threadScoper = ts;
     replicationQueue = rq;
@@ -143,13 +144,14 @@
     stateLog = sl;
     createdAt = System.nanoTime();
     metrics = m;
+    projectCache = pc;
     canceledWhileRunning = new AtomicBoolean(false);
     maxRetries = p.getMaxRetries();
   }
 
   @Override
   public void cancel() {
-    repLog.info("Replication [{}] to {} was canceled", IdGenerator.format(id), getURI());
+    repLog.info("Replication [{}] to {} was canceled", HexFormat.fromInt(id), getURI());
     canceledByReplication();
     pool.pushWasCanceled(this);
   }
@@ -158,7 +160,7 @@
   public void setCanceledWhileRunning() {
     repLog.info(
         "Replication [{}] to {} was canceled while being executed",
-        IdGenerator.format(id),
+        HexFormat.fromInt(id),
         getURI());
     canceledWhileRunning.set(true);
   }
@@ -180,7 +182,7 @@
 
   @Override
   public String toString() {
-    String print = "[" + IdGenerator.format(id) + "] push " + uri;
+    String print = "[" + HexFormat.fromInt(id) + "] push " + uri;
 
     if (retryCount > 0) {
       print = "(retry " + retryCount + ") " + print;
@@ -301,7 +303,7 @@
     // we start replication (instead a new instance, with the same URI, is
     // created and scheduled for a future point in time.)
     //
-    MDC.put(ID_MDC_KEY, IdGenerator.format(id));
+    MDC.put(ID_MDC_KEY, HexFormat.fromInt(id));
     RunwayStatus status = pool.requestRunway(this);
     if (!status.isAllowed()) {
       if (status.isCanceled()) {
@@ -310,7 +312,7 @@
         repLog.info(
             "Rescheduling replication to {} to avoid collision with the in-flight push [{}].",
             uri,
-            IdGenerator.format(status.getInFlightPushId()));
+            HexFormat.fromInt(status.getInFlightPushId()));
         pool.reschedule(this, Destination.RetryReason.COLLISION);
       }
       return;
@@ -464,19 +466,19 @@
 
   private List<RemoteRefUpdate> generateUpdates(Transport tn)
       throws IOException, PermissionBackendException {
-    ProjectControl pc;
-    try {
-      pc = pool.controlFor(projectName);
-    } catch (NoSuchProjectException e) {
+    ProjectState projectState = projectCache.checkedGet(projectName);
+    if (projectState == null) {
       return Collections.emptyList();
     }
 
     Map<String, Ref> local = git.getAllRefs();
     boolean filter;
+    PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName);
     try {
-      permissionBackend.user(pc.getUser()).project(projectName).check(ProjectPermission.READ);
+      projectState.checkStatePermitsRead();
+      forProject.check(ProjectPermission.READ);
       filter = false;
-    } catch (AuthException e) {
+    } catch (AuthException | ResourceConflictException e) {
       filter = true;
     }
     if (filter) {
@@ -493,7 +495,7 @@
         }
         local = n;
       }
-      local = refFilterFactory.create(pc.getProjectState(), git).filter(local, true);
+      local = forProject.filter(local, git, RefFilterOptions.builder().setFilterMeta(true).build());
     }
 
     return pushAllRefs ? doPushAll(tn, local) : doPushDelta(local);

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
index c71a792..ae0662d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java

@@ -14,7 +14,8 @@
 
 package com.googlesource.gerrit.plugins.replication;
 
-import com.google.gerrit.common.EventDispatcher;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.server.events.EventDispatcher;
 import com.google.gerrit.server.events.RefEvent;
 import com.google.gerrit.server.permissions.PermissionBackendException;
 import com.google.gwtorm.server.OrmException;
@@ -23,8 +24,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.eclipse.jgit.transport.RemoteRefUpdate;
 import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public abstract class PushResultProcessing {
 
@@ -159,7 +158,7 @@
   }
 
   public static class GitUpdateProcessing extends PushResultProcessing {
-    private static final Logger log = LoggerFactory.getLogger(GitUpdateProcessing.class);
+    private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
     private final EventDispatcher dispatcher;
 
@@ -189,7 +188,7 @@
       try {
         dispatcher.postEvent(event);
       } catch (OrmException | PermissionBackendException e) {
-        log.error("Cannot post event", e);
+        logger.atSevere().withCause(e).log("Cannot post event");
       }
     }
   }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
index 91fce7f..c3556af 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java

@@ -16,18 +16,11 @@
 
 import com.google.gerrit.server.CurrentUser;
 import com.google.gerrit.server.account.GroupMembership;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
 
 public class RemoteSiteUser extends CurrentUser {
-  public interface Factory {
-    RemoteSiteUser create(@Assisted GroupMembership authGroups);
-  }
-
   private final GroupMembership effectiveGroups;
 
-  @Inject
-  RemoteSiteUser(@Assisted GroupMembership authGroups) {
+  public RemoteSiteUser(GroupMembership authGroups) {
     effectiveGroups = authGroups;
   }
 
@@ -35,4 +28,10 @@
   public GroupMembership getEffectiveGroups() {
     return effectiveGroups;
   }
+
+  @Override
+  public Object getCacheKey() {
+    // Never cache a remote user
+    return new Object();
+  }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
new file mode 100644
index 0000000..dad1b0b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java

@@ -0,0 +1,104 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import java.io.IOException;
+import java.io.OutputStream;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.util.QuotedString;
+
+public class RemoteSsh implements AdminApi {
+
+  private final SshHelper sshHelper;
+  private URIish uri;
+
+  RemoteSsh(SshHelper sshHelper, URIish uri) {
+    this.sshHelper = sshHelper;
+    this.uri = uri;
+  }
+
+  @Override
+  public void createProject(Project.NameKey project, String head) {
+    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+    String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare";
+    if (head != null) {
+      cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head);
+    }
+    OutputStream errStream = sshHelper.newErrorBufferStream();
+    try {
+      sshHelper.executeRemoteSsh(uri, cmd, errStream);
+      repLog.info("Created remote repository: {}", uri);
+    } catch (IOException e) {
+      repLog.error(
+          "Error creating remote repository at {}:\n"
+              + "  Exception: {}\n"
+              + "  Command: {}\n"
+              + "  Output: {}",
+          uri,
+          e,
+          cmd,
+          errStream,
+          e);
+    }
+  }
+
+  @Override
+  public void deleteProject(Project.NameKey project) {
+    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+    String cmd = "rm -rf " + quotedPath;
+    OutputStream errStream = sshHelper.newErrorBufferStream();
+    try {
+      sshHelper.executeRemoteSsh(uri, cmd, errStream);
+      repLog.info("Deleted remote repository: {}", uri);
+    } catch (IOException e) {
+      repLog.error(
+          "Error deleting remote repository at {}:\n"
+              + "  Exception: {}\n"
+              + "  Command: {}\n"
+              + "  Output: {}",
+          uri,
+          e,
+          cmd,
+          errStream,
+          e);
+    }
+  }
+
+  @Override
+  public void updateHead(Project.NameKey project, String newHead) {
+    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+    String cmd =
+        "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead);
+    OutputStream errStream = sshHelper.newErrorBufferStream();
+    try {
+      sshHelper.executeRemoteSsh(uri, cmd, errStream);
+    } catch (IOException e) {
+      repLog.error(
+          "Error updating HEAD of remote repository at {} to {}:\n"
+              + "  Exception: {}\n"
+              + "  Command: {}\n"
+              + "  Output: {}",
+          uri,
+          newHead,
+          e,
+          cmd,
+          errStream,
+          e);
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index e94abbd..9693e2d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java

@@ -14,6 +14,7 @@
 package com.googlesource.gerrit.plugins.replication;
 
 import com.google.gerrit.server.git.WorkQueue;
+import java.nio.file.Path;
 import java.util.List;
 
 public interface ReplicationConfig {
@@ -32,6 +33,8 @@
 
   boolean isEmpty();
 
+  Path getEventsDirectory();
+
   int shutdown();
 
   void startup(WorkQueue workQueue);

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index ee1f16d..46d2f66 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java

@@ -15,8 +15,11 @@
 
 import static java.util.stream.Collectors.toList;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.extensions.annotations.PluginData;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.inject.Inject;
@@ -35,24 +38,28 @@
 import org.eclipse.jgit.transport.RemoteConfig;
 import org.eclipse.jgit.transport.URIish;
 import org.eclipse.jgit.util.FS;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 @Singleton
 public class ReplicationFileBasedConfig implements ReplicationConfig {
-  private static final Logger log = LoggerFactory.getLogger(ReplicationFileBasedConfig.class);
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   private List<Destination> destinations;
+  private final SitePaths site;
   private Path cfgPath;
   private boolean replicateAllOnPluginStart;
   private boolean defaultForceUpdate;
   private final FileBasedConfig config;
+  private final Path pluginDataDir;
 
   @Inject
-  public ReplicationFileBasedConfig(SitePaths site, DestinationFactory destinationFactory)
+  public ReplicationFileBasedConfig(
+      SitePaths site, Destination.Factory destinationFactory, @PluginData Path pluginDataDir)
       throws ConfigInvalidException, IOException {
+    this.site = site;
     this.cfgPath = site.etc_dir.resolve("replication.config");
     this.config = new FileBasedConfig(cfgPath.toFile(), FS.DETECTED);
     this.destinations = allDestinations(destinationFactory);
+    this.pluginDataDir = pluginDataDir;
   }
 
   /*
@@ -79,14 +86,14 @@
     return destinations.stream().filter(Objects::nonNull).filter(filter).collect(toList());
   }
 
-  private List<Destination> allDestinations(DestinationFactory destinationFactory)
+  private List<Destination> allDestinations(Destination.Factory destinationFactory)
       throws ConfigInvalidException, IOException {
     if (!config.getFile().exists()) {
-      log.warn("Config file {} does not exist; not replicating", config.getFile());
+      logger.atWarning().log("Config file %s does not exist; not replicating", config.getFile());
       return Collections.emptyList();
     }
     if (config.getFile().length() == 0) {
-      log.info("Config file {} is empty; not replicating", config.getFile());
+      logger.atInfo().log("Config file %s is empty; not replicating", config.getFile());
       return Collections.emptyList();
     }
 
@@ -100,7 +107,7 @@
           String.format("Cannot read %s: %s", config.getFile(), e.getMessage()), e);
     }
 
-    replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", true);
+    replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", false);
 
     defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
 
@@ -180,6 +187,15 @@
     return destinations.isEmpty();
   }
 
+  @Override
+  public Path getEventsDirectory() {
+    String eventsDirectory = config.getString("replication", null, "eventsDirectory");
+    if (!Strings.isNullOrEmpty(eventsDirectory)) {
+      return site.resolve(eventsDirectory);
+    }
+    return pluginDataDir;
+  }
+
   Path getCfgPath() {
     return cfgPath;
   }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
index 7b3486b..05bbb03 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java

@@ -15,7 +15,7 @@
 package com.googlesource.gerrit.plugins.replication;
 
 import com.google.gerrit.common.data.AccessSection;
-import com.google.gerrit.reviewdb.client.Project.NameKey;
+import com.google.gerrit.reviewdb.client.Project;
 import java.util.Collections;
 import java.util.List;
 
@@ -46,7 +46,7 @@
     projectPatterns = patterns;
   }
 
-  public boolean matches(NameKey name) {
+  public boolean matches(Project.NameKey name) {
     if (projectPatterns.isEmpty()) {
       return true;
     }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index f30e13d..1405a12 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java

@@ -34,8 +34,11 @@
 class ReplicationModule extends AbstractModule {
   @Override
   protected void configure() {
-    bind(DestinationFactory.class).in(Scopes.SINGLETON);
+    install(new FactoryModuleBuilder().build(Destination.Factory.class));
     bind(ReplicationQueue.class).in(Scopes.SINGLETON);
+    bind(LifecycleListener.class)
+        .annotatedWith(UniqueAnnotations.create())
+        .to(ReplicationQueue.class);
 
     DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class);
     DynamicSet.bind(binder(), NewProjectCreatedListener.class).to(ReplicationQueue.class);
@@ -55,14 +58,17 @@
         .to(StartReplicationCapability.class);
 
     install(new FactoryModuleBuilder().build(PushAll.Factory.class));
-    install(new FactoryModuleBuilder().build(RemoteSiteUser.Factory.class));
+    install(new FactoryModuleBuilder().build(ReplicationState.Factory.class));
 
     bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class);
-    bind(ReplicationStateListener.class).to(ReplicationStateLogger.class);
+    DynamicSet.setOf(binder(), ReplicationStateListener.class);
+    DynamicSet.bind(binder(), ReplicationStateListener.class).to(ReplicationStateLogger.class);
 
     EventTypes.register(RefReplicatedEvent.TYPE, RefReplicatedEvent.class);
     EventTypes.register(RefReplicationDoneEvent.TYPE, RefReplicationDoneEvent.class);
     EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class);
     bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class);
+
+    bind(AdminApiFactory.class);
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 30aff44..541a595 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java

@@ -14,8 +14,10 @@
 
 package com.googlesource.gerrit.plugins.replication;
 
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
+
 import com.google.common.base.Strings;
-import com.google.gerrit.common.EventDispatcher;
 import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
 import com.google.gerrit.extensions.events.HeadUpdatedListener;
 import com.google.gerrit.extensions.events.LifecycleListener;
@@ -23,23 +25,17 @@
 import com.google.gerrit.extensions.events.ProjectDeletedListener;
 import com.google.gerrit.extensions.registration.DynamicItem;
 import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.events.EventDispatcher;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.inject.Inject;
 import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
 import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.RefUpdate;
-import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.transport.URIish;
-import org.eclipse.jgit.util.QuotedString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -68,32 +64,38 @@
   }
 
   private final WorkQueue workQueue;
-  private final SshHelper sshHelper;
   private final DynamicItem<EventDispatcher> dispatcher;
   private final ReplicationConfig config;
-  private final GerritSshApi gerritAdmin;
+  private final AdminApiFactory adminApiFactory;
+  private final ReplicationState.Factory replicationStateFactory;
+  private final EventsStorage eventsStorage;
   private volatile boolean running;
 
   @Inject
   ReplicationQueue(
       WorkQueue wq,
-      SshHelper sh,
-      GerritSshApi ga,
+      AdminApiFactory aaf,
       ReplicationConfig rc,
       DynamicItem<EventDispatcher> dis,
-      ReplicationStateListener sl) {
+      ReplicationStateListeners sl,
+      ReplicationState.Factory rsf,
+      EventsStorage es) {
     workQueue = wq;
-    sshHelper = sh;
     dispatcher = dis;
     config = rc;
     stateLog = sl;
-    gerritAdmin = ga;
+    adminApiFactory = aaf;
+    replicationStateFactory = rsf;
+    eventsStorage = es;
   }
 
   @Override
   public void start() {
-    config.startup(workQueue);
-    running = true;
+    if (!running) {
+      config.startup(workQueue);
+      running = true;
+      firePendingEvents();
+    }
   }
 
   @Override
@@ -127,23 +129,37 @@
 
   @Override
   public void onGitReferenceUpdated(GitReferenceUpdatedListener.Event event) {
-    ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get()));
+    onGitReferenceUpdated(event.getProjectName(), event.getRefName());
+  }
+
+  private void onGitReferenceUpdated(String projectName, String refName) {
+    ReplicationState state =
+        replicationStateFactory.create(new GitUpdateProcessing(dispatcher.get()));
     if (!running) {
       stateLog.warn("Replication plugin did not finish startup before event", state);
       return;
     }
 
-    Project.NameKey project = new Project.NameKey(event.getProjectName());
+    Project.NameKey project = new Project.NameKey(projectName);
     for (Destination cfg : config.getDestinations(FilterType.ALL)) {
-      if (cfg.wouldPushProject(project) && cfg.wouldPushRef(event.getRefName())) {
+      if (cfg.wouldPushProject(project) && cfg.wouldPushRef(refName)) {
+        String eventKey = eventsStorage.persist(projectName, refName);
+        state.setEventKey(eventKey);
         for (URIish uri : cfg.getURIs(project, null)) {
-          cfg.schedule(project, event.getRefName(), uri, state);
+          cfg.schedule(project, refName, uri, state);
         }
       }
     }
     state.markAllPushTasksScheduled();
   }
 
+  private void firePendingEvents() {
+    for (EventsStorage.ReplicateRefUpdate e : eventsStorage.list()) {
+      repLog.info("Firing pending event {}", e);
+      onGitReferenceUpdated(e.project, e.ref);
+    }
+  }
+
   @Override
   public void onNewProjectCreated(NewProjectCreatedListener.Event event) {
     Project.NameKey projectName = new Project.NameKey(event.getProjectName());
@@ -234,192 +250,41 @@
   }
 
   private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
-    if (isGerrit(replicateURI)) {
-      gerritAdmin.createProject(replicateURI, projectName, head);
-    } else if (!replicateURI.isRemote()) {
-      createLocally(replicateURI, head);
-    } else if (isSSH(replicateURI)) {
-      createRemoteSsh(replicateURI, head);
-    } else {
-      repLog.warn(
-          "Cannot create new project on remote site {}."
-              + " Only local paths and SSH URLs are supported"
-              + " for remote repository creation",
-          replicateURI);
-      return false;
-    }
-    return true;
-  }
-
-  private static void createLocally(URIish uri, String head) {
-    try (Repository repo = new FileRepository(uri.getPath())) {
-      repo.create(true /* bare */);
-
-      if (head != null && head.startsWith(Constants.R_REFS)) {
-        RefUpdate u = repo.updateRef(Constants.HEAD);
-        u.disableRefLog();
-        u.link(head);
-      }
-      repLog.info("Created local repository: {}", uri);
-    } catch (IOException e) {
-      repLog.error("Error creating local repository {}:\n", uri.getPath(), e);
-    }
-  }
-
-  private void createRemoteSsh(URIish uri, String head) {
-    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
-    String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare";
-    if (head != null) {
-      cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head);
-    }
-    OutputStream errStream = sshHelper.newErrorBufferStream();
-    try {
-      sshHelper.executeRemoteSsh(uri, cmd, errStream);
-      repLog.info("Created remote repository: {}", uri);
-    } catch (IOException e) {
-      repLog.error(
-          "Error creating remote repository at {}:\n"
-              + "  Exception: {}\n"
-              + "  Command: {}\n"
-              + "  Output: {}",
-          uri,
-          e,
-          cmd,
-          errStream,
-          e);
-    }
-  }
-
-  private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
-    if (isGerrit(replicateURI)) {
-      gerritAdmin.deleteProject(replicateURI, projectName);
-      repLog.info("Deleted remote repository: " + replicateURI);
-    } else if (!replicateURI.isRemote()) {
-      deleteLocally(replicateURI);
-    } else if (isSSH(replicateURI)) {
-      deleteRemoteSsh(replicateURI);
-    } else {
-      repLog.warn(
-          "Cannot delete project on remote site {}. "
-              + "Only local paths and SSH URLs are supported"
-              + " for remote repository deletion",
-          replicateURI);
-    }
-  }
-
-  private static void deleteLocally(URIish uri) {
-    try {
-      recursivelyDelete(new File(uri.getPath()));
-      repLog.info("Deleted local repository: {}", uri);
-    } catch (IOException e) {
-      repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
-    }
-  }
-
-  private static void recursivelyDelete(File dir) throws IOException {
-    File[] contents = dir.listFiles();
-    if (contents != null) {
-      for (File d : contents) {
-        if (d.isDirectory()) {
-          recursivelyDelete(d);
-        } else {
-          if (!d.delete()) {
-            throw new IOException("Failed to delete: " + d.getAbsolutePath());
-          }
-        }
-      }
-    }
-    if (!dir.delete()) {
-      throw new IOException("Failed to delete: " + dir.getAbsolutePath());
-    }
-  }
-
-  private void deleteRemoteSsh(URIish uri) {
-    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
-    String cmd = "rm -rf " + quotedPath;
-    OutputStream errStream = sshHelper.newErrorBufferStream();
-    try {
-      sshHelper.executeRemoteSsh(uri, cmd, errStream);
-      repLog.info("Deleted remote repository: {}", uri);
-    } catch (IOException e) {
-      repLog.error(
-          "Error deleting remote repository at {}:\n"
-              + "  Exception: {}\n"
-              + "  Command: {}\n"
-              + "  Output: {}",
-          uri,
-          e,
-          cmd,
-          errStream,
-          e);
-    }
-  }
-
-  private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
-    if (isGerrit(replicateURI)) {
-      gerritAdmin.updateHead(replicateURI, projectName, newHead);
-    } else if (!replicateURI.isRemote()) {
-      updateHeadLocally(replicateURI, newHead);
-    } else if (isSSH(replicateURI)) {
-      updateHeadRemoteSsh(replicateURI, newHead);
-    } else {
-      repLog.warn(
-          "Cannot update HEAD of project on remote site {}."
-              + " Only local paths and SSH URLs are supported"
-              + " for remote HEAD update.",
-          replicateURI);
-    }
-  }
-
-  private void updateHeadRemoteSsh(URIish uri, String newHead) {
-    String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
-    String cmd =
-        "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead);
-    OutputStream errStream = sshHelper.newErrorBufferStream();
-    try {
-      sshHelper.executeRemoteSsh(uri, cmd, errStream);
-    } catch (IOException e) {
-      repLog.error(
-          "Error updating HEAD of remote repository at {} to {}:\n"
-              + "  Exception: {}\n"
-              + "  Command: {}\n"
-              + "  Output: {}",
-          uri,
-          newHead,
-          e,
-          cmd,
-          errStream,
-          e);
-    }
-  }
-
-  private static void updateHeadLocally(URIish uri, String newHead) {
-    try (Repository repo = new FileRepository(uri.getPath())) {
-      if (newHead != null) {
-        RefUpdate u = repo.updateRef(Constants.HEAD);
-        u.link(newHead);
-      }
-    } catch (IOException e) {
-      repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
-    }
-  }
-
-  private static boolean isSSH(URIish uri) {
-    String scheme = uri.getScheme();
-    if (!uri.isRemote()) {
-      return false;
-    }
-    if (scheme != null && scheme.toLowerCase().contains("ssh")) {
+    Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+    if (adminApi.isPresent()) {
+      adminApi.get().createProject(projectName, head);
       return true;
     }
-    if (scheme == null && uri.getHost() != null && uri.getPath() != null) {
-      return true;
-    }
+
+    warnCannotPerform("create new project", replicateURI);
     return false;
   }
 
-  private static boolean isGerrit(URIish uri) {
-    String scheme = uri.getScheme();
-    return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+  private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
+    Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+    if (adminApi.isPresent()) {
+      adminApi.get().deleteProject(projectName);
+      return;
+    }
+
+    warnCannotPerform("delete project", replicateURI);
+  }
+
+  private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
+    Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+    if (adminApi.isPresent()) {
+      adminApi.get().updateHead(projectName, newHead);
+      return;
+    }
+
+    warnCannotPerform("update HEAD of project", replicateURI);
+  }
+
+  private void warnCannotPerform(String op, URIish uri) {
+    repLog.warn(
+        "Cannot {} on remote site {}."
+            + "Only local paths and SSH URLs are supported for this operation",
+        op,
+        uri);
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
index 7268709..301219f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java

@@ -15,7 +15,6 @@
 package com.googlesource.gerrit.plugins.replication;
 
 import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.client.Project.NameKey;
 import com.google.gerrit.server.events.RefEvent;
 
 public class ReplicationScheduledEvent extends RefEvent {
@@ -38,7 +37,7 @@
   }
 
   @Override
-  public NameKey getProjectNameKey() {
+  public Project.NameKey getProjectNameKey() {
     return new Project.NameKey(project);
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
index 86557e2..ec878db 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java

@@ -16,6 +16,8 @@
 
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.Table;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -23,7 +25,13 @@
 import org.eclipse.jgit.transport.URIish;
 
 public class ReplicationState {
+
+  public interface Factory {
+    ReplicationState create(PushResultProcessing processing);
+  }
+
   private boolean allScheduled;
+  private final EventsStorage eventsStorage;
   private final PushResultProcessing pushResultProcessing;
 
   private final Lock countingLock = new ReentrantLock();
@@ -49,7 +57,11 @@
   private int totalPushTasksCount;
   private int finishedPushTasksCount;
 
-  public ReplicationState(PushResultProcessing processing) {
+  private String eventKey;
+
+  @AssistedInject
+  ReplicationState(EventsStorage storage, @Assisted PushResultProcessing processing) {
+    eventsStorage = storage;
     pushResultProcessing = processing;
     statusByProjectRef = HashBasedTable.create();
   }
@@ -133,10 +145,17 @@
   }
 
   private void doRefPushTasksCompleted(RefReplicationStatus refStatus) {
+    deleteEvent();
     pushResultProcessing.onRefReplicatedToAllNodes(
         refStatus.project, refStatus.ref, refStatus.nodesToReplicateCount);
   }
 
+  private void deleteEvent() {
+    if (eventKey != null) {
+      eventsStorage.delete(eventKey);
+    }
+  }
+
   private RefReplicationStatus getRefStatus(String project, String ref) {
     if (!statusByProjectRef.contains(project, ref)) {
       RefReplicationStatus refStatus = new RefReplicationStatus(project, ref);
@@ -173,4 +192,8 @@
       return name().toLowerCase().replace("_", "-");
     }
   }
+
+  public void setEventKey(String eventKey) {
+    this.eventKey = eventKey;
+  }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListeners.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListeners.java
new file mode 100644
index 0000000..d7bf227
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateListeners.java

@@ -0,0 +1,48 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.extensions.registration.DynamicSet;
+import com.google.inject.Inject;
+
+public class ReplicationStateListeners implements ReplicationStateListener {
+  private final DynamicSet<ReplicationStateListener> listeners;
+
+  @Inject
+  ReplicationStateListeners(DynamicSet<ReplicationStateListener> stateListeners) {
+    this.listeners = stateListeners;
+  }
+
+  @Override
+  public void warn(String msg, ReplicationState... states) {
+    for (ReplicationStateListener listener : listeners) {
+      listener.warn(msg, states);
+    }
+  }
+
+  @Override
+  public void error(String msg, ReplicationState... states) {
+    for (ReplicationStateListener listener : listeners) {
+      listener.error(msg, states);
+    }
+  }
+
+  @Override
+  public void error(String msg, Throwable t, ReplicationState... states) {
+    for (ReplicationStateListener listener : listeners) {
+      listener.error(msg, t, states);
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
index 2b0c16b..c518091 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java

@@ -17,6 +17,7 @@
 import com.google.gerrit.server.config.SitePaths;
 import com.google.inject.Inject;
 import java.io.IOException;
+import java.util.Objects;
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -49,8 +50,8 @@
 
   @Override
   public SecureCredentialsProvider create(String remoteName) {
-    String user = config.getString("remote", remoteName, "username");
-    String pass = config.getString("remote", remoteName, "password");
+    String user = Objects.toString(config.getString("remote", remoteName, "username"), "");
+    String pass = Objects.toString(config.getString("remote", remoteName, "password"), "");
     return new SecureCredentialsProvider(user, pass);
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
index c4294a9..62b4036 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java

@@ -20,7 +20,7 @@
 import org.eclipse.jgit.transport.URIish;
 
 /** Looks up a remote's password in secure.config. */
-class SecureCredentialsProvider extends CredentialsProvider {
+public class SecureCredentialsProvider extends CredentialsProvider {
   private final String cfgUser;
   private final String cfgPass;
 

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
index 5aa0861..77bc285 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java

@@ -51,13 +51,15 @@
 
   @Inject private PushAll.Factory pushFactory;
 
+  @Inject private ReplicationState.Factory replicationStateFactory;
+
   @Override
   protected void run() throws Failure {
     if (all && projectPatterns.size() > 0) {
       throw new UnloggedFailure(1, "error: cannot combine --all and PROJECT");
     }
 
-    ReplicationState state = new ReplicationState(new CommandProcessing(this));
+    ReplicationState state = replicationStateFactory.create(new CommandProcessing(this));
     Future<?> future = null;
 
     ReplicationFilter projectFilter;

diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index c066513..2359599 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md

@@ -13,6 +13,11 @@
   sudo su -c 'ssh mirror1.us.some.org echo' gerrit2
 ```
 
+*NOTE:* make sure the local user's ssh keys format is PEM, here how to generate them:
+```
+  ssh-keygen -m PEM -t rsa -C "your_email@example.com"
+```
+
 <a name="example_file">
 Next, create `$site_path/etc/replication.config` as a Git-style config
 file, for example to replicate in parallel to four different hosts:</a>
@@ -65,7 +70,7 @@
 
 gerrit.replicateOnStartup
 :	If true, replicates to all remotes on startup to ensure they
-	are in-sync with this server.  By default, true.
+	are in-sync with this server.  By default, false.
 
 gerrit.autoReload
 :	If true, automatically reloads replication destinations and settings
@@ -116,6 +121,17 @@
 
 	By default, pushes are retried indefinitely.
 
+replication.eventsDirectory
+: Directory where replication events are persisted
+
+	When scheduling a replication, the replication event is persisted
+	under this directory. When the replication is done, the event is deleted.
+	If plugin is stopped before all scheduled replications are done, the
+	persisted events will not be deleted. When the plugin is started again,
+	it will trigger all replications found under this directory.
+
+	When not set, defaults to the plugin's data directory.
+
 remote.NAME.url
 :	Address of the remote server to push to.  Multiple URLs may be
 	specified within a single remote block, listing different
@@ -290,7 +306,7 @@
 	If the remote site was not available at the moment when a new
 	project was created, it will be created if during the replication
 	of a ref it is found to be missing.
-	
+
 	If false, repositories are never created automatically on this
 	remote.
 
@@ -390,7 +406,7 @@
 File `~/.ssh/config`
 --------------------
 
-If present, Gerrit reads and caches `~/.ssh/config` at startup, and
+Gerrit reads and caches the `~/.ssh/config` at startup, and
 supports most SSH configuration options.  For example:
 
 ```
@@ -404,6 +420,15 @@
     PreferredAuthentications publickey
 ```
 
+*IdentityFile* and *PreferredAuthentications* must be defined for all the hosts.
+Here an example of the minimum `~/.ssh/config` needed:
+
+```
+  Host *
+    IdentityFile ~/.ssh/id_rsa
+    PreferredAuthentications publickey
+```
+
 Supported options:
 
   * Host

diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 337bd1d..5fa7b98 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java

@@ -22,8 +22,8 @@
 import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
 
-import com.google.gerrit.common.EventDispatcher;
 import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.events.EventDispatcher;
 import com.google.gerrit.server.permissions.PermissionBackendException;
 import com.google.gwtorm.client.KeyUtil;
 import com.google.gwtorm.server.OrmException;

diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java
index 193af1e..cf6715e 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java

@@ -32,12 +32,15 @@
 
   private ReplicationState replicationState;
   private PushResultProcessing pushResultProcessingMock;
+  private EventsStorage eventsStorage;
 
   @Before
   public void setUp() throws Exception {
     pushResultProcessingMock = createNiceMock(PushResultProcessing.class);
     replay(pushResultProcessingMock);
-    replicationState = new ReplicationState(pushResultProcessingMock);
+    eventsStorage = createNiceMock(EventsStorage.class);
+    replay(eventsStorage);
+    replicationState = new ReplicationState(eventsStorage, pushResultProcessingMock);
   }
 
   @Test