diff --git a/BUILD b/BUILD
index cd530c9..1ed5cf3 100644
--- a/BUILD
+++ b/BUILD
@@ -25,6 +25,18 @@
     timeout = "long",
     srcs = glob([
         "src/test/java/**/*Test.java",
+    ]),
+    tags = ["replication"],
+    visibility = ["//visibility:public"],
+    deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [
+        ":replication__plugin",
+        ":replication_util",
+    ],
+)
+
+junit_tests(
+    name = "replication_it",
+    srcs = glob([
         "src/test/java/**/*IT.java",
     ]),
     tags = ["replication"],
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
index 431c7d2..c3556af 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
@@ -28,4 +28,10 @@
   public GroupMembership getEffectiveGroups() {
     return effectiveGroups;
   }
+
+  @Override
+  public Object getCacheKey() {
+    // Never cache a remote user
+    return new Object();
+  }
 }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
index e0e3380..55c6a9c 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -123,8 +123,6 @@
 
     Project.NameKey sourceProject = createTestProject("foo");
 
-    assertThat(listIncompleteTasks("refs/meta/config")).hasSize(1);
-
     WaitUtil.waitUntil(
         () -> nonEmptyProjectExists(Project.nameKey(sourceProject + "replica.git")),
         TEST_NEW_PROJECT_TIMEOUT);
@@ -172,8 +170,6 @@
     RevCommit sourceCommit = pushResult.getCommit();
     String sourceRef = pushResult.getPatchSet().refName();
 
-    assertThat(listIncompleteTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(1);
-
     try (Repository repo = repoManager.openRepository(targetProject)) {
       waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
 
@@ -195,8 +191,6 @@
     input.revision = master;
     gApi.projects().name(project.get()).branch(newBranch).create(input);
 
-    assertThat(listIncompleteTasks("refs/heads/(mybranch|master)")).hasSize(2);
-
     try (Repository repo = repoManager.openRepository(targetProject);
         Repository sourceRepo = repoManager.openRepository(project)) {
       waitUntil(() -> checkedGetRef(repo, newBranch) != null);
@@ -221,8 +215,6 @@
     RevCommit sourceCommit = pushResult.getCommit();
     String sourceRef = pushResult.getPatchSet().refName();
 
-    assertThat(listIncompleteTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(2);
-
     try (Repository repo1 = repoManager.openRepository(targetProject1);
         Repository repo2 = repoManager.openRepository(targetProject2)) {
       waitUntil(
@@ -245,10 +237,12 @@
     createTestProject(project + "replica1");
     createTestProject(project + "replica2");
 
-    setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
-    setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
-    config.setInt("remote", "foo1", "replicationDelay", TEST_REPLICATION_DELAY * 100);
-    config.setInt("remote", "foo2", "replicationDelay", TEST_REPLICATION_DELAY * 100);
+    FileBasedConfig dest1 = setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
+    FileBasedConfig dest2 = setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
+    dest1.setInt("remote", null, "replicationDelay", TEST_REPLICATION_DELAY * 100);
+    dest2.setInt("remote", null, "replicationDelay", TEST_REPLICATION_DELAY * 100);
+    dest1.save();
+    dest2.save();
     reloadConfig();
 
     createChange();
@@ -596,13 +590,14 @@
         remoteName, Arrays.asList(replicaSuffix), project, replicationDelay, mirror);
   }
 
-  private void setReplicationDestination(
+  private FileBasedConfig setReplicationDestination(
       String remoteName, List<String> replicaSuffixes, Optional<String> project)
       throws IOException {
-    setReplicationDestination(remoteName, replicaSuffixes, project, TEST_REPLICATION_DELAY, false);
+    return setReplicationDestination(
+        remoteName, replicaSuffixes, project, TEST_REPLICATION_DELAY, false);
   }
 
-  private void setReplicationDestination(
+  private FileBasedConfig setReplicationDestination(
       String remoteName,
       List<String> replicaSuffixes,
       Optional<String> project,
@@ -620,6 +615,7 @@
     project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj));
     config.setBoolean("gerrit", null, "autoReload", true);
     config.save();
+    return config;
   }
 
   private void setProjectDeletionReplication(String remoteName, boolean replicateProjectDeletion)