Filter out non existing refs from fetch
Exclude ref from fetch task when ref is not present in the local
repository or has been removed in the global-refdb and therefore is
set to zeros.
P.S. The situation where the entry is not on the global-refdb is
in doubt, because we don't really know if fetching the refs would
succeed or not. The best approach is to fetch anyway and leave the
Git protocol to decide. That allows the fetching from instances when
the global-refdb has been initialised or for new repos that have
been imported but are not yet fully tracked on the global-refdb.
Furthermore, the global-refdb tracks only the mutable ref so
we cannot assume that everything is tracked.
Bug: Issue 16824
Change-Id: I06a3bb73a805046ec5a8bc582586e7db4d213171
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
index e8cfd69..8d101b7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
@@ -20,6 +20,7 @@
import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -37,6 +38,7 @@
@Singleton
public class MultisiteReplicationFetchFilter implements ReplicationFetchFilter {
+ private static final String ZERO_ID_NAME = ObjectId.zeroId().name();
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
public static final int MIN_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS = 1000;
public static final int RANDOM_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS = 1000;
@@ -57,6 +59,7 @@
gitRepositoryManager.openRepository(Project.nameKey(projectName))) {
RefDatabase refDb = repository.getRefDatabase();
return refs.stream()
+ .filter(ref -> !hasBeenRemovedFromGlobalRefDb(projectName, ref))
.filter(
ref -> {
Optional<ObjectId> localRefOid =
@@ -81,6 +84,31 @@
}
}
+ /* If the ref to fetch has been set to all zeros on the global-refdb, it means
+ * that whatever is the situation locally, we do not need to fetch it:
+ * - If the remote still has it, fetching it will be useless because the global
+ * state is that the ref should be removed.
+ * - If the remote doesn't have it anymore, trying to fetch the ref won't do
+ * anything because you can't just remove local refs by fetching.
+ */
+ private boolean hasBeenRemovedFromGlobalRefDb(String projectName, String ref) {
+ if (foundAsZeroInSharedRefDb(Project.nameKey(projectName), ref)) {
+ repLog.info(
+ "{}:{} is found as zeros (removed) in shared-refdb thus will NOT BE fetched",
+ projectName,
+ ref);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean foundAsZeroInSharedRefDb(NameKey projectName, String ref) {
+ return sharedRefDb
+ .get(projectName, ref, String.class)
+ .map(r -> ZERO_ID_NAME.equals(r))
+ .orElse(false);
+ }
+
private Optional<ObjectId> getSha1IfUpToDateWithGlobalRefDb(
Repository repository,
String projectName,
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilterTest.java
index 7abb513..ad7ae3c 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilterTest.java
@@ -27,10 +27,12 @@
import com.google.gerrit.testing.InMemoryTestEnvironment;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import java.util.Optional;
import java.util.Set;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -123,6 +125,85 @@
verify(sharedRefDatabaseMock, times(2)).isUpToDate(any(), any());
}
+ @Test
+ public void shouldNotFilterOutWhenMissingInTheSharedRefDb() throws Exception {
+ String temporaryOutdated = "refs/heads/temporaryOutdated";
+ newRef(temporaryOutdated);
+
+ Set<String> refsToFetch = Set.of(temporaryOutdated);
+
+ MultisiteReplicationFetchFilter fetchFilter =
+ new MultisiteReplicationFetchFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ Set<String> filteredRefsToFetch = fetchFilter.filter(project, refsToFetch);
+
+ assertThat(filteredRefsToFetch).hasSize(1);
+ }
+
+ @Test
+ public void shouldFilterOutWhenRefIsDeletedInTheSharedRefDb() throws Exception {
+ String temporaryOutdated = "refs/heads/temporaryOutdated";
+ newRef(temporaryOutdated);
+
+ Set<String> refsToFetch = Set.of(temporaryOutdated);
+ doReturn(Optional.of(ObjectId.zeroId().getName()))
+ .when(sharedRefDatabaseMock)
+ .get(eq(projectName), any(), any());
+
+ MultisiteReplicationFetchFilter fetchFilter =
+ new MultisiteReplicationFetchFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ Set<String> filteredRefsToFetch = fetchFilter.filter(project, refsToFetch);
+
+ assertThat(filteredRefsToFetch).hasSize(0);
+ verify(sharedRefDatabaseMock).get(eq(projectName), any(), any());
+ }
+
+ @Test
+ public void shouldNotFilterOutWhenRefIsMissingOnlyInTheLocalRepository() throws Exception {
+ String refObjectId = "0000000000000000000000000000000000000001";
+ String nonExistingLocalRef = "refs/heads/temporaryOutdated";
+
+ Set<String> refsToFetch = Set.of(nonExistingLocalRef);
+ doReturn(Optional.of(refObjectId))
+ .when(sharedRefDatabaseMock)
+ .get(eq(projectName), any(), any());
+
+ MultisiteReplicationFetchFilter fetchFilter =
+ new MultisiteReplicationFetchFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ Set<String> filteredRefsToFetch = fetchFilter.filter(project, refsToFetch);
+
+ assertThat(filteredRefsToFetch).hasSize(1);
+ }
+
+ @Test
+ public void shouldNotFilterOutRefThatDoesntExistLocallyOrInSharedRefDb() throws Exception {
+ String nonExisting = "refs/heads/non-existing-ref";
+
+ Set<String> refsToFetch = Set.of(nonExisting);
+
+ MultisiteReplicationFetchFilter fetchFilter =
+ new MultisiteReplicationFetchFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ Set<String> filteredRefsToFetch = fetchFilter.filter(project, refsToFetch);
+
+ assertThat(filteredRefsToFetch).hasSize(1);
+ }
+
+ @Test
+ public void shouldFilterOutRefMissingInTheLocalRepositoryAndDeletedInSharedRefDb()
+ throws Exception {
+ String nonExistingLocalRef = "refs/heads/temporaryOutdated";
+
+ Set<String> refsToFetch = Set.of(nonExistingLocalRef);
+ doReturn(Optional.of(ObjectId.zeroId().getName()))
+ .when(sharedRefDatabaseMock)
+ .get(eq(projectName), any(), any());
+
+ MultisiteReplicationFetchFilter fetchFilter =
+ new MultisiteReplicationFetchFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ Set<String> filteredRefsToFetch = fetchFilter.filter(project, refsToFetch);
+
+ assertThat(filteredRefsToFetch).hasSize(0);
+ }
+
private void newRef(String refName) throws Exception {
repo.branch(refName).commit().create();
}
