Merge "Update gitignore" into stable-3.5
diff --git a/example-setup/broker/Dockerfile b/example-setup/broker/Dockerfile
index abc12cf..1b3038c 100644
--- a/example-setup/broker/Dockerfile
+++ b/example-setup/broker/Dockerfile
@@ -1,4 +1,4 @@
-FROM gerritcodereview/gerrit:3.5.4-almalinux8
+FROM gerritcodereview/gerrit:3.5.5-almalinux8
USER root
diff --git a/example-setup/broker/configs/replication.config.template b/example-setup/broker/configs/replication.config.template
index d81662f..d464586 100644
--- a/example-setup/broker/configs/replication.config.template
+++ b/example-setup/broker/configs/replication.config.template
@@ -1,6 +1,6 @@
[gerrit]
autoReload = true
- replicateOnStartup = false
+ replicateOnStartup = $REPLICATE_ON_STARTUP
[replication]
excludeRefs = ^refs/users/\\d\\d/\\d+/edit-\\d+/\\d+$
lockErrorMaxRetries = 5
diff --git a/example-setup/broker/docker-compose.yaml b/example-setup/broker/docker-compose.yaml
index 4f7f99f..705aea6 100644
--- a/example-setup/broker/docker-compose.yaml
+++ b/example-setup/broker/docker-compose.yaml
@@ -10,6 +10,7 @@
- DEBUG_PORT=5005
- BROKER_HOST=broker
- BROKER_PORT=9092
+ - REPLICATE_ON_STARTUP=false
ports:
- "8080:8080"
- "29418:29418"
@@ -26,6 +27,7 @@
- DEBUG_PORT=5006
- BROKER_HOST=broker
- BROKER_PORT=9092
+ - REPLICATE_ON_STARTUP=true
ports:
- "8081:8080"
- "29419:29418"
diff --git a/example-setup/broker/entrypoint.sh b/example-setup/broker/entrypoint.sh
index 95a750d..1d958e4 100755
--- a/example-setup/broker/entrypoint.sh
+++ b/example-setup/broker/entrypoint.sh
@@ -13,14 +13,14 @@
cat /var/gerrit/etc/gerrit.config.template | envsubst > /var/gerrit/etc/gerrit.config
}
-setup_replication_config
-setup_gerrit_config
-
ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
echo "Init phase ..."
java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit
+setup_replication_config
+setup_gerrit_config
+
echo "Reindexing phase ..."
java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
diff --git a/example-setup/http/Dockerfile b/example-setup/http/Dockerfile
index 5f0d660..552a114 100644
--- a/example-setup/http/Dockerfile
+++ b/example-setup/http/Dockerfile
@@ -1,4 +1,4 @@
-FROM gerritcodereview/gerrit:3.5.4-almalinux8
+FROM gerritcodereview/gerrit:3.5.5-almalinux8
USER root
diff --git a/example-setup/http/configs/replication.config.template b/example-setup/http/configs/replication.config.template
index 7e81055..6768146 100644
--- a/example-setup/http/configs/replication.config.template
+++ b/example-setup/http/configs/replication.config.template
@@ -1,6 +1,6 @@
[gerrit]
autoReload = true
- replicateOnStartup = false
+ replicateOnStartup = $REPLICATE_ON_STARTUP
[replication]
excludeRefs = ^refs/users/\\d\\d/\\d+/edit-\\d+/\\d+$
lockErrorMaxRetries = 5
diff --git a/example-setup/http/docker-compose.yaml b/example-setup/http/docker-compose.yaml
index af90341..ccb6b86 100644
--- a/example-setup/http/docker-compose.yaml
+++ b/example-setup/http/docker-compose.yaml
@@ -8,6 +8,7 @@
- REMOTE=replica-1
- REMOTE_URL=gerrit2
- DEBUG_PORT=5005
+ - REPLICATE_ON_STARTUP=false
ports:
- "8080:8080"
- "29418:29418"
@@ -20,6 +21,7 @@
- REMOTE=primary
- REMOTE_URL=gerrit1
- DEBUG_PORT=5006
+ - REPLICATE_ON_STARTUP=true
ports:
- "8081:8080"
- "29419:29418"
diff --git a/example-setup/http/entrypoint.sh b/example-setup/http/entrypoint.sh
index 95a750d..1d958e4 100755
--- a/example-setup/http/entrypoint.sh
+++ b/example-setup/http/entrypoint.sh
@@ -13,14 +13,14 @@
cat /var/gerrit/etc/gerrit.config.template | envsubst > /var/gerrit/etc/gerrit.config
}
-setup_replication_config
-setup_gerrit_config
-
ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
echo "Init phase ..."
java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit
+setup_replication_config
+setup_gerrit_config
+
echo "Reindexing phase ..."
java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
index 78745bb..d41dd8f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication.pull;
import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.metrics.Counter0;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.MetricMaker;
@@ -28,6 +29,8 @@
private final Timer1<String> executionTime;
private final Timer1<String> end2EndTime;
+ private final Counter0 maxApiPayloadSizeReachedCounter;
+
@Inject
ApplyObjectMetrics(@PluginName String pluginName, MetricMaker metricMaker) {
Field<String> field =
@@ -53,6 +56,13 @@
.setCumulative()
.setUnit(Description.Units.MILLISECONDS),
field);
+ maxApiPayloadSizeReachedCounter =
+ metricMaker.newCounter(
+ "apply_object_max_api_payload_reached",
+ new Description(
+ "Number of apply object operation with payload larger than maxApiPayloadSize")
+ .setRate()
+ .setUnit("errors"));
}
/**
@@ -74,4 +84,9 @@
public Timer1.Context<String> startEnd2End(String name) {
return end2EndTime.start(name);
}
+
+ /** Increment metric when ref size is larger than maxApiPayloadSize. */
+ public void incrementMaxPayloadSizeReached() {
+ maxApiPayloadSizeReachedCounter.increment();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
index d8da65e..43db907 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -35,6 +35,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
import java.util.Collection;
@@ -91,6 +92,7 @@
private final int maxLockRetries;
private int lockRetryCount;
private final int id;
+ private String taskIdHex;
private final long createdAt;
private final FetchReplicationMetrics metrics;
private final AtomicBoolean canceledWhileRunning;
@@ -119,6 +121,7 @@
lockRetryCount = 0;
maxLockRetries = pool.getLockErrorMaxRetries();
id = ig.next();
+ taskIdHex = HexFormat.fromInt(id);
stateLog = sl;
createdAt = System.nanoTime();
metrics = m;
@@ -158,7 +161,7 @@
@Override
public String toString() {
- String print = "[" + HexFormat.fromInt(id) + "] fetch " + uri;
+ String print = "[" + taskIdHex + "] fetch " + uri;
if (retryCount > 0) {
print = "(retry " + retryCount + ") " + print;
@@ -288,13 +291,19 @@
if (!pool.requestRunway(this)) {
if (!canceled) {
repLog.info(
- "Rescheduling replication to {} to avoid collision with an in-flight fetch.", uri);
+ "Rescheduling [{}] replication to {} to avoid collision with an in-flight fetch.",
+ taskIdHex,
+ uri);
pool.reschedule(this, Source.RetryReason.COLLISION);
}
return;
}
- repLog.info("Replication from {} started...", uri);
+ repLog.info(
+ "Replication [{}] from {} started for refs [{}] ...",
+ taskIdHex,
+ uri,
+ String.join(",", getRefs()));
Timer1.Context<String> context = metrics.start(config.getName());
try {
long startedAt = context.getStartTime();
@@ -308,7 +317,8 @@
.flatMap(metrics -> metrics.stop(config.getName()))
.map(NANOSECONDS::toMillis);
repLog.info(
- "Replication from {} completed in {}ms, {}ms delay, {} retries{}",
+ "Replication [{}] from {} completed in {}ms, {}ms delay, {} retries{}",
+ taskIdHex,
uri,
elapsed,
delay,
@@ -324,16 +334,19 @@
// does not exist. In this case NoRemoteRepositoryException is not
// raised.
String msg = e.getMessage();
- repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
+ repLog.error(
+ "Cannot replicate [{}] {}; Remote repository error: {}", taskIdHex, projectName, msg);
} catch (NotSupportedException e) {
stateLog.error("Cannot replicate from " + uri, e, getStatesAsArray());
+ } catch (PermanentTransportException e) {
+ repLog.error(
+ String.format("Terminal failure. Cannot replicate [%s] from %s", taskIdHex, uri), e);
} catch (TransportException e) {
- Throwable cause = e.getCause();
if (e instanceof LockFailureException) {
lockRetryCount++;
// The LockFailureException message contains both URI and reason
// for this failure.
- repLog.error("Cannot replicate from {}: {}", uri, e.getMessage());
+ repLog.error("Cannot replicate [{}] from {}: {}", taskIdHex, uri, e.getMessage());
// The remote fetch operation should be retried.
if (lockRetryCount <= maxLockRetries) {
@@ -344,16 +357,17 @@
}
} else {
repLog.error(
- "Giving up after {} occurrences of this error: {} during replication from {}",
+ "Giving up after {} occurrences of this error: {} during replication from [{}] {}",
lockRetryCount,
e.getMessage(),
+ taskIdHex,
uri);
}
} else {
if (canceledWhileRunning.get()) {
logCanceledWhileRunningException(e);
} else {
- repLog.error("Cannot replicate from {}", uri, e);
+ repLog.error("Cannot replicate [{}] from {}", taskIdHex, uri, e);
// The remote fetch operation should be retried.
pool.reschedule(this, Source.RetryReason.TRANSPORT_ERROR);
}
@@ -371,7 +385,7 @@
}
private void logCanceledWhileRunningException(TransportException e) {
- repLog.info("Cannot replicate from {}. It was canceled while running", uri, e);
+ repLog.info("Cannot replicate [{}] from {}. It was canceled while running", taskIdHex, uri, e);
}
private void runImpl() throws IOException {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
index db46b23..cd6a0ea 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -55,9 +55,13 @@
private GitRepositoryManager gitRepositoryManager;
private Long maxRefSize;
private final int maxDepth;
+ private ApplyObjectMetrics metrics;
@Inject
- public RevisionReader(GitRepositoryManager gitRepositoryManager, ReplicationConfig cfg) {
+ public RevisionReader(
+ GitRepositoryManager gitRepositoryManager,
+ ReplicationConfig cfg,
+ ApplyObjectMetrics metrics) {
this.gitRepositoryManager = gitRepositoryManager;
this.maxRefSize =
cfg.getConfig()
@@ -65,6 +69,7 @@
this.maxDepth =
cfg.getConfig()
.getInt("replication", CONFIG_MAX_API_HISTORY_DEPTH, DEFAULT_MAX_API_HISTORY_DEPTH);
+ this.metrics = metrics;
}
public Optional<RevisionData> read(
@@ -146,6 +151,7 @@
return Optional.of(new RevisionData(parentObjectIds, commitRev, treeRev, blobs));
} catch (LargeObjectException e) {
+ metrics.incrementMaxPayloadSizeReached();
repLog.trace(
"Ref {} size for project {} is greater than configured '{}'",
refName,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index 3170eb5..3d9f5c3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -623,6 +623,10 @@
return configSettingsAllowReplication(project);
}
+ public boolean wouldCreateProject(Project.NameKey project) {
+ return configSettingsAllowReplication(project);
+ }
+
private boolean configSettingsAllowReplication(Project.NameKey project) {
// by default fetch all projects
List<String> projects = config.getProjects();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
index d86287a..f897012 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -25,7 +25,6 @@
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gerrit.server.permissions.RefPermission;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.Inject;
@@ -41,6 +40,7 @@
import java.io.IOException;
import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.URIish;
@@ -83,6 +83,12 @@
throw new ResourceNotFoundException(String.format("Project %s was not found", name));
}
+ Optional<Ref> ref = getRef(name, refName);
+ if (!ref.isPresent()) {
+ logger.atFine().log("Ref %s was not found in project %s", refName, name);
+ return;
+ }
+
Source source =
sourcesCollection
.getByRemoteName(sourceLabel)
@@ -93,15 +99,9 @@
URIish sourceUri = source.getURI(name);
try {
- projectState.get().checkStatePermitsWrite();
- permissionBackend
- .currentUser()
- .project(projectState.get().getNameKey())
- .ref(refName)
- .check(RefPermission.DELETE);
Context.setLocalEvent(true);
- deleteRef(name, refName);
+ deleteRef(name, ref.get());
eventDispatcher
.get()
@@ -145,17 +145,24 @@
}
}
- private RefUpdateState deleteRef(Project.NameKey name, String refName) throws IOException {
+ private Optional<Ref> getRef(Project.NameKey repo, String refName) throws IOException {
+ try (Repository repository = gitManager.openRepository(repo)) {
+ Ref ref = repository.exactRef(refName);
+ return Optional.ofNullable(ref);
+ }
+ }
+ private RefUpdateState deleteRef(Project.NameKey name, Ref ref) throws IOException {
try (Repository repository = gitManager.openRepository(name)) {
+
RefUpdate.Result result;
- RefUpdate u = repository.updateRef(refName);
- u.setExpectedOldObjectId(repository.exactRef(refName).getObjectId());
+ RefUpdate u = repository.updateRef(ref.getName());
+ u.setExpectedOldObjectId(ref.getObjectId());
u.setNewObjectId(ObjectId.zeroId());
u.setForceUpdate(true);
result = u.delete();
- return new RefUpdateState(refName, result);
+ return new RefUpdateState(ref.getName(), result);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
index 2214fb3..8711379 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -22,6 +22,7 @@
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.index.project.ProjectIndexer;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
@@ -49,15 +50,18 @@
private final GerritConfigOps gerritConfigOps;
private final Provider<CurrentUser> userProvider;
private final PermissionBackend permissionBackend;
+ private final ProjectIndexer projectIndexer;
@Inject
ProjectInitializationAction(
GerritConfigOps gerritConfigOps,
Provider<CurrentUser> userProvider,
- PermissionBackend permissionBackend) {
+ PermissionBackend permissionBackend,
+ ProjectIndexer projectIndexer) {
this.gerritConfigOps = gerritConfigOps;
this.userProvider = userProvider;
this.permissionBackend = permissionBackend;
+ this.projectIndexer = projectIndexer;
}
@Override
@@ -106,6 +110,10 @@
}
LocalFS localFS = new LocalFS(maybeUri.get());
Project.NameKey projectNameKey = Project.NameKey.parse(projectName);
- return localFS.createProject(projectNameKey, RefNames.HEAD);
+ if (localFS.createProject(projectNameKey, RefNames.HEAD)) {
+ projectIndexer.index(projectNameKey);
+ return true;
+ }
+ return false;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index c1ffa44..2ea8a33 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -22,45 +22,62 @@
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.config.GerritInstanceId;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventListener;
import com.google.gerrit.server.events.ProjectCreatedEvent;
+import com.google.gerrit.server.events.ProjectEvent;
import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob.Factory;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
public class StreamEventListener implements EventListener {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final String ZERO_ID_NAME = ObjectId.zeroId().name();
- private String instanceId;
- private WorkQueue workQueue;
- private ProjectInitializationAction projectInitializationAction;
-
- private Factory fetchJobFactory;
+ private final DeleteRefCommand deleteCommand;
+ private final ExcludedRefsFilter refsFilter;
+ private final Factory fetchJobFactory;
+ private final ProjectInitializationAction projectInitializationAction;
private final Provider<PullReplicationApiRequestMetrics> metricsProvider;
+ private final SourcesCollection sources;
+ private final String instanceId;
+ private final WorkQueue workQueue;
@Inject
public StreamEventListener(
@Nullable @GerritInstanceId String instanceId,
+ DeleteRefCommand deleteCommand,
ProjectInitializationAction projectInitializationAction,
WorkQueue workQueue,
FetchJob.Factory fetchJobFactory,
- Provider<PullReplicationApiRequestMetrics> metricsProvider) {
+ Provider<PullReplicationApiRequestMetrics> metricsProvider,
+ SourcesCollection sources,
+ ExcludedRefsFilter excludedRefsFilter) {
this.instanceId = instanceId;
+ this.deleteCommand = deleteCommand;
this.projectInitializationAction = projectInitializationAction;
this.workQueue = workQueue;
this.fetchJobFactory = fetchJobFactory;
this.metricsProvider = metricsProvider;
+ this.sources = sources;
+ this.refsFilter = excludedRefsFilter;
requireNonNull(
Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty");
@@ -79,40 +96,101 @@
}
public void fetchRefsForEvent(Event event) throws AuthException, PermissionBackendException {
- if (!instanceId.equals(event.instanceId)) {
- PullReplicationApiRequestMetrics metrics = metricsProvider.get();
- metrics.start(event);
- if (event instanceof RefUpdatedEvent) {
- RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
- if (!isProjectDelete(refUpdatedEvent)) {
- fetchRefsAsync(
- refUpdatedEvent.getRefName(),
- refUpdatedEvent.instanceId,
- refUpdatedEvent.getProjectNameKey(),
- metrics);
- }
+ if (instanceId.equals(event.instanceId) || !shouldReplicateProject(event)) {
+ return;
+ }
+
+ PullReplicationApiRequestMetrics metrics = metricsProvider.get();
+ metrics.start(event);
+ if (event instanceof RefUpdatedEvent) {
+ RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
+ if (!isRefToBeReplicated(refUpdatedEvent.getRefName())) {
+ logger.atFine().log(
+ "Skipping excluded ref '%s' for project '%s'",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ return;
}
- if (event instanceof ProjectCreatedEvent) {
- ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
- try {
- projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
- fetchRefsAsync(
- FetchOne.ALL_REFS,
- projectCreatedEvent.instanceId,
- projectCreatedEvent.getProjectNameKey(),
- metrics);
- } catch (AuthException | PermissionBackendException e) {
- logger.atSevere().withCause(e).log(
- "Cannot initialise project:%s", projectCreatedEvent.projectName);
- throw e;
- }
+
+ if (isProjectDelete(refUpdatedEvent)) {
+ return;
+ }
+
+ if (isRefDelete(refUpdatedEvent)) {
+ deleteRef(refUpdatedEvent);
+ return;
+ }
+
+ fetchRefsAsync(
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId,
+ refUpdatedEvent.getProjectNameKey(),
+ metrics);
+ } else if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+ try {
+ projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
+ fetchRefsAsync(
+ FetchOne.ALL_REFS,
+ projectCreatedEvent.instanceId,
+ projectCreatedEvent.getProjectNameKey(),
+ metrics);
+ } catch (AuthException | PermissionBackendException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot initialise project:%s", projectCreatedEvent.projectName);
+ throw e;
}
}
}
+ private void deleteRef(RefUpdatedEvent refUpdatedEvent) {
+ try {
+ deleteCommand.deleteRef(
+ refUpdatedEvent.getProjectNameKey(),
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId);
+ } catch (IOException | RestApiException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot delete ref %s project:%s",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private boolean isRefToBeReplicated(String refName) {
+ return !refsFilter.match(refName);
+ }
+
+ private boolean shouldReplicateProject(Event event) {
+ if (!(event instanceof ProjectEvent)) {
+ return false;
+ }
+
+ Optional<Source> maybeSource =
+ sources.getAll().stream()
+ .filter(s -> s.getRemoteConfigName().equals(event.instanceId))
+ .findFirst();
+
+ if (!maybeSource.isPresent()) {
+ return false;
+ }
+
+ Source source = maybeSource.get();
+ if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+
+ return source.isCreateMissingRepositories()
+ && source.wouldCreateProject(projectCreatedEvent.getProjectNameKey());
+ }
+
+ ProjectEvent projectEvent = (ProjectEvent) event;
+ return source.wouldFetchProject(projectEvent.getProjectNameKey());
+ }
+
+ private boolean isRefDelete(RefUpdatedEvent event) {
+ return ZERO_ID_NAME.equals(event.refUpdate.get().newRev);
+ }
+
private boolean isProjectDelete(RefUpdatedEvent event) {
- return RefNames.isConfigRef(event.getRefName())
- && ObjectId.zeroId().equals(ObjectId.fromString(event.refUpdate.get().newRev));
+ return RefNames.isConfigRef(event.getRefName()) && isRefDelete(event);
}
protected void fetchRefsAsync(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
index 89972cf..74ad9fc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
+import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.*;
@@ -51,6 +52,13 @@
private FetchResult fetchVia(Transport tn, List<RefSpec> fetchRefSpecs) throws IOException {
repLog.info("Fetch references {} from {}", fetchRefSpecs, uri);
- return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ try {
+ return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ } catch (TransportException e) {
+ if (PermanentTransportException.isPermanentFailure(e)) {
+ throw new PermanentTransportException("Terminal fetch failure", e);
+ }
+ throw e;
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
new file mode 100644
index 0000000..1d96a02
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.fetch;
+
+import com.jcraft.jsch.JSchException;
+import org.eclipse.jgit.errors.TransportException;
+import org.eclipse.jgit.internal.JGitText;
+
+public class PermanentTransportException extends TransportException {
+ private static final long serialVersionUID = 1L;
+
+ public PermanentTransportException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public static boolean isPermanentFailure(TransportException e) {
+ Throwable cause = e.getCause();
+ String message = e.getMessage();
+ return (cause instanceof JSchException && cause.getMessage().startsWith("UnknownHostKey:"))
+ || message.matches(JGitText.get().remoteDoesNotHaveSpec.replaceAll("\\{0\\}", ".+"));
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 51b10d8..b76585e 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -119,6 +119,18 @@
By default, fetches are retried indefinitely.
+ Note that only transient errors will be retried, whilst persistent errors will
+ cause a terminal failure, and the fetch will not be scheduled again. This is
+ only supported for JGit, not cGit. Currently, only the following failures are
+ considered permanent:
+
+ - UnknownHostKey: thrown by Jsch when establishing an SSH connection for an
+ unknown host.
+ - Jgit transport exception when the remote ref does not exist. The assumption
+ here is that the remote ref does not exist so it is not worth retrying. If the
+ exception arisen as a consequence of some ACLs (mis)configuration, then after
+ fixing the ACLs, an explicit replication must be manually triggered.
+
replication.instanceLabel
: Remote configuration name of the current server.
This label is passed as a part of the payload to notify other
@@ -380,6 +392,9 @@
By default, use replication.maxRetries.
+ Note that not all fetch failures are retriable. Please refer
+ to `replication.maxRetries` for more information on this.
+
remote.NAME.threads
: Number of worker threads to dedicate to fetching to the
repositories described by this remote. Each thread can fetch
@@ -558,6 +573,14 @@
remote.NAME.password
: Password to use for HTTP authentication on this remote.
+In both cases, the Global Capability `Access Database` [1] needs to be allowed
+in order to permit all `All-Users`' refs to be replicated. When _basic auth_ is
+used, the capability must be assigned to the `remote.NAME.username` used in
+configuration, whilst for _bearer token_, it needs to be assigned to
+the `Pull-replication Internal User` user.
+
+[1] https://gerrit-review.googlesource.com/Documentation/access-control.html#capability_accessDatabase
+
File `~/.ssh/config`
--------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
index 406cd8c..3f40848 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
@@ -22,18 +22,12 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.SkipProjectClone;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
-import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.projects.BranchInput;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.inject.Inject;
import com.google.inject.Scopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
@@ -46,12 +40,8 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.io.IOException;
import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.time.Duration;
import java.util.List;
-import java.util.function.Supplier;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Ref;
@@ -68,27 +58,8 @@
@TestPlugin(
name = "pull-replication",
sysModule = "com.googlesource.gerrit.plugins.replication.pull.CGitFetchIT$TestModule")
-public class CGitFetchIT extends LightweightPluginDaemonTest {
+public class CGitFetchIT extends FetchITBase {
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final int TEST_REPLICATION_DELAY = 60;
- private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
-
- @Inject private SitePaths sitePaths;
- @Inject private ProjectOperations projectOperations;
- private FetchFactory fetchFactory;
- private Path gitPath;
- private Path testRepoPath;
-
- @Override
- public void setUpTestPlugin() throws Exception {
- gitPath = sitePaths.site_path.resolve("git");
- testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
-
- super.setUpTestPlugin();
- fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
- }
@Test
public void shouldFetchRef() throws Exception {
@@ -245,27 +216,6 @@
}
}
- private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
- WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
- }
-
- private Ref getRef(Repository repo, String branchName) throws IOException {
- return repo.getRefDatabase().exactRef(branchName);
- }
-
- private Ref checkedGetRef(Repository repo, String branchName) {
- try {
- return repo.getRefDatabase().exactRef(branchName);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
- return null;
- }
- }
-
- private Project.NameKey createTestProject(String name) throws Exception {
- return projectOperations.newProject().name(name).create();
- }
-
@SuppressWarnings("unused")
private static class TestModule extends FactoryModule {
@Override
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
new file mode 100644
index 0000000..2a11717
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
@@ -0,0 +1,73 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.function.Supplier;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+
+public abstract class FetchITBase extends LightweightPluginDaemonTest {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private static final int TEST_REPLICATION_DELAY = 60;
+ private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ FetchFactory fetchFactory;
+ private Path gitPath;
+ Path testRepoPath;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+ testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
+
+ super.setUpTestPlugin();
+ fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
+ }
+
+ void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
+ WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
+ }
+
+ Ref getRef(Repository repo, String branchName) throws IOException {
+ return repo.getRefDatabase().exactRef(branchName);
+ }
+
+ Ref checkedGetRef(Repository repo, String branchName) {
+ try {
+ return repo.getRefDatabase().exactRef(branchName);
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
+ return null;
+ }
+ }
+
+ Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
new file mode 100644
index 0000000..ec695a6
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
@@ -0,0 +1,85 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Scopes;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
+import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import java.net.URISyntaxException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.JGitFetchIT$TestModule")
+public class JGitFetchIT extends FetchITBase {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+
+ @Test(expected = PermanentTransportException.class)
+ public void shouldThrowPermanentTransportExceptionWhenRefDoesNotExists() throws Exception {
+
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String nonExistingRef = "refs/changes/02/20000/1:refs/changes/02/20000/1";
+ try (Repository repo = repoManager.openRepository(project)) {
+ Fetch objectUnderTest = fetchFactory.create(new URIish(testRepoPath.toString()), repo);
+ objectUnderTest.fetch(Lists.newArrayList(new RefSpec(nonExistingRef)));
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class TestModule extends FactoryModule {
+ @Override
+ protected void configure() {
+ Config cf = new Config();
+ cf.setInt("remote", "test_config", "timeout", 0);
+ try {
+ RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config");
+ SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf);
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(CredentialsFactory.class)
+ .to(AutoReloadSecureCredentialsFactoryDecorator.class)
+ .in(Scopes.SINGLETON);
+
+ bind(SourceConfiguration.class).toInstance(sourceConfig);
+ install(
+ new FactoryModuleBuilder()
+ .implement(Fetch.class, JGitFetch.class)
+ .implement(Fetch.class, FetchClientImplementation.class, JGitFetch.class)
+ .build(FetchFactory.class));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
new file mode 100644
index 0000000..fb4fb04
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
@@ -0,0 +1,42 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import com.jcraft.jsch.JSchException;
+import org.eclipse.jgit.errors.TransportException;
+import org.junit.Test;
+
+public class PermanentFailureExceptionTest {
+
+ @Test
+ public void shouldConsiderSchUnknownHostAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException(
+ "SSH error", new JSchException("UnknownHostKey: some.place"))))
+ .isTrue();
+ }
+
+ @Test
+ public void shouldConsiderNotExistingRefsAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException("Remote does not have refs/heads/foo available for fetch.")))
+ .isTrue();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
index 2adf72f..f1f9e44 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -17,6 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -57,6 +58,8 @@
public class DeleteRefCommandTest {
private static final String TEST_SOURCE_LABEL = "test-source-label";
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
+
+ private static final String NON_EXISTING_REF_NAME = "refs/changes/01/11101/1";
private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project");
private static URIish TEST_REMOTE_URI;
@@ -88,9 +91,6 @@
when(sourceCollection.getByRemoteName(TEST_SOURCE_LABEL)).thenReturn(Optional.of(source));
TEST_REMOTE_URI = new URIish("git://some.remote.uri");
when(source.getURI(TEST_PROJECT_NAME)).thenReturn(TEST_REMOTE_URI);
- when(permissionBackend.currentUser()).thenReturn(currentUser);
- when(currentUser.project(any())).thenReturn(forProject);
- when(forProject.ref(any())).thenReturn(forRef);
when(gitManager.openRepository(any())).thenReturn(repository);
when(repository.updateRef(any())).thenReturn(refUpdate);
when(repository.getRefDatabase()).thenReturn(refDb);
@@ -119,4 +119,13 @@
assertThat(fetchEvent.getProjectNameKey()).isEqualTo(TEST_PROJECT_NAME);
assertThat(fetchEvent.getRefName()).isEqualTo(TEST_REF_NAME);
}
+
+ @Test
+ public void shouldHandleNonExistingRef() throws Exception {
+ when(refDb.exactRef(anyString())).thenReturn(null);
+
+ objectUnderTest.deleteRef(TEST_PROJECT_NAME, NON_EXISTING_REF_NAME, TEST_SOURCE_LABEL);
+
+ verify(eventDispatcher, never()).postEvent(any());
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
index 77f05a1..47037d6 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
@@ -31,7 +31,6 @@
import org.junit.Test;
public class ProjectInitializationActionIT extends ActionITBase {
- public static final String INVALID_TEST_PROJECT_NAME = "\0";
@Inject private ProjectOperations projectOperations;
@Test
@@ -150,18 +149,6 @@
@Test
@GerritConfig(name = "container.replica", value = "true")
- public void shouldReturnBadRequestIfProjectNameIsInvalidAndCannotBeCreatedWhenNodeIsAReplica()
- throws Exception {
- url = getURLWithAuthenticationPrefix(INVALID_TEST_PROJECT_NAME);
- httpClientFactory
- .create(source)
- .execute(
- withBasicAuthenticationAsAdmin(createPutRequestWithHeaders()),
- assertHttpResponseCode(HttpServletResponse.SC_BAD_REQUEST));
- }
-
- @Test
- @GerritConfig(name = "container.replica", value = "true")
public void shouldReturnBadRequestIfContentNotSetWhenNodeIsAReplica() throws Exception {
httpClientFactory
.create(source)
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
index c673011..92314a4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -21,9 +21,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Lists;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.data.RefUpdateAttribute;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.ProjectCreatedEvent;
@@ -31,10 +33,15 @@
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
@@ -51,6 +58,7 @@
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
private static final String TEST_PROJECT = "test-project";
private static final String INSTANCE_ID = "node_instance_id";
+ private static final String NEW_REV = "0000000000000000000000000000000000000001";
private static final String REMOTE_INSTANCE_ID = "remote_node_instance_id";
@Mock private ProjectInitializationAction projectInitializationAction;
@@ -58,8 +66,12 @@
@Mock private ScheduledExecutorService executor;
@Mock private FetchJob fetchJob;
@Mock private FetchJob.Factory fetchJobFactory;
+ @Mock private DeleteRefCommand deleteRefCommand;
@Captor ArgumentCaptor<Input> inputCaptor;
@Mock private PullReplicationApiRequestMetrics metrics;
+ @Mock private SourcesCollection sources;
+ @Mock private Source source;
+ @Mock private ExcludedRefsFilter refsFilter;
private StreamEventListener objectUnderTest;
@@ -68,9 +80,22 @@
when(workQueue.getDefaultQueue()).thenReturn(executor);
when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any(), any()))
.thenReturn(fetchJob);
+ when(sources.getAll()).thenReturn(Lists.newArrayList(source));
+ when(source.wouldFetchProject(any())).thenReturn(true);
+ when(source.wouldCreateProject(any())).thenReturn(true);
+ when(source.isCreateMissingRepositories()).thenReturn(true);
+ when(source.getRemoteConfigName()).thenReturn(REMOTE_INSTANCE_ID);
+ when(refsFilter.match(any())).thenReturn(false);
objectUnderTest =
new StreamEventListener(
- INSTANCE_ID, projectInitializationAction, workQueue, fetchJobFactory, () -> metrics);
+ INSTANCE_ID,
+ deleteRefCommand,
+ projectInitializationAction,
+ workQueue,
+ fetchJobFactory,
+ () -> metrics,
+ sources,
+ refsFilter);
}
@Test
@@ -80,6 +105,7 @@
objectUnderTest.onEvent(event);
verify(executor, never()).submit(any(Runnable.class));
+ verify(sources, never()).getAll();
}
@Test
@@ -99,11 +125,49 @@
}
@Test
+ public void shouldSkipEventWhenNotOnAllowedProjectsList() {
+ when(source.wouldFetchProject(any())).thenReturn(false);
+
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
+ public void shouldDeleteRefForRefDeleteEvent() throws IOException, RestApiException {
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.newRev = ObjectId.zeroId().getName();
+ refUpdate.project = TEST_PROJECT;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(deleteRefCommand)
+ .deleteRef(Project.nameKey(TEST_PROJECT), refUpdate.refName, REMOTE_INSTANCE_ID);
+ }
+
+ @Test
public void shouldScheduleFetchJobForRefUpdateEvent() {
RefUpdatedEvent event = new RefUpdatedEvent();
RefUpdateAttribute refUpdate = new RefUpdateAttribute();
refUpdate.refName = TEST_REF_NAME;
refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
event.instanceId = REMOTE_INSTANCE_ID;
event.refUpdate = () -> refUpdate;
@@ -120,6 +184,24 @@
}
@Test
+ public void shouldSkipRefUpdateEventForExcludedRef() {
+ when(refsFilter.match(any())).thenReturn(true);
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
public void shouldCreateProjectForProjectCreatedEvent()
throws AuthException, PermissionBackendException {
ProjectCreatedEvent event = new ProjectCreatedEvent();
@@ -132,6 +214,34 @@
}
@Test
+ public void shouldNotCreateProjectWhenCreateMissingRepositoriesNotSet()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
+ public void shouldNotCreateProjectWhenReplicationNotAllowed()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
public void shouldScheduleAllRefsFetchForProjectCreatedEvent() {
ProjectCreatedEvent event = new ProjectCreatedEvent();
event.instanceId = REMOTE_INSTANCE_ID;
