Merge branch 'stable-3.6' into stable-3.7
* stable-3.6:
Set the config files after Gerrit init step
Make sure project is indexed after creation
Allow creation of new projects
Introduce concept of permanent failure
Document need for Access Database capability
Do not try to delete inexistent refs
Make variable used by constructor final
Stream events listener should respect excluded refs param
Stream events listener should respect `remote.NAME.projects` param
Remove unnecessary test
Add ref deletion functionality to the StreamEventsListener
Add guard clauses to improve readability
Use Gerrit 3.5.5 in example setup
Return empty list when prefix is not matched in suggest
Add metric to count when ref size is larger that maxApiPayloadSize
Improve fetch logging by adding replication task id and refs list
Change the base docker image in the broker topology.
Update docker http/broker topologies.
Listen to GitBatchRefUpdateListener for a set of events
Pull-replication does not require DELETE refs permissions.
Move Gerrit init and reindexing steps to entrypoint.sh.
Fix init step for pull-replication.
Add init step for pull-replication
Add example using broker notification (Kafka)
Set correct 3.6 broker version
Set correct 3.5 broker version
Give execute permission to entrypoint.sh
Remove unused env variables
Expose debug ports in Gerrit
Do no update Gerrit rpm
Remove /a from fetch url in replication.config.template
Add docker-compose to spinup test environment
Updated Dockerfile to latest Gerrit version (3.7.2)
Change-Id: I55b5c3fb31588f963335e6eeda058a24487be415
diff --git a/BUILD b/BUILD
index e1f435d..f6a6e48 100644
--- a/BUILD
+++ b/BUILD
@@ -9,6 +9,7 @@
"Implementation-URL: https://gerrit-review.googlesource.com/#/admin/projects/plugins/pull-replication",
"Gerrit-PluginName: pull-replication",
"Gerrit-Module: com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ "Gerrit-InitStep: com.googlesource.gerrit.plugins.replication.pull.InitPlugin",
"Gerrit-SshModule: com.googlesource.gerrit.plugins.replication.pull.SshModule",
"Gerrit-HttpModule: com.googlesource.gerrit.plugins.replication.pull.api.HttpModule",
],
diff --git a/example-setup/Dockerfile b/example-setup/broker/Dockerfile
similarity index 61%
rename from example-setup/Dockerfile
rename to example-setup/broker/Dockerfile
index 350aec0..acc34b9 100644
--- a/example-setup/Dockerfile
+++ b/example-setup/broker/Dockerfile
@@ -1,18 +1,17 @@
-FROM gerritcodereview/gerrit:3.7.0-almalinux8
+FROM gerritcodereview/gerrit:3.7.2-almalinux8
USER root
RUN yum install -y gettext
-ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
-
-RUN java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit && \
- java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
-
RUN git config -f /var/gerrit/etc/secure.config --add auth.bearerToken "theSecretBearerToken"
COPY --chown=gerrit:gerrit pull-replication.jar /var/gerrit/plugins/pull-replication.jar
COPY --chown=gerrit:gerrit pull-replication.jar /var/gerrit/lib/pull-replication.jar
+
+COPY --chown=gerrit:gerrit events-kafka.jar /var/gerrit/plugins/events-kafka.jar
+COPY --chown=gerrit:gerrit libevents-broker.jar /var/gerrit/lib/libevents-broker.jar
+
COPY --chown=gerrit:gerrit entrypoint.sh /tmp/
COPY --chown=gerrit:gerrit configs/replication.config.template /var/gerrit/etc/
COPY --chown=gerrit:gerrit configs/gerrit.config.template /var/gerrit/etc/
diff --git a/example-setup/broker/README.md b/example-setup/broker/README.md
new file mode 100644
index 0000000..d50e097
--- /dev/null
+++ b/example-setup/broker/README.md
@@ -0,0 +1,20 @@
+# What is this for?
+
+This docker compose sets up primary and replica nodes using pull-replication to
+replicate with notifications over a broker. In this case our broker
+implementation is Kafka.
+
+Copy the pull-replication, events-kafka, and events-broker artefacts to test
+into this directory:
+
+```bash
+cp $GERRIT_HOME/bazel-bin/plugins/pull-replication/pull-replication.jar .
+cp $GERRIT_HOME/bazel-bin/plugins/events-kafka/events-kafka.jar .
+cp $GERRIT_HOME/bazel-bin/plugins/events-broker/libevents-broker.jar .
+```
+
+Start up the application using docker compose:
+
+```bash
+docker-compose up
+```
diff --git a/example-setup/broker/configs/gerrit.config.template b/example-setup/broker/configs/gerrit.config.template
new file mode 100644
index 0000000..de65519
--- /dev/null
+++ b/example-setup/broker/configs/gerrit.config.template
@@ -0,0 +1,40 @@
+[gerrit]
+ basePath = git
+ serverId = 69ec38f0-350e-4d9c-96d4-bc956f2faaac
+ canonicalWebUrl = http://localhost:8080
+ installModule = com.gerritforge.gerrit.eventbroker.BrokerApiModule
+ instanceId = $INSTANCE_ID
+[container]
+ javaOptions = "-Dflogger.backend_factory=com.google.common.flogger.backend.log4j.Log4jBackendFactory#getInstance"
+ javaOptions = "-Dflogger.logging_context=com.google.gerrit.server.logging.LoggingContext#getInstance"
+ javaOptions = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:$DEBUG_PORT"
+ replica = $REPLICA
+[index]
+ type = LUCENE
+[auth]
+ type = DEVELOPMENT_BECOME_ANY_ACCOUNT
+[receive]
+ enableSignedPush = false
+[sendemail]
+ smtpServer = localhost
+[sshd]
+ listenAddress = *:29418
+ advertisedAddress = *:29418
+[httpd]
+ listenUrl = http://*:8080/
+ requestLog = true
+[cache]
+ directory = cache
+[plugins]
+ allowRemoteAdmin = true
+[plugin "events-kafka"]
+ sendAsync = true
+ bootstrapServers = $BROKER_HOST:$BROKER_PORT
+ groupId = $INSTANCE_ID
+ numberOfSubscribers = 6
+ securityProtocol = PLAINTEXT
+ pollingIntervalMs = 1000
+ enableAutoCommit = true
+ autoCommitIntervalMs = 1000
+ autoOffsetReset = latest
+ sendStreamEvents = true
\ No newline at end of file
diff --git a/example-setup/configs/replication.config.template b/example-setup/broker/configs/replication.config.template
similarity index 77%
copy from example-setup/configs/replication.config.template
copy to example-setup/broker/configs/replication.config.template
index 7e81055..d464586 100644
--- a/example-setup/configs/replication.config.template
+++ b/example-setup/broker/configs/replication.config.template
@@ -1,17 +1,17 @@
[gerrit]
autoReload = true
- replicateOnStartup = false
+ replicateOnStartup = $REPLICATE_ON_STARTUP
[replication]
excludeRefs = ^refs/users/\\d\\d/\\d+/edit-\\d+/\\d+$
lockErrorMaxRetries = 5
maxRetries = 100
useCGitClient = false
consumeStreamEvents = false
- syncRefs="ALL REFS ASYNC"
- maxApiPayloadSize=40000
+ eventBrokerTopic = gerrit
+ syncRefs = "ALL REFS ASYNC"
+ maxApiPayloadSize = 40000
[remote "$REMOTE"]
url = http://$REMOTE_URL:8080/#{name}#.git
- apiUrl = http://$REMOTE_URL:8080
fetch = +refs/*:refs/*
mirror = true
timeout = 60 # In seconds
@@ -22,4 +22,4 @@
createMissingRepositories = true
replicateProjectDeletions = true
replicateHiddenProjects = true
- tagopt= --no-tags
\ No newline at end of file
+ tagopt = --no-tags
\ No newline at end of file
diff --git a/example-setup/broker/docker-compose.yaml b/example-setup/broker/docker-compose.yaml
new file mode 100644
index 0000000..705aea6
--- /dev/null
+++ b/example-setup/broker/docker-compose.yaml
@@ -0,0 +1,60 @@
+version: '3'
+services:
+ gerrit1:
+ build: .
+ environment:
+ - INSTANCE_ID=primary
+ - REPLICA=false
+ - REMOTE=replica-1
+ - REMOTE_URL=gerrit2
+ - DEBUG_PORT=5005
+ - BROKER_HOST=broker
+ - BROKER_PORT=9092
+ - REPLICATE_ON_STARTUP=false
+ ports:
+ - "8080:8080"
+ - "29418:29418"
+ - "5005:5005"
+ depends_on:
+ - broker
+ gerrit2:
+ build: .
+ environment:
+ - INSTANCE_ID=replica-1
+ - REPLICA=true
+ - REMOTE=primary
+ - REMOTE_URL=gerrit1
+ - DEBUG_PORT=5006
+ - BROKER_HOST=broker
+ - BROKER_PORT=9092
+ - REPLICATE_ON_STARTUP=true
+ ports:
+ - "8081:8080"
+ - "29419:29418"
+ - "5006:5006"
+ depends_on:
+ - broker
+ - gerrit1
+
+ zookeeper:
+ image: confluentinc/cp-zookeeper:7.3.0
+ container_name: zookeeper
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 2181
+ ZOOKEEPER_TICK_TIME: 2000
+
+ broker:
+ image: confluentinc/cp-kafka:7.3.0
+ container_name: broker
+ ports:
+ - "9092:9092"
+ depends_on:
+ - zookeeper
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_INTERNAL://broker:29092
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
\ No newline at end of file
diff --git a/example-setup/entrypoint.sh b/example-setup/broker/entrypoint.sh
similarity index 64%
rename from example-setup/entrypoint.sh
rename to example-setup/broker/entrypoint.sh
index 412c487..1d958e4 100755
--- a/example-setup/entrypoint.sh
+++ b/example-setup/broker/entrypoint.sh
@@ -13,8 +13,16 @@
cat /var/gerrit/etc/gerrit.config.template | envsubst > /var/gerrit/etc/gerrit.config
}
+ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
+
+echo "Init phase ..."
+java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit
+
setup_replication_config
setup_gerrit_config
+echo "Reindexing phase ..."
+java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
+
echo "Running Gerrit ..."
exec /var/gerrit/bin/gerrit.sh run
\ No newline at end of file
diff --git a/example-setup/Dockerfile b/example-setup/http/Dockerfile
similarity index 61%
copy from example-setup/Dockerfile
copy to example-setup/http/Dockerfile
index 350aec0..3a68b59 100644
--- a/example-setup/Dockerfile
+++ b/example-setup/http/Dockerfile
@@ -1,14 +1,9 @@
-FROM gerritcodereview/gerrit:3.7.0-almalinux8
+FROM gerritcodereview/gerrit:3.7.2-almalinux8
USER root
RUN yum install -y gettext
-ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
-
-RUN java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit && \
- java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
-
RUN git config -f /var/gerrit/etc/secure.config --add auth.bearerToken "theSecretBearerToken"
COPY --chown=gerrit:gerrit pull-replication.jar /var/gerrit/plugins/pull-replication.jar
diff --git a/example-setup/README.md b/example-setup/http/README.md
similarity index 100%
rename from example-setup/README.md
rename to example-setup/http/README.md
diff --git a/example-setup/configs/gerrit.config.template b/example-setup/http/configs/gerrit.config.template
similarity index 100%
rename from example-setup/configs/gerrit.config.template
rename to example-setup/http/configs/gerrit.config.template
diff --git a/example-setup/configs/replication.config.template b/example-setup/http/configs/replication.config.template
similarity index 93%
rename from example-setup/configs/replication.config.template
rename to example-setup/http/configs/replication.config.template
index 7e81055..6768146 100644
--- a/example-setup/configs/replication.config.template
+++ b/example-setup/http/configs/replication.config.template
@@ -1,6 +1,6 @@
[gerrit]
autoReload = true
- replicateOnStartup = false
+ replicateOnStartup = $REPLICATE_ON_STARTUP
[replication]
excludeRefs = ^refs/users/\\d\\d/\\d+/edit-\\d+/\\d+$
lockErrorMaxRetries = 5
diff --git a/example-setup/docker-compose.yaml b/example-setup/http/docker-compose.yaml
similarity index 88%
rename from example-setup/docker-compose.yaml
rename to example-setup/http/docker-compose.yaml
index af90341..ccb6b86 100644
--- a/example-setup/docker-compose.yaml
+++ b/example-setup/http/docker-compose.yaml
@@ -8,6 +8,7 @@
- REMOTE=replica-1
- REMOTE_URL=gerrit2
- DEBUG_PORT=5005
+ - REPLICATE_ON_STARTUP=false
ports:
- "8080:8080"
- "29418:29418"
@@ -20,6 +21,7 @@
- REMOTE=primary
- REMOTE_URL=gerrit1
- DEBUG_PORT=5006
+ - REPLICATE_ON_STARTUP=true
ports:
- "8081:8080"
- "29419:29418"
diff --git a/example-setup/entrypoint.sh b/example-setup/http/entrypoint.sh
similarity index 64%
copy from example-setup/entrypoint.sh
copy to example-setup/http/entrypoint.sh
index 412c487..1d958e4 100755
--- a/example-setup/entrypoint.sh
+++ b/example-setup/http/entrypoint.sh
@@ -13,8 +13,16 @@
cat /var/gerrit/etc/gerrit.config.template | envsubst > /var/gerrit/etc/gerrit.config
}
+ARG JAVA_OPTS='--add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED'
+
+echo "Init phase ..."
+java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit
+
setup_replication_config
setup_gerrit_config
+echo "Reindexing phase ..."
+java $JAVA_OPTS -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
+
echo "Running Gerrit ..."
exec /var/gerrit/bin/gerrit.sh run
\ No newline at end of file
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index f1f162e..3bf728d 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl
@@ -3,6 +3,6 @@
def external_plugin_deps():
maven_jar(
name = "events-broker",
- artifact = "com.gerritforge:events-broker:3.4.0.4",
- sha1 = "8d361d863382290e33828116e65698190118d0f1",
+ artifact = "com.gerritforge:events-broker:3.6.0-rc3",
+ sha1 = "cb398afa4f76367be5c62b99a7ffce74ae1d3d8b",
)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
index 78745bb..d41dd8f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication.pull;
import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.metrics.Counter0;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.MetricMaker;
@@ -28,6 +29,8 @@
private final Timer1<String> executionTime;
private final Timer1<String> end2EndTime;
+ private final Counter0 maxApiPayloadSizeReachedCounter;
+
@Inject
ApplyObjectMetrics(@PluginName String pluginName, MetricMaker metricMaker) {
Field<String> field =
@@ -53,6 +56,13 @@
.setCumulative()
.setUnit(Description.Units.MILLISECONDS),
field);
+ maxApiPayloadSizeReachedCounter =
+ metricMaker.newCounter(
+ "apply_object_max_api_payload_reached",
+ new Description(
+ "Number of apply object operation with payload larger than maxApiPayloadSize")
+ .setRate()
+ .setUnit("errors"));
}
/**
@@ -74,4 +84,9 @@
public Timer1.Context<String> startEnd2End(String name) {
return end2EndTime.start(name);
}
+
+ /** Increment metric when ref size is larger than maxApiPayloadSize. */
+ public void incrementMaxPayloadSizeReached() {
+ maxApiPayloadSizeReachedCounter.increment();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
index eb543a0..43db907 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -35,6 +35,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
import java.util.Collection;
@@ -91,6 +92,7 @@
private final int maxLockRetries;
private int lockRetryCount;
private final int id;
+ private String taskIdHex;
private final long createdAt;
private final FetchReplicationMetrics metrics;
private final AtomicBoolean canceledWhileRunning;
@@ -119,6 +121,7 @@
lockRetryCount = 0;
maxLockRetries = pool.getLockErrorMaxRetries();
id = ig.next();
+ taskIdHex = HexFormat.fromInt(id);
stateLog = sl;
createdAt = System.nanoTime();
metrics = m;
@@ -158,7 +161,7 @@
@Override
public String toString() {
- String print = "[" + HexFormat.fromInt(id) + "] fetch " + uri;
+ String print = "[" + taskIdHex + "] fetch " + uri;
if (retryCount > 0) {
print = "(retry " + retryCount + ") " + print;
@@ -288,13 +291,19 @@
if (!pool.requestRunway(this)) {
if (!canceled) {
repLog.info(
- "Rescheduling replication to {} to avoid collision with an in-flight fetch.", uri);
+ "Rescheduling [{}] replication to {} to avoid collision with an in-flight fetch.",
+ taskIdHex,
+ uri);
pool.reschedule(this, Source.RetryReason.COLLISION);
}
return;
}
- repLog.info("Replication from {} started...", uri);
+ repLog.info(
+ "Replication [{}] from {} started for refs [{}] ...",
+ taskIdHex,
+ uri,
+ String.join(",", getRefs()));
Timer1.Context<String> context = metrics.start(config.getName());
try {
long startedAt = context.getStartTime();
@@ -308,7 +317,8 @@
.flatMap(metrics -> metrics.stop(config.getName()))
.map(NANOSECONDS::toMillis);
repLog.info(
- "Replication from {} completed in {}ms, {}ms delay, {} retries{}",
+ "Replication [{}] from {} completed in {}ms, {}ms delay, {} retries{}",
+ taskIdHex,
uri,
elapsed,
delay,
@@ -324,15 +334,19 @@
// does not exist. In this case NoRemoteRepositoryException is not
// raised.
String msg = e.getMessage();
- repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
+ repLog.error(
+ "Cannot replicate [{}] {}; Remote repository error: {}", taskIdHex, projectName, msg);
} catch (NotSupportedException e) {
stateLog.error("Cannot replicate from " + uri, e, getStatesAsArray());
+ } catch (PermanentTransportException e) {
+ repLog.error(
+ String.format("Terminal failure. Cannot replicate [%s] from %s", taskIdHex, uri), e);
} catch (TransportException e) {
if (e instanceof LockFailureException) {
lockRetryCount++;
// The LockFailureException message contains both URI and reason
// for this failure.
- repLog.error("Cannot replicate from {}: {}", uri, e.getMessage());
+ repLog.error("Cannot replicate [{}] from {}: {}", taskIdHex, uri, e.getMessage());
// The remote fetch operation should be retried.
if (lockRetryCount <= maxLockRetries) {
@@ -343,16 +357,17 @@
}
} else {
repLog.error(
- "Giving up after {} occurrences of this error: {} during replication from {}",
+ "Giving up after {} occurrences of this error: {} during replication from [{}] {}",
lockRetryCount,
e.getMessage(),
+ taskIdHex,
uri);
}
} else {
if (canceledWhileRunning.get()) {
logCanceledWhileRunningException(e);
} else {
- repLog.error("Cannot replicate from {}", uri, e);
+ repLog.error("Cannot replicate [{}] from {}", taskIdHex, uri, e);
// The remote fetch operation should be retried.
pool.reschedule(this, Source.RetryReason.TRANSPORT_ERROR);
}
@@ -370,7 +385,7 @@
}
private void logCanceledWhileRunningException(TransportException e) {
- repLog.info("Cannot replicate from {}. It was canceled while running", uri, e);
+ repLog.info("Cannot replicate [{}] from {}. It was canceled while running", taskIdHex, uri, e);
}
private void runImpl() throws IOException {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/InitPlugin.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/InitPlugin.java
new file mode 100644
index 0000000..5539451
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/InitPlugin.java
@@ -0,0 +1,102 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.googlesource.gerrit.plugins.replication.pull.auth.PullReplicationGroupBackend.INTERNAL_GROUP_NAME;
+import static com.googlesource.gerrit.plugins.replication.pull.auth.PullReplicationGroupBackend.INTERNAL_GROUP_UUID;
+
+import com.google.gerrit.common.data.GlobalCapability;
+import com.google.gerrit.entities.GroupReference;
+import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.pgm.init.api.AllProjectsConfig;
+import com.google.gerrit.pgm.init.api.ConsoleUI;
+import com.google.gerrit.pgm.init.api.InitStep;
+import com.google.inject.Inject;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.ArrayUtils;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+
+public class InitPlugin implements InitStep {
+
+ private final String pluginName;
+ private final BearerTokenProvider bearerTokenProvider;
+ private final ConsoleUI ui;
+ private final AllProjectsConfig allProjectsConfig;
+
+ private static final String CAPABILITY_SECTION = "capability";
+
+ @Inject
+ InitPlugin(
+ @PluginName String pluginName,
+ BearerTokenProvider bearerTokenProvider,
+ ConsoleUI ui,
+ AllProjectsConfig allProjectsConfig) {
+ this.pluginName = pluginName;
+ this.bearerTokenProvider = bearerTokenProvider;
+ this.ui = ui;
+ this.allProjectsConfig = allProjectsConfig;
+ }
+
+ @Override
+ public void run() {}
+
+ @Override
+ public void postRun() throws Exception {
+ ui.header("%s initialization", pluginName);
+
+ if (!bearerTokenProvider.get().isPresent()) {
+ ui.message(
+ "The %s plugin is not configured to use bearer token. If you are using basic auth, remember to grant the '%s' global capability to all relevant users\n",
+ pluginName, GlobalCapability.ACCESS_DATABASE);
+ return;
+ }
+
+ ui.message(
+ "Granting '%s' global capability to the '%s' user\n",
+ GlobalCapability.ACCESS_DATABASE, INTERNAL_GROUP_NAME);
+
+ upsertCapability();
+
+ ui.message(
+ "'%s' global capability granted to user '%s'\n",
+ GlobalCapability.ACCESS_DATABASE, INTERNAL_GROUP_NAME);
+ }
+
+ private void upsertCapability() throws ConfigInvalidException, IOException {
+ String pullReplicationGroup = "group " + INTERNAL_GROUP_NAME;
+
+ Config cfg = allProjectsConfig.load().getConfig();
+ String[] groupsWithAccessDatabase =
+ cfg.getStringList(CAPABILITY_SECTION, null, GlobalCapability.ACCESS_DATABASE);
+
+ if (Arrays.stream(groupsWithAccessDatabase).noneMatch(pullReplicationGroup::equals)) {
+ cfg.setStringList(
+ CAPABILITY_SECTION,
+ null,
+ GlobalCapability.ACCESS_DATABASE,
+ Arrays.stream(ArrayUtils.add(groupsWithAccessDatabase, pullReplicationGroup))
+ .collect(Collectors.toList()));
+
+ allProjectsConfig
+ .getGroups()
+ .put(
+ INTERNAL_GROUP_UUID, GroupReference.create(INTERNAL_GROUP_UUID, INTERNAL_GROUP_NAME));
+
+ allProjectsConfig.save(pluginName, "Init step");
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
index cd19be4..08a402e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -19,7 +19,7 @@
import com.google.common.eventbus.EventBus;
import com.google.gerrit.extensions.annotations.Exports;
import com.google.gerrit.extensions.config.CapabilityDefinition;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
@@ -123,7 +123,7 @@
.annotatedWith(UniqueAnnotations.create())
.to(ReplicationQueue.class);
- DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class);
+ DynamicSet.bind(binder(), GitBatchRefUpdateListener.class).to(ReplicationQueue.class);
bind(ConfigParser.class).to(SourceConfigParser.class).in(Scopes.SINGLETON);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index f5b27ee..990c82c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -20,7 +20,7 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.entities.RefNames;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
@@ -65,7 +65,7 @@
public class ReplicationQueue
implements ObservableQueue,
LifecycleListener,
- GitReferenceUpdatedListener,
+ GitBatchRefUpdateListener,
ProjectDeletedListener,
HeadUpdatedListener {
@@ -147,17 +147,24 @@
}
@Override
- public void onGitReferenceUpdated(GitReferenceUpdatedListener.Event event) {
- if (isRefToBeReplicated(event.getRefName())) {
- repLog.info(
- "Ref event received: {} on project {}:{} - {} => {}",
- refUpdateType(event),
- event.getProjectName(),
- event.getRefName(),
- event.getOldObjectId(),
- event.getNewObjectId());
- fire(ReferenceUpdatedEvent.from(event));
- }
+ public void onGitBatchRefUpdate(GitBatchRefUpdateListener.Event event) {
+ event.getUpdatedRefs().stream()
+ .sorted(ReplicationQueue::sortByMetaRefAsLast)
+ .forEachOrdered(
+ updateRef -> {
+ String refName = updateRef.getRefName();
+
+ if (isRefToBeReplicated(refName)) {
+ repLog.info(
+ "Ref event received: {} on project {}:{} - {} => {}",
+ refUpdateType(updateRef),
+ event.getProjectName(),
+ refName,
+ updateRef.getOldObjectId(),
+ updateRef.getNewObjectId());
+ fire(ReferenceUpdatedEvent.from(event.getProjectName(), updateRef));
+ }
+ });
}
@Override
@@ -170,11 +177,17 @@
source.getApis().forEach(apiUrl -> source.scheduleDeleteProject(apiUrl, project)));
}
- private static String refUpdateType(GitReferenceUpdatedListener.Event event) {
- String forcedPrefix = event.isNonFastForward() ? "FORCED " : " ";
- if (event.isCreate()) {
+ private static int sortByMetaRefAsLast(UpdatedRef a, @SuppressWarnings("unused") UpdatedRef b) {
+ repLog.info("sortByMetaRefAsLast(" + a.getRefName() + " <=> " + b.getRefName());
+ return Boolean.compare(
+ RefNames.isNoteDbMetaRef(a.getRefName()), RefNames.isNoteDbMetaRef(b.getRefName()));
+ }
+
+ private static String refUpdateType(UpdatedRef updateRef) {
+ String forcedPrefix = updateRef.isNonFastForward() ? "FORCED " : " ";
+ if (updateRef.isCreate()) {
return forcedPrefix + "CREATE";
- } else if (event.isDelete()) {
+ } else if (updateRef.isDelete()) {
return forcedPrefix + "DELETE";
} else {
return forcedPrefix + "UPDATE";
@@ -518,12 +531,12 @@
projectName, refName, objectId, isDelete);
}
- static ReferenceUpdatedEvent from(GitReferenceUpdatedListener.Event event) {
+ static ReferenceUpdatedEvent from(String projectName, UpdatedRef updateRef) {
return ReferenceUpdatedEvent.create(
- event.getProjectName(),
- event.getRefName(),
- ObjectId.fromString(event.getNewObjectId()),
- event.isDelete());
+ projectName,
+ updateRef.getRefName(),
+ ObjectId.fromString(updateRef.getNewObjectId()),
+ updateRef.isDelete());
}
public abstract String projectName();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
index db46b23..cd6a0ea 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -55,9 +55,13 @@
private GitRepositoryManager gitRepositoryManager;
private Long maxRefSize;
private final int maxDepth;
+ private ApplyObjectMetrics metrics;
@Inject
- public RevisionReader(GitRepositoryManager gitRepositoryManager, ReplicationConfig cfg) {
+ public RevisionReader(
+ GitRepositoryManager gitRepositoryManager,
+ ReplicationConfig cfg,
+ ApplyObjectMetrics metrics) {
this.gitRepositoryManager = gitRepositoryManager;
this.maxRefSize =
cfg.getConfig()
@@ -65,6 +69,7 @@
this.maxDepth =
cfg.getConfig()
.getInt("replication", CONFIG_MAX_API_HISTORY_DEPTH, DEFAULT_MAX_API_HISTORY_DEPTH);
+ this.metrics = metrics;
}
public Optional<RevisionData> read(
@@ -146,6 +151,7 @@
return Optional.of(new RevisionData(parentObjectIds, commitRev, treeRev, blobs));
} catch (LargeObjectException e) {
+ metrics.incrementMaxPayloadSizeReached();
repLog.trace(
"Ref {} size for project {} is greater than configured '{}'",
refName,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index 3170eb5..3d9f5c3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -623,6 +623,10 @@
return configSettingsAllowReplication(project);
}
+ public boolean wouldCreateProject(Project.NameKey project) {
+ return configSettingsAllowReplication(project);
+ }
+
private boolean configSettingsAllowReplication(Project.NameKey project) {
// by default fetch all projects
List<String> projects = config.getProjects();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
index fd62910..e49c8b6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -25,7 +25,6 @@
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gerrit.server.permissions.RefPermission;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.Inject;
@@ -40,6 +39,7 @@
import java.io.IOException;
import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.URIish;
@@ -79,6 +79,12 @@
throw new ResourceNotFoundException(String.format("Project %s was not found", name));
}
+ Optional<Ref> ref = getRef(name, refName);
+ if (!ref.isPresent()) {
+ logger.atFine().log("Ref %s was not found in project %s", refName, name);
+ return;
+ }
+
Source source =
sourcesCollection
.getByRemoteName(sourceLabel)
@@ -89,15 +95,9 @@
URIish sourceUri = source.getURI(name);
try {
- projectState.get().checkStatePermitsWrite();
- permissionBackend
- .currentUser()
- .project(projectState.get().getNameKey())
- .ref(refName)
- .check(RefPermission.DELETE);
Context.setLocalEvent(true);
- deleteRef(name, refName);
+ deleteRef(name, ref.get());
eventDispatcher
.get()
@@ -141,17 +141,24 @@
}
}
- private RefUpdateState deleteRef(Project.NameKey name, String refName) throws IOException {
+ private Optional<Ref> getRef(Project.NameKey repo, String refName) throws IOException {
+ try (Repository repository = gitManager.openRepository(repo)) {
+ Ref ref = repository.exactRef(refName);
+ return Optional.ofNullable(ref);
+ }
+ }
+ private RefUpdateState deleteRef(Project.NameKey name, Ref ref) throws IOException {
try (Repository repository = gitManager.openRepository(name)) {
+
RefUpdate.Result result;
- RefUpdate u = repository.updateRef(refName);
- u.setExpectedOldObjectId(repository.exactRef(refName).getObjectId());
+ RefUpdate u = repository.updateRef(ref.getName());
+ u.setExpectedOldObjectId(ref.getObjectId());
u.setNewObjectId(ObjectId.zeroId());
u.setForceUpdate(true);
result = u.delete();
- return new RefUpdateState(refName, result);
+ return new RefUpdateState(ref.getName(), result);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
index 2214fb3..8711379 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -22,6 +22,7 @@
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.index.project.ProjectIndexer;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
@@ -49,15 +50,18 @@
private final GerritConfigOps gerritConfigOps;
private final Provider<CurrentUser> userProvider;
private final PermissionBackend permissionBackend;
+ private final ProjectIndexer projectIndexer;
@Inject
ProjectInitializationAction(
GerritConfigOps gerritConfigOps,
Provider<CurrentUser> userProvider,
- PermissionBackend permissionBackend) {
+ PermissionBackend permissionBackend,
+ ProjectIndexer projectIndexer) {
this.gerritConfigOps = gerritConfigOps;
this.userProvider = userProvider;
this.permissionBackend = permissionBackend;
+ this.projectIndexer = projectIndexer;
}
@Override
@@ -106,6 +110,10 @@
}
LocalFS localFS = new LocalFS(maybeUri.get());
Project.NameKey projectNameKey = Project.NameKey.parse(projectName);
- return localFS.createProject(projectNameKey, RefNames.HEAD);
+ if (localFS.createProject(projectNameKey, RefNames.HEAD)) {
+ projectIndexer.index(projectNameKey);
+ return true;
+ }
+ return false;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/auth/PullReplicationGroupBackend.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/auth/PullReplicationGroupBackend.java
index 7bcf3d2..ffd3c1f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/auth/PullReplicationGroupBackend.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/auth/PullReplicationGroupBackend.java
@@ -32,7 +32,7 @@
/** Backend to expose the pull-replication internal user group membership. */
@Singleton
-class PullReplicationGroupBackend extends AbstractGroupBackend {
+public class PullReplicationGroupBackend extends AbstractGroupBackend {
public static final AccountGroup.UUID INTERNAL_GROUP_UUID =
AccountGroup.uuid("pullreplication:internal-user");
public static final String INTERNAL_GROUP_NAME = "Pull-replication Internal User";
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index c1ffa44..2ea8a33 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -22,45 +22,62 @@
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.config.GerritInstanceId;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventListener;
import com.google.gerrit.server.events.ProjectCreatedEvent;
+import com.google.gerrit.server.events.ProjectEvent;
import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob.Factory;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
public class StreamEventListener implements EventListener {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final String ZERO_ID_NAME = ObjectId.zeroId().name();
- private String instanceId;
- private WorkQueue workQueue;
- private ProjectInitializationAction projectInitializationAction;
-
- private Factory fetchJobFactory;
+ private final DeleteRefCommand deleteCommand;
+ private final ExcludedRefsFilter refsFilter;
+ private final Factory fetchJobFactory;
+ private final ProjectInitializationAction projectInitializationAction;
private final Provider<PullReplicationApiRequestMetrics> metricsProvider;
+ private final SourcesCollection sources;
+ private final String instanceId;
+ private final WorkQueue workQueue;
@Inject
public StreamEventListener(
@Nullable @GerritInstanceId String instanceId,
+ DeleteRefCommand deleteCommand,
ProjectInitializationAction projectInitializationAction,
WorkQueue workQueue,
FetchJob.Factory fetchJobFactory,
- Provider<PullReplicationApiRequestMetrics> metricsProvider) {
+ Provider<PullReplicationApiRequestMetrics> metricsProvider,
+ SourcesCollection sources,
+ ExcludedRefsFilter excludedRefsFilter) {
this.instanceId = instanceId;
+ this.deleteCommand = deleteCommand;
this.projectInitializationAction = projectInitializationAction;
this.workQueue = workQueue;
this.fetchJobFactory = fetchJobFactory;
this.metricsProvider = metricsProvider;
+ this.sources = sources;
+ this.refsFilter = excludedRefsFilter;
requireNonNull(
Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty");
@@ -79,40 +96,101 @@
}
public void fetchRefsForEvent(Event event) throws AuthException, PermissionBackendException {
- if (!instanceId.equals(event.instanceId)) {
- PullReplicationApiRequestMetrics metrics = metricsProvider.get();
- metrics.start(event);
- if (event instanceof RefUpdatedEvent) {
- RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
- if (!isProjectDelete(refUpdatedEvent)) {
- fetchRefsAsync(
- refUpdatedEvent.getRefName(),
- refUpdatedEvent.instanceId,
- refUpdatedEvent.getProjectNameKey(),
- metrics);
- }
+ if (instanceId.equals(event.instanceId) || !shouldReplicateProject(event)) {
+ return;
+ }
+
+ PullReplicationApiRequestMetrics metrics = metricsProvider.get();
+ metrics.start(event);
+ if (event instanceof RefUpdatedEvent) {
+ RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
+ if (!isRefToBeReplicated(refUpdatedEvent.getRefName())) {
+ logger.atFine().log(
+ "Skipping excluded ref '%s' for project '%s'",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ return;
}
- if (event instanceof ProjectCreatedEvent) {
- ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
- try {
- projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
- fetchRefsAsync(
- FetchOne.ALL_REFS,
- projectCreatedEvent.instanceId,
- projectCreatedEvent.getProjectNameKey(),
- metrics);
- } catch (AuthException | PermissionBackendException e) {
- logger.atSevere().withCause(e).log(
- "Cannot initialise project:%s", projectCreatedEvent.projectName);
- throw e;
- }
+
+ if (isProjectDelete(refUpdatedEvent)) {
+ return;
+ }
+
+ if (isRefDelete(refUpdatedEvent)) {
+ deleteRef(refUpdatedEvent);
+ return;
+ }
+
+ fetchRefsAsync(
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId,
+ refUpdatedEvent.getProjectNameKey(),
+ metrics);
+ } else if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+ try {
+ projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
+ fetchRefsAsync(
+ FetchOne.ALL_REFS,
+ projectCreatedEvent.instanceId,
+ projectCreatedEvent.getProjectNameKey(),
+ metrics);
+ } catch (AuthException | PermissionBackendException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot initialise project:%s", projectCreatedEvent.projectName);
+ throw e;
}
}
}
+ private void deleteRef(RefUpdatedEvent refUpdatedEvent) {
+ try {
+ deleteCommand.deleteRef(
+ refUpdatedEvent.getProjectNameKey(),
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId);
+ } catch (IOException | RestApiException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot delete ref %s project:%s",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private boolean isRefToBeReplicated(String refName) {
+ return !refsFilter.match(refName);
+ }
+
+ private boolean shouldReplicateProject(Event event) {
+ if (!(event instanceof ProjectEvent)) {
+ return false;
+ }
+
+ Optional<Source> maybeSource =
+ sources.getAll().stream()
+ .filter(s -> s.getRemoteConfigName().equals(event.instanceId))
+ .findFirst();
+
+ if (!maybeSource.isPresent()) {
+ return false;
+ }
+
+ Source source = maybeSource.get();
+ if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+
+ return source.isCreateMissingRepositories()
+ && source.wouldCreateProject(projectCreatedEvent.getProjectNameKey());
+ }
+
+ ProjectEvent projectEvent = (ProjectEvent) event;
+ return source.wouldFetchProject(projectEvent.getProjectNameKey());
+ }
+
+ private boolean isRefDelete(RefUpdatedEvent event) {
+ return ZERO_ID_NAME.equals(event.refUpdate.get().newRev);
+ }
+
private boolean isProjectDelete(RefUpdatedEvent event) {
- return RefNames.isConfigRef(event.getRefName())
- && ObjectId.zeroId().equals(ObjectId.fromString(event.refUpdate.get().newRev));
+ return RefNames.isConfigRef(event.getRefName()) && isRefDelete(event);
}
protected void fetchRefsAsync(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
index 89972cf..74ad9fc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
+import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.*;
@@ -51,6 +52,13 @@
private FetchResult fetchVia(Transport tn, List<RefSpec> fetchRefSpecs) throws IOException {
repLog.info("Fetch references {} from {}", fetchRefSpecs, uri);
- return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ try {
+ return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ } catch (TransportException e) {
+ if (PermanentTransportException.isPermanentFailure(e)) {
+ throw new PermanentTransportException("Terminal fetch failure", e);
+ }
+ throw e;
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
new file mode 100644
index 0000000..9aebb31
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
@@ -0,0 +1,35 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.fetch;
+
+import org.apache.sshd.common.SshException;
+import org.eclipse.jgit.errors.TransportException;
+import org.eclipse.jgit.internal.JGitText;
+
+public class PermanentTransportException extends TransportException {
+ private static final long serialVersionUID = 1L;
+
+ public PermanentTransportException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public static boolean isPermanentFailure(TransportException e) {
+ Throwable cause = e.getCause();
+ String message = e.getMessage();
+ return (cause instanceof SshException
+ && cause.getMessage().startsWith("Failed (UnsupportedCredentialItem) to execute:"))
+ || message.matches(JGitText.get().remoteDoesNotHaveSpec.replaceAll("\\{0\\}", ".+"));
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 51b10d8..b76585e 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -119,6 +119,18 @@
By default, fetches are retried indefinitely.
+ Note that only transient errors will be retried, whilst persistent errors will
+ cause a terminal failure, and the fetch will not be scheduled again. This is
+ only supported for JGit, not cGit. Currently, only the following failures are
+ considered permanent:
+
+ - UnknownHostKey: thrown by Jsch when establishing an SSH connection for an
+ unknown host.
+ - Jgit transport exception when the remote ref does not exist. The assumption
+ here is that the remote ref does not exist so it is not worth retrying. If the
+ exception arisen as a consequence of some ACLs (mis)configuration, then after
+ fixing the ACLs, an explicit replication must be manually triggered.
+
replication.instanceLabel
: Remote configuration name of the current server.
This label is passed as a part of the payload to notify other
@@ -380,6 +392,9 @@
By default, use replication.maxRetries.
+ Note that not all fetch failures are retriable. Please refer
+ to `replication.maxRetries` for more information on this.
+
remote.NAME.threads
: Number of worker threads to dedicate to fetching to the
repositories described by this remote. Each thread can fetch
@@ -558,6 +573,14 @@
remote.NAME.password
: Password to use for HTTP authentication on this remote.
+In both cases, the Global Capability `Access Database` [1] needs to be allowed
+in order to permit all `All-Users`' refs to be replicated. When _basic auth_ is
+used, the capability must be assigned to the `remote.NAME.username` used in
+configuration, whilst for _bearer token_, it needs to be assigned to
+the `Pull-replication Internal User` user.
+
+[1] https://gerrit-review.googlesource.com/Documentation/access-control.html#capability_accessDatabase
+
File `~/.ssh/config`
--------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
index 406cd8c..3f40848 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
@@ -22,18 +22,12 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.SkipProjectClone;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
-import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.projects.BranchInput;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.inject.Inject;
import com.google.inject.Scopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
@@ -46,12 +40,8 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.io.IOException;
import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.time.Duration;
import java.util.List;
-import java.util.function.Supplier;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Ref;
@@ -68,27 +58,8 @@
@TestPlugin(
name = "pull-replication",
sysModule = "com.googlesource.gerrit.plugins.replication.pull.CGitFetchIT$TestModule")
-public class CGitFetchIT extends LightweightPluginDaemonTest {
+public class CGitFetchIT extends FetchITBase {
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final int TEST_REPLICATION_DELAY = 60;
- private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
-
- @Inject private SitePaths sitePaths;
- @Inject private ProjectOperations projectOperations;
- private FetchFactory fetchFactory;
- private Path gitPath;
- private Path testRepoPath;
-
- @Override
- public void setUpTestPlugin() throws Exception {
- gitPath = sitePaths.site_path.resolve("git");
- testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
-
- super.setUpTestPlugin();
- fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
- }
@Test
public void shouldFetchRef() throws Exception {
@@ -245,27 +216,6 @@
}
}
- private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
- WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
- }
-
- private Ref getRef(Repository repo, String branchName) throws IOException {
- return repo.getRefDatabase().exactRef(branchName);
- }
-
- private Ref checkedGetRef(Repository repo, String branchName) {
- try {
- return repo.getRefDatabase().exactRef(branchName);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
- return null;
- }
- }
-
- private Project.NameKey createTestProject(String name) throws Exception {
- return projectOperations.newProject().name(name).create();
- }
-
@SuppressWarnings("unused")
private static class TestModule extends FactoryModule {
@Override
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
index 43331dc..69549aa 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
@@ -17,10 +17,13 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.changes.NotifyHandling;
import com.google.gerrit.extensions.common.AccountInfo;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener.UpdatedRef;
+import java.util.Set;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.transport.ReceiveCommand;
-public class FakeGitReferenceUpdatedEvent implements GitReferenceUpdatedListener.Event {
+public class FakeGitReferenceUpdatedEvent implements GitBatchRefUpdateListener.Event {
private final String projectName;
private final String ref;
private final String oldObjectId;
@@ -46,36 +49,6 @@
}
@Override
- public String getRefName() {
- return ref;
- }
-
- @Override
- public String getOldObjectId() {
- return oldObjectId;
- }
-
- @Override
- public String getNewObjectId() {
- return newObjectId;
- }
-
- @Override
- public boolean isCreate() {
- return type == ReceiveCommand.Type.CREATE;
- }
-
- @Override
- public boolean isDelete() {
- return type == ReceiveCommand.Type.DELETE;
- }
-
- @Override
- public boolean isNonFastForward() {
- return type == ReceiveCommand.Type.UPDATE_NONFASTFORWARD;
- }
-
- @Override
public AccountInfo getUpdater() {
return null;
}
@@ -91,4 +64,46 @@
public NotifyHandling getNotify() {
return NotifyHandling.ALL;
}
+
+ @Override
+ public Set<UpdatedRef> getUpdatedRefs() {
+ return Set.of(
+ new GitBatchRefUpdateListener.UpdatedRef() {
+
+ @Override
+ public String getRefName() {
+ return ref;
+ }
+
+ @Override
+ public String getOldObjectId() {
+ return ObjectId.zeroId().getName();
+ }
+
+ @Override
+ public String getNewObjectId() {
+ return newObjectId;
+ }
+
+ @Override
+ public boolean isCreate() {
+ return type == ReceiveCommand.Type.CREATE;
+ }
+
+ @Override
+ public boolean isDelete() {
+ return type == ReceiveCommand.Type.DELETE;
+ }
+
+ @Override
+ public boolean isNonFastForward() {
+ return type == ReceiveCommand.Type.UPDATE_NONFASTFORWARD;
+ }
+ });
+ }
+
+ @Override
+ public Set<String> getRefNames() {
+ return Set.of(ref);
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
new file mode 100644
index 0000000..2a11717
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
@@ -0,0 +1,73 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.function.Supplier;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+
+public abstract class FetchITBase extends LightweightPluginDaemonTest {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private static final int TEST_REPLICATION_DELAY = 60;
+ private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ FetchFactory fetchFactory;
+ private Path gitPath;
+ Path testRepoPath;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+ testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
+
+ super.setUpTestPlugin();
+ fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
+ }
+
+ void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
+ WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
+ }
+
+ Ref getRef(Repository repo, String branchName) throws IOException {
+ return repo.getRefDatabase().exactRef(branchName);
+ }
+
+ Ref checkedGetRef(Repository repo, String branchName) {
+ try {
+ return repo.getRefDatabase().exactRef(branchName);
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
+ return null;
+ }
+ }
+
+ Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
new file mode 100644
index 0000000..ec695a6
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
@@ -0,0 +1,85 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Scopes;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
+import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import java.net.URISyntaxException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.JGitFetchIT$TestModule")
+public class JGitFetchIT extends FetchITBase {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+
+ @Test(expected = PermanentTransportException.class)
+ public void shouldThrowPermanentTransportExceptionWhenRefDoesNotExists() throws Exception {
+
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String nonExistingRef = "refs/changes/02/20000/1:refs/changes/02/20000/1";
+ try (Repository repo = repoManager.openRepository(project)) {
+ Fetch objectUnderTest = fetchFactory.create(new URIish(testRepoPath.toString()), repo);
+ objectUnderTest.fetch(Lists.newArrayList(new RefSpec(nonExistingRef)));
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class TestModule extends FactoryModule {
+ @Override
+ protected void configure() {
+ Config cf = new Config();
+ cf.setInt("remote", "test_config", "timeout", 0);
+ try {
+ RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config");
+ SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf);
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(CredentialsFactory.class)
+ .to(AutoReloadSecureCredentialsFactoryDecorator.class)
+ .in(Scopes.SINGLETON);
+
+ bind(SourceConfiguration.class).toInstance(sourceConfig);
+ install(
+ new FactoryModuleBuilder()
+ .implement(Fetch.class, JGitFetch.class)
+ .implement(Fetch.class, FetchClientImplementation.class, JGitFetch.class)
+ .build(FetchFactory.class));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
new file mode 100644
index 0000000..b44e92b
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
@@ -0,0 +1,44 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import org.apache.sshd.common.SshException;
+import org.eclipse.jgit.errors.TransportException;
+import org.junit.Test;
+
+public class PermanentFailureExceptionTest {
+
+ @Test
+ public void shouldConsiderSchUnknownHostAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException(
+ "SSH error",
+ new SshException(
+ "Failed (UnsupportedCredentialItem) to execute: some.commands"))))
+ .isTrue();
+ }
+
+ @Test
+ public void shouldConsiderNotExistingRefsAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException("Remote does not have refs/heads/foo available for fetch.")))
+ .isTrue();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
index ee5876f..ff8265f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
@@ -27,7 +27,7 @@
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.projects.BranchInput;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator;
@@ -105,14 +105,14 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
@@ -141,14 +141,14 @@
RevCommit sourceCommit = pushResult.getCommit();
final String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
@@ -174,14 +174,14 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
branchRevision,
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project);
Repository sourceRepo = repoManager.openRepository(project)) {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
index 86b22b2..6fd5cb3 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -30,7 +30,7 @@
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.api.changes.NotifyHandling;
import com.google.gerrit.extensions.api.projects.BranchInput;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.restapi.RestApiException;
@@ -91,14 +91,14 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
@@ -124,14 +124,14 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
branchRevision,
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project);
Repository sourceRepo = repoManager.openRepository(project)) {
@@ -167,14 +167,14 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
branchRevision,
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, newBranch) != null);
@@ -193,14 +193,14 @@
assertThat(pushedRefs).hasSize(1);
assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK);
- GitReferenceUpdatedListener.Event forcedPushEvent =
+ GitBatchRefUpdateListener.Event forcedPushEvent =
new FakeGitReferenceUpdatedEvent(
project,
newBranch,
branchRevision,
amendedCommit.getId().getName(),
ReceiveCommand.Type.UPDATE_NONFASTFORWARD);
- pullReplicationQueue.onGitReferenceUpdated(forcedPushEvent);
+ pullReplicationQueue.onGitBatchRefUpdate(forcedPushEvent);
try (Repository repo = repoManager.openRepository(project);
Repository sourceRepo = repoManager.openRepository(project)) {
@@ -232,14 +232,14 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
@@ -273,14 +273,14 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
branchRevision,
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project);
Repository sourceRepo = repoManager.openRepository(project)) {
@@ -364,14 +364,14 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
index d8e5947..2d1f51f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
@@ -21,7 +21,7 @@
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.acceptance.config.GerritConfig;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
@@ -82,14 +82,14 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- GitReferenceUpdatedListener.Event event =
+ GitBatchRefUpdateListener.Event event =
new FakeGitReferenceUpdatedEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
sourceCommit.getId().getName(),
ReceiveCommand.Type.CREATE);
- pullReplicationQueue.onGitReferenceUpdated(event);
+ pullReplicationQueue.onGitBatchRefUpdate(event);
try (Repository repo = repoManager.openRepository(project)) {
waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 927aec2..e7de264 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -21,6 +21,7 @@
import static org.mockito.Mockito.anyInt;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -33,8 +34,8 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.changes.NotifyHandling;
import com.google.gerrit.extensions.common.AccountInfo;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
-import com.google.gerrit.extensions.events.GitReferenceUpdatedListener.Event;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener;
+import com.google.gerrit.extensions.events.GitBatchRefUpdateListener.UpdatedRef;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.metrics.DisabledMetricMaker;
@@ -54,6 +55,8 @@
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.http.client.ClientProtocolException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.lib.ObjectId;
@@ -64,6 +67,7 @@
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
+import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@@ -161,57 +165,84 @@
@Test
public void shouldCallSendObjectWhenMetaRef() throws ClientProtocolException, IOException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ TestEvent event = new TestEvent("refs/changes/01/1/meta");
objectUnderTest.start();
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any());
}
@Test
public void shouldCallInitProjectWhenProjectIsMissing() throws IOException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ TestEvent event = new TestEvent("refs/changes/01/1/meta");
when(httpResult.isSuccessful()).thenReturn(false);
when(httpResult.isProjectMissing(any())).thenReturn(true);
when(source.isCreateMissingRepositories()).thenReturn(true);
objectUnderTest.start();
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).initProject(any(), any());
}
@Test
public void shouldNotCallInitProjectWhenReplicateNewRepositoriesNotSet() throws IOException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ TestEvent event = new TestEvent("refs/changes/01/1/meta");
when(httpResult.isSuccessful()).thenReturn(false);
when(httpResult.isProjectMissing(any())).thenReturn(true);
when(source.isCreateMissingRepositories()).thenReturn(false);
objectUnderTest.start();
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient, never()).initProject(any(), any());
}
@Test
public void shouldCallSendObjectWhenPatchSetRef() throws ClientProtocolException, IOException {
- Event event = new TestEvent("refs/changes/01/1/1");
+ TestEvent event = new TestEvent("refs/changes/01/1/1");
objectUnderTest.start();
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any());
}
@Test
+ public void shouldCallSendObjectReorderingRefsHavingMetaAtTheEnd()
+ throws ClientProtocolException, IOException {
+ sendRefUpdatedEvents("refs/changes/01/1/meta", "refs/changes/01/1/1");
+ verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ }
+
+ @Test
+ public void shouldCallSendObjectKeepingMetaAtTheEnd()
+ throws ClientProtocolException, IOException {
+ sendRefUpdatedEvents("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ }
+
+ private void sendRefUpdatedEvents(String firstRef, String secondRef) {
+ objectUnderTest.start();
+ objectUnderTest.onGitBatchRefUpdate(new TestEvent(firstRef, secondRef));
+ }
+
+ private void verifySendObjectOrdering(String firstRef, String secondRef)
+ throws ClientProtocolException, IOException {
+ InOrder inOrder = inOrder(fetchRestApiClient);
+
+ inOrder.verify(fetchRestApiClient).callSendObjects(any(), eq(firstRef), any(), any());
+ inOrder.verify(fetchRestApiClient).callSendObjects(any(), eq(secondRef), any(), any());
+ }
+
+ @Test
public void shouldFallbackToCallFetchWhenIOException()
throws ClientProtocolException, IOException, LargeObjectException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ TestEvent event = new TestEvent("refs/changes/01/1/meta");
objectUnderTest.start();
when(revReader.read(any(), any(), anyString(), anyInt())).thenThrow(IOException.class);
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).callFetch(any(), anyString(), any());
}
@@ -219,12 +250,12 @@
@Test
public void shouldFallbackToCallFetchWhenLargeRef()
throws ClientProtocolException, IOException, LargeObjectException {
- Event event = new TestEvent("refs/changes/01/1/1");
+ TestEvent event = new TestEvent("refs/changes/01/1/1");
objectUnderTest.start();
when(revReader.read(any(), any(), anyString(), anyInt())).thenReturn(Optional.empty());
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).callFetch(any(), anyString(), any());
}
@@ -232,7 +263,7 @@
@Test
public void shouldFallbackToCallFetchWhenParentObjectIsMissing()
throws ClientProtocolException, IOException {
- Event event = new TestEvent("refs/changes/01/1/1");
+ TestEvent event = new TestEvent("refs/changes/01/1/1");
objectUnderTest.start();
when(httpResult.isSuccessful()).thenReturn(false);
@@ -240,7 +271,7 @@
when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any()))
.thenReturn(httpResult);
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient).callFetch(any(), anyString(), any());
}
@@ -248,7 +279,7 @@
@Test
public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnMetaRef()
throws ClientProtocolException, IOException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ TestEvent event = new TestEvent("refs/changes/01/1/meta");
objectUnderTest.start();
when(httpResult.isSuccessful()).thenReturn(false, true);
@@ -256,7 +287,7 @@
when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any()))
.thenReturn(httpResult);
- objectUnderTest.onGitReferenceUpdated(event);
+ objectUnderTest.onGitBatchRefUpdate(event);
verify(fetchRestApiClient, times(2))
.callSendObjects(any(), anyString(), revisionsDataCaptor.capture(), any());
@@ -291,16 +322,16 @@
() -> revReader,
applyObjectMetrics,
fetchMetrics);
- Event event = new TestEvent("refs/multi-site/version");
- objectUnderTest.onGitReferenceUpdated(event);
+ TestEvent event = new TestEvent("refs/multi-site/version");
+ objectUnderTest.onGitBatchRefUpdate(event);
verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo);
}
@Test
public void shouldSkipEventWhenStarredChangesRef() {
- Event event = new TestEvent("refs/starred-changes/41/2941/1000000");
- objectUnderTest.onGitReferenceUpdated(event);
+ TestEvent event = new TestEvent("refs/starred-changes/41/2941/1000000");
+ objectUnderTest.onGitBatchRefUpdate(event);
verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo);
}
@@ -365,24 +396,22 @@
return createTempDirectory(prefix);
}
- private class TestEvent implements GitReferenceUpdatedListener.Event {
+ private static class TestEvent implements GitBatchRefUpdateListener.Event {
private String refName;
private String projectName;
- private ObjectId newObjectId;
+ private List<UpdatedRef> refs;
- public TestEvent(String refName) {
- this(refName, "defaultProject", ObjectId.zeroId());
+ public TestEvent(String... refNames) {
+ this(
+ "defaultProject",
+ Arrays.stream(refNames)
+ .map(refName -> updateRef(refName, ObjectId.zeroId()))
+ .collect(Collectors.toUnmodifiableList()));
}
- public TestEvent(String refName, String projectName, ObjectId newObjectId) {
- this.refName = refName;
+ public TestEvent(String projectName, List<UpdatedRef> refs) {
this.projectName = projectName;
- this.newObjectId = newObjectId;
- }
-
- @Override
- public String getRefName() {
- return refName;
+ this.refs = refs;
}
@Override
@@ -396,34 +425,55 @@
}
@Override
- public String getOldObjectId() {
- return ObjectId.zeroId().getName();
- }
-
- @Override
- public String getNewObjectId() {
- return newObjectId.getName();
- }
-
- @Override
- public boolean isCreate() {
- return false;
- }
-
- @Override
- public boolean isDelete() {
- return false;
- }
-
- @Override
- public boolean isNonFastForward() {
- return false;
- }
-
- @Override
public AccountInfo getUpdater() {
return null;
}
+
+ @Override
+ public Set<UpdatedRef> getUpdatedRefs() {
+ return refs.stream().collect(Collectors.toSet());
+ }
+
+ private static final GitBatchRefUpdateListener.UpdatedRef updateRef(
+ String refName, ObjectId refObjectId) {
+ return new GitBatchRefUpdateListener.UpdatedRef() {
+
+ @Override
+ public String getRefName() {
+ return refName;
+ }
+
+ @Override
+ public String getOldObjectId() {
+ return ObjectId.zeroId().getName();
+ }
+
+ @Override
+ public String getNewObjectId() {
+ return refObjectId.getName();
+ }
+
+ @Override
+ public boolean isCreate() {
+ return false;
+ }
+
+ @Override
+ public boolean isDelete() {
+ return false;
+ }
+
+ @Override
+ public boolean isNonFastForward() {
+ return false;
+ }
+ };
+ }
+
+ @Override
+ public Set<String> getRefNames() {
+ return Set.of(refName);
+ }
}
private class FakeProjectDeletedEvent implements ProjectDeletedListener.Event {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
index efa931f..fc1b02c 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -17,6 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -57,6 +58,8 @@
public class DeleteRefCommandTest {
private static final String TEST_SOURCE_LABEL = "test-source-label";
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
+
+ private static final String NON_EXISTING_REF_NAME = "refs/changes/01/11101/1";
private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project");
private static URIish TEST_REMOTE_URI;
@@ -88,9 +91,6 @@
when(sourceCollection.getByRemoteName(TEST_SOURCE_LABEL)).thenReturn(Optional.of(source));
TEST_REMOTE_URI = new URIish("git://some.remote.uri");
when(source.getURI(TEST_PROJECT_NAME)).thenReturn(TEST_REMOTE_URI);
- when(permissionBackend.currentUser()).thenReturn(currentUser);
- when(currentUser.project(any())).thenReturn(forProject);
- when(forProject.ref(any())).thenReturn(forRef);
when(gitManager.openRepository(any())).thenReturn(repository);
when(repository.updateRef(any())).thenReturn(refUpdate);
when(repository.getRefDatabase()).thenReturn(refDb);
@@ -118,4 +118,13 @@
assertThat(fetchEvent.getProjectNameKey()).isEqualTo(TEST_PROJECT_NAME);
assertThat(fetchEvent.getRefName()).isEqualTo(TEST_REF_NAME);
}
+
+ @Test
+ public void shouldHandleNonExistingRef() throws Exception {
+ when(refDb.exactRef(anyString())).thenReturn(null);
+
+ objectUnderTest.deleteRef(TEST_PROJECT_NAME, NON_EXISTING_REF_NAME, TEST_SOURCE_LABEL);
+
+ verify(eventDispatcher, never()).postEvent(any());
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
index 77f05a1..47037d6 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
@@ -31,7 +31,6 @@
import org.junit.Test;
public class ProjectInitializationActionIT extends ActionITBase {
- public static final String INVALID_TEST_PROJECT_NAME = "\0";
@Inject private ProjectOperations projectOperations;
@Test
@@ -150,18 +149,6 @@
@Test
@GerritConfig(name = "container.replica", value = "true")
- public void shouldReturnBadRequestIfProjectNameIsInvalidAndCannotBeCreatedWhenNodeIsAReplica()
- throws Exception {
- url = getURLWithAuthenticationPrefix(INVALID_TEST_PROJECT_NAME);
- httpClientFactory
- .create(source)
- .execute(
- withBasicAuthenticationAsAdmin(createPutRequestWithHeaders()),
- assertHttpResponseCode(HttpServletResponse.SC_BAD_REQUEST));
- }
-
- @Test
- @GerritConfig(name = "container.replica", value = "true")
public void shouldReturnBadRequestIfContentNotSetWhenNodeIsAReplica() throws Exception {
httpClientFactory
.create(source)
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
index c673011..92314a4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -21,9 +21,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Lists;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.data.RefUpdateAttribute;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.ProjectCreatedEvent;
@@ -31,10 +33,15 @@
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
@@ -51,6 +58,7 @@
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
private static final String TEST_PROJECT = "test-project";
private static final String INSTANCE_ID = "node_instance_id";
+ private static final String NEW_REV = "0000000000000000000000000000000000000001";
private static final String REMOTE_INSTANCE_ID = "remote_node_instance_id";
@Mock private ProjectInitializationAction projectInitializationAction;
@@ -58,8 +66,12 @@
@Mock private ScheduledExecutorService executor;
@Mock private FetchJob fetchJob;
@Mock private FetchJob.Factory fetchJobFactory;
+ @Mock private DeleteRefCommand deleteRefCommand;
@Captor ArgumentCaptor<Input> inputCaptor;
@Mock private PullReplicationApiRequestMetrics metrics;
+ @Mock private SourcesCollection sources;
+ @Mock private Source source;
+ @Mock private ExcludedRefsFilter refsFilter;
private StreamEventListener objectUnderTest;
@@ -68,9 +80,22 @@
when(workQueue.getDefaultQueue()).thenReturn(executor);
when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any(), any()))
.thenReturn(fetchJob);
+ when(sources.getAll()).thenReturn(Lists.newArrayList(source));
+ when(source.wouldFetchProject(any())).thenReturn(true);
+ when(source.wouldCreateProject(any())).thenReturn(true);
+ when(source.isCreateMissingRepositories()).thenReturn(true);
+ when(source.getRemoteConfigName()).thenReturn(REMOTE_INSTANCE_ID);
+ when(refsFilter.match(any())).thenReturn(false);
objectUnderTest =
new StreamEventListener(
- INSTANCE_ID, projectInitializationAction, workQueue, fetchJobFactory, () -> metrics);
+ INSTANCE_ID,
+ deleteRefCommand,
+ projectInitializationAction,
+ workQueue,
+ fetchJobFactory,
+ () -> metrics,
+ sources,
+ refsFilter);
}
@Test
@@ -80,6 +105,7 @@
objectUnderTest.onEvent(event);
verify(executor, never()).submit(any(Runnable.class));
+ verify(sources, never()).getAll();
}
@Test
@@ -99,11 +125,49 @@
}
@Test
+ public void shouldSkipEventWhenNotOnAllowedProjectsList() {
+ when(source.wouldFetchProject(any())).thenReturn(false);
+
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
+ public void shouldDeleteRefForRefDeleteEvent() throws IOException, RestApiException {
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.newRev = ObjectId.zeroId().getName();
+ refUpdate.project = TEST_PROJECT;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(deleteRefCommand)
+ .deleteRef(Project.nameKey(TEST_PROJECT), refUpdate.refName, REMOTE_INSTANCE_ID);
+ }
+
+ @Test
public void shouldScheduleFetchJobForRefUpdateEvent() {
RefUpdatedEvent event = new RefUpdatedEvent();
RefUpdateAttribute refUpdate = new RefUpdateAttribute();
refUpdate.refName = TEST_REF_NAME;
refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
event.instanceId = REMOTE_INSTANCE_ID;
event.refUpdate = () -> refUpdate;
@@ -120,6 +184,24 @@
}
@Test
+ public void shouldSkipRefUpdateEventForExcludedRef() {
+ when(refsFilter.match(any())).thenReturn(true);
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
public void shouldCreateProjectForProjectCreatedEvent()
throws AuthException, PermissionBackendException {
ProjectCreatedEvent event = new ProjectCreatedEvent();
@@ -132,6 +214,34 @@
}
@Test
+ public void shouldNotCreateProjectWhenCreateMissingRepositoriesNotSet()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
+ public void shouldNotCreateProjectWhenReplicationNotAllowed()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
public void shouldScheduleAllRefsFetchForProjectCreatedEvent() {
ProjectCreatedEvent event = new ProjectCreatedEvent();
event.instanceId = REMOTE_INSTANCE_ID;
