Google Git
Sign in
gerrit/plugins/multi-site/08ad640d485cb5cdbd5ca3d3d22af3a9ea3f82ef^1..08ad640d485cb5cdbd5ca3d3d22af3a9ea3f82ef/.
commit
08ad640d485cb5cdbd5ca3d3d22af3a9ea3f82ef
[log]
author
Fabio Ponciroli <ponch78@gmail.com>
Wed Aug 13 11:34:04 2025 -0600
committer
Fabio Ponciroli <ponch78@gmail.com>
Wed Aug 13 11:34:04 2025 -0600
tree
a82de00befffc632b47fb5177ddd8ea5d872cd3a
parent
e60dc7bcac8dd17607b82e95ad11ea3db31b9275 [diff]
parent
2f428baa1389bb820c4378f3ca2bb4112deffa03 [diff]
Merge branch 'stable-3.10' into stable-3.11

* stable-3.10:
  Throw lockFailureException when RefDBLock occurs
  Add 3rd site in the local test environment
  Use fanout replication config for the local test environment
  Sort ref-names before locking them in filterAndLock

Change-Id: Ie85b9562c46a99722979174d66ba393d77d16618

diff --git a/setup_local_env/configs/replication.config b/setup_local_env/configs/replication.config
index ff0c248..7aa26fc 100644
--- a/setup_local_env/configs/replication.config
+++ b/setup_local_env/configs/replication.config

@@ -1,15 +1,3 @@
-[remote "$REPLICA_INSTANCE_ID"]
-    $PULL_REPLICATION_URL
-    $PULL_REPLICATION_API_URL
-    fetch = +refs/*:refs/*
-    mirror = true
-    timeout = 600
-    rescheduleDelay = 15
-    replicationDelay = $REPLICATION_DELAY_SEC
-    createMissingRepositories = true
-    replicateProjectDeletions = true
-    replicateHiddenProjects = true
-    threads = 5
 [gerrit]
     autoReload = true
     replicateOnStartup = false

diff --git a/setup_local_env/configs/replication.remote b/setup_local_env/configs/replication.remote
new file mode 100644
index 0000000..5724708
--- /dev/null
+++ b/setup_local_env/configs/replication.remote

@@ -0,0 +1,12 @@
+[remote]
+    $PULL_REPLICATION_URL
+    $PULL_REPLICATION_API_URL
+    fetch = +refs/*:refs/*
+    mirror = true
+    timeout = 600
+    rescheduleDelay = 15
+    replicationDelay = $REPLICATION_DELAY_SEC
+    createMissingRepositories = true
+    replicateProjectDeletions = true
+    replicateHiddenProjects = true
+    threads = 5
\ No newline at end of file

diff --git a/setup_local_env/haproxy-config/haproxy.cfg b/setup_local_env/haproxy-config/haproxy.cfg
index de4f709..eb8bce4 100644
--- a/setup_local_env/haproxy-config/haproxy.cfg
+++ b/setup_local_env/haproxy-config/haproxy.cfg

@@ -45,6 +45,7 @@
     http-check expect status 200
     server node1 $HA_GERRIT_SITE1_HOSTNAME:$HA_GERRIT_SITE1_HTTPD_PORT check inter 10s
     server node2 $HA_GERRIT_SITE2_HOSTNAME:$HA_GERRIT_SITE2_HTTPD_PORT check inter 10s backup
+    server node3 $HA_GERRIT_SITE3_HOSTNAME:$HA_GERRIT_SITE3_HTTPD_PORT check inter 10s backup
 
 backend write-backendnodes
     mode http
@@ -56,6 +57,7 @@
     http-check expect status 200
     server node1 $HA_GERRIT_SITE1_HOSTNAME:$HA_GERRIT_SITE1_HTTPD_PORT check inter 10s
     server node2 $HA_GERRIT_SITE2_HOSTNAME:$HA_GERRIT_SITE2_HTTPD_PORT check inter 10s backup
+    server node3 $HA_GERRIT_SITE3_HOSTNAME:$HA_GERRIT_SITE3_HTTPD_PORT check inter 10s backup
 
 backend ssh
     mode tcp
@@ -66,4 +68,5 @@
     timeout server 5m
     server ssh_node1 $HA_GERRIT_SITE1_HOSTNAME:$HA_GERRIT_SITE1_SSHD_PORT check inter 10s check port $HA_GERRIT_SITE1_HTTPD_PORT inter 10s
     server ssh_node2 $HA_GERRIT_SITE2_HOSTNAME:$HA_GERRIT_SITE2_SSHD_PORT check inter 10s check port $HA_GERRIT_SITE2_HTTPD_PORT inter 10s backup
+    server ssh_node3 $HA_GERRIT_SITE3_HOSTNAME:$HA_GERRIT_SITE3_SSHD_PORT check inter 10s check port $HA_GERRIT_SITE2_HTTPD_PORT inter 10s backup
 

diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh
index f56ef90..bab281a 100755
--- a/setup_local_env/setup.sh
+++ b/setup_local_env/setup.sh

@@ -78,6 +78,23 @@
   done
 }
 
+function copy_replication_remote_file {
+  fanout_template_file=$SCRIPT_DIR/configs/replication.remote
+
+  CONFIG_TEST_SITE=$1
+  export REPLICATION_HTTPD_PORT=$2
+  export REPLICATION_HOSTNAME=$3
+  export PULL_REPLICATION_URL=$(get_pull_replication_url $REPLICATION_HOSTNAME)
+  export PULL_REPLICATION_API_URL=$(get_pull_replication_api_url $REPLICATION_HOSTNAME)
+  export REPLICA_INSTANCE_ID=$4
+  fanout_file=$REPLICA_INSTANCE_ID.config
+
+  mkdir -p $CONFIG_TEST_SITE/replication
+
+  echo "Replacing variables for file $fanout_template_file and copying to $CONFIG_TEST_SITE/replication/$fanout_file"
+  cat $fanout_template_file | envsubst | sed 's/#{name}#/${name}/g' > $CONFIG_TEST_SITE/replication/$fanout_file
+}
+
 function start_ha_proxy {
 
   export HA_GERRIT_CANONICAL_HOSTNAME=$GERRIT_CANONICAL_HOSTNAME
@@ -87,11 +104,14 @@
 
   export HA_GERRIT_SITE1_HOSTNAME=$GERRIT_1_HOSTNAME
   export HA_GERRIT_SITE2_HOSTNAME=$GERRIT_2_HOSTNAME
+  export HA_GERRIT_SITE3_HOSTNAME=$GERRIT_3_HOSTNAME
   export HA_GERRIT_SITE1_HTTPD_PORT=$GERRIT_1_HTTPD_PORT
   export HA_GERRIT_SITE2_HTTPD_PORT=$GERRIT_2_HTTPD_PORT
+  export HA_GERRIT_SITE3_HTTPD_PORT=$GERRIT_3_HTTPD_PORT
 
   export HA_GERRIT_SITE1_SSHD_PORT=$GERRIT_1_SSHD_PORT
   export HA_GERRIT_SITE2_SSHD_PORT=$GERRIT_2_SSHD_PORT
+  export HA_GERRIT_SITE3_SSHD_PORT=$GERRIT_3_SSHD_PORT
 
   cat $SCRIPT_DIR/haproxy-config/haproxy.cfg | envsubst > $HA_PROXY_CONFIG_DIR/haproxy.cfg
 
@@ -133,13 +153,28 @@
   CONFIG_TEST_SITE_2=$LOCATION_TEST_SITE_2/etc
   GERRIT_SITE2_REMOTE_DEBUG_PORT="5006"
   GERRIT_SITE2_INSTANCE_ID="instance-2"
+  # SITE 3
+  GERRIT_SITE3_HOSTNAME=$7
+  GERRIT_SITE3_HTTPD_PORT=$8
+  GERRIT_SITE3_SSHD_PORT=$9
+  CONFIG_TEST_SITE_3=$LOCATION_TEST_SITE_3/etc
+  GERRIT_SITE3_REMOTE_DEBUG_PORT="5007"
+  GERRIT_SITE3_INSTANCE_ID="instance-3"
 
   # Set config SITE1
   copy_config_files $CONFIG_TEST_SITE_1 $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_SSHD_PORT $GERRIT_SITE2_HTTPD_PORT $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE1_REMOTE_DEBUG_PORT $GERRIT_SITE1_INSTANCE_ID $GERRIT_SITE2_INSTANCE_ID
-
+  copy_replication_remote_file $CONFIG_TEST_SITE_1 $GERRIT_SITE2_HTTPD_PORT $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_INSTANCE_ID
+  copy_replication_remote_file $CONFIG_TEST_SITE_1 $GERRIT_SITE3_HTTPD_PORT $GERRIT_SITE3_HOSTNAME $GERRIT_SITE3_INSTANCE_ID
 
   # Set config SITE2
   copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT $GERRIT_SITE2_INSTANCE_ID $GERRIT_SITE1_INSTANCE_ID
+  copy_replication_remote_file $CONFIG_TEST_SITE_2 $GERRIT_SITE1_HTTPD_PORT $GERRIT_SITE1_HOSTNAME $GERRIT_SITE1_INSTANCE_ID
+  copy_replication_remote_file $CONFIG_TEST_SITE_2 $GERRIT_SITE3_HTTPD_PORT $GERRIT_SITE3_HOSTNAME $GERRIT_SITE3_INSTANCE_ID
+
+  # Set config SITE3
+  copy_config_files $CONFIG_TEST_SITE_3 $GERRIT_SITE3_HTTPD_PORT $LOCATION_TEST_SITE_3 $GERRIT_SITE3_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $GERRIT_SITE1_HOSTNAME $GERRIT_SITE3_HOSTNAME $GERRIT_SITE3_REMOTE_DEBUG_PORT $GERRIT_SITE3_INSTANCE_ID $GERRIT_SITE1_INSTANCE_ID
+  copy_replication_remote_file $CONFIG_TEST_SITE_3 $GERRIT_SITE1_HTTPD_PORT $GERRIT_SITE1_HOSTNAME $GERRIT_SITE1_INSTANCE_ID
+  copy_replication_remote_file $CONFIG_TEST_SITE_3 $GERRIT_SITE2_HTTPD_PORT $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_INSTANCE_ID
 }
 
 function is_docker_desktop {
@@ -406,10 +441,13 @@
 export GERRIT_CANONICAL_PORT=${GERRIT_CANONICAL_PORT:-"8080"}
 GERRIT_1_HOSTNAME=${GERRIT_1_HOSTNAME:-"localhost"}
 GERRIT_2_HOSTNAME=${GERRIT_2_HOSTNAME:-"localhost"}
+GERRIT_3_HOSTNAME=${GERRIT_3_HOSTNAME:-"localhost"}
 GERRIT_1_HTTPD_PORT=${GERRIT_1_HTTPD_PORT:-"18080"}
 GERRIT_2_HTTPD_PORT=${GERRIT_2_HTTPD_PORT:-"18081"}
+GERRIT_3_HTTPD_PORT=${GERRIT_3_HTTPD_PORT:-"18082"}
 GERRIT_1_SSHD_PORT=${GERRIT_1_SSHD_PORT:-"39418"}
 GERRIT_2_SSHD_PORT=${GERRIT_2_SSHD_PORT:-"49418"}
+GERRIT_3_SSHD_PORT=${GERRIT_3_SSHD_PORT:-"59418"}
 export REPLICATION_DELAY_SEC=${REPLICATION_DELAY_SEC:-"5"}
 export SSH_ADVERTISED_PORT=${SSH_ADVERTISED_PORT:-"29418"}
 HTTPS_ENABLED=${HTTPS_ENABLED:-"false"}
@@ -418,6 +456,7 @@
 export COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
 LOCATION_TEST_SITE_1=$COMMON_LOCATION/instance-1
 LOCATION_TEST_SITE_2=$COMMON_LOCATION/instance-2
+LOCATION_TEST_SITE_3=$COMMON_LOCATION/instance-3
 HA_PROXY_CONFIG_DIR=$COMMON_LOCATION/ha-proxy-config
 HA_PROXY_CERTIFICATES_DIR="$HA_PROXY_CONFIG_DIR/certificates"
 PROMETHEUS_CONFIG_DIR=$COMMON_LOCATION/prometheus-config
@@ -430,7 +469,7 @@
 export FAKE_NFS=$COMMON_LOCATION/fake_nfs
 
 if [ "$JUST_CLEANUP_ENV" = "true" ];then
-  cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
+  cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $LOCATION_TEST_SITE_3 $COMMON_LOCATION
   exit 0
 fi
 
@@ -500,10 +539,10 @@
 # New installation
 if [ $NEW_INSTALLATION = "true" ]; then
 
-  cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
+  cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $LOCATION_TEST_SITE_3 $COMMON_LOCATION
 
   echo "Setting up directories"
-  mkdir -p $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $HA_PROXY_CERTIFICATES_DIR $FAKE_NFS
+  mkdir -p $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $LOCATION_TEST_SITE_3 $HA_PROXY_CERTIFICATES_DIR $FAKE_NFS
   java -jar $DEPLOYMENT_LOCATION/gerrit.war init --batch --no-auto-start --install-all-plugins --dev -d $LOCATION_TEST_SITE_1
 
   # Deploying TLS certificates
@@ -544,18 +583,19 @@
   # Replicating environment
   echo "Replicating environment"
   cp -fR $LOCATION_TEST_SITE_1/* $LOCATION_TEST_SITE_2
+  cp -fR $LOCATION_TEST_SITE_1/* $LOCATION_TEST_SITE_3
 
-  echo "Link pullreplication plugin"
-  ln -s $LOCATION_TEST_SITE_1/plugins/pull-replication.jar $LOCATION_TEST_SITE_1/lib/pull-replication.jar
-  ln -s $LOCATION_TEST_SITE_2/plugins/pull-replication.jar $LOCATION_TEST_SITE_2/lib/pull-replication.jar
+  for location_test_site in $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $LOCATION_TEST_SITE_3
+  do
+    echo "Link pullreplication plugin on $location_test_site"
+    ln -s $location_test_site/plugins/pull-replication.jar $location_test_site/lib/pull-replication.jar
 
-  echo "Link multi-site library to plugin directory"
-  ln -s $LOCATION_TEST_SITE_1/lib/multi-site.jar $LOCATION_TEST_SITE_1/plugins/multi-site.jar
-  ln -s $LOCATION_TEST_SITE_2/lib/multi-site.jar $LOCATION_TEST_SITE_2/plugins/multi-site.jar
+    echo "Link multi-site library to plugin directory on $location_test_site"
+    ln -s $location_test_site/lib/multi-site.jar $location_test_site/plugins/multi-site.jar
 
-  echo "Copy pull-replication plugin"
-  cp -f $DEPLOYMENT_LOCATION/pull-replication.jar $LOCATION_TEST_SITE_1/plugins/pull-replication.jar
-  cp -f $DEPLOYMENT_LOCATION/pull-replication.jar $LOCATION_TEST_SITE_2/plugins/pull-replication.jar
+    echo "Copy pull-replication plugin on $location_test_site"
+    cp -f $DEPLOYMENT_LOCATION/pull-replication.jar $location_test_site/plugins/pull-replication.jar
+  done
 
 fi
 
@@ -577,16 +617,15 @@
 prepare_broker_data
 
 echo "Re-deploying configuration files"
-deploy_config_files $GERRIT_1_HOSTNAME $GERRIT_1_HTTPD_PORT $GERRIT_1_SSHD_PORT $GERRIT_2_HOSTNAME $GERRIT_2_HTTPD_PORT $GERRIT_2_SSHD_PORT
-echo "Move replication and delete-project plugins to /lib on gerrit site 1"
-mv $LOCATION_TEST_SITE_1/plugins/{replication,delete-project}.jar $LOCATION_TEST_SITE_1/lib/.
-echo "Starting gerrit site 1"
-$LOCATION_TEST_SITE_1/bin/gerrit.sh restart
-echo "Move replication plugin to /lib on gerrit site 2"
-mv $LOCATION_TEST_SITE_2/plugins/{replication,delete-project}.jar $LOCATION_TEST_SITE_2/lib/.
-echo "Starting gerrit site 2"
-$LOCATION_TEST_SITE_2/bin/gerrit.sh restart
+deploy_config_files $GERRIT_1_HOSTNAME $GERRIT_1_HTTPD_PORT $GERRIT_1_SSHD_PORT $GERRIT_2_HOSTNAME $GERRIT_2_HTTPD_PORT $GERRIT_2_SSHD_PORT $GERRIT_3_HOSTNAME $GERRIT_3_HTTPD_PORT $GERRIT_3_SSHD_PORT
 
+for location_test_site in $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $LOCATION_TEST_SITE_3
+do
+  echo "Move replication and delete-project plugins to /lib on gerrit $location_test_site"
+  mv $location_test_site/plugins/{replication,delete-project}.jar $location_test_site/lib/.
+  echo "Starting gerrit $location_test_site"
+  $location_test_site/bin/gerrit.sh restart
+done
 
 if [[ $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}' | wc -l) -lt 1 ]];then
   echo "Starting haproxy"
@@ -604,10 +643,12 @@
 echo "GERRIT HA-PROXY: $GERRIT_CANONICAL_WEB_URL"
 echo "GERRIT-1: http://$GERRIT_1_HOSTNAME:$GERRIT_1_HTTPD_PORT"
 echo "GERRIT-2: http://$GERRIT_2_HOSTNAME:$GERRIT_2_HTTPD_PORT"
+echo "GERRIT-3: http://$GERRIT_3_HOSTNAME:$GERRIT_3_HTTPD_PORT"
 echo "Prometheus: http://localhost:9090"
 echo
 echo "Site-1: $LOCATION_TEST_SITE_1"
 echo "Site-2: $LOCATION_TEST_SITE_2"
+echo "Site-3: $LOCATION_TEST_SITE_3"
 echo
 echo "$HTTPS_CLONE_MSG"
 echo

diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
index 89310bf..8423d2e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java

@@ -19,6 +19,7 @@
 import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
 import com.gerritforge.gerrit.globalrefdb.RefDbLockException;
 import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.entities.Project;
@@ -26,6 +27,7 @@
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.googlesource.gerrit.plugins.multisite.Configuration;
+import com.googlesource.gerrit.plugins.replication.pull.FetchOne.LockFailureException;
 import com.googlesource.gerrit.plugins.replication.pull.ReplicationFetchFilter;
 import java.io.IOException;
 import java.util.Collections;
@@ -34,6 +36,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.stream.Collectors;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Ref;
@@ -96,20 +99,25 @@
 
   @Override
   public Map<String, AutoCloseable> filterAndLock(String projectName, Set<String> fetchRefs)
-      throws RefDbLockException {
+      throws LockFailureException {
+    // Use a SortedSet to ensure that they are always locked in the same order and therefore
+    // prevents potential deadlocks where multiple fetch tasks are mutually locking each other
+    // by locking different refs in different orders.
+    SortedSet<String> sortedFetchRefs = ImmutableSortedSet.copyOf(fetchRefs);
     Project.NameKey projectKey = Project.nameKey(projectName);
     Set<String> filteredRefs = new HashSet<>();
     Map<String, AutoCloseable> refLocks = new HashMap<>();
     try {
-      for (String ref : fetchRefs) {
+      for (String ref : sortedFetchRefs) {
         refLocks.put(ref, sharedRefDb.lockLocalRef(projectKey, ref));
       }
-      filteredRefs.addAll(filter(projectName, fetchRefs));
+      filteredRefs.addAll(filter(projectName, sortedFetchRefs));
     } catch (RefDbLockException lockException) {
       filteredRefs.clear();
-      throw lockException;
+      throw new LockFailureException(
+          "Unable to lock refs " + sortedFetchRefs + " for project " + projectName, lockException);
     } finally {
-      for (String excludedRef : Sets.difference(fetchRefs, filteredRefs)) {
+      for (String excludedRef : Sets.difference(sortedFetchRefs, filteredRefs)) {
         AutoCloseable excludedLock = refLocks.remove(excludedRef);
         if (excludedLock != null) {
           try {