Allow multi-site configuration of gerrit masters

By specifying MULTISITE_ENABLED=true and providing a list of kafka brokers
and a zookeeper connection string the dual-master recipe can be configured
to be part of a multi-site deployment.

Feature: Issue 13237
Change-Id: I27b9190d567a67a0970226bf77f4cacb3e13bec7
diff --git a/.gitignore b/.gitignore
index cb80d80..bec44c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,10 @@
 setup.env
 gerrit/plugins/*.jar
+gerrit/lib/*.jar
 gerrit/etc/*key*
 gerrit.config
 secure.config
 replication.config
+zookeeper-refdb.config
 .idea
 **/*.pem
diff --git a/Makefile.common b/Makefile.common
index 202fab1..f80d22a 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -49,4 +49,14 @@
 endif
 ifdef SMTP_SSL_VERIFY
 		$(eval SMTP_OPTIONAL_PARAMS := $(SMTP_OPTIONAL_PARAMS) ParameterKey=SMTPSslVerify,ParameterValue=$(SMTP_SSL_VERIFY))
-endif
\ No newline at end of file
+endif
+
+set-optional-params-multisite:
+ifdef MULTISITE_ENABLED
+		$(eval MULTISITE_OPTIONAL_PARAMS := ParameterKey=MultiSiteEnabled,ParameterValue=$(MULTISITE_ENABLED))
+		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteKafkaBrokers,ParameterValue=\"$(MULTISITE_KAFKA_BROKERS)\")
+		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteZookeeperConnectString,ParameterValue=$(MULTISITE_ZOOKEEPER_CONNECT_STRING))
+endif
+ifdef MULTISITE_ZOOKEEPER_ROOT_NODE
+		$(eval MULTISITE_OPTIONAL_PARAMS := $(MULTISITE_OPTIONAL_PARAMS) ParameterKey=MultiSiteZookeeperRootNode,ParameterValue=$(MULTISITE_ZOOKEEPER_ROOT_NODE))
+endif
diff --git a/common.env b/common.env
index 9411c5b..0860ce3 100644
--- a/common.env
+++ b/common.env
@@ -7,8 +7,13 @@
 GERRIT_VERSION=3.2
 GERRIT_PATCH=3
 GERRIT_BRANCH=stable-$(GERRIT_VERSION)
+
+# Multi-site
+EVENTSBROKER_LIB_VER=3.2.0-rc4
+
 # Gerrit CI
 GERRIT_CI=https://gerrit-ci.gerritforge.com/view/Plugins-$(GERRIT_BRANCH)/job
+MAVEN_ARCHIVE=https://repo1.maven.org/maven2/com/gerritforge
 LAST_BUILD=lastSuccessfulBuild/artifact/bazel-bin/plugins
 
 # Image directories
diff --git a/dual-master/Makefile b/dual-master/Makefile
index 47c639f..8e202db 100644
--- a/dual-master/Makefile
+++ b/dual-master/Makefile
@@ -70,7 +70,7 @@
 		ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
 		$(CLUSTER_OPTIONAL_PARAMS)
 
-service-master-1: set-optional-params-metrics-cloudwatch set-optional-params-smtp
+service-master-1: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-optional-params-multisite
 ifdef GERRIT_MASTER1_INSTANCE_ID
 		$(eval MASTER1_SERVICE_OPTIONAL_PARAMS := $(MASTER1_SERVICE_OPTIONAL_PARAMS) ParameterKey=InstanceId,ParameterValue=$(GERRIT_MASTER1_INSTANCE_ID))
 endif
@@ -115,12 +115,13 @@
 		ParameterKey=GerritCPU,ParameterValue=$(GERRIT_CPU) \
 		ParameterKey=GerritHeapLimit,ParameterValue=$(GERRIT_HEAP_LIMIT) \
 		ParameterKey=JgitCacheSize,ParameterValue=$(JGIT_CACHE_SIZE) \
+		$(MULTISITE_OPTIONAL_PARAMS) \
 		$(REMOTE_OPTIONAL_PARAMS) \
 		$(MASTER1_SERVICE_OPTIONAL_PARAMS) \
 		$(METRICS_CW_OPTIONAL_PARAMS) \
 		$(SMTP_OPTIONAL_PARAMS)
 
-service-master-2: set-optional-params-metrics-cloudwatch set-optional-params-smtp
+service-master-2: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-optional-params-multisite
 ifdef GERRIT_MASTER2_INSTANCE_ID
 		$(eval MASTER2_SERVICE_OPTIONAL_PARAMS := $(MASTER2_SERVICE_OPTIONAL_PARAMS) ParameterKey=InstanceId,ParameterValue=$(GERRIT_MASTER2_INSTANCE_ID))
 endif
@@ -166,6 +167,7 @@
 		ParameterKey=GerritCPU,ParameterValue=$(GERRIT_CPU) \
 		ParameterKey=GerritHeapLimit,ParameterValue=$(GERRIT_HEAP_LIMIT) \
 		ParameterKey=JgitCacheSize,ParameterValue=$(JGIT_CACHE_SIZE) \
+		$(MULTISITE_OPTIONAL_PARAMS) \
 		$(REMOTE_OPTIONAL_PARAMS) \
 		$(MASTER2_SERVICE_OPTIONAL_PARAMS) \
 		$(METRICS_CW_OPTIONAL_PARAMS) \
diff --git a/dual-master/README.md b/dual-master/README.md
index f612e28..519aa05 100644
--- a/dual-master/README.md
+++ b/dual-master/README.md
@@ -130,6 +130,40 @@
 and writing sides of Git replication: by enabling both of them, it is possible to
 establish replication to a remote Git site.
 
+#### MULTI-SITE
+
+This recipe supports multi-site. Multi-site is a specific configuration of Gerrit
+that allows it to be part of distributed multi-master of multiple Gerrit clusters.
+No storage is shared among the Gerrit sites: syncing happens thanks to two
+channels:
+
+* The `replication` plugin allow alignment of git data (see [replication service](#replication-service))
+for how to enable this.
+* The `multi-site` group of plugins and resources allow the coordination and the exchange
+of gerrit specific events that are produced and consumed by the members of the multi-site deployment.
+(See the [multi-site design](https://gerrit.googlesource.com/plugins/multi-site/+/refs/heads/stable-3.2/DESIGN.md)
+for more information on this.
+
+##### Requirements
+* Kafka brokers and Zookeeper are required by this recipe and are expected to exist
+and accessible with server-side TLS security enabled by the master instances
+resulting from the deployment of this recipe.
+* Replication service must be enabled to allow syncing of Git data.
+
+These are the parameters that can be specified to enable/disable multi-site:
+
+* `MULTISITE_ENABLED`: Optional. Whether this Gerrit is part of a multi-site
+cluster deployment. "false" by default.
+* `MULTISITE_ZOOKEEPER_CONNECT_STRING`: Required when "MULTISITE_ENABLED=true".
+Connection string to Zookeeper.
+* `MULTISITE_KAFKA_BROKERS`: Required when "MULTISITE_ENABLED=true".
+Comma separated list of Kafka broker hosts (host:port)
+to use for publishing events to the message broker.
+* `MULTISITE_ZOOKEEPER_ROOT_NODE` Optional. Root node to use in Zookeeper to
+store/retrieve information.
+Constraint: a slash-separated ('/') string not starting with a slash ('/')
+"gerrit/multi-site" by default.
+
 ### 2 - Deploy
 
 * Create the cluster, services and DNS routing stacks:
diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index fd39589..9f9ee7a 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -175,6 +175,21 @@
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/gc_log\",
                       \"timezone\": \"UTC\"
                       },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/sharedref_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/sharedref_log\",
+                      \"timezone\": \"UTC\"
+                      },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/message_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/message_log\",
+                      \"timezone\": \"UTC\"
+                      },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/websession_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/websession_log\",
+                      \"timezone\": \"UTC\"
+                      },
                       {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/audit_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/audit_log\",
@@ -200,6 +215,21 @@
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/gc_log\",
                       \"timezone\": \"UTC\"
                       },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/sharedref_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/sharedref_log\",
+                      \"timezone\": \"UTC\"
+                      },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/message_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/message_log\",
+                      \"timezone\": \"UTC\"
+                      },
+                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/websession_log\",
+                      \"log_group_name\": \"${AWS::StackName}\",
+                      \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/websession_log\",
+                      \"timezone\": \"UTC\"
+                      },
                       {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/audit_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/audit_log\",
diff --git a/dual-master/cf-service-master.yml b/dual-master/cf-service-master.yml
index 1f61c3b..2b80436 100644
--- a/dual-master/cf-service-master.yml
+++ b/dual-master/cf-service-master.yml
@@ -156,6 +156,24 @@
     Description: The fully qualified domain name of a remote replication target
     Type: String
     Default: ''
+  MultiSiteEnabled:
+    Description: Whether this gerrit is part of a multi-site cluster deployment
+    Type: String
+    Default: false
+  MultiSiteKafkaBrokers:
+    Description: Comma separated list of Kafka broker hosts (host:port) to use for publishing events to the message broker
+    Type: CommaDelimitedList
+    Default: ''
+  MultiSiteZookeeperConnectString:
+    Description: Connection string to Zookeeper
+    Type: String
+    Default: ''
+  MultiSiteZookeeperRootNode:
+    Description: Root node to use in Zookeeper to store/retrieve information
+    Type: String
+    ConstraintDescription: Choose a slash-separated ('/') string not starting with a slash ('/')
+    AllowedPattern: '^[^\/].*'
+    Default: 'gerrit/multi-site'
 
 Mappings:
   Gerrit:
@@ -284,6 +302,15 @@
                       Value: !Join [',', !Ref MetricsCloudwatchExcludeMetrics]
                     - Name: REMOTE_REPLICATION_TARGET_HOST
                       Value: !Ref RemoteReplicationTargetHost
+                    - Name: MULTISITE_ENABLED
+                      Value: !Ref MultiSiteEnabled
+                    - Name: MULTISITE_KAFKA_BROKERS
+                      Value: !Join [',', !Ref MultiSiteKafkaBrokers]
+                    - Name: MULTISITE_ZOOKEEPER_CONNECT_STRING
+                      Value: !Ref MultiSiteZookeeperConnectString
+                    - Name: MULTISITE_ZOOKEEPER_ROOT_NODE
+                      Value: !Ref MultiSiteZookeeperRootNode
+
                   MountPoints:
                     - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Git']
                       ContainerPath: /var/gerrit/git
diff --git a/dual-master/setup.env.template b/dual-master/setup.env.template
index 716edae..99bbf84 100644
--- a/dual-master/setup.env.template
+++ b/dual-master/setup.env.template
@@ -1,3 +1,8 @@
+MULTISITE_ENABLED:=false
+MULTISITE_ZOOKEEPER_CONNECT_STRING:=zookeeper-yourcompany.com:2181
+MULTISITE_KAFKA_BROKERS:=kafka0-yourcompany.com:9092,kafka1-yourcompany.com:9092
+MULTISITE_ZOOKEEPER_ROOT_NODE:=gerrit/multi-site
+
 CLUSTER_DESIRED_CAPACITY:=3
 CLUSTER_INSTANCE_TYPE:=m4.2xlarge
 SERVICE_MASTER1_STACK_NAME:=$(AWS_PREFIX)-service-master-1
diff --git a/gerrit/Dockerfile b/gerrit/Dockerfile
index 26dec1f..f7f8537 100644
--- a/gerrit/Dockerfile
+++ b/gerrit/Dockerfile
@@ -15,6 +15,7 @@
 USER gerrit
 
 COPY --chown=gerrit:gerrit plugins /var/gerrit/plugins
+COPY --chown=gerrit:gerrit lib /var/gerrit/lib
 COPY --chown=gerrit:gerrit etc /var/gerrit/etc
 
 # Install AWS cli
diff --git a/gerrit/Makefile b/gerrit/Makefile
index 026e31f..3b78794 100644
--- a/gerrit/Makefile
+++ b/gerrit/Makefile
@@ -7,10 +7,10 @@
 
 gerrit-get-plugins:
 	# Make sure Prometheus Metrics exporter plugin is installed
-	mkdir -p $(@D)/plugins
+	mkdir -p $(@D)/{plugins,lib}
 
-	@echo "Ensure no plugins are left from previous deployments"
-	rm -f $(@D)/plugins/*.jar
+	@echo "Ensure no plugins or libraries are left from previous deployments"
+	rm -f $(@D)/{plugins,lib}/*.jar
 
 	@echo "Downloading metrics-reporter-prometheus plugin $(GERRIT_BRANCH)"
 	wget $(GERRIT_CI)/plugin-metrics-reporter-prometheus-bazel-master-$(GERRIT_BRANCH)/$(LAST_BUILD)/metrics-reporter-prometheus/metrics-reporter-prometheus.jar \
@@ -37,6 +37,39 @@
 	-O ./plugins/metrics-reporter-cloudwatch.jar \
 	|| { echo >&2 "Cannot download metrics-reporter-cloudwatch plugin: Check internet connection. Aborting"; exit 1; }
 
+ifeq ($(MULTISITE_ENABLED),true)
+	@echo "Downloading multi-site plugin $(GERRIT_BRANCH)"
+	wget $(GERRIT_CI)/plugin-multi-site-bazel-$(GERRIT_BRANCH)/$(LAST_BUILD)/multi-site/multi-site.jar \
+	-O ./plugins/multi-site.jar \
+	|| { echo >&2 "Cannot download multi-site plugin: Check internet connection. Aborting"; exit 1; }
+
+	@echo "Symlink multi-site plugin into lib"
+	ln -sf ../plugins/multi-site.jar ./lib/multi-site.jar
+
+	@echo "Symlink replication plugin into lib"
+	ln -sf ../plugins/replication.jar lib/replication.jar
+
+	@echo "Downloading events-broker library"
+	wget $(MAVEN_ARCHIVE)/events-broker/$(EVENTSBROKER_LIB_VER)/events-broker-$(EVENTSBROKER_LIB_VER).jar \
+	-O lib/events-broker.jar \
+	|| { echo >&2 "Cannot download events-broker-library: Check internet connection. Aborting"; exit 1; }
+
+	@echo "Downloading kafka-events plugin $(GERRIT_BRANCH)"
+	wget $(GERRIT_CI)/plugin-kafka-events-bazel-$(GERRIT_BRANCH)/$(LAST_BUILD)/kafka-events/kafka-events.jar \
+	-O ./plugins/kafka-events.jar \
+	|| { echo >&2 "Cannot download kafka-events plugin: Check internet connection. Aborting"; exit 1; }
+
+	@echo "Downloading zookeper-refdb-3.5 plugin $(GERRIT_BRANCH)"
+	wget $(GERRIT_CI)/plugin-zookeeper-refdb-bazel-$(GERRIT_BRANCH)/$(LAST_BUILD)/zookeeper-refdb/zookeeper-refdb-zk-3.5.jar \
+	-O ./plugins/zookeeper-refdb.jar \
+	|| { echo >&2 "Cannot download zookeeper-refdb plugin: Check internet connection. Aborting"; exit 1; }
+
+	@echo "Downloading websession-broker plugin $(GERRIT_BRANCH)"
+	wget $(GERRIT_CI)/plugin-websession-broker-bazel-master-$(GERRIT_BRANCH)/$(LAST_BUILD)/websession-broker/websession-broker.jar \
+	-O ./plugins/websession-broker.jar \
+	|| { echo >&2 "Cannot download websession-broker plugin: Check internet connection. Aborting"; exit 1; }
+endif
+
 gerrit-build:
 	cat Dockerfile | \
 		GERRIT_VERSION=$(GERRIT_VERSION) GERRIT_PATCH=$(GERRIT_PATCH) envsubst | \
diff --git a/gerrit/entrypoint.sh b/gerrit/entrypoint.sh
index 4ceb995..1235531 100755
--- a/gerrit/entrypoint.sh
+++ b/gerrit/entrypoint.sh
@@ -8,6 +8,15 @@
 if [ $CONTAINER_SLAVE ]; then
   echo "Slave mode..."
 
+  echo "Ensure master specific plugins and libraries are not installed:"
+  for jar in "lib/multi-site.jar" "plugins/multi-site.jar" "lib/replication.jar" \
+    "lib/events-broker.jar" "plugins/kafka-events.jar" "plugins/zookeeper-refdb.jar" \
+    "plugins/websession-broker.jar" "plugins/high-availability.jar"
+  do
+    echo "rm -f /var/gerrit/$jar"
+    rm -f /var/gerrit/"$jar"
+  done
+
   if [ ! -d /var/gerrit/git/All-Projects.git ] ||
      [ ! -d /var/gerrit/git/All-Users.git ] ||
      [ `git --git-dir=/var/gerrit/git/All-Projects.git show-ref | wc -l` -eq 0 ] ||
diff --git a/gerrit/etc/gerrit.config.template b/gerrit/etc/gerrit.config.template
index 772fa8c..6b76753 100644
--- a/gerrit/etc/gerrit.config.template
+++ b/gerrit/etc/gerrit.config.template
@@ -3,6 +3,12 @@
 	canonicalWebUrl = http://localhost
 	serverId = 0f56469f-dfe4-4f28-aca7-803eba7845e0
 	instanceId = {{ GERRIT_INSTANCE_ID }}
+
+{% if MULTISITE_ENABLED is defined and MULTISITE_ENABLED == "true" %}
+	installDbModule = com.googlesource.gerrit.plugins.multisite.GitModule
+	installModule = com.googlesource.gerrit.plugins.multisite.Module
+	installModule = com.gerritforge.gerrit.eventbroker.BrokerApiModule
+{% endif %}
 [core]
 	packedGitLimit = {{ JGIT_CACHE_SIZE }}
 [database]
@@ -89,3 +95,13 @@
     excludeMetrics = caches/.*
 
 {% endif %}
+{% if MULTISITE_ENABLED is defined and MULTISITE_ENABLED == "true" %}
+[plugin "kafka-events"]
+    sendAsync = true
+    bootstrapServers = {{ MULTISITE_KAFKA_BROKERS }}
+    groupId = {{ GERRIT_INSTANCE_ID }}
+    numberOfSubscribers = 6
+    autoCommitIntervalMs = 1000
+    maxRequestSize = 10485760
+    securityProtocol = SSL
+{% endif %}
diff --git a/gerrit/etc/high-availability.config.template b/gerrit/etc/high-availability.config.template
index ff94d0d..b26f4d6 100644
--- a/gerrit/etc/high-availability.config.template
+++ b/gerrit/etc/high-availability.config.template
@@ -6,3 +6,9 @@
 
 [peerInfo "static"]
   url = {{ HA_PEER_URL }}
+
+{% if MULTISITE_ENABLED is defined and MULTISITE_ENABLED == "true" %}
+[websession]
+  synchronize = false
+{% endif %}
+
diff --git a/gerrit/etc/multi-site.config b/gerrit/etc/multi-site.config
new file mode 100644
index 0000000..7bf8c32
--- /dev/null
+++ b/gerrit/etc/multi-site.config
@@ -0,0 +1,4 @@
+[index]
+  maxTries = 6
+  retryInterval = 30000
+  numStripedLocks = 100
\ No newline at end of file
diff --git a/gerrit/etc/replication.config.template b/gerrit/etc/replication.config.template
index f2216a3..6a16b5e 100644
--- a/gerrit/etc/replication.config.template
+++ b/gerrit/etc/replication.config.template
@@ -4,7 +4,7 @@
 
 [gerrit]
   autoReload = true
-  replicateOnStartup = true
+  replicateOnStartup = {{ REPLICATE_ON_STARTUP }}
 
 [remote "slave-1"]
   url = {{ SLAVE_1_URL }}
diff --git a/gerrit/etc/zookeeper-refdb.config.template b/gerrit/etc/zookeeper-refdb.config.template
new file mode 100644
index 0000000..7e0b649
--- /dev/null
+++ b/gerrit/etc/zookeeper-refdb.config.template
@@ -0,0 +1,4 @@
+[ref-database "zookeeper"]
+  connectString = {{ MULTISITE_ZOOKEEPER_CONNECT_STRING }}
+  sslConnection = true
+  rootNode = {{ MULTISITE_ZOOKEEPER_ROOT_NODE }}
\ No newline at end of file
diff --git a/gerrit/setup_gerrit.py b/gerrit/setup_gerrit.py
index 7b6759a..79cfa1f 100755
--- a/gerrit/setup_gerrit.py
+++ b/gerrit/setup_gerrit.py
@@ -8,6 +8,7 @@
 
 setupReplication = (os.getenv('SETUP_REPLICATION') == 'true')
 setupHA = (os.getenv('SETUP_HA') == 'true')
+setupMultiSite = (os.getenv('MULTISITE_ENABLED') == 'true')
 
 def get_secret(secret_name):
     # Create a Secrets Manager client
@@ -157,7 +158,9 @@
         'METRICS_CLOUDWATCH_JVM_ENABLED': os.getenv('METRICS_CLOUDWATCH_JVM_ENABLED'),
         'METRICS_CLOUDWATCH_INITIAL_DELAY': os.getenv('METRICS_CLOUDWATCH_INITIAL_DELAY'),
         'METRICS_CLOUDWATCH_DRY_RUN': os.getenv('METRICS_CLOUDWATCH_DRY_RUN'),
-        'METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST': os.getenv('METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST')
+        'METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST': os.getenv('METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST'),
+        'MULTISITE_ENABLED': os.getenv('MULTISITE_ENABLED'),
+        'MULTISITE_KAFKA_BROKERS': os.getenv('MULTISITE_KAFKA_BROKERS')
     })
     f.write(template.render(config_for_template))
 
@@ -169,12 +172,17 @@
     with open(GERRIT_CONFIG_DIRECTORY + "replication.config", 'w', encoding='utf-8') as f:
         SLAVE_FQDN = os.getenv('SLAVE_SUBDOMAIN') + "." + os.getenv('HOSTED_ZONE_NAME')
         REMOTE_TARGET = os.getenv('REMOTE_REPLICATION_TARGET_HOST', '')
+        # In a multi-site setup, the very first replication needs to be
+        # triggered manually from site-A to site-B, once the latter is ready,
+        # thus "REPLICATE_ON_STARTUP" needs to be disabled
+        REPLICATE_ON_STARTUP = "false" if setupMultiSite else "true"
         f.write(template.render(
                 SLAVE_1_URL="git://" + SLAVE_FQDN + ":" + os.getenv('GIT_PORT') + "/${name}.git",
                 SLAVE_1_AMDIN_URL="ssh://gerrit@" + SLAVE_FQDN + ":" + os.getenv('GIT_SSH_PORT') + "/var/gerrit/git/${name}.git",
                 REMOTE_TARGET=REMOTE_TARGET,
                 REMOTE_TARGET_URL="git://" + REMOTE_TARGET + ":" + os.getenv('GIT_PORT') + "/${name}.git",
                 REMOTE_ADMIN_TARGET_URL="ssh://gerrit@" + REMOTE_TARGET + ":" + os.getenv('GIT_SSH_PORT') + "/var/gerrit/git/${name}.git",
+                REPLICATE_ON_STARTUP=REPLICATE_ON_STARTUP
                 ))
 
 if (setupHA):
@@ -182,4 +190,20 @@
           GERRIT_CONFIG_DIRECTORY + "high-availability.config'")
     template = env.get_template("high-availability.config.template")
     with open(GERRIT_CONFIG_DIRECTORY + "high-availability.config", 'w', encoding='utf-8') as f:
-        f.write(template.render(HA_PEER_URL=os.getenv('HA_PEER_URL')))
+        f.write(template.render(
+            HA_PEER_URL=os.getenv('HA_PEER_URL'),
+            MULTISITE_ENABLED=os.getenv('MULTISITE_ENABLED')
+        ))
+
+if setupMultiSite:
+    CONFIGURATION_FILE = "zookeeper-refdb.config"
+    CONFIGURATION_TARGET = GERRIT_CONFIG_DIRECTORY + CONFIGURATION_FILE
+    TEMPLATE_FILE = CONFIGURATION_FILE + ".template"
+
+    print("*** "+ CONFIGURATION_TARGET)
+    template = env.get_template("zookeeper-refdb.config.template")
+    with open(CONFIGURATION_TARGET, 'w', encoding='utf-8') as f:
+        f.write(template.render(
+            MULTISITE_ZOOKEEPER_CONNECT_STRING=os.getenv('MULTISITE_ZOOKEEPER_CONNECT_STRING'),
+            MULTISITE_ZOOKEEPER_ROOT_NODE=os.getenv('MULTISITE_ZOOKEEPER_ROOT_NODE')
+        ))