Merge "Reformat Comment to HumanComment"
diff --git a/.gitignore b/.gitignore
index dbd3157..73f4353 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,12 +1,15 @@
+# LC_COLLATE=C sort
+
.DS_Store
+/.apt_generated/
/.classpath
+/.primary_build_tool
/.project
/.settings/
-/.primary_build_tool
+
/bazel-bin
/bazel-out
/bazel-reviewers
/bazel-testlogs
-/bazel-multi-site
+
/eclipse-out/
-/.apt_generated/
diff --git a/DESIGN.md b/DESIGN.md
index 0995a14..270b7af 100644
--- a/DESIGN.md
+++ b/DESIGN.md
@@ -510,6 +510,60 @@
set of refs in Read Only state across all the cluster if the RW node is failing after having
sent the request to the Ref-DB but before persisting this request into its `git` layer.
+#### Geo located Gerrit master election
+
+Once you go multi-site multi-master you can improve the latency of your calls by
+serving traffic from the closest server to your user.
+
+Whether you are running your infrastructure in the cloud or on premise you have different solutions you can look at.
+
+##### AWS
+
+Route53 AWS DNS service offers the opportunity of doing [Geo Proximity](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html#routing-policy-geoproximity)
+routing using [Traffic Flow](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/traffic-flow.html).
+
+Traffic flow is a tool which allows the definition of traffic policies and rules via a UI. Traffic rules are of different types, among which *Geoproximity* rules.
+
+When creating geoproximity rules for your resources you can specify one of the following values for each rule:
+
+* If you're using AWS resources, the AWS Region that you created the resource in
+* If you're using non-AWS resources, the latitude and longitude of the resource.
+
+This allows you to have an hybrid cloud-on premise infrastructure.
+
+You can define quite complex failover rules to ensure high availability of your system ([here](https://pasteboard.co/ILFSd5Y.png) an example).
+
+Overall the service provided is pretty much a smart reverse-proxy, if you want more
+complex routing strategies you will still need a proper Load Balancer.
+
+##### GCE
+
+GCE [doesn't offer](https://cloud.google.com/docs/compare/aws/networking#dns) a Geographical based routing, but it implicitly has geo-located DNS entries
+when distributing your application among different zones.
+
+The Load Balancer will balance the traffic to the [nearest available instance](https://cloud.google.com/load-balancing/docs/backend-service#backend_services_and_regions)
+, but this is not configurable and the app server has to be in GC.
+
+Hybrid architectures are supported but would make things more complicated,
+hence this solution is probably worthy only when the Gerrit instances are running in GC.
+
+##### On premise
+
+If you are going for an on premise solution and using HAProxy as Load Balancer,
+it is easy to define static ACL based on IP ranges and use them to route your traffic.
+
+This [blogpost](https://icicimov.github.io/blog/devops/Haproxy-GeoIP/) explains how to achieve it.
+
+On top of that, you want to define a DNS entry per zone and use the ACLs you just defined to
+issue redirection of the calls to most appropiate zone.
+
+You will have to add to your frontend definition your redirection strategy, i.e.:
+
+```
+http-request redirect code 307 prefix https://review-eu.gerrithub.io if acl_EU
+http-request redirect code 307 prefix https://review-am.gerrithub.io if acl_NA
+```
+
# Next steps in the roadmap
## Step-1: Fill the gaps in multi-site Stage #7 implementation:
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..f008749
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,2 @@
+pluginPipeline(formatCheckId: 'gerritforge:multi-site-format-47168e90078b0b3f11401610930e82830e76bff7',
+ buildCheckId: 'gerritforge:multi-site-47168e90078b0b3f11401610930e82830e76bff7')
diff --git a/dockerised_local_env/.gitignore b/dockerised_local_env/.gitignore
index c88a11c..bc7d3f3 100644
--- a/dockerised_local_env/.gitignore
+++ b/dockerised_local_env/.gitignore
@@ -1,25 +1,30 @@
-gerrit-1/plugins
-gerrit-1/db
-gerrit-1/git
-gerrit-1/logs
-gerrit-1/index
-gerrit-1/data
-gerrit-1/bin
-gerrit-1/etc
-gerrit-1/tmp
-gerrit-1/lib
-gerrit-1/ssh/known_hosts
-gerrit-2/plugins
-gerrit-2/db
-gerrit-2/git
-gerrit-2/logs
-gerrit-2/index
-gerrit-2/data
-gerrit-2/bin
-gerrit-2/etc
-gerrit-2/tmp
-gerrit-2/lib
-gerrit-2/ssh/known_hosts
-gerrit-2/bin
-syslog-sidecar/logs
-syslog-sidecar/socket
+# LC_COLLATE=C sort
+
+/gerrit-1/bin/
+/gerrit-1/data/
+/gerrit-1/db/
+/gerrit-1/etc/
+/gerrit-1/git/
+/gerrit-1/index/
+/gerrit-1/lib/
+/gerrit-1/logs/
+/gerrit-1/plugins/
+/gerrit-1/ssh/known_hosts
+/gerrit-1/tmp/
+
+/gerrit-2/bin/
+/gerrit-2/data/
+/gerrit-2/db/
+/gerrit-2/etc/
+/gerrit-2/git/
+/gerrit-2/index/
+/gerrit-2/lib/
+/gerrit-2/logs/
+/gerrit-2/plugins/
+/gerrit-2/ssh/known_hosts
+/gerrit-2/tmp/
+
+/gerrit-common/shared-dir/
+
+/syslog-sidecar/logs/
+/syslog-sidecar/socket/
diff --git a/dockerised_local_env/Makefile b/dockerised_local_env/Makefile
index 0b6b7e3..d48a672 100644
--- a/dockerised_local_env/Makefile
+++ b/dockerised_local_env/Makefile
@@ -20,6 +20,7 @@
download: gerrit plugin_websession_flatfile \
plugin_healthcheck \
+ plugin_delete_project \
plugin_multi_site
@@ -35,12 +36,17 @@
plugin_multi_site: prepare
$(WGET) $(CI_URL)/plugin-multi-site-bazel-stable-2.16/lastSuccessfulBuild/artifact/bazel-bin/plugins/multi-site/multi-site.jar -P $(GERRIT_1_LIB_DIRECTORY)
- cp $(GERRIT_1_LIB_DIRECTORY)/multi-site.jar $(GERRIT_2_LIB_DIRECTORY)/multi-site.jar
+ cp $(GERRIT_1_PLUGINS_DIRECTORY)/replication.jar $(GERRIT_1_LIB_DIRECTORY)
+ cp $(GERRIT_1_LIB_DIRECTORY)/*.jar $(GERRIT_2_LIB_DIRECTORY)
plugin_healthcheck: prepare
$(WGET) $(CI_URL)/plugin-healthcheck-bazel-stable-2.16/lastSuccessfulBuild/artifact/bazel-bin/plugins/healthcheck/healthcheck.jar -P $(GERRIT_1_PLUGINS_DIRECTORY)
cp $(GERRIT_1_PLUGINS_DIRECTORY)/healthcheck.jar $(GERRIT_2_PLUGINS_DIRECTORY)/healthcheck.jar
+plugin_delete_project: prepare
+ $(WGET) $(CI_URL)/plugin-delete-project-bazel-stable-2.16/lastSuccessfulBuild/artifact/bazel-bin/plugins/delete-project/delete-project.jar -P $(GERRIT_1_PLUGINS_DIRECTORY)
+ cp $(GERRIT_1_PLUGINS_DIRECTORY)/delete-project.jar $(GERRIT_2_PLUGINS_DIRECTORY)/delete-project.jar
+
build:
docker build -t $(MYDIR) ./gerrit-1
docker build -t $(MYDIR) ./gerrit-2
diff --git a/dockerised_local_env/README.md b/dockerised_local_env/README.md
index 099be72..92be0cd 100644
--- a/dockerised_local_env/README.md
+++ b/dockerised_local_env/README.md
@@ -2,13 +2,19 @@
## Prerequisites
- * envsubst:
+* envsubst:
```bash
brew install gettext
brew link --force gettext
```
+* wget:
+
+```bash
+brew install wget
+```
+
## Instructions
The docker compose provided in this directory is meant to orchestrate the spin up
@@ -33,3 +39,26 @@
```bash
make restart_gerrit_1 # (or make restart_gerrit_2)
```
+
+## How to test
+
+Consider the
+[instructions](https://gerrit-review.googlesource.com/Documentation/dev-e2e-tests.html)
+on how to use Gerrit core's Gatling framework, to run non-core test scenarios
+such as this plugin one below:
+
+```bash
+sbt "gatling:testOnly com.googlesource.gerrit.plugins.multisite.scenarios.CloneUsingMultiGerrit1"
+```
+
+This is a scenario that can serve as an example for how to start testing a
+multi-site Gerrit system, here such as this dockerized one. That scenario tries
+to clone a project created on this dockerized multi Gerrit, from gerrit-1 (port
+8081). The scenario therefore expects Gerrit multi-site to have properly
+synchronized the new project from the up node gerrit-2 to gerrit-1. That
+project gets deleted after by the (so aggregate) scenario.
+
+Scenario scala source files and their companion json resource ones are stored
+under the usual src/test directories. That structure follows the scala package
+one from the scenario classes. The core framework expects such a directory
+structure for both the scala and resources (json data) files.
diff --git a/dockerised_local_env/gerrit-2/docker-entrypoint.sh b/dockerised_local_env/gerrit-2/docker-entrypoint.sh
index 532377c..06f928f 100755
--- a/dockerised_local_env/gerrit-2/docker-entrypoint.sh
+++ b/dockerised_local_env/gerrit-2/docker-entrypoint.sh
@@ -11,7 +11,11 @@
echo "Waiting for gerrit1 server to become available."
sleep 120
+
+ chmod go-r /var/gerrit/.ssh/id_rsa
+ ssh-keyscan -t rsa -p 29418 gerrit-1 > /var/gerrit/.ssh/known_hosts
ssh -p 29418 admin@gerrit-1 replication start
+
echo "Waiting for replication to complete."
sleep 30
fi
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index 2bbf1aa..30bfbc1 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl
@@ -9,12 +9,12 @@
maven_jar(
name = "global-refdb",
- artifact = "com.gerritforge:global-refdb:3.1.0-rc1",
- sha1 = "61fc8defaed9c364e6bfa101563e434fcc70038f",
+ artifact = "com.gerritforge:global-refdb:3.1.2",
+ sha1 = "6ddee3de0f3fe9254453118ae1eca481ec03e957"
)
maven_jar(
name = "events-broker",
- artifact = "com.gerritforge:events-broker:3.1.4",
- sha1 = "5672908dde0bd02cabc95efe34a8d8507d44b6ac",
+ artifact = "com.gerritforge:events-broker:3.2.0-rc4",
+ sha1 = "53e3f862ac2c2196dba716756ac9586f4b63af47",
)
diff --git a/setup_local_env/README.md b/setup_local_env/README.md
index 2df0c2b..441f98d 100644
--- a/setup_local_env/README.md
+++ b/setup_local_env/README.md
@@ -2,27 +2,35 @@
This script configures a full environment to simulate a Gerrit Multi-Site setup.
The environment is composed by:
- - 2 gerrit instances deployed by default in /tmp
- - 1 kafka node and 1 zookeeper node
- - 1 HA-PROXY
+
+- 2 gerrit instances deployed by default in /tmp
+- 1 kafka node and 1 zookeeper node
+- 1 HA-PROXY
## Requirements
- - java
- - docker and docker-compose
- - wget
- - envsubst
- - haproxy
+
+- java
+- docker and docker-compose
+- wget
+- envsubst
+- haproxy
## Examples
+
Simplest setup with all default values and cleanup previous deployment
+
```bash
sh setup_local_env/setup.sh --release-war-file /path/to/release.war --multisite-plugin-file /path/to/multi-site.jar
```
+
Cleanup the previous deployments
+
```bash
sh setup_local_env/setup.sh --just-cleanup-env true
```
+
Help
+
```bash
Usage: sh setup.sh [--option ]
@@ -52,6 +60,7 @@
```
## Limitations
- - Assumes the ssh replication is done always on port 22 on both instances
- - When cloning projects via ssh, public keys entries are added to `known_hosts`
- - Clean up the old entries when doing a new deploymet, otherwise just use HTTP
\ No newline at end of file
+
+- Assumes the ssh replication is done always on port 22 on both instances
+- When cloning projects via ssh, public keys entries are added to `known_hosts`
+ - Clean up the old entries when doing a new deployment, otherwise just use HTTP
diff --git a/setup_local_env/configs/gerrit.config b/setup_local_env/configs/gerrit.config
index 6092cc1..f9eca89 100644
--- a/setup_local_env/configs/gerrit.config
+++ b/setup_local_env/configs/gerrit.config
@@ -32,6 +32,7 @@
advertisedAddress = *:$SSH_ADVERTISED_PORT
[httpd]
listenUrl = proxy-$HTTP_PROTOCOL://*:$GERRIT_HTTPD_PORT/
+ requestLog = true
[cache]
directory = cache
[plugins]
@@ -39,11 +40,14 @@
[plugin "websession-flatfile"]
directory = $FAKE_NFS
[plugin "kafka-events"]
+ sendAsync = true
bootstrapServers = localhost:$KAFKA_PORT
groupId = $KAFKA_GROUP_ID
- numberOfSubscribers = 5
+ numberOfSubscribers = 6
securityProtocol = PLAINTEXT
pollingIntervalMs = 1000
enableAutoCommit = true
autoCommitIntervalMs = 1000
autoOffsetReset = latest
+[plugin "metrics-reporter-prometheus"]
+ prometheusBearerToken = token
diff --git a/setup_local_env/configs/multi-site.config b/setup_local_env/configs/multi-site.config
index 442c37e..9571513 100644
--- a/setup_local_env/configs/multi-site.config
+++ b/setup_local_env/configs/multi-site.config
@@ -4,6 +4,8 @@
[broker]
indexEventTopic = gerrit_index
+ batchIndexEventTopic = gerrit_batch_index
streamEventTopic = gerrit_stream
projectListEventTopic = gerrit_list_project
cacheEventTopic = gerrit_cache_eviction
+
diff --git a/setup_local_env/configs/prometheus.yml b/setup_local_env/configs/prometheus.yml
new file mode 100644
index 0000000..8eaa989
--- /dev/null
+++ b/setup_local_env/configs/prometheus.yml
@@ -0,0 +1,17 @@
+global:
+ scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+ evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+scrape_configs:
+ - job_name: 'metrics'
+ scheme: http
+ metrics_path: '/plugins/metrics-reporter-prometheus/metrics'
+ params:
+ format: ['prometheus']
+ bearer_token: token
+ scrape_interval: 5s
+ static_configs:
+ - targets: ['$GERRIT_SITE_HOST:18080','$GERRIT_SITE_HOST:18081']
+ labels:
+ env: 'unit'
diff --git a/setup_local_env/configs/zookeeper.config b/setup_local_env/configs/zookeeper-refdb.config
similarity index 100%
rename from setup_local_env/configs/zookeeper.config
rename to setup_local_env/configs/zookeeper-refdb.config
diff --git a/setup_local_env/docker-compose.kafka-broker.yaml b/setup_local_env/docker-compose.yaml
similarity index 63%
rename from setup_local_env/docker-compose.kafka-broker.yaml
rename to setup_local_env/docker-compose.yaml
index b7e91f0..c386d46 100644
--- a/setup_local_env/docker-compose.kafka-broker.yaml
+++ b/setup_local_env/docker-compose.yaml
@@ -13,3 +13,11 @@
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ prometheus:
+ image: prom/prometheus:v2.16.0
+ user: root
+ volumes:
+ - $COMMON_LOCATION/prometheus.yml:/etc/prometheus/prometheus.yml
+ ports:
+ - "9090:9090"
+ network_mode: $NETWORK_MODE
diff --git a/setup_local_env/prometheus-config/prometheus.yml b/setup_local_env/prometheus-config/prometheus.yml
new file mode 100644
index 0000000..e6d3ea1
--- /dev/null
+++ b/setup_local_env/prometheus-config/prometheus.yml
@@ -0,0 +1,14 @@
+# my global config
+global:
+ scrape_interval: 10s
+ evaluation_interval: 10s
+
+scrape_configs:
+ - job_name: gerrit
+ static_configs:
+ - targets: ['localhost:18080','localhost:18081']
+ metrics_path: '/plugins/metrics-reporter-prometheus/metrics'
+ params:
+ format: ['prometheus']
+ bearer_token: token
+
diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh
index 5d4ca83..9438965 100755
--- a/setup_local_env/setup.sh
+++ b/setup_local_env/setup.sh
@@ -16,6 +16,10 @@
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+GERRIT_BRANCH=master
+GERRIT_CI=https://gerrit-ci.gerritforge.com/view/Plugins-$GERRIT_BRANCH/job
+LAST_BUILD=lastSuccessfulBuild/artifact/bazel-bin/plugins
+EVENTS_BROKER_VER=`grep 'com.gerritforge:events-broker' $(dirname $0)/../external_plugin_deps.bzl | cut -d '"' -f 2 | cut -d ':' -f 3`
function check_application_requirements {
type haproxy >/dev/null 2>&1 || { echo >&2 "Require haproxy but it's not installed. Aborting."; exit 1; }
@@ -70,6 +74,7 @@
cat $file | envsubst | sed 's/#{name}#/${name}/g' > $CONFIG_TEST_SITE/$file_name
done
}
+
function start_ha_proxy {
export HA_GERRIT_CANONICAL_HOSTNAME=$GERRIT_CANONICAL_HOSTNAME
@@ -123,12 +128,26 @@
copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT $GERRIT_SITE2_KAFKA_GROUP_ID
}
+function is_docker_desktop {
+ echo $(docker info | grep "Operating System: Docker Desktop" | wc -l)
+}
+
+function docker_host_env {
+ IS_DOCKER_DESKTOP=$(is_docker_desktop)
+ if [ "$IS_DOCKER_DESKTOP" = "1" ];then
+ echo "mac"
+ else
+ echo "linux"
+ fi
+}
+
function cleanup_environment {
echo "Killing existing HA-PROXY setup"
kill $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}') 2> /dev/null
- echo "Stopping kafka and zk"
- docker-compose -f $SCRIPT_DIR/docker-compose.kafka-broker.yaml down 2> /dev/null
+
+ echo "Stopping docker containers"
+ docker-compose -f $SCRIPT_DIR/docker-compose.yaml down 2> /dev/null
echo "Stopping GERRIT instances"
$1/bin/gerrit.sh stop 2> /dev/null
@@ -290,11 +309,12 @@
export SSH_ADVERTISED_PORT=${SSH_ADVERTISED_PORT:-"29418"}
HTTPS_ENABLED=${HTTPS_ENABLED:-"false"}
-COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
+export COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
LOCATION_TEST_SITE_1=$COMMON_LOCATION/instance-1
LOCATION_TEST_SITE_2=$COMMON_LOCATION/instance-2
HA_PROXY_CONFIG_DIR=$COMMON_LOCATION/ha-proxy-config
HA_PROXY_CERTIFICATES_DIR="$HA_PROXY_CONFIG_DIR/certificates"
+PROMETHEUS_CONFIG_DIR=$COMMON_LOCATION/prometheus-config
RELEASE_WAR_FILE_LOCATION=${RELEASE_WAR_FILE_LOCATION:-bazel-bin/release.war}
MULTISITE_LIB_LOCATION=${MULTISITE_LIB_LOCATION:-bazel-bin/plugins/multi-site/multi-site.jar}
@@ -320,32 +340,37 @@
cp -f $MULTISITE_LIB_LOCATION $DEPLOYMENT_LOCATION/multi-site.jar >/dev/null 2>&1 || { echo >&2 "$MULTISITE_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; }
fi
if [ $DOWNLOAD_WEBSESSION_PLUGIN = "true" ];then
- echo "Downloading websession-broker plugin"
- wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-websession-broker-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/websession-broker/websession-broker.jar \
+ echo "Downloading websession-broker plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-websession-broker-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/websession-broker/websession-broker.jar \
-O $DEPLOYMENT_LOCATION/websession-broker.jar || { echo >&2 "Cannot download websession-broker plugin: Check internet connection. Abort\
ing"; exit 1; }
- wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-healthcheck-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/healthcheck/healthcheck.jar \
+ wget $GERRIT_CI/plugin-healthcheck-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/healthcheck/healthcheck.jar \
-O $DEPLOYMENT_LOCATION/healthcheck.jar || { echo >&2 "Cannot download healthcheck plugin: Check internet connection. Abort\
ing"; exit 1; }
else
echo "Without the websession-broker; user login via haproxy will fail."
fi
-echo "Downloading zookeeper plugin master"
- wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-zookeeper-gh-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/zookeeper/zookeeper.jar \
- -O $DEPLOYMENT_LOCATION/zookeeper.jar || { echo >&2 "Cannot download zookeeper plugin: Check internet connection. Abort\
+echo "Downloading zookeeper plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-zookeeper-refdb-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/zookeeper-refdb/zookeeper-refdb.jar \
+ -O $DEPLOYMENT_LOCATION/zookeeper-refdb.jar || { echo >&2 "Cannot download zookeeper plugin: Check internet connection. Abort\
ing"; exit 1; }
-echo "Downloading events-broker library"
- wget https://repo1.maven.org/maven2/com/gerritforge/events-broker/3.1.4/events-broker-3.1.4.jar \
+echo "Downloading events-broker library $GERRIT_BRANCH"
+ wget https://repo1.maven.org/maven2/com/gerritforge/events-broker/$EVENTS_BROKER_VER/events-broker-$EVENTS_BROKER_VER.jar \
-O $DEPLOYMENT_LOCATION/events-broker.jar || { echo >&2 "Cannot download events-broker library: Check internet connection. Abort\
ing"; exit 1; }
-echo "Downloading kafka-events plugin master"
- wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-kafka-events-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/kafka-events/kafka-events.jar \
+echo "Downloading kafka-events plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-kafka-events-bazel-$GERRIT_BRANCH/$LAST_BUILD/kafka-events/kafka-events.jar \
-O $DEPLOYMENT_LOCATION/kafka-events.jar || { echo >&2 "Cannot download kafka-events plugin: Check internet connection. Abort\
ing"; exit 1; }
+echo "Downloading metrics-reporter-prometheus plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-metrics-reporter-prometheus-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/metrics-reporter-prometheus/metrics-reporter-prometheus.jar \
+ -O $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar || { echo >&2 "Cannot download metrics-reporter-prometheus plugin: Check internet connection. Abort\
+ing"; exit 1; }
+
if [ "$REPLICATION_TYPE" = "ssh" ];then
echo "Using 'SSH' replication type"
echo "Make sure ~/.ssh/authorized_keys and ~/.ssh/known_hosts are configured correctly"
@@ -383,7 +408,7 @@
cp -f $DEPLOYMENT_LOCATION/healthcheck.jar $LOCATION_TEST_SITE_1/plugins/healthcheck.jar
echo "Copy zookeeper plugin"
- cp -f $DEPLOYMENT_LOCATION/zookeeper.jar $LOCATION_TEST_SITE_1/plugins/zookeeper.jar
+ cp -f $DEPLOYMENT_LOCATION/zookeeper-refdb.jar $LOCATION_TEST_SITE_1/plugins/zookeeper-refdb.jar
echo "Copy events broker library"
cp -f $DEPLOYMENT_LOCATION/events-broker.jar $LOCATION_TEST_SITE_1/lib/events-broker.jar
@@ -391,6 +416,9 @@
echo "Copy kafka events plugin"
cp -f $DEPLOYMENT_LOCATION/kafka-events.jar $LOCATION_TEST_SITE_1/plugins/kafka-events.jar
+ echo "Copy metrics-reporter-prometheus plugin"
+ cp -f $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar $LOCATION_TEST_SITE_1/plugins/metrics-reporter-prometheus.jar
+
echo "Re-indexing"
java -jar $DEPLOYMENT_LOCATION/gerrit.war reindex -d $LOCATION_TEST_SITE_1
# Replicating environment
@@ -406,12 +434,23 @@
ln -s $LOCATION_TEST_SITE_2/lib/multi-site.jar $LOCATION_TEST_SITE_2/plugins/multi-site.jar
fi
+DOCKER_HOST_ENV=$(docker_host_env)
+echo "Docker host environment: $DOCKER_HOST_ENV"
+if [ "$DOCKER_HOST_ENV" = "mac" ];then
+ export GERRIT_SITE_HOST="host.docker.internal"
+ export NETWORK_MODE="bridge"
+else
+ export GERRIT_SITE_HOST="localhost"
+ export NETWORK_MODE="host"
+fi
+
+cat $SCRIPT_DIR/configs/prometheus.yml | envsubst > $COMMON_LOCATION/prometheus.yml
IS_KAFKA_RUNNING=$(check_if_kafka_is_running)
if [ $IS_KAFKA_RUNNING -lt 1 ];then
echo "Starting zk and kafka"
- docker-compose -f $SCRIPT_DIR/docker-compose.kafka-broker.yaml up -d
+ docker-compose -f $SCRIPT_DIR/docker-compose.yaml up -d
echo "Waiting for kafka to start..."
while [[ $(check_if_kafka_is_running) -lt 1 ]];do sleep 10s; done
fi
@@ -442,6 +481,7 @@
echo "GERRIT HA-PROXY: $GERRIT_CANONICAL_WEB_URL"
echo "GERRIT-1: http://$GERRIT_1_HOSTNAME:$GERRIT_1_HTTPD_PORT"
echo "GERRIT-2: http://$GERRIT_2_HOSTNAME:$GERRIT_2_HTTPD_PORT"
+echo "Prometheus: http://localhost:9090"
echo
echo "Site-1: $LOCATION_TEST_SITE_1"
echo "Site-2: $LOCATION_TEST_SITE_2"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java
new file mode 100644
index 0000000..c2c4b46
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java
@@ -0,0 +1,48 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite;
+
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.util.SystemLog;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.apache.log4j.PatternLayout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class Log4jProjectVersionLogger extends LibModuleLogFile implements ProjectVersionLogger {
+ private static final String LOG_NAME = "project_version_log";
+ private final Logger verLog;
+
+ @Inject
+ public Log4jProjectVersionLogger(SystemLog systemLog) {
+ super(systemLog, LOG_NAME, new PatternLayout("[%d{ISO8601}] [%t] %-5p : %m%n"));
+ this.verLog = LoggerFactory.getLogger(LOG_NAME);
+ }
+
+ @Override
+ public void log(Project.NameKey projectName, long currentVersion, long replicationLag) {
+ if (replicationLag > 0) {
+ verLog.warn(
+ "{ \"project\":\"{}\", \"version\":{}, \"lag\":{} }",
+ projectName,
+ currentVersion,
+ replicationLag);
+ } else {
+ verLog.info("{ \"project\":\"{}\", \"version\":{} }", projectName, currentVersion);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
index 6a73d19..003ce5b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
@@ -100,6 +100,19 @@
}
@Override
+ public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {
+ if (newRefValue != null) {
+ sharedRefDBLog.info(
+ gson.toJson(
+ new SharedRefLogEntry.UpdateRef(
+ project, refName, safeToString(currRef), safeToString(newRefValue), null, null)));
+ } else {
+ sharedRefDBLog.info(
+ gson.toJson(new SharedRefLogEntry.DeleteRef(project, refName, safeToString(currRef))));
+ }
+ }
+
+ @Override
public void logProjectDelete(String project) {
sharedRefDBLog.info(gson.toJson(new SharedRefLogEntry.DeleteProject(project)));
}
@@ -118,4 +131,11 @@
public void setLogger(Logger logger) {
this.sharedRefDBLog = logger;
}
+
+ private <T> String safeToString(T currRef) {
+ if (currRef == null) {
+ return "<null>";
+ }
+ return currRef.toString();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java
new file mode 100644
index 0000000..6ababb6
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java
@@ -0,0 +1,22 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite;
+
+import com.google.gerrit.entities.Project;
+
+public interface ProjectVersionLogger {
+
+ public void log(Project.NameKey projectName, long currentVersion, long replicationLag);
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
index f15a4f2..ddf9d84 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
@@ -21,10 +21,13 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.NoopSharedRefDatabase;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
public class SharedRefDatabaseWrapper implements GlobalRefDatabase {
+ private static final GlobalRefDatabase NOOP_REFDB = new NoopSharedRefDatabase();
@Inject(optional = true)
private DynamicItem<GlobalRefDatabase> sharedRefDbDynamicItem;
@@ -59,6 +62,16 @@
}
@Override
+ public <T> boolean compareAndPut(Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
+ boolean succeeded = sharedRefDb().compareAndPut(project, refName, currValue, newValue);
+ if (succeeded) {
+ sharedRefLogger.logRefUpdate(project.get(), refName, currValue, newValue);
+ }
+ return succeeded;
+ }
+
+ @Override
public AutoCloseable lockRef(Project.NameKey project, String refName)
throws GlobalRefDbLockException {
AutoCloseable locker = sharedRefDb().lockRef(project, refName);
@@ -77,7 +90,13 @@
sharedRefLogger.logProjectDelete(project.get());
}
+ @Override
+ public <T> Optional<T> get(Project.NameKey nameKey, String s, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return sharedRefDb().get(nameKey, s, clazz);
+ }
+
private GlobalRefDatabase sharedRefDb() {
- return sharedRefDbDynamicItem.get();
+ return Optional.ofNullable(sharedRefDbDynamicItem).map(di -> di.get()).orElse(NOOP_REFDB);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
index 51f9ff0..3853af6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
@@ -21,6 +21,8 @@
void logRefUpdate(String project, Ref currRef, ObjectId newRefValue);
+ <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue);
+
void logProjectDelete(String project);
void logLockAcquisition(String project, String refName);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/cache/CachePatternMatcher.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/cache/CachePatternMatcher.java
index f815c92..cd86848 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/cache/CachePatternMatcher.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/cache/CachePatternMatcher.java
@@ -27,13 +27,7 @@
class CachePatternMatcher {
private static final List<String> DEFAULT_PATTERNS =
ImmutableList.of(
- "^accounts.*",
- "^groups.*",
- "ldap_groups",
- "ldap_usernames",
- "projects",
- "sshkeys",
- "web_sessions");
+ "^accounts.*", "^groups.*", "ldap_groups", "ldap_usernames", "projects", "sshkeys");
private final Pattern pattern;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
index ec6072c..816edc9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
@@ -74,6 +74,7 @@
msgLog.log(Direction.CONSUME, topic, event);
eventRouter.route(event.getEvent());
subscriberMetrics.incrementSubscriberConsumedMessage();
+ subscriberMetrics.updateReplicationStatusMetrics(event);
} catch (IOException e) {
logger.atSevere().withCause(e).log(
"Malformed event '%s': [Exception: %s]", event.getHeader());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java
new file mode 100644
index 0000000..5bbaee0
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java
@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.consumer;
+
+import com.google.gerrit.extensions.registration.DynamicSet;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
+import com.googlesource.gerrit.plugins.multisite.InstanceId;
+import com.googlesource.gerrit.plugins.multisite.MessageLogger;
+import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
+import com.googlesource.gerrit.plugins.multisite.forwarder.router.IndexEventRouter;
+import java.util.UUID;
+
+@Singleton
+public class BatchIndexEventSubscriber extends AbstractSubcriber {
+ @Inject
+ public BatchIndexEventSubscriber(
+ IndexEventRouter eventRouter,
+ DynamicSet<DroppedEventListener> droppedEventListeners,
+ @InstanceId UUID instanceId,
+ MessageLogger msgLog,
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
+ }
+
+ @Override
+ protected EventTopic getTopic() {
+ return EventTopic.BATCH_INDEX_TOPIC;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
index 696cf03..49d470a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
@@ -34,8 +34,7 @@
MessageLogger msgLog,
SubscriberMetrics subscriberMetrics,
Configuration cfg) {
- super(
- eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
index f14dde5..d858b89 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.gerritforge.gerrit.eventbroker.BrokerApi;
-import com.google.common.collect.Lists;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.registration.DynamicItem;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
index bf8b33d..6ff0969 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
@@ -34,8 +34,7 @@
MessageLogger msgLog,
SubscriberMetrics subscriberMetrics,
Configuration cfg) {
- super(
- eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
index 39ace0e..20c355e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
@@ -34,8 +34,7 @@
MessageLogger msgLog,
SubscriberMetrics subscriberMetrics,
Configuration cfg) {
- super(
- eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
index 36f618e..4459859 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
@@ -14,25 +14,53 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.entities.Project;
import com.google.gerrit.metrics.Counter1;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.MetricMaker;
+import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.MultiSiteMetrics;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
+import com.googlesource.gerrit.plugins.replication.RefReplicatedEvent;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
+import com.googlesource.gerrit.plugins.replication.ReplicationScheduledEvent;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
@Singleton
public class SubscriberMetrics extends MultiSiteMetrics {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String SUBSCRIBER_SUCCESS_COUNTER = "subscriber_msg_consumer_counter";
private static final String SUBSCRIBER_FAILURE_COUNTER =
"subscriber_msg_consumer_failure_counter";
+ private static final String REPLICATION_LAG_SEC =
+ "multi_site/subscriber/subscriber_replication_status/sec_behind";
private final Counter1<String> subscriberSuccessCounter;
private final Counter1<String> subscriberFailureCounter;
+ private final ProjectVersionLogger verLogger;
+
+ private final Map<String, Long> replicationStatusPerProject = new HashMap<>();
+ private final Map<String, Long> localVersionPerProject = new HashMap<>();
+
+ private ProjectVersionRefUpdate projectVersionRefUpdate;
@Inject
- public SubscriberMetrics(MetricMaker metricMaker) {
+ public SubscriberMetrics(
+ MetricMaker metricMaker,
+ ProjectVersionRefUpdate projectVersionRefUpdate,
+ ProjectVersionLogger verLogger) {
+ this.projectVersionRefUpdate = projectVersionRefUpdate;
this.subscriberSuccessCounter =
metricMaker.newCounter(
"multi_site/subscriber/subscriber_message_consumer_counter",
@@ -47,6 +75,19 @@
.setRate()
.setUnit("errors"),
stringField(SUBSCRIBER_FAILURE_COUNTER, "Subscriber failed to consume messages count"));
+ metricMaker.newCallbackMetric(
+ REPLICATION_LAG_SEC,
+ Long.class,
+ new Description("Replication lag (sec)").setGauge().setUnit(Description.Units.SECONDS),
+ () -> {
+ Collection<Long> lags = replicationStatusPerProject.values();
+ if (lags.isEmpty()) {
+ return 0L;
+ }
+ return Collections.max(lags);
+ });
+
+ this.verLogger = verLogger;
}
public void incrementSubscriberConsumedMessage() {
@@ -56,4 +97,45 @@
public void incrementSubscriberFailedToConsumeMessage() {
subscriberFailureCounter.increment(SUBSCRIBER_FAILURE_COUNTER);
}
+
+ public void updateReplicationStatusMetrics(EventMessage eventMessage) {
+ Event event = eventMessage.getEvent();
+ if (event instanceof RefReplicationDoneEvent) {
+ RefReplicationDoneEvent replicationDone = (RefReplicationDoneEvent) event;
+ updateReplicationLagMetrics(
+ replicationDone.getProjectNameKey(), replicationDone.getRefName());
+ } else if (event instanceof RefReplicatedEvent) {
+ RefReplicatedEvent replicated = (RefReplicatedEvent) event;
+ updateReplicationLagMetrics(replicated.getProjectNameKey(), replicated.getRefName());
+ } else if (event instanceof ReplicationScheduledEvent) {
+ ReplicationScheduledEvent updated = (ReplicationScheduledEvent) event;
+ updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+ } else if (event instanceof RefUpdatedEvent) {
+ RefUpdatedEvent updated = (RefUpdatedEvent) event;
+ updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+ }
+ }
+
+ private void updateReplicationLagMetrics(Project.NameKey projectName, String ref) {
+ Optional<Long> remoteVersion =
+ projectVersionRefUpdate.getProjectRemoteVersion(projectName.get());
+ Optional<Long> localVersion = projectVersionRefUpdate.getProjectLocalVersion(projectName.get());
+ if (remoteVersion.isPresent() && localVersion.isPresent()) {
+ long lag = remoteVersion.get() - localVersion.get();
+
+ if (!localVersion.get().equals(localVersionPerProject.get(projectName.get()))
+ || lag != replicationStatusPerProject.get(projectName.get())) {
+ logger.atFine().log(
+ "Published replication lag metric for project '%s' of %d sec(s) [local-ref=%d global-ref=%d]",
+ projectName, lag, localVersion.get(), remoteVersion.get());
+ replicationStatusPerProject.put(projectName.get(), lag);
+ localVersionPerProject.put(projectName.get(), localVersion.get());
+ verLogger.log(projectName, localVersion.get(), lag);
+ }
+ } else {
+ logger.atFine().log(
+ "Did not publish replication lag metric for %s because the %s version is not defined",
+ projectName, localVersion.isPresent() ? "remote" : "local");
+ }
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
index afd4d09..09adb18 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
@@ -28,6 +28,7 @@
DynamicSet.setOf(binder(), DroppedEventListener.class);
DynamicSet.bind(binder(), AbstractSubcriber.class).to(IndexEventSubscriber.class);
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(BatchIndexEventSubscriber.class);
DynamicSet.bind(binder(), AbstractSubcriber.class).to(StreamEventSubscriber.class);
DynamicSet.bind(binder(), AbstractSubcriber.class).to(CacheEvictionEventSubscriber.class);
DynamicSet.bind(binder(), AbstractSubcriber.class).to(ProjectUpdateEventSubscriber.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
index 8889935..1c0c644 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
@@ -17,6 +17,7 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.events.EventListener;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
import java.util.concurrent.Executor;
public class EventModule extends LifecycleModule {
@@ -26,5 +27,6 @@
bind(Executor.class).annotatedWith(EventExecutor.class).toProvider(EventExecutorProvider.class);
listener().to(EventExecutorProvider.class);
DynamicSet.bind(binder(), EventListener.class).to(EventHandler.class);
+ DynamicSet.bind(binder(), EventListener.class).to(ProjectVersionRefUpdate.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
index c00c571..5212aa4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
@@ -20,7 +20,12 @@
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.AccountIndexEvent;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* Index an account using {@link AccountIndexer}. This class is meant to be used on the receiving
@@ -31,12 +36,15 @@
@Singleton
public class ForwardedIndexAccountHandler
extends ForwardedIndexingHandler<Account.Id, AccountIndexEvent> {
+
private final AccountIndexer indexer;
+ private Map<Account.Id, Operation> accountsToIndex;
@Inject
ForwardedIndexAccountHandler(AccountIndexer indexer, Configuration config) {
super(config.index().numStripedLocks());
this.indexer = indexer;
+ this.accountsToIndex = new HashMap<>();
}
@Override
@@ -49,4 +57,29 @@
protected void doDelete(Account.Id id, Optional<AccountIndexEvent> event) {
throw new UnsupportedOperationException("Delete from account index not supported");
}
+
+ public synchronized void indexAsync(Account.Id id, Operation operation) {
+ accountsToIndex.put(id, operation);
+ }
+
+ public synchronized void doAsyncIndex() {
+ accountsToIndex =
+ accountsToIndex.entrySet().stream()
+ .filter(e -> !checkedIndex(e))
+ .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
+ }
+
+ private boolean checkedIndex(Map.Entry<Account.Id, Operation> account) {
+ try {
+ index(account.getKey(), account.getValue(), Optional.empty());
+ return true;
+ } catch (IOException e) {
+ log.error("Account {} index failed", account.getKey(), e);
+ return false;
+ }
+ }
+
+ public Set<Account.Id> pendingAccountsToIndex() {
+ return accountsToIndex.keySet();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParser.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParser.java
index 922da1a..0bbdada 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParser.java
@@ -14,7 +14,7 @@
package com.googlesource.gerrit.plugins.multisite.forwarder;
-import com.google.common.base.Strings;
+import com.google.common.base.MoreObjects;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.AccountGroup;
import com.google.gerrit.server.events.EventGson;
@@ -31,31 +31,43 @@
this.gson = gson;
}
- public Object fromJson(String cacheName, String jsonString) {
- JsonElement json = gson.fromJson(Strings.nullToEmpty(jsonString), JsonElement.class);
+ @SuppressWarnings("cast")
+ public Object fromJson(String cacheName, Object json) {
Object key;
// Need to add a case for 'adv_bases'
switch (cacheName) {
case Constants.ACCOUNTS:
- key = Account.id(json.getAsJsonObject().get("id").getAsInt());
+ key = Account.id(jsonElement(json).getAsJsonObject().get("id").getAsInt());
break;
case Constants.GROUPS:
- key = AccountGroup.id(json.getAsJsonObject().get("id").getAsInt());
+ key = AccountGroup.id(jsonElement(json).getAsJsonObject().get("id").getAsInt());
break;
case Constants.GROUPS_BYINCLUDE:
case Constants.GROUPS_MEMBERS:
- key = AccountGroup.uuid(json.getAsJsonObject().get("uuid").getAsString());
+ key = AccountGroup.uuid(jsonElement(json).getAsJsonObject().get("uuid").getAsString());
break;
case Constants.PROJECT_LIST:
- key = gson.fromJson(json, Object.class);
+ key = gson.fromJson(nullToEmpty(json).toString(), Object.class);
break;
default:
- try {
- key = gson.fromJson(json, String.class);
- } catch (Exception e) {
- key = gson.fromJson(json, Object.class);
+ if (json instanceof String) {
+ key = (String) json;
+ } else {
+ try {
+ key = gson.fromJson(nullToEmpty(json).toString().trim(), String.class);
+ } catch (Exception e) {
+ key = gson.fromJson(nullToEmpty(json).toString(), Object.class);
+ }
}
}
return key;
}
+
+ private JsonElement jsonElement(Object json) {
+ return gson.fromJson(nullToEmpty(json), JsonElement.class);
+ }
+
+ private static String nullToEmpty(Object value) {
+ return MoreObjects.firstNonNull(value, "").toString().trim();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/IndexEventForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/IndexEventForwarder.java
index 701c2fe..5c1f444 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/IndexEventForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/IndexEventForwarder.java
@@ -19,10 +19,18 @@
public interface IndexEventForwarder {
/**
- * Publish an indexing event to the broker.
+ * Publish an indexing event to the broker using interactive topic.
*
* @param event the details of the index event.
* @return true if successful, otherwise false.
*/
boolean index(IndexEvent event);
+
+ /**
+ * Publish an indexing event to the broker using batch topic.
+ *
+ * @param event the details of the index event.
+ * @return true if successful, otherwise false.
+ */
+ boolean batchIndex(IndexEvent event);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
index 0b4252d..a86c62e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
@@ -35,4 +35,9 @@
public boolean index(IndexEvent event) {
return broker.send(EventTopic.INDEX_TOPIC.topic(cfg), event);
}
+
+ @Override
+ public boolean batchIndex(IndexEvent event) {
+ return broker.send(EventTopic.BATCH_INDEX_TOPIC.topic(cfg), event);
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
index eaa2df9..4e7a781 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
@@ -18,6 +18,7 @@
public enum EventTopic {
INDEX_TOPIC("GERRIT.EVENT.INDEX", "indexEvent"),
+ BATCH_INDEX_TOPIC("GERRIT.EVENT.BATCH.INDEX", "batchIndexEvent"),
CACHE_TOPIC("GERRIT.EVENT.CACHE", "cacheEvent"),
PROJECT_LIST_TOPIC("GERRIT.EVENT.PROJECT.LIST", "projectListEvent"),
STREAM_EVENT_TOPIC("GERRIT.EVENT.STREAM", "streamEvent");
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/CacheEvictionEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/CacheEvictionEventRouter.java
index 8c86c0c..0fb0c0a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/CacheEvictionEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/CacheEvictionEventRouter.java
@@ -34,8 +34,7 @@
@Override
public void route(CacheEvictionEvent cacheEvictionEvent) throws CacheNotFoundException {
- Object parsedKey =
- gsonParser.fromJson(cacheEvictionEvent.cacheName, cacheEvictionEvent.key.toString());
+ Object parsedKey = gsonParser.fromJson(cacheEvictionEvent.cacheName, cacheEvictionEvent.key);
cacheEvictionHanlder.evict(CacheEntry.from(cacheEvictionEvent.cacheName, parsedKey));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
index dcaaea2..202fb42 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
@@ -17,7 +17,10 @@
import static com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexingHandler.Operation.DELETE;
import static com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexingHandler.Operation.INDEX;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Account;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.AllUsersName;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexAccountHandler;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexChangeHandler;
@@ -29,25 +32,32 @@
import com.googlesource.gerrit.plugins.multisite.forwarder.events.GroupIndexEvent;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.IndexEvent;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ProjectIndexEvent;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
import java.io.IOException;
import java.util.Optional;
+import java.util.Set;
-public class IndexEventRouter implements ForwardedEventRouter<IndexEvent> {
+public class IndexEventRouter implements ForwardedEventRouter<IndexEvent>, LifecycleListener {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
private final ForwardedIndexAccountHandler indexAccountHandler;
private final ForwardedIndexChangeHandler indexChangeHandler;
private final ForwardedIndexGroupHandler indexGroupHandler;
private final ForwardedIndexProjectHandler indexProjectHandler;
+ private final AllUsersName allUsersName;
@Inject
public IndexEventRouter(
ForwardedIndexAccountHandler indexAccountHandler,
ForwardedIndexChangeHandler indexChangeHandler,
ForwardedIndexGroupHandler indexGroupHandler,
- ForwardedIndexProjectHandler indexProjectHandler) {
+ ForwardedIndexProjectHandler indexProjectHandler,
+ AllUsersName allUsersName) {
this.indexAccountHandler = indexAccountHandler;
this.indexChangeHandler = indexChangeHandler;
this.indexGroupHandler = indexGroupHandler;
this.indexProjectHandler = indexProjectHandler;
+ this.allUsersName = allUsersName;
}
@Override
@@ -61,8 +71,7 @@
Optional.of(changeIndexEvent));
} else if (sourceEvent instanceof AccountIndexEvent) {
AccountIndexEvent accountIndexEvent = (AccountIndexEvent) sourceEvent;
- indexAccountHandler.index(
- Account.id(accountIndexEvent.accountId), INDEX, Optional.of(accountIndexEvent));
+ indexAccountHandler.indexAsync(Account.id(accountIndexEvent.accountId), INDEX);
} else if (sourceEvent instanceof GroupIndexEvent) {
GroupIndexEvent groupIndexEvent = (GroupIndexEvent) sourceEvent;
indexGroupHandler.index(groupIndexEvent.groupUUID, INDEX, Optional.of(groupIndexEvent));
@@ -75,4 +84,34 @@
String.format("Cannot route event %s", sourceEvent.getType()));
}
}
+
+ public void onRefReplicated(RefReplicationDoneEvent replicationEvent) throws IOException {
+ if (replicationEvent.getProjectNameKey().equals(allUsersName)) {
+ Account.Id accountId = Account.Id.fromRef(replicationEvent.getRefName());
+ if (accountId != null) {
+ indexAccountHandler.index(accountId, INDEX, Optional.empty());
+ } else {
+ indexAccountHandler.doAsyncIndex();
+ }
+ }
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public void stop() {
+ Set<Account.Id> accountsToIndex = indexAccountHandler.pendingAccountsToIndex();
+ if (!accountsToIndex.isEmpty()) {
+ logger.atWarning().log("Forcing reindex of accounts %s upon shutdown", accountsToIndex);
+ indexAccountHandler.doAsyncIndex();
+ }
+
+ Set<Account.Id> accountsIndexFailed = indexAccountHandler.pendingAccountsToIndex();
+ if (!accountsIndexFailed.isEmpty()) {
+ logger.atSevere().log(
+ "The accounts %s failed to be indexed and their Lucene index is stale",
+ accountsIndexFailed);
+ }
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/RouterModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/RouterModule.java
index 91dfc53..bac907e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/RouterModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/RouterModule.java
@@ -14,13 +14,14 @@
package com.googlesource.gerrit.plugins.multisite.forwarder.router;
-import com.google.inject.AbstractModule;
+import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.inject.Scopes;
-public class RouterModule extends AbstractModule {
+public class RouterModule extends LifecycleModule {
@Override
protected void configure() {
bind(IndexEventRouter.class).in(Scopes.SINGLETON);
+ listener().to(IndexEventRouter.class).in(Scopes.SINGLETON);
bind(CacheEvictionEventRouter.class).in(Scopes.SINGLETON);
bind(ProjectListUpdateRouter.class).in(Scopes.SINGLETON);
bind(StreamEventRouter.class).in(Scopes.SINGLETON);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
index d911c00..4ef3426 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
@@ -18,17 +18,32 @@
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedEventHandler;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
+import java.io.IOException;
public class StreamEventRouter implements ForwardedEventRouter<Event> {
private final ForwardedEventHandler streamEventHandler;
+ private final IndexEventRouter indexEventRouter;
@Inject
- public StreamEventRouter(ForwardedEventHandler streamEventHandler) {
+ public StreamEventRouter(
+ ForwardedEventHandler streamEventHandler, IndexEventRouter indexEventRouter) {
this.streamEventHandler = streamEventHandler;
+ this.indexEventRouter = indexEventRouter;
}
@Override
- public void route(Event sourceEvent) throws PermissionBackendException {
+ public void route(Event sourceEvent) throws PermissionBackendException, IOException {
+ if (RefReplicationDoneEvent.TYPE.equals(sourceEvent.getType())) {
+ /* TODO: We currently explicitly ignore the status and result of the replication
+ * event because there isn't a reliable way to understand if the current node was
+ * the replication target and was successful or not.
+ *
+ * It is better to risk to reindex once more rather than missing a reindexing event.
+ */
+ indexEventRouter.onRefReplicated((RefReplicationDoneEvent) sourceEvent);
+ }
+
streamEventHandler.dispatch(sourceEvent);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
index cbdb936..60611f4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
@@ -79,7 +79,7 @@
@Override
public Optional<ChangeNotes> getChangeNotes() {
try (ManualRequestContext ctx = oneOffReqCtx.open()) {
- this.changeNotes = Optional.ofNullable(changeFinder.findOne(changeId));
+ this.changeNotes = changeFinder.findOne(changeId);
return changeNotes;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/IndexEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/IndexEventHandler.java
index ee3ddbc..ed33efa 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/IndexEventHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/IndexEventHandler.java
@@ -98,10 +98,17 @@
private void executeIndexChangeTask(String projectName, int id) {
if (!Context.isForwardedEvent()) {
ChangeChecker checker = changeChecker.create(projectName + "~" + id);
+
try {
checker
.newIndexEvent(projectName, id, false)
- .map(event -> new IndexChangeTask(event))
+ .map(
+ event -> {
+ if (Thread.currentThread().getName().contains("Batch")) {
+ return new BatchIndexChangeTask(event);
+ }
+ return new IndexChangeTask(event);
+ })
.ifPresent(
task -> {
if (queuedTasks.add(task)) {
@@ -164,6 +171,37 @@
}
}
+ class BatchIndexChangeTask extends IndexTask {
+ private final ChangeIndexEvent changeIndexEvent;
+
+ BatchIndexChangeTask(ChangeIndexEvent changeIndexEvent) {
+ this.changeIndexEvent = changeIndexEvent;
+ }
+
+ @Override
+ public void execute() {
+ forwarders.forEach(f -> f.batchIndex(changeIndexEvent));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ IndexChangeTask that = (IndexChangeTask) o;
+ return Objects.equal(changeIndexEvent, that.changeIndexEvent);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(changeIndexEvent);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Index change %s in target instance", changeIndexEvent.changeId);
+ }
+ }
+
class IndexAccountTask extends IndexTask {
private final AccountIndexEvent accountIndexEvent;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ProjectCheckerImpl.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ProjectCheckerImpl.java
index 548ccc0..d9851f3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ProjectCheckerImpl.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ProjectCheckerImpl.java
@@ -28,6 +28,6 @@
@Override
public boolean isProjectUpToDate(Project.NameKey projectName) {
- return projectCache.get(projectName) != null;
+ return projectCache.get(projectName).isPresent();
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
index b6f4033..f831671 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
@@ -15,76 +15,141 @@
package com.googlesource.gerrit.plugins.multisite.validation;
import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
+import com.google.common.base.Preconditions;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.replication.ReplicationPushFilter;
+import java.io.IOException;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class MultisiteReplicationPushFilter implements ReplicationPushFilter {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String REF_META_SUFFIX = "/meta";
+ public static final int MIN_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS = 1000;
+ public static final int RANDOM_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS = 1000;
+
static final String REPLICATION_LOG_NAME = "replication_log";
static final Logger repLog = LoggerFactory.getLogger(REPLICATION_LOG_NAME);
private final SharedRefDatabaseWrapper sharedRefDb;
+ private final GitRepositoryManager gitRepositoryManager;
@Inject
- public MultisiteReplicationPushFilter(SharedRefDatabaseWrapper sharedRefDb) {
+ public MultisiteReplicationPushFilter(
+ SharedRefDatabaseWrapper sharedRefDb, GitRepositoryManager gitRepositoryManager) {
this.sharedRefDb = sharedRefDb;
+ this.gitRepositoryManager = gitRepositoryManager;
}
@Override
public List<RemoteRefUpdate> filter(String projectName, List<RemoteRefUpdate> remoteUpdatesList) {
Set<String> outdatedChanges = new HashSet<>();
- List<RemoteRefUpdate> filteredRefUpdates =
- remoteUpdatesList.stream()
- .filter(
- refUpdate -> {
- String ref = refUpdate.getSrcRef();
- try {
- if (sharedRefDb.isUpToDate(
- Project.nameKey(projectName),
- new ObjectIdRef.Unpeeled(
- Ref.Storage.NETWORK, ref, refUpdate.getNewObjectId()))) {
- return true;
+ try (Repository repository =
+ gitRepositoryManager.openRepository(Project.nameKey(projectName))) {
+ List<RemoteRefUpdate> filteredRefUpdates =
+ remoteUpdatesList.stream()
+ .filter(
+ refUpdate -> {
+ boolean refUpToDate = isUpToDateWithRetry(projectName, repository, refUpdate);
+ if (!refUpToDate) {
+ repLog.warn(
+ "{} is not up-to-date with the shared-refdb and thus will NOT BE replicated",
+ refUpdate);
+ if (refUpdate.getSrcRef().endsWith(REF_META_SUFFIX)) {
+ outdatedChanges.add(getRootChangeRefPrefix(refUpdate.getSrcRef()));
+ }
}
- repLog.warn(
- "{} is not up-to-date with the shared-refdb and thus will NOT BE replicated",
- refUpdate);
- } catch (GlobalRefDbLockException e) {
- repLog.warn(
- "{} is locked on shared-refdb and thus will NOT BE replicated", refUpdate);
- }
- if (ref.endsWith(REF_META_SUFFIX)) {
- outdatedChanges.add(getRootChangeRefPrefix(ref));
- }
- return false;
- })
- .collect(Collectors.toList());
+ return refUpToDate;
+ })
+ .collect(Collectors.toList());
- return filteredRefUpdates.stream()
- .filter(
- refUpdate -> {
- if (outdatedChanges.contains(changePrefix(refUpdate.getSrcRef()))) {
- repLog.warn(
- "{} belongs to an outdated /meta ref and thus will NOT BE replicated",
- refUpdate);
- return false;
- }
- return true;
- })
- .collect(Collectors.toList());
+ return filteredRefUpdates.stream()
+ .filter(
+ refUpdate -> {
+ if (outdatedChanges.contains(changePrefix(refUpdate.getSrcRef()))) {
+ repLog.warn(
+ "{} belongs to an outdated /meta ref and thus will NOT BE replicated",
+ refUpdate);
+ return false;
+ }
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ } catch (IOException ioe) {
+ String message = String.format("Error while opening project: '%s'", projectName);
+ repLog.error(message);
+ logger.atSevere().withCause(ioe).log(message);
+ return Collections.emptyList();
+ }
+ }
+
+ private boolean isUpToDateWithRetry(
+ String projectName, Repository repository, RemoteRefUpdate refUpdate) {
+ String ref = refUpdate.getSrcRef();
+ try {
+ if (sharedRefDb.isUpToDate(
+ Project.nameKey(projectName),
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, ref, refUpdate.getNewObjectId()))) {
+ return true;
+ }
+
+ randomSleepForMitigatingConditionWhereLocalRefHaveJustBeenChanged(
+ projectName, refUpdate, ref);
+
+ return sharedRefDb.isUpToDate(
+ Project.nameKey(projectName),
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, ref, getNotNullExactRef(repository, ref)));
+ } catch (GlobalRefDbLockException gle) {
+ String message =
+ String.format("%s is locked on shared-refdb and thus will NOT BE replicated", ref);
+ repLog.error(message);
+ logger.atSevere().withCause(gle).log(message);
+ return false;
+ } catch (IOException ioe) {
+ String message =
+ String.format("Error while extracting ref '%s' for project '%s'", ref, projectName);
+ repLog.error(message);
+ logger.atSevere().withCause(ioe).log(message);
+ return false;
+ }
+ }
+
+ private void randomSleepForMitigatingConditionWhereLocalRefHaveJustBeenChanged(
+ String projectName, RemoteRefUpdate refUpdate, String ref) {
+ int randomSleepTimeMsec =
+ MIN_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS
+ + new Random().nextInt(RANDOM_WAIT_BEFORE_RELOAD_LOCAL_VERSION_MS);
+ repLog.debug(
+ String.format(
+ "'%s' is not up-to-date for project '%s' [local='%s']. Reload local ref in '%d ms' and re-check",
+ ref, projectName, refUpdate.getNewObjectId(), randomSleepTimeMsec));
+ try {
+ Thread.sleep(randomSleepTimeMsec);
+ } catch (InterruptedException ie) {
+ String message =
+ String.format("Error while waiting for next check for '%s', ref '%s'", projectName, ref);
+ repLog.error(message);
+ logger.atWarning().withCause(ie).log(message);
+ }
}
private String changePrefix(String changeRef) {
@@ -106,4 +171,10 @@
return changeMetaRef;
}
+
+ private ObjectId getNotNullExactRef(Repository repository, String refName) throws IOException {
+ Ref ref = repository.exactRef(refName);
+ Preconditions.checkNotNull(ref);
+ return ref.getObjectId();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
new file mode 100644
index 0000000..73bab49
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
@@ -0,0 +1,304 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
+
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
+import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.EventListener;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.notedb.IntBlob;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
+
+@Singleton
+public class ProjectVersionRefUpdate implements EventListener {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final Set<RefUpdate.Result> SUCCESSFUL_RESULTS =
+ ImmutableSet.of(RefUpdate.Result.NEW, RefUpdate.Result.FORCED, RefUpdate.Result.NO_CHANGE);
+
+ public static final String MULTI_SITE_VERSIONING_REF = "refs/multi-site/version";
+ public static final String MULTI_SITE_VERSIONING_VALUE_REF = "refs/multi-site/version/value";
+ public static final Ref NULL_PROJECT_VERSION_REF =
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId());
+
+ private final GitRepositoryManager gitRepositoryManager;
+ private final GitReferenceUpdated gitReferenceUpdated;
+ private final ProjectVersionLogger verLogger;
+
+ protected final SharedRefDatabaseWrapper sharedRefDb;
+
+ @Inject
+ public ProjectVersionRefUpdate(
+ GitRepositoryManager gitRepositoryManager,
+ SharedRefDatabaseWrapper sharedRefDb,
+ GitReferenceUpdated gitReferenceUpdated,
+ ProjectVersionLogger verLogger) {
+ this.gitRepositoryManager = gitRepositoryManager;
+ this.sharedRefDb = sharedRefDb;
+ this.gitReferenceUpdated = gitReferenceUpdated;
+ this.verLogger = verLogger;
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ logger.atFine().log("Processing event type: " + event.type);
+ // Producer of the Event use RefUpdatedEvent to trigger the version update
+ if (!Context.isForwardedEvent() && event instanceof RefUpdatedEvent) {
+ updateProducerProjectVersionUpdate((RefUpdatedEvent) event);
+ }
+ }
+
+ private boolean isSpecialRefName(String refName) {
+ return refName.startsWith(RefNames.REFS_SEQUENCES)
+ || refName.startsWith(RefNames.REFS_STARRED_CHANGES)
+ || refName.equals(MULTI_SITE_VERSIONING_REF);
+ }
+
+ private void updateProducerProjectVersionUpdate(RefUpdatedEvent refUpdatedEvent) {
+ String refName = refUpdatedEvent.getRefName();
+
+ if (isSpecialRefName(refName)) {
+ logger.atFine().log(
+ "Found a special ref name %s, skipping update for %s",
+ refName, refUpdatedEvent.getProjectNameKey().get());
+ return;
+ }
+ try {
+ Project.NameKey projectNameKey = refUpdatedEvent.getProjectNameKey();
+ long newVersion = getCurrentGlobalVersionNumber();
+
+ Optional<RefUpdate> newProjectVersionRefUpdate =
+ updateLocalProjectVersion(projectNameKey, newVersion);
+
+ if (newProjectVersionRefUpdate.isPresent()) {
+ verLogger.log(projectNameKey, newVersion, 0L);
+
+ if (updateSharedProjectVersion(
+ projectNameKey, newProjectVersionRefUpdate.get().getNewObjectId(), newVersion)) {
+ gitReferenceUpdated.fire(projectNameKey, newProjectVersionRefUpdate.get(), null);
+ }
+ } else {
+ logger.atWarning().log(
+ "Ref %s not found on projet %s: skipping project version update",
+ refUpdatedEvent.getRefName(), projectNameKey);
+ }
+ } catch (LocalProjectVersionUpdateException | SharedProjectVersionUpdateException e) {
+ logger.atSevere().withCause(e).log(
+ "Issue encountered when updating version for project "
+ + refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private RefUpdate getProjectVersionRefUpdate(Repository repository, Long version)
+ throws IOException {
+ RefUpdate refUpdate = repository.getRefDatabase().newUpdate(MULTI_SITE_VERSIONING_REF, false);
+ refUpdate.setNewObjectId(getNewId(repository, version));
+ refUpdate.setForceUpdate(true);
+ return refUpdate;
+ }
+
+ private ObjectId getNewId(Repository repository, Long version) throws IOException {
+ ObjectInserter ins = repository.newObjectInserter();
+ ObjectId newId = ins.insert(OBJ_BLOB, Long.toString(version).getBytes(UTF_8));
+ ins.flush();
+ return newId;
+ }
+
+ private boolean updateSharedProjectVersion(
+ Project.NameKey projectNameKey, ObjectId newObjectId, Long newVersion)
+ throws SharedProjectVersionUpdateException {
+
+ Ref sharedRef =
+ sharedRefDb
+ .get(projectNameKey, MULTI_SITE_VERSIONING_REF, String.class)
+ .map(
+ (String objectId) ->
+ new ObjectIdRef.Unpeeled(
+ Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.fromString(objectId)))
+ .orElse(
+ new ObjectIdRef.Unpeeled(
+ Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId()));
+ Optional<Long> sharedVersion =
+ sharedRefDb
+ .get(projectNameKey, MULTI_SITE_VERSIONING_VALUE_REF, String.class)
+ .map(Long::parseLong);
+
+ try {
+ if (sharedVersion.isPresent() && sharedVersion.get() >= newVersion) {
+ logger.atWarning().log(
+ String.format(
+ "NOT Updating project %s version %s (value=%d) in shared ref-db because is more recent than the local one %s (value=%d) ",
+ projectNameKey.get(),
+ newObjectId,
+ newVersion,
+ sharedRef.getObjectId().getName(),
+ sharedVersion.get()));
+ return false;
+ }
+
+ logger.atFine().log(
+ String.format(
+ "Updating shared project %s version to %s (value=%d)",
+ projectNameKey.get(), newObjectId, newVersion));
+
+ boolean success = sharedRefDb.compareAndPut(projectNameKey, sharedRef, newObjectId);
+ if (!success) {
+ String message =
+ String.format(
+ "Project version blob update failed for %s. Current value %s, new value: %s",
+ projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId);
+ logger.atSevere().log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+
+ success =
+ sharedRefDb.compareAndPut(
+ projectNameKey,
+ MULTI_SITE_VERSIONING_VALUE_REF,
+ sharedVersion.map(Object::toString).orElse(null),
+ newVersion.toString());
+ if (!success) {
+ String message =
+ String.format(
+ "Project version update failed for %s. Current value %s, new value: %s",
+ projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId);
+ logger.atSevere().log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+
+ return true;
+ } catch (GlobalRefDbSystemError refDbSystemError) {
+ String message =
+ String.format(
+ "Error while updating shared project version for %s. Current value %s, new value: %s. Error: %s",
+ projectNameKey.get(),
+ sharedRef.getObjectId(),
+ newObjectId,
+ refDbSystemError.getMessage());
+ logger.atSevere().withCause(refDbSystemError).log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+ }
+
+ public Optional<Long> getProjectLocalVersion(String projectName) {
+ try (Repository repository =
+ gitRepositoryManager.openRepository(Project.NameKey.parse(projectName))) {
+ Optional<IntBlob> blob = IntBlob.parse(repository, MULTI_SITE_VERSIONING_REF);
+ if (blob.isPresent()) {
+ Long repoVersion = Integer.toUnsignedLong(blob.get().value());
+ logger.atFine().log("Local project '%s' has version %d", projectName, repoVersion);
+ return Optional.of(repoVersion);
+ }
+ } catch (RepositoryNotFoundException re) {
+ logger.atFine().log("Project '%s' not found", projectName);
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Cannot read local project '%s' version", projectName);
+ }
+ return Optional.empty();
+ }
+
+ public Optional<Long> getProjectRemoteVersion(String projectName) {
+ Optional<String> globalVersion =
+ sharedRefDb.get(
+ Project.NameKey.parse(projectName), MULTI_SITE_VERSIONING_VALUE_REF, String.class);
+ return globalVersion.flatMap(longString -> getLongValueOf(longString));
+ }
+
+ private Object safeGetObjectId(Ref currentRef) {
+ return currentRef == null ? "null" : currentRef.getObjectId();
+ }
+
+ private Optional<Long> getLongValueOf(String longString) {
+ try {
+ return Optional.ofNullable(Long.parseLong(longString));
+ } catch (NumberFormatException e) {
+ logger.atSevere().withCause(e).log(
+ "Unable to parse timestamp value %s into Long", longString);
+ return Optional.empty();
+ }
+ }
+
+ private Optional<RefUpdate> updateLocalProjectVersion(
+ Project.NameKey projectNameKey, long newVersionNumber)
+ throws LocalProjectVersionUpdateException {
+ logger.atFine().log(
+ "Updating local version for project %s with version %d",
+ projectNameKey.get(), newVersionNumber);
+ try (Repository repository = gitRepositoryManager.openRepository(projectNameKey)) {
+ RefUpdate refUpdate = getProjectVersionRefUpdate(repository, newVersionNumber);
+ RefUpdate.Result result = refUpdate.update();
+ if (!isSuccessful(result)) {
+ String message =
+ String.format(
+ "RefUpdate failed with result %s for: project=%s, version=%d",
+ result.name(), projectNameKey.get(), newVersionNumber);
+ logger.atSevere().log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+
+ return Optional.of(refUpdate);
+ } catch (IOException e) {
+ String message = "Cannot create versioning command for " + projectNameKey.get();
+ logger.atSevere().withCause(e).log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ }
+
+ private long getCurrentGlobalVersionNumber() {
+ return System.currentTimeMillis() / 1000;
+ }
+
+ private Boolean isSuccessful(RefUpdate.Result result) {
+ return SUCCESSFUL_RESULTS.contains(result);
+ }
+
+ public static class LocalProjectVersionUpdateException extends Exception {
+ private static final long serialVersionUID = 7649956232401457023L;
+
+ public LocalProjectVersionUpdateException(String projectName) {
+ super("Cannot update local project version of " + projectName);
+ }
+ }
+
+ public static class SharedProjectVersionUpdateException extends Exception {
+ private static final long serialVersionUID = -9153858177700286314L;
+
+ public SharedProjectVersionUpdateException(String projectName) {
+ super("Cannot update shared project version of " + projectName);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
index 312562c..dcc9030 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
@@ -86,7 +86,8 @@
public RefUpdate.Result executeRefUpdate(
RefUpdate refUpdate, NoParameterFunction<RefUpdate.Result> refUpdateFunction)
throws IOException {
- if (refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
+ if (isProjectVersionUpdate(refUpdate.getName())
+ || refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
return refUpdateFunction.invoke();
}
@@ -105,6 +106,13 @@
return null;
}
+ private Boolean isProjectVersionUpdate(String refName) {
+ Boolean isProjectVersionUpdate =
+ refName.equals(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ logger.atFine().log("Is project version update? " + isProjectVersionUpdate);
+ return isProjectVersionUpdate;
+ }
+
private <T extends Throwable> void softFailBasedOnEnforcement(T e, EnforcePolicy policy)
throws T {
logger.atWarning().withCause(e).log(
@@ -153,6 +161,11 @@
sharedRefDb.compareAndPut(
Project.nameKey(projectName), refPair.compareRef, refPair.putValue);
} catch (GlobalRefDbSystemError e) {
+ logger.atWarning().withCause(e).log(
+ "Not able to persist the data in Zookeeper for project '{}' and ref '{}', message: {}",
+ projectName,
+ refPair.getName(),
+ e.getMessage());
throw new SharedDbSplitBrainException(errorMessage, e);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
index 1719f38..481d288 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
@@ -21,7 +21,9 @@
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.LockWrapper;
+import com.googlesource.gerrit.plugins.multisite.Log4jProjectVersionLogger;
import com.googlesource.gerrit.plugins.multisite.Log4jSharedRefLogger;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.SharedRefLogger;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
@@ -45,6 +47,7 @@
bind(SharedRefDatabaseWrapper.class).in(Scopes.SINGLETON);
bind(SharedRefLogger.class).to(Log4jSharedRefLogger.class);
+ bind(ProjectVersionLogger.class).to(Log4jProjectVersionLogger.class);
factory(LockWrapper.Factory.class);
factory(MultiSiteRepository.Factory.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
index b82d7d9..1530838 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
@@ -18,6 +18,7 @@
import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
import com.google.gerrit.entities.Project;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
@@ -35,6 +36,12 @@
}
@Override
+ public <T> boolean compareAndPut(Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
+ return false;
+ }
+
+ @Override
public AutoCloseable lockRef(Project.NameKey project, String refName)
throws GlobalRefDbLockException {
return () -> {};
@@ -47,4 +54,10 @@
@Override
public void remove(Project.NameKey project) throws GlobalRefDbSystemError {}
+
+ @Override
+ public <T> Optional<T> get(Project.NameKey project, String refName, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return Optional.empty();
+ }
}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index 42249b1..b6de0a9 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -93,10 +93,6 @@
`metric=plugins/multi-site/multi_site/subscriber/subscriber_message_consumer_failure_counter/subscriber_msg_consumer_poll_failure_counter, type=com.codahale.metrics.Meter`
-* Subscriber replication status (latest replication Epoch time in seconds) per instance
+* Subscriber replication lag (sec behind the producer)
-`metric=plugins/multi-site/multi_site/subscriber/subscriber_replication_status/instance_latest_replication_epochtime_secs, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
-
-* Subscriber replication status (latest replication Epoch time in seconds) per project
-
-`metric=plugins/multi-site/multi_site/subscriber/subscriber_replication_status/latest_replication_epochtime_secs_<projectName>, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
+`metric=site/multi_site/subscriber/subscriber_replication_status/sec_behind, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
diff --git a/src/test/README.md b/src/test/README.md
new file mode 100644
index 0000000..0fe89c6
--- /dev/null
+++ b/src/test/README.md
@@ -0,0 +1,8 @@
+# About this directory structure
+
+Refer to ../../dockerised_local_env/README.md for more about these directory structures:
+
+```bash
+ ./resources/com
+ ./scala
+```
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java
new file mode 100644
index 0000000..99148fa
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java
@@ -0,0 +1,98 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.consumer;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+import com.google.common.base.Suppliers;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.metrics.MetricMaker;
+import com.google.gerrit.server.data.RefUpdateAttribute;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
+import java.util.Optional;
+import java.util.UUID;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SubscriberMetricsTest {
+ private static final String A_TEST_PROJECT_NAME = "test-project";
+ private static final Project.NameKey A_TEST_PROJECT_NAME_KEY =
+ Project.nameKey(A_TEST_PROJECT_NAME);
+
+ @Mock private SharedRefDatabaseWrapper sharedRefDb;
+ @Mock private GitReferenceUpdated gitReferenceUpdated;
+ @Mock private MetricMaker metricMaker;
+ @Mock private ProjectVersionLogger verLogger;
+ @Mock private ProjectVersionRefUpdate projectVersionRefUpdate;
+ private SubscriberMetrics metrics;
+ private EventMessage.Header msgHeader;
+
+ @Before
+ public void setup() throws Exception {
+ msgHeader = new EventMessage.Header(UUID.randomUUID(), UUID.randomUUID());
+ metrics = new SubscriberMetrics(metricMaker, projectVersionRefUpdate, verLogger);
+ }
+
+ @Test
+ public void shouldLogProjectVersionWhenReceivingRefUpdatedEventWithoutLag() {
+ Optional<Long> globalRefDbVersion = Optional.of(System.currentTimeMillis() / 1000);
+ when(projectVersionRefUpdate.getProjectRemoteVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+ when(projectVersionRefUpdate.getProjectLocalVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+
+ EventMessage eventMessage = new EventMessage(msgHeader, newRefUpdateEvent());
+
+ metrics.updateReplicationStatusMetrics(eventMessage);
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, globalRefDbVersion.get(), 0);
+ }
+
+ @Test
+ public void shouldLogProjectVersionWhenReceivingRefUpdatedEventWithALag() {
+ Optional<Long> globalRefDbVersion = Optional.of(System.currentTimeMillis() / 1000);
+ long replicationLag = 60;
+ when(projectVersionRefUpdate.getProjectRemoteVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion.map(ts -> ts + replicationLag));
+ when(projectVersionRefUpdate.getProjectLocalVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+
+ EventMessage eventMessage = new EventMessage(msgHeader, newRefUpdateEvent());
+
+ metrics.updateReplicationStatusMetrics(eventMessage);
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, globalRefDbVersion.get(), replicationLag);
+ }
+
+ private RefUpdatedEvent newRefUpdateEvent() {
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.project = A_TEST_PROJECT_NAME;
+ refUpdate.refName = "refs/heads/foo";
+ refUpdate.newRev = "591727cfec5174368a7829f79741c41683d84c89";
+ RefUpdatedEvent refUpdateEvent = new RefUpdatedEvent();
+ refUpdateEvent.refUpdate = Suppliers.ofInstance(refUpdate);
+ return refUpdateEvent;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
index c919df8..a632b74 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
@@ -46,4 +46,13 @@
verify(cacheEvictionHandler).evict(CacheEntry.from(event.cacheName, event.key));
}
+
+ @Test
+ public void routerShouldSendEventsToTheAppropriateHandler_ProjectCacheEvictionWithSlash()
+ throws Exception {
+ final CacheEvictionEvent event = new CacheEvictionEvent("cache", "some/project");
+ router.route(event);
+
+ verify(cacheEvictionHandler).evict(CacheEntry.from(event.cacheName, event.key));
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
index 923abb2..31ca13d 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
@@ -19,6 +19,8 @@
import static org.mockito.Mockito.verifyZeroInteractions;
import com.google.gerrit.entities.Account;
+import com.google.gerrit.server.config.AllUsersName;
+import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedEventHandler;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexAccountHandler;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexChangeHandler;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexGroupHandler;
@@ -30,6 +32,8 @@
import com.googlesource.gerrit.plugins.multisite.forwarder.events.IndexEvent;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ProjectIndexEvent;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.IndexEventRouter;
+import com.googlesource.gerrit.plugins.multisite.forwarder.router.StreamEventRouter;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
import java.util.Optional;
import org.junit.Before;
import org.junit.Test;
@@ -45,12 +49,18 @@
@Mock private ForwardedIndexChangeHandler indexChangeHandler;
@Mock private ForwardedIndexGroupHandler indexGroupHandler;
@Mock private ForwardedIndexProjectHandler indexProjectHandler;
+ @Mock private ForwardedEventHandler forwardedEventHandler;
+ private AllUsersName allUsersName = new AllUsersName("All-Users");
@Before
public void setUp() {
router =
new IndexEventRouter(
- indexAccountHandler, indexChangeHandler, indexGroupHandler, indexProjectHandler);
+ indexAccountHandler,
+ indexChangeHandler,
+ indexGroupHandler,
+ indexProjectHandler,
+ allUsersName);
}
@Test
@@ -59,15 +69,30 @@
router.route(event);
verify(indexAccountHandler)
- .index(
- Account.id(event.accountId),
- ForwardedIndexingHandler.Operation.INDEX,
- Optional.of(event));
+ .indexAsync(Account.id(event.accountId), ForwardedIndexingHandler.Operation.INDEX);
verifyZeroInteractions(indexChangeHandler, indexGroupHandler, indexProjectHandler);
}
@Test
+ public void streamEventRouterShouldTriggerAccountIndexFlush() throws Exception {
+
+ StreamEventRouter streamEventRouter = new StreamEventRouter(forwardedEventHandler, router);
+
+ final AccountIndexEvent event = new AccountIndexEvent(1);
+ router.route(event);
+
+ verify(indexAccountHandler)
+ .indexAsync(Account.id(event.accountId), ForwardedIndexingHandler.Operation.INDEX);
+
+ verifyZeroInteractions(indexChangeHandler, indexGroupHandler, indexProjectHandler);
+
+ streamEventRouter.route(new RefReplicationDoneEvent(allUsersName.get(), "refs/any", 1));
+
+ verify(indexAccountHandler).doAsyncIndex();
+ }
+
+ @Test
public void routerShouldSendEventsToTheAppropriateHandler_GroupIndex() throws Exception {
final String groupId = "12";
final GroupIndexEvent event = new GroupIndexEvent(groupId);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
index 605eb45..3b4bec5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
@@ -22,6 +22,7 @@
import com.google.gerrit.server.events.CommentAddedEvent;
import com.google.gerrit.server.util.time.TimeUtil;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedEventHandler;
+import com.googlesource.gerrit.plugins.multisite.forwarder.router.IndexEventRouter;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.StreamEventRouter;
import org.junit.Before;
import org.junit.Test;
@@ -34,10 +35,11 @@
private StreamEventRouter router;
@Mock private ForwardedEventHandler streamEventHandler;
+ @Mock private IndexEventRouter indexEventRouter;
@Before
public void setUp() {
- router = new StreamEventRouter(streamEventHandler);
+ router = new StreamEventRouter(streamEventHandler, indexEventRouter);
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParserTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParserTest.java
index 19ae621..12bdb74 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParserTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/GsonParserTest.java
@@ -53,8 +53,7 @@
@Test
public void stringParse() {
String key = "key";
- String json = gson.toJson(key);
- assertThat(key).isEqualTo(gsonParser.fromJson(Constants.PROJECTS, json));
+ assertThat(key).isEqualTo(gsonParser.fromJson(Constants.PROJECTS, key));
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
index 37d5008..047a1c5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
@@ -33,4 +33,7 @@
@Override
public void logLockRelease(String project, String refName) {}
+
+ @Override
+ public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
index 46eb424..fcbafbb 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
@@ -14,7 +14,6 @@
package com.googlesource.gerrit.plugins.multisite.validation;
-import static com.google.gerrit.testing.GerritJUnit.assertThrows;
import static java.util.Arrays.asList;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
new file mode 100644
index 0000000..2eefb5a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
@@ -0,0 +1,266 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF;
+import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.google.gerrit.server.project.ProjectConfig;
+import com.google.gerrit.testing.InMemoryRepositoryManager;
+import com.google.gerrit.testing.InMemoryTestEnvironment;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProjectVersionRefUpdateTest implements RefFixture {
+
+ @Rule public InMemoryTestEnvironment testEnvironment = new InMemoryTestEnvironment();
+
+ @Mock RefUpdatedEvent refUpdatedEvent;
+ @Mock SharedRefDatabaseWrapper sharedRefDb;
+ @Mock GitReferenceUpdated gitReferenceUpdated;
+ @Mock ProjectVersionLogger verLogger;
+
+ @Inject private ProjectConfig.Factory projectConfigFactory;
+ @Inject private InMemoryRepositoryManager repoManager;
+
+ private TestRepository<InMemoryRepository> repo;
+ private ProjectConfig project;
+ private RevCommit masterCommit;
+
+ @Before
+ public void setUp() throws Exception {
+ InMemoryRepository inMemoryRepo = repoManager.createRepository(A_TEST_PROJECT_NAME_KEY);
+ project = projectConfigFactory.create(A_TEST_PROJECT_NAME_KEY);
+ project.load(inMemoryRepo);
+ repo = new TestRepository<>(inMemoryRepo);
+ masterCommit = repo.branch("master").commit().create();
+ }
+
+ @After
+ public void tearDown() {
+ Context.unsetForwardedEvent();
+ }
+
+ @Test
+ public void producerShouldUpdateProjectVersionUponRefUpdatedEvent() throws IOException {
+ Context.setForwardedEvent(false);
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63"));
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1)));
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldUpdateProjectVersionUponForcedPushRefUpdatedEvent() throws Exception {
+ Context.setForwardedEvent(false);
+
+ Thread.sleep(1000L);
+ RevCommit masterPlusOneCommit = repo.branch("master").commit().create();
+
+ Thread.sleep(1000L);
+ repo.branch("master").update(masterCommit);
+
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63"));
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1)));
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterPlusOneCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldCreateNewProjectVersionWhenMissingUponRefUpdatedEvent()
+ throws IOException {
+ Context.setForwardedEvent(false);
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.empty());
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.empty());
+
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), isNull(), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldNotUpdateProjectVersionUponSequenceRefUpdatedEvent() throws Exception {
+ producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(RefNames.REFS_SEQUENCES);
+ }
+
+ @Test
+ public void producerShouldNotUpdateProjectVersionUponStarredChangesRefUpdatedEvent()
+ throws Exception {
+ producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(RefNames.REFS_STARRED_CHANGES);
+ }
+
+ private void producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(String magicRefPrefix)
+ throws Exception {
+ String magicRefName = magicRefPrefix + "/foo";
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(magicRefName);
+ repo.branch(magicRefName).commit().create();
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+
+ verifyZeroInteractions(verLogger);
+ }
+
+ @Test
+ public void shouldNotUpdateProjectVersionWhenProjectDoesntExist() throws IOException {
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(Project.nameKey("aNonExistentProject"));
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+
+ verifyZeroInteractions(verLogger);
+ }
+
+ @Test
+ public void getRemoteProjectVersionShouldReturnCorrectValue() {
+ when(sharedRefDb.get(A_TEST_PROJECT_NAME_KEY, MULTI_SITE_VERSIONING_VALUE_REF, String.class))
+ .thenReturn(Optional.of("123"));
+
+ Optional<Long> version =
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .getProjectRemoteVersion(A_TEST_PROJECT_NAME);
+
+ assertThat(version.isPresent()).isTrue();
+ assertThat(version.get()).isEqualTo(123L);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
index eefb685..de93545 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
@@ -23,6 +23,7 @@
import com.google.gerrit.entities.Project;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.SharedRefLogger;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedDbSplitBrainException;
@@ -44,6 +45,8 @@
@Mock SharedRefDatabaseWrapper sharedRefDb;
+ @Mock SharedRefLogger sharedRefLogger;
+
@Mock RefDatabase localRefDb;
@Mock ValidationMetrics validationMetrics;
@@ -71,14 +74,17 @@
doReturn(refName).when(refUpdate).getName();
lenient().doReturn(oldUpdateRef.getObjectId()).when(refUpdate).getOldObjectId();
- refUpdateValidator =
- new RefUpdateValidator(
- sharedRefDb,
- validationMetrics,
- defaultRefEnforcement,
- new DummyLockWrapper(),
- A_TEST_PROJECT_NAME,
- localRefDb);
+ refUpdateValidator = newRefUpdateValidator(sharedRefDb);
+ }
+
+ @Test
+ public void validationShouldSucceedWhenSharedRefDbIsNoop() throws Exception {
+ SharedRefDatabaseWrapper noopSharedRefDbWrapper = new SharedRefDatabaseWrapper(sharedRefLogger);
+
+ Result result =
+ newRefUpdateValidator(noopSharedRefDbWrapper)
+ .executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
+ assertThat(result).isEqualTo(RefUpdate.Result.NEW);
}
@Test
@@ -186,4 +192,14 @@
.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
assertThat(result).isEqualTo(RefUpdate.Result.LOCK_FAILURE);
}
+
+ private RefUpdateValidator newRefUpdateValidator(SharedRefDatabaseWrapper refDbWrapper) {
+ return new RefUpdateValidator(
+ refDbWrapper,
+ validationMetrics,
+ defaultRefEnforcement,
+ new DummyLockWrapper(),
+ A_TEST_PROJECT_NAME,
+ localRefDb);
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
index 72e6431..7e4593b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
@@ -18,36 +18,61 @@
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase;
import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.testing.InMemoryRepositoryManager;
+import com.google.gerrit.testing.InMemoryTestEnvironment;
+import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.DisabledSharedRefLogger;
import com.googlesource.gerrit.plugins.multisite.validation.MultisiteReplicationPushFilter;
-import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
+import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.transport.RemoteRefUpdate;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
-public class MultisiteReplicationPushFilterTest {
+public class MultisiteReplicationPushFilterTest extends LocalDiskRepositoryTestCase
+ implements RefFixture {
+
+ @Rule public InMemoryTestEnvironment testEnvironment = new InMemoryTestEnvironment();
@Mock SharedRefDatabaseWrapper sharedRefDatabaseMock;
- String project = "fooProject";
- Project.NameKey projectName = Project.nameKey(project);
+ @Inject private InMemoryRepositoryManager gitRepositoryManager;
+
+ String project = A_TEST_PROJECT_NAME;
+ Project.NameKey projectName = A_TEST_PROJECT_NAME_KEY;
+
+ private TestRepository<InMemoryRepository> repo;
+
+ @Before
+ public void setUp() throws Exception {
+ InMemoryRepository inMemoryRepo =
+ gitRepositoryManager.createRepository(A_TEST_PROJECT_NAME_KEY);
+ repo = new TestRepository<>(inMemoryRepo);
+ }
@Test
public void shouldReturnAllRefUpdatesWhenAllUpToDate() throws Exception {
@@ -56,7 +81,7 @@
doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
MultisiteReplicationPushFilter pushFilter =
- new MultisiteReplicationPushFilter(sharedRefDatabaseMock);
+ new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
List<RemoteRefUpdate> filteredRefUpdates = pushFilter.filter(project, refUpdates);
assertThat(filteredRefUpdates).containsExactlyElementsIn(refUpdates);
@@ -70,13 +95,42 @@
SharedRefDatabaseWrapper sharedRefDatabase = newSharedRefDatabase(outdatedRef.getSrcRef());
MultisiteReplicationPushFilter pushFilter =
- new MultisiteReplicationPushFilter(sharedRefDatabase);
+ new MultisiteReplicationPushFilter(sharedRefDatabase, gitRepositoryManager);
List<RemoteRefUpdate> filteredRefUpdates = pushFilter.filter(project, refUpdates);
assertThat(filteredRefUpdates).containsExactly(refUpToDate);
}
@Test
+ public void shouldLoadLocalVersionAndNotFilter() throws Exception {
+ RemoteRefUpdate temporaryOutdated = refUpdate("refs/heads/temporaryOutdated");
+ List<RemoteRefUpdate> refUpdates = Collections.singletonList(temporaryOutdated);
+ doReturn(false).doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
+
+ MultisiteReplicationPushFilter pushFilter =
+ new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ List<RemoteRefUpdate> filteredRefUpdates = pushFilter.filter(project, refUpdates);
+
+ assertThat(filteredRefUpdates).containsExactly(temporaryOutdated);
+ verify(sharedRefDatabaseMock, times(2)).isUpToDate(any(), any());
+ }
+
+ @Test
+ public void shouldLoadLocalVersionAndFilter() throws Exception {
+ RemoteRefUpdate temporaryOutdated = refUpdate("refs/heads/temporaryOutdated");
+ repo.branch("refs/heads/temporaryOutdated").commit().create();
+ List<RemoteRefUpdate> refUpdates = Collections.singletonList(temporaryOutdated);
+ doReturn(false).doReturn(false).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
+
+ MultisiteReplicationPushFilter pushFilter =
+ new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
+ List<RemoteRefUpdate> filteredRefUpdates = pushFilter.filter(project, refUpdates);
+
+ assertThat(filteredRefUpdates).isEmpty();
+ verify(sharedRefDatabaseMock, times(2)).isUpToDate(any(), any());
+ }
+
+ @Test
public void shouldFilterOutAllOutdatedChangesRef() throws Exception {
RemoteRefUpdate refUpToDate = refUpdate("refs/heads/uptodate");
RemoteRefUpdate refChangeUpToDate = refUpdate("refs/changes/25/1225/2");
@@ -87,7 +141,7 @@
SharedRefDatabaseWrapper sharedRefDatabase = newSharedRefDatabase(changeMetaRef.getSrcRef());
MultisiteReplicationPushFilter pushFilter =
- new MultisiteReplicationPushFilter(sharedRefDatabase);
+ new MultisiteReplicationPushFilter(sharedRefDatabase, gitRepositoryManager);
List<RemoteRefUpdate> filteredRefUpdates = pushFilter.filter(project, refUpdates);
assertThat(filteredRefUpdates).containsExactly(refUpToDate, refChangeUpToDate);
@@ -118,6 +172,13 @@
}
@Override
+ public <T> boolean compareAndPut(
+ Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
+ return false;
+ }
+
+ @Override
public AutoCloseable lockRef(Project.NameKey project, String refName)
throws GlobalRefDbLockException {
return null;
@@ -125,15 +186,22 @@
@Override
public void remove(Project.NameKey project) throws GlobalRefDbSystemError {}
+
+ @Override
+ public <T> Optional<T> get(Project.NameKey project, String refName, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return Optional.empty();
+ }
};
return new SharedRefDatabaseWrapper(
DynamicItem.itemOf(GlobalRefDatabase.class, sharedRefDatabase),
new DisabledSharedRefLogger());
}
- private RemoteRefUpdate refUpdate(String refName) throws IOException {
+ private RemoteRefUpdate refUpdate(String refName) throws Exception {
ObjectId srcObjId = ObjectId.fromString("0000000000000000000000000000000000000001");
Ref srcRef = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, refName, srcObjId);
+ repo.branch(refName).commit().create();
return new RemoteRefUpdate(null, srcRef, "origin", false, "origin", srcObjId);
}
}
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.json
new file mode 100644
index 0000000..34d4ca0
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.json
@@ -0,0 +1,6 @@
+[
+ {
+ "url": "http://HOSTNAME:HTTP_PORT1/_PROJECT",
+ "cmd": "clone"
+ }
+]
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit-body.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit-body.json
new file mode 100644
index 0000000..23bf26c
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit-body.json
@@ -0,0 +1,5 @@
+{
+ "project": "${project}",
+ "branch": "master",
+ "subject": "Change"
+}
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.json
new file mode 100644
index 0000000..c267ab3
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.json
@@ -0,0 +1,6 @@
+[
+ {
+ "url": "http://HOSTNAME:HTTP_PORT/a/changes/",
+ "project": "_PROJECT"
+ }
+]
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit-body.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit-body.json
new file mode 100644
index 0000000..bcf4708
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit-body.json
@@ -0,0 +1,3 @@
+{
+ "create_empty_commit": "true"
+}
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.json
new file mode 100644
index 0000000..40e5a45
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.json
@@ -0,0 +1,5 @@
+[
+ {
+ "url": "http://HOSTNAME:HTTP_PORT/a/projects/PROJECT"
+ }
+]
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.json
new file mode 100644
index 0000000..da1a058
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.json
@@ -0,0 +1,4 @@
+[
+ {
+ }
+]
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.json
new file mode 100644
index 0000000..4b7f52e
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.json
@@ -0,0 +1,6 @@
+[
+ {
+ "url": "http://HOSTNAME:HTTP_PORT1/a/changes/",
+ "number": "NUMBER"
+ }
+]
diff --git a/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.json b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.json
new file mode 100644
index 0000000..7cc8293
--- /dev/null
+++ b/src/test/resources/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.json
@@ -0,0 +1,5 @@
+[
+ {
+ "url": "http://HOSTNAME:HTTP_PORT/a/projects/PROJECT/delete-project~delete"
+ }
+]
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.scala
new file mode 100644
index 0000000..ba54314
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CloneUsingMultiGerrit1.scala
@@ -0,0 +1,59 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.GitSimulation
+import io.gatling.core.Predef.{atOnceUsers, _}
+import io.gatling.core.feeder.FeederBuilder
+import io.gatling.core.structure.ScenarioBuilder
+
+import scala.concurrent.duration._
+
+class CloneUsingMultiGerrit1 extends GitSimulation {
+ private val data: FeederBuilder = jsonFile(resource).convert(keys).queue
+ private var default: String = name
+
+ def this(default: String) {
+ this()
+ this.default = default
+ }
+
+ override def replaceOverride(in: String): String = {
+ val next = replaceProperty("http_port1", 8081, in)
+ replaceKeyWith("_project", default, next)
+ }
+
+ val test: ScenarioBuilder = scenario(unique)
+ .feed(data)
+ .exec(gitRequest)
+
+ private val createProject = new CreateProjectUsingMultiGerrit(default)
+ private val deleteProject = new DeleteProjectUsingMultiGerrit(default)
+
+ setUp(
+ createProject.test.inject(
+ nothingFor(stepWaitTime(createProject) seconds),
+ atOnceUsers(1)
+ ),
+ test.inject(
+ nothingFor(stepWaitTime(this) seconds),
+ atOnceUsers(1)
+ ),
+ deleteProject.test.inject(
+ nothingFor(createProject.maxExecutionTime + maxExecutionTime seconds),
+ atOnceUsers(1)
+ ),
+ ).protocols(gitProtocol, httpProtocol)
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.scala
new file mode 100644
index 0000000..f6e610b
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateChangeUsingMultiGerrit.scala
@@ -0,0 +1,64 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.GerritSimulation
+import io.gatling.core.Predef.{atOnceUsers, _}
+import io.gatling.core.feeder.FeederBuilder
+import io.gatling.core.structure.ScenarioBuilder
+import io.gatling.http.Predef._
+
+import scala.concurrent.duration._
+
+class CreateChangeUsingMultiGerrit extends GerritSimulation {
+ private val data: FeederBuilder = jsonFile(resource).convert(keys).queue
+ private val default: String = name
+ private val numberKey = "_number"
+
+ override def relativeRuntimeWeight = 10
+
+ private val test: ScenarioBuilder = scenario(unique)
+ .feed(data)
+ .exec(httpRequest
+ .body(ElFileBody(body)).asJson
+ .check(regex("\"" + numberKey + "\":(\\d+),").saveAs(numberKey)))
+ .exec(session => {
+ deleteChange.number = Some(session(numberKey).as[Int])
+ session
+ })
+
+ private val createProject = new CreateProjectUsingMultiGerrit(default)
+ private val deleteProject = new DeleteProjectUsingMultiGerrit(default)
+ private val deleteChange = new DeleteChangeUsingMultiGerrit1
+
+ setUp(
+ createProject.test.inject(
+ nothingFor(stepWaitTime(createProject) seconds),
+ atOnceUsers(1)
+ ),
+ test.inject(
+ nothingFor(stepWaitTime(this) seconds),
+ atOnceUsers(1)
+ ),
+ deleteChange.test.inject(
+ nothingFor(stepWaitTime(deleteChange) seconds),
+ atOnceUsers(1)
+ ).protocols(deleteChange.httpForReplica),
+ deleteProject.test.inject(
+ nothingFor(stepWaitTime(deleteProject) seconds),
+ atOnceUsers(1)
+ ),
+ ).protocols(httpProtocol)
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.scala
new file mode 100644
index 0000000..5fd3f41
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerrit.scala
@@ -0,0 +1,40 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.ProjectSimulation
+import io.gatling.core.Predef._
+import io.gatling.core.feeder.FeederBuilder
+import io.gatling.core.structure.ScenarioBuilder
+
+class CreateProjectUsingMultiGerrit extends ProjectSimulation {
+ private val data: FeederBuilder = jsonFile(resource).convert(keys).queue
+
+ override def relativeRuntimeWeight = 15
+
+ def this(default: String) {
+ this()
+ this.default = default
+ }
+
+ val test: ScenarioBuilder = scenario(unique)
+ .feed(data)
+ .exec(httpRequest.body(RawFileBody(body)).asJson)
+
+ setUp(
+ test.inject(
+ atOnceUsers(1)
+ )).protocols(httpProtocol)
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.scala
new file mode 100644
index 0000000..83f0f79
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/CreateProjectUsingMultiGerritTwice.scala
@@ -0,0 +1,53 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.GitSimulation
+import io.gatling.core.Predef.{atOnceUsers, _}
+
+import scala.concurrent.duration._
+
+class CreateProjectUsingMultiGerritTwice extends GitSimulation {
+ private val default: String = name
+
+ private val createProject = new CreateProjectUsingMultiGerrit(default)
+ private val deleteProject = new DeleteProjectUsingMultiGerrit(default)
+ private val createItAgain = new CreateProjectUsingMultiGerrit(default)
+ private val verifyProject = new CloneUsingMultiGerrit1(default)
+ private val deleteItAfter = new DeleteProjectUsingMultiGerrit(default)
+
+ setUp(
+ createProject.test.inject(
+ nothingFor(stepWaitTime(createProject) seconds),
+ atOnceUsers(1)
+ ),
+ deleteProject.test.inject(
+ nothingFor(stepWaitTime(deleteProject) seconds),
+ atOnceUsers(1)
+ ),
+ createItAgain.test.inject(
+ nothingFor(stepWaitTime(createItAgain) seconds),
+ atOnceUsers(1)
+ ),
+ verifyProject.test.inject(
+ nothingFor(stepWaitTime(verifyProject) seconds),
+ atOnceUsers(1)
+ ),
+ deleteItAfter.test.inject(
+ nothingFor(stepWaitTime(deleteItAfter) seconds),
+ atOnceUsers(1)
+ ),
+ ).protocols(gitProtocol, httpProtocol)
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.scala
new file mode 100644
index 0000000..e086efe
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteChangeUsingMultiGerrit1.scala
@@ -0,0 +1,54 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.GerritSimulation
+import com.typesafe.config.ConfigFactory
+import io.gatling.core.Predef.{atOnceUsers, _}
+import io.gatling.core.feeder.FeederBuilder
+import io.gatling.core.structure.ScenarioBuilder
+import io.gatling.http.Predef.http
+import io.gatling.http.protocol.HttpProtocolBuilder
+
+class DeleteChangeUsingMultiGerrit1 extends GerritSimulation {
+ private val data: FeederBuilder = jsonFile(resource).convert(keys).queue
+ var number: Option[Int] = None
+
+ override def relativeRuntimeWeight = 10
+
+ override def replaceOverride(in: String): String = {
+ replaceProperty("http_port1", 8081, in)
+ }
+
+ val httpForReplica: HttpProtocolBuilder = http.basicAuth(
+ conf.httpConfiguration.userName,
+ ConfigFactory.load().getString("http.password_replica"))
+
+ val test: ScenarioBuilder = scenario(unique)
+ .feed(data)
+ .exec(session => {
+ if (number.nonEmpty) {
+ session.set("number", number.get)
+ } else {
+ session
+ }
+ })
+ .exec(http(unique).delete("${url}${number}"))
+
+ setUp(
+ test.inject(
+ atOnceUsers(1)
+ )).protocols(httpForReplica)
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.scala b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.scala
new file mode 100644
index 0000000..f199e5e
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/multisite/scenarios/DeleteProjectUsingMultiGerrit.scala
@@ -0,0 +1,40 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.scenarios
+
+import com.google.gerrit.scenarios.ProjectSimulation
+import io.gatling.core.Predef._
+import io.gatling.core.feeder.FeederBuilder
+import io.gatling.core.structure.ScenarioBuilder
+
+class DeleteProjectUsingMultiGerrit extends ProjectSimulation {
+ private val data: FeederBuilder = jsonFile(resource).convert(keys).queue
+
+ override def relativeRuntimeWeight = 10
+
+ def this(default: String) {
+ this()
+ this.default = default
+ }
+
+ val test: ScenarioBuilder = scenario(unique)
+ .feed(data)
+ .exec(httpRequest)
+
+ setUp(
+ test.inject(
+ atOnceUsers(1)
+ )).protocols(httpProtocol)
+}
