Merge branch 'stable-2.16' into stable-3.0
* stable-2.16:
Jenkinsfile: use gerrit-ci-library pipeline for plugin validation
e2e-tests: Replace FileBasedFeederBuilder with FeederBuilder declaration
Change-Id: I819b1940f3beacd08e5453c07c95076a1f98b140
diff --git a/BUILD b/BUILD
index c933c98..ec555d5 100644
--- a/BUILD
+++ b/BUILD
@@ -17,12 +17,9 @@
],
resources = glob(["src/main/resources/**/*"]),
deps = [
+ "@global-refdb//jar",
+ "@events-broker//jar",
":replication-neverlink",
- "@curator-client//jar",
- "@curator-framework//jar",
- "@curator-recipes//jar",
- "@kafka-client//jar",
- "@zookeeper//jar",
],
)
@@ -51,13 +48,9 @@
visibility = ["//visibility:public"],
exports = PLUGIN_DEPS + PLUGIN_TEST_DEPS + [
":multi-site__plugin",
- "@curator-client//jar",
- "@curator-framework//jar",
- "@curator-test//jar",
- "@mockito//jar",
- "@testcontainers-kafka//jar",
"@wiremock//jar",
- "//lib/testcontainers",
+ "@global-refdb//jar",
+ "@events-broker//jar",
"//plugins/replication",
],
)
diff --git a/README.md b/README.md
index 49e96df..0bf8486 100644
--- a/README.md
+++ b/README.md
@@ -78,29 +78,10 @@
```
[gerrit]
+ installDbModule = com.googlesource.gerrit.plugins.multisite.GitModule
installModule = com.googlesource.gerrit.plugins.multisite.Module
```
-Create the `$GERRIT_SITE/etc/multi-site.config` on all Gerrit servers with the
-following basic settings:
-
-```
-[kafka]
- bootstrapServers = <kafka-host>:<kafka-port>
-
-[kafka "publisher"]
- enabled = true
-
-[kafka "subscriber"]
- enabled = true
-
-[ref-database]
- enabled = true
-
-[ref-database "zookeeper"]
- connectString = "localhost:2181"
-```
-
For more details on the configuration settings, please refer to the
[multi-site configuration documentation](src/main/resources/Documentation/config.md).
diff --git a/dockerised_local_env/gerrit-common/multi-site.config b/dockerised_local_env/gerrit-common/multi-site.config
index 04b9c2c..deec00f 100644
--- a/dockerised_local_env/gerrit-common/multi-site.config
+++ b/dockerised_local_env/gerrit-common/multi-site.config
@@ -12,14 +12,10 @@
cacheEventTopic = gerrit_cache_eviction
[kafka "subscriber"]
- enabled = true
pollingIntervalMs = 1000
KafkaProp-enableAutoCommit = true
KafkaProp-autoCommitIntervalMs = 1000
KafkaProp-autoOffsetReset = latest
-[kafka "publisher"]
- enabled = true
-
[ref-database "zookeeper"]
connectString = "zookeeper:2181"
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index 369f6e0..c10e32a 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl
@@ -8,76 +8,13 @@
)
maven_jar(
- name = "mockito",
- artifact = "org.mockito:mockito-core:2.27.0",
- sha1 = "835fc3283b481f4758b8ef464cd560c649c08b00",
- deps = [
- "@byte-buddy//jar",
- "@byte-buddy-agent//jar",
- "@objenesis//jar",
- ],
- )
-
- BYTE_BUDDY_VER = "1.9.10"
-
- maven_jar(
- name = "byte-buddy",
- artifact = "net.bytebuddy:byte-buddy:" + BYTE_BUDDY_VER,
- sha1 = "211a2b4d3df1eeef2a6cacf78d74a1f725e7a840",
+ name = "global-refdb",
+ artifact = "com.gerritforge:global-refdb:3.0.2",
+ sha1 = "293a807bd82a284c215213b442b3930258e01f5e"
)
maven_jar(
- name = "byte-buddy-agent",
- artifact = "net.bytebuddy:byte-buddy-agent:" + BYTE_BUDDY_VER,
- sha1 = "9674aba5ee793e54b864952b001166848da0f26b",
- )
-
- maven_jar(
- name = "objenesis",
- artifact = "org.objenesis:objenesis:2.6",
- sha1 = "639033469776fd37c08358c6b92a4761feb2af4b",
- )
-
- maven_jar(
- name = "kafka-client",
- artifact = "org.apache.kafka:kafka-clients:2.1.0",
- sha1 = "34d9983705c953b97abb01e1cd04647f47272fe5",
- )
-
- maven_jar(
- name = "testcontainers-kafka",
- artifact = "org.testcontainers:kafka:1.11.3",
- sha1 = "932d1baa2541f218b1b44a0546ae83d530011468",
- )
-
- CURATOR_VER = "4.2.0"
-
- maven_jar(
- name = "curator-test",
- artifact = "org.apache.curator:curator-test:" + CURATOR_VER,
- sha1 = "98ac2dd69b8c07dcaab5e5473f93fdb9e320cd73",
- )
-
- maven_jar(
- name = "curator-framework",
- artifact = "org.apache.curator:curator-framework:" + CURATOR_VER,
- sha1 = "5b1cc87e17b8fe4219b057f6025662a693538861",
- )
-
- maven_jar(
- name = "curator-recipes",
- artifact = "org.apache.curator:curator-recipes:" + CURATOR_VER,
- sha1 = "7f775be5a7062c2477c51533b9d008f70411ba8e",
- )
-
- maven_jar(
- name = "curator-client",
- artifact = "org.apache.curator:curator-client:" + CURATOR_VER,
- sha1 = "d5d50930b8dd189f92c40258a6ba97675fea3e15",
- )
-
- maven_jar(
- name = "zookeeper",
- artifact = "org.apache.zookeeper:zookeeper:3.4.14",
- sha1 = "c114c1e1c8172a7cd3f6ae39209a635f7a06c1a1",
+ name = "events-broker",
+ artifact = "com.gerritforge:events-broker:3.0.5",
+ sha1 = "7abf72d2252f975baff666fbbf28b7036767aa81",
)
diff --git a/setup_local_env/configs/gerrit.config b/setup_local_env/configs/gerrit.config
index 18eeca4..f9eca89 100644
--- a/setup_local_env/configs/gerrit.config
+++ b/setup_local_env/configs/gerrit.config
@@ -2,8 +2,9 @@
basePath = git
serverId = 69ec38f0-350e-4d9c-96d4-bc956f2faaac
canonicalWebUrl = $GERRIT_CANONICAL_WEB_URL
+ installModule = com.gerritforge.gerrit.eventbroker.BrokerApiModule # events-broker module to setup BrokerApi dynamic item
installModule = com.googlesource.gerrit.plugins.multisite.Module # multi-site needs to be a gerrit lib
-
+ installDbModule = com.googlesource.gerrit.plugins.multisite.GitModule
[database]
type = h2
database = $LOCATION_TEST_SITE/db/ReviewDB
@@ -31,9 +32,22 @@
advertisedAddress = *:$SSH_ADVERTISED_PORT
[httpd]
listenUrl = proxy-$HTTP_PROTOCOL://*:$GERRIT_HTTPD_PORT/
+ requestLog = true
[cache]
directory = cache
[plugins]
allowRemoteAdmin = true
[plugin "websession-flatfile"]
directory = $FAKE_NFS
+[plugin "kafka-events"]
+ sendAsync = true
+ bootstrapServers = localhost:$KAFKA_PORT
+ groupId = $KAFKA_GROUP_ID
+ numberOfSubscribers = 6
+ securityProtocol = PLAINTEXT
+ pollingIntervalMs = 1000
+ enableAutoCommit = true
+ autoCommitIntervalMs = 1000
+ autoOffsetReset = latest
+[plugin "metrics-reporter-prometheus"]
+ prometheusBearerToken = token
diff --git a/setup_local_env/configs/multi-site.config b/setup_local_env/configs/multi-site.config
index 6b631a2..9571513 100644
--- a/setup_local_env/configs/multi-site.config
+++ b/setup_local_env/configs/multi-site.config
@@ -1,21 +1,11 @@
[index]
maxTries = 50
retryInterval = 30000
-[kafka]
- bootstrapServers = localhost:$KAFKA_PORT
- securityProtocol = PLAINTEXT
- indexEventTopic = gerrit_index
- batchIndexEventTopic = gerrit_batch_index
- streamEventTopic = gerrit_stream
- projectListEventTopic = gerrit_list_project
- cacheEventTopic = gerrit_cache_eviction
-[kafka "subscriber"]
- enabled = true
- pollingIntervalMs = 1000
- KafkaProp-enableAutoCommit = true
- KafkaProp-autoCommitIntervalMs = 1000
- KafkaProp-autoOffsetReset = latest
-[kafka "publisher"]
- enabled = true
-[ref-database "zookeeper"]
- connectString = localhost:$ZK_PORT
+
+[broker]
+ indexEventTopic = gerrit_index
+ batchIndexEventTopic = gerrit_batch_index
+ streamEventTopic = gerrit_stream
+ projectListEventTopic = gerrit_list_project
+ cacheEventTopic = gerrit_cache_eviction
+
diff --git a/setup_local_env/configs/prometheus.yml b/setup_local_env/configs/prometheus.yml
new file mode 100644
index 0000000..8eaa989
--- /dev/null
+++ b/setup_local_env/configs/prometheus.yml
@@ -0,0 +1,17 @@
+global:
+ scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+ evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+scrape_configs:
+ - job_name: 'metrics'
+ scheme: http
+ metrics_path: '/plugins/metrics-reporter-prometheus/metrics'
+ params:
+ format: ['prometheus']
+ bearer_token: token
+ scrape_interval: 5s
+ static_configs:
+ - targets: ['$GERRIT_SITE_HOST:18080','$GERRIT_SITE_HOST:18081']
+ labels:
+ env: 'unit'
diff --git a/setup_local_env/configs/zookeeper-refdb.config b/setup_local_env/configs/zookeeper-refdb.config
new file mode 100644
index 0000000..2c84a05
--- /dev/null
+++ b/setup_local_env/configs/zookeeper-refdb.config
@@ -0,0 +1,2 @@
+[ref-database "zookeeper"]
+ connectString = localhost:$ZK_PORT
\ No newline at end of file
diff --git a/setup_local_env/docker-compose.kafka-broker.yaml b/setup_local_env/docker-compose.yaml
similarity index 63%
rename from setup_local_env/docker-compose.kafka-broker.yaml
rename to setup_local_env/docker-compose.yaml
index b7e91f0..c386d46 100644
--- a/setup_local_env/docker-compose.kafka-broker.yaml
+++ b/setup_local_env/docker-compose.yaml
@@ -13,3 +13,11 @@
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ prometheus:
+ image: prom/prometheus:v2.16.0
+ user: root
+ volumes:
+ - $COMMON_LOCATION/prometheus.yml:/etc/prometheus/prometheus.yml
+ ports:
+ - "9090:9090"
+ network_mode: $NETWORK_MODE
diff --git a/setup_local_env/prometheus-config/prometheus.yml b/setup_local_env/prometheus-config/prometheus.yml
new file mode 100644
index 0000000..e6d3ea1
--- /dev/null
+++ b/setup_local_env/prometheus-config/prometheus.yml
@@ -0,0 +1,14 @@
+# my global config
+global:
+ scrape_interval: 10s
+ evaluation_interval: 10s
+
+scrape_configs:
+ - job_name: gerrit
+ static_configs:
+ - targets: ['localhost:18080','localhost:18081']
+ metrics_path: '/plugins/metrics-reporter-prometheus/metrics'
+ params:
+ format: ['prometheus']
+ bearer_token: token
+
diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh
index e6ebcd7..a97afc3 100755
--- a/setup_local_env/setup.sh
+++ b/setup_local_env/setup.sh
@@ -16,252 +16,273 @@
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+GERRIT_BRANCH=stable-3.0
+GERRIT_CI=https://gerrit-ci.gerritforge.com/view/Plugins-$GERRIT_BRANCH/job
+LAST_BUILD=lastSuccessfulBuild/artifact/bazel-bin/plugins
function check_application_requirements {
- type haproxy >/dev/null 2>&1 || { echo >&2 "Require haproxy but it's not installed. Aborting."; exit 1; }
- type java >/dev/null 2>&1 || { echo >&2 "Require java but it's not installed. Aborting."; exit 1; }
- type docker >/dev/null 2>&1 || { echo >&2 "Require docker but it's not installed. Aborting."; exit 1; }
- type docker-compose >/dev/null 2>&1 || { echo >&2 "Require docker-compose but it's not installed. Aborting."; exit 1; }
- type wget >/dev/null 2>&1 || { echo >&2 "Require wget but it's not installed. Aborting."; exit 1; }
- type envsubst >/dev/null 2>&1 || { echo >&2 "Require envsubst but it's not installed. Aborting."; exit 1; }
- type openssl >/dev/null 2>&1 || { echo >&2 "Require openssl but it's not installed. Aborting."; exit 1; }
+ type haproxy >/dev/null 2>&1 || { echo >&2 "Require haproxy but it's not installed. Aborting."; exit 1; }
+ type java >/dev/null 2>&1 || { echo >&2 "Require java but it's not installed. Aborting."; exit 1; }
+ type docker >/dev/null 2>&1 || { echo >&2 "Require docker but it's not installed. Aborting."; exit 1; }
+ type docker-compose >/dev/null 2>&1 || { echo >&2 "Require docker-compose but it's not installed. Aborting."; exit 1; }
+ type wget >/dev/null 2>&1 || { echo >&2 "Require wget but it's not installed. Aborting."; exit 1; }
+ type envsubst >/dev/null 2>&1 || { echo >&2 "Require envsubst but it's not installed. Aborting."; exit 1; }
+ type openssl >/dev/null 2>&1 || { echo >&2 "Require openssl but it's not installed. Aborting."; exit 1; }
}
function get_replication_url {
- REPLICATION_LOCATION_TEST_SITE=$1
- REPLICATION_HOSTNAME=$2
- USER=$REPLICATION_SSH_USER
+ REPLICATION_LOCATION_TEST_SITE=$1
+ REPLICATION_HOSTNAME=$2
+ USER=$REPLICATION_SSH_USER
- if [ "$REPLICATION_TYPE" = "file" ];then
- echo "url = file://$REPLICATION_LOCATION_TEST_SITE/git/#{name}#.git"
- elif [ "$REPLICATION_TYPE" = "ssh" ];then
- echo "url = ssh://$USER@$REPLICATION_HOSTNAME:$REPLICATION_LOCATION_TEST_SITE/git/#{name}#.git"
- fi
+ if [ "$REPLICATION_TYPE" = "file" ];then
+ echo "url = file://$REPLICATION_LOCATION_TEST_SITE/git/#{name}#.git"
+ elif [ "$REPLICATION_TYPE" = "ssh" ];then
+ echo "url = ssh://$USER@$REPLICATION_HOSTNAME:$REPLICATION_LOCATION_TEST_SITE/git/#{name}#.git"
+ fi
}
function deploy_tls_certificates {
- echo "Deplying certificates in $HA_PROXY_CERTIFICATES_DIR..."
- openssl req -new -newkey rsa:2048 -x509 -sha256 -days 365 -nodes \
- -out $HA_PROXY_CERTIFICATES_DIR/MyCertificate.crt \
- -keyout $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.key \
- -subj "/C=GB/ST=London/L=London/O=Gerrit Org/OU=IT Department/CN=localhost"
- cat $HA_PROXY_CERTIFICATES_DIR/MyCertificate.crt $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.key | tee $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.pem
+ echo "Deplying certificates in $HA_PROXY_CERTIFICATES_DIR..."
+ openssl req -new -newkey rsa:2048 -x509 -sha256 -days 365 -nodes \
+ -out $HA_PROXY_CERTIFICATES_DIR/MyCertificate.crt \
+ -keyout $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.key \
+ -subj "/C=GB/ST=London/L=London/O=Gerrit Org/OU=IT Department/CN=localhost"
+ cat $HA_PROXY_CERTIFICATES_DIR/MyCertificate.crt $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.key | tee $HA_PROXY_CERTIFICATES_DIR/GerritLocalKey.pem
}
function copy_config_files {
- for file in `ls $SCRIPT_DIR/configs/*.config`
- do
- file_name=`basename $file`
+ for file in `ls $SCRIPT_DIR/configs/*.config`
+ do
+ file_name=`basename $file`
- CONFIG_TEST_SITE=$1
- export GERRIT_HTTPD_PORT=$2
- export LOCATION_TEST_SITE=$3
- export GERRIT_SSHD_PORT=$4
- export REPLICATION_HTTPD_PORT=$5
- export REPLICATION_LOCATION_TEST_SITE=$6
- export GERRIT_HOSTNAME=$7
- export REPLICATION_HOSTNAME=$8
- export REMOTE_DEBUG_PORT=$9
- export REPLICATION_URL=$(get_replication_url $REPLICATION_LOCATION_TEST_SITE $REPLICATION_HOSTNAME)
+ CONFIG_TEST_SITE=$1
+ export GERRIT_HTTPD_PORT=$2
+ export LOCATION_TEST_SITE=$3
+ export GERRIT_SSHD_PORT=$4
+ export REPLICATION_HTTPD_PORT=$5
+ export REPLICATION_LOCATION_TEST_SITE=$6
+ export GERRIT_HOSTNAME=$7
+ export REPLICATION_HOSTNAME=$8
+ export REMOTE_DEBUG_PORT=$9
+ export KAFKA_GROUP_ID=${10}
+ export REPLICATION_URL=$(get_replication_url $REPLICATION_LOCATION_TEST_SITE $REPLICATION_HOSTNAME)
- echo "Replacing variables for file $file and copying to $CONFIG_TEST_SITE/$file_name"
+ echo "Replacing variables for file $file and copying to $CONFIG_TEST_SITE/$file_name"
- cat $file | envsubst | sed 's/#{name}#/${name}/g' > $CONFIG_TEST_SITE/$file_name
- done
+ cat $file | envsubst | sed 's/#{name}#/${name}/g' > $CONFIG_TEST_SITE/$file_name
+ done
}
+
function start_ha_proxy {
- export HA_GERRIT_CANONICAL_HOSTNAME=$GERRIT_CANONICAL_HOSTNAME
- export HA_GERRIT_CANONICAL_PORT=$GERRIT_CANONICAL_PORT
+ export HA_GERRIT_CANONICAL_HOSTNAME=$GERRIT_CANONICAL_HOSTNAME
+ export HA_GERRIT_CANONICAL_PORT=$GERRIT_CANONICAL_PORT
- export HA_HTTPS_BIND=$HTTPS_BIND
+ export HA_HTTPS_BIND=$HTTPS_BIND
- export HA_GERRIT_SITE1_HOSTNAME=$GERRIT_1_HOSTNAME
- export HA_GERRIT_SITE2_HOSTNAME=$GERRIT_2_HOSTNAME
- export HA_GERRIT_SITE1_HTTPD_PORT=$GERRIT_1_HTTPD_PORT
- export HA_GERRIT_SITE2_HTTPD_PORT=$GERRIT_2_HTTPD_PORT
+ export HA_GERRIT_SITE1_HOSTNAME=$GERRIT_1_HOSTNAME
+ export HA_GERRIT_SITE2_HOSTNAME=$GERRIT_2_HOSTNAME
+ export HA_GERRIT_SITE1_HTTPD_PORT=$GERRIT_1_HTTPD_PORT
+ export HA_GERRIT_SITE2_HTTPD_PORT=$GERRIT_2_HTTPD_PORT
- export HA_GERRIT_SITE1_SSHD_PORT=$GERRIT_1_SSHD_PORT
- export HA_GERRIT_SITE2_SSHD_PORT=$GERRIT_2_SSHD_PORT
+ export HA_GERRIT_SITE1_SSHD_PORT=$GERRIT_1_SSHD_PORT
+ export HA_GERRIT_SITE2_SSHD_PORT=$GERRIT_2_SSHD_PORT
- cat $SCRIPT_DIR/haproxy-config/haproxy.cfg | envsubst > $HA_PROXY_CONFIG_DIR/haproxy.cfg
+ cat $SCRIPT_DIR/haproxy-config/haproxy.cfg | envsubst > $HA_PROXY_CONFIG_DIR/haproxy.cfg
- echo "Starting HA-PROXY..."
- echo "THE SCRIPT LOCATION $SCRIPT_DIR"
- echo "THE HA SCRIPT_LOCATION $HA_SCRIPT_DIR"
- haproxy -f $HA_PROXY_CONFIG_DIR/haproxy.cfg &
+ echo "Starting HA-PROXY..."
+ echo "THE SCRIPT LOCATION $SCRIPT_DIR"
+ echo "THE HA SCRIPT_LOCATION $HA_SCRIPT_DIR"
+ haproxy -f $HA_PROXY_CONFIG_DIR/haproxy.cfg &
}
function deploy_config_files {
- # KAFKA configuration
- export KAFKA_PORT=9092
+ # KAFKA configuration
+ export KAFKA_PORT=9092
- # ZK configuration
- export ZK_PORT=2181
+ # ZK configuration
+ export ZK_PORT=2181
- # SITE 1
- GERRIT_SITE1_HOSTNAME=$1
- GERRIT_SITE1_HTTPD_PORT=$2
- GERRIT_SITE1_SSHD_PORT=$3
- CONFIG_TEST_SITE_1=$LOCATION_TEST_SITE_1/etc
- GERRIT_SITE1_REMOTE_DEBUG_PORT="5005"
- # SITE 2
- GERRIT_SITE2_HOSTNAME=$4
- GERRIT_SITE2_HTTPD_PORT=$5
- GERRIT_SITE2_SSHD_PORT=$6
- CONFIG_TEST_SITE_2=$LOCATION_TEST_SITE_2/etc
- GERRIT_SITE2_REMOTE_DEBUG_PORT="5006"
+ # SITE 1
+ GERRIT_SITE1_HOSTNAME=$1
+ GERRIT_SITE1_HTTPD_PORT=$2
+ GERRIT_SITE1_SSHD_PORT=$3
+ CONFIG_TEST_SITE_1=$LOCATION_TEST_SITE_1/etc
+ GERRIT_SITE1_REMOTE_DEBUG_PORT="5005"
+ GERRIT_SITE1_KAFKA_GROUP_ID="instance-1"
+ # SITE 2
+ GERRIT_SITE2_HOSTNAME=$4
+ GERRIT_SITE2_HTTPD_PORT=$5
+ GERRIT_SITE2_SSHD_PORT=$6
+ CONFIG_TEST_SITE_2=$LOCATION_TEST_SITE_2/etc
+ GERRIT_SITE2_REMOTE_DEBUG_PORT="5006"
+ GERRIT_SITE2_KAFKA_GROUP_ID="instance-2"
- # Set config SITE1
- copy_config_files $CONFIG_TEST_SITE_1 $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_SSHD_PORT $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE1_REMOTE_DEBUG_PORT
+ # Set config SITE1
+ copy_config_files $CONFIG_TEST_SITE_1 $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_SSHD_PORT $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE1_REMOTE_DEBUG_PORT $GERRIT_SITE1_KAFKA_GROUP_ID
- # Set config SITE2
- copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT
+ # Set config SITE2
+ copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT $GERRIT_SITE2_KAFKA_GROUP_ID
+}
+
+function is_docker_desktop {
+ echo $(docker info | grep "Operating System: Docker Desktop" | wc -l)
+}
+
+function docker_host_env {
+ IS_DOCKER_DESKTOP=$(is_docker_desktop)
+ if [ "$IS_DOCKER_DESKTOP" = "1" ];then
+ echo "mac"
+ else
+ echo "linux"
+ fi
}
function cleanup_environment {
- echo "Killing existing HA-PROXY setup"
- kill $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}') 2> /dev/null
- echo "Stoping kafka and zk"
- docker-compose -f $SCRIPT_DIR/docker-compose.kafka-broker.yaml down 2> /dev/null
+ echo "Killing existing HA-PROXY setup"
+ kill $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}') 2> /dev/null
- echo "Stoping GERRIT instances"
- $1/bin/gerrit.sh stop 2> /dev/null
- $2/bin/gerrit.sh stop 2> /dev/null
+ echo "Stopping docker containers"
+ docker-compose -f $SCRIPT_DIR/docker-compose.yaml down 2> /dev/null
- echo "REMOVING setup directory $3"
- rm -rf $3 2> /dev/null
+ echo "Stopping GERRIT instances"
+ $1/bin/gerrit.sh stop 2> /dev/null
+ $2/bin/gerrit.sh stop 2> /dev/null
+
+ echo "REMOVING setup directory $3"
+ rm -rf $3 2> /dev/null
}
function check_if_kafka_is_running {
- echo $(docker inspect kafka_test_node 2> /dev/null | grep '"Running": true' | wc -l)
+ echo $(docker inspect kafka_test_node 2> /dev/null | grep '"Running": true' | wc -l)
}
while [ $# -ne 0 ]
do
case "$1" in
"--help" )
- echo "Usage: sh $0 [--option $value]"
- echo
- echo "[--release-war-file] Location to release.war file"
- echo "[--multisite-lib-file] Location to lib multi-site.jar file"
- echo
- echo "[--new-deployment] Cleans up previous gerrit deployment and re-installs it. default true"
- echo "[--get-websession-plugin] Download websession-flatfile plugin from CI lastSuccessfulBuild; default true"
- echo "[--deployment-location] Base location for the test deployment; default /tmp"
- echo
- echo "[--gerrit-canonical-host] The default host for Gerrit to be accessed through; default localhost"
- echo "[--gerrit-canonical-port] The default port for Gerrit to be accessed throug; default 8080"
- echo
- echo "[--gerrit-ssh-advertised-port] Gerrit Instance 1 sshd port; default 29418"
- echo
- echo "[--gerrit1-httpd-port] Gerrit Instance 1 http port; default 18080"
- echo "[--gerrit1-sshd-port] Gerrit Instance 1 sshd port; default 39418"
- echo
- echo "[--gerrit2-httpd-port] Gerrit Instance 2 http port; default 18081"
- echo "[--gerrit2-sshd-port] Gerrit Instance 2 sshd port; default 49418"
- echo
- echo "[--replication-type] Options [file,ssh]; default ssh"
- echo "[--replication-ssh-user] SSH user for the replication plugin; default $(whoami)"
- echo "[--replication-delay] Replication delay across the two instances in seconds"
- echo
- echo "[--just-cleanup-env] Cleans up previous deployment; default false"
- echo
- echo "[--enabled-https] Enabled https; default true"
- echo
- exit 0
+ echo "Usage: sh $0 [--option $value]"
+ echo
+ echo "[--release-war-file] Location to release.war file"
+ echo "[--multisite-lib-file] Location to lib multi-site.jar file"
+ echo
+ echo "[--new-deployment] Cleans up previous gerrit deployment and re-installs it. default true"
+ echo "[--get-websession-plugin] Download websession-broker plugin from CI lastSuccessfulBuild; default true"
+ echo "[--deployment-location] Base location for the test deployment; default /tmp"
+ echo
+ echo "[--gerrit-canonical-host] The default host for Gerrit to be accessed through; default localhost"
+ echo "[--gerrit-canonical-port] The default port for Gerrit to be accessed throug; default 8080"
+ echo
+ echo "[--gerrit-ssh-advertised-port] Gerrit Instance 1 sshd port; default 29418"
+ echo
+ echo "[--gerrit1-httpd-port] Gerrit Instance 1 http port; default 18080"
+ echo "[--gerrit1-sshd-port] Gerrit Instance 1 sshd port; default 39418"
+ echo
+ echo "[--gerrit2-httpd-port] Gerrit Instance 2 http port; default 18081"
+ echo "[--gerrit2-sshd-port] Gerrit Instance 2 sshd port; default 49418"
+ echo
+ echo "[--replication-type] Options [file,ssh]; default ssh"
+ echo "[--replication-ssh-user] SSH user for the replication plugin; default $(whoami)"
+ echo "[--replication-delay] Replication delay across the two instances in seconds"
+ echo
+ echo "[--just-cleanup-env] Cleans up previous deployment; default false"
+ echo
+ echo "[--enabled-https] Enabled https; default true"
+ echo
+ exit 0
;;
"--new-deployment")
NEW_INSTALLATION=$2
- shift
- shift
+ shift
+ shift
;;
"--get-websession-plugin")
- DOWNLOAD_WEBSESSION_FLATFILE=$2
- shift
- shift
+ DOWNLOAD_WEBSESSION_PLUGIN=$2
+ shift
+ shift
;;
"--deployment-location" )
- DEPLOYMENT_LOCATION=$2
- shift
- shift
+ DEPLOYMENT_LOCATION=$2
+ shift
+ shift
;;
"--release-war-file" )
- RELEASE_WAR_FILE_LOCATION=$2
- shift
- shift
+ RELEASE_WAR_FILE_LOCATION=$2
+ shift
+ shift
;;
"--multisite-lib-file" )
- MULTISITE_LIB_LOCATION=$2
- shift
- shift
+ MULTISITE_LIB_LOCATION=$2
+ shift
+ shift
;;
"--gerrit-canonical-host" )
- export GERRIT_CANONICAL_HOSTNAME=$2
- shift
- shift
+ export GERRIT_CANONICAL_HOSTNAME=$2
+ shift
+ shift
;;
"--gerrit-canonical-port" )
- export GERRIT_CANONICAL_PORT=$2
- shift
- shift
+ export GERRIT_CANONICAL_PORT=$2
+ shift
+ shift
;;
"--gerrit-ssh-advertised-port" )
- export SSH_ADVERTISED_PORT=$2
- shift
- shift
+ export SSH_ADVERTISED_PORT=$2
+ shift
+ shift
;;
"--gerrit1-httpd-port" )
- GERRIT_1_HTTPD_PORT=$2
- shift
- shift
+ GERRIT_1_HTTPD_PORT=$2
+ shift
+ shift
;;
"--gerrit2-httpd-port" )
- GERRIT_2_HTTPD_PORT=$2
- shift
- shift
+ GERRIT_2_HTTPD_PORT=$2
+ shift
+ shift
;;
"--gerrit1-sshd-port" )
- GERRIT_1_SSHD_PORT=$2
- shift
- shift
+ GERRIT_1_SSHD_PORT=$2
+ shift
+ shift
;;
"--gerrit2-sshd-port" )
- GERRIT_2_SSHD_PORT=$2
- shift
- shift
+ GERRIT_2_SSHD_PORT=$2
+ shift
+ shift
;;
"--replication-ssh-user" )
- export REPLICATION_SSH_USER=$2
- shift
- shift
+ export REPLICATION_SSH_USER=$2
+ shift
+ shift
;;
"--replication-type")
- export REPLICATION_TYPE=$2
- shift
- shift
+ export REPLICATION_TYPE=$2
+ shift
+ shift
;;
"--replication-delay")
- export REPLICATION_DELAY_SEC=$2
- shift
- shift
+ export REPLICATION_DELAY_SEC=$2
+ shift
+ shift
;;
"--just-cleanup-env" )
- JUST_CLEANUP_ENV=$2
- shift
- shift
+ JUST_CLEANUP_ENV=$2
+ shift
+ shift
;;
"--enabled-https" )
- HTTPS_ENABLED=$2
- shift
- shift
+ HTTPS_ENABLED=$2
+ shift
+ shift
;;
- * )
- echo "Unknown option argument: $1"
- shift
- shift
+ * )
+ echo "Unknown option argument: $1"
+ shift
+ shift
;;
esac
done
@@ -271,7 +292,7 @@
# Defaults
NEW_INSTALLATION=${NEW_INSTALLATION:-"true"}
-DOWNLOAD_WEBSESSION_FLATFILE=${DOWNLOAD_WEBSESSION_FLATFILE:-"true"}
+DOWNLOAD_WEBSESSION_PLUGIN=${DOWNLOAD_WEBSESSION_PLUGIN:-"true"}
DEPLOYMENT_LOCATION=${DEPLOYMENT_LOCATION:-"/tmp"}
export GERRIT_CANONICAL_HOSTNAME=${GERRIT_CANONICAL_HOSTNAME:-"localhost"}
export GERRIT_CANONICAL_PORT=${GERRIT_CANONICAL_PORT:-"8080"}
@@ -287,11 +308,12 @@
export SSH_ADVERTISED_PORT=${SSH_ADVERTISED_PORT:-"29418"}
HTTPS_ENABLED=${HTTPS_ENABLED:-"false"}
-COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
+export COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
LOCATION_TEST_SITE_1=$COMMON_LOCATION/instance-1
LOCATION_TEST_SITE_2=$COMMON_LOCATION/instance-2
HA_PROXY_CONFIG_DIR=$COMMON_LOCATION/ha-proxy-config
HA_PROXY_CERTIFICATES_DIR="$HA_PROXY_CONFIG_DIR/certificates"
+PROMETHEUS_CONFIG_DIR=$COMMON_LOCATION/prometheus-config
RELEASE_WAR_FILE_LOCATION=${RELEASE_WAR_FILE_LOCATION:-bazel-bin/release.war}
MULTISITE_LIB_LOCATION=${MULTISITE_LIB_LOCATION:-bazel-bin/plugins/multi-site/multi-site.jar}
@@ -300,93 +322,136 @@
export FAKE_NFS=$COMMON_LOCATION/fake_nfs
if [ "$JUST_CLEANUP_ENV" = "true" ];then
- cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
- exit 0
+ cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
+ exit 0
fi
if [ -z $RELEASE_WAR_FILE_LOCATION ];then
- echo "A release.war file is required. Usage: sh $0 --release-war-file /path/to/release.war"
- exit 1
+ echo "A release.war file is required. Usage: sh $0 --release-war-file /path/to/release.war"
+ exit 1
else
- cp -f $RELEASE_WAR_FILE_LOCATION $DEPLOYMENT_LOCATION/gerrit.war >/dev/null 2>&1 || { echo >&2 "$RELEASE_WAR_FILE_LOCATION: Not able to copy the file. Aborting"; exit 1; }
+ cp -f $RELEASE_WAR_FILE_LOCATION $DEPLOYMENT_LOCATION/gerrit.war >/dev/null 2>&1 || { echo >&2 "$RELEASE_WAR_FILE_LOCATION: Not able to copy the file. Aborting"; exit 1; }
fi
if [ -z $MULTISITE_LIB_LOCATION ];then
- echo "The multi-site library is required. Usage: sh $0 --multisite-lib-file /path/to/multi-site.jar"
- exit 1
+ echo "The multi-site library is required. Usage: sh $0 --multisite-lib-file /path/to/multi-site.jar"
+ exit 1
else
- cp -f $MULTISITE_LIB_LOCATION $DEPLOYMENT_LOCATION/multi-site.jar >/dev/null 2>&1 || { echo >&2 "$MULTISITE_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; }
+ cp -f $MULTISITE_LIB_LOCATION $DEPLOYMENT_LOCATION/multi-site.jar >/dev/null 2>&1 || { echo >&2 "$MULTISITE_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; }
fi
-if [ $DOWNLOAD_WEBSESSION_FLATFILE = "true" ];then
- echo "Downloading websession-flatfile plugin stable 2.16"
- wget https://gerrit-ci.gerritforge.com/view/Plugins-stable-2.16/job/plugin-websession-flatfile-bazel-master-stable-2.16/lastSuccessfulBuild/artifact/bazel-bin/plugins/websession-flatfile/websession-flatfile.jar \
- -O $DEPLOYMENT_LOCATION/websession-flatfile.jar || { echo >&2 "Cannot download websession-flatfile plugin: Check internet connection. Abort\
+if [ $DOWNLOAD_WEBSESSION_PLUGIN = "true" ];then
+ echo "Downloading websession-broker plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-websession-broker-bazel-$GERRIT_BRANCH/$LAST_BUILD/websession-broker/websession-broker.jar \
+ -O $DEPLOYMENT_LOCATION/websession-broker.jar || { echo >&2 "Cannot download websession-broker plugin: Check internet connection. Abort\
ing"; exit 1; }
- wget https://gerrit-ci.gerritforge.com/view/Plugins-stable-2.16/job/plugin-healthcheck-bazel-stable-2.16/lastSuccessfulBuild/artifact/bazel-bin/plugins/healthcheck/healthcheck.jar \
- -O $DEPLOYMENT_LOCATION/healthcheck.jar || { echo >&2 "Cannot download healthcheck plugin: Check internet connection. Abort\
+ wget $GERRIT_CI/plugin-healthcheck-bazel-$GERRIT_BRANCH/$LAST_BUILD/healthcheck/healthcheck.jar \
+ -O $DEPLOYMENT_LOCATION/healthcheck.jar || { echo >&2 "Cannot download healthcheck plugin: Check internet connection. Abort\
ing"; exit 1; }
else
- echo "Without the websession-flatfile; user login via haproxy will fail."
+ echo "Without the websession-broker; user login via haproxy will fail."
fi
+echo "Downloading zookeeper plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-zookeeper-refdb-bazel-$GERRIT_BRANCH/$LAST_BUILD/zookeeper-refdb/zookeeper-refdb.jar \
+ -O $DEPLOYMENT_LOCATION/zookeeper-refdb.jar || { echo >&2 "Cannot download zookeeper plugin: Check internet connection. Abort\
+ing"; exit 1; }
+
+echo "Downloading events-broker library $GERRIT_BRANCH"
+ wget https://repo1.maven.org/maven2/com/gerritforge/events-broker/3.0.5/events-broker-3.0.5.jar \
+ -O $DEPLOYMENT_LOCATION/events-broker.jar || { echo >&2 "Cannot download events-broker library: Check internet connection. Abort\
+ing"; exit 1; }
+
+echo "Downloading kafka-events plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-kafka-events-bazel-$GERRIT_BRANCH/$LAST_BUILD/kafka-events/kafka-events.jar \
+ -O $DEPLOYMENT_LOCATION/kafka-events.jar || { echo >&2 "Cannot download kafka-events plugin: Check internet connection. Abort\
+ing"; exit 1; }
+
+echo "Downloading metrics-reporter-prometheus plugin $GERRIT_BRANCH"
+ wget $GERRIT_CI/plugin-metrics-reporter-prometheus-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/metrics-reporter-prometheus/metrics-reporter-prometheus.jar \
+ -O $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar || { echo >&2 "Cannot download metrics-reporter-prometheus plugin: Check internet connection. Abort\
+ing"; exit 1; }
+
if [ "$REPLICATION_TYPE" = "ssh" ];then
- echo "Using 'SSH' replication type"
- echo "Make sure ~/.ssh/authorized_keys and ~/.ssh/known_hosts are configured correctly"
+ echo "Using 'SSH' replication type"
+ echo "Make sure ~/.ssh/authorized_keys and ~/.ssh/known_hosts are configured correctly"
fi
if [ "$HTTPS_ENABLED" = "true" ];then
- export HTTP_PROTOCOL="https"
- export GERRIT_CANONICAL_WEB_URL="$HTTP_PROTOCOL://$GERRIT_CANONICAL_HOSTNAME/"
- export HTTPS_BIND="bind *:443 ssl crt $HA_PROXY_CONFIG_DIR/certificates/GerritLocalKey.pem"
- HTTPS_CLONE_MSG="Using self-signed certificates, to clone via https - 'git config --global http.sslVerify false'"
+ export HTTP_PROTOCOL="https"
+ export GERRIT_CANONICAL_WEB_URL="$HTTP_PROTOCOL://$GERRIT_CANONICAL_HOSTNAME/"
+ export HTTPS_BIND="bind *:443 ssl crt $HA_PROXY_CONFIG_DIR/certificates/GerritLocalKey.pem"
+ HTTPS_CLONE_MSG="Using self-signed certificates, to clone via https - 'git config --global http.sslVerify false'"
else
- export HTTP_PROTOCOL="http"
- export GERRIT_CANONICAL_WEB_URL="$HTTP_PROTOCOL://$GERRIT_CANONICAL_HOSTNAME:$GERRIT_CANONICAL_PORT/"
+ export HTTP_PROTOCOL="http"
+ export GERRIT_CANONICAL_WEB_URL="$HTTP_PROTOCOL://$GERRIT_CANONICAL_HOSTNAME:$GERRIT_CANONICAL_PORT/"
fi
# New installation
if [ $NEW_INSTALLATION = "true" ]; then
- cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
+ cleanup_environment $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $COMMON_LOCATION
- echo "Setting up directories"
- mkdir -p $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $HA_PROXY_CERTIFICATES_DIR $FAKE_NFS
- java -jar $DEPLOYMENT_LOCATION/gerrit.war init --batch --no-auto-start --install-all-plugins --dev -d $LOCATION_TEST_SITE_1
+ echo "Setting up directories"
+ mkdir -p $LOCATION_TEST_SITE_1 $LOCATION_TEST_SITE_2 $HA_PROXY_CERTIFICATES_DIR $FAKE_NFS
+ java -jar $DEPLOYMENT_LOCATION/gerrit.war init --batch --no-auto-start --install-all-plugins --dev -d $LOCATION_TEST_SITE_1
- # Deploying TLS certificates
- if [ "$HTTPS_ENABLED" = "true" ];then deploy_tls_certificates;fi
+ # Deploying TLS certificates
+ if [ "$HTTPS_ENABLED" = "true" ];then deploy_tls_certificates;fi
- echo "Copy multi-site library to lib directory"
- cp -f $DEPLOYMENT_LOCATION/multi-site.jar $LOCATION_TEST_SITE_1/lib/multi-site.jar
+ echo "Copy multi-site library to lib directory"
+ cp -f $DEPLOYMENT_LOCATION/multi-site.jar $LOCATION_TEST_SITE_1/lib/multi-site.jar
- echo "Copy websession-flatfile plugin"
- cp -f $DEPLOYMENT_LOCATION/websession-flatfile.jar $LOCATION_TEST_SITE_1/plugins/websession-flatfile.jar
+ echo "Copy websession-broker plugin"
+ cp -f $DEPLOYMENT_LOCATION/websession-broker.jar $LOCATION_TEST_SITE_1/plugins/websession-broker.jar
- echo "Copy healthcheck plugin"
- cp -f $DEPLOYMENT_LOCATION/healthcheck.jar $LOCATION_TEST_SITE_1/plugins/healthcheck.jar
+ echo "Copy healthcheck plugin"
+ cp -f $DEPLOYMENT_LOCATION/healthcheck.jar $LOCATION_TEST_SITE_1/plugins/healthcheck.jar
- echo "Re-indexing"
- java -jar $DEPLOYMENT_LOCATION/gerrit.war reindex -d $LOCATION_TEST_SITE_1
- # Replicating environment
- echo "Replicating environment"
- cp -fR $LOCATION_TEST_SITE_1/* $LOCATION_TEST_SITE_2
+ echo "Copy zookeeper plugin"
+ cp -f $DEPLOYMENT_LOCATION/zookeeper-refdb.jar $LOCATION_TEST_SITE_1/plugins/zookeeper-refdb.jar
- echo "Link replication plugin"
- ln -s $LOCATION_TEST_SITE_1/plugins/replication.jar $LOCATION_TEST_SITE_1/lib/replication.jar
- ln -s $LOCATION_TEST_SITE_2/plugins/replication.jar $LOCATION_TEST_SITE_2/lib/replication.jar
+ echo "Copy events broker library"
+ cp -f $DEPLOYMENT_LOCATION/events-broker.jar $LOCATION_TEST_SITE_1/lib/events-broker.jar
- echo "Link multi-site library to plugin directory"
- ln -s $LOCATION_TEST_SITE_1/lib/multi-site.jar $LOCATION_TEST_SITE_1/plugins/multi-site.jar
- ln -s $LOCATION_TEST_SITE_2/lib/multi-site.jar $LOCATION_TEST_SITE_2/plugins/multi-site.jar
+ echo "Copy kafka events plugin"
+ cp -f $DEPLOYMENT_LOCATION/kafka-events.jar $LOCATION_TEST_SITE_1/plugins/kafka-events.jar
+
+ echo "Copy metrics-reporter-prometheus plugin"
+ cp -f $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar $LOCATION_TEST_SITE_1/plugins/metrics-reporter-prometheus.jar
+
+ echo "Re-indexing"
+ java -jar $DEPLOYMENT_LOCATION/gerrit.war reindex -d $LOCATION_TEST_SITE_1
+ # Replicating environment
+ echo "Replicating environment"
+ cp -fR $LOCATION_TEST_SITE_1/* $LOCATION_TEST_SITE_2
+
+ echo "Link replication plugin"
+ ln -s $LOCATION_TEST_SITE_1/plugins/replication.jar $LOCATION_TEST_SITE_1/lib/replication.jar
+ ln -s $LOCATION_TEST_SITE_2/plugins/replication.jar $LOCATION_TEST_SITE_2/lib/replication.jar
+
+ echo "Link multi-site library to plugin directory"
+ ln -s $LOCATION_TEST_SITE_1/lib/multi-site.jar $LOCATION_TEST_SITE_1/plugins/multi-site.jar
+ ln -s $LOCATION_TEST_SITE_2/lib/multi-site.jar $LOCATION_TEST_SITE_2/plugins/multi-site.jar
fi
+DOCKER_HOST_ENV=$(docker_host_env)
+echo "Docker host environment: $DOCKER_HOST_ENV"
+if [ "$DOCKER_HOST_ENV" = "mac" ];then
+ export GERRIT_SITE_HOST="host.docker.internal"
+ export NETWORK_MODE="bridge"
+else
+ export GERRIT_SITE_HOST="localhost"
+ export NETWORK_MODE="host"
+fi
+
+cat $SCRIPT_DIR/configs/prometheus.yml | envsubst > $COMMON_LOCATION/prometheus.yml
IS_KAFKA_RUNNING=$(check_if_kafka_is_running)
if [ $IS_KAFKA_RUNNING -lt 1 ];then
- echo "Starting zk and kafka"
- docker-compose -f $SCRIPT_DIR/docker-compose.kafka-broker.yaml up -d
- echo "Waiting for kafka to start..."
- while [[ $(check_if_kafka_is_running) -lt 1 ]];do sleep 10s; done
+ echo "Starting zk and kafka"
+ docker-compose -f $SCRIPT_DIR/docker-compose.yaml up -d
+ echo "Waiting for kafka to start..."
+ while [[ $(check_if_kafka_is_running) -lt 1 ]];do sleep 10s; done
fi
echo "Re-deploying configuration files"
@@ -398,8 +463,8 @@
if [[ $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}' | wc -l) -lt 1 ]];then
- echo "Starting haproxy"
- start_ha_proxy
+ echo "Starting haproxy"
+ start_ha_proxy
fi
echo "==============================="
@@ -415,6 +480,7 @@
echo "GERRIT HA-PROXY: $GERRIT_CANONICAL_WEB_URL"
echo "GERRIT-1: http://$GERRIT_1_HOSTNAME:$GERRIT_1_HTTPD_PORT"
echo "GERRIT-2: http://$GERRIT_2_HOSTNAME:$GERRIT_2_HTTPD_PORT"
+echo "Prometheus: http://localhost:9090"
echo
echo "Site-1: $LOCATION_TEST_SITE_1"
echo "Site-2: $LOCATION_TEST_SITE_2"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
index ca01471..ebe28d4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
@@ -18,6 +18,7 @@
import static com.google.common.base.Suppliers.ofInstance;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
@@ -49,7 +50,6 @@
static final String INSTANCE_ID_FILE = "instanceId.data";
static final String THREAD_POOL_SIZE_KEY = "threadPoolSize";
static final int DEFAULT_THREAD_POOL_SIZE = 4;
- static final String ENABLE_KEY = "enabled";
private static final String REPLICATION_CONFIG = "replication.config";
// common parameters to cache and index sections
@@ -63,6 +63,7 @@
private final Supplier<Index> index;
private final Supplier<SharedRefDatabase> sharedRefDb;
private final Supplier<Collection<Message>> replicationConfigValidation;
+ private final Supplier<Broker> broker;
private final Config multiSiteConfig;
@Inject
@@ -79,6 +80,7 @@
event = memoize(() -> new Event(lazyMultiSiteCfg));
index = memoize(() -> new Index(lazyMultiSiteCfg));
sharedRefDb = memoize(() -> new SharedRefDatabase(lazyMultiSiteCfg));
+ broker = memoize(() -> new Broker(lazyMultiSiteCfg));
}
public Config getMultiSiteConfig() {
@@ -101,6 +103,10 @@
return index.get();
}
+ public Broker broker() {
+ return broker.get();
+ }
+
public Collection<Message> validate() {
return replicationConfigValidation.get();
}
@@ -162,6 +168,7 @@
public static class SharedRefDatabase {
public static final String SECTION = "ref-database";
+ public static final String ENABLE_KEY = "enabled";
public static final String SUBSECTION_ENFORCEMENT_RULES = "enforcementRules";
private final boolean enabled;
@@ -278,6 +285,19 @@
}
}
+ public static class Broker {
+ static final String BROKER_SECTION = "broker";
+ private final Config cfg;
+
+ Broker(Supplier<Config> cfgSupplier) {
+ cfg = cfgSupplier.get();
+ }
+
+ public String getTopic(String topicKey, String defValue) {
+ return MoreObjects.firstNonNull(cfg.getString(BROKER_SECTION, null, topicKey), defValue);
+ }
+ }
+
static boolean getBoolean(
Supplier<Config> cfg, String section, String subsection, String name, boolean defaultValue) {
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/GitModule.java
similarity index 64%
rename from src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java
rename to src/main/java/com/googlesource/gerrit/plugins/multisite/GitModule.java
index ff932da..4f7205d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/GitModule.java
@@ -14,21 +14,22 @@
package com.googlesource.gerrit.plugins.multisite;
-import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.inject.AbstractModule;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.validation.ValidationModule;
-@Singleton
-public class GerritNoteDbStatus implements NoteDbStatus {
- private final NotesMigration notesMigration;
+public class GitModule extends AbstractModule {
+ private final Configuration config;
@Inject
- public GerritNoteDbStatus(NotesMigration notesMigration) {
- this.notesMigration = notesMigration;
+ public GitModule(Configuration config) {
+ this.config = config;
}
@Override
- public boolean enabled() {
- return notesMigration.commitChangeWrites();
+ protected void configure() {
+ if (config.getSharedRefDb().isEnabled()) {
+ install(new ValidationModule(config));
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jMessageLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jMessageLogger.java
index db7dd63..7c88655 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jMessageLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jMessageLogger.java
@@ -14,12 +14,14 @@
package com.googlesource.gerrit.plugins.multisite;
+import com.gerritforge.gerrit.eventbroker.EventGsonProvider;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
import com.google.gerrit.extensions.systemstatus.ServerInformation;
import com.google.gerrit.server.util.PluginLogFile;
import com.google.gerrit.server.util.SystemLog;
+import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
import org.apache.log4j.PatternLayout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,15 +30,18 @@
public class Log4jMessageLogger extends PluginLogFile implements MessageLogger {
private static final String LOG_NAME = "message_log";
private final Logger msgLog;
+ private final Gson gson;
@Inject
- public Log4jMessageLogger(SystemLog systemLog, ServerInformation serverInfo) {
+ public Log4jMessageLogger(
+ SystemLog systemLog, ServerInformation serverInfo, EventGsonProvider gsonProvider) {
super(systemLog, serverInfo, LOG_NAME, new PatternLayout("[%d{ISO8601}] [%t] %-5p : %m%n"));
- msgLog = LoggerFactory.getLogger(LOG_NAME);
+ this.msgLog = LoggerFactory.getLogger(LOG_NAME);
+ this.gson = gsonProvider.get();
}
@Override
- public void log(Direction direction, SourceAwareEventWrapper event) {
- msgLog.info("{} Header[{}] Body[{}]", direction, event.getHeader(), event.getBody());
+ public void log(Direction direction, String topic, EventMessage event) {
+ msgLog.info("{} {} {}", direction, topic, gson.toJson(event));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java
new file mode 100644
index 0000000..7e38e06
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jProjectVersionLogger.java
@@ -0,0 +1,48 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite;
+
+import com.google.gerrit.reviewdb.client.Project.NameKey;
+import com.google.gerrit.server.util.SystemLog;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.apache.log4j.PatternLayout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class Log4jProjectVersionLogger extends LibModuleLogFile implements ProjectVersionLogger {
+ private static final String LOG_NAME = "project_version_log";
+ private final Logger verLog;
+
+ @Inject
+ public Log4jProjectVersionLogger(SystemLog systemLog) {
+ super(systemLog, LOG_NAME, new PatternLayout("[%d{ISO8601}] [%t] %-5p : %m%n"));
+ this.verLog = LoggerFactory.getLogger(LOG_NAME);
+ }
+
+ @Override
+ public void log(NameKey projectName, long currentVersion, long replicationLag) {
+ if (replicationLag > 0) {
+ verLog.warn(
+ "{ \"project\":\"{}\", \"version\":{}, \"lag\":{} }",
+ projectName,
+ currentVersion,
+ replicationLag);
+ } else {
+ verLog.info("{ \"project\":\"{}\", \"version\":{} }", projectName, currentVersion);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
index 8830e19..95202b5 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
@@ -20,9 +20,9 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.common.GitPerson;
+import com.google.gerrit.json.OutputFormat;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.CommonConverters;
-import com.google.gerrit.server.OutputFormat;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.util.SystemLog;
import com.google.gson.Gson;
@@ -101,6 +101,19 @@
}
@Override
+ public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {
+ if (newRefValue != null) {
+ sharedRefDBLog.info(
+ gson.toJson(
+ new SharedRefLogEntry.UpdateRef(
+ project, refName, safeToString(currRef), safeToString(newRefValue), null, null)));
+ } else {
+ sharedRefDBLog.info(
+ gson.toJson(new SharedRefLogEntry.DeleteRef(project, refName, safeToString(currRef))));
+ }
+ }
+
+ @Override
public void logProjectDelete(String project) {
sharedRefDBLog.info(gson.toJson(new SharedRefLogEntry.DeleteProject(project)));
}
@@ -119,4 +132,11 @@
public void setLogger(Logger logger) {
this.sharedRefDBLog = logger;
}
+
+ private <T> String safeToString(T currRef) {
+ if (currRef == null) {
+ return "<null>";
+ }
+ return currRef.toString();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/MessageLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/MessageLogger.java
index 8b07115..b1f3e79 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/MessageLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/MessageLogger.java
@@ -14,7 +14,7 @@
package com.googlesource.gerrit.plugins.multisite;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
public interface MessageLogger {
@@ -23,5 +23,5 @@
CONSUME;
}
- public void log(Direction direction, SourceAwareEventWrapper event);
+ public void log(Direction direction, String topic, EventMessage event);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Module.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Module.java
index 5f4f635..708e707 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Module.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Module.java
@@ -14,25 +14,22 @@
package com.googlesource.gerrit.plugins.multisite;
-import com.google.common.annotations.VisibleForTesting;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase;
+import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.SitePaths;
-import com.google.gson.Gson;
import com.google.inject.CreationException;
import com.google.inject.Inject;
import com.google.inject.Provides;
-import com.google.inject.ProvisionException;
+import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.spi.Message;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerModule;
-import com.googlesource.gerrit.plugins.multisite.broker.GsonProvider;
import com.googlesource.gerrit.plugins.multisite.cache.CacheModule;
import com.googlesource.gerrit.plugins.multisite.event.EventModule;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwarderModule;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.RouterModule;
import com.googlesource.gerrit.plugins.multisite.index.IndexModule;
-import com.googlesource.gerrit.plugins.multisite.validation.ValidationModule;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.NoopSharedRefDatabase;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
@@ -47,44 +44,26 @@
public class Module extends LifecycleModule {
private static final Logger log = LoggerFactory.getLogger(Module.class);
private Configuration config;
- private NoteDbStatus noteDb;
- private BrokerModule brokerModule;
- private final boolean disableGitRepositoryValidation;
@Inject
- public Module(Configuration config, NoteDbStatus noteDb, BrokerModule brokerModule) {
- this(config, noteDb, brokerModule, false);
- }
-
- // TODO: It is not possible to properly test the libModules in Gerrit.
- // Disable the Git repository validation during integration test and then build the necessary
- // support
- // in Gerrit for it.
- @VisibleForTesting
- public Module(
- Configuration config,
- NoteDbStatus noteDb,
- BrokerModule brokerModule,
- boolean disableGitRepositoryValidation) {
+ public Module(Configuration config) {
this.config = config;
- this.noteDb = noteDb;
- this.brokerModule = brokerModule;
- this.disableGitRepositoryValidation = disableGitRepositoryValidation;
}
@Override
protected void configure() {
- if (!noteDb.enabled()) {
- throw new ProvisionException(
- "Gerrit is still running on ReviewDb: please migrate to NoteDb "
- + "and then reload the multi-site plugin.");
- }
Collection<Message> validationErrors = config.validate();
if (!validationErrors.isEmpty()) {
throw new CreationException(validationErrors);
}
+ DynamicItem.itemOf(binder(), GlobalRefDatabase.class);
+ DynamicItem.bind(binder(), GlobalRefDatabase.class)
+ .to(NoopSharedRefDatabase.class)
+ .in(Scopes.SINGLETON);
+ log.info("Shared ref-db engine: none");
+
listener().to(Log4jMessageLogger.class);
bind(MessageLogger.class).to(Log4jMessageLogger.class);
@@ -100,18 +79,7 @@
install(new IndexModule());
}
- install(brokerModule);
-
install(new RouterModule());
-
- install(
- new ValidationModule(
- config, disableGitRepositoryValidation || !config.getSharedRefDb().isEnabled()));
-
- bind(Gson.class)
- .annotatedWith(BrokerGson.class)
- .toProvider(GsonProvider.class)
- .in(Singleton.class);
}
@Provides
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/NoteDbStatus.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/NoteDbStatus.java
deleted file mode 100644
index f47e503..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/NoteDbStatus.java
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite;
-
-import com.google.inject.ImplementedBy;
-
-/** Returns the status of changes migration. */
-@ImplementedBy(GerritNoteDbStatus.class)
-public interface NoteDbStatus {
-
- /**
- * Status of NoteDb migration.
- *
- * @return true if Gerrit has been migrated to NoteDb
- */
- boolean enabled();
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java
index d445251..2eaa5d9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java
@@ -1,4 +1,4 @@
-// Copyright (C) 2015 The Android Open Source Project
+// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,43 +14,37 @@
package com.googlesource.gerrit.plugins.multisite;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.extensions.events.ProjectDeletedListener;
+import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.inject.Inject;
import com.google.inject.Scopes;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
-import com.googlesource.gerrit.plugins.multisite.kafka.KafkaBrokerApi;
-import com.googlesource.gerrit.plugins.multisite.kafka.KafkaBrokerModule;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZkValidationModule;
+import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
+import com.googlesource.gerrit.plugins.multisite.consumer.MultiSiteConsumerRunner;
+import com.googlesource.gerrit.plugins.multisite.consumer.SubscriberModule;
+import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerForwarderModule;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectDeletedSharedDbCleanup;
public class PluginModule extends LifecycleModule {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
private Configuration config;
- private ZkValidationModule zkValidationModule;
- private KafkaBrokerModule kafkaBrokerModule;
@Inject
- public PluginModule(
- Configuration config,
- ZkValidationModule zkValidationModule,
- KafkaBrokerModule kafkaBrokerModule) {
+ public PluginModule(Configuration config) {
this.config = config;
- this.zkValidationModule = zkValidationModule;
- this.kafkaBrokerModule = kafkaBrokerModule;
}
@Override
protected void configure() {
+ bind(BrokerApiWrapper.class).in(Scopes.SINGLETON);
+ install(new SubscriberModule());
+
+ install(new BrokerForwarderModule());
+ listener().to(MultiSiteConsumerRunner.class);
+
if (config.getSharedRefDb().isEnabled()) {
- logger.atInfo().log("Shared ref-db engine: Zookeeper");
- install(zkValidationModule);
+ listener().to(PluginStartup.class);
+ DynamicSet.bind(binder(), ProjectDeletedListener.class)
+ .to(ProjectDeletedSharedDbCleanup.class);
}
-
- DynamicItem.bind(binder(), BrokerApi.class).to(KafkaBrokerApi.class).in(Scopes.SINGLETON);
- listener().to(KafkaBrokerApi.class);
-
- install(kafkaBrokerModule);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginStartup.java
similarity index 60%
copy from src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java
copy to src/main/java/com/googlesource/gerrit/plugins/multisite/PluginStartup.java
index ff932da..33b54d2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/GerritNoteDbStatus.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginStartup.java
@@ -14,21 +14,25 @@
package com.googlesource.gerrit.plugins.multisite;
-import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import com.google.inject.Injector;
-@Singleton
-public class GerritNoteDbStatus implements NoteDbStatus {
- private final NotesMigration notesMigration;
+public class PluginStartup implements LifecycleListener {
+ private SharedRefDatabaseWrapper sharedRefDb;
+ private Injector injector;
@Inject
- public GerritNoteDbStatus(NotesMigration notesMigration) {
- this.notesMigration = notesMigration;
+ public PluginStartup(SharedRefDatabaseWrapper sharedRefDb, Injector injector) {
+ this.sharedRefDb = sharedRefDb;
+ this.injector = injector;
}
@Override
- public boolean enabled() {
- return notesMigration.commitChangeWrites();
+ public void start() {
+ injector.injectMembers(sharedRefDb);
}
+
+ @Override
+ public void stop() {}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerSession.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java
similarity index 65%
rename from src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerSession.java
rename to src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java
index b04fbff..0ae1cc2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerSession.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/ProjectVersionLogger.java
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 The Android Open Source Project
+// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,15 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.broker;
+package com.googlesource.gerrit.plugins.multisite;
-public interface BrokerSession {
+import com.google.gerrit.reviewdb.client.Project;
- boolean isOpen();
+public interface ProjectVersionLogger {
- void connect();
-
- void disconnect();
-
- boolean publish(String topic, String payload);
+ public void log(Project.NameKey projectName, long currentVersion, long replicationLag);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
index e6ebc08..0ea78a9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
@@ -14,60 +14,87 @@
package com.googlesource.gerrit.plugins.multisite;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedLockException;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import java.io.IOException;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
-public class SharedRefDatabaseWrapper implements SharedRefDatabase {
+public class SharedRefDatabaseWrapper implements GlobalRefDatabase {
- private final DynamicItem<SharedRefDatabase> sharedRefDbDynamicItem;
+ @Inject(optional = true)
+ private DynamicItem<GlobalRefDatabase> sharedRefDbDynamicItem;
+
private final SharedRefLogger sharedRefLogger;
@Inject
- public SharedRefDatabaseWrapper(
- DynamicItem<SharedRefDatabase> sharedRefDbDynamicItem, SharedRefLogger sharedRefLogger) {
- this.sharedRefDbDynamicItem = sharedRefDbDynamicItem;
+ public SharedRefDatabaseWrapper(SharedRefLogger sharedRefLogger) {
this.sharedRefLogger = sharedRefLogger;
}
+ @VisibleForTesting
+ public SharedRefDatabaseWrapper(
+ DynamicItem<GlobalRefDatabase> sharedRefDbDynamicItem, SharedRefLogger sharedRefLogger) {
+ this.sharedRefLogger = sharedRefLogger;
+ this.sharedRefDbDynamicItem = sharedRefDbDynamicItem;
+ }
+
@Override
- public boolean isUpToDate(String project, Ref ref) throws SharedLockException {
+ public boolean isUpToDate(Project.NameKey project, Ref ref) throws GlobalRefDbLockException {
return sharedRefDb().isUpToDate(project, ref);
}
@Override
- public boolean compareAndPut(String project, Ref currRef, ObjectId newRefValue)
- throws IOException {
+ public boolean compareAndPut(Project.NameKey project, Ref currRef, ObjectId newRefValue)
+ throws GlobalRefDbSystemError {
boolean succeeded = sharedRefDb().compareAndPut(project, currRef, newRefValue);
if (succeeded) {
- sharedRefLogger.logRefUpdate(project, currRef, newRefValue);
+ sharedRefLogger.logRefUpdate(project.get(), currRef, newRefValue);
}
return succeeded;
}
@Override
- public AutoCloseable lockRef(String project, String refName) throws SharedLockException {
+ public <T> boolean compareAndPut(Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
+ boolean succeeded = sharedRefDb().compareAndPut(project, refName, currValue, newValue);
+ if (succeeded) {
+ sharedRefLogger.logRefUpdate(project.get(), refName, currValue, newValue);
+ }
+ return succeeded;
+ }
+
+ @Override
+ public AutoCloseable lockRef(Project.NameKey project, String refName)
+ throws GlobalRefDbLockException {
AutoCloseable locker = sharedRefDb().lockRef(project, refName);
- sharedRefLogger.logLockAcquisition(project, refName);
+ sharedRefLogger.logLockAcquisition(project.get(), refName);
return locker;
}
@Override
- public boolean exists(String project, String refName) {
+ public boolean exists(Project.NameKey project, String refName) {
return sharedRefDb().exists(project, refName);
}
@Override
- public void removeProject(String project) throws IOException {
- sharedRefDb().removeProject(project);
- sharedRefLogger.logProjectDelete(project);
+ public void remove(Project.NameKey project) throws GlobalRefDbSystemError {
+ sharedRefDb().remove(project);
+ sharedRefLogger.logProjectDelete(project.get());
}
- private SharedRefDatabase sharedRefDb() {
+ @Override
+ public <T> Optional<T> get(Project.NameKey nameKey, String s, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return sharedRefDb().get(nameKey, s, clazz);
+ }
+
+ private GlobalRefDatabase sharedRefDb() {
return sharedRefDbDynamicItem.get();
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
index 51f9ff0..3853af6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
@@ -21,6 +21,8 @@
void logRefUpdate(String project, Ref currRef, ObjectId newRefValue);
+ <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue);
+
void logProjectDelete(String project);
void logLockAcquisition(String project, String refName);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/ZookeeperConfig.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/ZookeeperConfig.java
deleted file mode 100644
index 35471fa..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/ZookeeperConfig.java
+++ /dev/null
@@ -1,247 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Suppliers.memoize;
-import static com.google.common.base.Suppliers.ofInstance;
-
-import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
-import com.google.gerrit.server.config.SitePaths;
-import java.io.IOException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.BoundedExponentialBackoffRetry;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ZookeeperConfig {
- private static final Logger log = LoggerFactory.getLogger(ZookeeperConfig.class);
- public static final int defaultSessionTimeoutMs;
- public static final int defaultConnectionTimeoutMs;
- public static final String DEFAULT_ZK_CONNECT = "localhost:2181";
- private final int DEFAULT_RETRY_POLICY_BASE_SLEEP_TIME_MS = 1000;
- private final int DEFAULT_RETRY_POLICY_MAX_SLEEP_TIME_MS = 3000;
- private final int DEFAULT_RETRY_POLICY_MAX_RETRIES = 3;
- private final int DEFAULT_CAS_RETRY_POLICY_BASE_SLEEP_TIME_MS = 100;
- private final int DEFAULT_CAS_RETRY_POLICY_MAX_SLEEP_TIME_MS = 300;
- private final int DEFAULT_CAS_RETRY_POLICY_MAX_RETRIES = 3;
- private final int DEFAULT_TRANSACTION_LOCK_TIMEOUT = 1000;
-
- static {
- CuratorFrameworkFactory.Builder b = CuratorFrameworkFactory.builder();
- defaultSessionTimeoutMs = b.getSessionTimeoutMs();
- defaultConnectionTimeoutMs = b.getConnectionTimeoutMs();
- }
-
- public static final String SUBSECTION = "zookeeper";
- public static final String KEY_CONNECT_STRING = "connectString";
- public static final String KEY_SESSION_TIMEOUT_MS = "sessionTimeoutMs";
- public static final String KEY_CONNECTION_TIMEOUT_MS = "connectionTimeoutMs";
- public static final String KEY_RETRY_POLICY_BASE_SLEEP_TIME_MS = "retryPolicyBaseSleepTimeMs";
- public static final String KEY_RETRY_POLICY_MAX_SLEEP_TIME_MS = "retryPolicyMaxSleepTimeMs";
- public static final String KEY_RETRY_POLICY_MAX_RETRIES = "retryPolicyMaxRetries";
- public static final String KEY_ROOT_NODE = "rootNode";
- public final String KEY_CAS_RETRY_POLICY_BASE_SLEEP_TIME_MS = "casRetryPolicyBaseSleepTimeMs";
- public final String KEY_CAS_RETRY_POLICY_MAX_SLEEP_TIME_MS = "casRetryPolicyMaxSleepTimeMs";
- public final String KEY_CAS_RETRY_POLICY_MAX_RETRIES = "casRetryPolicyMaxRetries";
- public final String TRANSACTION_LOCK_TIMEOUT_KEY = "transactionLockTimeoutMs";
-
- private final String connectionString;
- private final String root;
- private final int sessionTimeoutMs;
- private final int connectionTimeoutMs;
- private final int baseSleepTimeMs;
- private final int maxSleepTimeMs;
- private final int maxRetries;
- private final int casBaseSleepTimeMs;
- private final int casMaxSleepTimeMs;
- private final int casMaxRetries;
-
- public static final String SECTION = "ref-database";
- private final Long transactionLockTimeOut;
-
- private CuratorFramework build;
-
- public ZookeeperConfig(Config zkCfg) {
- Supplier<Config> lazyZkConfig = lazyLoad(zkCfg);
- connectionString =
- getString(lazyZkConfig, SECTION, SUBSECTION, KEY_CONNECT_STRING, DEFAULT_ZK_CONNECT);
- root = getString(lazyZkConfig, SECTION, SUBSECTION, KEY_ROOT_NODE, "gerrit/multi-site");
- sessionTimeoutMs =
- getInt(lazyZkConfig, SECTION, SUBSECTION, KEY_SESSION_TIMEOUT_MS, defaultSessionTimeoutMs);
- connectionTimeoutMs =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_CONNECTION_TIMEOUT_MS,
- defaultConnectionTimeoutMs);
-
- baseSleepTimeMs =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_RETRY_POLICY_BASE_SLEEP_TIME_MS,
- DEFAULT_RETRY_POLICY_BASE_SLEEP_TIME_MS);
-
- maxSleepTimeMs =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_RETRY_POLICY_MAX_SLEEP_TIME_MS,
- DEFAULT_RETRY_POLICY_MAX_SLEEP_TIME_MS);
-
- maxRetries =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_RETRY_POLICY_MAX_RETRIES,
- DEFAULT_RETRY_POLICY_MAX_RETRIES);
-
- casBaseSleepTimeMs =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_CAS_RETRY_POLICY_BASE_SLEEP_TIME_MS,
- DEFAULT_CAS_RETRY_POLICY_BASE_SLEEP_TIME_MS);
-
- casMaxSleepTimeMs =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_CAS_RETRY_POLICY_MAX_SLEEP_TIME_MS,
- DEFAULT_CAS_RETRY_POLICY_MAX_SLEEP_TIME_MS);
-
- casMaxRetries =
- getInt(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- KEY_CAS_RETRY_POLICY_MAX_RETRIES,
- DEFAULT_CAS_RETRY_POLICY_MAX_RETRIES);
-
- transactionLockTimeOut =
- getLong(
- lazyZkConfig,
- SECTION,
- SUBSECTION,
- TRANSACTION_LOCK_TIMEOUT_KEY,
- DEFAULT_TRANSACTION_LOCK_TIMEOUT);
-
- checkArgument(StringUtils.isNotEmpty(connectionString), "zookeeper.%s contains no servers");
- }
-
- public CuratorFramework buildCurator() {
- if (build == null) {
- this.build =
- CuratorFrameworkFactory.builder()
- .connectString(connectionString)
- .sessionTimeoutMs(sessionTimeoutMs)
- .connectionTimeoutMs(connectionTimeoutMs)
- .retryPolicy(
- new BoundedExponentialBackoffRetry(baseSleepTimeMs, maxSleepTimeMs, maxRetries))
- .namespace(root)
- .build();
- this.build.start();
- }
-
- return this.build;
- }
-
- public Long getZkInterProcessLockTimeOut() {
- return transactionLockTimeOut;
- }
-
- public RetryPolicy buildCasRetryPolicy() {
- return new BoundedExponentialBackoffRetry(casBaseSleepTimeMs, casMaxSleepTimeMs, casMaxRetries);
- }
-
- private static FileBasedConfig getConfigFile(SitePaths sitePaths, String configFileName) {
- return new FileBasedConfig(sitePaths.etc_dir.resolve(configFileName).toFile(), FS.DETECTED);
- }
-
- private long getLong(
- Supplier<Config> cfg, String section, String subSection, String name, long defaultValue) {
- try {
- return cfg.get().getLong(section, subSection, name, defaultValue);
- } catch (IllegalArgumentException e) {
- log.error("invalid value for {}; using default value {}", name, defaultValue);
- log.debug("Failed to retrieve long value: {}", e.getMessage(), e);
- return defaultValue;
- }
- }
-
- private int getInt(
- Supplier<Config> cfg, String section, String subSection, String name, int defaultValue) {
- try {
- return cfg.get().getInt(section, subSection, name, defaultValue);
- } catch (IllegalArgumentException e) {
- log.error("invalid value for {}; using default value {}", name, defaultValue);
- log.debug("Failed to retrieve integer value: {}", e.getMessage(), e);
- return defaultValue;
- }
- }
-
- private Supplier<Config> lazyLoad(Config config) {
- if (config instanceof FileBasedConfig) {
- return memoize(
- () -> {
- FileBasedConfig fileConfig = (FileBasedConfig) config;
- String fileConfigFileName = fileConfig.getFile().getPath();
- try {
- log.info("Loading configuration from {}", fileConfigFileName);
- fileConfig.load();
- } catch (IOException | ConfigInvalidException e) {
- log.error("Unable to load configuration from " + fileConfigFileName, e);
- }
- return fileConfig;
- });
- }
- return ofInstance(config);
- }
-
- private boolean getBoolean(
- Supplier<Config> cfg, String section, String subsection, String name, boolean defaultValue) {
- try {
- return cfg.get().getBoolean(section, subsection, name, defaultValue);
- } catch (IllegalArgumentException e) {
- log.error("invalid value for {}; using default value {}", name, defaultValue);
- log.debug("Failed to retrieve boolean value: {}", e.getMessage(), e);
- return defaultValue;
- }
- }
-
- private String getString(
- Supplier<Config> cfg, String section, String subsection, String name, String defaultValue) {
- String value = cfg.get().getString(section, subsection, name);
- if (!Strings.isNullOrEmpty(value)) {
- return value;
- }
- return defaultValue;
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApi.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApi.java
deleted file mode 100644
index 35350e9..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApi.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker;
-
-import com.google.gerrit.server.events.Event;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import java.util.function.Consumer;
-
-/** API for sending/receiving events through a message Broker. */
-public interface BrokerApi {
-
- /**
- * Send an event to a topic.
- *
- * @param topic
- * @param event
- * @return true if the event was successfully sent. False otherwise.
- */
- boolean send(String topic, Event event);
-
- /**
- * Receive asynchronously events from a topic.
- *
- * @param topic
- * @param eventConsumer
- */
- void receiveAync(String topic, Consumer<SourceAwareEventWrapper> eventConsumer);
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiNoOp.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiNoOp.java
deleted file mode 100644
index 19cf0f7..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiNoOp.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker;
-
-import com.google.gerrit.server.events.Event;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import java.util.function.Consumer;
-
-public class BrokerApiNoOp implements BrokerApi {
-
- @Override
- public boolean send(String topic, Event event) {
- return true;
- }
-
- @Override
- public void receiveAync(String topic, Consumer<SourceAwareEventWrapper> eventConsumer) {}
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapper.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapper.java
index e83fe53..71be5e6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapper.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapper.java
@@ -14,29 +14,53 @@
package com.googlesource.gerrit.plugins.multisite.broker;
+import com.gerritforge.gerrit.eventbroker.BrokerApi;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+import com.gerritforge.gerrit.eventbroker.TopicSubscriber;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.Event;
import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
+import com.googlesource.gerrit.plugins.multisite.InstanceId;
+import com.googlesource.gerrit.plugins.multisite.MessageLogger;
+import com.googlesource.gerrit.plugins.multisite.MessageLogger.Direction;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import java.util.Set;
+import java.util.UUID;
import java.util.function.Consumer;
public class BrokerApiWrapper implements BrokerApi {
private final DynamicItem<BrokerApi> apiDelegate;
private final BrokerMetrics metrics;
+ private final MessageLogger msgLog;
+ private final UUID instanceId;
@Inject
- public BrokerApiWrapper(DynamicItem<BrokerApi> apiDelegate, BrokerMetrics metrics) {
+ public BrokerApiWrapper(
+ DynamicItem<BrokerApi> apiDelegate,
+ BrokerMetrics metrics,
+ MessageLogger msgLog,
+ @InstanceId UUID instanceId) {
this.apiDelegate = apiDelegate;
this.metrics = metrics;
+ this.msgLog = msgLog;
+ this.instanceId = instanceId;
+ }
+
+ public boolean send(String topic, Event event) {
+ return send(topic, apiDelegate.get().newMessage(instanceId, event));
}
@Override
- public boolean send(String topic, Event event) {
+ public boolean send(String topic, EventMessage message) {
+ if (Context.isForwardedEvent()) {
+ return true;
+ }
boolean succeeded = false;
try {
- succeeded = apiDelegate.get().send(topic, event);
+ succeeded = apiDelegate.get().send(topic, message);
} finally {
if (succeeded) {
+ msgLog.log(Direction.PUBLISH, topic, message);
metrics.incrementBrokerPublishedMessage();
} else {
metrics.incrementBrokerFailedToPublishMessage();
@@ -46,7 +70,22 @@
}
@Override
- public void receiveAync(String topic, Consumer<SourceAwareEventWrapper> eventConsumer) {
- apiDelegate.get().receiveAync(topic, eventConsumer);
+ public void receiveAsync(String topic, Consumer<EventMessage> messageConsumer) {
+ apiDelegate.get().receiveAsync(topic, messageConsumer);
+ }
+
+ @Override
+ public void disconnect() {
+ apiDelegate.get().disconnect();
+ }
+
+ @Override
+ public Set<TopicSubscriber> topicSubscribers() {
+ return apiDelegate.get().topicSubscribers();
+ }
+
+ @Override
+ public void replayAllEvents(String topic) {
+ apiDelegate.get().replayAllEvents(topic);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerGson.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerGson.java
deleted file mode 100644
index 219aa96..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerGson.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker;
-
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import com.google.inject.BindingAnnotation;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-@Retention(RUNTIME)
-@Target(PARAMETER)
-@BindingAnnotation
-public @interface BrokerGson {}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerModule.java
deleted file mode 100644
index 6983984..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerModule.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker;
-
-import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.inject.AbstractModule;
-import com.google.inject.Scopes;
-import com.googlesource.gerrit.plugins.multisite.consumer.SubscriberModule;
-
-public class BrokerModule extends AbstractModule {
-
- @Override
- protected void configure() {
- DynamicItem.itemOf(binder(), BrokerApi.class);
- DynamicItem.bind(binder(), BrokerApi.class).to(BrokerApiNoOp.class).in(Scopes.SINGLETON);
-
- bind(BrokerApiWrapper.class).in(Scopes.SINGLETON);
-
- install(new SubscriberModule());
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/GsonProvider.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/GsonProvider.java
deleted file mode 100644
index 0791e6a..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/GsonProvider.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker;
-
-import com.google.common.base.Supplier;
-import com.google.gerrit.server.events.Event;
-import com.google.gerrit.server.events.EventDeserializer;
-import com.google.gerrit.server.events.SupplierDeserializer;
-import com.google.gerrit.server.events.SupplierSerializer;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.inject.Provider;
-
-public class GsonProvider implements Provider<Gson> {
- @Override
- public Gson get() {
- return new GsonBuilder()
- .registerTypeAdapter(Event.class, new EventDeserializer())
- .registerTypeAdapter(Supplier.class, new SupplierSerializer())
- .registerTypeAdapter(Supplier.class, new SupplierDeserializer())
- .create();
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisher.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisher.java
deleted file mode 100644
index ba5b532..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisher.java
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker.kafka;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.server.events.Event;
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.InstanceId;
-import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.MessageLogger.Direction;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerSession;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
-import java.util.UUID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Singleton
-public class BrokerPublisher implements LifecycleListener {
- protected final Logger log = LoggerFactory.getLogger(getClass());
-
- private final BrokerSession session;
- private final Gson gson;
- private final UUID instanceId;
- private final MessageLogger msgLog;
-
- @Inject
- public BrokerPublisher(
- BrokerSession session,
- @BrokerGson Gson gson,
- @InstanceId UUID instanceId,
- MessageLogger msgLog) {
- this.session = session;
- this.gson = gson;
- this.instanceId = instanceId;
- this.msgLog = msgLog;
- }
-
- @Override
- public void start() {
- if (!session.isOpen()) {
- session.connect();
- }
- }
-
- @Override
- public void stop() {
- if (session.isOpen()) {
- session.disconnect();
- }
- }
-
- public boolean publish(String topic, Event event) {
- if (Context.isForwardedEvent()) {
- return true;
- }
-
- SourceAwareEventWrapper brokerEvent = toBrokerEvent(event);
- Boolean eventPublished = session.publish(topic, getPayload(brokerEvent));
- if (eventPublished) {
- msgLog.log(Direction.PUBLISH, brokerEvent);
- }
- return eventPublished;
- }
-
- private String getPayload(SourceAwareEventWrapper event) {
- return gson.toJson(event);
- }
-
- private SourceAwareEventWrapper toBrokerEvent(Event event) {
- JsonObject body = eventToJson(event);
- return new SourceAwareEventWrapper(
- new SourceAwareEventWrapper.EventHeader(
- UUID.randomUUID(), event.getType(), instanceId, event.eventCreatedOn),
- body);
- }
-
- @VisibleForTesting
- public JsonObject eventToJson(Event event) {
- return gson.toJsonTree(event).getAsJsonObject();
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/KafkaSession.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/KafkaSession.java
deleted file mode 100644
index 6594544..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/KafkaSession.java
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker.kafka;
-
-import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.InstanceId;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerSession;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class KafkaSession implements BrokerSession {
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaSession.class);
- private KafkaConfiguration properties;
- private final UUID instanceId;
- private volatile Producer<String, String> producer;
-
- @Inject
- public KafkaSession(KafkaConfiguration kafkaConfig, @InstanceId UUID instanceId) {
- this.properties = kafkaConfig;
- this.instanceId = instanceId;
- }
-
- @Override
- public boolean isOpen() {
- if (producer != null) {
- return true;
- }
- return false;
- }
-
- @Override
- public void connect() {
- if (isOpen()) {
- LOGGER.debug("Already connected.");
- return;
- }
-
- LOGGER.info("Connect to {}...", properties.getKafka().getBootstrapServers());
- /* Need to make sure that the thread of the running connection uses
- * the correct class loader otherwize you can endup with hard to debug
- * ClassNotFoundExceptions
- */
- setConnectionClassLoader();
- producer = new KafkaProducer<>(properties.kafkaPublisher());
- LOGGER.info("Connection established.");
- }
-
- private void setConnectionClassLoader() {
- Thread.currentThread().setContextClassLoader(KafkaSession.class.getClassLoader());
- }
-
- @Override
- public void disconnect() {
- LOGGER.info("Disconnecting...");
- if (producer != null) {
- LOGGER.info("Closing Producer {}...", producer);
- producer.close();
- }
- producer = null;
- }
-
- @Override
- public boolean publish(String topic, String payload) {
- return publishToTopic(properties.getKafka().getTopicAlias(EventTopic.of(topic)), payload);
- }
-
- private boolean publishToTopic(String topic, String payload) {
- Future<RecordMetadata> future =
- producer.send(new ProducerRecord<>(topic, instanceId.toString(), payload));
- try {
- RecordMetadata metadata = future.get();
- LOGGER.debug("The offset of the record we just sent is: {}", metadata.offset());
- return true;
- } catch (InterruptedException | ExecutionException e) {
- LOGGER.error("Cannot send the message", e);
- return false;
- }
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
index 2308c0e..816edc9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/AbstractSubcriber.java
@@ -14,77 +14,74 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gson.Gson;
-import com.google.gwtorm.server.OrmException;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
import com.googlesource.gerrit.plugins.multisite.MessageLogger.Direction;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.CacheNotFoundException;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.ForwardedEventRouter;
import java.io.IOException;
import java.util.UUID;
+import java.util.function.Consumer;
-public abstract class AbstractSubcriber implements Runnable {
+public abstract class AbstractSubcriber {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
- private final BrokerApi brokerApi;
private final ForwardedEventRouter eventRouter;
private final DynamicSet<DroppedEventListener> droppedEventListeners;
- private final Gson gson;
private final UUID instanceId;
private final MessageLogger msgLog;
private SubscriberMetrics subscriberMetrics;
+ private final Configuration cfg;
+ private final String topic;
public AbstractSubcriber(
- BrokerApiWrapper brokerApi,
ForwardedEventRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gson,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
this.eventRouter = eventRouter;
this.droppedEventListeners = droppedEventListeners;
- this.gson = gson;
this.instanceId = instanceId;
this.msgLog = msgLog;
this.subscriberMetrics = subscriberMetrics;
- this.brokerApi = brokerApi;
- }
-
- @Override
- public void run() {
- brokerApi.receiveAync(getTopic().topic(), this::processRecord);
+ this.cfg = cfg;
+ this.topic = getTopic().topic(cfg);
}
protected abstract EventTopic getTopic();
- private void processRecord(SourceAwareEventWrapper event) {
+ public Consumer<EventMessage> getConsumer() {
+ return this::processRecord;
+ }
- if (event.getHeader().getSourceInstanceId().equals(instanceId)) {
+ private void processRecord(EventMessage event) {
+
+ if (event.getHeader().sourceInstanceId.equals(instanceId)) {
logger.atFiner().log(
"Dropping event %s produced by our instanceId %s",
event.toString(), instanceId.toString());
droppedEventListeners.forEach(l -> l.onEventDropped(event));
} else {
try {
- msgLog.log(Direction.CONSUME, event);
- eventRouter.route(event.getEventBody(gson));
+ msgLog.log(Direction.CONSUME, topic, event);
+ eventRouter.route(event.getEvent());
subscriberMetrics.incrementSubscriberConsumedMessage();
+ subscriberMetrics.updateReplicationStatusMetrics(event);
} catch (IOException e) {
logger.atSevere().withCause(e).log(
- "Malformed event '%s': [Exception: %s]", event.getHeader().getEventType());
+ "Malformed event '%s': [Exception: %s]", event.getHeader());
subscriberMetrics.incrementSubscriberFailedToConsumeMessage();
- } catch (PermissionBackendException | OrmException | CacheNotFoundException e) {
+ } catch (PermissionBackendException | CacheNotFoundException e) {
logger.atSevere().withCause(e).log(
- "Cannot handle message %s: [Exception: %s]", event.getHeader().getEventType());
+ "Cannot handle message %s: [Exception: %s]", event.getHeader());
subscriberMetrics.incrementSubscriberFailedToConsumeMessage();
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java
index 9c430c6..5bbaee0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/BatchIndexEventSubscriber.java
@@ -15,13 +15,11 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.IndexEventRouter;
import java.util.UUID;
@@ -30,21 +28,13 @@
public class BatchIndexEventSubscriber extends AbstractSubcriber {
@Inject
public BatchIndexEventSubscriber(
- BrokerApiWrapper brokerApi,
IndexEventRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gsonProvider,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
- super(
- brokerApi,
- eventRouter,
- droppedEventListeners,
- gsonProvider,
- instanceId,
- msgLog,
- subscriberMetrics);
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/CacheEvictionEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/CacheEvictionEventSubscriber.java
index 572d1a3..eae66b4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/CacheEvictionEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/CacheEvictionEventSubscriber.java
@@ -15,13 +15,11 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.CacheEvictionEventRouter;
import java.util.UUID;
@@ -30,21 +28,14 @@
public class CacheEvictionEventSubscriber extends AbstractSubcriber {
@Inject
public CacheEvictionEventSubscriber(
- BrokerApiWrapper brokerApi,
CacheEvictionEventRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gsonProvider,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
- super(
- brokerApi,
- eventRouter,
- droppedEventListeners,
- gsonProvider,
- instanceId,
- msgLog,
- subscriberMetrics);
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ConsumerExecutor.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ConsumerExecutor.java
deleted file mode 100644
index 936d07a..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ConsumerExecutor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.consumer;
-
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import com.google.inject.BindingAnnotation;
-import java.lang.annotation.Retention;
-
-@Retention(RUNTIME)
-@BindingAnnotation
-public @interface ConsumerExecutor {}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/DroppedEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/DroppedEventListener.java
index 680e8ed..6f4680c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/DroppedEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/DroppedEventListener.java
@@ -14,11 +14,13 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+
public interface DroppedEventListener {
/**
* Invoked when any event is dropped.
*
* @param event information about the event.
*/
- void onEventDropped(SourceAwareEventWrapper event);
+ void onEventDropped(EventMessage event);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
index df55040..49d470a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/IndexEventSubscriber.java
@@ -15,13 +15,11 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.IndexEventRouter;
import java.util.UUID;
@@ -30,21 +28,13 @@
public class IndexEventSubscriber extends AbstractSubcriber {
@Inject
public IndexEventSubscriber(
- BrokerApiWrapper brokerApi,
IndexEventRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gsonProvider,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
- super(
- brokerApi,
- eventRouter,
- droppedEventListeners,
- gsonProvider,
- instanceId,
- msgLog,
- subscriberMetrics);
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
index 1778961..d858b89 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/MultiSiteConsumerRunner.java
@@ -14,36 +14,41 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.gerritforge.gerrit.eventbroker.BrokerApi;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import java.util.concurrent.ExecutorService;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
@Singleton
public class MultiSiteConsumerRunner implements LifecycleListener {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final DynamicSet<AbstractSubcriber> consumers;
- private final ExecutorService executor;
+ private DynamicItem<BrokerApi> brokerApi;
+ private Configuration cfg;
@Inject
public MultiSiteConsumerRunner(
- @ConsumerExecutor ExecutorService executor, DynamicSet<AbstractSubcriber> consumers) {
+ DynamicItem<BrokerApi> brokerApi,
+ DynamicSet<AbstractSubcriber> consumers,
+ Configuration cfg) {
this.consumers = consumers;
- this.executor = executor;
+ this.brokerApi = brokerApi;
+ this.cfg = cfg;
}
@Override
public void start() {
logger.atInfo().log("starting consumers");
- consumers.forEach(c -> executor.execute(c));
+ consumers.forEach(
+ consumer ->
+ brokerApi.get().receiveAsync(consumer.getTopic().topic(cfg), consumer.getConsumer()));
}
@Override
- public void stop() {
- logger.atInfo().log("shutting down consumers");
- executor.shutdown();
- }
+ public void stop() {}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
index 5c42ea6..6ff0969 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ProjectUpdateEventSubscriber.java
@@ -15,13 +15,11 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.ProjectListUpdateRouter;
import java.util.UUID;
@@ -30,15 +28,13 @@
public class ProjectUpdateEventSubscriber extends AbstractSubcriber {
@Inject
public ProjectUpdateEventSubscriber(
- BrokerApiWrapper brokerApi,
ProjectListUpdateRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gson,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
- super(
- brokerApi, eventRouter, droppedEventListeners, gson, instanceId, msgLog, subscriberMetrics);
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SourceAwareEventWrapper.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SourceAwareEventWrapper.java
deleted file mode 100644
index b8bd0d8..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SourceAwareEventWrapper.java
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.consumer;
-
-import static java.util.Objects.requireNonNull;
-
-import com.google.gerrit.server.events.Event;
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import java.util.UUID;
-
-public class SourceAwareEventWrapper {
-
- private final EventHeader header;
- private final JsonObject body;
-
- public EventHeader getHeader() {
- return header;
- }
-
- public JsonObject getBody() {
- return body;
- }
-
- public Event getEventBody(Gson gson) {
- return gson.fromJson(this.body, Event.class);
- }
-
- public static class EventHeader {
- private final UUID eventId;
- private final String eventType;
- private final UUID sourceInstanceId;
- private final Long eventCreatedOn;
-
- public EventHeader(UUID eventId, String eventType, UUID sourceInstanceId, Long eventCreatedOn) {
- this.eventId = eventId;
- this.eventType = eventType;
- this.sourceInstanceId = sourceInstanceId;
- this.eventCreatedOn = eventCreatedOn;
- }
-
- public UUID getEventId() {
- return eventId;
- }
-
- public String getEventType() {
- return eventType;
- }
-
- public UUID getSourceInstanceId() {
- return sourceInstanceId;
- }
-
- public Long getEventCreatedOn() {
- return eventCreatedOn;
- }
-
- public void validate() {
- requireNonNull(eventId, "EventId cannot be null");
- requireNonNull(eventType, "EventType cannot be null");
- requireNonNull(sourceInstanceId, "Source Instance ID cannot be null");
- }
-
- @Override
- public String toString() {
- return "{"
- + "eventId="
- + eventId
- + ", eventType='"
- + eventType
- + '\''
- + ", sourceInstanceId="
- + sourceInstanceId
- + ", eventCreatedOn="
- + eventCreatedOn
- + '}';
- }
- }
-
- public SourceAwareEventWrapper(EventHeader header, JsonObject body) {
- this.header = header;
- this.body = body;
- }
-
- public void validate() {
- requireNonNull(header, "Header cannot be null");
- requireNonNull(body, "Body cannot be null");
- header.validate();
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
index b48ab31..20c355e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/StreamEventSubscriber.java
@@ -15,13 +15,11 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.InstanceId;
import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.router.StreamEventRouter;
import java.util.UUID;
@@ -30,15 +28,13 @@
public class StreamEventSubscriber extends AbstractSubcriber {
@Inject
public StreamEventSubscriber(
- BrokerApiWrapper brokerApi,
StreamEventRouter eventRouter,
DynamicSet<DroppedEventListener> droppedEventListeners,
- @BrokerGson Gson gson,
@InstanceId UUID instanceId,
MessageLogger msgLog,
- SubscriberMetrics subscriberMetrics) {
- super(
- brokerApi, eventRouter, droppedEventListeners, gson, instanceId, msgLog, subscriberMetrics);
+ SubscriberMetrics subscriberMetrics,
+ Configuration cfg) {
+ super(eventRouter, droppedEventListeners, instanceId, msgLog, subscriberMetrics, cfg);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
index 996b581..1474262 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
@@ -14,28 +14,53 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.metrics.Counter1;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.MetricMaker;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
+import com.googlesource.gerrit.plugins.replication.RefReplicatedEvent;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
+import com.googlesource.gerrit.plugins.replication.ReplicationScheduledEvent;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
@Singleton
public class SubscriberMetrics {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String SUBSCRIBER_SUCCESS_COUNTER = "subscriber_msg_consumer_counter";
private static final String SUBSCRIBER_FAILURE_COUNTER =
"subscriber_msg_consumer_failure_counter";
- private static final String SUBSCRIBER_POLL_FAILURE_COUNTER =
- "subscriber_msg_consumer_poll_failure_counter";
+ private static final String REPLICATION_LAG_SEC =
+ "multi_site/subscriber/subscriber_replication_status/sec_behind";
private final Counter1<String> subscriberSuccessCounter;
private final Counter1<String> subscriberFailureCounter;
- private final Counter1<String> subscriberPollFailureCounter;
+ private final ProjectVersionLogger verLogger;
+
+ private final Map<String, Long> replicationStatusPerProject = new HashMap<>();
+ private final Map<String, Long> localVersionPerProject = new HashMap<>();
+
+ private ProjectVersionRefUpdate projectVersionRefUpdate;
@Inject
- public SubscriberMetrics(MetricMaker metricMaker) {
+ public SubscriberMetrics(
+ MetricMaker metricMaker,
+ ProjectVersionRefUpdate projectVersionRefUpdate,
+ ProjectVersionLogger verLogger) {
+ this.projectVersionRefUpdate = projectVersionRefUpdate;
this.subscriberSuccessCounter =
metricMaker.newCounter(
"multi_site/subscriber/subscriber_message_consumer_counter",
@@ -51,15 +76,19 @@
.setUnit("errors"),
Field.ofString(
SUBSCRIBER_FAILURE_COUNTER, "Subscriber failed to consume messages count"));
+ metricMaker.newCallbackMetric(
+ REPLICATION_LAG_SEC,
+ Long.class,
+ new Description("Replication lag (sec)").setGauge().setUnit(Description.Units.SECONDS),
+ () -> {
+ Collection<Long> lags = replicationStatusPerProject.values();
+ if (lags.isEmpty()) {
+ return 0L;
+ }
+ return Collections.max(lags);
+ });
- this.subscriberPollFailureCounter =
- metricMaker.newCounter(
- "multi_site/subscriber/subscriber_message_consumer_poll_failure_counter",
- new Description("Number of failed attempts to poll messages by the subscriber")
- .setRate()
- .setUnit("errors"),
- Field.ofString(
- SUBSCRIBER_POLL_FAILURE_COUNTER, "Subscriber failed to poll messages count"));
+ this.verLogger = verLogger;
}
public void incrementSubscriberConsumedMessage() {
@@ -70,7 +99,44 @@
subscriberFailureCounter.increment(SUBSCRIBER_FAILURE_COUNTER);
}
- public void incrementSubscriberFailedToPollMessages() {
- subscriberPollFailureCounter.increment(SUBSCRIBER_POLL_FAILURE_COUNTER);
+ public void updateReplicationStatusMetrics(EventMessage eventMessage) {
+ Event event = eventMessage.getEvent();
+ if (event instanceof RefReplicationDoneEvent) {
+ RefReplicationDoneEvent replicationDone = (RefReplicationDoneEvent) event;
+ updateReplicationLagMetrics(
+ replicationDone.getProjectNameKey(), replicationDone.getRefName());
+ } else if (event instanceof RefReplicatedEvent) {
+ RefReplicatedEvent replicated = (RefReplicatedEvent) event;
+ updateReplicationLagMetrics(replicated.getProjectNameKey(), replicated.getRefName());
+ } else if (event instanceof ReplicationScheduledEvent) {
+ ReplicationScheduledEvent updated = (ReplicationScheduledEvent) event;
+ updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+ } else if (event instanceof RefUpdatedEvent) {
+ RefUpdatedEvent updated = (RefUpdatedEvent) event;
+ updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+ }
+ }
+
+ private void updateReplicationLagMetrics(Project.NameKey projectName, String ref) {
+ Optional<Long> remoteVersion =
+ projectVersionRefUpdate.getProjectRemoteVersion(projectName.get());
+ Optional<Long> localVersion = projectVersionRefUpdate.getProjectLocalVersion(projectName.get());
+ if (remoteVersion.isPresent() && localVersion.isPresent()) {
+ long lag = remoteVersion.get() - localVersion.get();
+
+ if (!localVersion.get().equals(localVersionPerProject.get(projectName.get()))
+ || lag != replicationStatusPerProject.get(projectName.get())) {
+ logger.atFine().log(
+ "Published replication lag metric for project '%s' of %d sec(s) [local-ref=%d global-ref=%d]",
+ projectName, lag, localVersion.get(), remoteVersion.get());
+ replicationStatusPerProject.put(projectName.get(), lag);
+ localVersionPerProject.put(projectName.get(), localVersion.get());
+ verLogger.log(projectName, localVersion.get(), lag);
+ }
+ } else {
+ logger.atFine().log(
+ "Did not publish replication lag metric for %s because the %s version is not defined",
+ projectName, localVersion.isPresent() ? "remote" : "local");
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
index 0a0c350..09adb18 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberModule.java
@@ -16,22 +16,21 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.lifecycle.LifecycleModule;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.MultiSiteEvent;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
public class SubscriberModule extends LifecycleModule {
@Override
protected void configure() {
MultiSiteEvent.registerEventTypes();
- bind(ExecutorService.class)
- .annotatedWith(ConsumerExecutor.class)
- .toInstance(Executors.newFixedThreadPool(EventTopic.values().length));
- listener().to(MultiSiteConsumerRunner.class);
DynamicSet.setOf(binder(), AbstractSubcriber.class);
DynamicSet.setOf(binder(), DroppedEventListener.class);
+
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(IndexEventSubscriber.class);
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(BatchIndexEventSubscriber.class);
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(StreamEventSubscriber.class);
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(CacheEvictionEventSubscriber.class);
+ DynamicSet.bind(binder(), AbstractSubcriber.class).to(ProjectUpdateEventSubscriber.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
index 8889935..1c0c644 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
@@ -17,6 +17,7 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.events.EventListener;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
import java.util.concurrent.Executor;
public class EventModule extends LifecycleModule {
@@ -26,5 +27,6 @@
bind(Executor.class).annotatedWith(EventExecutor.class).toProvider(EventExecutorProvider.class);
listener().to(EventExecutorProvider.class);
DynamicSet.bind(binder(), EventListener.class).to(EventHandler.class);
+ DynamicSet.bind(binder(), EventListener.class).to(ProjectVersionRefUpdate.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedAwareEventBroker.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedAwareEventBroker.java
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedAwareEventBroker.java
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandler.java
index 85dab30..278ba4e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandler.java
@@ -19,7 +19,6 @@
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.slf4j.Logger;
@@ -47,9 +46,8 @@
* Dispatch an event in the local node, event will not be forwarded to the other node.
*
* @param event The event to dispatch
- * @throws OrmException If an error occur while retrieving the change the event belongs to.
*/
- public void dispatch(Event event) throws OrmException, PermissionBackendException {
+ public void dispatch(Event event) throws PermissionBackendException {
try (ManualRequestContext ctx = oneOffCtx.open()) {
Context.setForwardedEvent(true);
log.debug("dispatching event {}", event.getType());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
index fb7aef1..6d1d6cf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexAccountHandler.java
@@ -16,7 +16,6 @@
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.index.account.AccountIndexer;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.Configuration;
@@ -49,7 +48,7 @@
}
@Override
- protected void doIndex(Account.Id id, Optional<AccountIndexEvent> event) throws IOException {
+ protected void doIndex(Account.Id id, Optional<AccountIndexEvent> event) {
indexer.index(id);
log.debug("Account {} successfully indexed", id);
}
@@ -74,7 +73,7 @@
try {
index(account.getKey(), account.getValue(), Optional.empty());
return true;
- } catch (IOException | OrmException e) {
+ } catch (IOException e) {
log.error("Account {} index failed", account.getKey(), e);
return false;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandler.java
index 1913b00..8340a5f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandler.java
@@ -18,10 +18,8 @@
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
-import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.Configuration;
@@ -30,7 +28,6 @@
import com.googlesource.gerrit.plugins.multisite.index.ChangeChecker;
import com.googlesource.gerrit.plugins.multisite.index.ChangeCheckerImpl;
import com.googlesource.gerrit.plugins.multisite.index.ForwardedIndexExecutor;
-import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
@@ -71,72 +68,48 @@
}
@Override
- protected void doIndex(String id, Optional<ChangeIndexEvent> indexEvent)
- throws IOException, OrmException {
+ protected void doIndex(String id, Optional<ChangeIndexEvent> indexEvent) {
doIndex(id, indexEvent, 0);
}
- private void doIndex(String id, Optional<ChangeIndexEvent> indexEvent, int retryCount)
- throws IOException, OrmException {
- try {
- ChangeChecker checker = changeCheckerFactory.create(id);
- Optional<ChangeNotes> changeNotes = checker.getChangeNotes();
- if (changeNotes.isPresent()) {
- ChangeNotes notes = changeNotes.get();
- reindex(notes);
+ private void doIndex(String id, Optional<ChangeIndexEvent> indexEvent, int retryCount) {
+ ChangeChecker checker = changeCheckerFactory.create(id);
+ Optional<ChangeNotes> changeNotes = checker.getChangeNotes();
+ if (changeNotes.isPresent()) {
+ ChangeNotes notes = changeNotes.get();
+ reindex(notes);
- if (checker.isChangeUpToDate(indexEvent)) {
- if (retryCount > 0) {
- log.warn("Change {} has been eventually indexed after {} attempt(s)", id, retryCount);
- } else {
- log.debug("Change {} successfully indexed", id);
- }
+ if (checker.isChangeUpToDate(indexEvent)) {
+ if (retryCount > 0) {
+ log.warn("Change {} has been eventually indexed after {} attempt(s)", id, retryCount);
} else {
- log.warn(
- "Change {} seems too old compared to the event timestamp (event={} >> change-Ts={})",
- id,
- indexEvent,
- checker);
- rescheduleIndex(id, indexEvent, retryCount + 1);
+ log.debug("Change {} successfully indexed", id);
}
} else {
log.warn(
- "Change {} not present yet in local Git repository (event={}) after {} attempt(s)",
+ "Change {} seems too old compared to the event timestamp (event={} >> change-Ts={})",
id,
indexEvent,
- retryCount);
- if (!rescheduleIndex(id, indexEvent, retryCount + 1)) {
- log.error(
- "Change {} could not be found in the local Git repository (event={})",
- id,
- indexEvent);
- }
+ checker);
+ rescheduleIndex(id, indexEvent, retryCount + 1);
}
- } catch (Exception e) {
- if (isCausedByNoSuchChangeException(e)) {
- indexer.delete(parseChangeId(id));
- log.warn("Error trying to index Change {}. Deleted from index", id, e);
- return;
+ } else {
+ log.warn(
+ "Change {} not present yet in local Git repository (event={}) after {} attempt(s)",
+ id,
+ indexEvent,
+ retryCount);
+ if (!rescheduleIndex(id, indexEvent, retryCount + 1)) {
+ log.error(
+ "Change {} could not be found in the local Git repository (event={})", id, indexEvent);
}
-
- throw e;
}
}
- private static boolean isCausedByNoSuchChangeException(Throwable throwable) {
- while (throwable != null) {
- if (throwable instanceof NoSuchChangeException) {
- return true;
- }
- throwable = throwable.getCause();
- }
- return false;
- }
-
- private void reindex(ChangeNotes notes) throws IOException, OrmException {
+ private void reindex(ChangeNotes notes) {
try (ManualRequestContext ctx = oneOffCtx.open()) {
notes.reload();
- indexer.index(ctx.getReviewDbProvider().get(), notes.getChange());
+ indexer.index(notes.getChange());
}
}
@@ -172,7 +145,7 @@
}
@Override
- protected void doDelete(String id, Optional<ChangeIndexEvent> indexEvent) throws IOException {
+ protected void doDelete(String id, Optional<ChangeIndexEvent> indexEvent) {
indexer.delete(parseChangeId(id));
log.debug("Change {} successfully deleted from index", id);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexGroupHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexGroupHandler.java
index 76ce260..368dffe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexGroupHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexGroupHandler.java
@@ -16,12 +16,10 @@
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.index.group.GroupIndexer;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.GroupIndexEvent;
-import java.io.IOException;
import java.util.Optional;
/**
@@ -41,8 +39,7 @@
}
@Override
- protected void doIndex(String uuid, Optional<GroupIndexEvent> event)
- throws IOException, OrmException {
+ protected void doIndex(String uuid, Optional<GroupIndexEvent> event) {
indexer.index(new AccountGroup.UUID(uuid));
log.debug("Group {} successfully indexed", uuid);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexProjectHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexProjectHandler.java
index ff2e111..e5f7e10 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexProjectHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexProjectHandler.java
@@ -22,7 +22,6 @@
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ProjectIndexEvent;
import com.googlesource.gerrit.plugins.multisite.index.ForwardedIndexExecutor;
import com.googlesource.gerrit.plugins.multisite.index.ProjectChecker;
-import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -58,15 +57,14 @@
}
@Override
- protected void doIndex(String projectName, Optional<ProjectIndexEvent> event) throws IOException {
+ protected void doIndex(String projectName, Optional<ProjectIndexEvent> event) {
if (!attemptIndex(projectName, event)) {
log.warn("First Attempt failed, scheduling again after {} msecs", retryInterval);
rescheduleIndex(projectName, event, 1);
}
}
- public boolean attemptIndex(String projectName, Optional<ProjectIndexEvent> event)
- throws IOException {
+ public boolean attemptIndex(String projectName, Optional<ProjectIndexEvent> event) {
log.debug("Attempt to index project {}, event: [{}]", projectName, event);
final Project.NameKey projectNameKey = new Project.NameKey(projectName);
if (projectChecker.isProjectUpToDate(projectNameKey)) {
@@ -97,17 +95,13 @@
indexExecutor.schedule(
() -> {
Context.setForwardedEvent(true);
- try {
- if (!attemptIndex(projectName, event)) {
- log.warn(
- "Attempt {} to index project {} failed, scheduling again after {} msecs",
- retryCount,
- projectName,
- retryInterval);
- rescheduleIndex(projectName, event, retryCount + 1);
- }
- } catch (IOException e) {
- log.warn("Project {} could not be indexed", projectName, e);
+ if (!attemptIndex(projectName, event)) {
+ log.warn(
+ "Attempt {} to index project {} failed, scheduling again after {} msecs",
+ retryCount,
+ projectName,
+ retryInterval);
+ rescheduleIndex(projectName, event, retryCount + 1);
}
},
retryInterval,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexingHandler.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexingHandler.java
index 67662f6..de6b836 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexingHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexingHandler.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.multisite.forwarder;
import com.google.common.util.concurrent.Striped;
-import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.locks.Lock;
@@ -43,9 +42,9 @@
private final Striped<Lock> idLocks;
- protected abstract void doIndex(T id, Optional<E> indexEvent) throws IOException, OrmException;
+ protected abstract void doIndex(T id, Optional<E> indexEvent);
- protected abstract void doDelete(T id, Optional<E> indexEvent) throws IOException;
+ protected abstract void doDelete(T id, Optional<E> indexEvent);
protected ForwardedIndexingHandler(int lockStripes) {
idLocks = Striped.lock(lockStripes);
@@ -58,9 +57,8 @@
* @param operation The operation to do; index or delete
* @param event The index event details.
* @throws IOException If an error occur while indexing.
- * @throws OrmException If an error occur while retrieving a change related to the item to index
*/
- public void index(T id, Operation operation, Optional<E> event) throws IOException, OrmException {
+ public void index(T id, Operation operation, Optional<E> event) throws IOException {
log.debug("{} {} {}", operation, id, event);
try {
Context.setForwardedEvent(true);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerCacheEvictionForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerCacheEvictionForwarder.java
index dff21c1..b32e5ae 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerCacheEvictionForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerCacheEvictionForwarder.java
@@ -16,7 +16,7 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
import com.googlesource.gerrit.plugins.multisite.forwarder.CacheEvictionForwarder;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.CacheEvictionEvent;
@@ -24,15 +24,17 @@
@Singleton
public class BrokerCacheEvictionForwarder implements CacheEvictionForwarder {
- private final BrokerApi broker;
+ private final BrokerApiWrapper broker;
+ private final Configuration cfg;
@Inject
- BrokerCacheEvictionForwarder(BrokerApiWrapper broker) {
+ BrokerCacheEvictionForwarder(BrokerApiWrapper broker, Configuration cfg) {
this.broker = broker;
+ this.cfg = cfg;
}
@Override
public boolean evict(CacheEvictionEvent event) {
- return broker.send(EventTopic.CACHE_TOPIC.topic(), event);
+ return broker.send(EventTopic.CACHE_TOPIC.topic(cfg), event);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerForwarderModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerForwarderModule.java
new file mode 100644
index 0000000..6bd6437
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerForwarderModule.java
@@ -0,0 +1,33 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.forwarder.broker;
+
+import com.google.gerrit.extensions.registration.DynamicSet;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.googlesource.gerrit.plugins.multisite.forwarder.CacheEvictionForwarder;
+import com.googlesource.gerrit.plugins.multisite.forwarder.IndexEventForwarder;
+import com.googlesource.gerrit.plugins.multisite.forwarder.ProjectListUpdateForwarder;
+import com.googlesource.gerrit.plugins.multisite.forwarder.StreamEventForwarder;
+
+public class BrokerForwarderModule extends LifecycleModule {
+ @Override
+ protected void configure() {
+ DynamicSet.bind(binder(), IndexEventForwarder.class).to(BrokerIndexEventForwarder.class);
+ DynamicSet.bind(binder(), CacheEvictionForwarder.class).to(BrokerCacheEvictionForwarder.class);
+ DynamicSet.bind(binder(), ProjectListUpdateForwarder.class)
+ .to(BrokerProjectListUpdateForwarder.class);
+ DynamicSet.bind(binder(), StreamEventForwarder.class).to(BrokerStreamEventForwarder.class);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
index 214d47d..a86c62e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerIndexEventForwarder.java
@@ -15,27 +15,29 @@
package com.googlesource.gerrit.plugins.multisite.forwarder.broker;
import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
import com.googlesource.gerrit.plugins.multisite.forwarder.IndexEventForwarder;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.IndexEvent;
public class BrokerIndexEventForwarder implements IndexEventForwarder {
- private final BrokerApi broker;
+ private final BrokerApiWrapper broker;
+ private final Configuration cfg;
@Inject
- BrokerIndexEventForwarder(BrokerApiWrapper broker) {
+ BrokerIndexEventForwarder(BrokerApiWrapper broker, Configuration cfg) {
this.broker = broker;
+ this.cfg = cfg;
}
@Override
public boolean index(IndexEvent event) {
- return broker.send(EventTopic.INDEX_TOPIC.topic(), event);
+ return broker.send(EventTopic.INDEX_TOPIC.topic(cfg), event);
}
@Override
public boolean batchIndex(IndexEvent event) {
- return broker.send(EventTopic.BATCH_INDEX_TOPIC.topic(), event);
+ return broker.send(EventTopic.BATCH_INDEX_TOPIC.topic(cfg), event);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerProjectListUpdateForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerProjectListUpdateForwarder.java
index 1a8b652..34e0300 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerProjectListUpdateForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerProjectListUpdateForwarder.java
@@ -18,22 +18,24 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
import com.googlesource.gerrit.plugins.multisite.forwarder.ProjectListUpdateForwarder;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ProjectListUpdateEvent;
@Singleton
public class BrokerProjectListUpdateForwarder implements ProjectListUpdateForwarder {
- private final BrokerApi broker;
+ private final BrokerApiWrapper broker;
+ private final Configuration cfg;
@Inject
- BrokerProjectListUpdateForwarder(BrokerApiWrapper broker) {
+ BrokerProjectListUpdateForwarder(BrokerApiWrapper broker, Configuration cfg) {
this.broker = broker;
+ this.cfg = cfg;
}
@Override
public boolean updateProjectList(ProjectListUpdateEvent event) {
- return broker.send(PROJECT_LIST_TOPIC.topic(), event);
+ return broker.send(PROJECT_LIST_TOPIC.topic(cfg), event);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerStreamEventForwarder.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerStreamEventForwarder.java
index ed3a717..9ff4688 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerStreamEventForwarder.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/broker/BrokerStreamEventForwarder.java
@@ -17,22 +17,24 @@
import com.google.gerrit.server.events.Event;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper;
import com.googlesource.gerrit.plugins.multisite.forwarder.StreamEventForwarder;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
@Singleton
public class BrokerStreamEventForwarder implements StreamEventForwarder {
- private final BrokerApi broker;
+ private final BrokerApiWrapper broker;
+ private final Configuration cfg;
@Inject
- BrokerStreamEventForwarder(BrokerApiWrapper broker) {
+ BrokerStreamEventForwarder(BrokerApiWrapper broker, Configuration cfg) {
this.broker = broker;
+ this.cfg = cfg;
}
@Override
public boolean send(Event event) {
- return broker.send(EventTopic.STREAM_EVENT_TOPIC.topic(), event);
+ return broker.send(EventTopic.STREAM_EVENT_TOPIC.topic(cfg), event);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
index 11e77a9..4e7a781 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/events/EventTopic.java
@@ -14,6 +14,8 @@
package com.googlesource.gerrit.plugins.multisite.forwarder.events;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
+
public enum EventTopic {
INDEX_TOPIC("GERRIT.EVENT.INDEX", "indexEvent"),
BATCH_INDEX_TOPIC("GERRIT.EVENT.BATCH.INDEX", "batchIndexEvent"),
@@ -29,18 +31,14 @@
this.aliasKey = aliasKey;
}
- public String topic() {
- return topic;
+ public String topic(Configuration config) {
+ return config.broker().getTopic(topicAliasKey(), topic);
}
public String topicAliasKey() {
return aliasKey + "Topic";
}
- public String enabledKey() {
- return aliasKey + "Enabled";
- }
-
public static EventTopic of(String topicString) {
EventTopic[] topics = EventTopic.values();
for (EventTopic topic : topics) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/ForwardedEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/ForwardedEventRouter.java
index 139020b..f9ad0c9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/ForwardedEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/ForwardedEventRouter.java
@@ -15,11 +15,10 @@
package com.googlesource.gerrit.plugins.multisite.forwarder.router;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.server.OrmException;
import com.googlesource.gerrit.plugins.multisite.forwarder.CacheNotFoundException;
import java.io.IOException;
public interface ForwardedEventRouter<EventType> {
void route(EventType sourceEvent)
- throws IOException, OrmException, PermissionBackendException, CacheNotFoundException;
+ throws IOException, PermissionBackendException, CacheNotFoundException;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
index 45c8e53..6ab8c72 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/IndexEventRouter.java
@@ -22,7 +22,6 @@
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Account.Id;
import com.google.gerrit.server.config.AllUsersName;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexAccountHandler;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexChangeHandler;
@@ -63,7 +62,7 @@
}
@Override
- public void route(IndexEvent sourceEvent) throws IOException, OrmException {
+ public void route(IndexEvent sourceEvent) throws IOException {
if (sourceEvent instanceof ChangeIndexEvent) {
ChangeIndexEvent changeIndexEvent = (ChangeIndexEvent) sourceEvent;
ForwardedIndexingHandler.Operation operation = changeIndexEvent.deleted ? DELETE : INDEX;
@@ -87,8 +86,7 @@
}
}
- public void onRefReplicated(RefReplicationDoneEvent replicationEvent)
- throws IOException, OrmException {
+ public void onRefReplicated(RefReplicationDoneEvent replicationEvent) throws IOException {
if (replicationEvent.getProjectNameKey().equals(allUsersName)) {
Account.Id accountId = Account.Id.fromRef(replicationEvent.getRefName());
if (accountId != null) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
index 1ff6992..4ef3426 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/forwarder/router/StreamEventRouter.java
@@ -16,7 +16,6 @@
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedEventHandler;
import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
@@ -34,8 +33,7 @@
}
@Override
- public void route(Event sourceEvent)
- throws OrmException, PermissionBackendException, IOException {
+ public void route(Event sourceEvent) throws PermissionBackendException, IOException {
if (RefReplicationDoneEvent.TYPE.equals(sourceEvent.getType())) {
/* TODO: We currently explicitly ignore the status and result of the replication
* event because there isn't a reliable way to understand if the current node was
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeChecker.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeChecker.java
index 1b0fea8..3646b3a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeChecker.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeChecker.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.multisite.index;
import com.google.gerrit.server.notedb.ChangeNotes;
-import com.google.gwtorm.server.OrmException;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ChangeIndexEvent;
import java.io.IOException;
import java.util.Optional;
@@ -27,9 +26,8 @@
* Return the Change nodes read from ReviewDb or NoteDb.
*
* @return notes of the Change
- * @throws OrmException if ReviewDb or NoteDb cannot be opened
*/
- public Optional<ChangeNotes> getChangeNotes() throws OrmException;
+ public Optional<ChangeNotes> getChangeNotes();
/**
* Create a new index event POJO associated with the current Change.
@@ -40,21 +38,17 @@
* index
* @return new IndexEvent
* @throws IOException if the current Change cannot read
- * @throws OrmException if ReviewDb cannot be opened
*/
public Optional<ChangeIndexEvent> newIndexEvent(String projectName, int changeId, boolean deleted)
- throws IOException, OrmException;
+ throws IOException;
/**
* Check if the local Change is aligned with the indexEvent received.
*
* @param indexEvent indexing event
* @return true if the local Change is up-to-date, false otherwise.
- * @throws IOException if an I/O error occurred while reading the local Change
- * @throws OrmException if the local ReviewDb cannot be opened
*/
- public boolean isChangeUpToDate(Optional<ChangeIndexEvent> indexEvent)
- throws IOException, OrmException;
+ public boolean isChangeUpToDate(Optional<ChangeIndexEvent> indexEvent);
/**
* Return the last computed up-to-date Change time-stamp.
@@ -63,7 +57,6 @@
*
* @return the Change timestamp epoch in seconds
* @throws IOException if an I/O error occurred while reading the local Change
- * @throws OrmException if the local ReviewDb cannot be opened
*/
- public Optional<Long> getComputedChangeTs() throws IOException, OrmException;
+ public Optional<Long> getComputedChangeTs() throws IOException;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
index 8cb2fec..f1e80cc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/index/ChangeCheckerImpl.java
@@ -14,16 +14,15 @@
package com.googlesource.gerrit.plugins.multisite.index;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Comment;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.change.ChangeFinder;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
-import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ChangeIndexEvent;
@@ -66,7 +65,7 @@
@Override
public Optional<ChangeIndexEvent> newIndexEvent(String projectName, int changeId, boolean deleted)
- throws IOException, OrmException {
+ throws IOException {
return getComputedChangeTs()
.map(
ts -> {
@@ -78,7 +77,7 @@
}
@Override
- public Optional<ChangeNotes> getChangeNotes() throws OrmException {
+ public Optional<ChangeNotes> getChangeNotes() {
try (ManualRequestContext ctx = oneOffReqCtx.open()) {
this.changeNotes = Optional.ofNullable(changeFinder.findOne(changeId));
return changeNotes;
@@ -86,8 +85,7 @@
}
@Override
- public boolean isChangeUpToDate(Optional<ChangeIndexEvent> indexEvent)
- throws IOException, OrmException {
+ public boolean isChangeUpToDate(Optional<ChangeIndexEvent> indexEvent) {
getComputedChangeTs();
if (!computedChangeTs.isPresent()) {
log.warn("Unable to compute last updated ts for change {}", changeId);
@@ -108,7 +106,7 @@
}
@Override
- public Optional<Long> getComputedChangeTs() throws IOException, OrmException {
+ public Optional<Long> getComputedChangeTs() {
if (!computedChangeTs.isPresent()) {
computedChangeTs = computeLastChangeTs();
}
@@ -117,17 +115,12 @@
@Override
public String toString() {
- try {
- return "change-id="
- + changeId
- + "@"
- + getComputedChangeTs().map(ChangeIndexEvent::format)
- + "/"
- + getBranchTargetSha();
- } catch (IOException | OrmException e) {
- log.error("Unable to render change {}", changeId, e);
- return "change-id=" + changeId;
- }
+ return "change-id="
+ + changeId
+ + "@"
+ + getComputedChangeTs().map(ChangeIndexEvent::format)
+ + "/"
+ + getBranchTargetSha();
}
private String getBranchTargetSha() {
@@ -147,21 +140,19 @@
}
}
- private Optional<Long> computeLastChangeTs() throws OrmException {
- try (ReviewDb db = oneOffReqCtx.open().getReviewDbProvider().get()) {
- return getChangeNotes().map(notes -> getTsFromChangeAndDraftComments(db, notes));
- }
+ private Optional<Long> computeLastChangeTs() {
+ return getChangeNotes().map(notes -> getTsFromChangeAndDraftComments(notes));
}
- private long getTsFromChangeAndDraftComments(ReviewDb db, ChangeNotes notes) {
+ private long getTsFromChangeAndDraftComments(ChangeNotes notes) {
Change change = notes.getChange();
Timestamp changeTs = change.getLastUpdatedOn();
try {
- for (Comment comment : commentsUtil.draftByChange(db, changeNotes.get())) {
+ for (Comment comment : commentsUtil.draftByChange(changeNotes.get())) {
Timestamp commentTs = comment.writtenOn;
changeTs = commentTs.after(changeTs) ? commentTs : changeTs;
}
- } catch (OrmException e) {
+ } catch (StorageException e) {
log.warn("Unable to access draft comments for change {}", change, e);
}
return changeTs.getTime() / 1000;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerApi.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerApi.java
deleted file mode 100644
index ff23b78..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerApi.java
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka;
-
-import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.server.events.Event;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerApi;
-import com.googlesource.gerrit.plugins.multisite.broker.kafka.BrokerPublisher;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import com.googlesource.gerrit.plugins.multisite.kafka.consumer.KafkaEventSubscriber;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Consumer;
-
-public class KafkaBrokerApi implements BrokerApi, LifecycleListener {
-
- private final BrokerPublisher publisher;
- private final Provider<KafkaEventSubscriber> subscriberProvider;
- private List<KafkaEventSubscriber> subscribers;
-
- @Inject
- public KafkaBrokerApi(
- BrokerPublisher publisher, Provider<KafkaEventSubscriber> subscriberProvider) {
- this.publisher = publisher;
- this.subscriberProvider = subscriberProvider;
- subscribers = new ArrayList<>();
- }
-
- @Override
- public boolean send(String topic, Event event) {
- return publisher.publish(topic, event);
- }
-
- @Override
- public void receiveAync(String topic, Consumer<SourceAwareEventWrapper> eventConsumer) {
- KafkaEventSubscriber subscriber = subscriberProvider.get();
- synchronized (subscribers) {
- subscribers.add(subscriber);
- }
- subscriber.subscribe(EventTopic.of(topic), eventConsumer);
- }
-
- @Override
- public void start() {}
-
- @Override
- public void stop() {
- for (KafkaEventSubscriber subscriber : subscribers) {
- subscriber.shutdown();
- }
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerModule.java
deleted file mode 100644
index ad04fb8..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaBrokerModule.java
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka;
-
-import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.inject.Inject;
-import com.google.inject.TypeLiteral;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerSession;
-import com.googlesource.gerrit.plugins.multisite.broker.kafka.BrokerPublisher;
-import com.googlesource.gerrit.plugins.multisite.broker.kafka.KafkaSession;
-import com.googlesource.gerrit.plugins.multisite.consumer.AbstractSubcriber;
-import com.googlesource.gerrit.plugins.multisite.consumer.BatchIndexEventSubscriber;
-import com.googlesource.gerrit.plugins.multisite.consumer.CacheEvictionEventSubscriber;
-import com.googlesource.gerrit.plugins.multisite.consumer.IndexEventSubscriber;
-import com.googlesource.gerrit.plugins.multisite.consumer.ProjectUpdateEventSubscriber;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import com.googlesource.gerrit.plugins.multisite.consumer.StreamEventSubscriber;
-import com.googlesource.gerrit.plugins.multisite.forwarder.CacheEvictionForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.IndexEventForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.ProjectListUpdateForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.StreamEventForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerCacheEvictionForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerIndexEventForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerProjectListUpdateForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerStreamEventForwarder;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import com.googlesource.gerrit.plugins.multisite.kafka.consumer.KafkaEventDeserializer;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.Deserializer;
-
-public class KafkaBrokerModule extends LifecycleModule {
- private KafkaConfiguration config;
-
- @Inject
- public KafkaBrokerModule(KafkaConfiguration config) {
- this.config = config;
- }
-
- @Override
- protected void configure() {
- if (config.kafkaSubscriber().enabled()) {
- bind(new TypeLiteral<Deserializer<byte[]>>() {}).toInstance(new ByteArrayDeserializer());
- bind(new TypeLiteral<Deserializer<SourceAwareEventWrapper>>() {})
- .to(KafkaEventDeserializer.class);
-
- if (config.kafkaSubscriber().enabledEvent(EventTopic.INDEX_TOPIC)) {
- DynamicSet.bind(binder(), AbstractSubcriber.class).to(IndexEventSubscriber.class);
- }
- if (config.kafkaSubscriber().enabledEvent(EventTopic.BATCH_INDEX_TOPIC)) {
- DynamicSet.bind(binder(), AbstractSubcriber.class).to(BatchIndexEventSubscriber.class);
- }
- if (config.kafkaSubscriber().enabledEvent(EventTopic.STREAM_EVENT_TOPIC)) {
- DynamicSet.bind(binder(), AbstractSubcriber.class).to(StreamEventSubscriber.class);
- }
- if (config.kafkaSubscriber().enabledEvent(EventTopic.CACHE_TOPIC)) {
- DynamicSet.bind(binder(), AbstractSubcriber.class).to(CacheEvictionEventSubscriber.class);
- }
- if (config.kafkaSubscriber().enabledEvent(EventTopic.PROJECT_LIST_TOPIC)) {
- DynamicSet.bind(binder(), AbstractSubcriber.class).to(ProjectUpdateEventSubscriber.class);
- }
- }
-
- if (config.kafkaPublisher().enabled()) {
- listener().to(BrokerPublisher.class);
- bind(BrokerSession.class).to(KafkaSession.class);
-
- if (config.kafkaPublisher().enabledEvent(EventTopic.INDEX_TOPIC)) {
- DynamicSet.bind(binder(), IndexEventForwarder.class).to(BrokerIndexEventForwarder.class);
- }
- if (config.kafkaPublisher().enabledEvent(EventTopic.CACHE_TOPIC)) {
- DynamicSet.bind(binder(), CacheEvictionForwarder.class)
- .to(BrokerCacheEvictionForwarder.class);
- }
- if (config.kafkaPublisher().enabledEvent(EventTopic.PROJECT_LIST_TOPIC)) {
- DynamicSet.bind(binder(), ProjectListUpdateForwarder.class)
- .to(BrokerProjectListUpdateForwarder.class);
- }
- if (config.kafkaPublisher().enabledEvent(EventTopic.STREAM_EVENT_TOPIC)) {
- DynamicSet.bind(binder(), StreamEventForwarder.class).to(BrokerStreamEventForwarder.class);
- }
- }
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfiguration.java
deleted file mode 100644
index dee7382..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfiguration.java
+++ /dev/null
@@ -1,283 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka;
-
-import static com.google.common.base.Suppliers.memoize;
-import static com.google.common.base.Suppliers.ofInstance;
-
-import com.google.common.base.CaseFormat;
-import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.Configuration;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Singleton
-public class KafkaConfiguration {
-
- private static final Logger log = LoggerFactory.getLogger(KafkaConfiguration.class);
- static final String KAFKA_PROPERTY_PREFIX = "KafkaProp-";
- static final String KAFKA_SECTION = "kafka";
- static final String ENABLE_KEY = "enabled";
- private static final String DEFAULT_KAFKA_BOOTSTRAP_SERVERS = "localhost:9092";
- private static final boolean DEFAULT_ENABLE_PROCESSING = true;
- private static final int DEFAULT_POLLING_INTERVAL_MS = 1000;
-
- private final Supplier<KafkaSubscriber> subscriber;
- private final Supplier<Kafka> kafka;
- private final Supplier<KafkaPublisher> publisher;
-
- @Inject
- public KafkaConfiguration(Configuration configuration) {
- Supplier<Config> lazyCfg = lazyLoad(configuration.getMultiSiteConfig());
- kafka = memoize(() -> new Kafka(lazyCfg));
- publisher = memoize(() -> new KafkaPublisher(lazyCfg));
- subscriber = memoize(() -> new KafkaSubscriber(lazyCfg));
- }
-
- public Kafka getKafka() {
- return kafka.get();
- }
-
- public KafkaSubscriber kafkaSubscriber() {
- return subscriber.get();
- }
-
- private static void applyKafkaConfig(
- Supplier<Config> configSupplier, String subsectionName, Properties target) {
- Config config = configSupplier.get();
- for (String section : config.getSubsections(KAFKA_SECTION)) {
- if (section.equals(subsectionName)) {
- for (String name : config.getNames(KAFKA_SECTION, section, true)) {
- if (name.startsWith(KAFKA_PROPERTY_PREFIX)) {
- Object value = config.getString(KAFKA_SECTION, subsectionName, name);
- String configProperty = name.replaceFirst(KAFKA_PROPERTY_PREFIX, "");
- String propName =
- CaseFormat.LOWER_CAMEL
- .to(CaseFormat.LOWER_HYPHEN, configProperty)
- .replaceAll("-", ".");
- log.info("[{}] Setting kafka property: {} = {}", subsectionName, propName, value);
- target.put(propName, value);
- }
- }
- }
- }
- target.put(
- "bootstrap.servers",
- getString(
- configSupplier,
- KAFKA_SECTION,
- null,
- "bootstrapServers",
- DEFAULT_KAFKA_BOOTSTRAP_SERVERS));
- }
-
- private static String getString(
- Supplier<Config> cfg, String section, String subsection, String name, String defaultValue) {
- String value = cfg.get().getString(section, subsection, name);
- if (!Strings.isNullOrEmpty(value)) {
- return value;
- }
- return defaultValue;
- }
-
- private static Map<EventTopic, Boolean> eventsEnabled(
- Supplier<Config> config, String subsection) {
- Map<EventTopic, Boolean> eventsEnabled = new HashMap<>();
- for (EventTopic topic : EventTopic.values()) {
- eventsEnabled.put(
- topic,
- config
- .get()
- .getBoolean(
- KAFKA_SECTION, subsection, topic.enabledKey(), DEFAULT_ENABLE_PROCESSING));
- }
- return eventsEnabled;
- }
-
- public KafkaPublisher kafkaPublisher() {
- return publisher.get();
- }
-
- private Supplier<Config> lazyLoad(Config config) {
- if (config instanceof FileBasedConfig) {
- return memoize(
- () -> {
- FileBasedConfig fileConfig = (FileBasedConfig) config;
- String fileConfigFileName = fileConfig.getFile().getPath();
- try {
- log.info("Loading configuration from {}", fileConfigFileName);
- fileConfig.load();
- } catch (IOException | ConfigInvalidException e) {
- log.error("Unable to load configuration from " + fileConfigFileName, e);
- }
- return fileConfig;
- });
- }
- return ofInstance(config);
- }
-
- public static class Kafka {
- private final Map<EventTopic, String> eventTopics;
- private final String bootstrapServers;
-
- Kafka(Supplier<Config> config) {
- this.bootstrapServers =
- getString(
- config, KAFKA_SECTION, null, "bootstrapServers", DEFAULT_KAFKA_BOOTSTRAP_SERVERS);
-
- this.eventTopics = new HashMap<>();
- for (EventTopic eventTopic : EventTopic.values()) {
- eventTopics.put(
- eventTopic,
- getString(config, KAFKA_SECTION, null, eventTopic.topicAliasKey(), eventTopic.topic()));
- }
- }
-
- public String getTopicAlias(EventTopic topic) {
- return eventTopics.get(topic);
- }
-
- public String getBootstrapServers() {
- return bootstrapServers;
- }
-
- private static String getString(
- Supplier<Config> cfg, String section, String subsection, String name, String defaultValue) {
- String value = cfg.get().getString(section, subsection, name);
- if (!Strings.isNullOrEmpty(value)) {
- return value;
- }
- return defaultValue;
- }
- }
-
- public static class KafkaPublisher extends Properties {
- private static final long serialVersionUID = 0L;
-
- public static final String KAFKA_STRING_SERIALIZER = StringSerializer.class.getName();
-
- public static final String KAFKA_PUBLISHER_SUBSECTION = "publisher";
- public static final boolean DEFAULT_BROKER_ENABLED = false;
-
- private final boolean enabled;
- private final Map<EventTopic, Boolean> eventsEnabled;
-
- private KafkaPublisher(Supplier<Config> cfg) {
- enabled =
- cfg.get()
- .getBoolean(
- KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, ENABLE_KEY, DEFAULT_BROKER_ENABLED);
-
- eventsEnabled = eventsEnabled(cfg, KAFKA_PUBLISHER_SUBSECTION);
-
- if (enabled) {
- setDefaults();
- applyKafkaConfig(cfg, KAFKA_PUBLISHER_SUBSECTION, this);
- }
- }
-
- private void setDefaults() {
- put("acks", "all");
- put("retries", 10);
- put("batch.size", 16384);
- put("linger.ms", 1);
- put("buffer.memory", 33554432);
- put("key.serializer", KAFKA_STRING_SERIALIZER);
- put("value.serializer", KAFKA_STRING_SERIALIZER);
- put("reconnect.backoff.ms", 5000L);
- }
-
- public boolean enabled() {
- return enabled;
- }
-
- public boolean enabledEvent(EventTopic eventType) {
- return eventsEnabled.get(eventType);
- }
- }
-
- public static class KafkaSubscriber extends Properties {
- private static final long serialVersionUID = 1L;
-
- static final String KAFKA_SUBSCRIBER_SUBSECTION = "subscriber";
-
- private final boolean enabled;
- private final Integer pollingInterval;
- private Map<EventTopic, Boolean> eventsEnabled;
- private final Config cfg;
-
- public KafkaSubscriber(Supplier<Config> configSupplier) {
- this.cfg = configSupplier.get();
-
- this.pollingInterval =
- cfg.getInt(
- KAFKA_SECTION,
- KAFKA_SUBSCRIBER_SUBSECTION,
- "pollingIntervalMs",
- DEFAULT_POLLING_INTERVAL_MS);
-
- enabled = cfg.getBoolean(KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, ENABLE_KEY, false);
-
- eventsEnabled = eventsEnabled(configSupplier, KAFKA_SUBSCRIBER_SUBSECTION);
-
- if (enabled) {
- applyKafkaConfig(configSupplier, KAFKA_SUBSCRIBER_SUBSECTION, this);
- }
- }
-
- public boolean enabled() {
- return enabled;
- }
-
- public boolean enabledEvent(EventTopic topic) {
- return eventsEnabled.get(topic);
- }
-
- public Properties initPropsWith(UUID instanceId) {
- String groupId =
- getString(
- cfg, KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, "groupId", instanceId.toString());
- this.put("group.id", groupId);
-
- return this;
- }
-
- public Integer getPollingInterval() {
- return pollingInterval;
- }
-
- private String getString(
- Config cfg, String section, String subsection, String name, String defaultValue) {
- String value = cfg.getString(section, subsection, name);
- if (!Strings.isNullOrEmpty(value)) {
- return value;
- }
- return defaultValue;
- }
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaConsumerFactory.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaConsumerFactory.java
deleted file mode 100644
index 9a5e19f..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaConsumerFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration;
-import java.util.UUID;
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.Deserializer;
-
-@Singleton
-public class KafkaConsumerFactory {
- private KafkaConfiguration config;
-
- @Inject
- public KafkaConsumerFactory(KafkaConfiguration configuration) {
- this.config = configuration;
- }
-
- public Consumer<byte[], byte[]> create(Deserializer<byte[]> keyDeserializer, UUID instanceId) {
- return new KafkaConsumer<>(
- config.kafkaSubscriber().initPropsWith(instanceId),
- keyDeserializer,
- new ByteArrayDeserializer());
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializer.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializer.java
deleted file mode 100644
index 38b8d61..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
-
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerGson;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import java.util.Map;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-
-@Singleton
-public class KafkaEventDeserializer implements Deserializer<SourceAwareEventWrapper> {
-
- private final StringDeserializer stringDeserializer = new StringDeserializer();
- private Gson gson;
-
- // To be used when providing this deserializer with class name (then need to add a configuration
- // entry to set the gson.provider
- public KafkaEventDeserializer() {}
-
- @Inject
- public KafkaEventDeserializer(@BrokerGson Gson gson) {
- this.gson = gson;
- }
-
- @Override
- public void configure(Map<String, ?> configs, boolean isKey) {}
-
- @Override
- public SourceAwareEventWrapper deserialize(String topic, byte[] data) {
- final SourceAwareEventWrapper result =
- gson.fromJson(stringDeserializer.deserialize(topic, data), SourceAwareEventWrapper.class);
-
- result.validate();
-
- return result;
- }
-
- @Override
- public void close() {}
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventSubscriber.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventSubscriber.java
deleted file mode 100644
index c72fa0d..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventSubscriber.java
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.server.util.ManualRequestContext;
-import com.google.gerrit.server.util.OneOffRequestContext;
-import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.InstanceId;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import com.googlesource.gerrit.plugins.multisite.consumer.SubscriberMetrics;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration;
-import java.time.Duration;
-import java.util.Collections;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.common.errors.WakeupException;
-import org.apache.kafka.common.serialization.Deserializer;
-
-public class KafkaEventSubscriber {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private final Consumer<byte[], byte[]> consumer;
- private final OneOffRequestContext oneOffCtx;
- private final AtomicBoolean closed = new AtomicBoolean(false);
-
- private final Deserializer<SourceAwareEventWrapper> valueDeserializer;
- private final KafkaConfiguration configuration;
- private final SubscriberMetrics subscriberMetrics;
-
- @Inject
- public KafkaEventSubscriber(
- KafkaConfiguration configuration,
- KafkaConsumerFactory consumerFactory,
- Deserializer<byte[]> keyDeserializer,
- Deserializer<SourceAwareEventWrapper> valueDeserializer,
- @InstanceId UUID instanceId,
- OneOffRequestContext oneOffCtx,
- SubscriberMetrics subscriberMetrics) {
-
- this.configuration = configuration;
- this.oneOffCtx = oneOffCtx;
- this.subscriberMetrics = subscriberMetrics;
-
- final ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(KafkaEventSubscriber.class.getClassLoader());
- this.consumer = consumerFactory.create(keyDeserializer, instanceId);
- } finally {
- Thread.currentThread().setContextClassLoader(previousClassLoader);
- }
- this.valueDeserializer = valueDeserializer;
- }
-
- public void subscribe(
- EventTopic evenTopic, java.util.function.Consumer<SourceAwareEventWrapper> messageProcessor) {
- try {
- final String topic = configuration.getKafka().getTopicAlias(evenTopic);
- logger.atInfo().log(
- "Kafka consumer subscribing to topic alias [%s] for event topic [%s]", topic, evenTopic);
- consumer.subscribe(Collections.singleton(topic));
- while (!closed.get()) {
- ConsumerRecords<byte[], byte[]> consumerRecords =
- consumer.poll(Duration.ofMillis(configuration.kafkaSubscriber().getPollingInterval()));
- consumerRecords.forEach(
- consumerRecord -> {
- try (ManualRequestContext ctx = oneOffCtx.open()) {
- SourceAwareEventWrapper event =
- valueDeserializer.deserialize(consumerRecord.topic(), consumerRecord.value());
- messageProcessor.accept(event);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log(
- "Malformed event '%s': [Exception: %s]",
- new String(consumerRecord.value(), UTF_8));
- subscriberMetrics.incrementSubscriberFailedToConsumeMessage();
- }
- });
- }
- } catch (WakeupException e) {
- // Ignore exception if closing
- if (!closed.get()) throw e;
- } catch (Exception e) {
- subscriberMetrics.incrementSubscriberFailedToPollMessages();
- throw e;
- } finally {
- consumer.close();
- }
- }
-
- public void shutdown() {
- closed.set(true);
- consumer.wakeup();
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidator.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidator.java
index 4e6bef5..cfe585d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidator.java
@@ -19,7 +19,7 @@
import com.google.inject.assistedinject.Assisted;
import com.googlesource.gerrit.plugins.multisite.LockWrapper;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.OutOfSyncException;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
import java.io.IOException;
@@ -29,6 +29,8 @@
import java.util.stream.Stream;
import org.eclipse.jgit.lib.BatchRefUpdate;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.transport.ReceiveCommand;
@@ -99,6 +101,13 @@
refsToUpdate = compareAndGetLatestLocalRefs(refsToUpdate, locks);
delegateUpdate.invoke();
updateSharedRefDb(batchRefUpdate.getCommands().stream(), refsToUpdate);
+ } catch (OutOfSyncException e) {
+ List<ReceiveCommand> receiveCommands = batchRefUpdate.getCommands();
+ logger.atWarning().withCause(e).log(
+ String.format(
+ "Batch ref-update failing because node is out of sync with the shared ref-db. Set all commands Result to LOCK_FAILURE [%d]",
+ receiveCommands.size()));
+ receiveCommands.forEach((command) -> command.setResult(ReceiveCommand.Result.LOCK_FAILURE));
}
}
@@ -124,7 +133,7 @@
try {
switch (command.getType()) {
case CREATE:
- return new RefPair(SharedRefDatabase.nullRef(command.getRefName()), getNewRef(command));
+ return new RefPair(nullRef(command.getRefName()), getNewRef(command));
case UPDATE:
case UPDATE_NONFASTFORWARD:
@@ -155,4 +164,8 @@
}
return latestRefsToUpdate;
}
+
+ private static final Ref nullRef(String refName) {
+ return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, ObjectId.zeroId());
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabase.java
index 630b091..e1d1c65 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabase.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabase.java
@@ -99,11 +99,6 @@
}
@Override
- public Ref getRef(String name) throws IOException {
- return refDatabase.getRef(name);
- }
-
- @Override
public String toString() {
return refDatabase.toString();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
index b85ec89..b666c02 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationPushFilter.java
@@ -14,6 +14,7 @@
package com.googlesource.gerrit.plugins.multisite.validation;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
import com.google.common.base.Preconditions;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Project;
@@ -21,8 +22,6 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedLockException;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
import com.googlesource.gerrit.plugins.replication.ReplicationPushFilter;
import java.io.IOException;
import java.util.Collections;
@@ -32,6 +31,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.RemoteRefUpdate;
@@ -61,6 +61,7 @@
@Override
public List<RemoteRefUpdate> filter(String projectName, List<RemoteRefUpdate> remoteUpdatesList) {
Set<String> outdatedChanges = new HashSet<>();
+
try (Repository repository =
gitRepositoryManager.openRepository(Project.NameKey.parse(projectName))) {
List<RemoteRefUpdate> filteredRefUpdates =
@@ -106,7 +107,8 @@
String ref = refUpdate.getSrcRef();
try {
if (sharedRefDb.isUpToDate(
- projectName, SharedRefDatabase.newRef(ref, refUpdate.getNewObjectId()))) {
+ new Project.NameKey(projectName),
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, ref, refUpdate.getNewObjectId()))) {
return true;
}
@@ -114,12 +116,13 @@
projectName, refUpdate, ref);
return sharedRefDb.isUpToDate(
- projectName, SharedRefDatabase.newRef(ref, getNotNullExactRef(repository, ref)));
- } catch (SharedLockException sle) {
+ new Project.NameKey(projectName),
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, ref, getNotNullExactRef(repository, ref)));
+ } catch (GlobalRefDbLockException gle) {
String message =
String.format("%s is locked on shared-refdb and thus will NOT BE replicated", ref);
repLog.error(message);
- logger.atSevere().withCause(sle).log(message);
+ logger.atSevere().withCause(gle).log(message);
return false;
} catch (IOException ioe) {
String message =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanup.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanup.java
index 78e1cea..9c93793 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanup.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanup.java
@@ -14,11 +14,12 @@
package com.googlesource.gerrit.plugins.multisite.validation;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
-import java.io.IOException;
public class ProjectDeletedSharedDbCleanup implements ProjectDeletedListener {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -41,8 +42,8 @@
"Deleting project '%s'. Will perform a cleanup in Shared-Ref database.", projectName);
try {
- sharedDb.removeProject(projectName);
- } catch (IOException e) {
+ sharedDb.remove(new Project.NameKey(projectName));
+ } catch (GlobalRefDbSystemError e) {
validationMetrics.incrementSplitBrain();
logger.atSevere().withCause(e).log(
"Project '%s' deleted from GIT but it was not able to cleanup"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
new file mode 100644
index 0000000..3b82582
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
@@ -0,0 +1,304 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
+
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.client.RefNames;
+import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.EventListener;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.notedb.IntBlob;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
+
+@Singleton
+public class ProjectVersionRefUpdate implements EventListener {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final Set<RefUpdate.Result> SUCCESSFUL_RESULTS =
+ ImmutableSet.of(RefUpdate.Result.NEW, RefUpdate.Result.FORCED, RefUpdate.Result.NO_CHANGE);
+
+ public static final String MULTI_SITE_VERSIONING_REF = "refs/multi-site/version";
+ public static final String MULTI_SITE_VERSIONING_VALUE_REF = "refs/multi-site/version/value";
+ public static final Ref NULL_PROJECT_VERSION_REF =
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId());
+
+ private final GitRepositoryManager gitRepositoryManager;
+ private final GitReferenceUpdated gitReferenceUpdated;
+ private final ProjectVersionLogger verLogger;
+
+ protected final SharedRefDatabaseWrapper sharedRefDb;
+
+ @Inject
+ public ProjectVersionRefUpdate(
+ GitRepositoryManager gitRepositoryManager,
+ SharedRefDatabaseWrapper sharedRefDb,
+ GitReferenceUpdated gitReferenceUpdated,
+ ProjectVersionLogger verLogger) {
+ this.gitRepositoryManager = gitRepositoryManager;
+ this.sharedRefDb = sharedRefDb;
+ this.gitReferenceUpdated = gitReferenceUpdated;
+ this.verLogger = verLogger;
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ logger.atFine().log("Processing event type: " + event.type);
+ // Producer of the Event use RefUpdatedEvent to trigger the version update
+ if (!Context.isForwardedEvent() && event instanceof RefUpdatedEvent) {
+ updateProducerProjectVersionUpdate((RefUpdatedEvent) event);
+ }
+ }
+
+ private boolean isSpecialRefName(String refName) {
+ return refName.startsWith(RefNames.REFS_SEQUENCES)
+ || refName.startsWith(RefNames.REFS_STARRED_CHANGES)
+ || refName.equals(MULTI_SITE_VERSIONING_REF);
+ }
+
+ private void updateProducerProjectVersionUpdate(RefUpdatedEvent refUpdatedEvent) {
+ String refName = refUpdatedEvent.getRefName();
+
+ if (isSpecialRefName(refName)) {
+ logger.atFine().log(
+ "Found a special ref name %s, skipping update for %s",
+ refName, refUpdatedEvent.getProjectNameKey().get());
+ return;
+ }
+ try {
+ Project.NameKey projectNameKey = refUpdatedEvent.getProjectNameKey();
+ long newVersion = getCurrentGlobalVersionNumber();
+
+ Optional<RefUpdate> newProjectVersionRefUpdate =
+ updateLocalProjectVersion(projectNameKey, newVersion);
+
+ if (newProjectVersionRefUpdate.isPresent()) {
+ verLogger.log(projectNameKey, newVersion, 0L);
+
+ if (updateSharedProjectVersion(
+ projectNameKey, newProjectVersionRefUpdate.get().getNewObjectId(), newVersion)) {
+ gitReferenceUpdated.fire(projectNameKey, newProjectVersionRefUpdate.get(), null);
+ }
+ } else {
+ logger.atWarning().log(
+ "Ref %s not found on projet %s: skipping project version update",
+ refUpdatedEvent.getRefName(), projectNameKey);
+ }
+ } catch (LocalProjectVersionUpdateException | SharedProjectVersionUpdateException e) {
+ logger.atSevere().withCause(e).log(
+ "Issue encountered when updating version for project "
+ + refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private RefUpdate getProjectVersionRefUpdate(Repository repository, Long version)
+ throws IOException {
+ RefUpdate refUpdate = repository.getRefDatabase().newUpdate(MULTI_SITE_VERSIONING_REF, false);
+ refUpdate.setNewObjectId(getNewId(repository, version));
+ refUpdate.setForceUpdate(true);
+ return refUpdate;
+ }
+
+ private ObjectId getNewId(Repository repository, Long version) throws IOException {
+ ObjectInserter ins = repository.newObjectInserter();
+ ObjectId newId = ins.insert(OBJ_BLOB, Long.toString(version).getBytes(UTF_8));
+ ins.flush();
+ return newId;
+ }
+
+ private boolean updateSharedProjectVersion(
+ Project.NameKey projectNameKey, ObjectId newObjectId, Long newVersion)
+ throws SharedProjectVersionUpdateException {
+
+ Ref sharedRef =
+ sharedRefDb
+ .get(projectNameKey, MULTI_SITE_VERSIONING_REF, String.class)
+ .map(
+ (String objectId) ->
+ new ObjectIdRef.Unpeeled(
+ Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.fromString(objectId)))
+ .orElse(
+ new ObjectIdRef.Unpeeled(
+ Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId()));
+ Optional<Long> sharedVersion =
+ sharedRefDb
+ .get(projectNameKey, MULTI_SITE_VERSIONING_VALUE_REF, String.class)
+ .map(Long::parseLong);
+
+ try {
+ if (sharedVersion.isPresent() && sharedVersion.get() >= newVersion) {
+ logger.atWarning().log(
+ String.format(
+ "NOT Updating project %s version %s (value=%d) in shared ref-db because is more recent than the local one %s (value=%d) ",
+ projectNameKey.get(),
+ newObjectId,
+ newVersion,
+ sharedRef.getObjectId().getName(),
+ sharedVersion.get()));
+ return false;
+ }
+
+ logger.atFine().log(
+ String.format(
+ "Updating shared project %s version to %s (value=%d)",
+ projectNameKey.get(), newObjectId, newVersion));
+
+ boolean success = sharedRefDb.compareAndPut(projectNameKey, sharedRef, newObjectId);
+ if (!success) {
+ String message =
+ String.format(
+ "Project version blob update failed for %s. Current value %s, new value: %s",
+ projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId);
+ logger.atSevere().log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+
+ success =
+ sharedRefDb.compareAndPut(
+ projectNameKey,
+ MULTI_SITE_VERSIONING_VALUE_REF,
+ sharedVersion.map(Object::toString).orElse(null),
+ newVersion.toString());
+ if (!success) {
+ String message =
+ String.format(
+ "Project version update failed for %s. Current value %s, new value: %s",
+ projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId);
+ logger.atSevere().log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+
+ return true;
+ } catch (GlobalRefDbSystemError refDbSystemError) {
+ String message =
+ String.format(
+ "Error while updating shared project version for %s. Current value %s, new value: %s. Error: %s",
+ projectNameKey.get(),
+ sharedRef.getObjectId(),
+ newObjectId,
+ refDbSystemError.getMessage());
+ logger.atSevere().withCause(refDbSystemError).log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+ }
+
+ public Optional<Long> getProjectLocalVersion(String projectName) {
+ try (Repository repository =
+ gitRepositoryManager.openRepository(Project.NameKey.parse(projectName))) {
+ Optional<IntBlob> blob = IntBlob.parse(repository, MULTI_SITE_VERSIONING_REF);
+ if (blob.isPresent()) {
+ Long repoVersion = Integer.toUnsignedLong(blob.get().value());
+ logger.atFine().log("Local project '%s' has version %d", projectName, repoVersion);
+ return Optional.of(repoVersion);
+ }
+ } catch (RepositoryNotFoundException re) {
+ logger.atFine().log("Project '%s' not found", projectName);
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Cannot read local project '%s' version", projectName);
+ }
+ return Optional.empty();
+ }
+
+ public Optional<Long> getProjectRemoteVersion(String projectName) {
+ Optional<String> globalVersion =
+ sharedRefDb.get(
+ Project.NameKey.parse(projectName), MULTI_SITE_VERSIONING_VALUE_REF, String.class);
+ return globalVersion.flatMap(longString -> getLongValueOf(longString));
+ }
+
+ private Object safeGetObjectId(Ref currentRef) {
+ return currentRef == null ? "null" : currentRef.getObjectId();
+ }
+
+ private Optional<Long> getLongValueOf(String longString) {
+ try {
+ return Optional.ofNullable(Long.parseLong(longString));
+ } catch (NumberFormatException e) {
+ logger.atSevere().withCause(e).log(
+ "Unable to parse timestamp value %s into Long", longString);
+ return Optional.empty();
+ }
+ }
+
+ private Optional<RefUpdate> updateLocalProjectVersion(
+ Project.NameKey projectNameKey, long newVersionNumber)
+ throws LocalProjectVersionUpdateException {
+ logger.atFine().log(
+ "Updating local version for project %s with version %d",
+ projectNameKey.get(), newVersionNumber);
+ try (Repository repository = gitRepositoryManager.openRepository(projectNameKey)) {
+ RefUpdate refUpdate = getProjectVersionRefUpdate(repository, newVersionNumber);
+ RefUpdate.Result result = refUpdate.update();
+ if (!isSuccessful(result)) {
+ String message =
+ String.format(
+ "RefUpdate failed with result %s for: project=%s, version=%d",
+ result.name(), projectNameKey.get(), newVersionNumber);
+ logger.atSevere().log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+
+ return Optional.of(refUpdate);
+ } catch (IOException e) {
+ String message = "Cannot create versioning command for " + projectNameKey.get();
+ logger.atSevere().withCause(e).log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ }
+
+ private long getCurrentGlobalVersionNumber() {
+ return System.currentTimeMillis() / 1000;
+ }
+
+ private Boolean isSuccessful(RefUpdate.Result result) {
+ return SUCCESSFUL_RESULTS.contains(result);
+ }
+
+ public static class LocalProjectVersionUpdateException extends Exception {
+ private static final long serialVersionUID = 7649956232401457023L;
+
+ public LocalProjectVersionUpdateException(String projectName) {
+ super("Cannot update local project version of " + projectName);
+ }
+ }
+
+ public static class SharedProjectVersionUpdateException extends Exception {
+ private static final long serialVersionUID = -9153858177700286314L;
+
+ public SharedProjectVersionUpdateException(String projectName) {
+ super("Cannot update shared project version of " + projectName);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
index e66b3ba..76c1a67 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
@@ -14,10 +14,10 @@
package com.googlesource.gerrit.plugins.multisite.validation;
-import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase.nullRef;
-
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
import com.google.common.base.MoreObjects;
import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.googlesource.gerrit.plugins.multisite.LockWrapper;
@@ -25,12 +25,12 @@
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.OutOfSyncException;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedDbSplitBrainException;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedLockException;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
import java.io.IOException;
import java.util.HashMap;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
@@ -86,7 +86,8 @@
public RefUpdate.Result executeRefUpdate(
RefUpdate refUpdate, NoParameterFunction<RefUpdate.Result> refUpdateFunction)
throws IOException {
- if (refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
+ if (isProjectVersionUpdate(refUpdate.getName())
+ || refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
return refUpdateFunction.invoke();
}
@@ -105,6 +106,13 @@
return null;
}
+ private Boolean isProjectVersionUpdate(String refName) {
+ Boolean isProjectVersionUpdate =
+ refName.equals(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ logger.atFine().log("Is project version update? " + isProjectVersionUpdate);
+ return isProjectVersionUpdate;
+ }
+
private <T extends Throwable> void softFailBasedOnEnforcement(T e, EnforcePolicy policy)
throws T {
logger.atWarning().withCause(e).log(
@@ -127,6 +135,11 @@
updateSharedDbOrThrowExceptionFor(refPairForUpdate);
}
return result;
+ } catch (OutOfSyncException e) {
+ logger.atWarning().withCause(e).log(
+ String.format("Local node is out of sync with ref-db: %s", e.getMessage()));
+
+ return RefUpdate.Result.LOCK_FAILURE;
}
}
@@ -144,8 +157,10 @@
projectName, refPair.getName(), refPair.putValue);
boolean succeeded;
try {
- succeeded = sharedRefDb.compareAndPut(projectName, refPair.compareRef, refPair.putValue);
- } catch (IOException e) {
+ succeeded =
+ sharedRefDb.compareAndPut(
+ new Project.NameKey(projectName), refPair.compareRef, refPair.putValue);
+ } catch (GlobalRefDbSystemError e) {
logger.atWarning().withCause(e).log(
"Not able to persist the data in Zookeeper for project '{}' and ref '{}', message: {}",
projectName,
@@ -171,14 +186,17 @@
String.format("%s-%s", projectName, refName),
() ->
lockWrapperFactory.create(
- projectName, refName, sharedRefDb.lockRef(projectName, refName)));
+ projectName,
+ refName,
+ sharedRefDb.lockRef(new Project.NameKey(projectName), refName)));
RefPair latestRefPair = getLatestLocalRef(refPair);
- if (sharedRefDb.isUpToDate(projectName, latestRefPair.compareRef)) {
+ if (sharedRefDb.isUpToDate(new Project.NameKey(projectName), latestRefPair.compareRef)) {
return latestRefPair;
}
- if (isNullRef(latestRefPair.compareRef) || sharedRefDb.exists(projectName, refName)) {
+ if (isNullRef(latestRefPair.compareRef)
+ || sharedRefDb.exists(new Project.NameKey(projectName), refName)) {
validationMetrics.incrementSplitBrainPrevention();
softFailBasedOnEnforcement(
@@ -198,6 +216,10 @@
latestRef == null ? nullRef(refPair.getName()) : latestRef, refPair.putValue);
}
+ private Ref nullRef(String name) {
+ return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, name, ObjectId.zeroId());
+ }
+
protected boolean isSuccessful(RefUpdate.Result result) {
switch (result) {
case NEW:
@@ -224,7 +246,7 @@
}
protected Ref getCurrentRef(String refName) throws IOException {
- return MoreObjects.firstNonNull(refDb.getRef(refName), SharedRefDatabase.nullRef(refName));
+ return MoreObjects.firstNonNull(refDb.getRef(refName), nullRef(refName));
}
public static class CloseableSet<T extends AutoCloseable> implements AutoCloseable {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
index e331819..481d288 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
@@ -16,19 +16,18 @@
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.LockWrapper;
+import com.googlesource.gerrit.plugins.multisite.Log4jProjectVersionLogger;
import com.googlesource.gerrit.plugins.multisite.Log4jSharedRefLogger;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.SharedRefLogger;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.NoopSharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.replication.ReplicationExtensionPointModule;
import com.googlesource.gerrit.plugins.replication.ReplicationPushFilter;
@@ -37,24 +36,18 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final Configuration cfg;
- private final boolean disableGitRepositoryValidation;
- public ValidationModule(Configuration cfg, boolean disableGitRepositoryValidation) {
+ public ValidationModule(Configuration cfg) {
this.cfg = cfg;
- this.disableGitRepositoryValidation = disableGitRepositoryValidation;
}
@Override
protected void configure() {
install(new ReplicationExtensionPointModule());
- DynamicItem.itemOf(binder(), SharedRefDatabase.class);
- DynamicItem.bind(binder(), SharedRefDatabase.class)
- .to(NoopSharedRefDatabase.class)
- .in(Scopes.SINGLETON);
- logger.atInfo().log("Shared ref-db engine: none");
-
+ bind(SharedRefDatabaseWrapper.class).in(Scopes.SINGLETON);
bind(SharedRefLogger.class).to(Log4jSharedRefLogger.class);
+ bind(ProjectVersionLogger.class).to(Log4jProjectVersionLogger.class);
factory(LockWrapper.Factory.class);
factory(MultiSiteRepository.Factory.class);
@@ -64,12 +57,10 @@
factory(RefUpdateValidator.Factory.class);
factory(BatchRefUpdateValidator.Factory.class);
+ bind(GitRepositoryManager.class).to(MultiSiteGitRepositoryManager.class);
DynamicItem.bind(binder(), ReplicationPushFilter.class)
.to(MultisiteReplicationPushFilter.class);
- if (!disableGitRepositoryValidation) {
- bind(GitRepositoryManager.class).to(MultiSiteGitRepositoryManager.class);
- }
if (cfg.getSharedRefDb().getEnforcementRules().isEmpty()) {
bind(SharedRefEnforcement.class).to(DefaultSharedRefEnforcement.class).in(Scopes.SINGLETON);
} else {
@@ -77,6 +68,5 @@
.to(CustomSharedRefEnforcementByProject.class)
.in(Scopes.SINGLETON);
}
- DynamicSet.bind(binder(), ProjectDeletedListener.class).to(ProjectDeletedSharedDbCleanup.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ZkConnectionConfig.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ZkConnectionConfig.java
deleted file mode 100644
index 0339f01..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ZkConnectionConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation;
-
-import org.apache.curator.RetryPolicy;
-
-public class ZkConnectionConfig {
-
- public final RetryPolicy curatorRetryPolicy;
- public final Long transactionLockTimeout;
-
- public ZkConnectionConfig(RetryPolicy curatorRetryPolicy, Long transactionLockTimeout) {
- this.curatorRetryPolicy = curatorRetryPolicy;
- this.transactionLockTimeout = transactionLockTimeout;
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
index e40688f..ef8d9b8 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabase.java
@@ -14,30 +14,50 @@
package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
+import com.google.gerrit.reviewdb.client.Project;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
-public class NoopSharedRefDatabase implements SharedRefDatabase {
+public class NoopSharedRefDatabase implements GlobalRefDatabase {
+
@Override
- public boolean isUpToDate(String project, Ref ref) {
+ public boolean isUpToDate(Project.NameKey project, Ref ref) throws GlobalRefDbLockException {
return true;
}
@Override
- public boolean compareAndPut(String project, Ref currRef, ObjectId newRefValue) {
+ public boolean compareAndPut(Project.NameKey project, Ref currRef, ObjectId newRefValue)
+ throws GlobalRefDbSystemError {
return true;
}
@Override
- public AutoCloseable lockRef(String project, String refName) {
- return () -> {};
- }
-
- @Override
- public boolean exists(String project, String refName) {
+ public <T> boolean compareAndPut(Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
return false;
}
@Override
- public void removeProject(String project) {}
+ public AutoCloseable lockRef(Project.NameKey project, String refName)
+ throws GlobalRefDbLockException {
+ return () -> {};
+ }
+
+ @Override
+ public boolean exists(Project.NameKey project, String refName) {
+ return false;
+ }
+
+ @Override
+ public void remove(Project.NameKey project) throws GlobalRefDbSystemError {}
+
+ @Override
+ public <T> Optional<T> get(Project.NameKey project, String refName, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return Optional.empty();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/SharedRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/SharedRefDatabase.java
deleted file mode 100644
index a93efcf..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/SharedRefDatabase.java
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
-
-import java.io.IOException;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectIdRef;
-import org.eclipse.jgit.lib.Ref;
-
-public interface SharedRefDatabase {
-
- /** A null ref that isn't associated to any name. */
- Ref NULL_REF = nullRef(null);
-
- /**
- * Create a new in-memory ref name associated with an NULL object id.
- *
- * @param refName ref name
- * @return the new NULL ref object
- */
- static Ref nullRef(String refName) {
- return new Ref() {
-
- @Override
- public String getName() {
- return refName;
- }
-
- @Override
- public boolean isSymbolic() {
- return false;
- }
-
- @Override
- public Ref getLeaf() {
- return null;
- }
-
- @Override
- public Ref getTarget() {
- return null;
- }
-
- @Override
- public ObjectId getObjectId() {
- return ObjectId.zeroId();
- }
-
- @Override
- public ObjectId getPeeledObjectId() {
- return ObjectId.zeroId();
- }
-
- @Override
- public boolean isPeeled() {
- return false;
- }
-
- @Override
- public Storage getStorage() {
- return Storage.NEW;
- }
- };
- }
-
- /**
- * Create a new in-memory Ref name associated with an objectId.
- *
- * @param refName ref name
- * @param objectId object id
- */
- static Ref newRef(String refName, ObjectId objectId) {
- return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId);
- }
-
- /**
- * Verify in shared db if Ref is the most recent
- *
- * @param project project name of the ref
- * @param ref to be checked against shared-ref db
- * @return true if it is; false otherwise
- * @throws SharedLockException if there was a problem locking the resource
- */
- boolean isUpToDate(String project, Ref ref) throws SharedLockException;
-
- /**
- * Compare a reference, and put if it matches.
- *
- * <p>Two reference match if and only if they satisfy the following:
- *
- * <ul>
- * <li>If one reference is a symbolic ref, the other one should be a symbolic ref.
- * <li>If both are symbolic refs, the target names should be same.
- * <li>If both are object ID refs, the object IDs should be same.
- * </ul>
- *
- * @param project project name of the ref
- * @param currRef old value to compare to. If the reference is expected to not exist the old value
- * has a storage of {@link org.eclipse.jgit.lib.Ref.Storage#NEW} and an ObjectId value of
- * {@code null}.
- * @param newRefValue new reference to store.
- * @return true if the put was successful; false otherwise.
- * @throws java.io.IOException the reference cannot be put due to a system error.
- */
- boolean compareAndPut(String project, Ref currRef, ObjectId newRefValue) throws IOException;
-
- /**
- * Lock a reference for writing.
- *
- * @param project project name
- * @param refName ref to lock
- * @return lock object
- * @throws SharedLockException if the lock cannot be obtained
- */
- AutoCloseable lockRef(String project, String refName) throws SharedLockException;
-
- /**
- * Verify if the DB contains a value for the specific project and ref name
- *
- * @param project
- * @param refName
- * @return true if the ref exists on the project
- */
- boolean exists(String project, String refName);
-
- /**
- * Clean project path from SharedRefDatabase
- *
- * @param project project name
- * @throws IOException
- */
- void removeProject(String project) throws IOException;
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabase.java
deleted file mode 100644
index 1179045..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabase.java
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright (C) 2012 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-import com.google.common.flogger.FluentLogger;
-import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.multisite.validation.ZkConnectionConfig;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedLockException;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.atomic.AtomicValue;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicValue;
-import org.apache.curator.framework.recipes.locks.InterProcessMutex;
-import org.apache.curator.framework.recipes.locks.Locker;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Ref;
-
-public class ZkSharedRefDatabase implements SharedRefDatabase {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private final CuratorFramework client;
- private final RetryPolicy retryPolicy;
-
- private final Long transactionLockTimeOut;
-
- @Inject
- public ZkSharedRefDatabase(CuratorFramework client, ZkConnectionConfig connConfig) {
- this.client = client;
- this.retryPolicy = connConfig.curatorRetryPolicy;
- this.transactionLockTimeOut = connConfig.transactionLockTimeout;
- }
-
- @Override
- public boolean isUpToDate(String project, Ref ref) throws SharedLockException {
- if (!exists(project, ref.getName())) {
- return true;
- }
-
- try {
- final byte[] valueInZk = client.getData().forPath(pathFor(project, ref.getName()));
-
- // Assuming this is a delete node NULL_REF
- if (valueInZk == null) {
- logger.atInfo().log(
- "%s:%s not found in Zookeeper, assumed as delete node NULL_REF",
- project, ref.getName());
- return false;
- }
-
- ObjectId objectIdInSharedRefDb = readObjectId(valueInZk);
- Boolean isUpToDate = objectIdInSharedRefDb.equals(ref.getObjectId());
-
- if (!isUpToDate) {
- logger.atWarning().log(
- "%s:%s is out of sync: local=%s zk=%s",
- project, ref.getName(), ref.getObjectId(), objectIdInSharedRefDb);
- }
-
- return isUpToDate;
- } catch (Exception e) {
- throw new SharedLockException(project, ref.getName(), e);
- }
- }
-
- @Override
- public void removeProject(String project) throws IOException {
- try {
- client.delete().deletingChildrenIfNeeded().forPath("/" + project);
- } catch (Exception e) {
- throw new IOException(String.format("Not able to delete project '%s'", project), e);
- }
- }
-
- @Override
- public boolean exists(String project, String refName) throws ZookeeperRuntimeException {
- try {
- return client.checkExists().forPath(pathFor(project, refName)) != null;
- } catch (Exception e) {
- throw new ZookeeperRuntimeException("Failed to check if path exists in Zookeeper", e);
- }
- }
-
- @Override
- public Locker lockRef(String project, String refName) throws SharedLockException {
- InterProcessMutex refPathMutex =
- new InterProcessMutex(client, "/locks" + pathFor(project, refName));
- try {
- return new Locker(refPathMutex, transactionLockTimeOut, MILLISECONDS);
- } catch (Exception e) {
- throw new SharedLockException(project, refName, e);
- }
- }
-
- @Override
- public boolean compareAndPut(String projectName, Ref oldRef, ObjectId newRefValue)
- throws IOException {
-
- final DistributedAtomicValue distributedRefValue =
- new DistributedAtomicValue(client, pathFor(projectName, oldRef), retryPolicy);
-
- try {
- if (oldRef == NULL_REF) {
- return distributedRefValue.initialize(writeObjectId(newRefValue));
- }
- final ObjectId newValue = newRefValue == null ? ObjectId.zeroId() : newRefValue;
- final AtomicValue<byte[]> newDistributedValue =
- distributedRefValue.compareAndSet(
- writeObjectId(oldRef.getObjectId()), writeObjectId(newValue));
-
- if (!newDistributedValue.succeeded() && refNotInZk(projectName, oldRef)) {
- return distributedRefValue.initialize(writeObjectId(newRefValue));
- }
-
- return newDistributedValue.succeeded();
- } catch (Exception e) {
- logger.atWarning().withCause(e).log(
- "Error trying to perform CAS at path %s", pathFor(projectName, oldRef));
- throw new IOException(
- String.format("Error trying to perform CAS at path %s", pathFor(projectName, oldRef)), e);
- }
- }
-
- private boolean refNotInZk(String projectName, Ref oldRef) throws Exception {
- return client.checkExists().forPath(pathFor(projectName, oldRef)) == null;
- }
-
- static String pathFor(String projectName, Ref oldRef) {
- return pathFor(projectName, oldRef.getName());
- }
-
- static String pathFor(String projectName, String refName) {
- return "/" + projectName + "/" + refName;
- }
-
- static ObjectId readObjectId(byte[] value) {
- return ObjectId.fromString(value, 0);
- }
-
- static byte[] writeObjectId(ObjectId value) {
- return ObjectId.toString(value).getBytes(StandardCharsets.US_ASCII);
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkValidationModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkValidationModule.java
deleted file mode 100644
index f83b4d9..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkValidationModule.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.inject.AbstractModule;
-import com.google.inject.Inject;
-import com.google.inject.Scopes;
-import com.googlesource.gerrit.plugins.multisite.Configuration;
-import com.googlesource.gerrit.plugins.multisite.ZookeeperConfig;
-import com.googlesource.gerrit.plugins.multisite.validation.ZkConnectionConfig;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import org.apache.curator.framework.CuratorFramework;
-
-public class ZkValidationModule extends AbstractModule {
-
- private ZookeeperConfig cfg;
-
- @Inject
- public ZkValidationModule(Configuration cfg) {
- this.cfg = new ZookeeperConfig(cfg.getMultiSiteConfig());
- }
-
- @Override
- protected void configure() {
- DynamicItem.bind(binder(), SharedRefDatabase.class)
- .to(ZkSharedRefDatabase.class)
- .in(Scopes.SINGLETON);
- bind(CuratorFramework.class).toInstance(cfg.buildCurator());
-
- bind(ZkConnectionConfig.class)
- .toInstance(
- new ZkConnectionConfig(cfg.buildCasRetryPolicy(), cfg.getZkInterProcessLockTimeOut()));
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperRuntimeException.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperRuntimeException.java
deleted file mode 100644
index 9f2951b..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperRuntimeException.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-/** Unable to communicate with Zookeeper */
-public class ZookeeperRuntimeException extends RuntimeException {
- private static final long serialVersionUID = 1L;
-
- public ZookeeperRuntimeException(String description, Throwable t) {
- super(description, t);
- }
-}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index 264c8e9..b6de0a9 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -3,12 +3,13 @@
the masters happens using the replication plugin and an external message
broker.
-This plugin allows Gerrit to publish and to consume events over a Kafka
+This plugin allows Gerrit to publish and to consume events over a
message broker for aligning with the other masters over different sites.
The masters must be:
-* migrated to NoteDb
+* events-broker library must be installed as a library module in the
+ `$GERRIT_SITE/lib` directory of all the masters
* connected to the same message broker
* behind a load balancer (e.g., HAProxy)
@@ -57,34 +58,12 @@
Prerequisites:
-* Kafka message broker deployed in a multi-master setup across all the sites
+* Message broker deployed in a multi-master setup across all the sites
For the masters:
* Install and configure @PLUGIN@ plugin
-Here is an example of minimal @PLUGIN@.config:
-
-For all the masters on all the sites:
-
-```
-[kafka]
- bootstrapServers = kafka-1:9092,kafka-2:9092,kafka-3:9092
- eventTopic = gerrit_index
-
-[kafka "publisher"]
- enable = true
- indexEventTopic = gerrit_index
- streamEventTopic = gerrit_stream
- cacheEvictionEventTopic = gerrit_cache_eviction
-
-[kafka "subscriber"]
- enable = true
- pollingIntervalMs = 1000
- autoCommitIntervalMs = 1000
-```
-
-
For further information and supported options, refer to [config](config.md)
documentation.
@@ -95,21 +74,25 @@
### Broker message publisher
* Broker message published count
-`metric=multi_site/broker/broker_message_publisher_counter/broker_msg_publisher_counter, type=com.codahale.metrics.Meter`
+`metric=plugins/multi-site/multi_site/broker/broker_message_publisher_counter/broker_msg_publisher_counter, type=com.codahale.metrics.Meter`
* Broker failed to publish message count
-`metric=multi_site/broker/broker_message_publisher_failure_counter/broker_msg_publisher_failure_counter, type=com.codahale.metrics.Meter`
+`metric=plugins/multi-site/multi_site/broker/broker_message_publisher_failure_counter/broker_msg_publisher_failure_counter, type=com.codahale.metrics.Meter`
### Message subscriber
* Subscriber message consumed count
-`multi_site/subscriber/subscriber_message_consumer_counter/subscriber_msg_consumer_counter, type=com.codahale.metrics.Meter`
+`metric=plugins/multi-site/multi_site/subscriber/subscriber_message_consumer_counter/subscriber_msg_consumer_counter, type=com.codahale.metrics.Meter`
* Subscriber failed to consume message count
-`multi_site/subscriber/subscriber_message_consumer_failure_counter/subscriber_msg_consumer_failure_counter, type=com.codahale.metrics.Meter`
+`metric=plugins/multi-site/multi_site/subscriber/subscriber_message_consumer_failure_counter/subscriber_msg_consumer_failure_counter, type=com.codahale.metrics.Meter`
* Subscriber failed to poll messages count
-`multi_site/subscriber/subscriber_message_consumer_poll_failure_counter/subscriber_msg_consumer_poll_failure_counter, type=com.codahale.metrics.Meter`
+`metric=plugins/multi-site/multi_site/subscriber/subscriber_message_consumer_failure_counter/subscriber_msg_consumer_poll_failure_counter, type=com.codahale.metrics.Meter`
+
+* Subscriber replication lag (sec behind the producer)
+
+`metric=site/multi_site/subscriber/subscriber_replication_status/sec_behind, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 227d0e8..662f46e 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -3,60 +3,8 @@
=========================
The @PLUGIN@ plugin must be installed as a library module in the
-`$GERRIT_SITE/lib` folder of all the instances and the following fields should
-be specified in the `$site_path/etc/@PLUGIN@.config` file:
-
-File '@PLUGIN@.config'
---------------------
-
-## Sample configuration.
-
-```
-[kafka]
- bootstrapServers = kafka-1:9092,kafka-2:9092,kafka-3:9092
-
- indexEventTopic = gerrit_index
- streamEventTopic = gerrit_stream
- cacheEventTopic = gerrit_cache_eviction
- projectListEventTopic = gerrit_project_list
-
-[kafka "publisher"]
- enabled = true
-
- indexEventEnabled = true
- cacheEventEnabled = true
- projectListEventEnabled = true
- streamEventEnabled = true
-
- KafkaProp-compressionType = none
- KafkaProp-deliveryTimeoutMs = 60000
-
-[kafka "subscriber"]
- enabled = true
- pollingIntervalMs = 1000
-
- KafkaProp-enableAutoCommit = true
- KafkaProp-autoCommitIntervalMs = 1000
- KafkaProp-autoCommitIntervalMs = 5000
-
- indexEventEnabled = true
- cacheEventEnabled = true
- projectListEventEnabled = true
- streamEventEnabled = true
-
-[ref-database "zookeeper"]
- connectString = "localhost:2181"
- rootNode = "/gerrit/multi-site"
- sessionTimeoutMs = 1000
- connectionTimeoutMs = 1000
- retryPolicyBaseSleepTimeMs = 1000
- retryPolicyMaxSleepTimeMs = 3000
- retryPolicyMaxRetries = 3
- casRetryPolicyBaseSleepTimeMs = 100
- casRetryPolicyMaxSleepTimeMs = 100
- casRetryPolicyMaxRetries = 3
- transactionLockTimeoutMs = 1000
-```
+`$GERRIT_SITE/lib` folder of all the instances. Configuration should
+be specified in the `$site_path/etc/@PLUGIN@.config` file.
## Configuration parameters
@@ -106,83 +54,22 @@
: The time interval in milliseconds between subsequent auto-retries.
Defaults to 30000 (30 seconds).
-```kafka.bootstrapServers```
-: List of Kafka broker hosts (host:port) to use for publishing events to the message
- broker
-
-```kafka.indexEventTopic```
-: Name of the Kafka topic to use for publishing indexing events
+```broker.indexEventTopic```
+: Name of the topic to use for publishing indexing events
Defaults to GERRIT.EVENT.INDEX
-```kafka.streamEventTopic```
-: Name of the Kafka topic to use for publishing stream events
+```broker.streamEventTopic```
+: Name of the topic to use for publishing stream events
Defaults to GERRIT.EVENT.STREAM
-```kafka.cacheEventTopic```
-: Name of the Kafka topic to use for publishing cache eviction events
+```broker.cacheEventTopic```
+: Name of the topic to use for publishing cache eviction events
Defaults to GERRIT.EVENT.CACHE
-```kafka.projectListEventTopic```
-: Name of the Kafka topic to use for publishing cache eviction events
+```broker.projectListEventTopic```
+: Name of the topic to use for publishing cache eviction events
Defaults to GERRIT.EVENT.PROJECT.LIST
-```kafka.publisher.indexEventEnabled```
-: Enable publication of index events, ignored when `kafka.publisher.enabled`
- is false
-
- Defaults: true
-
-```kafka.publisher.cacheEventEnabled```
-: Enable publication of cache events, ignored when `kafka.publisher.enabled`
- is false
-
- Defaults: true
-
-```kafka.publisher.projectListEventEnabled```
-: Enable publication of project list events, ignored when `kafka.publisher.enabled`
- is false
-
- Defaults: true
-
-```kafka.publisher.streamEventEnabled```
-: Enable publication of stream events, ignored when `kafka.publisher.enabled`
- is false
-
- Defaults: true
-
-```kafka.subscriber.enabled```
-: Enable consuming of events from Kafka
- Defaults: false
-
-```kafka.subscriber.indexEventEnabled```
-: Enable consumption of index events, ignored when `kafka.subscriber.enabled`
- is false
-
- Defaults: true
-
-```kafka.subscriber.cacheEventEnabled```
-: Enable consumption of cache events, ignored when `kafka.subscriber.enabled`
- is false
-
- Defaults: true
-
-```kafka.subscriber.projectListEventEnabled```
-: Enable consumption of project list events, ignored when `kafka.subscriber.enabled`
- is false
-
- Defaults: true
-
-```kafka.subscriber.streamEventEnabled```
-: Enable consumption of stream events, ignored when `kafka.subscriber.enabled`
- is false
-
- Defaults: true
-
-```kafka.subscriber.pollingIntervalMs```
-: Polling interval in milliseconds for checking incoming events
-
- Defaults: 1000
-
```ref-database.enabled```
: Enable the use of a shared ref-database
Defaults: true
@@ -213,88 +100,4 @@
Relax the alignment with the shared ref-database for AProject on refs/heads/feature.
- Defaults: No rules = All projects are REQUIRED to be consistent on all refs.
-
-```ref-database.zookeeper.connectString```
-: Connection string to Zookeeper
-
-```ref-database.zookeeper.rootNode```
-: Root node to use in Zookeeper to store/retrieve information
-
- Defaults: "/gerrit/multi-site"
-
-
-```ref-database.zookeeper.sessionTimeoutMs```
-: Zookeeper session timeout in milliseconds
-
- Defaults: 1000
-
-```ref-database.zookeeper.connectionTimeoutMs```
-: Zookeeper connection timeout in milliseconds
-
- Defaults: 1000
-
-```ref-database.zookeeper.retryPolicyBaseSleepTimeMs```
-: Configuration for the base sleep timeout in milliseconds of the
- BoundedExponentialBackoffRetry policy used for the Zookeeper connection
-
- Defaults: 1000
-
-```ref-database.zookeeper.retryPolicyMaxSleepTimeMs```
-: Configuration for the maximum sleep timeout in milliseconds of the
- BoundedExponentialBackoffRetry policy used for the Zookeeper connection
-
- Defaults: 3000
-
-```ref-database.zookeeper.retryPolicyMaxRetries```
-: Configuration for the maximum number of retries of the
- BoundedExponentialBackoffRetry policy used for the Zookeeper connection
-
- Defaults: 3
-
-```ref-database.zookeeper.casRetryPolicyBaseSleepTimeMs```
-: Configuration for the base sleep timeout in milliseconds of the
- BoundedExponentialBackoffRetry policy used for the Compare and Swap
- operations on Zookeeper
-
- Defaults: 1000
-
-```ref-database.zookeeper.casRetryPolicyMaxSleepTimeMs```
-: Configuration for the maximum sleep timeout in milliseconds of the
- BoundedExponentialBackoffRetry policy used for the Compare and Swap
- operations on Zookeeper
-
- Defaults: 3000
-
-```ref-database.zookeeper.casRetryPolicyMaxRetries```
-: Configuration for the maximum number of retries of the
- BoundedExponentialBackoffRetry policy used for the Compare and Swap
- operations on Zookeeper
-
- Defaults: 3
-
-```ref-database.zookeeper.transactionLockTimeoutMs```
-: Configuration for the Zookeeper Lock timeout (in milliseconds) used when reading data
- from Zookeeper, applying the git local changes and writing the new objectId
- into Zookeeper
-
- Defaults: 1000
-
-#### Custom kafka properties:
-
-In addition to the above settings, custom Kafka properties can be explicitly set
-for `publisher` and `subscriber`.
-In order to be acknowledged, these properties need to be prefixed with the
-`KafkaProp-` prefix and be formatted using camel case, as follows: `KafkaProp-yourPropertyValue`
-
-For example, if you want to set the `auto.commit.interval.ms` property for
-consumers, you need to configure this property as `KafkaProp-autoCommitIntervalMs`.
-
-**NOTE**: custom Kafka properties will be ignored when the relevant subsection is
-disabled (i.e. `kafka.subscriber.enabled` and/or `kafka.publisher.enabled` are
-set to `false`).
-
-The complete list of available settings can be found directly in the kafka website:
-
-* **Publisher**: https://kafka.apache.org/documentation/#producerconfigs
-* **Subscriber**: https://kafka.apache.org/documentation/#consumerconfigs
+ Defaults: No rules = All projects are REQUIRED to be consistent on all refs.
\ No newline at end of file
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/ModuleTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/ModuleTest.java
index 8b7a17b..2df60dd 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/ModuleTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/ModuleTest.java
@@ -17,7 +17,6 @@
import static com.google.common.truth.Truth.assertThat;
import com.google.gerrit.server.config.SitePaths;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerModule;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -37,16 +36,13 @@
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Configuration configMock;
- @Mock private NoteDbStatus noteDb;
- @Mock private BrokerModule brokerModule;
-
@Rule public TemporaryFolder tempFolder = new TemporaryFolder();
private Module module;
@Before
public void setup() {
- module = new Module(configMock, noteDb, brokerModule);
+ module = new Module(configMock);
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapperTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapperTest.java
index 2abc7b0..92fa101 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapperTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/BrokerApiWrapperTest.java
@@ -5,9 +5,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.gerritforge.gerrit.eventbroker.BrokerApi;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.Event;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
+import com.googlesource.gerrit.plugins.multisite.MessageLogger;
+import java.util.UUID;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -19,26 +21,30 @@
@Mock private BrokerMetrics brokerMetrics;
@Mock private BrokerApi brokerApi;
@Mock Event event;
+ @Mock MessageLogger msgLog;
+ private UUID instanceId = UUID.randomUUID();
+ private String topic = "index";
private BrokerApiWrapper objectUnderTest;
@Before
public void setUp() {
objectUnderTest =
- new BrokerApiWrapper(DynamicItem.itemOf(BrokerApi.class, brokerApi), brokerMetrics);
+ new BrokerApiWrapper(
+ DynamicItem.itemOf(BrokerApi.class, brokerApi), brokerMetrics, msgLog, instanceId);
}
@Test
public void shouldIncrementBrokerMetricCounterWhenMessagePublished() {
when(brokerApi.send(any(), any())).thenReturn(true);
- objectUnderTest.send(EventTopic.INDEX_TOPIC.topic(), event);
+ objectUnderTest.send(topic, event);
verify(brokerMetrics, only()).incrementBrokerPublishedMessage();
}
@Test
public void shouldIncrementBrokerFailedMetricCounterWhenMessagePublishingFailed() {
when(brokerApi.send(any(), any())).thenReturn(false);
- objectUnderTest.send(EventTopic.INDEX_TOPIC.topic(), event);
+ objectUnderTest.send(topic, event);
verify(brokerMetrics, only()).incrementBrokerFailedToPublishMessage();
}
@@ -47,7 +53,7 @@
when(brokerApi.send(any(), any()))
.thenThrow(new RuntimeException("Unexpected runtime exception"));
try {
- objectUnderTest.send(EventTopic.INDEX_TOPIC.topic(), event);
+ objectUnderTest.send(topic, event);
} catch (RuntimeException e) {
// expected
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisherTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisherTest.java
deleted file mode 100644
index c751a84..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/broker/kafka/BrokerPublisherTest.java
+++ /dev/null
@@ -1,178 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.broker.kafka;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.only;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import com.google.gerrit.extensions.client.ChangeKind;
-import com.google.gerrit.reviewdb.client.Account;
-import com.google.gerrit.reviewdb.client.Branch;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.server.data.AccountAttribute;
-import com.google.gerrit.server.data.ApprovalAttribute;
-import com.google.gerrit.server.events.CommentAddedEvent;
-import com.google.gerrit.server.events.Event;
-import com.google.gerrit.server.util.time.TimeUtil;
-import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.googlesource.gerrit.plugins.multisite.MessageLogger;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerMetrics;
-import com.googlesource.gerrit.plugins.multisite.broker.BrokerSession;
-import com.googlesource.gerrit.plugins.multisite.broker.GsonProvider;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import java.util.UUID;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-
-@RunWith(MockitoJUnitRunner.class)
-public class BrokerPublisherTest {
-
- @Mock private BrokerMetrics brokerMetrics;
- @Mock private BrokerSession session;
- @Mock private MessageLogger msgLog;
- private BrokerPublisher publisher;
-
- private Gson gson = new GsonProvider().get();
-
- private String accountName = "Foo Bar";
- private String accountEmail = "foo@bar.com";
- private String accountUsername = "foobar";
- private String approvalType = ChangeKind.REWORK.toString();
-
- private String approvalDescription = "ApprovalDescription";
- private String approvalValue = "+2";
- private String oldApprovalValue = "+1";
- private Long approvalGrantedOn = 123L;
- private String commentDescription = "Patch Set 1: Code-Review+2";
- private String projectName = "project";
- private String refName = "refs/heads/master";
- private String changeId = "Iabcd1234abcd1234abcd1234abcd1234abcd1234";
- private Long eventCreatedOn = 123L;
-
- @Before
- public void setUp() {
- publisher = new BrokerPublisher(session, gson, UUID.randomUUID(), msgLog);
- }
-
- @Test
- public void shouldSerializeCommentAddedEvent() {
-
- Event event = createSampleEvent();
-
- String expectedSerializedCommentEvent =
- "{\"author\": {\"name\": \""
- + accountName
- + "\",\"email\": \""
- + accountEmail
- + "\",\"username\": \""
- + accountUsername
- + "\"},\"approvals\": [{\"type\": \""
- + approvalType
- + "\",\"description\": \""
- + approvalDescription
- + "\",\"value\": \""
- + approvalValue
- + "\",\"oldValue\": \""
- + oldApprovalValue
- + "\",\"grantedOn\": "
- + approvalGrantedOn
- + ",\"by\": {\"name\": \""
- + accountName
- + "\",\"email\": \""
- + accountEmail
- + "\",\"username\": \""
- + accountUsername
- + "\"}}],\"comment\": \""
- + commentDescription
- + "\",\""
- + projectName
- + "\": {\"name\": \""
- + projectName
- + "\"},\"refName\": \""
- + refName
- + "\",\"changeKey\": {\"id\": \""
- + changeId
- + "\""
- + " },\"type\": \"comment-added\",\"eventCreatedOn\": "
- + eventCreatedOn
- + "}";
-
- JsonObject expectedCommentEventJsonObject =
- gson.fromJson(expectedSerializedCommentEvent, JsonElement.class).getAsJsonObject();
-
- assertThat(publisher.eventToJson(event).equals(expectedCommentEventJsonObject)).isTrue();
- }
-
- @Test
- public void shouldLogEventPublishedMessageWhenPublishingSucceed() {
- Event event = createSampleEvent();
- when(session.publish(any(), any())).thenReturn(true);
- publisher.publish(EventTopic.INDEX_TOPIC.topic(), event);
- verify(msgLog, only()).log(any(), any());
- }
-
- @Test
- public void shouldSkipEventPublishedLoggingWhenMessagePublishigFailed() {
- Event event = createSampleEvent();
- when(session.publish(any(), any())).thenReturn(false);
-
- publisher.publish(EventTopic.INDEX_TOPIC.topic(), event);
- verify(msgLog, never()).log(any(), any());
- }
-
- private Event createSampleEvent() {
- final Change change =
- new Change(
- new Change.Key(changeId),
- new Change.Id(1),
- new Account.Id(1),
- new Branch.NameKey(projectName, refName),
- TimeUtil.nowTs());
-
- CommentAddedEvent event = new CommentAddedEvent(change);
- AccountAttribute accountAttribute = new AccountAttribute();
- accountAttribute.email = accountEmail;
- accountAttribute.name = accountName;
- accountAttribute.username = accountUsername;
-
- event.eventCreatedOn = eventCreatedOn;
- event.approvals =
- () -> {
- ApprovalAttribute approvalAttribute = new ApprovalAttribute();
- approvalAttribute.value = approvalValue;
- approvalAttribute.oldValue = oldApprovalValue;
- approvalAttribute.description = approvalDescription;
- approvalAttribute.by = accountAttribute;
- approvalAttribute.type = ChangeKind.REWORK.toString();
- approvalAttribute.grantedOn = approvalGrantedOn;
-
- return new ApprovalAttribute[] {approvalAttribute};
- };
-
- event.author = () -> accountAttribute;
- event.comment = commentDescription;
-
- return event;
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java
new file mode 100644
index 0000000..ead3189
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java
@@ -0,0 +1,98 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.consumer;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.gerritforge.gerrit.eventbroker.EventMessage;
+import com.google.common.base.Suppliers;
+import com.google.gerrit.metrics.MetricMaker;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.data.RefUpdateAttribute;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
+import java.util.Optional;
+import java.util.UUID;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SubscriberMetricsTest {
+ private static final String A_TEST_PROJECT_NAME = "test-project";
+ private static final Project.NameKey A_TEST_PROJECT_NAME_KEY =
+ Project.nameKey(A_TEST_PROJECT_NAME);
+
+ @Mock private SharedRefDatabaseWrapper sharedRefDb;
+ @Mock private GitReferenceUpdated gitReferenceUpdated;
+ @Mock private MetricMaker metricMaker;
+ @Mock private ProjectVersionLogger verLogger;
+ @Mock private ProjectVersionRefUpdate projectVersionRefUpdate;
+ private SubscriberMetrics metrics;
+ private EventMessage.Header msgHeader;
+
+ @Before
+ public void setup() throws Exception {
+ msgHeader = new EventMessage.Header(UUID.randomUUID(), UUID.randomUUID());
+ metrics = new SubscriberMetrics(metricMaker, projectVersionRefUpdate, verLogger);
+ }
+
+ @Test
+ public void shouldLogProjectVersionWhenReceivingRefUpdatedEventWithoutLag() {
+ Optional<Long> globalRefDbVersion = Optional.of(System.currentTimeMillis() / 1000);
+ when(projectVersionRefUpdate.getProjectRemoteVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+ when(projectVersionRefUpdate.getProjectLocalVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+
+ EventMessage eventMessage = new EventMessage(msgHeader, newRefUpdateEvent());
+
+ metrics.updateReplicationStatusMetrics(eventMessage);
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, globalRefDbVersion.get(), 0);
+ }
+
+ @Test
+ public void shouldLogProjectVersionWhenReceivingRefUpdatedEventWithALag() {
+ Optional<Long> globalRefDbVersion = Optional.of(System.currentTimeMillis() / 1000);
+ long replicationLag = 60;
+ when(projectVersionRefUpdate.getProjectRemoteVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion.map(ts -> ts + replicationLag));
+ when(projectVersionRefUpdate.getProjectLocalVersion(A_TEST_PROJECT_NAME))
+ .thenReturn(globalRefDbVersion);
+
+ EventMessage eventMessage = new EventMessage(msgHeader, newRefUpdateEvent());
+
+ metrics.updateReplicationStatusMetrics(eventMessage);
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, globalRefDbVersion.get(), replicationLag);
+ }
+
+ private RefUpdatedEvent newRefUpdateEvent() {
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.project = A_TEST_PROJECT_NAME;
+ refUpdate.refName = "refs/heads/foo";
+ refUpdate.newRev = "591727cfec5174368a7829f79741c41683d84c89";
+ RefUpdatedEvent refUpdateEvent = new RefUpdatedEvent();
+ refUpdateEvent.refUpdate = Suppliers.ofInstance(refUpdate);
+ return refUpdateEvent;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/CacheEvictionEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
similarity index 96%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/CacheEvictionEventRouterTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
index 23cbd4d..9292262 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/CacheEvictionEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/CacheEvictionEventRouterTest.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
+package com.googlesource.gerrit.plugins.multisite.event;
import static org.mockito.Mockito.verify;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/IndexEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
similarity index 98%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/IndexEventRouterTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
index 81e832a..bf6f043 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/IndexEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/IndexEventRouterTest.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
+package com.googlesource.gerrit.plugins.multisite.event;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/ProjectListUpdateRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/ProjectListUpdateRouterTest.java
similarity index 95%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/ProjectListUpdateRouterTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/event/ProjectListUpdateRouterTest.java
index 00a239b..93daf92 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/ProjectListUpdateRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/ProjectListUpdateRouterTest.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
+package com.googlesource.gerrit.plugins.multisite.event;
import static org.mockito.Mockito.verify;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/StreamEventRouterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
similarity index 96%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/StreamEventRouterTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
index ee79e34..31afa79 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/StreamEventRouterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/event/StreamEventRouterTest.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
+package com.googlesource.gerrit.plugins.multisite.event;
import static org.mockito.Mockito.verify;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandlerTest.java
index 3b01c61..5c36ada 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandlerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedEventHandlerTest.java
@@ -19,11 +19,11 @@
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.verify;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.ProjectCreatedEvent;
import com.google.gerrit.server.util.OneOffRequestContext;
-import com.google.gwtorm.server.OrmException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -81,7 +81,7 @@
(Answer<Void>)
invocation -> {
assertThat(Context.isForwardedEvent()).isTrue();
- throw new OrmException("someMessage");
+ throw new StorageException("someMessage");
})
.when(dispatcherMock)
.postEvent(event);
@@ -89,8 +89,8 @@
assertThat(Context.isForwardedEvent()).isFalse();
try {
handler.dispatch(event);
- fail("should have throw an OrmException");
- } catch (OrmException e) {
+ fail("should have throw an StorageException");
+ } catch (StorageException e) {
assertThat(e.getMessage()).isEqualTo("someMessage");
}
assertThat(Context.isForwardedEvent()).isFalse();
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandlerTest.java
index 2793253..0a910c5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandlerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/forwarder/ForwardedIndexChangeHandlerTest.java
@@ -24,17 +24,15 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
import com.google.gerrit.server.util.time.TimeUtil;
-import com.google.gwtorm.server.OrmException;
-import com.google.inject.util.Providers;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.forwarder.ForwardedIndexingHandler.Operation;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.ChangeIndexEvent;
@@ -60,10 +58,8 @@
private static String TEST_CHANGE_ID = TEST_PROJECT + "~" + TEST_CHANGE_NUMBER;
private static final boolean CHANGE_EXISTS = true;
private static final boolean CHANGE_DOES_NOT_EXIST = false;
- private static final boolean DO_NOT_THROW_IO_EXCEPTION = false;
- private static final boolean DO_NOT_THROW_ORM_EXCEPTION = false;
- private static final boolean THROW_IO_EXCEPTION = true;
- private static final boolean THROW_ORM_EXCEPTION = true;
+ private static final boolean DO_NOT_THROW_STORAGE_EXCEPTION = false;
+ private static final boolean THROW_STORAGE_EXCEPTION = true;
private static final boolean CHANGE_UP_TO_DATE = true;
private static final boolean CHANGE_OUTDATED = false;
@@ -71,7 +67,6 @@
@Mock private ChangeIndexer indexerMock;
@Mock private OneOffRequestContext ctxMock;
@Mock private ManualRequestContext manualRequestContextMock;
- @Mock private ReviewDb dbMock;
@Mock private ChangeNotes changeNotes;
@Mock private Configuration configurationMock;
@Mock private Configuration.Index index;
@@ -87,7 +82,6 @@
@Before
public void setUp() throws Exception {
when(ctxMock.open()).thenReturn(manualRequestContextMock);
- when(manualRequestContextMock.getReviewDbProvider()).thenReturn(Providers.of(dbMock));
id = new Change.Id(TEST_CHANGE_NUMBER);
change = new Change(null, id, null, null, TimeUtil.nowTs());
when(changeNotes.getChange()).thenReturn(change);
@@ -103,7 +97,7 @@
public void changeIsIndexedWhenUpToDate() throws Exception {
setupChangeAccessRelatedMocks(CHANGE_EXISTS, CHANGE_UP_TO_DATE);
handler.index(TEST_CHANGE_ID, Operation.INDEX, Optional.empty());
- verify(indexerMock, times(1)).index(any(ReviewDb.class), any(Change.class));
+ verify(indexerMock, times(1)).index(any(Change.class));
}
@Test
@@ -111,7 +105,7 @@
setupChangeAccessRelatedMocks(CHANGE_EXISTS, CHANGE_OUTDATED);
handler.index(
TEST_CHANGE_ID, Operation.INDEX, Optional.of(new ChangeIndexEvent("foo", 1, false)));
- verify(indexerMock, times(1)).index(any(ReviewDb.class), any(Change.class));
+ verify(indexerMock, times(1)).index(any(Change.class));
}
@Test
@@ -125,22 +119,13 @@
setupChangeAccessRelatedMocks(CHANGE_DOES_NOT_EXIST, CHANGE_OUTDATED);
handler.index(TEST_CHANGE_ID, Operation.INDEX, Optional.empty());
verify(indexerMock, never()).delete(id);
- verify(indexerMock, never())
- .index(any(ReviewDb.class), any(Project.NameKey.class), any(Change.Id.class));
+ verify(indexerMock, never()).index(any(Project.NameKey.class), any(Change.Id.class));
}
@Test
- public void schemaThrowsExceptionWhenLookingUpForChange() throws Exception {
- setupChangeAccessRelatedMocks(CHANGE_EXISTS, THROW_ORM_EXCEPTION, CHANGE_UP_TO_DATE);
- exception.expect(OrmException.class);
- handler.index(TEST_CHANGE_ID, Operation.INDEX, Optional.empty());
- }
-
- @Test
- public void indexerThrowsIOExceptionTryingToIndexChange() throws Exception {
- setupChangeAccessRelatedMocks(
- CHANGE_EXISTS, DO_NOT_THROW_ORM_EXCEPTION, THROW_IO_EXCEPTION, CHANGE_UP_TO_DATE);
- exception.expect(IOException.class);
+ public void indexerThrowsStorageExceptionTryingToIndexChange() throws Exception {
+ setupChangeAccessRelatedMocks(CHANGE_EXISTS, THROW_STORAGE_EXCEPTION, CHANGE_UP_TO_DATE);
+ exception.expect(StorageException.class);
handler.index(TEST_CHANGE_ID, Operation.INDEX, Optional.empty());
}
@@ -156,13 +141,13 @@
return null;
})
.when(indexerMock)
- .index(any(ReviewDb.class), any(Change.class));
+ .index(any(Change.class));
assertThat(Context.isForwardedEvent()).isFalse();
handler.index(TEST_CHANGE_ID, Operation.INDEX, Optional.empty());
assertThat(Context.isForwardedEvent()).isFalse();
- verify(indexerMock, times(1)).index(any(ReviewDb.class), any(Change.class));
+ verify(indexerMock, times(1)).index(any(Change.class));
}
@Test
@@ -175,7 +160,7 @@
throw new IOException("someMessage");
})
.when(indexerMock)
- .index(any(ReviewDb.class), any(Change.class));
+ .index(any(Change.class));
assertThat(Context.isForwardedEvent()).isFalse();
try {
@@ -186,36 +171,22 @@
}
assertThat(Context.isForwardedEvent()).isFalse();
- verify(indexerMock, times(1)).index(any(ReviewDb.class), any(Change.class));
+ verify(indexerMock, times(1)).index(any(Change.class));
}
private void setupChangeAccessRelatedMocks(boolean changeExist, boolean changeUpToDate)
throws Exception {
- setupChangeAccessRelatedMocks(
- changeExist, DO_NOT_THROW_ORM_EXCEPTION, DO_NOT_THROW_IO_EXCEPTION, changeUpToDate);
+ setupChangeAccessRelatedMocks(changeExist, DO_NOT_THROW_STORAGE_EXCEPTION, changeUpToDate);
}
private void setupChangeAccessRelatedMocks(
- boolean changeExist, boolean ormException, boolean changeUpToDate)
- throws OrmException, IOException {
- setupChangeAccessRelatedMocks(
- changeExist, ormException, DO_NOT_THROW_IO_EXCEPTION, changeUpToDate);
- }
-
- private void setupChangeAccessRelatedMocks(
- boolean changeExists, boolean ormException, boolean ioException, boolean changeIsUpToDate)
- throws OrmException, IOException {
- if (ormException) {
- doThrow(new OrmException("")).when(ctxMock).open();
- }
-
+ boolean changeExists, boolean storageException, boolean changeIsUpToDate)
+ throws StorageException {
if (changeExists) {
when(changeCheckerFactoryMock.create(TEST_CHANGE_ID)).thenReturn(changeCheckerPresentMock);
when(changeCheckerPresentMock.getChangeNotes()).thenReturn(Optional.of(changeNotes));
- if (ioException) {
- doThrow(new IOException("io-error"))
- .when(indexerMock)
- .index(any(ReviewDb.class), any(Change.class));
+ if (storageException) {
+ doThrow(new StorageException("io-error")).when(indexerMock).index(any(Change.class));
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfigurationTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfigurationTest.java
deleted file mode 100644
index 84e1dd8..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/KafkaConfigurationTest.java
+++ /dev/null
@@ -1,200 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration.ENABLE_KEY;
-import static com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration.KAFKA_PROPERTY_PREFIX;
-import static com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration.KAFKA_SECTION;
-import static com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration.KafkaPublisher.KAFKA_PUBLISHER_SUBSECTION;
-import static com.googlesource.gerrit.plugins.multisite.kafka.KafkaConfiguration.KafkaSubscriber.KAFKA_SUBSCRIBER_SUBSECTION;
-
-import com.googlesource.gerrit.plugins.multisite.Configuration;
-import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventTopic;
-import org.eclipse.jgit.lib.Config;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.junit.MockitoJUnitRunner;
-
-@RunWith(MockitoJUnitRunner.class)
-public class KafkaConfigurationTest {
-
- private Config globalPluginConfig;
- private Configuration multiSiteConfig;
-
- @Before
- public void setup() {
- globalPluginConfig = new Config();
- multiSiteConfig = new Configuration(globalPluginConfig, new Config());
- }
-
- private KafkaConfiguration getConfiguration() {
- return new KafkaConfiguration(multiSiteConfig);
- }
-
- @Test
- public void kafkaSubscriberPropertiesAreSetWhenSectionIsEnabled() {
- final String kafkaPropertyName = KAFKA_PROPERTY_PREFIX + "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, ENABLE_KEY, true);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaSubscriber().getProperty("foo.bar.baz");
-
- assertThat(property.equals(kafkaPropertyValue)).isTrue();
- }
-
- @Test
- public void kafkaSubscriberPropertiesAreNotSetWhenSectionIsDisabled() {
- final String kafkaPropertyName = KAFKA_PROPERTY_PREFIX + "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, ENABLE_KEY, false);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaSubscriber().getProperty("foo.bar.baz");
-
- assertThat(property).isNull();
- }
-
- @Test
- public void kafkaSubscriberPropertiesAreIgnoredWhenPrefixIsNotSet() {
- final String kafkaPropertyName = "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, ENABLE_KEY, true);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_SUBSCRIBER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaSubscriber().getProperty("foo.bar.baz");
-
- assertThat(property).isNull();
- }
-
- @Test
- public void kafkaPublisherPropertiesAreSetWhenSectionIsEnabled() {
- final String kafkaPropertyName = KAFKA_PROPERTY_PREFIX + "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, ENABLE_KEY, true);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaPublisher().getProperty("foo.bar.baz");
-
- assertThat(property.equals(kafkaPropertyValue)).isTrue();
- }
-
- @Test
- public void kafkaPublisherPropertiesAreIgnoredWhenPrefixIsNotSet() {
- final String kafkaPropertyName = "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, ENABLE_KEY, true);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaPublisher().getProperty("foo.bar.baz");
-
- assertThat(property).isNull();
- }
-
- @Test
- public void kafkaPublisherPropertiesAreNotSetWhenSectionIsDisabled() {
- final String kafkaPropertyName = KAFKA_PROPERTY_PREFIX + "fooBarBaz";
- final String kafkaPropertyValue = "aValue";
- globalPluginConfig.setBoolean(KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, ENABLE_KEY, false);
- globalPluginConfig.setString(
- KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, kafkaPropertyName, kafkaPropertyValue);
-
- final String property = getConfiguration().kafkaPublisher().getProperty("foo.bar.baz");
-
- assertThat(property).isNull();
- }
-
- @Test
- public void shouldReturnKafkaTopicAliasForIndexTopic() {
- setKafkaTopicAlias("indexEventTopic", "gerrit_index");
- final String property = getConfiguration().getKafka().getTopicAlias(EventTopic.INDEX_TOPIC);
-
- assertThat(property).isEqualTo("gerrit_index");
- }
-
- @Test
- public void shouldReturnKafkaTopicAliasForStreamEventTopic() {
- setKafkaTopicAlias("streamEventTopic", "gerrit_stream_events");
- final String property =
- getConfiguration().getKafka().getTopicAlias(EventTopic.STREAM_EVENT_TOPIC);
-
- assertThat(property).isEqualTo("gerrit_stream_events");
- }
-
- @Test
- public void shouldReturnKafkaTopicAliasForProjectListEventTopic() {
- setKafkaTopicAlias("projectListEventTopic", "gerrit_project_list");
- final String property =
- getConfiguration().getKafka().getTopicAlias(EventTopic.PROJECT_LIST_TOPIC);
-
- assertThat(property).isEqualTo("gerrit_project_list");
- }
-
- @Test
- public void shouldReturnKafkaTopicAliasForCacheEventTopic() {
- setKafkaTopicAlias("cacheEventTopic", "gerrit_cache");
- final String property = getConfiguration().getKafka().getTopicAlias(EventTopic.CACHE_TOPIC);
-
- assertThat(property).isEqualTo("gerrit_cache");
- }
-
- @Test
- public void shouldReturnKafkaTopicEnabledForCacheEventTopic() {
- setKafkaTopicEnabled("cacheEventEnabled", false);
- final Boolean property =
- getConfiguration().kafkaPublisher().enabledEvent(EventTopic.CACHE_TOPIC);
- assertThat(property).isFalse();
- }
-
- @Test
- public void shouldReturnKafkaTopicEnabledForIndexTopic() {
- setKafkaTopicEnabled("indexEventEnabled", false);
- final Boolean property =
- getConfiguration().kafkaPublisher().enabledEvent(EventTopic.INDEX_TOPIC);
- assertThat(property).isFalse();
- }
-
- @Test
- public void shouldReturnKafkaTopicEnabledForStreamEventTopic() {
- setKafkaTopicEnabled("streamEventEnabled", false);
- final Boolean property =
- getConfiguration().kafkaPublisher().enabledEvent(EventTopic.STREAM_EVENT_TOPIC);
- assertThat(property).isFalse();
- }
-
- @Test
- public void shouldReturnKafkaTopicEnabledForProjectListEventTopic() {
- setKafkaTopicEnabled("projectListEventEnabled", false);
- final Boolean property =
- getConfiguration().kafkaPublisher().enabledEvent(EventTopic.PROJECT_LIST_TOPIC);
- assertThat(property).isFalse();
- }
-
- private void setKafkaTopicAlias(String topicKey, String topic) {
- globalPluginConfig.setString(KAFKA_SECTION, null, topicKey, topic);
- }
-
- private void setKafkaTopicEnabled(String topicEnabledKey, Boolean isEnabled) {
- globalPluginConfig.setBoolean(
- KAFKA_SECTION, KAFKA_PUBLISHER_SUBSECTION, topicEnabledKey, isEnabled);
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializerTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializerTest.java
deleted file mode 100644
index 239b586..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/kafka/consumer/KafkaEventDeserializerTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.kafka.consumer;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.gson.Gson;
-import com.googlesource.gerrit.plugins.multisite.broker.GsonProvider;
-import com.googlesource.gerrit.plugins.multisite.consumer.SourceAwareEventWrapper;
-import java.util.UUID;
-import org.junit.Before;
-import org.junit.Test;
-
-public class KafkaEventDeserializerTest {
- private KafkaEventDeserializer deserializer;
-
- @Before
- public void setUp() {
- final Gson gson = new GsonProvider().get();
- deserializer = new KafkaEventDeserializer(gson);
- }
-
- @Test
- public void kafkaEventDeserializerShouldParseAKafkaEvent() {
- final UUID eventId = UUID.randomUUID();
- final String eventType = "event-type";
- final UUID sourceInstanceId = UUID.randomUUID();
- final long eventCreatedOn = 10L;
- final String eventJson =
- String.format(
- "{ "
- + "\"header\": { \"eventId\": \"%s\", \"eventType\": \"%s\", \"sourceInstanceId\": \"%s\", \"eventCreatedOn\": %d },"
- + "\"body\": {}"
- + "}",
- eventId, eventType, sourceInstanceId, eventCreatedOn);
- final SourceAwareEventWrapper event =
- deserializer.deserialize("ignored", eventJson.getBytes(UTF_8));
-
- assertThat(event.getBody().entrySet()).isEmpty();
- assertThat(event.getHeader().getEventId()).isEqualTo(eventId);
- assertThat(event.getHeader().getEventType()).isEqualTo(eventType);
- assertThat(event.getHeader().getSourceInstanceId()).isEqualTo(sourceInstanceId);
- assertThat(event.getHeader().getEventCreatedOn()).isEqualTo(eventCreatedOn);
- }
-
- @Test(expected = RuntimeException.class)
- public void kafkaEventDeserializerShouldFailForInvalidJson() {
- deserializer.deserialize("ignored", "this is not a JSON string".getBytes(UTF_8));
- }
-
- @Test(expected = RuntimeException.class)
- public void kafkaEventDeserializerShouldFailForInvalidObjectButValidJSON() {
- deserializer.deserialize("ignored", "{}".getBytes(UTF_8));
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidatorTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidatorTest.java
index 5a68d2f..0960cf1 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidatorTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/BatchRefUpdateValidatorTest.java
@@ -14,27 +14,31 @@
package com.googlesource.gerrit.plugins.multisite.validation;
-import static junit.framework.TestCase.assertFalse;
+import static com.google.common.truth.Truth.assertThat;
import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
-import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.metrics.DisabledMetricMaker;
+import com.google.gerrit.reviewdb.client.Project;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZkSharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZookeeperTestContainerSupport;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
-import org.apache.curator.retry.RetryNTimes;
import org.eclipse.jgit.internal.storage.file.RefDirectory;
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.BatchRefUpdate;
import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
@@ -43,7 +47,11 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+@RunWith(MockitoJUnitRunner.class)
public class BatchRefUpdateValidatorTest extends LocalDiskRepositoryTestCase implements RefFixture {
@Rule public TestName nameRule = new TestName();
@@ -53,15 +61,15 @@
private RevCommit A;
private RevCommit B;
- ZookeeperTestContainerSupport zookeeperContainer;
- SharedRefDatabaseWrapper zkSharedRefDatabase;
+ @Mock SharedRefDatabaseWrapper sharedRefDatabase;
+
+ @Mock SharedRefEnforcement tmpRefEnforcement;
@Before
public void setup() throws Exception {
super.setUp();
gitRepoSetup();
- zookeeperAndPolicyEnforcementSetup();
}
private void gitRepoSetup() throws Exception {
@@ -72,24 +80,6 @@
B = repo.commit(repo.getRevWalk().parseCommit(A));
}
- private void zookeeperAndPolicyEnforcementSetup() {
- zookeeperContainer = new ZookeeperTestContainerSupport(false);
- int SLEEP_BETWEEN_RETRIES_MS = 30;
- long TRANSACTION_LOCK_TIMEOUT = 1000l;
- int NUMBER_OF_RETRIES = 5;
-
- zkSharedRefDatabase =
- new SharedRefDatabaseWrapper(
- DynamicItem.itemOf(
- SharedRefDatabase.class,
- new ZkSharedRefDatabase(
- zookeeperContainer.getCurator(),
- new ZkConnectionConfig(
- new RetryNTimes(NUMBER_OF_RETRIES, SLEEP_BETWEEN_RETRIES_MS),
- TRANSACTION_LOCK_TIMEOUT))),
- new DisabledSharedRefLogger());
- }
-
@Test
public void immutableChangeShouldNotBeWrittenIntoZk() throws Exception {
String AN_IMMUTABLE_REF = "refs/changes/01/1/1";
@@ -102,7 +92,8 @@
BatchRefUpdateValidator.executeBatchUpdateWithValidation(
batchRefUpdate, () -> execute(batchRefUpdate));
- assertFalse(zkSharedRefDatabase.exists(A_TEST_PROJECT_NAME, AN_IMMUTABLE_REF));
+ verify(sharedRefDatabase, never())
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
}
@Test
@@ -116,7 +107,34 @@
BatchRefUpdateValidator.executeBatchUpdateWithValidation(
batchRefUpdate, () -> execute(batchRefUpdate));
- assertFalse(zkSharedRefDatabase.exists(A_TEST_PROJECT_NAME, DRAFT_COMMENT));
+ verify(sharedRefDatabase, never())
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, newRef(DRAFT_COMMENT, A.getId()), B.getId());
+ }
+
+ @Test
+ public void validationShouldFailWhenLocalRefDbIsOutOfSync() throws Exception {
+ String AN_OUT_OF_SYNC_REF = "refs/changes/01/1/1";
+ BatchRefUpdate batchRefUpdate =
+ newBatchUpdate(
+ Collections.singletonList(new ReceiveCommand(A, B, AN_OUT_OF_SYNC_REF, UPDATE)));
+ BatchRefUpdateValidator batchRefUpdateValidator =
+ getRefValidatorForEnforcement(A_TEST_PROJECT_NAME, tmpRefEnforcement);
+
+ doReturn(SharedRefEnforcement.EnforcePolicy.REQUIRED)
+ .when(batchRefUpdateValidator.refEnforcement)
+ .getPolicy(A_TEST_PROJECT_NAME, AN_OUT_OF_SYNC_REF);
+ lenient()
+ .doReturn(false)
+ .when(sharedRefDatabase)
+ .isUpToDate(A_TEST_PROJECT_NAME_KEY, newRef(AN_OUT_OF_SYNC_REF, AN_OBJECT_ID_1));
+
+ batchRefUpdateValidator.executeBatchUpdateWithValidation(
+ batchRefUpdate, () -> execute(batchRefUpdate));
+
+ final List<ReceiveCommand> commands = batchRefUpdate.getCommands();
+ assertThat(commands.size()).isEqualTo(1);
+ commands.forEach(
+ (command) -> assertThat(command.getResult()).isEqualTo(ReceiveCommand.Result.LOCK_FAILURE));
}
private BatchRefUpdateValidator newDefaultValidator(String projectName) {
@@ -126,7 +144,7 @@
private BatchRefUpdateValidator getRefValidatorForEnforcement(
String projectName, SharedRefEnforcement sharedRefEnforcement) {
return new BatchRefUpdateValidator(
- zkSharedRefDatabase,
+ sharedRefDatabase,
new ValidationMetrics(new DisabledMetricMaker()),
sharedRefEnforcement,
new DummyLockWrapper(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
index 37d5008..047a1c5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
@@ -33,4 +33,7 @@
@Override
public void logLockRelease(String project, String refName) {}
+
+ @Override
+ public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/Log4jSharedRefLoggerTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/Log4jSharedRefLoggerTest.java
index f368d41..c6f0fc1 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/Log4jSharedRefLoggerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/Log4jSharedRefLoggerTest.java
@@ -18,10 +18,10 @@
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit;
+import com.google.gerrit.json.OutputFormat;
import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.server.OutputFormat;
-import com.google.gerrit.server.Sequences;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.notedb.Sequences;
import com.google.gerrit.server.util.SystemLog;
import com.google.gson.Gson;
import com.googlesource.gerrit.plugins.multisite.Log4jSharedRefLogger;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
index 730e558..187f686 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
@@ -15,17 +15,18 @@
package com.googlesource.gerrit.plugins.multisite.validation;
import static java.util.Arrays.asList;
-import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import java.io.IOException;
import java.util.Collections;
import org.eclipse.jgit.lib.BatchRefUpdate;
@@ -50,6 +51,7 @@
@Mock SharedRefDatabaseWrapper sharedRefDb;
@Mock BatchRefUpdate batchRefUpdate;
+ @Mock BatchRefUpdateValidator batchRefUpdateValidator;
@Mock RefDatabase refDatabase;
@Mock RevWalk revWalk;
@Mock ProgressMonitor progressMonitor;
@@ -110,30 +112,38 @@
setMockRequiredReturnValues();
// When compareAndPut against sharedDb succeeds
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
doReturn(true)
.when(sharedRefDb)
- .compareAndPut(eq(A_TEST_PROJECT_NAME), refEquals(oldRef), eq(newRef.getObjectId()));
+ .compareAndPut(eq(A_TEST_PROJECT_NAME_KEY), refEquals(oldRef), eq(newRef.getObjectId()));
multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
verify(sharedRefDb)
- .compareAndPut(eq(A_TEST_PROJECT_NAME), refEquals(oldRef), eq(newRef.getObjectId()));
+ .compareAndPut(eq(A_TEST_PROJECT_NAME_KEY), refEquals(oldRef), eq(newRef.getObjectId()));
}
private Ref refEquals(Ref oldRef) {
return argThat(new RefMatcher(oldRef));
}
- @Test
+ @Test(expected = IOException.class)
public void executeAndFailsWithExceptions() throws IOException {
+ multiSiteRefUpdate = getMultiSiteBatchRefUpdateWithMockedValidator();
+ doThrow(new IOException("IO Test Exception"))
+ .when(batchRefUpdateValidator)
+ .executeBatchUpdateWithValidation(any(), any());
+
+ multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
+ }
+
+ @Test
+ public void executeSuccessfullyWithNoExceptionsWhenOutOfSync() throws IOException {
setMockRequiredReturnValues();
- doReturn(true).when(sharedRefDb).exists(A_TEST_PROJECT_NAME, A_TEST_REF_NAME);
- doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
- try {
- multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
- fail("Expecting an IOException to be thrown");
- } catch (IOException e) {
- verify(validationMetrics).incrementSplitBrainPrevention();
- }
+ doReturn(true).when(sharedRefDb).exists(A_TEST_PROJECT_NAME_KEY, A_TEST_REF_NAME);
+ doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
+
+ multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
+
+ verify(validationMetrics).incrementSplitBrainPrevention();
}
@Test
@@ -163,6 +173,17 @@
return new MultiSiteBatchRefUpdate(batchRefValidatorFactory, A_TEST_PROJECT_NAME, refDatabase);
}
+ private MultiSiteBatchRefUpdate getMultiSiteBatchRefUpdateWithMockedValidator() {
+ BatchRefUpdateValidator.Factory batchRefValidatorFactory =
+ new BatchRefUpdateValidator.Factory() {
+ @Override
+ public BatchRefUpdateValidator create(String projectName, RefDatabase refDb) {
+ return batchRefUpdateValidator;
+ }
+ };
+ return new MultiSiteBatchRefUpdate(batchRefValidatorFactory, A_TEST_PROJECT_NAME, refDatabase);
+ }
+
protected static class RefMatcher implements ArgumentMatcher<Ref> {
private Ref left;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteGitRepositoryManagerTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteGitRepositoryManagerTest.java
index 82476a1..491ced4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteGitRepositoryManagerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteGitRepositoryManagerTest.java
@@ -18,7 +18,7 @@
import static org.mockito.Mockito.verify;
import com.google.gerrit.server.git.LocalDiskRepositoryManager;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import org.eclipse.jgit.lib.Repository;
import org.junit.Before;
import org.junit.Test;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabaseTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabaseTest.java
index b41378b..41b83e5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabaseTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefDatabaseTest.java
@@ -17,7 +17,7 @@
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.verify;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
import org.junit.Rule;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
index a9d8e76..16cda4e 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
@@ -24,8 +24,8 @@
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.RefUpdateValidator.Factory;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefUpdateStub;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefUpdateStub;
import java.io.IOException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
@@ -65,10 +65,10 @@
@Test
public void newUpdateShouldValidateAndSucceed() throws Exception {
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
doReturn(true)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, oldRef, newRef.getObjectId());
RefUpdate refUpdate = RefUpdateStub.forSuccessfulUpdate(oldRef, newRef.getObjectId());
@@ -82,7 +82,7 @@
@Test(expected = Exception.class)
public void newUpdateShouldValidateAndFailWithIOException() throws Exception {
- doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
RefUpdate refUpdate = RefUpdateStub.forSuccessfulUpdate(oldRef, newRef.getObjectId());
@@ -94,7 +94,7 @@
@Test
public void newUpdateShouldIncreaseRefUpdateFailureCountWhenFailing() throws IOException {
- doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
RefUpdate refUpdate = RefUpdateStub.forSuccessfulUpdate(oldRef, newRef.getObjectId());
@@ -113,10 +113,10 @@
public void newUpdateShouldNotIncreaseSplitBrainPreventedCounterIfFailingSharedDbPostUpdate()
throws IOException {
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
doReturn(false)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, oldRef, newRef.getObjectId());
RefUpdate refUpdate = RefUpdateStub.forSuccessfulUpdate(oldRef, newRef.getObjectId());
@@ -135,10 +135,10 @@
public void newUpdateShouldtIncreaseSplitBrainCounterIfFailingSharedDbPostUpdate()
throws IOException {
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
doReturn(false)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, oldRef, newRef.getObjectId());
RefUpdate refUpdate = RefUpdateStub.forSuccessfulUpdate(oldRef, newRef.getObjectId());
@@ -155,9 +155,11 @@
@Test
public void deleteShouldValidateAndSucceed() throws IOException {
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
- doReturn(true).when(sharedRefDb).compareAndPut(A_TEST_PROJECT_NAME, oldRef, ObjectId.zeroId());
+ doReturn(true)
+ .when(sharedRefDb)
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, oldRef, ObjectId.zeroId());
RefUpdate refUpdate = RefUpdateStub.forSuccessfulDelete(oldRef);
@@ -171,7 +173,7 @@
@Test
public void deleteShouldIncreaseRefUpdateFailureCountWhenFailing() throws IOException {
- doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, oldRef);
+ doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, oldRef);
RefUpdate refUpdate = RefUpdateStub.forSuccessfulDelete(oldRef);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRepositoryTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRepositoryTest.java
index eb05b30..15c596f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRepositoryTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRepositoryTest.java
@@ -18,7 +18,7 @@
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.verify;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import java.io.IOException;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate.Result;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanupTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanupTest.java
new file mode 100644
index 0000000..ba381a4
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectDeletedSharedDbCleanupTest.java
@@ -0,0 +1,45 @@
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import com.google.gerrit.extensions.api.changes.NotifyHandling;
+import com.google.gerrit.extensions.events.ProjectDeletedListener;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProjectDeletedSharedDbCleanupTest implements RefFixture {
+ @Rule public TestName nameRule = new TestName();
+
+ @Mock ValidationMetrics mockValidationMetrics;
+ @Mock SharedRefDatabaseWrapper sharedRefDatabase;
+
+ @Test
+ public void aDeleteProjectEventShouldCleanupProjectFromZk() throws Exception {
+ String projectName = A_TEST_PROJECT_NAME;
+ ProjectDeletedSharedDbCleanup projectDeletedSharedDbCleanup =
+ new ProjectDeletedSharedDbCleanup(sharedRefDatabase, mockValidationMetrics);
+
+ ProjectDeletedListener.Event event =
+ new ProjectDeletedListener.Event() {
+ @Override
+ public String getProjectName() {
+ return projectName;
+ }
+
+ @Override
+ public NotifyHandling getNotify() {
+ return NotifyHandling.NONE;
+ }
+ };
+
+ projectDeletedSharedDbCleanup.onProjectDeleted(event);
+
+ Mockito.verify(sharedRefDatabase, Mockito.times(1)).remove(A_TEST_PROJECT_NAME_KEY);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
new file mode 100644
index 0000000..6360227
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
@@ -0,0 +1,267 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF;
+import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.client.RefNames;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.google.gerrit.server.project.ProjectConfig;
+import com.google.gerrit.testing.InMemoryRepositoryManager;
+import com.google.gerrit.testing.InMemoryTestEnvironment;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProjectVersionRefUpdateTest implements RefFixture {
+
+ @Rule public InMemoryTestEnvironment testEnvironment = new InMemoryTestEnvironment();
+
+ @Mock RefUpdatedEvent refUpdatedEvent;
+ @Mock SharedRefDatabaseWrapper sharedRefDb;
+ @Mock GitReferenceUpdated gitReferenceUpdated;
+ @Mock ProjectVersionLogger verLogger;
+
+ @Inject private ProjectConfig.Factory projectConfigFactory;
+ @Inject private InMemoryRepositoryManager repoManager;
+
+ private TestRepository<InMemoryRepository> repo;
+ private ProjectConfig project;
+ private RevCommit masterCommit;
+
+ @Before
+ public void setUp() throws Exception {
+ InMemoryRepository inMemoryRepo = repoManager.createRepository(A_TEST_PROJECT_NAME_KEY);
+ project = projectConfigFactory.create(A_TEST_PROJECT_NAME_KEY);
+ project.load(inMemoryRepo);
+ repo = new TestRepository<>(inMemoryRepo);
+ masterCommit = repo.branch("master").commit().create();
+ }
+
+ @After
+ public void tearDown() {
+ Context.unsetForwardedEvent();
+ }
+
+ @Test
+ public void producerShouldUpdateProjectVersionUponRefUpdatedEvent() throws IOException {
+ Context.setForwardedEvent(false);
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63"));
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1)));
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldUpdateProjectVersionUponForcedPushRefUpdatedEvent() throws Exception {
+ Context.setForwardedEvent(false);
+
+ Thread.sleep(1000L);
+ RevCommit masterPlusOneCommit = repo.branch("master").commit().create();
+
+ Thread.sleep(1000L);
+ repo.branch("master").update(masterCommit);
+
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63"));
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1)));
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterPlusOneCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldCreateNewProjectVersionWhenMissingUponRefUpdatedEvent()
+ throws IOException {
+ Context.setForwardedEvent(false);
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF,
+ String.class))
+ .thenReturn(Optional.empty());
+ when(sharedRefDb.get(
+ A_TEST_PROJECT_NAME_KEY,
+ ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF,
+ String.class))
+ .thenReturn(Optional.empty());
+
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any()))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), isNull(), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ long storedVersion =
+ Long.parseLong(IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name()));
+ assertThat(storedVersion).isGreaterThan((long) masterCommit.getCommitTime());
+
+ verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0);
+ }
+
+ @Test
+ public void producerShouldNotUpdateProjectVersionUponSequenceRefUpdatedEvent() throws Exception {
+ producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(RefNames.REFS_SEQUENCES);
+ }
+
+ @Test
+ public void producerShouldNotUpdateProjectVersionUponStarredChangesRefUpdatedEvent()
+ throws Exception {
+ producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(RefNames.REFS_STARRED_CHANGES);
+ }
+
+ private void producerShouldNotUpdateProjectVersionUponMagicRefUpdatedEvent(String magicRefPrefix)
+ throws Exception {
+ String magicRefName = magicRefPrefix + "/foo";
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(magicRefName);
+ repo.branch(magicRefName).commit().create();
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+
+ verifyZeroInteractions(verLogger);
+ }
+
+ @Test
+ public void shouldNotUpdateProjectVersionWhenProjectDoesntExist() throws IOException {
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey())
+ .thenReturn(new Project.NameKey("aNonExistentProject"));
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+
+ verifyZeroInteractions(verLogger);
+ }
+
+ @Test
+ public void getRemoteProjectVersionShouldReturnCorrectValue() {
+ when(sharedRefDb.get(A_TEST_PROJECT_NAME_KEY, MULTI_SITE_VERSIONING_VALUE_REF, String.class))
+ .thenReturn(Optional.of("123"));
+
+ Optional<Long> version =
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb, gitReferenceUpdated, verLogger)
+ .getProjectRemoteVersion(A_TEST_PROJECT_NAME);
+
+ assertThat(version.isPresent()).isTrue();
+ assertThat(version.get()).isEqualTo(123L);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
index f0677b8..a9968c4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidatorTest.java
@@ -16,20 +16,17 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
+import com.google.gerrit.reviewdb.client.Project;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.OutOfSyncException;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedDbSplitBrainException;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
@@ -86,15 +83,18 @@
@Test
public void validationShouldSucceedWhenLocalRefDbIsUpToDate() throws Exception {
- lenient().doReturn(false).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, localRef);
lenient()
.doReturn(false)
.when(sharedRefDb)
- .compareAndPut(anyString(), any(Ref.class), any(ObjectId.class));
+ .isUpToDate(any(Project.NameKey.class), any(Ref.class));
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, localRef);
+ lenient()
+ .doReturn(false)
+ .when(sharedRefDb)
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
doReturn(true)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, localRef, newUpdateRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, localRef, newUpdateRef.getObjectId());
Result result = refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
@@ -104,14 +104,14 @@
@Test
public void sharedRefDbShouldBeUpdatedWithRefDeleted() throws Exception {
doReturn(ObjectId.zeroId()).when(refUpdate).getNewObjectId();
- doReturn(true).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
+ doReturn(true).when(sharedRefDb).isUpToDate(any(Project.NameKey.class), any(Ref.class));
lenient()
.doReturn(false)
.when(sharedRefDb)
- .compareAndPut(anyString(), any(Ref.class), any(ObjectId.class));
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
doReturn(true)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, localRef, ObjectId.zeroId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, localRef, ObjectId.zeroId());
doReturn(localRef).doReturn(null).when(localRefDb).getRef(refName);
Result result = refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.FORCED);
@@ -121,16 +121,16 @@
@Test
public void sharedRefDbShouldBeUpdatedWithNewRefCreated() throws Exception {
- Ref localNullRef = SharedRefDatabase.nullRef(refName);
+ Ref localNullRef = nullRef(refName);
- doReturn(true).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
+ doReturn(true).when(sharedRefDb).isUpToDate(any(Project.NameKey.class), any(Ref.class));
lenient()
.doReturn(false)
.when(sharedRefDb)
- .compareAndPut(anyString(), any(Ref.class), any(ObjectId.class));
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
doReturn(true)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, localNullRef, newUpdateRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, localNullRef, newUpdateRef.getObjectId());
doReturn(localNullRef).doReturn(newUpdateRef).when(localRefDb).getRef(refName);
Result result = refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
@@ -138,44 +138,52 @@
assertThat(result).isEqualTo(RefUpdate.Result.NEW);
}
- @Test(expected = OutOfSyncException.class)
- public void validationShouldFailWhenLocalRefDbIsNotUpToDate() throws Exception {
- lenient().doReturn(true).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
- doReturn(true).when(sharedRefDb).exists(A_TEST_PROJECT_NAME, refName);
- doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, localRef);
+ @Test
+ public void validationShouldFailWhenLocalRefDbIsOutOfSync() throws Exception {
+ lenient()
+ .doReturn(true)
+ .when(sharedRefDb)
+ .isUpToDate(any(Project.NameKey.class), any(Ref.class));
+ doReturn(true).when(sharedRefDb).exists(A_TEST_PROJECT_NAME_KEY, refName);
+ doReturn(false).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, localRef);
- refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
+ Result result = refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
+
+ assertThat(result).isEqualTo(Result.LOCK_FAILURE);
}
@Test(expected = SharedDbSplitBrainException.class)
public void shouldTrowSplitBrainWhenLocalRefDbIsUpToDateButFinalCompareAndPutIsFailing()
throws Exception {
- lenient().doReturn(false).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, localRef);
+ lenient()
+ .doReturn(false)
+ .when(sharedRefDb)
+ .isUpToDate(any(Project.NameKey.class), any(Ref.class));
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, localRef);
lenient()
.doReturn(true)
.when(sharedRefDb)
- .compareAndPut(anyString(), any(Ref.class), any(ObjectId.class));
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
doReturn(false)
.when(sharedRefDb)
- .compareAndPut(A_TEST_PROJECT_NAME, localRef, newUpdateRef.getObjectId());
+ .compareAndPut(A_TEST_PROJECT_NAME_KEY, localRef, newUpdateRef.getObjectId());
refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.NEW);
}
@Test
public void shouldNotUpdateSharedRefDbWhenFinalCompareAndPutIsFailing() throws Exception {
- lenient().doReturn(false).when(sharedRefDb).isUpToDate(anyString(), any(Ref.class));
- doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME, localRef);
+ lenient()
+ .doReturn(false)
+ .when(sharedRefDb)
+ .isUpToDate(any(Project.NameKey.class), any(Ref.class));
+ doReturn(true).when(sharedRefDb).isUpToDate(A_TEST_PROJECT_NAME_KEY, localRef);
Result result =
refUpdateValidator.executeRefUpdate(refUpdate, () -> RefUpdate.Result.LOCK_FAILURE);
- verify(sharedRefDb, never()).compareAndPut(anyString(), any(Ref.class), any(ObjectId.class));
+ verify(sharedRefDb, never())
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
assertThat(result).isEqualTo(RefUpdate.Result.LOCK_FAILURE);
}
-
- private Ref newRef(String refName, ObjectId objectId) {
- return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId);
- }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProjectTest.java
similarity index 94%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProjectTest.java
index d63436c..4ee3c85 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProjectTest.java
@@ -12,15 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
import static com.google.common.truth.Truth.assertThat;
-import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase.newRef;
import com.googlesource.gerrit.plugins.multisite.Configuration;
import com.googlesource.gerrit.plugins.multisite.Configuration.SharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
import java.util.Arrays;
import org.eclipse.jgit.lib.Config;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/DefaultSharedRefEnforcementTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/DefaultSharedRefEnforcementTest.java
similarity index 89%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/DefaultSharedRefEnforcementTest.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/DefaultSharedRefEnforcementTest.java
index 83fcf52..b462408 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/DefaultSharedRefEnforcementTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/DefaultSharedRefEnforcementTest.java
@@ -12,13 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
import static com.google.common.truth.Truth.assertThat;
-import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase.newRef;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
import org.eclipse.jgit.lib.Ref;
import org.junit.Test;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
index d6b92e4..fb8eb40 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/MultisiteReplicationPushFilterTest.java
@@ -21,19 +21,22 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException;
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.testing.InMemoryRepositoryManager;
import com.google.gerrit.testing.InMemoryTestEnvironment;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
import com.googlesource.gerrit.plugins.multisite.validation.DisabledSharedRefLogger;
import com.googlesource.gerrit.plugins.multisite.validation.MultisiteReplicationPushFilter;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
-import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
@@ -60,6 +63,7 @@
@Inject private InMemoryRepositoryManager gitRepositoryManager;
String project = A_TEST_PROJECT_NAME;
+ Project.NameKey projectName = A_TEST_PROJECT_NAME_KEY;
private TestRepository<InMemoryRepository> repo;
@@ -74,7 +78,7 @@
public void shouldReturnAllRefUpdatesWhenAllUpToDate() throws Exception {
List<RemoteRefUpdate> refUpdates =
Arrays.asList(refUpdate("refs/heads/foo"), refUpdate("refs/heads/bar"));
- doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(project), any());
+ doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
MultisiteReplicationPushFilter pushFilter =
new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
@@ -101,7 +105,7 @@
public void shouldLoadLocalVersionAndNotFilter() throws Exception {
RemoteRefUpdate temporaryOutdated = refUpdate("refs/heads/temporaryOutdated");
List<RemoteRefUpdate> refUpdates = Collections.singletonList(temporaryOutdated);
- doReturn(false).doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(project), any());
+ doReturn(false).doReturn(true).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
MultisiteReplicationPushFilter pushFilter =
new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
@@ -116,7 +120,7 @@
RemoteRefUpdate temporaryOutdated = refUpdate("refs/heads/temporaryOutdated");
repo.branch("refs/heads/temporaryOutdated").commit().create();
List<RemoteRefUpdate> refUpdates = Collections.singletonList(temporaryOutdated);
- doReturn(false).doReturn(false).when(sharedRefDatabaseMock).isUpToDate(eq(project), any());
+ doReturn(false).doReturn(false).when(sharedRefDatabaseMock).isUpToDate(eq(projectName), any());
MultisiteReplicationPushFilter pushFilter =
new MultisiteReplicationPushFilter(sharedRefDatabaseMock, gitRepositoryManager);
@@ -147,35 +151,50 @@
Set<String> rejectedSet = new HashSet<>();
rejectedSet.addAll(Arrays.asList(rejectedRefs));
- SharedRefDatabase sharedRefDatabase =
- new SharedRefDatabase() {
+ GlobalRefDatabase sharedRefDatabase =
+ new GlobalRefDatabase() {
@Override
- public void removeProject(String project) throws IOException {}
-
- @Override
- public AutoCloseable lockRef(String project, String refName) throws SharedLockException {
- return null;
- }
-
- @Override
- public boolean isUpToDate(String project, Ref ref) throws SharedLockException {
+ public boolean isUpToDate(Project.NameKey project, Ref ref)
+ throws GlobalRefDbLockException {
return !rejectedSet.contains(ref.getName());
}
@Override
- public boolean exists(String project, String refName) {
+ public boolean exists(Project.NameKey project, String refName) {
return true;
}
@Override
- public boolean compareAndPut(String project, Ref currRef, ObjectId newRefValue)
- throws IOException {
+ public boolean compareAndPut(Project.NameKey project, Ref currRef, ObjectId newRefValue)
+ throws GlobalRefDbSystemError {
return false;
}
+
+ @Override
+ public <T> boolean compareAndPut(
+ Project.NameKey project, String refName, T currValue, T newValue)
+ throws GlobalRefDbSystemError {
+ return false;
+ }
+
+ @Override
+ public AutoCloseable lockRef(Project.NameKey project, String refName)
+ throws GlobalRefDbLockException {
+ return null;
+ }
+
+ @Override
+ public void remove(Project.NameKey project) throws GlobalRefDbSystemError {}
+
+ @Override
+ public <T> Optional<T> get(Project.NameKey project, String refName, Class<T> clazz)
+ throws GlobalRefDbSystemError {
+ return Optional.empty();
+ }
};
return new SharedRefDatabaseWrapper(
- DynamicItem.itemOf(SharedRefDatabase.class, sharedRefDatabase),
+ DynamicItem.itemOf(GlobalRefDatabase.class, sharedRefDatabase),
new DisabledSharedRefLogger());
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabaseTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabaseTest.java
new file mode 100644
index 0000000..c63530a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/NoopSharedRefDatabaseTest.java
@@ -0,0 +1,28 @@
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.eclipse.jgit.lib.Ref;
+import org.junit.Test;
+
+public class NoopSharedRefDatabaseTest implements RefFixture {
+
+ private Ref sampleRef = newRef(A_TEST_REF_NAME, AN_OBJECT_ID_1);
+ private NoopSharedRefDatabase objectUnderTest = new NoopSharedRefDatabase();
+
+ @Test
+ public void isUpToDateShouldAlwaysReturnTrue() {
+ assertThat(objectUnderTest.isUpToDate(A_TEST_PROJECT_NAME_KEY, sampleRef)).isTrue();
+ }
+
+ @Test
+ public void compareAndPutShouldAlwaysReturnTrue() {
+ assertThat(objectUnderTest.compareAndPut(A_TEST_PROJECT_NAME_KEY, sampleRef, AN_OBJECT_ID_2))
+ .isTrue();
+ }
+
+ @Test
+ public void existsShouldAlwaysReturnFalse() {
+ assertThat(objectUnderTest.exists(A_TEST_PROJECT_NAME_KEY, A_TEST_REF_NAME)).isFalse();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefFixture.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefFixture.java
similarity index 84%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefFixture.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefFixture.java
index 72ea236..8ddbf5c 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefFixture.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefFixture.java
@@ -12,11 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.Ref;
import org.junit.Ignore;
@Ignore
@@ -41,4 +43,12 @@
default String testBranch() {
return "aTestBranch";
}
+
+ default Ref newRef(String refName, ObjectId objectId) {
+ return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId);
+ }
+
+ default Ref nullRef(String refName) {
+ return newRef(refName, ObjectId.zeroId());
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefSharedDatabaseTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefSharedDatabaseTest.java
index 57ab5e0..c9eeaa8 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefSharedDatabaseTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefSharedDatabaseTest.java
@@ -16,8 +16,8 @@
import static com.google.common.truth.Truth.assertThat;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Ref.Storage;
import org.junit.Rule;
@@ -38,7 +38,7 @@
ObjectId objectId = AN_OBJECT_ID_1;
String refName = aBranchRef();
- Ref aNewRef = SharedRefDatabase.newRef(refName, objectId);
+ Ref aNewRef = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId);
assertThat(aNewRef.getName()).isEqualTo(refName);
assertThat(aNewRef.getObjectId()).isEqualTo(objectId);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefUpdateStub.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefUpdateStub.java
similarity index 99%
rename from src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefUpdateStub.java
rename to src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefUpdateStub.java
index cec476e..7c1d7b4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/RefUpdateStub.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/RefUpdateStub.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
import java.io.IOException;
import org.apache.commons.lang.NotImplementedException;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseIT.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseIT.java
deleted file mode 100644
index e2bdfbc..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseIT.java
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import com.google.gerrit.acceptance.AbstractDaemonTest;
-import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.gerrit.metrics.DisabledMetricMaker;
-import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
-import com.googlesource.gerrit.plugins.multisite.validation.BatchRefUpdateValidator;
-import com.googlesource.gerrit.plugins.multisite.validation.DisabledSharedRefLogger;
-import com.googlesource.gerrit.plugins.multisite.validation.DummyLockWrapper;
-import com.googlesource.gerrit.plugins.multisite.validation.MultiSiteBatchRefUpdate;
-import com.googlesource.gerrit.plugins.multisite.validation.ValidationMetrics;
-import com.googlesource.gerrit.plugins.multisite.validation.ZkConnectionConfig;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
-import org.apache.curator.retry.RetryNTimes;
-import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
-import org.eclipse.jgit.lib.NullProgressMonitor;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.RefDatabase;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.transport.ReceiveCommand;
-import org.eclipse.jgit.transport.ReceiveCommand.Result;
-import org.eclipse.jgit.transport.ReceiveCommand.Type;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-public class ZkSharedRefDatabaseIT extends AbstractDaemonTest implements RefFixture {
- @Rule public TestName nameRule = new TestName();
-
- ZookeeperTestContainerSupport zookeeperContainer;
- SharedRefDatabaseWrapper zkSharedRefDatabase;
- SharedRefEnforcement refEnforcement;
-
- int SLEEP_BETWEEN_RETRIES_MS = 30;
- long TRANSACTION_LOCK_TIMEOUT = 1000l;
- int NUMBER_OF_RETRIES = 5;
-
- @Before
- public void setup() {
- refEnforcement = new DefaultSharedRefEnforcement();
- zookeeperContainer = new ZookeeperTestContainerSupport(false);
- zkSharedRefDatabase =
- new SharedRefDatabaseWrapper(
- DynamicItem.itemOf(
- SharedRefDatabase.class,
- new ZkSharedRefDatabase(
- zookeeperContainer.getCurator(),
- new ZkConnectionConfig(
- new RetryNTimes(NUMBER_OF_RETRIES, SLEEP_BETWEEN_RETRIES_MS),
- TRANSACTION_LOCK_TIMEOUT))),
- new DisabledSharedRefLogger());
- }
-
- @After
- public void cleanup() {
- zookeeperContainer.cleanup();
- }
-
- @Test
- public void sequenceOfGitUpdatesWithARejectionCausesZKCheckToFail() throws Exception {
- ObjectId commitObjectIdOne = commitBuilder().add("test_file.txt", "A").create().getId();
- ObjectId commitObjectIdTwo = commitBuilder().add("test_file.txt", "B").create().getId();
- ObjectId commitObjectIdThree = commitBuilder().add("test_file.txt", "A2").create().getId();
-
- ReceiveCommand aRefCreation =
- new ReceiveCommand(ObjectId.zeroId(), commitObjectIdOne, A_TEST_REF_NAME);
-
- ReceiveCommand aCommandThatWillBeRejectedByJGit =
- new ReceiveCommand(
- commitObjectIdOne, commitObjectIdTwo, A_TEST_REF_NAME, Type.UPDATE_NONFASTFORWARD);
-
- ReceiveCommand aCommandThatShouldSucceed =
- new ReceiveCommand(commitObjectIdOne, commitObjectIdThree, A_TEST_REF_NAME, Type.UPDATE);
-
- InMemoryRepository repository = testRepo.getRepository();
- try (RevWalk rw = new RevWalk(repository)) {
- newBatchRefUpdate(repository, aRefCreation).execute(rw, NullProgressMonitor.INSTANCE);
-
- // The rejection of this command should not leave the shared DB into an inconsistent state
- newBatchRefUpdate(repository, aCommandThatWillBeRejectedByJGit)
- .execute(rw, NullProgressMonitor.INSTANCE);
-
- // This command will succeed only if the previous one is not leaving any traces in the
- // shared ref DB
- newBatchRefUpdate(repository, aCommandThatShouldSucceed)
- .execute(rw, NullProgressMonitor.INSTANCE);
-
- assertThat(aRefCreation.getResult()).isEqualTo(Result.OK);
- assertThat(aCommandThatWillBeRejectedByJGit.getResult())
- .isEqualTo(Result.REJECTED_NONFASTFORWARD);
- assertThat(aCommandThatShouldSucceed.getResult()).isEqualTo(Result.OK);
- }
- }
-
- @Test
- public void aBatchWithOneFailedCommandShouldFailAllOtherCommands() throws Exception {
- ObjectId commitObjectIdOne = commitBuilder().add("test_file1.txt", "A").create().getId();
- ObjectId commitObjectIdTwo = commitBuilder().add("test_file1.txt", "B").create().getId();
- ObjectId commitObjectIdThree = commitBuilder().add("test_file2.txt", "C").create().getId();
-
- ReceiveCommand firstCommand =
- new ReceiveCommand(ObjectId.zeroId(), commitObjectIdOne, A_TEST_REF_NAME);
-
- ReceiveCommand aNonFastForwardUpdate =
- new ReceiveCommand(
- commitObjectIdOne, commitObjectIdTwo, A_TEST_REF_NAME, Type.UPDATE_NONFASTFORWARD);
-
- ReceiveCommand aNewCreate =
- new ReceiveCommand(ObjectId.zeroId(), commitObjectIdThree, "refs/for/master2");
-
- InMemoryRepository repository = testRepo.getRepository();
- try (RevWalk rw = new RevWalk(repository)) {
- newBatchRefUpdate(repository, firstCommand, aNonFastForwardUpdate, aNewCreate)
- .execute(rw, NullProgressMonitor.INSTANCE);
- }
-
- // All commands in batch failed because of the second one
- assertThat(firstCommand.getResult()).isEqualTo(Result.REJECTED_OTHER_REASON);
- assertThat(aNonFastForwardUpdate.getResult()).isEqualTo(Result.REJECTED_NONFASTFORWARD);
- assertThat(aNewCreate.getResult()).isEqualTo(Result.REJECTED_OTHER_REASON);
-
- // Zookeeper has been left untouched
- assertFalse(existsDataInZkForCommand(firstCommand));
- assertFalse(existsDataInZkForCommand(aNonFastForwardUpdate));
- assertFalse(existsDataInZkForCommand(aNewCreate));
- }
-
- @Test
- public void shouldBeSuccessfulWhenRefIsRecreated() throws Exception {
- ObjectId commitObjectIdOne = commitBuilder().add("test_file1.txt", "A").create().getId();
- ObjectId commitObjectIdTwo = commitBuilder().add("test_file1.txt", "B").create().getId();
-
- ReceiveCommand firstCommand =
- new ReceiveCommand(ObjectId.zeroId(), commitObjectIdOne, A_TEST_REF_NAME);
-
- ReceiveCommand deleteCommand =
- new ReceiveCommand(commitObjectIdOne, ObjectId.zeroId(), A_TEST_REF_NAME, Type.DELETE);
-
- ReceiveCommand secondCommand =
- new ReceiveCommand(ObjectId.zeroId(), commitObjectIdTwo, A_TEST_REF_NAME);
-
- InMemoryRepository repository = testRepo.getRepository();
- try (RevWalk rw = new RevWalk(repository)) {
-
- newBatchRefUpdate(repository, firstCommand).execute(rw, NullProgressMonitor.INSTANCE);
- newBatchRefUpdate(repository, deleteCommand).execute(rw, NullProgressMonitor.INSTANCE);
- newBatchRefUpdate(repository, secondCommand).execute(rw, NullProgressMonitor.INSTANCE);
- }
-
- assertTrue(existsDataInZkForCommand(secondCommand));
- }
-
- private boolean existsDataInZkForCommand(ReceiveCommand firstCommand) throws Exception {
- return zkSharedRefDatabase.exists(A_TEST_PROJECT_NAME, firstCommand.getRefName());
- }
-
- private MultiSiteBatchRefUpdate newBatchRefUpdate(
- Repository localGitRepo, ReceiveCommand... commands) {
-
- BatchRefUpdateValidator.Factory batchRefValidatorFactory =
- new BatchRefUpdateValidator.Factory() {
- @Override
- public BatchRefUpdateValidator create(String projectName, RefDatabase refDb) {
- return new BatchRefUpdateValidator(
- zkSharedRefDatabase,
- new ValidationMetrics(new DisabledMetricMaker()),
- new DefaultSharedRefEnforcement(),
- new DummyLockWrapper(),
- projectName,
- refDb);
- }
- };
-
- MultiSiteBatchRefUpdate result =
- new MultiSiteBatchRefUpdate(
- batchRefValidatorFactory, A_TEST_PROJECT_NAME, localGitRepo.getRefDatabase());
-
- result.setAllowNonFastForwards(false);
- for (ReceiveCommand command : commands) {
- result.addCommand(command);
- }
- return result;
- }
-
- @Override
- public String testBranch() {
- return "branch_" + nameRule.getMethodName();
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseTest.java
deleted file mode 100644
index bff41f1..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZkSharedRefDatabaseTest.java
+++ /dev/null
@@ -1,190 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.mockito.Mockito.mock;
-
-import com.google.gerrit.extensions.api.changes.NotifyHandling;
-import com.google.gerrit.extensions.events.ProjectDeletedListener;
-import com.google.gerrit.extensions.registration.DynamicItem;
-import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
-import com.googlesource.gerrit.plugins.multisite.validation.DisabledSharedRefLogger;
-import com.googlesource.gerrit.plugins.multisite.validation.ProjectDeletedSharedDbCleanup;
-import com.googlesource.gerrit.plugins.multisite.validation.ValidationMetrics;
-import com.googlesource.gerrit.plugins.multisite.validation.ZkConnectionConfig;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
-import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
-import org.apache.curator.retry.RetryNTimes;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Ref;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-public class ZkSharedRefDatabaseTest implements RefFixture {
- @Rule public TestName nameRule = new TestName();
-
- ZookeeperTestContainerSupport zookeeperContainer;
- SharedRefDatabaseWrapper zkSharedRefDatabase;
- SharedRefEnforcement refEnforcement;
-
- ValidationMetrics mockValidationMetrics;
-
- @Before
- public void setup() {
- refEnforcement = new DefaultSharedRefEnforcement();
- zookeeperContainer = new ZookeeperTestContainerSupport(false);
- int SLEEP_BETWEEN_RETRIES_MS = 30;
- long TRANSACTION_LOCK_TIMEOUT = 1000l;
- int NUMBER_OF_RETRIES = 5;
-
- zkSharedRefDatabase =
- new SharedRefDatabaseWrapper(
- DynamicItem.itemOf(
- SharedRefDatabase.class,
- new ZkSharedRefDatabase(
- zookeeperContainer.getCurator(),
- new ZkConnectionConfig(
- new RetryNTimes(NUMBER_OF_RETRIES, SLEEP_BETWEEN_RETRIES_MS),
- TRANSACTION_LOCK_TIMEOUT))),
- new DisabledSharedRefLogger());
-
- mockValidationMetrics = mock(ValidationMetrics.class);
- }
-
- @After
- public void cleanup() {
- zookeeperContainer.cleanup();
- }
-
- @Test
- public void shouldCompareAndPutSuccessfully() throws Exception {
- Ref oldRef = refOf(AN_OBJECT_ID_1);
- Ref newRef = refOf(AN_OBJECT_ID_2);
- String projectName = A_TEST_PROJECT_NAME;
-
- zookeeperContainer.createRefInZk(projectName, oldRef);
-
- assertThat(zkSharedRefDatabase.compareAndPut(projectName, oldRef, newRef.getObjectId()))
- .isTrue();
- }
-
- @Test
- public void shouldFetchLatestObjectIdInZk() throws Exception {
- Ref oldRef = refOf(AN_OBJECT_ID_1);
- Ref newRef = refOf(AN_OBJECT_ID_2);
- String projectName = A_TEST_PROJECT_NAME;
-
- zookeeperContainer.createRefInZk(projectName, oldRef);
-
- assertThat(zkSharedRefDatabase.compareAndPut(projectName, oldRef, newRef.getObjectId()))
- .isTrue();
-
- assertThat(zkSharedRefDatabase.isUpToDate(projectName, newRef)).isTrue();
- assertThat(zkSharedRefDatabase.isUpToDate(projectName, oldRef)).isFalse();
- }
-
- @Test
- public void shouldCompareAndPutWithNullOldRefSuccessfully() throws Exception {
- Ref oldRef = refOf(null);
- Ref newRef = refOf(AN_OBJECT_ID_2);
- String projectName = A_TEST_PROJECT_NAME;
-
- zookeeperContainer.createRefInZk(projectName, oldRef);
-
- assertThat(zkSharedRefDatabase.compareAndPut(projectName, oldRef, newRef.getObjectId()))
- .isTrue();
- }
-
- @Test
- public void compareAndPutShouldFailIfTheObjectionHasNotTheExpectedValue() throws Exception {
- String projectName = A_TEST_PROJECT_NAME;
-
- Ref oldRef = refOf(AN_OBJECT_ID_1);
- Ref expectedRef = refOf(AN_OBJECT_ID_2);
-
- zookeeperContainer.createRefInZk(projectName, oldRef);
-
- assertThat(zkSharedRefDatabase.compareAndPut(projectName, expectedRef, AN_OBJECT_ID_3))
- .isFalse();
- }
-
- private Ref refOf(ObjectId objectId) {
- return SharedRefDatabase.newRef(aBranchRef(), objectId);
- }
-
- @Test
- public void removeProjectShouldRemoveTheWholePathInZk() throws Exception {
- String projectName = A_TEST_PROJECT_NAME;
- Ref someRef = refOf(AN_OBJECT_ID_1);
-
- zookeeperContainer.createRefInZk(projectName, someRef);
-
- assertThat(zookeeperContainer.readRefValueFromZk(projectName, someRef))
- .isEqualTo(AN_OBJECT_ID_1);
-
- assertThat(getNumChildrenForPath("/")).isEqualTo(1);
-
- zkSharedRefDatabase.removeProject(projectName);
-
- assertThat(getNumChildrenForPath("/")).isEqualTo(0);
- }
-
- @Test
- public void aDeleteProjectEventShouldCleanupProjectFromZk() throws Exception {
- String projectName = A_TEST_PROJECT_NAME;
- Ref someRef = refOf(AN_OBJECT_ID_1);
- ProjectDeletedSharedDbCleanup projectDeletedSharedDbCleanup =
- new ProjectDeletedSharedDbCleanup(zkSharedRefDatabase, mockValidationMetrics);
-
- ProjectDeletedListener.Event event =
- new ProjectDeletedListener.Event() {
- @Override
- public String getProjectName() {
- return projectName;
- }
-
- @Override
- public NotifyHandling getNotify() {
- return NotifyHandling.NONE;
- }
- };
-
- zookeeperContainer.createRefInZk(projectName, someRef);
-
- assertThat(getNumChildrenForPath("/")).isEqualTo(1);
-
- projectDeletedSharedDbCleanup.onProjectDeleted(event);
-
- assertThat(getNumChildrenForPath("/")).isEqualTo(0);
- }
-
- @Override
- public String testBranch() {
- return "branch_" + nameRule.getMethodName();
- }
-
- private int getNumChildrenForPath(String path) throws Exception {
- return zookeeperContainer
- .getCurator()
- .checkExists()
- .forPath(String.format(path))
- .getNumChildren();
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperTestContainerSupport.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperTestContainerSupport.java
deleted file mode 100644
index 3237e3a..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/ZookeeperTestContainerSupport.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
-
-import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZkSharedRefDatabase.pathFor;
-import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZkSharedRefDatabase.writeObjectId;
-
-import com.googlesource.gerrit.plugins.multisite.ZookeeperConfig;
-import org.apache.curator.framework.CuratorFramework;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Ref;
-import org.junit.Ignore;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.wait.strategy.Wait;
-
-@Ignore
-public class ZookeeperTestContainerSupport {
-
- static class ZookeeperContainer extends GenericContainer<ZookeeperContainer> {
- public static String ZOOKEEPER_VERSION = "3.4.13";
-
- public ZookeeperContainer() {
- super("zookeeper:" + ZOOKEEPER_VERSION);
- }
- }
-
- private ZookeeperContainer container;
- private ZookeeperConfig configuration;
- private CuratorFramework curator;
-
- public CuratorFramework getCurator() {
- return curator;
- }
-
- public ZookeeperContainer getContainer() {
- return container;
- }
-
- @SuppressWarnings("resource")
- public ZookeeperTestContainerSupport(boolean migrationMode) {
- container = new ZookeeperContainer().withExposedPorts(2181).waitingFor(Wait.forListeningPort());
- container.start();
- Integer zkHostPort = container.getMappedPort(2181);
- Config sharedRefDbConfig = new Config();
- String connectString = container.getContainerIpAddress() + ":" + zkHostPort;
- sharedRefDbConfig.setBoolean("ref-database", null, "enabled", true);
- sharedRefDbConfig.setString("ref-database", "zookeeper", "connectString", connectString);
- sharedRefDbConfig.setString(
- "ref-database",
- ZookeeperConfig.SUBSECTION,
- ZookeeperConfig.KEY_CONNECT_STRING,
- connectString);
-
- configuration = new ZookeeperConfig(sharedRefDbConfig);
- this.curator = configuration.buildCurator();
- }
-
- public void cleanup() {
- this.curator.delete();
- this.container.stop();
- }
-
- public ObjectId readRefValueFromZk(String projectName, Ref ref) throws Exception {
- final byte[] bytes = curator.getData().forPath(pathFor(projectName, ref));
- return ZkSharedRefDatabase.readObjectId(bytes);
- }
-
- public void createRefInZk(String projectName, Ref ref) throws Exception {
- curator
- .create()
- .creatingParentContainersIfNeeded()
- .forPath(pathFor(projectName, ref), writeObjectId(ref.getObjectId()));
- }
-}
