Allow kinesis-events to be configured as events-broker
Originally kafka was the only implementation of events-broker, so it
made sense that the local setup of a multi-site environment was
hardcoding kafka as the only events-broker implementation.
With the upcoming of additional implementations however, the user should
be able to specify which flavour of broker implementation should be
installed.
Introduce the ability to install kinesis as opposed to kafka.
Bug: Issue 14274
Change-Id: I9c6be9ffeaca02116cd17759d5951581f3585336
diff --git a/setup_local_env/README.md b/setup_local_env/README.md
index 5b99996..0b1d00c 100644
--- a/setup_local_env/README.md
+++ b/setup_local_env/README.md
@@ -4,7 +4,8 @@
The environment is composed by:
- 2 gerrit instances deployed by default in /tmp
-- 1 kafka node and 1 zookeeper node
+- 1 zookeeper node
+- 1 Broker node (either kafka or kinesis)
- 1 HA-PROXY
## Requirements
@@ -14,15 +15,27 @@
- wget
- envsubst
- haproxy
+- aws-cli (only when broker_type is "kinesis")
## Examples
-Simplest setup with all default values and cleanup previous deployment
+Simplest setup with all default values and cleanup previous deployment. This
+will deploy kafka broker
```bash
sh setup_local_env/setup.sh --release-war-file /path/to/gerrit.war --multisite-lib-file /path/to/multi-site.jar
```
+Deploy Kinesis broker
+
+```bash
+sh setup_local_env/setup.sh \
+ --release-war-file /path/to/gerrit.war \
+ --multisite-lib-file /path/to/multi-site.jar \
+ --broker-type kinesis
+```
+
+
Cleanup the previous deployments
```bash
@@ -59,6 +72,8 @@
[--just-cleanup-env] Cleans up previous deployment; default false
[--enabled-https] Enabled https; default true
+
+[--broker_type] events broker type; either 'kafka' or 'kinesis'. Default 'kafka'
```
## Limitations
diff --git a/setup_local_env/configs/gerrit.config b/setup_local_env/configs/gerrit.config
index f9eca89..0f19578 100644
--- a/setup_local_env/configs/gerrit.config
+++ b/setup_local_env/configs/gerrit.config
@@ -41,13 +41,20 @@
directory = $FAKE_NFS
[plugin "kafka-events"]
sendAsync = true
- bootstrapServers = localhost:$KAFKA_PORT
- groupId = $KAFKA_GROUP_ID
+ bootstrapServers = localhost:$BROKER_PORT
+ groupId = $GROUP_ID
numberOfSubscribers = 6
securityProtocol = PLAINTEXT
pollingIntervalMs = 1000
enableAutoCommit = true
autoCommitIntervalMs = 1000
autoOffsetReset = latest
+[plugin "kinesis-events"]
+ numberOfSubscribers = 6
+ pollingIntervalMs = 1000
+ region = us-east-1
+ endpoint = http://localhost:$BROKER_PORT
+ applicationName = $GROUP_ID
+ initialPosition = trim_horizon
[plugin "metrics-reporter-prometheus"]
prometheusBearerToken = token
diff --git a/setup_local_env/docker-compose.yaml b/setup_local_env/docker-compose-core.yaml
similarity index 61%
rename from setup_local_env/docker-compose.yaml
rename to setup_local_env/docker-compose-core.yaml
index c386d46..dab168d 100644
--- a/setup_local_env/docker-compose.yaml
+++ b/setup_local_env/docker-compose-core.yaml
@@ -5,15 +5,8 @@
ports:
- "2181:2181"
container_name: zk_test_node
- kafka:
- image: wurstmeister/kafka:2.12-2.1.0
- ports:
- - "9092:9092"
- container_name: kafka_test_node
- environment:
- KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
- KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
prometheus:
+ container_name: prometheus_test_node
image: prom/prometheus:v2.16.0
user: root
volumes:
diff --git a/setup_local_env/docker-compose-kafka.yaml b/setup_local_env/docker-compose-kafka.yaml
new file mode 100644
index 0000000..8a31502
--- /dev/null
+++ b/setup_local_env/docker-compose-kafka.yaml
@@ -0,0 +1,15 @@
+version: '3'
+services:
+ kafka:
+ image: wurstmeister/kafka:2.12-2.1.0
+ ports:
+ - "9092:9092"
+ container_name: kafka_test_node
+ environment:
+ KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ networks:
+ - setup_local_env_default
+networks:
+ setup_local_env_default:
+ external: true
diff --git a/setup_local_env/docker-compose-kinesis.yaml b/setup_local_env/docker-compose-kinesis.yaml
new file mode 100644
index 0000000..321d760
--- /dev/null
+++ b/setup_local_env/docker-compose-kinesis.yaml
@@ -0,0 +1,16 @@
+version: '3'
+services:
+ kinesis:
+ image: localstack/localstack:0.12.8
+ ports:
+ - "4566:4566"
+ - "4751:4751"
+ container_name: kinesis_test_node
+ environment:
+ SERVICES: dynamodb,cloudwatch,kinesis
+ AWS_REGION: us-east-1
+ networks:
+ - setup_local_env_default
+networks:
+ setup_local_env_default:
+ external: true
\ No newline at end of file
diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh
index f25281c..5c9de6f 100755
--- a/setup_local_env/setup.sh
+++ b/setup_local_env/setup.sh
@@ -30,6 +30,9 @@
type wget >/dev/null 2>&1 || { echo >&2 "Require wget but it's not installed. Aborting."; exit 1; }
type envsubst >/dev/null 2>&1 || { echo >&2 "Require envsubst but it's not installed. Aborting."; exit 1; }
type openssl >/dev/null 2>&1 || { echo >&2 "Require openssl but it's not installed. Aborting."; exit 1; }
+ if [ "$BROKER_TYPE" = "kinesis" ]; then
+ type aws >/dev/null 2>&1 || { echo >&2 "Require aws-cli but it's not installed. Aborting."; exit 1; }
+ fi
}
function get_replication_url {
@@ -67,7 +70,7 @@
export GERRIT_HOSTNAME=$7
export REPLICATION_HOSTNAME=$8
export REMOTE_DEBUG_PORT=$9
- export KAFKA_GROUP_ID=${10}
+ export GROUP_ID=${10}
export REPLICATION_URL=$(get_replication_url $REPLICATION_LOCATION_TEST_SITE $REPLICATION_HOSTNAME)
echo "Replacing variables for file $file and copying to $CONFIG_TEST_SITE/$file_name"
@@ -99,10 +102,15 @@
haproxy -f $HA_PROXY_CONFIG_DIR/haproxy.cfg &
}
-function deploy_config_files {
- # KAFKA configuration
- export KAFKA_PORT=9092
+function export_broker_port {
+ if [ "$BROKER_TYPE" = "kinesis" ]; then
+ export BROKER_PORT=4566
+ elif [ "$BROKER_TYPE" = "kafka" ]; then
+ export BROKER_PORT=9092
+ fi
+}
+function deploy_config_files {
# ZK configuration
export ZK_PORT=2181
@@ -112,21 +120,21 @@
GERRIT_SITE1_SSHD_PORT=$3
CONFIG_TEST_SITE_1=$LOCATION_TEST_SITE_1/etc
GERRIT_SITE1_REMOTE_DEBUG_PORT="5005"
- GERRIT_SITE1_KAFKA_GROUP_ID="instance-1"
+ GERRIT_SITE1_GROUP_ID="instance-1"
# SITE 2
GERRIT_SITE2_HOSTNAME=$4
GERRIT_SITE2_HTTPD_PORT=$5
GERRIT_SITE2_SSHD_PORT=$6
CONFIG_TEST_SITE_2=$LOCATION_TEST_SITE_2/etc
GERRIT_SITE2_REMOTE_DEBUG_PORT="5006"
- GERRIT_SITE2_KAFKA_GROUP_ID="instance-2"
+ GERRIT_SITE2_GROUP_ID="instance-2"
# Set config SITE1
- copy_config_files $CONFIG_TEST_SITE_1 $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_SSHD_PORT $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE1_REMOTE_DEBUG_PORT $GERRIT_SITE1_KAFKA_GROUP_ID
+ copy_config_files $CONFIG_TEST_SITE_1 $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_SSHD_PORT $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE1_REMOTE_DEBUG_PORT $GERRIT_SITE1_GROUP_ID
# Set config SITE2
- copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT $GERRIT_SITE2_KAFKA_GROUP_ID
+ copy_config_files $CONFIG_TEST_SITE_2 $GERRIT_SITE2_HTTPD_PORT $LOCATION_TEST_SITE_2 $GERRIT_SITE2_SSHD_PORT $GERRIT_SITE1_HTTPD_PORT $LOCATION_TEST_SITE_1 $GERRIT_SITE1_HOSTNAME $GERRIT_SITE2_HOSTNAME $GERRIT_SITE2_REMOTE_DEBUG_PORT $GERRIT_SITE2_GROUP_ID
}
function is_docker_desktop {
@@ -142,13 +150,32 @@
fi
}
+function create_kinesis_streams {
+ for stream in "gerrit_batch_index" "gerrit_cache_eviction" "gerrit_index" "gerrit_list_project" "gerrit_stream" "gerrit_web_session" "gerrit"
+ do
+ create_kinesis_stream $stream
+ done
+}
+
+function create_kinesis_stream {
+ local stream=$1
+
+ echo "[KINESIS] Create stream $stream"
+ until aws --endpoint-url=http://localhost:$BROKER_PORT kinesis create-stream --shard-count 1 --stream-name "$stream"
+ do
+ echo "[KINESIS stream $stream] Creation failed. Retrying in 5 seconds..."
+ sleep 5s
+ done
+}
function cleanup_environment {
echo "Killing existing HA-PROXY setup"
kill $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}') 2> /dev/null
- echo "Stopping docker containers"
- docker-compose -f $SCRIPT_DIR/docker-compose.yaml down 2> /dev/null
+ echo "Stopping $BROKER_TYPE docker container"
+ docker-compose -f "${SCRIPT_DIR}/docker-compose-${BROKER_TYPE}.yaml" down 2> /dev/null
+ echo "Stopping core docker containers"
+ docker-compose -f "${SCRIPT_DIR}/docker-compose-core.yaml" down 2> /dev/null
echo "Stopping GERRIT instances"
$1/bin/gerrit.sh stop 2> /dev/null
@@ -158,8 +185,32 @@
rm -rf $3 2> /dev/null
}
-function check_if_kafka_is_running {
- echo $(docker inspect kafka_test_node 2> /dev/null | grep '"Running": true' | wc -l)
+function check_if_container_is_running {
+ local container=$1;
+ echo $(docker inspect "$container" 2> /dev/null | grep '"Running": true' | wc -l)
+}
+
+function ensure_docker_compose_is_up_and_running {
+ local log_label=$1
+ local container_name=$2
+ local docker_compose_file=$3
+
+ local is_container_running=$(check_if_container_is_running "$container_name")
+ if [ "$is_container_running" -lt 1 ];then
+ echo "[$log_label] Starting docker containers"
+ docker-compose -f "${SCRIPT_DIR}/${docker_compose_file}" up -d
+
+ echo "[$log_label] Waiting for docker containers to start..."
+ while [[ $(check_if_container_is_running "$container_name") -lt 1 ]];do sleep 10s; done
+ else
+ echo "[$log_label] Containers already running, nothing to do"
+ fi
+}
+
+function prepare_broker_data {
+ if [ "$BROKER_TYPE" = "kinesis" ]; then
+ create_kinesis_streams
+ fi
}
while [ $# -ne 0 ]
@@ -194,6 +245,8 @@
echo
echo "[--enabled-https] Enabled https; default true"
echo
+ echo "[--broker-type] events broker type; either 'kafka' or 'kinesis'. Default 'kafka'"
+ echo
exit 0
;;
"--new-deployment")
@@ -281,6 +334,15 @@
shift
shift
;;
+ "--broker-type" )
+ BROKER_TYPE=$2
+ shift
+ shift
+ if [ ! "$BROKER_TYPE" = "kafka" ] && [ ! "$BROKER_TYPE" = "kinesis" ]; then
+ echo >&2 "broker type: '$BROKER_TYPE' not valid. Please supply 'kafka' or 'kinesis'. Aborting"
+ exit 1
+ fi
+ ;;
* )
echo "Unknown option argument: $1"
shift
@@ -309,6 +371,7 @@
export REPLICATION_DELAY_SEC=${REPLICATION_DELAY_SEC:-"5"}
export SSH_ADVERTISED_PORT=${SSH_ADVERTISED_PORT:-"29418"}
HTTPS_ENABLED=${HTTPS_ENABLED:-"false"}
+BROKER_TYPE=${BROKER_TYPE:-"kafka"}
export COMMON_LOCATION=$DEPLOYMENT_LOCATION/gerrit_setup
LOCATION_TEST_SITE_1=$COMMON_LOCATION/instance-1
@@ -367,10 +430,19 @@
-O $DEPLOYMENT_LOCATION/events-broker.jar || { echo >&2 "Cannot download events-broker library: Check internet connection. Abort\
ing"; exit 1; }
+if [ "$BROKER_TYPE" = "kafka" ]; then
echo "Downloading kafka-events plugin $GERRIT_BRANCH"
wget $GERRIT_CI/plugin-kafka-events-bazel-$GERRIT_BRANCH/$LAST_BUILD/kafka-events/kafka-events.jar \
-O $DEPLOYMENT_LOCATION/kafka-events.jar || { echo >&2 "Cannot download kafka-events plugin: Check internet connection. Abort\
ing"; exit 1; }
+fi
+
+if [ "$BROKER_TYPE" = "kinesis" ]; then
+echo "Downloading kinesis-events plugin master (TODO: replace with $GERRIT_BRANCH, once we have build)"
+ wget $GERRIT_CI/plugin-kinesis-events-gh-bazel-master/$LAST_BUILD/kinesis-events/kinesis-events.jar \
+ -O $DEPLOYMENT_LOCATION/kinesis-events.jar || { echo >&2 "Cannot download kinesis-events plugin: Check internet connection. Abort\
+ing"; exit 1; }
+fi
echo "Downloading metrics-reporter-prometheus plugin $GERRIT_BRANCH"
wget $GERRIT_CI/plugin-metrics-reporter-prometheus-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/metrics-reporter-prometheus/metrics-reporter-prometheus.jar \
@@ -422,8 +494,8 @@
echo "Copy events broker library"
cp -f $DEPLOYMENT_LOCATION/events-broker.jar $LOCATION_TEST_SITE_1/lib/events-broker.jar
- echo "Copy kafka events plugin"
- cp -f $DEPLOYMENT_LOCATION/kafka-events.jar $LOCATION_TEST_SITE_1/plugins/kafka-events.jar
+ echo "Copy $BROKER_TYPE events plugin"
+ cp -f $DEPLOYMENT_LOCATION/$BROKER_TYPE-events.jar $LOCATION_TEST_SITE_1/plugins/$BROKER_TYPE-events.jar
echo "Copy metrics-reporter-prometheus plugin"
cp -f $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar $LOCATION_TEST_SITE_1/plugins/metrics-reporter-prometheus.jar
@@ -455,14 +527,10 @@
cat $SCRIPT_DIR/configs/prometheus.yml | envsubst > $COMMON_LOCATION/prometheus.yml
-IS_KAFKA_RUNNING=$(check_if_kafka_is_running)
-if [ $IS_KAFKA_RUNNING -lt 1 ];then
-
- echo "Starting zk and kafka"
- docker-compose -f $SCRIPT_DIR/docker-compose.yaml up -d
- echo "Waiting for kafka to start..."
- while [[ $(check_if_kafka_is_running) -lt 1 ]];do sleep 10s; done
-fi
+export_broker_port
+ensure_docker_compose_is_up_and_running "core" "prometheus_test_node" "docker-compose-core.yaml"
+ensure_docker_compose_is_up_and_running "$BROKER_TYPE" "${BROKER_TYPE}_test_node" "docker-compose-$BROKER_TYPE.yaml"
+prepare_broker_data
echo "Re-deploying configuration files"
deploy_config_files $GERRIT_1_HOSTNAME $GERRIT_1_HTTPD_PORT $GERRIT_1_SSHD_PORT $GERRIT_2_HOSTNAME $GERRIT_2_HTTPD_PORT $GERRIT_2_SSHD_PORT
