Ability to publish metrics to cloudwatch for single-master recipe
Allow single-master recipe to be configured so that metrics can be sent
to cloudwatch. This leverages the metrics-reporter-cloudwatch plugin[1].
[1] https://gerrit.googlesource.com/plugins/metrics-reporter-cloudwatch/
Feature: Issue 13212
Change-Id: I68c40346b67f353a626533a2268ec01a2742f419
diff --git a/Makefile.common b/Makefile.common
index 550d9ac..7b1caf5 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -16,3 +16,26 @@
aws s3api create-bucket $(CREATE_BUCKET_PARAMS)
aws s3 cp ../common-templates/cf-gerrit-task-execution-role.yml s3://$(TEMPLATE_BUCKET_NAME)/
aws s3 cp ../common-templates/cf-gerrit-network-stack.yml s3://$(TEMPLATE_BUCKET_NAME)/
+
+set-optional-params-metrics-cloudwatch:
+ifdef METRICS_CLOUDWATCH_ENABLED
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchEnabled,ParameterValue=$(METRICS_CLOUDWATCH_ENABLED))
+endif
+ifdef METRICS_CLOUDWATCH_NAMESPACE
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchNamespace,ParameterValue=$(METRICS_CLOUDWATCH_NAMESPACE))
+endif
+ifdef METRICS_CLOUDWATCH_RATE
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchRate,ParameterValue=$(METRICS_CLOUDWATCH_RATE))
+endif
+ifdef METRICS_CLOUDWATCH_INITIAL_DELAY
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchInitialDelay,ParameterValue=$(METRICS_CLOUDWATCH_INITIAL_DELAY))
+endif
+ifdef METRICS_CLOUDWATCH_JVM_ENABLED
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchJVMEnabled,ParameterValue=$(METRICS_CLOUDWATCH_JVM_ENABLED))
+endif
+ifdef METRICS_CLOUDWATCH_DRY_RUN
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchDryRun,ParameterValue=$(METRICS_CLOUDWATCH_DRY_RUN))
+endif
+ifdef METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST
+ $(eval METRICS_CW_OPTIONAL_PARAMS := $(METRICS_CW_OPTIONAL_PARAMS) ParameterKey=MetricsCloudwatchExcludeMetrics,ParameterValue=\"$(METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST)\")
+endif
\ No newline at end of file
diff --git a/common-templates/cf-gerrit-task-execution-role.yml b/common-templates/cf-gerrit-task-execution-role.yml
index 889111e..ddb6ded 100644
--- a/common-templates/cf-gerrit-task-execution-role.yml
+++ b/common-templates/cf-gerrit-task-execution-role.yml
@@ -29,6 +29,8 @@
# Allow the ECS tasks to upload logs to CloudWatch
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
+ # Allow the ECS tasks to push metrics to CloudWatch
+ - 'cloudwatch:PutMetricData'
Resource: '*'
- PolicyName: AmazonECSTaskSecretManagerRolePolicy
PolicyDocument:
diff --git a/gerrit/Makefile b/gerrit/Makefile
index 2e08594..1362c95 100644
--- a/gerrit/Makefile
+++ b/gerrit/Makefile
@@ -29,6 +29,11 @@
-O ./plugins/healthcheck.jar \
|| { echo >&2 "Cannot download healthcheck plugin: Check internet connection. Aborting"; exit 1; }
+ @echo "Downloading Metrics Reporter Cloudwatch plugin $(GERRIT_BRANCH)"
+ wget $(GERRIT_CI)/plugin-metrics-reporter-cloudwatch-bazel-master-$(GERRIT_BRANCH)/$(LAST_BUILD)/metrics-reporter-cloudwatch/metrics-reporter-cloudwatch.jar \
+ -O ./plugins/healthcheck.jar \
+ || { echo >&2 "Cannot download metrics-reporter-cloudwatch plugin: Check internet connection. Aborting"; exit 1; }
+
gerrit-build:
cat Dockerfile | \
GERRIT_VERSION=$(GERRIT_VERSION) GERRIT_PATCH=$(GERRIT_PATCH) envsubst | \
diff --git a/gerrit/etc/gerrit.config.template b/gerrit/etc/gerrit.config.template
index 1be0d54..c283e5a 100644
--- a/gerrit/etc/gerrit.config.template
+++ b/gerrit/etc/gerrit.config.template
@@ -55,6 +55,22 @@
[receive]
enableSignedPush = false
+{% if METRICS_CLOUDWATCH_ENABLED == "true" %}
+[plugin "metrics-reporter-cloudwatch"]
+ namespace = {{ METRICS_CLOUDWATCH_NAMESPACE }}
+ rate = {{ METRICS_CLOUDWATCH_RATE }}
+ jvmMetrics = {{ METRICS_CLOUDWATCH_JVM_ENABLED }}
+ initialDelay = {{ METRICS_CLOUDWATCH_INITIAL_DELAY }}
+ dryRun = {{ METRICS_CLOUDWATCH_DRY_RUN }}
+
+{% if METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST is defined and METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST|length %}
+ {% set excludedMetricsList = METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST.split(',') %}
+ {% for excluded in excludedMetricsList %}
+ excludeMetrics = {{ excluded }}
+ {% endfor %}
+{% endif %}
+{% endif %}
+
{% if PROMETHEUS_BEARER_TOKEN is defined %}
[plugin "javamelody"]
diff --git a/gerrit/setup_gerrit.py b/gerrit/setup_gerrit.py
index 2f15659..58b13d9 100755
--- a/gerrit/setup_gerrit.py
+++ b/gerrit/setup_gerrit.py
@@ -145,7 +145,14 @@
'SMTP_USER': os.getenv('SMTP_USER'),
'SMTP_DOMAIN': os.getenv('SMTP_DOMAIN'),
'GERRIT_HEAP_LIMIT': os.getenv('GERRIT_HEAP_LIMIT'),
- 'JGIT_CACHE_SIZE': os.getenv('JGIT_CACHE_SIZE')
+ 'JGIT_CACHE_SIZE': os.getenv('JGIT_CACHE_SIZE'),
+ 'METRICS_CLOUDWATCH_ENABLED': os.getenv('METRICS_CLOUDWATCH_ENABLED'),
+ 'METRICS_CLOUDWATCH_NAMESPACE': os.getenv('METRICS_CLOUDWATCH_NAMESPACE'),
+ 'METRICS_CLOUDWATCH_RATE': os.getenv('METRICS_CLOUDWATCH_RATE'),
+ 'METRICS_CLOUDWATCH_JVM_ENABLED': os.getenv('METRICS_CLOUDWATCH_JVM_ENABLED'),
+ 'METRICS_CLOUDWATCH_INITIAL_DELAY': os.getenv('METRICS_CLOUDWATCH_INITIAL_DELAY'),
+ 'METRICS_CLOUDWATCH_DRY_RUN': os.getenv('METRICS_CLOUDWATCH_DRY_RUN'),
+ 'METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST': os.getenv('METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST')
})
f.write(template.render(config_for_template))
diff --git a/single-master/Makefile b/single-master/Makefile
index d7b6051..688c8dc 100644
--- a/single-master/Makefile
+++ b/single-master/Makefile
@@ -37,7 +37,7 @@
ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
$(CLUSTER_OPTIONAL_PARAMS)
-service:
+service: set-optional-params-metrics-cloudwatch
ifdef LOAD_BALANCER_SCHEME
$(eval SERVICE_OPTIONAL_PARAMS := $(SERVICE_OPTIONAL_PARAMS) ParameterKey=LoadBalancerScheme,ParameterValue=$(LOAD_BALANCER_SCHEME))
endif
@@ -67,7 +67,8 @@
ParameterKey=GerritCPU,ParameterValue=$(GERRIT_CPU) \
ParameterKey=GerritHeapLimit,ParameterValue=$(GERRIT_HEAP_LIMIT) \
ParameterKey=JgitCacheSize,ParameterValue=$(JGIT_CACHE_SIZE) \
- $(SERVICE_OPTIONAL_PARAMS)
+ $(SERVICE_OPTIONAL_PARAMS) \
+ $(METRICS_CW_OPTIONAL_PARAMS)
dns-routing:
$(AWS_FC_COMMAND) create-stack \
diff --git a/single-master/README.md b/single-master/README.md
index eb470db..c703d85 100644
--- a/single-master/README.md
+++ b/single-master/README.md
@@ -45,6 +45,7 @@
### Monitoring
* Standard CloudWatch monitoring metrics for each component
+* Application level CloudWatch monitoring can be enabled as described [here](#cloudwatch-monitoring)
* Prometheus and Grafana stack is not available for this recipe yet. However the work has been done for
the dual-master recipe and it could be easily adapted (you can find the relevant issue
[here](https://bugs.chromium.org/p/gerrit/issues/detail?id=13092)).
@@ -72,6 +73,27 @@
* `SERVICE_STACK_NAME`: Optional. Name of the service stack. `gerrit-service` by default.
+#### CloudWatch Monitoring
+
+Application level metrics for CloudWatch are available through the
+[metrics-reporter-cloudwatch](https://gerrit.googlesource.com/plugins/metrics-reporter-cloudwatch/)
+plugin.
+
+* `METRICS_CLOUDWATCH_ENABLED`: Optional - Boolean.
+Whether to publish metrics to CloudWatch. Default: false
+* `METRICS_CLOUDWATCH_NAMESPACE`: Optional - String.
+The CloudWatch namespace for Gerrit metrics. Default: _gerrit_
+* `METRICS_CLOUDWATCH_RATE`: Optional - String.
+The rate at which metrics should be fired to AWS. Default: _60s_
+* `METRICS_CLOUDWATCH_INITIAL_DELAY`: Optional - String.
+The time to delay the first reporting execution. Default: _0_
+* `METRICS_CLOUDWATCH_JVM_ENABLED`: Optional - Boolean.
+Publish JVM metrics. Default: _false_
+* `METRICS_CLOUDWATCH_DRY_RUN`: Optional - Boolean.
+Log.DEBUG the metrics, rather than publishing. Default: _false_
+* `METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST`: Optional. Comma-separated list.
+ Regex patterns to exclude from publishing. Default: empty string.
+
### 2 - Deploy
* Create the cluster, service and DNS routing stacks:
diff --git a/single-master/cf-service.yml b/single-master/cf-service.yml
index cc8530c..2c513ab 100644
--- a/single-master/cf-service.yml
+++ b/single-master/cf-service.yml
@@ -117,6 +117,37 @@
SMTPDomain:
Description: Domain to be used in the From field
Type: String
+ MetricsCloudwatchEnabled:
+ Description: Whether gerrit metrics should be published to cloudwatch
+ Type: String
+ Default: false
+ AllowedValues: [true, false]
+ MetricsCloudwatchNamespace:
+ Description: The CloudWatch namespace for Gerrit metrics
+ Type: String
+ Default: gerrit
+ MetricsCloudwatchRate:
+ Description: The rate at which metrics should be fired to AWS
+ Type: String
+ Default: 60s
+ MetricsCloudwatchInitialDelay:
+ Description: The time to delay the first reporting execution
+ Type: String
+ Default: 0
+ MetricsCloudwatchJVMEnabled:
+ Description: Whether JVM metrics shoiuld be published to cloudwatch
+ Type: String
+ Default: false
+ AllowedValues: [true, false]
+ MetricsCloudwatchDryRun:
+ Description: The reporter will log.DEBUG the metrics, instead of doing a real POST to CloudWatch
+ Type: String
+ Default: false
+ AllowedValues: [true, false]
+ MetricsCloudwatchExcludeMetrics:
+ Description: Comma separated list of regex patterns to exclude metrics reported to CloudWatch
+ Type: CommaDelimitedList
+ Default: ''
Resources:
Service:
@@ -176,6 +207,20 @@
Value: !Ref SMTPUser
- Name: SMTP_DOMAIN
Value: !Ref SMTPDomain
+ - Name: METRICS_CLOUDWATCH_ENABLED
+ Value: !Ref MetricsCloudwatchEnabled
+ - Name: METRICS_CLOUDWATCH_NAMESPACE
+ Value: !Ref MetricsCloudwatchNamespace
+ - Name: METRICS_CLOUDWATCH_RATE
+ Value: !Ref MetricsCloudwatchRate
+ - Name: METRICS_CLOUDWATCH_INITIAL_DELAY
+ Value: !Ref MetricsCloudwatchInitialDelay
+ - Name: METRICS_CLOUDWATCH_JVM_ENABLED
+ Value: !Ref MetricsCloudwatchJVMEnabled
+ - Name: METRICS_CLOUDWATCH_DRY_RUN
+ Value: !Ref MetricsCloudwatchDryRun
+ - Name: METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST
+ Value: !Join [',', !Ref MetricsCloudwatchExcludeMetrics]
MountPoints:
- SourceVolume: !Ref GerritGitVolume
ContainerPath: /var/gerrit/git
diff --git a/single-master/setup.env.template b/single-master/setup.env.template
index 2d15394..a058d53 100644
--- a/single-master/setup.env.template
+++ b/single-master/setup.env.template
@@ -17,4 +17,12 @@
SMTP_SERVER:=yoursmtp.yourcompany.com
SMTP_USER:=smtpuser
-SMTP_DOMAIN:=mail.yourcompany.com
\ No newline at end of file
+SMTP_DOMAIN:=mail.yourcompany.com
+
+METRICS_CLOUDWATCH_ENABLED:=true
+METRICS_CLOUDWATCH_NAMESPACE:=gerrit
+METRICS_CLOUDWATCH_RATE:=60s
+METRICS_CLOUDWATCH_INITIAL_DELAY:=0s
+METRICS_CLOUDWATCH_JVM_ENABLED:=true
+METRICS_CLOUDWATCH_DRY_RUN:=false
+METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST:=foo.*,bar.*
\ No newline at end of file
