Allow to enable replication service
Introduce the ability to enable a replication service that will expose
port 9148 and 1022 via a NLB to accept git traffic. The data will be
persisted on EFS volume mounted by the gerrit masters.
This is a step forward towards modeling a dual-master recipe that
supports multi-site deployment, since it allows other sites to replicate
git data across regions via the replication service.
Feature: Issue 13287
Change-Id: Ib0ef28cc8be0b1a406480e7c1da9ab72ca16c37e
diff --git a/dual-master/Makefile b/dual-master/Makefile
index f298ed9..7a8a325 100644
--- a/dual-master/Makefile
+++ b/dual-master/Makefile
@@ -7,19 +7,26 @@
SERVICE_SLAVE_TEMPLATE:=cf-service-slave.yml
DNS_ROUTING_TEMPLATE:=cf-dns-route.yml
LOAD_BALANCER_TEMPLATE:=cf-service-lb.yml
+SERVICE_REPLICATION_TEMPLATE:=cf-service-replication.yml
AWS_FC_COMMAND=export AWS_PAGER=;aws cloudformation
.PHONY: create-all delete-all \
+ wait-for-replication-creation wait-for-service-replication-deletion service-replication delete-service-replication \
cluster cluster-keys service-master-1 service-master-2 service-slave dns-routing \
wait-for-cluster-creation wait-for-service-master-1-creation wait-for-service-master-2-creation wait-for-service-slave-creation wait-for-dns-routing-creation \
wait-for-cluster-deletion wait-for-service-master-1-deletion wait-for-service-master-2-deletion wait-for-service-slave-deletion wait-for-dns-routing-deletion \
service-lb wait-for-service-lb-deletion wait-for-service-lb-creation \
gerrit-build gerrit-publish haproxy-publish syslog-sidecar-publish
+ifeq ($(REPLICATION_SERVICE_ENABLED),true)
+optional_replication_targets=service-replication wait-for-service-replication-creation
+endif
+
create-all: upload-common-templates \
git-daemon-publish git-ssh-publish \
gerrit-publish haproxy-publish syslog-sidecar-publish \
cluster wait-for-cluster-creation \
+ $(optional_replication_targets) \
service-slave service-master-1 \
wait-for-service-master-1-creation wait-for-service-slave-creation \
service-master-2 wait-for-service-master-2-creation \
@@ -148,6 +155,22 @@
$(METRICS_CW_OPTIONAL_PARAMS) \
$(SMTP_OPTIONAL_PARAMS)
+service-replication:
+ $(AWS_FC_COMMAND) create-stack \
+ --stack-name $(SERVICE_REPLICATION_STACK_NAME) \
+ --capabilities CAPABILITY_IAM \
+ --template-body file://`pwd`/$(SERVICE_REPLICATION_TEMPLATE) \
+ --region $(AWS_REGION) \
+ --parameters \
+ ParameterKey=GitReplicationServiceName,ParameterValue=$(SERVICE_REPLICATION_STACK_NAME) \
+ ParameterKey=ClusterStackName,ParameterValue=$(CLUSTER_STACK_NAME) \
+ ParameterKey=TemplateBucketName,ParameterValue=$(TEMPLATE_BUCKET_NAME) \
+ ParameterKey=DockerRegistryUrl,ParameterValue=$(DOCKER_REGISTRY_URI) \
+ ParameterKey=DesiredCount,ParameterValue=$(SERVICE_REPLICATION_DESIRED_COUNT) \
+ ParameterKey=GerritKeyPrefix,ParameterValue=$(GERRIT_KEY_PREFIX) \
+ ParameterKey=HostedZoneName,ParameterValue=$(HOSTED_ZONE_NAME) \
+ ParameterKey=GitReplicationSubdomain,ParameterValue=$(GIT_REPLICATION_SUBDOMAIN)
+
service-slave: set-optional-params-metrics-cloudwatch
ifdef GERRIT_SLAVE_INSTANCE_ID
$(eval SLAVE_SERVICE_OPTIONAL_PARAMS := $(SLAVE_SERVICE_OPTIONAL_PARAMS) ParameterKey=InstanceId,ParameterValue=$(GERRIT_SLAVE_INSTANCE_ID))
@@ -202,6 +225,7 @@
ParameterKey=Subdomain,ParameterValue=$(LB_SUBDOMAIN) \
ParameterKey=HAProxyDockerImage,ParameterValue=aws-gerrit/haproxy:$(HAPROXY_HEAD_SHA1) \
ParameterKey=SidecarDockerImage,ParameterValue=aws-gerrit/syslog-sidecar:$(SYSLOG_HEAD_SHA1) \
+ $(REPLICATION_SERVICE_OPTIONAL_PARAMS) \
$(SERVICE_OPTIONAL_PARAMS)
dns-routing:
@@ -229,6 +253,13 @@
--region $(AWS_REGION)
@echo "*** Service stack '$(SERVICE_MASTER1_STACK_NAME)' created"
+wait-for-service-replication-creation:
+ @echo "*** Wait for service replication stack '$(SERVICE_REPLICATION_STACK_NAME)' creation"
+ $(AWS_FC_COMMAND) wait stack-create-complete \
+ --stack-name $(SERVICE_REPLICATION_STACK_NAME) \
+ --region $(AWS_REGION)
+ @echo "*** Service stack '$(SERVICE_REPLICATION_STACK_NAME)' created"
+
wait-for-service-master-2-creation:
@echo "*** Wait for service master stack '$(SERVICE_MASTER2_STACK_NAME)' creation"
$(AWS_FC_COMMAND) wait stack-create-complete \
@@ -299,6 +330,13 @@
--region $(AWS_REGION)
@echo "*** DNS routing stack '$(DNS_ROUTING_STACK_NAME)' deleted"
+wait-for-service-replication-deletion:
+ @echo "*** Wait for service replication stack '$(SERVICE_REPLICATION_STACK_NAME)' deletion"
+ $(AWS_FC_COMMAND) wait stack-delete-complete \
+ --stack-name $(SERVICE_REPLICATION_STACK_NAME) \
+ --region $(AWS_REGION)
+ @echo "*** Service stack '$(SERVICE_REPLICATION_STACK_NAME)' deleted"
+
delete-cluster:
$(AWS_FC_COMMAND) delete-stack \
--stack-name $(CLUSTER_STACK_NAME) \
@@ -324,6 +362,11 @@
--stack-name $(LOAD_BALANCER_STACK_NAME) \
--region $(AWS_REGION)
+delete-service-replication:
+ $(AWS_FC_COMMAND) delete-stack \
+ --stack-name $(SERVICE_REPLICATION_STACK_NAME) \
+ --region $(AWS_REGION)
+
delete-dns-routing:
$(AWS_FC_COMMAND) delete-stack \
--stack-name $(DNS_ROUTING_STACK_NAME) \
@@ -333,6 +376,7 @@
delete-service-lb wait-for-service-lb-deletion \
delete-service-master-1 delete-service-master-2 delete-service-slave \
wait-for-service-master-1-deletion wait-for-service-master-2-deletion wait-for-service-slave-deletion \
+ delete-service-replication wait-for-service-replication-deletion \
delete-cluster wait-for-cluster-deletion
gerrit-publish:
diff --git a/dual-master/README.md b/dual-master/README.md
index cd62d2e..3e592d0 100644
--- a/dual-master/README.md
+++ b/dual-master/README.md
@@ -12,6 +12,12 @@
* `cf-service-slave`: define the service stack running the gerrit replica
* `cf-service-lb`: define the LBs in front of gerrit masters (this includes haproxy as well as NLB)
+When the recipe enables the replication_service (see [docs](#replication-service))
+then these additional templates will be executed:
+
+* `cf-service-replication`: Define a replication stack that will allow git replication
+over the EFS volume, which is mounted by the master instances.
+
### Networking
* Single VPC:
@@ -100,6 +106,16 @@
* `GERRIT_MASTER2_INSTANCE_ID`: Optional. Identifier for the Gerrit master2 instance.
"gerrit-dual-master-MASTER2" by default.
+#### REPLICATION SERVICE
+
+* `REPLICATION_SERVICE_ENABLED`: Optional. Whether to expose a replication endpoint.
+"false" by default.
+* `SERVICE_REPLICATION_STACK_NAME`: Optional. The name of the replication service stack.
+"git-replication-service" by default.
+* `SERVICE_REPLICATION_DESIRED_COUNT`: Optional. Number of wanted replication tasks.
+"1" by default.
+* `GIT_REPLICATION_SUBDOMAIN`: Optional. The subdomain to use for the replication endpoint.
+"git-replication" by default.
### 2 - Deploy
@@ -119,6 +135,27 @@
and stored in a `pem` file on the current directory.
To use when ssh-ing into your instances as follow: `ssh -i cluster-keys.pem ec2-user@<ec2_instance_ip>`
+#### Replication-Service
+
+Optionally this recipe can be deployed so that replication onto the share EFS volume
+is available.
+
+By setting the environment variable `REPLICATION_SERVICE_ENABLED=true`, this recipe will
+set up and configure additional resources that will allow other other sites to replicate
+to a specific endpoint, exposed as:
+
+* For GIT replication
+`$(GIT_REPLICATION_SUBDOMAIN).$(HOSTED_ZONE_NAME):9148`
+
+* For Git Admin replication
+`$(GIT_REPLICATION_SUBDOMAIN).$(HOSTED_ZONE_NAME):1022`
+
+The service will persist git data on the same EFS volume mounted by the gerrit
+master1 and gerrit master2.
+
+Note that the replication endpoint is not internet-facing, thus replication requests
+must be coming from a peered VPC.
+
### Cleaning up
```
diff --git a/dual-master/cf-service-replication.yml b/dual-master/cf-service-replication.yml
new file mode 100644
index 0000000..e85baf5
--- /dev/null
+++ b/dual-master/cf-service-replication.yml
@@ -0,0 +1,232 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Service to allow replication onto EFS
+Parameters:
+ GitReplicationServiceName:
+ Description: The name of the Git replication service
+ Type: String
+ Default: git-replication-service
+ GitReplicationSubdomain:
+ Description: The subdomain of the Git replication endpoint
+ Type: String
+ Default: git-replication
+ HostedZoneName:
+ Description: The route53 HostedZoneName.
+ Type: String
+ GitAdminSSHContainerName:
+ Description: The name of the Git Admin SSH container
+ Type: String
+ Default: git-admin-ssh-container
+ GitAdminSSHContainerPort:
+ Description: Git Admin SSH Container port
+ Type: Number
+ Default: 22
+ GitAdminSSHHostPort:
+ Description: Git ssh port
+ Type: Number
+ Default: 1022
+ GitDaemonContainerName:
+ Description: The name of the Git Daemon container
+ Type: String
+ Default: git-daemon-container
+ GitDaemonContainerPort:
+ Description: Git Daemon Container port
+ Type: Number
+ Default: 9418
+ GitDaemonHostPort:
+ Description: Git Daemon Host port
+ Type: Number
+ Default: 9418
+ ClusterStackName:
+ Description: Stack name of the ECS cluster to deply the serivces
+ Type: String
+ Default: gerrit-cluster
+ TemplateBucketName:
+ Description: S3 bucket containing cloudformation templates
+ Type: String
+ EnvironmentName:
+ Description: An environment name used to build the log stream names
+ Type: String
+ Default: test
+ GitSSHDockerImage:
+ Description: Git SSH Docker image
+ Type: String
+ Default: aws-gerrit/git-ssh:latest
+ GitDaemonDockerImage:
+ Description: Git Daemon Docker image
+ Type: String
+ Default: aws-gerrit/git-daemon:latest
+ DockerRegistryUrl:
+ Description: Docker registry URL
+ Type: String
+ DesiredCount:
+ Description: How many instances of this task should we run across our cluster?
+ Type: Number
+ Default: 1
+ GerritKeyPrefix:
+ Description: Gerrit credentials keys prefix
+ Type: String
+ GerritGitVolume:
+ Description: Gerrit git volume name
+ Type: String
+ Default: gerrit-git-master
+
+Resources:
+ Service:
+ Type: AWS::ECS::Service
+ DependsOn:
+ - GitAdminSSHListener
+ - GitDaemonListener
+ Properties:
+ Cluster:
+ Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'ClusterName']]
+ DesiredCount: !Ref DesiredCount
+ TaskDefinition: !Ref TaskDefinition
+ LoadBalancers:
+ - ContainerName: !Ref GitAdminSSHContainerName
+ ContainerPort: !Ref GitAdminSSHContainerPort
+ TargetGroupArn: !Ref GitAdminSSHTargetGroup
+ - ContainerName: !Ref GitDaemonContainerName
+ ContainerPort: !Ref GitDaemonContainerPort
+ TargetGroupArn: !Ref GitDaemonTargetGroup
+ TaskDefinition:
+ Type: AWS::ECS::TaskDefinition
+ Properties:
+ Family: !Join ['', [!Ref GitReplicationServiceName, TaskDefinition]]
+ TaskRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
+ ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
+ NetworkMode: bridge
+ ContainerDefinitions:
+ - Name: !Ref GitAdminSSHContainerName
+ Essential: true
+ Image: !Sub '${DockerRegistryUrl}/${GitSSHDockerImage}'
+ Environment:
+ - Name: TZ
+ Value: US/Pacific
+ - Name: SSH_USERS
+ Value: gerrit:1000:1000
+ - Name: AWS_REGION
+ Value: !Ref AWS::Region
+ - Name: GERRIT_KEY_PREFIX
+ Value: !Ref GerritKeyPrefix
+ MountPoints:
+ - SourceVolume: !Ref GerritGitVolume
+ ContainerPath: /var/gerrit/git
+ Cpu: 256
+ Memory: 512
+ PortMappings:
+ - ContainerPort: !Ref GitAdminSSHContainerPort
+ HostPort: !Ref GitAdminSSHHostPort
+ Protocol: tcp
+ LogConfiguration:
+ LogDriver: awslogs
+ Options:
+ awslogs-group: !Ref ClusterStackName
+ awslogs-region: !Ref AWS::Region
+ awslogs-stream-prefix: !Ref EnvironmentName
+ - Name: !Ref GitDaemonContainerName
+ Essential: true
+ Image: !Sub '${DockerRegistryUrl}/${GitDaemonDockerImage}'
+ MountPoints:
+ - SourceVolume: !Ref GerritGitVolume
+ ContainerPath: /var/gerrit/git
+ Cpu: 256
+ Memory: 512
+ PortMappings:
+ - ContainerPort: !Ref GitDaemonContainerPort
+ HostPort: !Ref GitDaemonHostPort
+ Protocol: tcp
+ LogConfiguration:
+ LogDriver: awslogs
+ Options:
+ awslogs-group: !Ref ClusterStackName
+ awslogs-region: !Ref AWS::Region
+ awslogs-stream-prefix: !Ref EnvironmentName
+ Volumes:
+ - Name: !Ref GerritGitVolume
+ Host:
+ SourcePath: "/mnt/efs/gerrit-shared/git"
+
+ LoadBalancer:
+ Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+ Properties:
+ Type: network
+ Scheme: internal
+ Subnets:
+ - Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'PublicSubnetOne']]
+ Tags:
+ - Key: Name
+ Value: !Join ['-', [!Ref 'EnvironmentName', !Ref GitReplicationServiceName, 'nlb']]
+
+ GitAdminSSHTargetGroup:
+ Type: AWS::ElasticLoadBalancingV2::TargetGroup
+ DependsOn: LoadBalancer
+ Properties:
+ VpcId:
+ Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'VPCId']]
+ Port: !Ref GitAdminSSHHostPort
+ Protocol: TCP
+
+ GitDaemonTargetGroup:
+ Type: AWS::ElasticLoadBalancingV2::TargetGroup
+ DependsOn: LoadBalancer
+ Properties:
+ VpcId:
+ Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'VPCId']]
+ Port: !Ref GitDaemonHostPort
+ Protocol: TCP
+
+ GitAdminSSHListener:
+ Type: AWS::ElasticLoadBalancingV2::Listener
+ DependsOn: LoadBalancer
+ Properties:
+ DefaultActions:
+ - Type: forward
+ TargetGroupArn: !Ref GitAdminSSHTargetGroup
+ LoadBalancerArn: !Ref LoadBalancer
+ Port: !Ref GitAdminSSHHostPort
+ Protocol: TCP
+
+ GitDaemonListener:
+ Type: AWS::ElasticLoadBalancingV2::Listener
+ DependsOn: LoadBalancer
+ Properties:
+ DefaultActions:
+ - Type: forward
+ TargetGroupArn: !Ref GitDaemonTargetGroup
+ LoadBalancerArn: !Ref LoadBalancer
+ Port: !Ref GitDaemonHostPort
+ Protocol: TCP
+
+ ReplicationDnsRecord:
+ Type: AWS::Route53::RecordSet
+ Properties:
+ Name: !Sub '${GitReplicationSubdomain}.${HostedZoneName}'
+ HostedZoneName: !Sub '${HostedZoneName}.'
+ Comment: DNS name for the Replication service.
+ Type: A
+ AliasTarget:
+ DNSName: !GetAtt 'LoadBalancer.DNSName'
+ HostedZoneId: !GetAtt 'LoadBalancer.CanonicalHostedZoneID'
+ EvaluateTargetHealth: False
+
+ ECSTaskExecutionRoleStack:
+ Type: AWS::CloudFormation::Stack
+ Properties:
+ TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-gerrit-task-execution-role.yml'] ]
+ TimeoutInMinutes: '5'
+
+Outputs:
+ ReplicationLoadBalancerDNSName:
+ Description: The url of the replication load balancer
+ Value: !GetAtt 'LoadBalancer.DNSName'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ReplicationLoadBalancerDNSName' ] ]
+ ReplicationDNSRecord:
+ Description: Alias DNS record for the replication load balancer URL
+ Value: !Sub '${GitReplicationSubdomain}.${HostedZoneName}'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ReplicationDNSRecord' ] ]
\ No newline at end of file
diff --git a/dual-master/setup.env.template b/dual-master/setup.env.template
index 333bd89..e4a6c2b 100644
--- a/dual-master/setup.env.template
+++ b/dual-master/setup.env.template
@@ -2,6 +2,12 @@
CLUSTER_INSTANCE_TYPE:=m4.2xlarge
SERVICE_MASTER1_STACK_NAME:=$(AWS_PREFIX)-service-master-1
SERVICE_MASTER2_STACK_NAME:=$(AWS_PREFIX)-service-master-2
+
+REPLICATION_SERVICE_ENABLED:=false
+SERVICE_REPLICATION_STACK_NAME:=$(AWS_PREFIX)-service-replication
+SERVICE_REPLICATION_DESIRED_COUNT:=1
+GIT_REPLICATION_SUBDOMAIN:=$(AWS_PREFIX)-replication
+
SERVICE_SLAVE_STACK_NAME:=$(AWS_PREFIX)-service-slave
HTTP_HOST_PORT_MASTER2:=8081
SSH_HOST_PORT_MASTER2:=29419
