Add active-active network load balancer
Register both gerrit-1 and gerrit-2 behind the same network load
balancer to allow serving traffic in an active-active architecture.
To allow so, both gerrit-1 and gerrit-2 needs to expose the same host
ports in order to allow traffic forwarding from the same SSH and HTTP
listeners.
For this reason the HTTP_HOST_PORT_PRIMARY and SSH_HOST_PORT_PRIMARY
variables have been removed, as effectively redundant.
A new PRIMARIES_GERRIT_SUBDOMAIN variable has been introduced so that
the active-active subdomain can be configured.
Note that whilst the haproxy is still installed and available, the
ultimate goal is to remove it and to allow _only_ active/active traffic
to go through the newly introduced load balancer.
Before doing that however a global-refdb needs to be introduced and
target groups need to be configured to maintain sticky sessions.
Bug: Issue 15128
Change-Id: I0ba035d4570ca77b1193c46bb612b2c6ea28d0fb
diff --git a/dual-primary/Makefile b/dual-primary/Makefile
index 6a2e56c..9007fe5 100644
--- a/dual-primary/Makefile
+++ b/dual-primary/Makefile
@@ -76,6 +76,9 @@
ifdef PRIMARY_MAX_COUNT
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=PrimaryMaxCount,ParameterValue=$(PRIMARY_MAX_COUNT))
endif
+ifdef LOAD_BALANCER_SCHEME
+ $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=PrimariesGerritLoadBalancerScheme,ParameterValue=$(LOAD_BALANCER_SCHEME))
+endif
$(AWS_FC_COMMAND) create-stack \
--stack-name $(CLUSTER_STACK_NAME) \
@@ -85,6 +88,7 @@
--parameters \
ParameterKey=ECSKeyName,ParameterValue=$(CLUSTER_KEYS) \
ParameterKey=TemplateBucketName,ParameterValue=$(TEMPLATE_BUCKET_NAME) \
+ ParameterKey=PrimariesGerritCertificateArn,ParameterValue=$(SSL_CERTIFICATE_ARN) \
$(GERRIT_OPTIONAL_PARAMS_NETWORK) \
$(CLUSTER_OPTIONAL_PARAMS) \
$(GERRIT_OPTIONAL_PRIMARY_VOLUME) \
@@ -130,8 +134,6 @@
ParameterKey=ReplicaSubdomain,ParameterValue=$(REPLICA_SUBDOMAIN) \
ParameterKey=DockerRegistryUrl,ParameterValue=$(DOCKER_REGISTRY_URI) \
ParameterKey=CertificateArn,ParameterValue=$(SSL_CERTIFICATE_ARN) \
- ParameterKey=HTTPHostPort,ParameterValue=$(HTTP_HOST_PORT_PRIMARY1) \
- ParameterKey=SSHHostPort,ParameterValue=$(SSH_HOST_PORT_PRIMARY1) \
ParameterKey=GerritKeyPrefix,ParameterValue=$(GERRIT_KEY_PREFIX)\
ParameterKey=DockerImage,ParameterValue=aws-gerrit/gerrit:$(IMAGE_TAG) \
ParameterKey=PeerSubdomain,ParameterValue=$(PRIMARY2_SUBDOMAIN) \
@@ -188,8 +190,6 @@
ParameterKey=ReplicaSubdomain,ParameterValue=$(REPLICA_SUBDOMAIN) \
ParameterKey=DockerRegistryUrl,ParameterValue=$(DOCKER_REGISTRY_URI) \
ParameterKey=CertificateArn,ParameterValue=$(SSL_CERTIFICATE_ARN) \
- ParameterKey=HTTPHostPort,ParameterValue=$(HTTP_HOST_PORT_PRIMARY2) \
- ParameterKey=SSHHostPort,ParameterValue=$(SSH_HOST_PORT_PRIMARY2) \
ParameterKey=GerritKeyPrefix,ParameterValue=$(GERRIT_KEY_PREFIX)\
ParameterKey=DockerImage,ParameterValue=aws-gerrit/gerrit:$(IMAGE_TAG) \
ParameterKey=PeerSubdomain,ParameterValue=$(PRIMARY1_SUBDOMAIN) \
@@ -305,7 +305,10 @@
--parameters \
ParameterKey=Primary1ServiceStackName,ParameterValue=$(SERVICE_PRIMARY1_STACK_NAME) \
ParameterKey=Primary2ServiceStackName,ParameterValue=$(SERVICE_PRIMARY2_STACK_NAME) \
- ParameterKey=LBServiceStackName,ParameterValue=$(LOAD_BALANCER_STACK_NAME)
+ ParameterKey=LBServiceStackName,ParameterValue=$(LOAD_BALANCER_STACK_NAME) \
+ ParameterKey=ClusterStackName,ParameterValue=$(CLUSTER_STACK_NAME) \
+ ParameterKey=PrimariesGerritHostedZoneName,ParameterValue=$(HOSTED_ZONE_NAME) \
+ ParameterKey=PrimariesGerritSubdomain,ParameterValue=$(PRIMARIES_GERRIT_SUBDOMAIN)
dashboard:
ifeq ($(METRICS_CLOUDWATCH_ENABLED),true)
diff --git a/dual-primary/README.md b/dual-primary/README.md
index 9ce2085..f08f391 100644
--- a/dual-primary/README.md
+++ b/dual-primary/README.md
@@ -160,12 +160,10 @@
* `DASHBOARD_STACK_NAME` : Optional. Name of the dashboard stack. `gerrit-dashboard` by default.
* `PRIMARY1_SUBDOMAIN`: Optional. Name of the primary 1 sub domain. `gerrit-primary-1-demo` by default.
* `PRIMARY2_SUBDOMAIN`: Optional. Name of the primary 2 sub domain. `gerrit-primary-2-demo` by default.
-* `HTTP_HOST_PORT_PRIMARY1`: Optional. Gerrit Host HTTP port for primary1 (must be different from primary2). `9080` by default.
-* `SSH_HOST_PORT_PRIMARY1:`: Optional. Gerrit Host SSH port for primary1 (must be different from primary2). `29418` by default.
-* `HTTP_HOST_PORT_PRIMARY2`: Optional. Gerrit Host HTTP port for primary2 (must be different from primary1). `9080` by default.
-* `SSH_HOST_PORT_PRIMARY2:`: Optional. Gerrit Host SSH port for primary2 (must be different from primary1). `29418` by default.
* `REPLICA_SUBDOMAIN`: Mandatory. The subdomain of the Gerrit replica. For example: `<AWS_PREFIX>-replica`
* `LB_SUBDOMAIN`: Mandatory. The subdomain of the Gerrit load balancer. For example: `<AWS_PREFIX>-dual-primary`
+* `PRIMARIES_GERRIT_SUBDOMAIN`: Mandatory. The subdomain of the lb serving traffic to both primary gerrit instances.
+ For example: `<AWS_PREFIX>-primaries`
* `PRIMARY_FILESYSTEM_THROUGHPUT_MODE`: Optional. The throughput mode for the primary file system to be created.
default: `bursting`. More info [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-efs-filesystem.html)
* `PRIMARY_FILESYSTEM_PROVISIONED_THROUGHPUT_IN_MIBPS`: Optional. Only used when `PRIMARY_FILESYSTEM_THROUGHPUT_MODE` is set to `provisioned`.
diff --git a/dual-primary/cf-cluster.yml b/dual-primary/cf-cluster.yml
index cf729be..fd456ac 100644
--- a/dual-primary/cf-cluster.yml
+++ b/dual-primary/cf-cluster.yml
@@ -155,6 +155,14 @@
Default: 1
MinValue: 1
MaxValue: 10
+ PrimariesGerritLoadBalancerScheme:
+ Description: The schema of the load balancer serving requests to primary gerrit instances
+ Type: String
+ Default: internet-facing
+ AllowedValues: [internal, internet-facing]
+ PrimariesGerritCertificateArn:
+ Description: SSL Certificates ARN for the load balancer serving requests to the primary gerrit instances
+ Type: String
Conditions:
isProvisionedThroughput: !Equals [!Ref PrimaryFileSystemThroughputMode, "provisioned"]
@@ -167,6 +175,13 @@
- !Equals [!Ref Subnet1IdProp, ""]
- !Equals [!Ref Subnet2IdProp, ""]
+Mappings:
+ Gerrit:
+ Port:
+ HTTPS: 443
+ HTTP: 8080
+ SSH: 29418
+
Resources:
# ECS Resources
ECSCluster:
@@ -289,6 +304,65 @@
/opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource HAProxyECSAutoScalingGroup --region ${AWS::Region}
+ PrimariesGerritLoadBalancer:
+ Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+ Properties:
+ Type: network
+ Scheme: !Ref PrimariesGerritLoadBalancerScheme
+ Name: 'primary-gerrit-instances'
+ LoadBalancerAttributes:
+ - Key: 'load_balancing.cross_zone.enabled'
+ Value: true
+ Subnets:
+ - !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef, !Ref Subnet1IdProp]
+ - !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetTwoRef, !Ref Subnet2IdProp]
+
+ PrimariesGerritHTTPTargetGroup:
+ Type: AWS::ElasticLoadBalancingV2::TargetGroup
+ DependsOn: PrimariesGerritLoadBalancer
+ Properties:
+ VpcId: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.VPCRef, !Ref VPCIdProp]
+ Port: !FindInMap ['Gerrit', 'Port', 'HTTP']
+ Protocol: TCP
+ HealthCheckPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
+ HealthCheckProtocol: HTTP
+ HealthCheckPath: '/config/server/healthcheck~status'
+ Name: 'primaries-gerrit-http'
+
+ PrimariesGerritHTTPSListener:
+ Type: AWS::ElasticLoadBalancingV2::Listener
+ Properties:
+ Certificates:
+ - CertificateArn: !Ref PrimariesGerritCertificateArn
+ DefaultActions:
+ - Type: forward
+ TargetGroupArn: !Ref PrimariesGerritHTTPTargetGroup
+ LoadBalancerArn: !Ref PrimariesGerritLoadBalancer
+ Port: !FindInMap ['Gerrit', 'Port', 'HTTPS']
+ Protocol: TLS
+
+ PrimariesGerritSSHTargetGroup:
+ Type: AWS::ElasticLoadBalancingV2::TargetGroup
+ DependsOn: PrimariesGerritLoadBalancer
+ Properties:
+ VpcId: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.VPCRef, !Ref VPCIdProp]
+ Port: !FindInMap ['Gerrit', 'Port', 'SSH']
+ Protocol: TCP
+ HealthCheckPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
+ HealthCheckProtocol: HTTP
+ HealthCheckPath: '/config/server/healthcheck~status'
+ Name: 'primaries-gerrit-ssh'
+
+ PrimariesGerritSSHListener:
+ Type: AWS::ElasticLoadBalancingV2::Listener
+ Properties:
+ DefaultActions:
+ - Type: forward
+ TargetGroupArn: !Ref PrimariesGerritSSHTargetGroup
+ LoadBalancerArn: !Ref PrimariesGerritLoadBalancer
+ Port: !FindInMap ['Gerrit', 'Port', 'SSH']
+ Protocol: TCP
+
Primary1ASG:
Type: AWS::CloudFormation::Stack
Properties:
@@ -473,3 +547,23 @@
Value: !If [CreateReplicaEFS, !GetAtt ReplicaGitFileSystemPermanentStack.Outputs.FileSystemID, !Ref ReplicaFileSystemID ]
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ReplicaFileSystemID' ] ]
+ PrimariesGerritHTTPTargetGroup:
+ Description: The target group registering both gerrit1 and gerrit2 serving HTTP traffic
+ Value: !Ref 'PrimariesGerritHTTPTargetGroup'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrimariesGerritHTTPTargetGroup' ] ]
+ PrimariesGerritSSHTargetGroup:
+ Description: The target group registering both gerrit1 and gerrit2 serving SSH traffic
+ Value: !Ref 'PrimariesGerritSSHTargetGroup'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrimariesGerritSSHTargetGroup' ] ]
+ PrimariesGerritLoadBalancerDNSName:
+ Description: The DNS name of the load balancer serving requests to both gerrit1 and gerrit2
+ Value: !GetAtt 'PrimariesGerritLoadBalancer.DNSName'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrimariesGerritLoadBalancerDNSName' ] ]
+ PrimariesGerritCanonicalHostedZoneID:
+ Description: Canonical Hosted Zone ID of the load balancer serving requests to both gerrit1 and gerrit2
+ Value: !GetAtt 'PrimariesGerritLoadBalancer.CanonicalHostedZoneID'
+ Export:
+ Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrimariesGerritCanonicalHostedZoneID' ] ]
\ No newline at end of file
diff --git a/dual-primary/cf-dns-route.yml b/dual-primary/cf-dns-route.yml
index 5048f75..1755fbd 100644
--- a/dual-primary/cf-dns-route.yml
+++ b/dual-primary/cf-dns-route.yml
@@ -1,10 +1,14 @@
AWSTemplateFormatVersion: '2010-09-09'
Description: A stack for the Gerrit service Route53 routing.
Parameters:
+ ClusterStackName:
+ Description: Stack name of the Cluster
+ Type: String
+ Default: gerrit-cluster-stack
Primary1ServiceStackName:
- Description: Stack name of the ECS Primary Gerrit service
- Type: String
- Default: gerrit-service-primary-1
+ Description: Stack name of the ECS Primary Gerrit service
+ Type: String
+ Default: gerrit-service-primary-1
Primary2ServiceStackName:
Description: Stack name of the ECS Primary Gerrit service
Type: String
@@ -13,6 +17,12 @@
Description: Stack name of the ECS LB service
Type: String
Default: gerrit-service
+ PrimariesGerritSubdomain:
+ Description: The subdomain of the load balancer serving requests to primary gerrit instances
+ Type: String
+ PrimariesGerritHostedZoneName:
+ Description: The zone name of the load balancer serving requests to primary gerrit instances
+ Type: String
Resources:
Primary1DnsRecord:
@@ -84,3 +94,27 @@
Fn::ImportValue:
!Join [':', [!Ref 'LBServiceStackName', 'CanonicalHostedZoneID']]
EvaluateTargetHealth: False
+
+ PrimariesGerritDnsRecord:
+ Type: AWS::Route53::RecordSet
+ Properties:
+ Name:
+ !Join
+ - '.'
+ - - !Ref PrimariesGerritSubdomain
+ - !Ref PrimariesGerritHostedZoneName
+ HostedZoneName:
+ !Join
+ - ''
+ - - !Ref PrimariesGerritHostedZoneName
+ - '.'
+ Comment: DNS name for the load balancer serving requests to primary gerrit instances
+ Type: A
+ AliasTarget:
+ DNSName:
+ Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'PrimariesGerritLoadBalancerDNSName']]
+ HostedZoneId:
+ Fn::ImportValue:
+ !Join [':', [!Ref 'ClusterStackName', 'PrimariesGerritCanonicalHostedZoneID']]
+ EvaluateTargetHealth: False
\ No newline at end of file
diff --git a/dual-primary/cf-service-primary.yml b/dual-primary/cf-service-primary.yml
index 7aa35c6..d08c07a 100644
--- a/dual-primary/cf-service-primary.yml
+++ b/dual-primary/cf-service-primary.yml
@@ -23,14 +23,6 @@
DockerRegistryUrl:
Description: Docker registry URL
Type: String
- HTTPHostPort:
- Description: Gerrit HTTP port
- Type: Number
- Default: 8080
- SSHHostPort:
- Description: Gerrit SSH port
- Type: Number
- Default: 29418
CertificateArn:
Description: SSL Certificates ARN
Type: String
@@ -207,12 +199,9 @@
Logs: gerrit-logs
Service:
Name: gerrit-primary
- LoadBalancer:
- HTTPPort: 8080
- SSHPort: 29418
- Container:
- HTTPPort: 8080
- SSHPort: 29418
+ Port:
+ HTTP: 8080
+ SSH: 29418
Git:
Daemon:
Port: 9418
@@ -233,12 +222,21 @@
TaskDefinition: !Ref TaskDefinition
LoadBalancers:
- ContainerName: !FindInMap ['Gerrit', 'Service', 'Name']
- ContainerPort: !FindInMap ['Gerrit', 'Container', 'HTTPPort']
+ ContainerPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
TargetGroupArn: !Ref HTTPTargetGroup
- ContainerName: !FindInMap ['Gerrit', 'Service', 'Name']
- ContainerPort: !FindInMap ['Gerrit', 'Container', 'SSHPort']
+ ContainerPort: !FindInMap ['Gerrit', 'Port', 'SSH']
TargetGroupArn: !Ref SSHTargetGroup
-
+ - ContainerName: !FindInMap ['Gerrit', 'Service', 'Name']
+ ContainerPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
+ TargetGroupArn:
+ Fn::ImportValue:
+ !Join [ ':', [ !Ref 'ClusterStackName', 'PrimariesGerritHTTPTargetGroup' ] ]
+ - ContainerName: !FindInMap ['Gerrit', 'Service', 'Name']
+ ContainerPort: !FindInMap ['Gerrit', 'Port', 'SSH']
+ TargetGroupArn:
+ Fn::ImportValue:
+ !Join [ ':', [ !Ref 'ClusterStackName', 'PrimariesGerritSSHTargetGroup' ] ]
TaskDefinition:
Type: AWS::ECS::TaskDefinition
Properties:
@@ -259,7 +257,7 @@
- Name: HTTPD_LISTEN_URL
Value: !Sub
- 'proxy-https://*:${HTTPContainerPort}/'
- - { HTTPContainerPort: !FindInMap ['Gerrit', 'Container', 'HTTPPort'] }
+ - { HTTPContainerPort: !FindInMap ['Gerrit', 'Port', 'HTTP'] }
- Name: AWS_REGION
Value: !Ref AWS::Region
- Name: SETUP_REPLICATION
@@ -273,7 +271,7 @@
- Name: HA_PEER_URL
Value: !Sub
- 'http://${PeerSubdomain}.${HostedZoneName}:${HTTPGerritLBPort}'
- - { HTTPGerritLBPort: !FindInMap ['Gerrit', 'LoadBalancer', 'HTTPPort'] }
+ - { HTTPGerritLBPort: !FindInMap ['Gerrit', 'Port', 'HTTP'] }
- Name: HOSTED_ZONE_NAME
Value: !Ref HostedZoneName
- Name: REINDEX_AT_STARTUP
@@ -362,11 +360,11 @@
Cpu: !Ref GerritCPU
Memory: !Ref GerritRAM
PortMappings:
- - ContainerPort: !FindInMap ['Gerrit', 'Container', 'HTTPPort']
- HostPort: !Ref HTTPHostPort
+ - ContainerPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
+ HostPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
Protocol: tcp
- - ContainerPort: !FindInMap ['Gerrit', 'Container', 'SSHPort']
- HostPort: !Ref SSHHostPort
+ - ContainerPort: !FindInMap ['Gerrit', 'Port', 'SSH']
+ HostPort: !FindInMap ['Gerrit', 'Port', 'SSH']
Protocol: tcp
LogConfiguration:
LogDriver: awslogs
@@ -421,9 +419,9 @@
VpcId:
Fn::ImportValue:
!Join [':', [!Ref 'ClusterStackName', 'VPCId']]
- Port: !Ref HTTPHostPort
+ Port: !FindInMap ['Gerrit', 'Port', 'HTTP']
Protocol: TCP
- HealthCheckPort: !Ref HTTPHostPort
+ HealthCheckPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
HealthCheckProtocol: HTTP
HealthCheckPath: '/config/server/healthcheck~status'
Name: !Sub 'gerrit-${GerritInstanceNumber}-primary-http'
@@ -436,7 +434,7 @@
- Type: forward
TargetGroupArn: !Ref HTTPTargetGroup
LoadBalancerArn: !Ref LoadBalancer
- Port: !FindInMap ['Gerrit', 'LoadBalancer', 'HTTPPort']
+ Port: !FindInMap ['Gerrit', 'Port', 'HTTP']
Protocol: TCP
SSHTargetGroup:
@@ -446,9 +444,9 @@
VpcId:
Fn::ImportValue:
!Join [':', [!Ref 'ClusterStackName', 'VPCId']]
- Port: !Ref SSHHostPort
+ Port: !FindInMap ['Gerrit', 'Port', 'SSH']
Protocol: TCP
- HealthCheckPort: !Ref HTTPHostPort
+ HealthCheckPort: !FindInMap ['Gerrit', 'Port', 'HTTP']
HealthCheckProtocol: HTTP
HealthCheckPath: '/config/server/healthcheck~status'
Name: !Sub 'gerrit-${GerritInstanceNumber}-primary-ssh'
@@ -461,7 +459,7 @@
- Type: forward
TargetGroupArn: !Ref SSHTargetGroup
LoadBalancerArn: !Ref LoadBalancer
- Port: !FindInMap ['Gerrit', 'LoadBalancer', 'SSHPort']
+ Port: !FindInMap ['Gerrit', 'Port', 'SSH']
Protocol: TCP
ECSTaskExecutionRoleStack:
diff --git a/dual-primary/setup.env.template b/dual-primary/setup.env.template
index 78a215d..c282ff6 100644
--- a/dual-primary/setup.env.template
+++ b/dual-primary/setup.env.template
@@ -14,10 +14,6 @@
# REMOTE_REPLICATION_TARGET_HOST:=other-site-replication.yourcompany.com
SERVICE_REPLICA_STACK_NAME:=$(AWS_PREFIX)-service-replica
-HTTP_HOST_PORT_PRIMARY2:=8081
-SSH_HOST_PORT_PRIMARY2:=29419
-HTTP_HOST_PORT_PRIMARY1:=8080
-SSH_HOST_PORT_PRIMARY1:=29418
DNS_ROUTING_STACK_NAME:=$(AWS_PREFIX)-dns-routing
DASHBOARD_STACK_NAME:=$(AWS_PREFIX)-dashboard
LOAD_BALANCER_STACK_NAME:=$(AWS_PREFIX)-load-balancer
@@ -26,6 +22,7 @@
PRIMARY2_SUBDOMAIN:=$(AWS_PREFIX)-primary-2.gerrit-demo
REPLICA_SUBDOMAIN:=$(AWS_PREFIX)-replica.gerrit-demo
LB_SUBDOMAIN=$(AWS_PREFIX)-lb.gerrit-demo
+PRIMARIES_GERRIT_SUBDOMAIN=$(AWS_PREFIX)-primaries.gerrit-demo
DOCKER_REGISTRY_URI:=<yourAccountId>.dkr.ecr.us-east-1.amazonaws.com
SSL_CERTIFICATE_ARN=arn:aws:acm:us-east-1:<yourAccountId>:certificate/33e2c235-a4d1-42b7-b866-18d8d744975c
GERRIT_RAM=6000
