Place master1 and master2 onto two different ASGs
master1 and master2 should not be placed onto the same EC2 instance in
order to assure Gerrit reliability.
Separate master1 and master2 by placing them onto instances running
within two different autoscaling groups.
In order to avoid repetition, launch configuration and autoscaling
groups for masters have been moved to a nested template.
Bug: Issue 13587
Change-Id: If3ba1ad743a90bd3c2135022db87c2fc0afb5245
diff --git a/Makefile.common b/Makefile.common
index efcc0c3..645d35d 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -18,6 +18,7 @@
aws s3 cp ../common-templates/cf-gerrit-task-execution-role.yml s3://$(TEMPLATE_BUCKET_NAME)/
aws s3 cp ../common-templates/cf-gerrit-network-stack.yml s3://$(TEMPLATE_BUCKET_NAME)/
aws s3 cp ../common-templates/cf-gerrit-volume.yml s3://$(TEMPLATE_BUCKET_NAME)/
+ aws s3 cp ../common-templates/cf-master-asg.yml s3://$(TEMPLATE_BUCKET_NAME)/
set-optional-params-metrics-cloudwatch:
ifdef METRICS_CLOUDWATCH_ENABLED
diff --git a/common-templates/cf-master-asg.yml b/common-templates/cf-master-asg.yml
new file mode 100644
index 0000000..3d26c45
--- /dev/null
+++ b/common-templates/cf-master-asg.yml
@@ -0,0 +1,180 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Gerrit master launch configuration and autoscaling group
+Parameters:
+ EC2AMI:
+ Description: AMI ID for the EC2 instance hosting gerrit masters
+ Type: String
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ AllowedValues: [t2.micro, t2.small, t2.medium, t2.large, m3.medium, m3.large,
+ m3.xlarge, m3.2xlarge, m4.large, m4.xlarge, m4.2xlarge, m4.4xlarge, m4.10xlarge,
+ c4.large, c4.xlarge, c4.2xlarge, c4.4xlarge, c4.8xlarge, c3.large, c3.xlarge,
+ c3.2xlarge, c3.4xlarge, c3.8xlarge, r3.large, r3.xlarge, r3.2xlarge, r3.4xlarge,
+ r3.8xlarge, i2.xlarge, i2.2xlarge, i2.4xlarge, i2.8xlarge]
+ ConstraintDescription: Please choose a valid instance type.
+ ECSKeyName:
+ Type: String
+ Description: EC2 key pair name the cluter's instances
+ EnvironmentName:
+ Description: An environment name used to build the log stream names
+ Type: String
+ ECSCluster:
+ Description: The ECSCluster reference name to register gerrit masters to
+ Type: String
+ EC2SecurityGroup:
+ Description: security groups to assign to the instances in the Auto Scaling group hosting gerrit masters
+ Type: String
+ EC2InstanceProfile:
+ Description: The Amazon Resource Name (ARN) of the instance profile associated with the IAM role for the EC2 instances running gerrit masters
+ Type: String
+ GerritInstanceNumber:
+ Description: Whether this is master1 or master2
+ Type: Number
+ AllowedValues: [1,2]
+ FileSystem:
+ Description: The ID of the filesystem to share git data between gerrit master instances
+ Type: String
+ SubnetId:
+ Description: The subnet ID where gerrit master in the Auto Scaling group can be created
+ Type: String
+ LogGroupName:
+ Description: The log group name
+ Type: String
+ MasterMaxCount:
+ Description: The maximum number of EC2 instances in the master autoscaling group
+ Type: Number
+
+Resources:
+ MasterECSAutoScalingGroup:
+ Type: AWS::AutoScaling::AutoScalingGroup
+ Properties:
+ VPCZoneIdentifier:
+ - !Ref SubnetId
+ LaunchConfigurationName: !Ref 'MasterLaunchConfiguration'
+ MinSize: '1'
+ MaxSize: !Ref MasterMaxCount
+ DesiredCapacity: '1'
+ CreationPolicy:
+ ResourceSignal:
+ Timeout: PT15M
+ UpdatePolicy:
+ AutoScalingReplacingUpdate:
+ WillReplace: 'true'
+
+ MasterLaunchConfiguration:
+ Type: AWS::AutoScaling::LaunchConfiguration
+ Properties:
+ ImageId: !Ref 'EC2AMI'
+ SecurityGroups: [!Ref 'EC2SecurityGroup']
+ InstanceType: !Ref 'InstanceType'
+ IamInstanceProfile: !Ref 'EC2InstanceProfile'
+ KeyName: !Ref ECSKeyName
+ UserData:
+ Fn::Base64: !Sub |
+ #!/bin/bash -xe
+ export MASTER_ID=master-${GerritInstanceNumber}
+ echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
+ echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"$MASTER_ID\"} >> /etc/ecs/ecs.config
+
+ yum install -y aws-cfn-bootstrap nfs-utils wget
+
+ # EFS setting
+ DIR_TGT=/mnt/efs/gerrit-shared
+ mkdir -p $DIR_TGT
+ EC2_REGION=${AWS::Region}
+ # 169.254.169.254 link-local address, valid only from the instance, to retrieve meta-data information.
+ EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone`
+ EFS_FILE_SYSTEM_ID=${FileSystem}
+ DIR_SRC=$EC2_AVAIL_ZONE.$EFS_FILE_SYSTEM_ID.efs.$EC2_REGION.amazonaws.com
+
+ touch /home/ec2-user/echo.res
+ echo $MASTER_ID >> /home/ec2-user/echo.res
+ echo $EFS_FILE_SYSTEM_ID >> /home/ec2-user/echo.res
+ echo $EC2_AVAIL_ZONE >> /home/ec2-user/echo.res
+ echo $EC2_REGION >> /home/ec2-user/echo.res
+ echo $DIR_SRC >> /home/ec2-user/echo.res
+ echo $DIR_TGT >> /home/ec2-user/echo.res
+ MAX_RETRIES=20
+ for i in $(seq 1 $MAX_RETRIES); do
+ echo "Mounting EFS volume ($i/$MAX_RETRIES)..."
+ `mount -t nfs4 -o nfsvers=4.1,hard,timeo=600,retrans=2 $DIR_SRC:/ $DIR_TGT >> /home/ec2-user/echo.res` \
+ && s=0 && break || s=$? && sleep 5;
+ done; (exit $s)
+ mkdir -p $DIR_TGT/git
+ mkdir -p $DIR_TGT/high-availability
+ chown -R 1000:1000 $DIR_TGT
+ cp -p /etc/fstab /etc/fstab.back-$(date +%F)
+ echo -e \"$DIR_SRC:/ \t\t $DIR_TGT \t\t nfs \t\t defaults \t\t 0 \t\t 0\" | tee -a /etc/fstab
+ # Get the CloudWatch Logs agent
+ echo -e "
+ {\"logs\":
+ {\"logs_collected\":
+ {\"files\":
+ {\"collect_list\":
+ [
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/replication_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/replication_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/httpd_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/httpd_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/sshd_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/sshd_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/gc_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/gc_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/sharedref_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/sharedref_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/message_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/message_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/websession_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/websession_log\",
+ \"timezone\": \"UTC\"
+ },
+ {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/audit_log\",
+ \"log_group_name\": \"${LogGroupName}\",
+ \"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/audit_log\",
+ \"timezone\": \"UTC\"
+ }
+ ]
+ }
+ }
+ }
+ }" >> /home/ec2-user/gerritlogsaccess.json
+
+ # Install the CloudWatch Logs agent
+ wget https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
+ rpm -U ./amazon-cloudwatch-agent.rpm
+ /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/home/ec2-user/gerritlogsaccess.json -s
+
+ # Signal to CloudFormation aws-cfn-bootstrap has been correctly updated
+ /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource MasterECSAutoScalingGroup --region ${AWS::Region}
+
+Outputs:
+ MasterLaunchConfiguration:
+ Value: !Ref MasterLaunchConfiguration
+ Export:
+ Name: !Join [ '-', [ !Ref 'AWS::StackName', 'MasterLaunchConfiguration', !Ref GerritInstanceNumber ] ]
+
+ MasterECSAutoScalingGroup:
+ Value: !Ref MasterECSAutoScalingGroup
+ Export:
+ Name: !Join [ '-', [ !Ref 'AWS::StackName', 'MasterECSAutoScalingGroup', !Ref GerritInstanceNumber ] ]
+
diff --git a/dual-master/Makefile b/dual-master/Makefile
index 16ded99..f481ba3 100644
--- a/dual-master/Makefile
+++ b/dual-master/Makefile
@@ -69,6 +69,9 @@
ifdef HA_PROXY_DESIRED_COUNT
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=HAProxyDesiredCount,ParameterValue=$(HA_PROXY_DESIRED_COUNT))
endif
+ifdef MASTER_MAX_COUNT
+ $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=MasterMaxCount,ParameterValue=$(MASTER_MAX_COUNT))
+endif
$(AWS_FC_COMMAND) create-stack \
--stack-name $(CLUSTER_STACK_NAME) \
@@ -76,7 +79,6 @@
--template-body file://`pwd`/$(CLUSTER_TEMPLATE) \
--region $(AWS_REGION) \
--parameters \
- ParameterKey=DesiredCapacity,ParameterValue=$(CLUSTER_DESIRED_CAPACITY) \
ParameterKey=ECSKeyName,ParameterValue=$(CLUSTER_KEYS) \
ParameterKey=TemplateBucketName,ParameterValue=$(TEMPLATE_BUCKET_NAME) \
ParameterKey=InternetGatewayIdProp,ParameterValue=$(INTERNET_GATEWAY_ID) \
diff --git a/dual-master/README.md b/dual-master/README.md
index 6836d0f..bc3be10 100644
--- a/dual-master/README.md
+++ b/dual-master/README.md
@@ -89,7 +89,6 @@
* `DASHBOARD_STACK_NAME` : Optional. Name of the dashboard stack. `gerrit-dashboard` by default.
* `MASTER1_SUBDOMAIN`: Optional. Name of the master 1 sub domain. `gerrit-master-1-demo` by default.
* `MASTER2_SUBDOMAIN`: Optional. Name of the master 2 sub domain. `gerrit-master-2-demo` by default.
-* `CLUSTER_DESIRED_CAPACITY`: Optional. Number of EC2 instances composing the cluster. `1` by default.
* `HTTP_HOST_PORT_MASTER1`: Optional. Gerrit Host HTTP port for master1 (must be different from master2). `9080` by default.
* `SSH_HOST_PORT_MASTER1:`: Optional. Gerrit Host SSH port for master1 (must be different from master2). `29418` by default.
* `HTTP_HOST_PORT_MASTER2`: Optional. Gerrit Host HTTP port for master2 (must be different from master1). `9080` by default.
@@ -119,6 +118,9 @@
* `HA_PROXY_MAX_COUNT`: Optional. Maximum number of EC2 instances in the haproxy autoscaling group.
"2" by default. Minimum: "2".
+* `MASTER_MAX_COUNT`: Optional. Maximum number of EC2 instances in the master autoscaling group.
+"2" by default. Minimum: "2".
+
#### REPLICATION SERVICE
* `REPLICATION_SERVICE_ENABLED`: Optional. Whether to expose a replication endpoint.
diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index ee620a4..395ca41 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -6,14 +6,6 @@
TemplateBucketName:
Description: S3 bucket containing cloudformation templates
Type: String
- DesiredCapacity:
- Type: Number
- Default: '1'
- Description: Number of EC2 instances to launch in your ECS cluster.
- MaxSize:
- Type: Number
- Default: '6'
- Description: Maximum number of EC2 instances that can be launched in your ECS cluster.
ECSAMI:
Description: AMI ID
Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
@@ -75,6 +67,12 @@
Type: Number
MinValue: 2
Default: 2
+ MasterMaxCount:
+ Description: The maximum number of EC2 instances in the master autoscaling group
+ ConstraintDescription: number of haproxy must be at least 2
+ Type: Number
+ Default: 2
+ MinValue: 2
Conditions:
isProvisionedThroughput: !Equals [!Ref FileSystemThroughputMode, "provisioned"]
@@ -182,163 +180,43 @@
/opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource HAProxyECSAutoScalingGroup --region ${AWS::Region}
- # Autoscaling group. This launches the actual EC2 instances that will register
- # themselves as members of the cluster, and run the docker containers.
- MasterECSAutoScalingGroup:
- Type: AWS::AutoScaling::AutoScalingGroup
+ Master1ASG:
+ Type: AWS::CloudFormation::Stack
Properties:
- VPCZoneIdentifier:
- - !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
- LaunchConfigurationName: !Ref 'ContainerInstances'
- MinSize: '1'
- MaxSize: !Ref 'MaxSize'
- DesiredCapacity: !Ref 'DesiredCapacity'
- CreationPolicy:
- ResourceSignal:
- Timeout: PT15M
- UpdatePolicy:
- AutoScalingReplacingUpdate:
- WillReplace: 'true'
- ContainerInstances:
- Type: AWS::AutoScaling::LaunchConfiguration
- Properties:
- ImageId: !Ref 'ECSAMI'
- SecurityGroups: [!Ref 'EcsHostSecurityGroup']
- InstanceType: !Ref 'InstanceType'
- IamInstanceProfile: !Ref 'EC2InstanceProfile'
- KeyName: !Ref ECSKeyName
- UserData:
- Fn::Base64: !Sub |
- #!/bin/bash -xe
- echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
- echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"master\"} >> /etc/ecs/ecs.config
- # Make sure latest version of the helper scripts are installed as per recommendation:
- # https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
- yum install -y aws-cfn-bootstrap nfs-utils wget
- # EFS setting
- DIR_TGT=/mnt/efs/gerrit-shared
- mkdir -p $DIR_TGT
- EC2_REGION=${AWS::Region}
- # 169.254.169.254 link-local address, valid only from the instance, to retrieve meta-data information.
- EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone`
- EFS_FILE_SYSTEM_ID=${FileSystem}
- DIR_SRC=$EC2_AVAIL_ZONE.$EFS_FILE_SYSTEM_ID.efs.$EC2_REGION.amazonaws.com
+ TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-master-asg.yml'] ]
+ TimeoutInMinutes: '25'
+ Parameters:
+ GerritInstanceNumber: 1
+ EC2AMI: !Ref ECSAMI
+ InstanceType: !Ref InstanceType
+ ECSKeyName: !Ref ECSKeyName
+ EnvironmentName: !Ref EnvironmentName
+ ECSCluster: !Ref ECSCluster
+ EC2SecurityGroup: !Ref EcsHostSecurityGroup
+ EC2InstanceProfile: !Ref EC2InstanceProfile
+ FileSystem: !Ref FileSystem
+ SubnetId: !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
+ LogGroupName: !Ref AWS::StackName
+ MasterMaxCount: !Ref MasterMaxCount
- touch /home/ec2-user/echo.res
- echo $EFS_FILE_SYSTEM_ID >> /home/ec2-user/echo.res
- echo $EC2_AVAIL_ZONE >> /home/ec2-user/echo.res
- echo $EC2_REGION >> /home/ec2-user/echo.res
- echo $DIR_SRC >> /home/ec2-user/echo.res
- echo $DIR_TGT >> /home/ec2-user/echo.res
- MAX_RETRIES=20
- for i in $(seq 1 $MAX_RETRIES); do
- echo "Mounting EFS volume ($i/$MAX_RETRIES)..."
- `mount -t nfs4 -o nfsvers=4.1,hard,timeo=600,retrans=2 $DIR_SRC:/ $DIR_TGT >> /home/ec2-user/echo.res` \
- && s=0 && break || s=$? && sleep 5;
- done; (exit $s)
- mkdir -p $DIR_TGT/git
- mkdir -p $DIR_TGT/high-availability
- chown -R 1000:1000 $DIR_TGT
- cp -p /etc/fstab /etc/fstab.back-$(date +%F)
- echo -e \"$DIR_SRC:/ \t\t $DIR_TGT \t\t nfs \t\t defaults \t\t 0 \t\t 0\" | tee -a /etc/fstab
- # Get the CloudWatch Logs agent
- echo -e "
- {\"logs\":
- {\"logs_collected\":
- {\"files\":
- {\"collect_list\":
- [
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/replication_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/replication_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/httpd_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/httpd_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/sshd_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/sshd_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/gc_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/gc_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/sharedref_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/sharedref_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/message_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/message_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/websession_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/websession_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-2/_data/audit_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-2/audit_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/replication_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/replication_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/httpd_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/httpd_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/sshd_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/sshd_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/gc_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/gc_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/sharedref_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/sharedref_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/message_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/message_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/websession_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/websession_log\",
- \"timezone\": \"UTC\"
- },
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master-1/_data/audit_log\",
- \"log_group_name\": \"${AWS::StackName}\",
- \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master-1/audit_log\",
- \"timezone\": \"UTC\"
- }
- ]
- }
- }
- }
- }" >> /home/ec2-user/gerritlogsaccess.json
- # Install the CloudWatch Logs agent
- wget https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
- rpm -U ./amazon-cloudwatch-agent.rpm
- /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/home/ec2-user/gerritlogsaccess.json -s
- # Signal to CloudFormation aws-cfn-bootstrap has been correctly updated
- /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource MasterECSAutoScalingGroup --region ${AWS::Region}
+ Master2ASG:
+ Type: AWS::CloudFormation::Stack
+ Properties:
+ TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-master-asg.yml'] ]
+ TimeoutInMinutes: '25'
+ Parameters:
+ GerritInstanceNumber: 2
+ EC2AMI: !Ref ECSAMI
+ InstanceType: !Ref InstanceType
+ ECSKeyName: !Ref ECSKeyName
+ EnvironmentName: !Ref EnvironmentName
+ ECSCluster: !Ref ECSCluster
+ EC2SecurityGroup: !Ref EcsHostSecurityGroup
+ EC2InstanceProfile: !Ref EC2InstanceProfile
+ FileSystem: !Ref FileSystem
+ SubnetId: !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
+ LogGroupName: !Ref AWS::StackName
+ MasterMaxCount: !Ref MasterMaxCount
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
diff --git a/dual-master/cf-service-master.yml b/dual-master/cf-service-master.yml
index 8c18ffb..4566b62 100644
--- a/dual-master/cf-service-master.yml
+++ b/dual-master/cf-service-master.yml
@@ -231,7 +231,7 @@
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
PlacementConstraints:
- - Expression: !Sub 'attribute:target_group == master'
+ - Expression: !Sub 'attribute:target_group == master-${GerritInstanceNumber}'
Type: "memberOf"
ContainerDefinitions:
- Name: !FindInMap ['Gerrit', 'Service', 'Name']
diff --git a/dual-master/cf-service-replication.yml b/dual-master/cf-service-replication.yml
index 3fbc984..b6b656b 100644
--- a/dual-master/cf-service-replication.yml
+++ b/dual-master/cf-service-replication.yml
@@ -97,7 +97,7 @@
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
PlacementConstraints:
- - Expression: !Sub 'attribute:target_group == master'
+ - Expression: !Sub 'attribute:target_group =~ master.*'
Type: "memberOf"
ContainerDefinitions:
- Name: !Ref GitAdminSSHContainerName
diff --git a/dual-master/setup.env.template b/dual-master/setup.env.template
index d986e66..57e46aa 100644
--- a/dual-master/setup.env.template
+++ b/dual-master/setup.env.template
@@ -3,7 +3,6 @@
MULTISITE_KAFKA_BROKERS:=kafka0-yourcompany.com:9092,kafka1-yourcompany.com:9092
MULTISITE_ZOOKEEPER_ROOT_NODE:=gerrit/multi-site
-CLUSTER_DESIRED_CAPACITY:=3
CLUSTER_INSTANCE_TYPE:=m4.2xlarge
SERVICE_MASTER1_STACK_NAME:=$(AWS_PREFIX)-service-master-1
SERVICE_MASTER2_STACK_NAME:=$(AWS_PREFIX)-service-master-2
@@ -59,3 +58,4 @@
HA_PROXY_MAX_COUNT:=2
HA_PROXY_DESIRED_COUNT:=2
+MASTER_MAX_COUNT:=2
