Separate replicas from masters
Not having a specific placement strategy for replicas might cause them
to be deployed on the same ec2-instance as the masters.
If tasks are running on the same instance, both source and destination
IP are the same when being received by the target instance and this
causes packets to be dropped as per convention for L4 networking[1].
Ensure replicas are never deployed to the same ec2-instance as the
masters by creating a dedicated ASG for them and forcing the placement
on ec2-instances labelled as 'replica'.
[1]https://aws.amazon.com/premiumsupport/knowledge-center/target-connection-fails-load-balancer/
Bug: Issue 13585
Change-Id: I3e0cb3c0a85865f80129e6fa4d1763bd242a21a5
diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index 9f9ee7a..e6a2147 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -90,9 +90,51 @@
LogGroupName: !Ref AWS::StackName
RetentionInDays: 14
+ ReplicaECSAutoScalingGroup:
+ Type: AWS::AutoScaling::AutoScalingGroup
+ Properties:
+ VPCZoneIdentifier:
+ - !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
+ LaunchConfigurationName: !Ref 'ReplicaLaunchConfiguration'
+ MinSize: '1'
+ MaxSize: '1'
+ DesiredCapacity: '1'
+ CreationPolicy:
+ ResourceSignal:
+ Timeout: PT15M
+ UpdatePolicy:
+ AutoScalingReplacingUpdate:
+ WillReplace: 'true'
+
+ ReplicaLaunchConfiguration:
+ Type: AWS::AutoScaling::LaunchConfiguration
+ Properties:
+ ImageId: !Ref 'ECSAMI'
+ SecurityGroups: [!Ref 'EcsHostSecurityGroup']
+ InstanceType: !Ref 'InstanceType'
+ IamInstanceProfile: !Ref 'EC2InstanceProfile'
+ KeyName: !Ref ECSKeyName
+ UserData:
+ Fn::Base64: !Sub |
+ #!/bin/bash -xe
+ echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
+ echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"replica\"} >> /etc/ecs/ecs.config
+
+ # Make sure latest version of the helper scripts are installed as per recommendation:
+ # https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
+ yum install -y aws-cfn-bootstrap wget
+
+ # Install the CloudWatch Logs agent
+ wget https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
+ rpm -U ./amazon-cloudwatch-agent.rpm
+ /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s
+
+ /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ReplicaECSAutoScalingGroup --region ${AWS::Region}
+
+
# Autoscaling group. This launches the actual EC2 instances that will register
# themselves as members of the cluster, and run the docker containers.
- ECSAutoScalingGroup:
+ MasterECSAutoScalingGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Properties:
VPCZoneIdentifier:
@@ -119,6 +161,7 @@
Fn::Base64: !Sub |
#!/bin/bash -xe
echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
+ echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"master\"} >> /etc/ecs/ecs.config
# Make sure latest version of the helper scripts are installed as per recommendation:
# https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
yum install -y aws-cfn-bootstrap nfs-utils wget
@@ -245,7 +288,7 @@
rpm -U ./amazon-cloudwatch-agent.rpm
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/home/ec2-user/gerritlogsaccess.json -s
# Signal to CloudFormation aws-cfn-bootstrap has been correctly updated
- /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
+ /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource MasterECSAutoScalingGroup --region ${AWS::Region}
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
diff --git a/dual-master/cf-service-lb.yml b/dual-master/cf-service-lb.yml
index 257a5e9..42a09c4 100644
--- a/dual-master/cf-service-lb.yml
+++ b/dual-master/cf-service-lb.yml
@@ -108,6 +108,9 @@
TaskRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
+ PlacementConstraints:
+ - Expression: !Sub 'attribute:target_group == master'
+ Type: "memberOf"
ContainerDefinitions:
- Name: !Ref LBServiceName
Essential: true
diff --git a/dual-master/cf-service-master.yml b/dual-master/cf-service-master.yml
index b38df4a..8c18ffb 100644
--- a/dual-master/cf-service-master.yml
+++ b/dual-master/cf-service-master.yml
@@ -230,6 +230,9 @@
TaskRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
+ PlacementConstraints:
+ - Expression: !Sub 'attribute:target_group == master'
+ Type: "memberOf"
ContainerDefinitions:
- Name: !FindInMap ['Gerrit', 'Service', 'Name']
Essential: true
diff --git a/dual-master/cf-service-replication.yml b/dual-master/cf-service-replication.yml
index e85baf5..3fbc984 100644
--- a/dual-master/cf-service-replication.yml
+++ b/dual-master/cf-service-replication.yml
@@ -96,6 +96,9 @@
TaskRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
+ PlacementConstraints:
+ - Expression: !Sub 'attribute:target_group == master'
+ Type: "memberOf"
ContainerDefinitions:
- Name: !Ref GitAdminSSHContainerName
Essential: true
diff --git a/dual-master/cf-service-slave.yml b/dual-master/cf-service-slave.yml
index 9d4aa7d..4f27e14 100644
--- a/dual-master/cf-service-slave.yml
+++ b/dual-master/cf-service-slave.yml
@@ -213,6 +213,9 @@
TaskRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
NetworkMode: bridge
+ PlacementConstraints:
+ - Expression: !Sub 'attribute:target_group == replica'
+ Type: "memberOf"
ContainerDefinitions:
- Name: !Ref GerritServiceName
Essential: true
