commit 2e4ba2a65ec1cfb0e05db748815439eabd7308bf
author Antonio Barone <syntonyze@gmail.com> Tue Oct 27 18:16:01 2020 +0100
committer Antonio Barone <syntonyze@gmail.com> Wed Oct 28 11:13:08 2020 +0000
tree 7855964d39eb8df2098a8863ebed872fd2c5ba9c
parent 03578a6e74d6a3ae240102b4bf5b61344d1f8fec

Separate ha-proxy from masters

Not having a specific placement strategy for ha-proxy might cause them
to be deployed on the same ec2-instance as the masters.

If tasks are running on the same instance, both source and destination
IP are the same when being received by the target instance and this
causes packets to be dropped as per convention for L4 networking[1].

Ensure ha-proxies are never deployed to the same ec2-instance as the
masters by creating a dedicated ASG for them and forcing the placement
on ec2-instances labelled as 'haproxy'.

[1]https://aws.amazon.com/premiumsupport/knowledge-center/target-connection-fails-load-balancer/

Bug: Issue 13565
Change-Id: I38ad5d29db6bd61d8f0396fc1468b72be5c0adb7

dual-master/cf-cluster.yml

diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index e6a2147..e41b185 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -131,6 +131,46 @@
 
           /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ReplicaECSAutoScalingGroup --region ${AWS::Region}
 
+  HAProxyECSAutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    Properties:
+      VPCZoneIdentifier:
+        - !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
+      LaunchConfigurationName: !Ref 'HAProxyLaunchConfiguration'
+      MinSize: '1'
+      MaxSize: '1'
+      DesiredCapacity: '1'
+    CreationPolicy:
+      ResourceSignal:
+        Timeout: PT15M
+    UpdatePolicy:
+      AutoScalingReplacingUpdate:
+        WillReplace: 'true'
+
+  HAProxyLaunchConfiguration:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Properties:
+      ImageId: !Ref 'ECSAMI'
+      SecurityGroups: [!Ref 'EcsHostSecurityGroup']
+      InstanceType: !Ref 'InstanceType'
+      IamInstanceProfile: !Ref 'EC2InstanceProfile'
+      KeyName: !Ref ECSKeyName
+      UserData:
+        Fn::Base64: !Sub |
+          #!/bin/bash -xe
+          echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
+          echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"haproxy\"} >> /etc/ecs/ecs.config
+
+          # Make sure latest version of the helper scripts are installed as per recommendation:
+          # https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
+          yum install -y aws-cfn-bootstrap wget
+
+          # Install the CloudWatch Logs agent
+          wget https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
+          rpm -U ./amazon-cloudwatch-agent.rpm
+          /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s
+
+          /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource HAProxyECSAutoScalingGroup --region ${AWS::Region}
 
   # Autoscaling group. This launches the actual EC2 instances that will register
   # themselves as members of the cluster, and run the docker containers.

dual-master/cf-service-lb.yml

diff --git a/dual-master/cf-service-lb.yml b/dual-master/cf-service-lb.yml
index 42a09c4..27f41fa 100644
--- a/dual-master/cf-service-lb.yml
+++ b/dual-master/cf-service-lb.yml
@@ -109,7 +109,7 @@
             ExecutionRoleArn: !GetAtt ECSTaskExecutionRoleStack.Outputs.TaskExecutionRoleRef
             NetworkMode: bridge
             PlacementConstraints:
-                - Expression: !Sub 'attribute:target_group == master'
+                - Expression: !Sub 'attribute:target_group == haproxy'
                   Type: "memberOf"
             ContainerDefinitions:
                 - Name: !Ref LBServiceName