Make ha-proxy highly available
A single ha-proxy instance is a single point of failure.
Set ha-proxy to a minimum of 2 instances, for achieving
high-availability of the proxy layer.
Bug: Issue 13609
Change-Id: I0d8661666ad82e89e7b9d41a66e9b31b66db932d
diff --git a/dual-master/Makefile b/dual-master/Makefile
index 133dabe..16ded99 100644
--- a/dual-master/Makefile
+++ b/dual-master/Makefile
@@ -63,6 +63,12 @@
ifdef VPC_CIDR
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=VPCCIDR,ParameterValue=$(VPC_CIDR))
endif
+ifdef HA_PROXY_MAX_COUNT
+ $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=HAProxyMaxCount,ParameterValue=$(HA_PROXY_MAX_COUNT))
+endif
+ifdef HA_PROXY_DESIRED_COUNT
+ $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=HAProxyDesiredCount,ParameterValue=$(HA_PROXY_DESIRED_COUNT))
+endif
$(AWS_FC_COMMAND) create-stack \
--stack-name $(CLUSTER_STACK_NAME) \
@@ -237,6 +243,9 @@
ifdef LOAD_BALANCER_SCHEME
$(eval SERVICE_OPTIONAL_PARAMS := $(SERVICE_OPTIONAL_PARAMS) ParameterKey=LoadBalancerScheme,ParameterValue=$(LOAD_BALANCER_SCHEME))
endif
+ifdef HA_PROXY_DESIRED_COUNT
+ $(eval SERVICE_OPTIONAL_PARAMS := $(SERVICE_OPTIONAL_PARAMS) ParameterKey=DesiredCount,ParameterValue=$(HA_PROXY_DESIRED_COUNT))
+endif
$(AWS_FC_COMMAND) create-stack \
--stack-name $(LOAD_BALANCER_STACK_NAME) \
diff --git a/dual-master/README.md b/dual-master/README.md
index fbdb39e..6836d0f 100644
--- a/dual-master/README.md
+++ b/dual-master/README.md
@@ -108,6 +108,17 @@
* `GERRIT_MASTER2_INSTANCE_ID`: Optional. Identifier for the Gerrit master2 instance.
"gerrit-dual-master-MASTER2" by default.
+* `HA_PROXY_DESIRED_COUNT`: Optional. Desired number of haproxy services.
+"2" by default. Minimum: "2".
+
+*Note* ha-proxies are running on ec2 instances with a ratio of 1 to 1: each
+ec2 node hosts one and only one ha-proxy. By increasing the number of desired
+ha-proxies then, the size of the autoscaling group hosting them also increases
+accordingly.
+
+* `HA_PROXY_MAX_COUNT`: Optional. Maximum number of EC2 instances in the haproxy autoscaling group.
+"2" by default. Minimum: "2".
+
#### REPLICATION SERVICE
* `REPLICATION_SERVICE_ENABLED`: Optional. Whether to expose a replication endpoint.
diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index e41b185..ee620a4 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -65,6 +65,16 @@
Description: The fs throughput, measured in MiB/s. Valid values are 1-1024.
Type: Number
Default: 256
+ HAProxyMaxCount:
+ Description: The maximum number of EC2 instances in the haproxy autoscaling group
+ Type: Number
+ Default: 2
+ HAProxyDesiredCount:
+ Description: The desired number of haproxy instances
+ ConstraintDescription: number of haproxy must be at least 2
+ Type: Number
+ MinValue: 2
+ Default: 2
Conditions:
isProvisionedThroughput: !Equals [!Ref FileSystemThroughputMode, "provisioned"]
@@ -137,9 +147,9 @@
VPCZoneIdentifier:
- !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
LaunchConfigurationName: !Ref 'HAProxyLaunchConfiguration'
- MinSize: '1'
- MaxSize: '1'
- DesiredCapacity: '1'
+ MinSize: '2'
+ MaxSize: !Ref HAProxyMaxCount
+ DesiredCapacity: !Ref HAProxyDesiredCount
CreationPolicy:
ResourceSignal:
Timeout: PT15M
diff --git a/dual-master/cf-service-lb.yml b/dual-master/cf-service-lb.yml
index 27f41fa..d2795ff 100644
--- a/dual-master/cf-service-lb.yml
+++ b/dual-master/cf-service-lb.yml
@@ -29,7 +29,9 @@
DesiredCount:
Description: How many instances of this task should we run across our cluster?
Type: Number
- Default: 1
+ MinValue: 2
+ Default: 2
+ ConstraintDescription: number of haproxy must be at least 2
HTTPGerritPort:
Description: Gerrit HTTP port
Type: Number
diff --git a/dual-master/setup.env.template b/dual-master/setup.env.template
index 99bbf84..d986e66 100644
--- a/dual-master/setup.env.template
+++ b/dual-master/setup.env.template
@@ -56,3 +56,6 @@
METRICS_CLOUDWATCH_JVM_ENABLED:=true
METRICS_CLOUDWATCH_DRY_RUN:=false
METRICS_CLOUDWATCH_EXCLUDE_METRICS_LIST:=foo.*,bar.*
+
+HA_PROXY_MAX_COUNT:=2
+HA_PROXY_DESIRED_COUNT:=2
