Allow to specify EBS for dual-master recipe
When upgrading master instance on the EC2 clusters, the new master will
share the git data that's on EFS, but not index or cache data, that is
on EBS.
Reindexing will happen automatically (thanks to the high-availability
plugin), but it could potentially take long time to reindex all repos.
Allow to bring up new masters with a specific snapshot for the EBS data
so that reindexing is quicker.
Feature: Issue 13644
Change-Id: I4ecd1b99e2ba69b116f0559a893ca891d8009e7b
diff --git a/Makefile.common b/Makefile.common
index 11b76e8..d42b31b 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -80,4 +80,16 @@
set-optional-jgit-conf:
ifdef JGIT_OPEN_FILES
$(eval JGIT_OPTIONAL_PARAMS := ParameterKey=JgitOpenFiles,ParameterValue=$(JGIT_OPEN_FILES))
+endif
+
+set-optional-gerrit-master-volume:
+ $(eval GERRIT_OPTIONAL_MASTER_VOLUME=)
+ifdef GERRIT_VOLUME_ID
+ $(eval GERRIT_OPTIONAL_MASTER_VOLUME := $(GERRIT_OPTIONAL_MASTER_VOLUME) ParameterKey=GerritVolumeId,ParameterValue=$(GERRIT_VOLUME_ID))
+endif
+ifdef GERRIT_VOLUME_SNAPSHOT_ID
+ $(eval GERRIT_OPTIONAL_MASTER_VOLUME := $(GERRIT_OPTIONAL_MASTER_VOLUME) ParameterKey=GerritVolumeSnapshotId,ParameterValue=$(GERRIT_VOLUME_SNAPSHOT_ID))
+endif
+ifdef GERRIT_VOLUME_SIZE_IN_GIB
+ $(eval GERRIT_OPTIONAL_MASTER_VOLUME := $(GERRIT_OPTIONAL_MASTER_VOLUME) ParameterKey=GerritVolumeSizeInGiB,ParameterValue=$(GERRIT_VOLUME_SIZE_IN_GIB))
endif
\ No newline at end of file
diff --git a/common-templates/cf-master-asg.yml b/common-templates/cf-master-asg.yml
index 0840168..c099798 100644
--- a/common-templates/cf-master-asg.yml
+++ b/common-templates/cf-master-asg.yml
@@ -1,6 +1,9 @@
AWSTemplateFormatVersion: '2010-09-09'
Description: Gerrit master launch configuration and autoscaling group
Parameters:
+ TemplateBucketName:
+ Description: S3 bucket containing cloudformation templates
+ Type: String
EC2AMI:
Description: AMI ID for the EC2 instance hosting gerrit masters
Type: String
@@ -44,6 +47,21 @@
MasterMaxCount:
Description: The maximum number of EC2 instances in the master autoscaling group
Type: Number
+ GerritVolumeId:
+ Description: Existing Gerrit volume id
+ Type: String
+ GerritVolumeSnapshotId:
+ Description: Id of the EBS snapshot for Gerrit volume
+ Type: String
+ GerritVolumeAttachMaxRetries:
+ Description: Maximum number of retries when attaching Gerrit Volume
+ Type: Number
+ GerritVolumeAttachRetryDelay:
+ Description: The delay in seconds between Gerrit Volume attach attempts
+ Type: Number
+ GerritVolumeSizeInGiB:
+ Description: Gerrit volume size in GiB
+ Type: Number
Resources:
MasterECSAutoScalingGroup:
@@ -77,7 +95,46 @@
echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"$MASTER_ID\"} >> /etc/ecs/ecs.config
- yum install -y aws-cfn-bootstrap nfs-utils wget
+ yum install -y aws-cfn-bootstrap nfs-utils wget aws-cli xfsprogs
+
+ EC2_INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
+ VOLUME_ID=${GerritVolumeStack.Outputs.GerritVolumeRef}
+ echo "Volume Id: $VOLUME_ID"
+
+ aws ec2 attach-volume --region ${AWS::Region} --volume-id $VOLUME_ID --instance-id $EC2_INSTANCE_ID --device /dev/xvdg
+
+ for i in $(seq 1 ${GerritVolumeAttachMaxRetries}); do
+ echo "Waiting for volume $VOLUME_ID to be attached to the instace $EC2_INSTANCE_ID ($i/${GerritVolumeAttachMaxRetries})"
+ volumeStatus=`aws ec2 describe-volumes --region ${AWS::Region} --volume-ids $VOLUME_ID`
+ if [[ $volumeStatus =~ "\"State\": \"attached\"" ]]; then
+ echo "Volume $VOLUME_ID attached to the instace $EC2_INSTANCE_ID"
+ break
+ elif [[ "$i" -eq "${GerritVolumeAttachMaxRetries}" ]]; then
+ echo "Could not attach the volume $VOLUME_ID to the instace $EC2_INSTANCE_ID after ${GerritVolumeAttachMaxRetries} attempts"
+ exit 1
+ fi
+ sleep ${GerritVolumeAttachRetryDelay}
+ done
+
+ if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+ echo "Create file system for Gerrit volume"
+ mkfs -t xfs /dev/xvdg
+ fi
+
+ mkdir /gerrit-mount-point
+ mount /dev/xvdg /gerrit-mount-point
+
+ if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+ echo "Create Gerrit directories"
+ mkdir -p /gerrit-mount-point/gerrit-logs \
+ /gerrit-mount-point/gerrit-cache \
+ /gerrit-mount-point/gerrit-data \
+ /gerrit-mount-point/gerrit-git \
+ /gerrit-mount-point/gerrit-index \
+ /gerrit-mount-point/gerrit-db
+ fi
+
+ chown 1000:1000 -R /gerrit-mount-point
# EFS setting
DIR_TGT=/mnt/efs/gerrit-shared
@@ -113,42 +170,42 @@
{\"files\":
{\"collect_list\":
[
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/replication_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/replication_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/replication_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/httpd_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/httpd_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/httpd_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/sshd_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/sshd_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/sshd_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/gc_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/gc_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/gc_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/sharedref_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/sharedref_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/sharedref_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/message_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/message_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/message_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/websession_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/websession_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/websession_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-$MASTER_ID/_data/audit_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/audit_log\",
\"log_group_name\": \"${LogGroupName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/$MASTER_ID/audit_log\",
\"timezone\": \"UTC\"
@@ -167,6 +224,16 @@
# Signal to CloudFormation aws-cfn-bootstrap has been correctly updated
/opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource MasterECSAutoScalingGroup --region ${AWS::Region}
+ GerritVolumeStack:
+ Type: AWS::CloudFormation::Stack
+ Properties:
+ TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-gerrit-volume.yml'] ]
+ TimeoutInMinutes: '25'
+ Parameters:
+ GerritVolumeId: !Ref GerritVolumeId
+ GerritVolumeSnapshotId: !Ref GerritVolumeSnapshotId
+ GerritVolumeSizeInGiB: !Ref GerritVolumeSizeInGiB
+
Outputs:
MasterLaunchConfiguration:
Value: !Ref MasterLaunchConfiguration
diff --git a/dual-master/Makefile b/dual-master/Makefile
index ca00f03..6545d07 100644
--- a/dual-master/Makefile
+++ b/dual-master/Makefile
@@ -47,7 +47,7 @@
service-lb wait-for-service-lb-creation \
dns-routing wait-for-dns-routing-creation
-cluster: cluster-keys
+cluster: cluster-keys set-optional-gerrit-master-volume
ifdef CLUSTER_INSTANCE_TYPE
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE))
endif
@@ -84,7 +84,8 @@
ParameterKey=InternetGatewayIdProp,ParameterValue=$(INTERNET_GATEWAY_ID) \
ParameterKey=VPCIdProp,ParameterValue=$(VPC_ID) \
ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
- $(CLUSTER_OPTIONAL_PARAMS)
+ $(CLUSTER_OPTIONAL_PARAMS) \
+ $(GERRIT_OPTIONAL_MASTER_VOLUME)
service-master-1: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-optional-params-multisite set-ldap-account-pattern set-optional-gerrit-ulimits set-optional-jgit-conf
ifdef GERRIT_MASTER1_INSTANCE_ID
diff --git a/dual-master/README.md b/dual-master/README.md
index bc3be10..6343fba 100644
--- a/dual-master/README.md
+++ b/dual-master/README.md
@@ -121,6 +121,12 @@
* `MASTER_MAX_COUNT`: Optional. Maximum number of EC2 instances in the master autoscaling group.
"2" by default. Minimum: "2".
+* `GERRIT_VOLUME_ID` : Optional. Id of an extisting EBS volume. If empty, a new volume
+for Gerrit data will be created
+* `GERRIT_VOLUME_SNAPSHOT_ID` : Optional. Ignored if GERRIT_VOLUME_ID is not empty. Id of
+the EBS volume snapshot used to create new EBS volume for Gerrit data.
+* `GERRIT_VOLUME_SIZE_IN_GIB`: Optional. The size of the Gerrit data volume, in GiBs. `10` by default.
+
#### REPLICATION SERVICE
* `REPLICATION_SERVICE_ENABLED`: Optional. Whether to expose a replication endpoint.
diff --git a/dual-master/cf-cluster.yml b/dual-master/cf-cluster.yml
index 395ca41..90b898a 100644
--- a/dual-master/cf-cluster.yml
+++ b/dual-master/cf-cluster.yml
@@ -73,6 +73,26 @@
Type: Number
Default: 2
MinValue: 2
+ GerritVolumeId:
+ Description: Existing Gerrit volume id
+ Type: String
+ Default: ""
+ GerritVolumeSnapshotId:
+ Description: Id of the EBS snapshot for Gerrit volume
+ Type: String
+ Default: ""
+ GerritVolumeAttachMaxRetries:
+ Description: Maximum number of retries when attaching Gerrit Volume
+ Type: Number
+ Default: 5
+ GerritVolumeAttachRetryDelay:
+ Description: The delay in seconds between Gerrit Volume attach attempts
+ Type: Number
+ Default: 5
+ GerritVolumeSizeInGiB:
+ Description: Gerrit volume size in GiB
+ Type: Number
+ Default: 10
Conditions:
isProvisionedThroughput: !Equals [!Ref FileSystemThroughputMode, "provisioned"]
@@ -198,6 +218,12 @@
SubnetId: !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
LogGroupName: !Ref AWS::StackName
MasterMaxCount: !Ref MasterMaxCount
+ GerritVolumeAttachMaxRetries: !Ref GerritVolumeAttachMaxRetries
+ GerritVolumeAttachRetryDelay: !Ref GerritVolumeAttachRetryDelay
+ GerritVolumeId: !Ref GerritVolumeId
+ GerritVolumeSnapshotId: !Ref GerritVolumeSnapshotId
+ GerritVolumeSizeInGiB: !Ref GerritVolumeSizeInGiB
+ TemplateBucketName: !Ref TemplateBucketName
Master2ASG:
Type: AWS::CloudFormation::Stack
@@ -217,6 +243,12 @@
SubnetId: !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef
LogGroupName: !Ref AWS::StackName
MasterMaxCount: !Ref MasterMaxCount
+ GerritVolumeAttachMaxRetries: !Ref GerritVolumeAttachMaxRetries
+ GerritVolumeAttachRetryDelay: !Ref GerritVolumeAttachRetryDelay
+ GerritVolumeId: !Ref GerritVolumeId
+ GerritVolumeSnapshotId: !Ref GerritVolumeSnapshotId
+ GerritVolumeSizeInGiB: !Ref GerritVolumeSizeInGiB
+ TemplateBucketName: !Ref TemplateBucketName
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
@@ -255,6 +287,8 @@
- 'ecr:GetAuthorizationToken'
- 'ecr:BatchGetImage'
- 'ecr:GetDownloadUrlForLayer'
+ - 'ec2:AttachVolume'
+ - 'ec2:DescribeVolumes'
Resource: '*'
FileSystem:
Type: AWS::EFS::FileSystem
diff --git a/dual-master/cf-service-master.yml b/dual-master/cf-service-master.yml
index 6d8f49b..46f5ca5 100644
--- a/dual-master/cf-service-master.yml
+++ b/dual-master/cf-service-master.yml
@@ -194,13 +194,13 @@
Mappings:
Gerrit:
Volume:
- Git: gerrit-git-master
- Websessions: gerrit-ha-websessions-master
- Data: gerrit-data-master
- Index: gerrit-index-master
- Cache: gerrit-cache-master
- Db: gerrit-db-master
- Logs: gerrit-logs-master
+ Git: gerrit-git
+ Websessions: gerrit-ha-websessions
+ Data: gerrit-data
+ Index: gerrit-index
+ Cache: gerrit-cache
+ Db: gerrit-db
+ Logs: gerrit-logs
Service:
Name: gerrit-master
LoadBalancer:
@@ -343,15 +343,15 @@
ContainerPath: /var/gerrit/git
- SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Websessions']
ContainerPath: /var/gerrit/high-availability
- - SourceVolume: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Data'], !Ref 'GerritInstanceNumber']]
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Data']
ContainerPath: /var/gerrit/data
- - SourceVolume: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Index'], !Ref 'GerritInstanceNumber']]
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Index']
ContainerPath: /var/gerrit/index
- - SourceVolume: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Cache'], !Ref 'GerritInstanceNumber']]
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Cache']
ContainerPath: /var/gerrit/cache
- - SourceVolume: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Db'], !Ref 'GerritInstanceNumber']]
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Db']
ContainerPath: /var/gerrit/db
- - SourceVolume: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Logs'], !Ref 'GerritInstanceNumber']]
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Logs']
ContainerPath: /var/gerrit/logs
Cpu: !Ref GerritCPU
Memory: !Ref GerritRAM
@@ -375,42 +375,21 @@
- Name: !FindInMap ['Gerrit', 'Volume', 'Websessions']
Host:
SourcePath: "/mnt/efs/gerrit-shared/high-availability"
- - Name: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Db'], !Ref 'GerritInstanceNumber']]
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-db: !Join ['-', [!Ref EnvironmentName, !FindInMap ['Gerrit', 'Volume', 'Db'], !Ref 'GerritInstanceNumber']]
- - Name: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Data'], !Ref 'GerritInstanceNumber']]
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-data: !Join ['-', [!Ref EnvironmentName, !FindInMap ['Gerrit', 'Volume', 'Data'], !Ref 'GerritInstanceNumber']]
- - Name: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Cache'], !Ref 'GerritInstanceNumber']]
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-cache: !Join ['-', [!Ref EnvironmentName, !FindInMap ['Gerrit', 'Volume', 'Cache'], !Ref 'GerritInstanceNumber']]
- - Name: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Index'], !Ref 'GerritInstanceNumber']]
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-index: !Join ['-', [!Ref EnvironmentName, !FindInMap ['Gerrit', 'Volume', 'Index'], !Ref 'GerritInstanceNumber']]
- - Name: !Join ['-', [!FindInMap ['Gerrit', 'Volume', 'Logs'], !Ref 'GerritInstanceNumber']]
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-logs: !Join ['-', [!Ref EnvironmentName, !FindInMap ['Gerrit', 'Volume', 'Logs'], !Ref 'GerritInstanceNumber']]
-
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Db']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Db']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Data']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Data']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Cache']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Cache']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Index']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Index']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Logs']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Logs']]]
LoadBalancer:
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
diff --git a/single-master/Makefile b/single-master/Makefile
index dda38cc..0e1a4c4 100644
--- a/single-master/Makefile
+++ b/single-master/Makefile
@@ -19,7 +19,7 @@
service wait-for-service-creation \
dns-routing wait-for-dns-routing-creation
-cluster: cluster-keys
+cluster: cluster-keys set-optional-gerrit-master-volume
ifdef CLUSTER_INSTANCE_TYPE
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE))
endif
@@ -29,15 +29,6 @@
ifdef VPC_CIDR
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=VPCCIDR,ParameterValue=$(VPC_CIDR))
endif
-ifdef GERRIT_VOLUME_ID
- $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=GerritVolumeId,ParameterValue=$(GERRIT_VOLUME_ID))
-endif
-ifdef GERRIT_VOLUME_SNAPSHOT_ID
- $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=GerritVolumeSnapshotId,ParameterValue=$(GERRIT_VOLUME_SNAPSHOT_ID))
-endif
-ifdef GERRIT_VOLUME_SIZE_IN_GIB
- $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=GerritVolumeSizeInGiB,ParameterValue=$(GERRIT_VOLUME_SIZE_IN_GIB))
-endif
$(AWS_FC_COMMAND) create-stack \
--stack-name $(CLUSTER_STACK_NAME) \
@@ -50,7 +41,8 @@
ParameterKey=InternetGatewayIdProp,ParameterValue=$(INTERNET_GATEWAY_ID) \
ParameterKey=VPCIdProp,ParameterValue=$(VPC_ID) \
ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
- $(CLUSTER_OPTIONAL_PARAMS)
+ $(CLUSTER_OPTIONAL_PARAMS) \
+ $(GERRIT_OPTIONAL_MASTER_VOLUME)
service: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-ldap-account-pattern set-optional-gerrit-ulimits set-optional-jgit-conf
ifdef LOAD_BALANCER_SCHEME
