Allow master-slave to reuse existing EBS
Single-master and dual-master recipes have the ability to re-use an
existing EBS volume or snapshot to preserve data across deployment,
however master-slave recipe was never updated with such functionality
and it creates a new EBS volume at each deployment.
Integrate master-slave cloudformation template with the EBS volume stack
so that data can be preserved across deployments.
Feature: Issue 13880
Change-Id: Iec19168187537b5fb65c96f2d55ddd573cfafe02
diff --git a/master-slave/Makefile b/master-slave/Makefile
index 071378b..6e2b3d0 100644
--- a/master-slave/Makefile
+++ b/master-slave/Makefile
@@ -28,7 +28,7 @@
$(optional_dashboard_targets) \
dns-routing wait-for-dns-routing-creation
-cluster: cluster-keys
+cluster: cluster-keys set-optional-gerrit-master-volume
ifdef CLUSTER_INSTANCE_TYPE
$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE))
endif
@@ -53,7 +53,8 @@
ParameterKey=InternetGatewayIdProp,ParameterValue=$(INTERNET_GATEWAY_ID) \
ParameterKey=VPCIdProp,ParameterValue=$(VPC_ID) \
ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
- $(CLUSTER_OPTIONAL_PARAMS)
+ $(CLUSTER_OPTIONAL_PARAMS) \
+ $(GERRIT_OPTIONAL_MASTER_VOLUME)
service-master: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-ldap-account-pattern set-optional-gerrit-ulimits set-optional-jgit-conf
ifdef LOAD_BALANCER_SCHEME
diff --git a/master-slave/README.md b/master-slave/README.md
index f08f0f2..a288fae 100644
--- a/master-slave/README.md
+++ b/master-slave/README.md
@@ -80,6 +80,11 @@
"gerrit-master-slave-MASTER" by default.
* `GERRIT_SLAVE_INSTANCE_ID`: Optional. Identifier for the Gerrit slave instance.
"gerrit-master-slave-SLAVE" by default.
+* `GERRIT_VOLUME_ID` : Optional. Id of an extisting EBS volume. If empty, a new volume
+for Gerrit data will be created
+* `GERRIT_VOLUME_SNAPSHOT_ID` : Optional. Ignored if GERRIT_VOLUME_ID is not empty. Id of
+the EBS volume snapshot used to create new EBS volume for Gerrit data.
+* `GERRIT_VOLUME_SIZE_IN_GIB`: Optional. The size of the Gerrit data volume, in GiBs. `10` by default.
*NOTE*: if you are planning to run the monitoring stack, set the
`MASTER_MAX_COUNT` value to at least 2. The resources provided by
diff --git a/master-slave/cf-cluster.yml b/master-slave/cf-cluster.yml
index e56e341..674727f 100644
--- a/master-slave/cf-cluster.yml
+++ b/master-slave/cf-cluster.yml
@@ -51,6 +51,27 @@
Description: An environment name used to build the log stream names
Type: String
Default: test
+ GerritVolumeId:
+ Description: Existing Gerrit volume id
+ Type: String
+ Default: ""
+ GerritVolumeSnapshotId:
+ Description: Id of the EBS snapshot for Gerrit volume
+ Type: String
+ Default: ""
+ GerritVolumeAttachMaxRetries:
+ Description: Maximum number of retries when attaching Gerrit Volume
+ Type: Number
+ Default: 5
+ GerritVolumeAttachRetryDelay:
+ Description: The delay in seconds between Gerrit Volume attach attempts
+ Type: Number
+ Default: 5
+ GerritVolumeSizeInGiB:
+ Description: Gerrit volume size in GiB
+ Type: Number
+ Default: 10
+
Resources:
# ECS Resources
ECSCluster:
@@ -103,7 +124,48 @@
echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"master\"} >> /etc/ecs/ecs.config
# Make sure latest version of the helper scripts are installed as per recommendation:
# https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
- yum install -y aws-cfn-bootstrap wget
+ yum install -y aws-cfn-bootstrap wget aws-cli xfsprogs
+
+ # ATTACH EBS VOLUME
+ EC2_INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
+ VOLUME_ID=${GerritVolumeStack.Outputs.GerritVolumeRef}
+ echo "Volume Id: $VOLUME_ID"
+
+ aws ec2 attach-volume --region ${AWS::Region} --volume-id $VOLUME_ID --instance-id $EC2_INSTANCE_ID --device /dev/xvdg
+
+ for i in $(seq 1 ${GerritVolumeAttachMaxRetries}); do
+ echo "Waiting for volume $VOLUME_ID to be attached to the instace $EC2_INSTANCE_ID ($i/${GerritVolumeAttachMaxRetries})"
+ volumeStatus=`aws ec2 describe-volumes --region ${AWS::Region} --volume-ids $VOLUME_ID`
+ if [[ $volumeStatus =~ "\"State\": \"attached\"" ]]; then
+ echo "Volume $VOLUME_ID attached to the instace $EC2_INSTANCE_ID"
+ break
+ elif [[ "$i" -eq "${GerritVolumeAttachMaxRetries}" ]]; then
+ echo "Could not attach the volume $VOLUME_ID to the instace $EC2_INSTANCE_ID after ${GerritVolumeAttachMaxRetries} attempts"
+ exit 1
+ fi
+ sleep ${GerritVolumeAttachRetryDelay}
+ done
+
+ if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+ echo "Create file system for Gerrit volume"
+ mkfs -t xfs /dev/xvdg
+ fi
+
+ mkdir /gerrit-mount-point
+ mount /dev/xvdg /gerrit-mount-point
+
+ if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+ echo "Create Gerrit directories"
+ mkdir -p /gerrit-mount-point/gerrit-logs \
+ /gerrit-mount-point/gerrit-cache \
+ /gerrit-mount-point/gerrit-data \
+ /gerrit-mount-point/gerrit-git \
+ /gerrit-mount-point/gerrit-index \
+ /gerrit-mount-point/gerrit-db
+ fi
+
+ chown 1000:1000 -R /gerrit-mount-point
+
# Get the CloudWatch Logs agent
echo -e "
{\"logs\":
@@ -111,27 +173,27 @@
{\"files\":
{\"collect_list\":
[
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/replication_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/replication_log\",
\"log_group_name\": \"${AWS::StackName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/replication_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/httpd_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/httpd_log\",
\"log_group_name\": \"${AWS::StackName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/httpd_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/sshd_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/sshd_log\",
\"log_group_name\": \"${AWS::StackName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/sshd_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/gc_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/gc_log\",
\"log_group_name\": \"${AWS::StackName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/gc_log\",
\"timezone\": \"UTC\"
},
- {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/audit_log\",
+ {\"file_path\": \"/gerrit-mount-point/gerrit-logs/audit_log\",
\"log_group_name\": \"${AWS::StackName}\",
\"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/audit_log\",
\"timezone\": \"UTC\"
@@ -263,6 +325,8 @@
- 'ecr:GetAuthorizationToken'
- 'ecr:BatchGetImage'
- 'ecr:GetDownloadUrlForLayer'
+ - 'ec2:AttachVolume'
+ - 'ec2:DescribeVolumes'
Resource: '*'
ECSTaskNetworkStack:
Type: AWS::CloudFormation::Stack
@@ -276,6 +340,15 @@
SubnetIdProp: !Ref 'SubnetIdProp'
SubnetCIDR: !Ref 'SubnetCIDR'
+ GerritVolumeStack:
+ Type: AWS::CloudFormation::Stack
+ Properties:
+ TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-gerrit-volume.yml'] ]
+ TimeoutInMinutes: '25'
+ Parameters:
+ GerritVolumeId: !Ref 'GerritVolumeId'
+ GerritVolumeSnapshotId: !Ref 'GerritVolumeSnapshotId'
+ GerritVolumeSizeInGiB: !Ref 'GerritVolumeSizeInGiB'
Outputs:
ClusterName:
diff --git a/master-slave/cf-service-master.yml b/master-slave/cf-service-master.yml
index ba08b97..3c0cff4 100644
--- a/master-slave/cf-service-master.yml
+++ b/master-slave/cf-service-master.yml
@@ -74,30 +74,6 @@
GerritKeyPrefix:
Description: Gerrit credentials keys prefix
Type: String
- GerritGitVolume:
- Description: Gerrit git volume name
- Type: String
- Default: gerrit-git-master
- GerritDataVolume:
- Description: Gerrit data volume name
- Type: String
- Default: gerrit-data-master
- GerritIndexVolume:
- Description: Gerrit index volume name
- Type: String
- Default: gerrit-index-master
- GerritCacheVolume:
- Description: Gerrit cache volume name
- Type: String
- Default: gerrit-cache-master
- GerritDbVolume:
- Description: Gerrit db volume name
- Type: String
- Default: gerrit-db-master
- GerritLogsVolume:
- Description: Gerrit logs volume name
- Type: String
- Default: gerrit-logs-master
GerritRAM:
Description: RAM to allocate to the Gerrit container
Type: Number
@@ -201,6 +177,16 @@
Type: CommaDelimitedList
Default: ''
+Mappings:
+ Gerrit:
+ Volume:
+ Git: gerrit-git
+ Data: gerrit-data
+ Index: gerrit-index
+ Cache: gerrit-cache
+ Db: gerrit-db
+ Logs: gerrit-logs
+
Resources:
Service:
Type: AWS::ECS::Service
@@ -305,17 +291,17 @@
HardLimit: !Ref FileDescriptorsHardLimit
SoftLimit: !Ref FileDescriptorsSoftLimit
MountPoints:
- - SourceVolume: !Ref GerritGitVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Git']
ContainerPath: /var/gerrit/git
- - SourceVolume: !Ref GerritDataVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Data']
ContainerPath: /var/gerrit/data
- - SourceVolume: !Ref GerritIndexVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Index']
ContainerPath: /var/gerrit/index
- - SourceVolume: !Ref GerritCacheVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Cache']
ContainerPath: /var/gerrit/cache
- - SourceVolume: !Ref GerritDbVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Db']
ContainerPath: /var/gerrit/db
- - SourceVolume: !Ref GerritLogsVolume
+ - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Logs']
ContainerPath: /var/gerrit/logs
Cpu: !Ref GerritCPU
Memory: !Ref GerritRAM
@@ -333,49 +319,24 @@
awslogs-region: !Ref AWS::Region
awslogs-stream-prefix: !Ref EnvironmentName
Volumes:
- - Name: !Ref 'GerritDbVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-db: !Join ['-', [!Ref EnvironmentName, !Ref GerritDbVolume]]
- - Name: !Ref 'GerritGitVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-git: !Join ['-', [!Ref EnvironmentName, !Ref GerritGitVolume]]
- - Name: !Ref 'GerritDataVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-data: !Join ['-', [!Ref EnvironmentName, !Ref GerritDataVolume]]
- - Name: !Ref 'GerritCacheVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-cache: !Join ['-', [!Ref EnvironmentName, !Ref GerritCacheVolume]]
- - Name: !Ref 'GerritIndexVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-index: !Join ['-', [!Ref EnvironmentName, !Ref GerritIndexVolume]]
- - Name: !Ref 'GerritLogsVolume'
- DockerVolumeConfiguration:
- Scope: shared
- Autoprovision: true
- Driver: local
- Labels:
- gerrit-logs: !Join ['-', [!Ref EnvironmentName, !Ref GerritLogsVolume]]
-
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Db']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Db']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Git']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Git']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Data']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Data']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Cache']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Cache']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Index']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Index']]]
+ - Name: !FindInMap ['Gerrit', 'Volume', 'Logs']
+ Host:
+ SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Logs']]]
LoadBalancer:
Type: AWS::ElasticLoadBalancingV2::LoadBalancer