Allow master-slave to reuse existing EBS

Single-master and dual-master recipes have the ability to re-use an
existing EBS volume or snapshot to preserve data across deployment,
however master-slave recipe was never updated with such functionality
and it creates a new EBS volume at each deployment.

Integrate master-slave cloudformation template with the EBS volume stack
so that data can be preserved across deployments.

Feature: Issue 13880
Change-Id: Iec19168187537b5fb65c96f2d55ddd573cfafe02
diff --git a/master-slave/Makefile b/master-slave/Makefile
index 071378b..6e2b3d0 100644
--- a/master-slave/Makefile
+++ b/master-slave/Makefile
@@ -28,7 +28,7 @@
 						$(optional_dashboard_targets) \
 						dns-routing wait-for-dns-routing-creation
 
-cluster: cluster-keys
+cluster: cluster-keys set-optional-gerrit-master-volume
 ifdef CLUSTER_INSTANCE_TYPE
 		$(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE))
 endif
@@ -53,7 +53,8 @@
 		ParameterKey=InternetGatewayIdProp,ParameterValue=$(INTERNET_GATEWAY_ID) \
 		ParameterKey=VPCIdProp,ParameterValue=$(VPC_ID) \
 		ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \
-		$(CLUSTER_OPTIONAL_PARAMS)
+		$(CLUSTER_OPTIONAL_PARAMS) \
+		$(GERRIT_OPTIONAL_MASTER_VOLUME)
 
 service-master: set-optional-params-metrics-cloudwatch set-optional-params-smtp set-ldap-account-pattern set-optional-gerrit-ulimits set-optional-jgit-conf
 ifdef LOAD_BALANCER_SCHEME
diff --git a/master-slave/README.md b/master-slave/README.md
index f08f0f2..a288fae 100644
--- a/master-slave/README.md
+++ b/master-slave/README.md
@@ -80,6 +80,11 @@
 "gerrit-master-slave-MASTER" by default.
 * `GERRIT_SLAVE_INSTANCE_ID`: Optional. Identifier for the Gerrit slave instance.
 "gerrit-master-slave-SLAVE" by default.
+* `GERRIT_VOLUME_ID` : Optional. Id of an extisting EBS volume. If empty, a new volume
+for Gerrit data will be created
+* `GERRIT_VOLUME_SNAPSHOT_ID` : Optional. Ignored if GERRIT_VOLUME_ID is not empty. Id of
+the EBS volume snapshot used to create new EBS volume for Gerrit data.
+* `GERRIT_VOLUME_SIZE_IN_GIB`: Optional. The size of the Gerrit data volume, in GiBs. `10` by default.
 
 *NOTE*: if you are planning to run the monitoring stack, set the
 `MASTER_MAX_COUNT` value to at least 2. The resources provided by
diff --git a/master-slave/cf-cluster.yml b/master-slave/cf-cluster.yml
index e56e341..674727f 100644
--- a/master-slave/cf-cluster.yml
+++ b/master-slave/cf-cluster.yml
@@ -51,6 +51,27 @@
       Description: An environment name used to build the log stream names
       Type: String
       Default: test
+  GerritVolumeId:
+    Description: Existing Gerrit volume id
+    Type: String
+    Default: ""
+  GerritVolumeSnapshotId:
+    Description: Id of the EBS snapshot for Gerrit volume
+    Type: String
+    Default: ""
+  GerritVolumeAttachMaxRetries:
+    Description: Maximum number of retries when attaching Gerrit Volume
+    Type: Number
+    Default: 5
+  GerritVolumeAttachRetryDelay:
+    Description: The delay in seconds between Gerrit Volume attach attempts
+    Type: Number
+    Default: 5
+  GerritVolumeSizeInGiB:
+    Description: Gerrit volume size in GiB
+    Type: Number
+    Default: 10
+
 Resources:
   # ECS Resources
   ECSCluster:
@@ -103,7 +124,48 @@
           echo ECS_INSTANCE_ATTRIBUTES={\"target_group\":\"master\"} >> /etc/ecs/ecs.config
           # Make sure latest version of the helper scripts are installed as per recommendation:
           # https://github.com/awsdocs/aws-cloudformation-user-guide/blob/master/doc_source/cfn-helper-scripts-reference.md#using-the-latest-version
-          yum install -y aws-cfn-bootstrap wget
+          yum install -y aws-cfn-bootstrap wget aws-cli xfsprogs
+
+          # ATTACH EBS VOLUME
+          EC2_INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
+          VOLUME_ID=${GerritVolumeStack.Outputs.GerritVolumeRef}
+          echo "Volume Id: $VOLUME_ID"
+
+          aws ec2 attach-volume --region ${AWS::Region} --volume-id $VOLUME_ID --instance-id $EC2_INSTANCE_ID --device /dev/xvdg
+
+          for i in $(seq 1 ${GerritVolumeAttachMaxRetries}); do
+            echo "Waiting for volume $VOLUME_ID to be attached to the instace $EC2_INSTANCE_ID ($i/${GerritVolumeAttachMaxRetries})"
+            volumeStatus=`aws ec2 describe-volumes --region ${AWS::Region} --volume-ids $VOLUME_ID`
+            if [[ $volumeStatus =~ "\"State\": \"attached\"" ]]; then
+              echo "Volume $VOLUME_ID attached to the instace $EC2_INSTANCE_ID"
+              break
+            elif [[ "$i" -eq "${GerritVolumeAttachMaxRetries}" ]]; then
+              echo "Could not attach the volume $VOLUME_ID to the instace $EC2_INSTANCE_ID after ${GerritVolumeAttachMaxRetries} attempts"
+              exit 1
+            fi
+            sleep ${GerritVolumeAttachRetryDelay}
+          done
+
+          if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+            echo "Create file system for Gerrit volume"
+            mkfs -t xfs /dev/xvdg
+          fi
+
+          mkdir /gerrit-mount-point
+          mount /dev/xvdg /gerrit-mount-point
+
+          if [[ "${GerritVolumeId}" = "" && "${GerritVolumeSnapshotId}" = "" ]]; then
+            echo "Create Gerrit directories"
+            mkdir -p /gerrit-mount-point/gerrit-logs \
+             /gerrit-mount-point/gerrit-cache \
+             /gerrit-mount-point/gerrit-data \
+             /gerrit-mount-point/gerrit-git \
+             /gerrit-mount-point/gerrit-index \
+             /gerrit-mount-point/gerrit-db
+          fi
+
+          chown 1000:1000 -R /gerrit-mount-point
+
           # Get the CloudWatch Logs agent
           echo -e "
             {\"logs\":
@@ -111,27 +173,27 @@
                 {\"files\":
                   {\"collect_list\":
                     [
-                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/replication_log\",
+                      {\"file_path\": \"/gerrit-mount-point/gerrit-logs/replication_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/replication_log\",
                       \"timezone\": \"UTC\"
                       },
-                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/httpd_log\",
+                      {\"file_path\": \"/gerrit-mount-point/gerrit-logs/httpd_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/httpd_log\",
                       \"timezone\": \"UTC\"
                       },
-                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/sshd_log\",
+                      {\"file_path\": \"/gerrit-mount-point/gerrit-logs/sshd_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/sshd_log\",
                       \"timezone\": \"UTC\"
                       },
-                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/gc_log\",
+                      {\"file_path\": \"/gerrit-mount-point/gerrit-logs/gc_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/gc_log\",
                       \"timezone\": \"UTC\"
                       },
-                      {\"file_path\": \"/var/lib/docker/volumes/gerrit-logs-master/_data/audit_log\",
+                      {\"file_path\": \"/gerrit-mount-point/gerrit-logs/audit_log\",
                       \"log_group_name\": \"${AWS::StackName}\",
                       \"log_stream_name\": \"${EnvironmentName}/{instance_id}/master/audit_log\",
                       \"timezone\": \"UTC\"
@@ -263,6 +325,8 @@
               - 'ecr:GetAuthorizationToken'
               - 'ecr:BatchGetImage'
               - 'ecr:GetDownloadUrlForLayer'
+              - 'ec2:AttachVolume'
+              - 'ec2:DescribeVolumes'
             Resource: '*'
   ECSTaskNetworkStack:
     Type: AWS::CloudFormation::Stack
@@ -276,6 +340,15 @@
         SubnetIdProp: !Ref 'SubnetIdProp'
         SubnetCIDR: !Ref 'SubnetCIDR'
 
+  GerritVolumeStack:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-gerrit-volume.yml'] ]
+      TimeoutInMinutes: '25'
+      Parameters:
+        GerritVolumeId: !Ref 'GerritVolumeId'
+        GerritVolumeSnapshotId: !Ref 'GerritVolumeSnapshotId'
+        GerritVolumeSizeInGiB: !Ref 'GerritVolumeSizeInGiB'
 
 Outputs:
   ClusterName:
diff --git a/master-slave/cf-service-master.yml b/master-slave/cf-service-master.yml
index ba08b97..3c0cff4 100644
--- a/master-slave/cf-service-master.yml
+++ b/master-slave/cf-service-master.yml
@@ -74,30 +74,6 @@
   GerritKeyPrefix:
         Description: Gerrit credentials keys prefix
         Type: String
-  GerritGitVolume:
-      Description: Gerrit git volume name
-      Type: String
-      Default: gerrit-git-master
-  GerritDataVolume:
-      Description: Gerrit data volume name
-      Type: String
-      Default: gerrit-data-master
-  GerritIndexVolume:
-      Description: Gerrit index volume name
-      Type: String
-      Default: gerrit-index-master
-  GerritCacheVolume:
-      Description: Gerrit cache volume name
-      Type: String
-      Default: gerrit-cache-master
-  GerritDbVolume:
-      Description: Gerrit db volume name
-      Type: String
-      Default: gerrit-db-master
-  GerritLogsVolume:
-      Description: Gerrit logs volume name
-      Type: String
-      Default: gerrit-logs-master
   GerritRAM:
       Description: RAM to allocate to the Gerrit container
       Type: Number
@@ -201,6 +177,16 @@
     Type: CommaDelimitedList
     Default: ''
 
+Mappings:
+  Gerrit:
+    Volume:
+      Git: gerrit-git
+      Data: gerrit-data
+      Index: gerrit-index
+      Cache: gerrit-cache
+      Db: gerrit-db
+      Logs: gerrit-logs
+
 Resources:
     Service:
         Type: AWS::ECS::Service
@@ -305,17 +291,17 @@
                       HardLimit: !Ref FileDescriptorsHardLimit
                       SoftLimit: !Ref FileDescriptorsSoftLimit
                   MountPoints:
-                    - SourceVolume: !Ref GerritGitVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Git']
                       ContainerPath: /var/gerrit/git
-                    - SourceVolume: !Ref GerritDataVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Data']
                       ContainerPath: /var/gerrit/data
-                    - SourceVolume: !Ref GerritIndexVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Index']
                       ContainerPath: /var/gerrit/index
-                    - SourceVolume: !Ref GerritCacheVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Cache']
                       ContainerPath: /var/gerrit/cache
-                    - SourceVolume: !Ref GerritDbVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Db']
                       ContainerPath: /var/gerrit/db
-                    - SourceVolume: !Ref GerritLogsVolume
+                    - SourceVolume: !FindInMap ['Gerrit', 'Volume', 'Logs']
                       ContainerPath: /var/gerrit/logs
                   Cpu: !Ref GerritCPU
                   Memory: !Ref GerritRAM
@@ -333,49 +319,24 @@
                         awslogs-region: !Ref AWS::Region
                         awslogs-stream-prefix: !Ref EnvironmentName
             Volumes:
-              - Name: !Ref 'GerritDbVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-db: !Join ['-', [!Ref EnvironmentName, !Ref GerritDbVolume]]
-              - Name: !Ref 'GerritGitVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-git: !Join ['-', [!Ref EnvironmentName, !Ref GerritGitVolume]]
-              - Name: !Ref 'GerritDataVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-data: !Join ['-', [!Ref EnvironmentName, !Ref GerritDataVolume]]
-              - Name: !Ref 'GerritCacheVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-cache: !Join ['-', [!Ref EnvironmentName, !Ref GerritCacheVolume]]
-              - Name: !Ref 'GerritIndexVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-index: !Join ['-', [!Ref EnvironmentName, !Ref GerritIndexVolume]]
-              - Name: !Ref 'GerritLogsVolume'
-                DockerVolumeConfiguration:
-                  Scope: shared
-                  Autoprovision: true
-                  Driver: local
-                  Labels:
-                    gerrit-logs: !Join ['-', [!Ref EnvironmentName, !Ref GerritLogsVolume]]
-
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Db']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Db']]]
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Git']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Git']]]
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Data']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Data']]]
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Cache']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Cache']]]
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Index']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Index']]]
+              - Name: !FindInMap ['Gerrit', 'Volume', 'Logs']
+                Host:
+                  SourcePath: !Join ['/', ["/gerrit-mount-point", !FindInMap ['Gerrit', 'Volume', 'Logs']]]
 
     LoadBalancer:
         Type: AWS::ElasticLoadBalancingV2::LoadBalancer