dual-primary: share git data on replicas over EFS Store git data for replicas over EFS so that it can persist beyond the lifespan of a single instance. This is the first step in the direction of allowing replicas to scale. Feature: Issue 13619 Change-Id: I2c3e265577cfa35e8040a9858b628ce964d9657d
diff --git a/Makefile.common b/Makefile.common index e5d0d8f..4c166c5 100644 --- a/Makefile.common +++ b/Makefile.common
@@ -93,4 +93,16 @@ endif ifdef GERRIT_VOLUME_SIZE_IN_GIB $(eval GERRIT_OPTIONAL_PRIMARY_VOLUME := $(GERRIT_OPTIONAL_PRIMARY_VOLUME) ParameterKey=GerritVolumeSizeInGiB,ParameterValue=$(GERRIT_VOLUME_SIZE_IN_GIB)) +endif + +set-optional-params-for-replica-filesystem: + $(eval GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM=) +ifdef REPLICA_FILESYSTEM_ID + $(eval GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM := $(GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM) ParameterKey=ReplicaFileSystemID,ParameterValue=$(REPLICA_FILESYSTEM_ID)) +endif +ifdef REPLICA_FILESYSTEM_THROUGHPUT_MODE + $(eval GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM := $(GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM) ParameterKey=ReplicaFileSystemThroughputMode,ParameterValue=$(REPLICA_FILESYSTEM_THROUGHPUT_MODE)) +endif +ifdef REPLICA_FILESYSTEM_PROVISIONED_THROUGHPUT_IN_MIBPS + $(eval GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM := $(GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM) ParameterKey=ReplicaProvisionedThroughputInMibps,ParameterValue=$(REPLICA_FILESYSTEM_PROVISIONED_THROUGHPUT_IN_MIBPS)) endif \ No newline at end of file
diff --git a/common-templates/cf-efs-stack.yml b/common-templates/cf-efs-stack.yml index 3c465cc..643f987 100644 --- a/common-templates/cf-efs-stack.yml +++ b/common-templates/cf-efs-stack.yml
@@ -16,6 +16,9 @@ SecurityGroupCidrIp: Description: The IPv4 address range for the security group, in CIDR format Type: String + TagValue: + Description: A tag value for this EFS resource + Type: String Conditions: isProvisionedThroughput: !Equals [!Ref FileSystemThroughputMode, "provisioned"] @@ -28,7 +31,7 @@ ProvisionedThroughputInMibps: !If [isProvisionedThroughput, !Ref ProvisionedThroughputInMibps, !Ref "AWS::NoValue"] FileSystemTags: - Key: Name - Value: "multi-primary-git-repo" + Value: !Ref TagValue GitMountTarget: Type: AWS::EFS::MountTarget
diff --git a/dual-primary/Makefile b/dual-primary/Makefile index 7e2b6ab..309d997 100644 --- a/dual-primary/Makefile +++ b/dual-primary/Makefile
@@ -50,7 +50,7 @@ $(optional_git_gc_targets_creation) \ dns-routing wait-for-dns-routing-creation -cluster: cluster-keys set-optional-gerrit-primary-volume +cluster: cluster-keys set-optional-gerrit-primary-volume set-optional-params-for-replica-filesystem ifdef CLUSTER_INSTANCE_TYPE $(eval CLUSTER_OPTIONAL_PARAMS := $(CLUSTER_OPTIONAL_PARAMS) ParameterKey=InstanceType,ParameterValue=$(CLUSTER_INSTANCE_TYPE)) endif @@ -91,7 +91,8 @@ ParameterKey=VPCIdProp,ParameterValue=$(VPC_ID) \ ParameterKey=SubnetIdProp,ParameterValue=$(SUBNET_ID) \ $(CLUSTER_OPTIONAL_PARAMS) \ - $(GERRIT_OPTIONAL_PRIMARY_VOLUME) + $(GERRIT_OPTIONAL_PRIMARY_VOLUME) \ + $(GERRIT_OPTIONAL_PARAMS_REPLICA_FILESYSTEM) service-primary-1: set-optional-params-metrics-cloudwatch set-optional-params-smtp \ set-optional-params-multisite set-ldap-account-pattern \ @@ -494,7 +495,7 @@ @echo "" @echo -n "Are you sure you want to continue? [y/N] " && read ans && [ $${ans:-N} = y ] -delete-all-including-retained-stack: confirm-persistent-stack-deletion delete-all delete-git-persistent-stack delete-network-persistent-stack +delete-all-including-retained-stack: confirm-persistent-stack-deletion delete-all delete-git-persistent-stack delete-git-replica-persistent-stack delete-network-persistent-stack delete-git-persistent-stack: @@ -513,6 +514,23 @@ echo "No Git persistent stack found. Nothing to do." \ ) +delete-git-replica-persistent-stack: + + $(eval REPLICA_EFS_STACK_NAME := $(shell $(AWS_FC_COMMAND) list-stacks --stack-status-filter CREATE_COMPLETE --query "StackSummaries[*].StackName" | jq -r '.[]| select(startswith("$(CLUSTER_STACK_NAME)-ReplicaGitFileSystemPermanentStack"))')) + + $(if $(REPLICA_EFS_STACK_NAME), \ + $(AWS_FC_COMMAND) delete-stack \ + --stack-name $(REPLICA_EFS_STACK_NAME) \ + --region $(AWS_REGION) && \ + echo "*** Wait for Git persistent stack of replicas '$(REPLICA_EFS_STACK_NAME)' deletion" && \ + $(AWS_FC_COMMAND) wait stack-delete-complete \ + --stack-name $(REPLICA_EFS_STACK_NAME) \ + --region $(AWS_REGION) && \ + echo "*** Git persistent stack '$(EFSREPLICA_EFS_STACK_NAME_STACK_NAME)' deleted" \ + , \ + echo "No Git persistent stack for replicas found. Nothing to do." \ + ) + delete-network-persistent-stack: $(eval NETWORK_STACK_NAME=$(shell $(AWS_FC_COMMAND) list-stacks --stack-status-filter CREATE_COMPLETE --query "StackSummaries[*].StackName" | jq -r '.[]| select(startswith("$(CLUSTER_STACK_NAME)-ECSTaskNetworkStack"))'))
diff --git a/dual-primary/README.md b/dual-primary/README.md index 767bdab..7c3e989 100644 --- a/dual-primary/README.md +++ b/dual-primary/README.md
@@ -221,6 +221,27 @@ Default: `10m` high-availability docs [here](https://gerrit.googlesource.com/plugins/high-availability/+/refs/heads/master/src/main/resources/Documentation/config.md) +##### Shared filesystem for replicas + +Similarly to primary nodes, replicas share a data via an EFS filesystem which is +mounted under the `/var/gerrit/git` directory. This allows git data to persist +beyond the lifespan of a single instance and to be shared so that replicas can +scale down and up according to needs. + +* `REPLICA_FILESYSTEM_ID`: Optional. An existing EFS filesystem id to mount on replicas. + + If empty, a new EFS will be created to store git data. + Setting this value is required when deploying a dual-primary cluster using + existing data as well as performing blue/green deployments. + The nested stack will be *retained* when the cluster is deleted, so that + existing data can be used to perform blue/green deployments. + +* `REPLICA_FILESYSTEM_THROUGHPUT_MODE`: Optional. The throughput mode for the file system to be created. +default: `bursting`. More info [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-efs-filesystem.html) + +* `REPLICA_FILESYSTEM_PROVISIONED_THROUGHPUT_IN_MIBPS`: Optional. Only used when `REPLICA_FILESYSTEM_THROUGHPUT_MODE` is set to `provisioned`. +default: `256`. + #### REPLICATION SERVICE * `REPLICATION_SERVICE_ENABLED`: Optional. Whether to expose a replication endpoint.
diff --git a/dual-primary/cf-cluster.yml b/dual-primary/cf-cluster.yml index 22d0594..7923dc1 100644 --- a/dual-primary/cf-cluster.yml +++ b/dual-primary/cf-cluster.yml
@@ -93,10 +93,24 @@ Description: Gerrit volume size in GiB Type: Number Default: 10 + ReplicaFileSystemID: + Description: Gerrit replicas shared filesystem id + Type: String + Default: "" + ReplicaFileSystemThroughputMode: + Description: Gerrit replicas shared filesystem throughput mode + Type: String + Default: bursting + AllowedValues: [bursting, provisioned] + ReplicaProvisionedThroughputInMibps: + Description: Gerrit replicas shared filesystem throughput, measured in MiB/s. Valid values are 1-1024. + Type: Number + Default: 256 Conditions: isProvisionedThroughput: !Equals [!Ref FileSystemThroughputMode, "provisioned"] CreateEFS: !Equals [!Ref FileSystemID, ""] + CreateReplicaEFS: !Equals [!Ref ReplicaFileSystemID, ""] NetworkStackNeeded: !Or - !Equals [!Ref VPCIdProp, ""] - !And @@ -308,6 +322,22 @@ PublicSubnet: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef, !Ref SubnetIdProp] SecurityGroupVPCID: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.VPCRef, !Ref VPCIdProp] SecurityGroupCidrIp: !Ref SubnetCIDR + TagValue: "efs-for-gerrit-primaries" + + ReplicaGitFileSystemPermanentStack: + Type: AWS::CloudFormation::Stack + DeletionPolicy: Retain + Condition: CreateReplicaEFS + Properties: + TemplateURL: !Join [ '', ['https://', !Ref TemplateBucketName, '.s3.amazonaws.com/cf-efs-stack.yml'] ] + TimeoutInMinutes: '25' + Parameters: + FileSystemThroughputMode: !Ref ReplicaFileSystemThroughputMode + ProvisionedThroughputInMibps: !Ref ReplicaProvisionedThroughputInMibps + PublicSubnet: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.PublicSubnetOneRef, !Ref SubnetIdProp] + SecurityGroupVPCID: !If [NetworkStackNeeded, !GetAtt ECSTaskNetworkStack.Outputs.VPCRef, !Ref VPCIdProp] + SecurityGroupCidrIp: !Ref SubnetCIDR + TagValue: "efs-for-gerrit-replicas" ECSTaskNetworkStack: Type: AWS::CloudFormation::Stack @@ -344,3 +374,8 @@ Value: !GetAtt ECSCluster.Arn Export: Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ClusterArn' ] ] + ReplicaFileSystemID: + Description: The ID of the EFS to be mounted by replicas + Value: !If [CreateReplicaEFS, !GetAtt ReplicaGitFileSystemPermanentStack.Outputs.FileSystemID, !Ref ReplicaFileSystemID ] + Export: + Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ReplicaFileSystemID' ] ]
diff --git a/dual-primary/cf-service-replica.yml b/dual-primary/cf-service-replica.yml index 58ed57d..ff59a9a 100644 --- a/dual-primary/cf-service-replica.yml +++ b/dual-primary/cf-service-replica.yml
@@ -359,12 +359,10 @@ Labels: gerrit-db: !Join ['-', [!Ref EnvironmentName, !Ref GerritDbVolume]] - Name: !Ref 'GerritGitVolume' - DockerVolumeConfiguration: - Scope: shared - Autoprovision: true - Driver: local - Labels: - gerrit-git: !Join ['-', [!Ref EnvironmentName, !Ref GerritGitVolume]] + EFSVolumeConfiguration: + FilesystemId: + Fn::ImportValue: + !Join [':', [!Ref 'ClusterStackName', 'ReplicaFileSystemID']] - Name: !Ref 'GerritDataVolume' DockerVolumeConfiguration: Scope: shared