Add export-logs operation for all recipes

Allow to export logs to S3 by issuing the command:

```
make eport-logs
```

Bug: Issue 14412
Change-Id: I1574c6969bdef307aae3c88d1cc32063f2fbff84
diff --git a/Makefile.common b/Makefile.common
index 47c08a5..a5be73a 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -1,3 +1,5 @@
+include ../operations/export-logs/Makefile
+
 SHELL := /bin/bash
 
 ROOT_DIR=$(dir $(realpath $(filter %Makefile.common,$(MAKEFILE_LIST))))
diff --git a/README.md b/README.md
index 84d0f44..ff94659 100644
--- a/README.md
+++ b/README.md
@@ -97,4 +97,9 @@
 test/gerrit-primary/bb21cb504ca44150b770ca05e922e332
 ```
 
-The task name can be found in the Amazon ECS console's `Task` section.
\ No newline at end of file
+The task name can be found in the Amazon ECS console's `Task` section.
+
+## Operations
+
+A set of utilities to perform operational tasks is also provided.
+Refer to the relevant [documentation](./operations/Operations.md) for details on this.
\ No newline at end of file
diff --git a/common.env b/common.env
index dc77a70..225f182 100644
--- a/common.env
+++ b/common.env
@@ -34,3 +34,6 @@
 # Elastic Container Service
 CLUSTER_STACK_NAME:=$(AWS_PREFIX)-cluster
 
+# S3 export logs
+S3_EXPORT_LOGS_BUCKET_NAME:=$(AWS_PREFIX)-s3-export-logs
+
diff --git a/operations/Operations.md b/operations/Operations.md
new file mode 100644
index 0000000..35d96e0
--- /dev/null
+++ b/operations/Operations.md
@@ -0,0 +1,66 @@
+## Operations
+
+#### Export logs to S3
+
+All logs, for all recipes are streamed to [cloudwatch logs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html)
+and can be accessed via the cloudwatch console.
+
+In some occasions however it is useful to export logs so that they
+can be shared, analyzed, manipulated outside the AWS environment.
+
+To do so, a make rule is provided for all recipes, to export logs
+to an S3 bucket.
+
+```bash
+make \
+  [AWS_REGION=<region>] \
+  [AWS_PREFIX=<prefix>] \
+  [EXPORT_FROM_SECONDS=<epoch_secs>] \
+  [S3_EXPORT_LOGS_BUCKET_NAME=<bucket>] \
+  export-logs
+```
+
+*`AWS_REGION`: Optional. Defaults to the value set in your [common.env](../common.env)
+*`AWS_PREFIX`: Optional. Defaults to the value set in your [common.env](../common.env)
+*`EXPORT_FROM_SECONDS`. Optional. The start time of the range for the request,
+  expressed as the number of seconds after Jan 1, 1970 00:00:00 UTC.
+*`S3_EXPORT_LOGS_BUCKET_NAME`: Optional. Defaults to `$(AWS_PREFIX)-s3-export-logs`
+
+Note: this command assumes that the bucket already exists and that is configured
+with the relevant policy allowing cloudwatch to export logs into it (see
+[permissions](#permissions)) for more information on how to set this up.
+
+Alternatively you can create, setup the bucket and export the logs in one command.
+
+```bash
+make \
+  [AWS_REGION=<region>] \
+  [AWS_PREFIX=<prefix>] \
+  [EXPORT_FROM_SECONDS=<epoch_secs>] \
+  [S3_EXPORT_LOGS_BUCKET_NAME=<bucket>] \
+  setup-bucket-and-export-logs
+```
+
+By default the created bucket will allow all items in the bucket to be public
+by default. If this is not what you want you should create your bucket through
+other means or update its policy after it has been created.
+
+The output of this command provides the full url of the main exported logs.
+Namely `sshd_log`, `httpd_log` and `error_log`.
+
+For example:
+
+```
+***************LOG URLS **********************
+https://export_bucket.s3.amazonaws.com/gerrit-export-logs-ts-1619182385/f9bac8f3-65a0-4334-8062-6fcdf62f4c59/test-i-0832d22ba2efbf450-sshd_log/000000.gz
+https://export_bucket.s3.amazonaws.com/gerrit-export-logs-ts-1619182385/f9bac8f3-65a0-4334-8062-6fcdf62f4c59/test-i-08cdd41be6c9bc623-error_log/000000.gz
+https://export_bucket.s3.amazonaws.com/gerrit-export-logs-ts-1619182385/f9bac8f3-65a0-4334-8062-6fcdf62f4c59/test-i-08cdd41be6c9bc623-httpd_log/000000.gz
+**********************************************
+```
+
+##### Permissions
+In order to execute the command, the issuing profile needs to have
+the ability to have full access to S3 and to CloudWatch, as detailed
+in the official [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/S3ExportTasks.html)
+
+
diff --git a/operations/export-logs/Makefile b/operations/export-logs/Makefile
new file mode 100644
index 0000000..d7abdb5
--- /dev/null
+++ b/operations/export-logs/Makefile
@@ -0,0 +1,59 @@
+now = $(shell date +%s)
+destination-prefix-name = gerrit-export-logs-ts-$(now)
+EXPORT_FROM_SECONDS ?= 0000000000
+
+create-s3-export-logs-bucket:
+	$(eval CREATE_S3_EXPORT_PARAMS_LOGS_PARAMS := --bucket $(S3_EXPORT_LOGS_BUCKET_NAME))
+ifneq ("$(AWS_REGION)", "us-east-1")
+		$(eval CREATE_S3_EXPORT_PARAMS_LOGS_PARAMS := $(CREATE_S3_EXPORT_PARAMS_LOGS_PARAMS) --create-bucket-configuration LocationConstraint=$(AWS_REGION))
+endif
+	@echo "*** Create bucket $(S3_EXPORT_LOGS_BUCKET_NAME)"
+	$(AWS) s3api create-bucket $(CREATE_S3_EXPORT_PARAMS_LOGS_PARAMS)
+
+set-bucket-permissions:
+	@echo "*** Set permissions to bucket $(S3_EXPORT_LOGS_BUCKET_NAME)"
+	cat ../operations/export-logs/s3.bucket.permissions.yaml | \
+		AWS_REGION=$(AWS_REGION) EXPORT_LOGS_BUCKET=$(S3_EXPORT_LOGS_BUCKET_NAME) \
+		 envsubst > /tmp/$(S3_EXPORT_LOGS_BUCKET_NAME).policy.yaml
+
+	$(AWS) s3api put-bucket-policy --bucket $(S3_EXPORT_LOGS_BUCKET_NAME) \
+		--policy file:///tmp/$(S3_EXPORT_LOGS_BUCKET_NAME).policy.yaml
+
+	$(AWS) s3api put-bucket-ownership-controls --bucket $(S3_EXPORT_LOGS_BUCKET_NAME) \
+		--ownership-controls '{ "Rules": [ { "ObjectOwnership": "BucketOwnerPreferred" } ] }'
+
+setup-s3-bucket: create-s3-export-logs-bucket set-bucket-permissions
+
+launch-export-task:
+	$(eval DESTINATION_PREFIX := $(destination-prefix-name))
+	$(eval TASK_ID := $(shell $(AWS) logs create-export-task --task-name $(DESTINATION_PREFIX) \
+		--log-group-name $(CLUSTER_STACK_NAME) \
+		--destination $(S3_EXPORT_LOGS_BUCKET_NAME) \
+		--from $(EXPORT_FROM_SECONDS)000 --to $(now)000 \
+		--destination-prefix $(DESTINATION_PREFIX) | jq -r '.taskId'))
+
+	@echo "Launched export task id $(TASK_ID) from $(EXPORT_FROM_SECONDS)000 to $(now)000"
+
+wait_for_export:
+	while [[ $$(aws logs describe-export-tasks --task-id "$(TASK_ID)" | jq -r '.exportTasks[0].status.code') =~ RUNNING|PENDING|PENDING_CANCEL ]]; do \
+  		echo "Wait for task $(DESTINATION_PREFIX) (id: $(TASK_ID)) to complete"; \
+  		sleep 5; \
+  	done;
+
+	@echo "Export logs to $(S3_EXPORT_LOGS_BUCKET_NAME)/$(DESTINATION_PREFIX) terminated."
+	$(AWS) logs describe-export-tasks --task-id "$(TASK_ID)" | jq -r '.exportTasks[0].status.code'
+
+output_log_urls:
+	@echo
+	@echo "*************** MAIN LOGS URLS **********************"
+	@for i in \
+		$(shell aws s3api list-objects --bucket $(S3_EXPORT_LOGS_BUCKET_NAME) \
+			--prefix "$(DESTINATION_PREFIX)/$(TASK_ID)" | \
+			 jq -r '.Contents[] | select(.Key|test("httpd_log|sshd_log|error_log")) | .Key'); do \
+		  echo https://$(S3_EXPORT_LOGS_BUCKET_NAME).s3.amazonaws.com/$$i; \
+		done;
+	@echo "**********************************************"
+	@echo
+
+export-logs: launch-export-task wait_for_export output_log_urls
+setup-bucket-and-export-logs: setup-s3-bucket export-logs
\ No newline at end of file
diff --git a/operations/export-logs/s3.bucket.permissions.yaml b/operations/export-logs/s3.bucket.permissions.yaml
new file mode 100644
index 0000000..4d33b17
--- /dev/null
+++ b/operations/export-logs/s3.bucket.permissions.yaml
@@ -0,0 +1,24 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "s3:GetBucketAcl",
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::$EXPORT_LOGS_BUCKET",
+      "Principal": { "Service": "logs.$AWS_REGION.amazonaws.com" }
+    },
+    {
+      "Action": "s3:PutObject" ,
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::$EXPORT_LOGS_BUCKET/*",
+      "Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" } },
+      "Principal": { "Service": "logs.$AWS_REGION.amazonaws.com" }
+    },
+    {
+        "Action": "s3:GetObject",
+        "Effect": "Allow",
+        "Resource": "arn:aws:s3:::$EXPORT_LOGS_BUCKET/*",
+        "Principal": "*"
+    }
+  ]
+}
\ No newline at end of file