Add gcs-upload role
This role can be used to upload build artifacts to GCS.
Change-Id: Id7b58e168e45503a04a7f3cbfd41e237e2594c5b
diff --git a/roles/gcs-upload/README.rst b/roles/gcs-upload/README.rst
new file mode 100644
index 0000000..7ed6967
--- /dev/null
+++ b/roles/gcs-upload/README.rst
@@ -0,0 +1,31 @@
+Upload files to GCS
+
+ container: "{{ gcs_upload_container }}"
+ credentials_file: "{{ gcs_upload_credentials_file }}"
+ project: "{{ gcs_upload_project }}"
+ root: "{{ gcs_upload_root }}"
+ prefix: "{{ gcs_upload_prefix }}"
+
+**Role Variables**
+
+.. zuul:rolevar:: gcs_upload_container
+
+ The name of the container to upload to.
+
+.. zuul:rolevar:: gcs_upload_credentials_file
+ :default: /authdaemon/token
+
+ The token file to use for authentication.
+
+.. zuul:rolevar:: gcs_upload_project
+
+ The project name to use with the auth token.
+
+.. zuul:rolevar:: gcs_upload_prefix
+ :default: ''
+
+ A path prefix to add before each file.
+
+.. zuul:rolevar:: gcs_upload_root
+
+ The root of the directory to upload.
diff --git a/roles/gcs-upload/defaults/main.yaml b/roles/gcs-upload/defaults/main.yaml
new file mode 100644
index 0000000..9f1004e
--- /dev/null
+++ b/roles/gcs-upload/defaults/main.yaml
@@ -0,0 +1,3 @@
+gcs_upload_credentials_file: /authdaemon/token
+gcs_upload_prefix: ''
+gcs_upload_cache_control: null
diff --git a/roles/gcs-upload/library/gcs_upload.py b/roles/gcs-upload/library/gcs_upload.py
new file mode 100644
index 0000000..a33e56a
--- /dev/null
+++ b/roles/gcs-upload/library/gcs_upload.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+#
+# Copyright 2014 Rackspace Australia
+# Copyright 2018-2019 Red Hat, Inc
+# Copyright 2021 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# Make coding more python3-ish
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+"""
+Utility to upload files to google
+Run this from the CLI from the zuul/jobs/roles directory with:
+ python -m gcs-upload.library.gcs_upload
+"""
+
+import argparse
+import datetime
+import json
+import logging
+import os
+import time
+try:
+ import queue as queuelib
+except ImportError:
+ import Queue as queuelib
+import sys
+import threading
+
+from google.cloud import storage
+import google.auth.compute_engine.credentials as gce_cred
+import mimetypes
+
+from ansible.module_utils.basic import AnsibleModule
+
+POST_ATTEMPTS = 3
+MAX_UPLOAD_THREADS = 24
+
+
+def retry_function(func):
+ for attempt in range(1, POST_ATTEMPTS + 1):
+ try:
+ return func()
+ except Exception:
+ if attempt >= POST_ATTEMPTS:
+ raise
+ else:
+ logging.exception("Error on attempt %d" % attempt)
+ time.sleep(attempt * 10)
+
+
+class Credentials(gce_cred.Credentials):
+ def _set_path(self, path):
+ """Call this after initialization"""
+ self._path = path
+ self.refresh(None)
+
+ def refresh(self, request):
+ with open(self._path) as f:
+ data = json.loads(f.read())
+ self.token = data['access_token']
+ self.expiry = (datetime.datetime.utcnow() +
+ datetime.timedelta(seconds=data['expires_in']))
+
+ def with_scopes(self, *args, **kw):
+ ret = super(Credentials, self).with_scopes(*args, **kw)
+ ret._set_path(self._path)
+ return ret
+
+
+class Uploader():
+ def __init__(self, client, container, prefix, cache_control):
+ self.client = client
+ self.prefix = prefix or ''
+ if self.prefix and not self.prefix.endswith('/'):
+ self.prefix += '/'
+ self.cache_control = cache_control
+ self.bucket = client.bucket(container)
+
+ def upload(self, file_list):
+ """Spin up thread pool to upload to storage"""
+ num_threads = min(len(file_list), MAX_UPLOAD_THREADS)
+ threads = []
+ queue = queuelib.Queue()
+ # add items to queue
+ for f in file_list:
+ queue.put(f)
+
+ for x in range(num_threads):
+ t = threading.Thread(target=self.post_thread, args=(queue,))
+ threads.append(t)
+ t.start()
+
+ for t in threads:
+ t.join()
+
+ def post_thread(self, queue):
+ while True:
+ try:
+ file_detail = queue.get_nowait()
+ logging.debug("%s: processing job %s",
+ threading.current_thread(),
+ file_detail)
+ retry_function(lambda: self._post_file(file_detail))
+ except IOError:
+ # Do our best to attempt to upload all the files
+ logging.exception("Error opening file")
+ continue
+ except queuelib.Empty:
+ # No more work to do
+ return
+
+ def _post_file(self, file_detail):
+ full_path, relative_path = file_detail
+ relative_path = self.prefix + relative_path
+ data = open(full_path, 'rb')
+ blob = self.bucket.blob(relative_path)
+ if self.cache_control:
+ blob.cache_control = self.cache_control
+ mime_guess, encoding = mimetypes.guess_type(full_path)
+ mimetype = mime_guess if mime_guess else 'application/octet-stream'
+ blob.upload_from_file(data, content_type=mimetype)
+
+
+def run(container, prefix, root, credentials_file=None, project=None,
+ cache_control=None):
+ if credentials_file:
+ cred = Credentials()
+ cred._set_path(credentials_file)
+ client = storage.Client(credentials=cred, project=project)
+ else:
+ client = storage.Client()
+
+ file_list = []
+ if root.endswith('/'):
+ root = root[:-1]
+ for path, folders, files in os.walk(root):
+ for filename in files:
+ full_path = os.path.join(path, filename)
+ relative_path = full_path[len(root)+1:]
+ file_list.append((full_path, relative_path))
+
+ uploader = Uploader(client, container, prefix, cache_control)
+ uploader.upload(file_list)
+ return file_list
+
+
+def ansible_main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ container=dict(required=True, type='str'),
+ root=dict(required=True, type='str'),
+ prefix=dict(type='str'),
+ credentials_file=dict(type='str'),
+ project=dict(type='str'),
+ cache_control=dict(type='str'),
+ )
+ )
+
+ p = module.params
+ file_list = run(p.get('container'), p.get('prefix'), p.get('root'),
+ credentials_file=p.get('credentials_file'),
+ project=p.get('project'),
+ cache_control=p.get('cache_control'))
+ module.exit_json(changed=True, file_list=file_list)
+
+
+def cli_main():
+ parser = argparse.ArgumentParser(
+ description="Upload files to Google Cloud Storage"
+ )
+ parser.add_argument('--verbose', action='store_true',
+ help='show debug information')
+ parser.add_argument('--credentials-file',
+ help='A file with Google Cloud credentials')
+ parser.add_argument('--project',
+ help='Name of the Google Cloud project (required for '
+ 'credential file)')
+ parser.add_argument('container',
+ help='Name of the container to use when uploading')
+ parser.add_argument('prefix',
+ help='The prefix under the container root')
+ parser.add_argument('cache_control',
+ help='The cache-control header to set')
+ parser.add_argument('root',
+ help='The root of the directory to upload')
+
+ args = parser.parse_args()
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ logging.captureWarnings(True)
+
+ file_list = run(args.container, args.prefix, args.root,
+ credentials_file=args.credentials_file,
+ project=args.project,
+ cache_control=args.cache_control)
+
+
+if __name__ == '__main__':
+ if sys.stdin.isatty():
+ cli_main()
+ else:
+ ansible_main()
diff --git a/roles/gcs-upload/tasks/main.yaml b/roles/gcs-upload/tasks/main.yaml
new file mode 100644
index 0000000..61a3219
--- /dev/null
+++ b/roles/gcs-upload/tasks/main.yaml
@@ -0,0 +1,10 @@
+- name: Upload to GCS
+ delegate_to: localhost
+ gcs_upload:
+ container: "{{ gcs_upload_container }}"
+ credentials_file: "{{ gcs_upload_credentials_file }}"
+ project: "{{ gcs_upload_project }}"
+ root: "{{ gcs_upload_root }}"
+ prefix: "{{ gcs_upload_prefix }}"
+ cache_control: "{{ gcs_upload_cache_control }}"
+ register: gcs_upload_results
