Remove upload-logs-gcs role
This has merged upstream in zuul-jobs, so we no longer need the
local copy.
Change-Id: I88fd192deccff95e0e4e586d958313d5a44c7588
diff --git a/roles/upload-logs-gcs/README.rst b/roles/upload-logs-gcs/README.rst
deleted file mode 100644
index 0b1e355..0000000
--- a/roles/upload-logs-gcs/README.rst
+++ /dev/null
@@ -1,63 +0,0 @@
-Upload logs to Google Cloud Storage
-
-Before using this role, create at least one bucket and set up
-appropriate access controls or lifecycle events. This role will not
-automatically create buckets (though it will configure CORS policies).
-
-This role requires the ``google-cloud-storage`` Python package to be
-installed in the Ansible environment on the Zuul executor. It uses
-Google Cloud Application Default Credentials.
-
-**Role Variables**
-
-.. zuul:rolevar:: zuul_site_upload_logs
- :default: true
-
- Controls when logs are uploaded. true, the default, means always
- upload logs. false means never upload logs. 'failure' means to only
- upload logs when the job has failed.
-
- .. note:: Intended to be set by admins via site-variables.
-
-.. zuul:rolevar:: zuul_log_partition
- :default: false
-
- If set to true, then the first component of the log path will be
- removed from the object name and added to the bucket name, so that
- logs for different changes are distributed across a large number of
- buckets.
-
-.. zuul:rolevar:: zuul_log_container
-
- This role *will not* create buckets which do not already exist. If
- partitioning is not enabled, this is the name of the bucket which
- will be used. If partitioning is enabled, then this will be used
- as the prefix for the bucket name which will be separated from the
- partition name by an underscore. For example, "logs_42" would be
- the bucket name for partition 42.
-
- Note that you will want to set this to a value that uniquely
- identifies your Zuul installation.
-
-.. zuul:rolevar:: zuul_log_path
- :default: Generated by the role `set-zuul-log-path-fact`
-
- Prepend this path to the object names when uploading.
-
-.. zuul:rolevar:: zuul_log_create_indexes
- :default: true
-
- Whether to create `index.html` files with directory indexes. If set
- to false, Swift containers can be marked with a `Web-Listings=true`
- property to activate Swift's own directory indexing.
-
-.. zuul:rolevar:: zuul_log_path_shard_build
- :default: false
-
- This var is consumed by set-zuul-log-path-fact which
- upload-logs-gcs calls into. If you set this you will get log paths
- prefixed with the first three characters of the build uuid. This
- will improve log file sharding.
-
- More details can be found at
- :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
diff --git a/roles/upload-logs-gcs/__init__.py b/roles/upload-logs-gcs/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/roles/upload-logs-gcs/__init__.py
+++ /dev/null
diff --git a/roles/upload-logs-gcs/defaults/main.yaml b/roles/upload-logs-gcs/defaults/main.yaml
deleted file mode 100644
index 4d7c551..0000000
--- a/roles/upload-logs-gcs/defaults/main.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-zuul_log_partition: false
-zuul_log_container_public: true
-zuul_log_create_indexes: true
diff --git a/roles/upload-logs-gcs/library/__init__.py b/roles/upload-logs-gcs/library/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/roles/upload-logs-gcs/library/__init__.py
+++ /dev/null
diff --git a/roles/upload-logs-gcs/library/zuul_google_storage_upload.py b/roles/upload-logs-gcs/library/zuul_google_storage_upload.py
deleted file mode 100755
index 774a786..0000000
--- a/roles/upload-logs-gcs/library/zuul_google_storage_upload.py
+++ /dev/null
@@ -1,862 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright 2014 Rackspace Australia
-# Copyright 2018-2019 Red Hat, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Make coding more python3-ish
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-
-"""
-Utility to upload files to google
-"""
-
-import argparse
-import datetime
-import gzip
-import io
-import json
-import logging
-import mimetypes
-import os
-try:
- import queue as queuelib
-except ImportError:
- import Queue as queuelib
-import shutil
-import stat
-import sys
-import tempfile
-import threading
-import time
-try:
- import urllib.parse as urlparse
-except ImportError:
- import urllib as urlparse
-import zlib
-import collections
-
-from google.cloud import storage
-import google.auth.compute_engine.credentials as gce_cred
-
-from ansible.module_utils.basic import AnsibleModule
-
-try:
- # Python 3.3+
- from collections.abc import Sequence
-except ImportError:
- from collections import Sequence
-
-mimetypes.init()
-mimetypes.add_type('text/plain', '.yaml')
-
-MAX_UPLOAD_THREADS = 24
-POST_ATTEMPTS = 3
-
-# Map mime types to apache icons
-APACHE_MIME_ICON_MAP = {
- '_default': 'unknown.png',
- 'application/gzip': 'compressed.png',
- 'application/directory': 'folder.png',
- 'text/html': 'text.png',
- 'text/plain': 'text.png',
-}
-
-# Map mime types to apache icons
-APACHE_FILE_ICON_MAP = {
- '..': 'back.png',
-}
-
-# These icon files are from the Apache project and are in the public
-# domain.
-ICON_IMAGES = {
- 'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
- '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB'
- 'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9'
- 'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj'
- 'tgSXKYeLBgAAAABJRU5ErkJggg==',
- 'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM'
- 'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/'
- 'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///'
- 'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM'
- '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM'
- 'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM'
- 'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ'
- '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ'
- 'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z'
- 'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm'
- 'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm'
- 'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm'
- 'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz'
- 'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz'
- 'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A'
- '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA'
- 'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA'
- 'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3'
- 'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA'
- 'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d'
- '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS'
- 'TlP///////////////////////////////////////////////8A'
- 'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5'
- '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ'
- 'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN'
- 'aR7rMEBYAAAAAElFTkSuQmCC',
- 'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
- '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE'
- 'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI'
- 'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF'
- 'TkSuQmCC',
- 'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
- '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja'
- 'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ'
- 'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C'
- 'YII=',
- 'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
- '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja'
- 'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq'
- 'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN'
- 'AupSdoFsAAAAAElFTkSuQmCC'}
-
-
-# Begin vendored code
-# This code is licensed under the Public Domain/CC0 and comes from
-# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py
-# Code was modified:
-# removed type annotations to support python2.
-# removed use of *, somearg for positional anonymous args.
-# Default compression level to 9.
-
-class GZIPCompressedStream(io.RawIOBase):
- def __init__(self, stream, compression_level=9):
- assert 1 <= compression_level <= 9
-
- self._compression_level = compression_level
- self._stream = stream
-
- self._compressed_stream = io.BytesIO()
- self._compressor = gzip.GzipFile(
- mode='wb',
- fileobj=self._compressed_stream,
- compresslevel=compression_level
- )
-
- # because of the GZIP header written by `GzipFile.__init__`:
- self._compressed_stream.seek(0)
- self.count = 0
-
- def read(self, length):
- r = super().read(length)
- self.count += len(r)
- return r
-
- def tell(self):
- return self.count
-
- @property
- def compression_level(self):
- return self._compression_level
-
- @property
- def stream(self):
- return self._stream
-
- def readable(self):
- return True
-
- def _read_compressed_into(self, b):
- buf = self._compressed_stream.read(len(b))
- b[:len(buf)] = buf
- return len(buf)
-
- def readinto(self, b):
- b = memoryview(b)
-
- offset = 0
- size = len(b)
- while offset < size:
- offset += self._read_compressed_into(b[offset:])
- if offset < size:
- # self._compressed_buffer now empty
- if self._compressor.closed:
- # nothing to compress anymore
- break
- # compress next bytes
- self._read_n_compress(size)
-
- return offset
-
- def _read_n_compress(self, size):
- assert size > 0
-
- data = self._stream.read(size)
-
- # rewind buffer to the start to free up memory
- # (because anything currently in the buffer should be already
- # streamed off the object)
- self._compressed_stream.seek(0)
- self._compressed_stream.truncate(0)
-
- if data:
- self._compressor.write(data)
- else:
- # this will write final data (will flush zlib with Z_FINISH)
- self._compressor.close()
-
- # rewind to the buffer start
- self._compressed_stream.seek(0)
-
- def __repr__(self):
- return (
- '{self.__class__.__name__}('
- '{self.stream!r}, '
- 'compression_level={self.compression_level!r}'
- ')'
- ).format(self=self)
-
-# End vendored code
-
-
-def get_mime_icon(mime, filename=''):
- icon = (APACHE_FILE_ICON_MAP.get(filename) or
- APACHE_MIME_ICON_MAP.get(mime) or
- APACHE_MIME_ICON_MAP['_default'])
- return "data:image/png;base64,%s" % ICON_IMAGES[icon]
-
-
-def retry_function(func):
- for attempt in range(1, POST_ATTEMPTS + 1):
- try:
- return func()
- except Exception:
- if attempt >= POST_ATTEMPTS:
- raise
- else:
- logging.exception("Error on attempt %d" % attempt)
- time.sleep(attempt * 10)
-
-
-def sizeof_fmt(num, suffix='B'):
- # From http://stackoverflow.com/questions/1094841/
- # reusable-library-to-get-human-readable-version-of-file-size
- for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Y', suffix)
-
-
-class Credentials(gce_cred.Credentials):
- def __init__(self, path):
- self._path = path
- self.refresh(None)
-
- def refresh(self, request):
- with open(self._path) as f:
- data = json.loads(f.read())
- self.token = data['access_token']
- self.expiry = (datetime.datetime.utcnow() +
- datetime.timedelta(seconds=data['expires_in']))
-
-
-class FileDetail():
- """
- Used to generate indexes with links or as the file path
- to push to storage.
- """
-
- def __init__(self, full_path, relative_path, filename=None):
- """
- Args:
- full_path (str): The absolute path to the file on disk.
- relative_path (str): The relative path from the artifacts source
- used for links.
- filename (str): An optional alternate filename in links.
- """
- # Make FileNotFoundError exception to be compatible with python2
- try:
- FileNotFoundError # noqa: F823
- except NameError:
- FileNotFoundError = OSError
-
- self.full_path = full_path
- if filename is None:
- self.filename = os.path.basename(full_path)
- else:
- self.filename = filename
- self.relative_path = relative_path
-
- if self.full_path and os.path.isfile(self.full_path):
- mime_guess, encoding = mimetypes.guess_type(self.full_path)
- self.mimetype = mime_guess if mime_guess else 'text/plain'
- self.encoding = encoding
- self.folder = False
- else:
- self.mimetype = 'application/directory'
- self.encoding = None
- self.folder = True
- try:
- st = os.stat(self.full_path)
- self.last_modified = time.gmtime(st[stat.ST_MTIME])
- self.size = st[stat.ST_SIZE]
- except (FileNotFoundError, TypeError):
- self.last_modified = time.gmtime(0)
- self.size = 0
-
- def __repr__(self):
- t = 'Folder' if self.folder else 'File'
- return '<%s %s>' % (t, self.relative_path)
-
-
-class FileList(Sequence):
- '''A collection of FileDetail objects
-
- This is a list-like group of FileDetail objects, intended to be
- used as a context manager around the upload process.
- '''
- def __init__(self):
- self.file_list = []
- self.file_list.append(FileDetail(None, '', ''))
- self.tempdirs = []
-
- def __enter__(self):
- return self
-
- def __exit__(self, type, value, traceback):
- for tempdir in self.tempdirs:
- shutil.rmtree(tempdir)
-
- def __getitem__(self, item):
- return self.file_list.__getitem__(item)
-
- def __len__(self):
- return self.file_list.__len__()
-
- def get_tempdir(self):
- '''Get a temporary directory
-
- Returns path to a private temporary directory which will be
- cleaned on exit
- '''
- tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp')
- self.tempdirs.append(tempdir)
- return tempdir
-
- @staticmethod
- def _path_in_tree(root, path):
- full_path = os.path.realpath(os.path.abspath(
- os.path.expanduser(path)))
- if not full_path.startswith(root):
- logging.debug("Skipping path outside root: %s" % (path,))
- return False
- return True
-
- def add(self, file_path):
- """
- Generate a list of files to upload to storage. Recurses through
- directories
- """
-
- # file_list: A list of FileDetails to push to storage
- file_list = []
-
- if os.path.isfile(file_path):
- relative_path = os.path.basename(file_path)
- file_list.append(FileDetail(file_path, relative_path))
- elif os.path.isdir(file_path):
- original_root = os.path.realpath(os.path.abspath(
- os.path.expanduser(file_path)))
-
- parent_dir = os.path.dirname(file_path)
- if not file_path.endswith('/'):
- filename = os.path.basename(file_path)
- full_path = file_path
- relative_name = os.path.relpath(full_path, parent_dir)
- file_list.append(FileDetail(full_path, relative_name,
- filename))
- # TODO: this will copy the result of symlinked files, but
- # it won't follow directory symlinks. If we add that, we
- # should ensure that we don't loop.
- for path, folders, files in os.walk(file_path):
- # Sort folder in-place so that we recurse in order.
- files.sort(key=lambda x: x.lower())
- folders.sort(key=lambda x: x.lower())
- # relative_path: The path between the given directory
- # and the one being currently walked.
- relative_path = os.path.relpath(path, parent_dir)
-
- for filename in folders:
- full_path = os.path.join(path, filename)
- if not self._path_in_tree(original_root, full_path):
- continue
- relative_name = os.path.relpath(full_path, parent_dir)
- file_list.append(FileDetail(full_path, relative_name,
- filename))
-
- for filename in files:
- full_path = os.path.join(path, filename)
- if not self._path_in_tree(original_root, full_path):
- continue
- relative_name = os.path.relpath(full_path, parent_dir)
- file_detail = FileDetail(full_path, relative_name)
- file_list.append(file_detail)
-
- self.file_list += file_list
-
-
-class Indexer():
- """Index a FileList
-
- Functions to generate indexes and other collated data for a
- FileList
-
- - make_indexes() : make index.html in folders
- """
- def __init__(self, file_list):
- '''
- Args:
- file_list (FileList): A FileList object with all files
- to be indexed.
- '''
- assert isinstance(file_list, FileList)
- self.file_list = file_list
-
- def _make_index_file(self, folder_links, title, tempdir, append_footer):
- """Writes an index into a file for pushing"""
- for file_details in folder_links:
- # Do not generate an index file if one exists already.
- # This may be the case when uploading other machine generated
- # content like python coverage info.
- if self.index_filename == file_details.filename:
- return
- index_content = self._generate_log_index(
- folder_links, title, append_footer)
- fd = open(os.path.join(tempdir, self.index_filename), 'w')
- fd.write(index_content)
- return os.path.join(tempdir, self.index_filename)
-
- def _generate_log_index(self, folder_links, title, append_footer):
- """Create an index of logfiles and links to them"""
-
- output = '<html><head><title>%s</title></head><body>\n' % title
- output += '<h1>%s</h1>\n' % title
- output += '<table><tr><th></th><th>Name</th><th>Last Modified</th>'
- output += '<th>Size</th></tr>'
-
- file_details_to_append = None
- for file_details in folder_links:
- output += '<tr>'
- output += (
- '<td><img alt="[ ]" title="%(m)s" src="%(i)s"></img></td>' % ({
- 'm': file_details.mimetype,
- 'i': get_mime_icon(file_details.mimetype,
- file_details.filename),
- }))
- filename = file_details.filename
- link_filename = filename
- if file_details.folder:
- filename += '/'
- link_filename += '/index.html'
- output += '<td><a href="%s">%s</a></td>' % (
- urlparse.quote(link_filename),
- filename)
- output += '<td>%s</td>' % time.asctime(
- file_details.last_modified)
- size = sizeof_fmt(file_details.size, suffix='')
- output += '<td style="text-align: right">%s</td>' % size
- output += '</tr>\n'
-
- if (append_footer and
- append_footer in file_details.filename):
- file_details_to_append = file_details
-
- output += '</table>'
-
- if file_details_to_append:
- output += '<br /><hr />'
- try:
- with open(file_details_to_append.full_path, 'r') as f:
- output += f.read()
- except IOError:
- logging.exception("Error opening file for appending")
-
- output += '</body></html>\n'
- return output
-
- def make_indexes(self, create_parent_links=True,
- create_topdir_parent_link=False,
- append_footer='index_footer.html'):
- '''Make index.html files
-
- Iterate the file list and crete index.html files for folders
-
- Args:
- create_parent_links (bool): Create parent links
- create_topdir_parent_link (bool): Create topdir parent link
- append_footer (str): Filename of a footer to append to each
- generated page
-
- Return:
- No value, the self.file_list will be updated
- '''
- self.index_filename = 'index.html'
-
- folders = collections.OrderedDict()
- for f in self.file_list:
- if f.folder:
- folders[f.relative_path] = []
- folder = os.path.dirname(os.path.dirname(
- f.relative_path + '/'))
- if folder == '/':
- folder = ''
- else:
- folder = os.path.dirname(f.relative_path)
- folders[folder].append(f)
-
- indexes = {}
- parent_file_detail = FileDetail(None, '..', '..')
- for folder, files in folders.items():
- # Don't add the pseudo-top-directory
- if files and files[0].full_path is None:
- files = files[1:]
- if create_topdir_parent_link:
- files = [parent_file_detail] + files
- elif create_parent_links:
- files = [parent_file_detail] + files
-
- # Do generate a link to the parent directory
- full_path = self._make_index_file(files, 'Index of %s' % (folder,),
- self.file_list.get_tempdir(),
- append_footer)
-
- if full_path:
- filename = os.path.basename(full_path)
- relative_name = os.path.join(folder, filename)
- indexes[folder] = FileDetail(full_path, relative_name)
-
- # This appends the index file at the end of the group of files
- # for each directory.
- new_list = []
- last_dirname = None
- for f in reversed(list(self.file_list)):
- if f.folder:
- relative_path = f.relative_path + '/'
- else:
- relative_path = f.relative_path
- dirname = os.path.dirname(relative_path)
- if dirname == '/':
- dirname = ''
- if dirname != last_dirname:
- index = indexes.pop(dirname, None)
- if index:
- new_list.append(index)
- last_dirname = dirname
- new_list.append(f)
- new_list.reverse()
- self.file_list.file_list = new_list
-
-
-class GzipFilter():
- chunk_size = 16384
-
- def __init__(self, infile):
- self.gzipfile = GZIPCompressedStream(infile)
- self.done = False
-
- def __iter__(self):
- return self
-
- def __next__(self):
- if self.done:
- self.gzipfile.close()
- raise StopIteration()
- data = self.gzipfile.read(self.chunk_size)
- if not data:
- self.done = True
- return data
-
-
-class DeflateFilter():
- chunk_size = 16384
-
- def __init__(self, infile):
- self.infile = infile
- self.encoder = zlib.compressobj()
- self.done = False
-
- def __iter__(self):
- return self
-
- def __next__(self):
- if self.done:
- raise StopIteration()
- ret = b''
- while True:
- data = self.infile.read(self.chunk_size)
- if data:
- ret = self.encoder.compress(data)
- if ret:
- break
- else:
- self.done = True
- ret = self.encoder.flush()
- break
- return ret
-
-
-class Uploader():
- def __init__(self, client, container, prefix=None,
- dry_run=False):
-
- self.dry_run = dry_run
- if dry_run:
- self.url = 'http://dry-run-url.com/a/path/'
- return
-
- self.client = client
- self.prefix = prefix or ''
-
- self.bucket = client.bucket(container)
- cors = [{
- 'method': ['GET', 'HEAD'],
- 'origin': ['*']
- }]
- self.bucket.cors = cors
- self.bucket.website = {"mainPageSuffix": "index.html"}
- self.bucket.update()
-
- self.url = os.path.join('https://storage.googleapis.com/',
- container, self.prefix)
-
- def upload(self, file_list):
- """Spin up thread pool to upload to storage"""
-
- if self.dry_run:
- return
-
- num_threads = min(len(file_list), MAX_UPLOAD_THREADS)
- threads = []
- queue = queuelib.Queue()
- # add items to queue
- for f in file_list:
- queue.put(f)
-
- for x in range(num_threads):
- t = threading.Thread(target=self.post_thread, args=(queue,))
- threads.append(t)
- t.start()
-
- for t in threads:
- t.join()
-
- def post_thread(self, queue):
- while True:
- try:
- file_detail = queue.get_nowait()
- logging.debug("%s: processing job %s",
- threading.current_thread(),
- file_detail)
- retry_function(lambda: self._post_file(file_detail))
- except IOError:
- # Do our best to attempt to upload all the files
- logging.exception("Error opening file")
- continue
- except queuelib.Empty:
- # No more work to do
- return
-
- @staticmethod
- def _is_text_type(mimetype):
- # We want to compress all text types.
- if mimetype.startswith('text/'):
- return True
-
- # Further compress types that typically contain text but are no
- # text sub type.
- compress_types = [
- 'application/json',
- 'image/svg+xml',
- ]
- if mimetype in compress_types:
- return True
- return False
-
- def _post_file(self, file_detail):
- relative_path = os.path.join(self.prefix, file_detail.relative_path)
- content_encoding = None
-
- if not file_detail.folder:
- if (file_detail.encoding is None and
- self._is_text_type(file_detail.mimetype)):
- content_encoding = 'gzip'
- data = GZIPCompressedStream(open(file_detail.full_path, 'rb'))
- else:
- if (not file_detail.filename.endswith(".gz") and
- file_detail.encoding):
- # Don't apply gzip encoding to files that we receive as
- # already gzipped. The reason for this is storage will
- # serve this back to users as an uncompressed file if they
- # don't set an accept-encoding that includes gzip. This
- # can cause problems when the desired file state is
- # compressed as with .tar.gz tarballs.
- content_encoding = file_detail.encoding
- data = open(file_detail.full_path, 'rb')
- else:
- data = ''
- relative_path = relative_path.rstrip('/')
- if relative_path == '':
- relative_path = '/'
- blob = self.bucket.blob(relative_path)
- if content_encoding:
- blob.content_encoding = content_encoding
- if hasattr(data, 'tell'):
- upload = blob.upload_from_file
- else:
- upload = blob.upload_from_string
- upload(data, content_type=file_detail.mimetype)
-
-
-def run(container, files,
- indexes=True, parent_links=True, topdir_parent_link=False,
- partition=False, footer='index_footer.html',
- prefix=None, dry_run=False, credentials_file=None):
-
- if credentials_file:
- cred = Credentials(credentials_file)
- client = storage.Client(credentials=cred)
- else:
- client = storage.Client()
-
- if prefix:
- prefix = prefix.lstrip('/')
- if partition and prefix:
- parts = prefix.split('/')
- if len(parts) > 1:
- container += '_' + parts[0]
- prefix = '/'.join(parts[1:])
-
- # Create the objects to make sure the arguments are sound.
- with FileList() as file_list:
- # Scan the files.
- for file_path in files:
- file_list.add(file_path)
-
- indexer = Indexer(file_list)
-
- # (Possibly) make indexes.
- if indexes:
- indexer.make_indexes(create_parent_links=parent_links,
- create_topdir_parent_link=topdir_parent_link,
- append_footer=footer)
-
- logging.debug("List of files prepared to upload:")
- for x in file_list:
- logging.debug(x)
-
- # Upload.
- uploader = Uploader(client, container, prefix, dry_run)
- uploader.upload(file_list)
- return uploader.url
-
-
-def ansible_main():
- module = AnsibleModule(
- argument_spec=dict(
- container=dict(required=True, type='str'),
- files=dict(required=True, type='list'),
- partition=dict(type='bool', default=False),
- indexes=dict(type='bool', default=True),
- parent_links=dict(type='bool', default=True),
- topdir_parent_link=dict(type='bool', default=False),
- footer=dict(type='str'),
- prefix=dict(type='str'),
- credentials_file=dict(type='str'),
- )
- )
-
- p = module.params
- url = run(p.get('container'), p.get('files'),
- indexes=p.get('indexes'),
- parent_links=p.get('parent_links'),
- topdir_parent_link=p.get('topdir_parent_link'),
- partition=p.get('partition'),
- footer=p.get('footer'),
- prefix=p.get('prefix'),
- credentials_file=p.get('credentials_file'))
- module.exit_json(changed=True,
- url=url)
-
-
-def cli_main():
- parser = argparse.ArgumentParser(
- description="Upload files to Google Cloud Storage"
- )
- parser.add_argument('--verbose', action='store_true',
- help='show debug information')
- parser.add_argument('--no-indexes', action='store_true',
- help='do not generate any indexes at all')
- parser.add_argument('--no-parent-links', action='store_true',
- help='do not include links back to a parent dir')
- parser.add_argument('--create-topdir-parent-link', action='store_true',
- help='include a link in the root directory of the '
- 'files to the parent directory which may be the '
- 'index of all results')
- parser.add_argument('--partition', action='store_true',
- help='partition the prefix into multiple containers')
- parser.add_argument('--append-footer', default='index_footer.html',
- help='when generating an index, if the given file is '
- 'present in a directory, append it to the index '
- '(set to "none" to disable)')
- parser.add_argument('--prefix',
- help='Prepend this path to the object names when '
- 'uploading')
- parser.add_argument('--dry-run', action='store_true',
- help='do not attempt to create containers or upload, '
- 'useful with --verbose for debugging')
- parser.add_argument('--credentials_file',
- help='A file with Google cloud credentials')
- parser.add_argument('container',
- help='Name of the container to use when uploading')
- parser.add_argument('files', nargs='+',
- help='the file(s) to upload with recursive glob '
- 'matching when supplied as a string')
-
- args = parser.parse_args()
-
- if args.verbose:
- logging.basicConfig(level=logging.DEBUG)
- logging.captureWarnings(True)
-
- append_footer = args.append_footer
- if append_footer.lower() == 'none':
- append_footer = None
-
- url = run(args.container, args.files,
- indexes=not args.no_indexes,
- parent_links=not args.no_parent_links,
- topdir_parent_link=args.create_topdir_parent_link,
- partition=args.partition,
- footer=append_footer,
- prefix=args.prefix,
- dry_run=args.dry_run,
- credentials_file=args.credentials_file)
- print(url)
-
-
-if __name__ == '__main__':
- if sys.stdin.isatty():
- cli_main()
- else:
- ansible_main()
diff --git a/roles/upload-logs-gcs/tasks/main.yaml b/roles/upload-logs-gcs/tasks/main.yaml
deleted file mode 100644
index 53746e9..0000000
--- a/roles/upload-logs-gcs/tasks/main.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-- name: Set zuul-log-path fact
- include_role:
- name: set-zuul-log-path-fact
- when: zuul_log_path is not defined
-
-# Always upload (true), never upload (false) or only on failure ('failure')
-- when: zuul_site_upload_logs | default(true) | bool or
- (zuul_site_upload_logs == 'failure' and not zuul_success | bool)
- block:
- # Use chmod instead of file because ansible 2.5 file with recurse and
- # follow can't really handle symlinks to .
- - name: Ensure logs are readable before uploading
- delegate_to: localhost
- command: "chmod -R u=rwX,g=rX,o=rX {{ zuul.executor.log_root }}/"
- # ANSIBLE0007 chmod used in place of argument mode to file
- tags:
- - skip_ansible_lint
-
- - name: Upload logs to Google Cloud Storage
- delegate_to: localhost
- zuul_google_storage_upload:
- partition: "{{ zuul_log_partition }}"
- container: "{{ zuul_log_container }}"
- prefix: "{{ zuul_log_path }}"
- indexes: "{{ zuul_log_create_indexes }}"
- credentials_file: "{{ zuul_log_credentials_file }}"
- files:
- - "{{ zuul.executor.log_root }}/"
- register: upload_results
-
-- name: Return log URL to Zuul
- delegate_to: localhost
- zuul_return:
- data:
- zuul:
- log_url: "{{ upload_results.url }}/"
- when: upload_results is defined
