tools/plugins.py

from __future__ import print_function
import argparse
import getpass
import os
import re
import requests
import sys
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import Enum, IntEnum
from functools import partial
from operator import attrgetter
from typing import List

from jinja2 import Template
from pygerrit2 import Anonymous, GerritRestAPI, HTTPBasicAuth, HTTPBasicAuthFromNetrc
from tqdm import tqdm


BRANCHES = ["master"] + [
    f"stable-{version}" for version in ["3.8", "3.7", "3.6"]
]
CI = "https://gerrit-ci.gerritforge.com"
GERRIT = "https://gerrit-review.googlesource.com"
GITILES = "https://gerrit.googlesource.com"

CORE_MAINTAINERS_ID = "google:AI2Pq9rwJtXWrKQ9Q62CcHSid7ngIF2hCfJ4bSpVquX_P2z5kFM6v9s"
CORE_MAINTAINERS_NAME = "Core maintainers"

BRANCH_MARK = "⌥"
GREEN_CHECK_MARK = "✅"
LOCK = "🔒"
RED_CROSS = "❌"
SQUARE = "⃞"


class BuildResult(Enum):
    """Build result for a plugin"""

    UNAVAILABLE = "unavailable"
    SUCCESSFUL = "successful"
    FAILED = "failed"

    def render(self):
        if self == BuildResult.SUCCESSFUL:
            return GREEN_CHECK_MARK
        elif self == BuildResult.FAILED:
            return RED_CROSS
        else:
            return SQUARE


class PluginState(IntEnum):
    ACTIVE = 1
    READ_ONLY = 2

    def render(self):
        if self == PluginState.ACTIVE:
            return GREEN_CHECK_MARK
        else:
            return LOCK


class Branch:
    """Branch of a plugin repository"""

    name: str
    build: BuildResult
    present: bool

    def __init__(self, name, build, present=True):
        self.name = name
        self.build = build
        self.present = present

    @classmethod
    def missing(cls, name):
        return cls(name, BuildResult.UNAVAILABLE, False)

    def render(self):
        return BRANCH_MARK if self.present else SQUARE


@dataclass
class Plugin:
    """Gerrit plugin"""

    name: str
    parent: str
    state: PluginState
    owner_group_ids: List[str]
    owner_names: List[str]
    empty: bool
    description: str
    all_changes_count: int
    recent_changes_count: int
    branches: List[Branch]

    def render_empty(self):
        return SQUARE if self.empty else BRANCH_MARK


@dataclass(frozen=True)
class Account:
    """Gerrit account"""

    id: str
    name: str
    email: str


class Plugins:
    """Class to retrieve data about Gerrit plugins and render the plugins page"""

    @staticmethod
    def _parse_options():
        parser = argparse.ArgumentParser(
            formatter_class=argparse.ArgumentDefaultsHelpFormatter
        )
        group = parser.add_mutually_exclusive_group()

        group.add_argument(
            "-n",
            "--netrc",
            dest="netrc",
            action="store_true",
            help="use credentials from .netrc",
        )
        group.add_argument(
            "-a",
            "--anonymous",
            dest="anonymous",
            action="store_true",
            help="use anonymous access, i.e. no credentials",
        )
        parser.add_argument(
            "-t",
            "--threads",
            dest="threads",
            default=1,
            type=int,
            help="number of threads to fetch data from Gerrit concurrently",
        )
        return parser.parse_args()

    @staticmethod
    def _render_header():
        header = "|Name|State|Repo|Changes (last 3 months/all)|Description|Maintainers"
        dashes = "|----|-----|----|---------------------------|-----------|---"
        spacer = "|    |     |    |                           |           |   "

        links = "\n"
        for b in BRANCHES:
            header += "|Branch|CI"
            dashes += "|-----:|--"
            spacer += f"|[{b}]|"
            links += f"[{b}]: {CI}/view/Plugins-{b}\n"

        return (header, dashes, spacer, links)

    @staticmethod
    def _render_template():
        data = {
            "permalink": "plugins",
            "updated": time.strftime("%A %d %B at %H:%M:%S %Z", time.gmtime()),
        }

        template_path = os.path.join(
            os.path.dirname(os.path.realpath(__file__)), "plugins.md.template"
        )
        template = Template(open(template_path).read())
        return template.render(data=data)

    @staticmethod
    def _get_matrix_header(state, is_empty):
        if state == PluginState.ACTIVE:
            if is_empty:
                return "Not Started"
            else:
                return "Active"
        else:
            if is_empty:
                return "Deprecated, not started"
            else:
                return "Deprecated"

    def __init__(self):
        self.options = self._parse_options()
        auth = self._authenticate()
        self.api = GerritRestAPI(url=GERRIT, auth=auth)
        self.plugins = list()
        self.maintainers = defaultdict(list)
        self._create_plugins()
        self.plugins = sorted(self.plugins, key=attrgetter("state", "empty"))

    def __iter__(self):
        return iter(self.plugins)

    def _create_plugin(self, plugin_list: dict, builds, p):
        """Create a plugin by fetching its data from Gerrit"""
        name = p[len("plugins/") :]
        plugin = plugin_list[p]

        if plugin["state"] == "ACTIVE":
            state = PluginState.ACTIVE
            changes = self._get_recent_changes_count(p)
            branches = self._get_branch_results(plugin["id"], name, builds)
        else:
            state = PluginState.READ_ONLY
            changes = 0
            branches = [Branch.missing(branch) for branch in BRANCHES]

        description = (
            plugin["description"].split("\n")[0].rstrip(r"\.")
            if "description" in plugin
            else ""
        )

        parent, owner_group_ids = self._get_meta_data(name)
        maintainers, maintainers_csv = self._get_owner_names(
            parent, name, owner_group_ids
        )
        plugin = Plugin(
            name=name,
            parent=parent,
            state=state,
            owner_group_ids=owner_group_ids,
            owner_names=maintainers_csv,
            empty=self._is_project_empty(p),
            description=description,
            all_changes_count=self._get_all_changes_count(p),
            recent_changes_count=changes,
            branches=branches,
        )
        return plugin, maintainers

    def _create_plugins(self):
        """Create plugins by fetching plugin data from Gerrit"""
        # Set an explicit limit to get more results than the index default limit
        # which is applied if no limit is set and which is 100 for gerrit-review
        # TODO: Instead of setting a high limit paginate over the results (see
        # https://issues.gerritcodereview.com/issues/296837507)
        plugin_list = self.api.get("/projects/?p=plugins%2f&d&limit=500")
        builds = requests.get(
            f"{CI}/api/json?pretty=true&tree=jobs[name,lastBuild[result]]"
        ).json()
        creator = partial(self._create_plugin, plugin_list, builds)
        with ThreadPoolExecutor(max_workers=self.options.threads) as executor:
            results = list(
                tqdm(executor.map(creator, plugin_list), total=len(plugin_list))
            )
            for (plugin, maintainers) in results:
                self.plugins.append(plugin)
                for m in maintainers:
                    self.maintainers[m.name].append((m, plugin.name))

    def _authenticate(self):
        if self.options.netrc:
            return HTTPBasicAuthFromNetrc(url=GERRIT)
        elif self.options.anonymous:
            return Anonymous()
        else:
            return self._authenticate_interactive()

    def _authenticate_interactive(self):
        username = os.environ.get("username")
        password = os.environ.get("password")
        while not username:
            username = input("user: ")
        while not password:
            password = getpass.getpass("password: ")
        auth = HTTPBasicAuth(username, password)
        return auth

    def _get_branches(self, pluginId):
        branchList = self.api.get(f"/projects/{pluginId}/branches/")
        pluginBranches = []
        for b in branchList:
            if b["ref"].startswith("refs/heads/"):
                ref = b["ref"]
                pluginBranches += [ref[len("refs/heads/") :]]
        return pluginBranches

    def _get_branch_results(self, pluginId, pluginName, builds):
        pluginBranches = self._get_branches(pluginId)
        branches = list()
        for branch in BRANCHES:
            string = fr"^plugin-{pluginName}-[a-z|-]*{branch}$"
            pattern = re.compile(string)
            result = BuildResult.UNAVAILABLE
            for job in builds["jobs"]:
                if pattern.match(job["name"]):
                    result = (
                        BuildResult.SUCCESSFUL
                        if job["lastBuild"] and job["lastBuild"]["result"] == "SUCCESS"
                        else BuildResult.FAILED
                    )
                    branches.append(Branch(branch, result, branch in pluginBranches))
                    break
            if result == BuildResult.UNAVAILABLE:
                branches.append(Branch.missing(branch))
        return branches

    def _get_all_changes_count(self, pluginName):
        changes = self.api.get(f"/changes/?q=project:{pluginName}")
        return len(changes)

    def _get_recent_changes_count(self, pluginName):
        changes = self.api.get(f"/changes/?q=project:{pluginName}+-age:3months")
        return len(changes)

    def _get_meta_data(self, pluginName):
        path = requests.utils.quote(f"plugins/{pluginName}", safe="")
        permissions = self.api.get(f"projects/{path}/access")
        parent = permissions["inherits_from"]["name"]
        try:
            owner_group_ids = permissions["local"]["refs/*"]["permissions"]["owner"][
                "rules"
            ].keys()
        except KeyError:
            # no owner group defined
            owner_group_ids = list()
        return parent, owner_group_ids

    def _get_owner_names(self, parent, name, owner_group_ids):
        accounts = set()
        external_groups = set()
        all_owner_group_ids = set(owner_group_ids)
        # add subgroups if any
        for id in owner_group_ids:
            if id == CORE_MAINTAINERS_ID:
                continue
            else:
                try:
                    subgroups = self.api.get(f"/groups/{id}/groups")
                    for s in subgroups:
                        all_owner_group_ids.add(s.get("id"))
                except requests.HTTPError:
                    print(
                        f"Failed to read subgroup {id} of owner group of plugin {name}",
                        file=sys.stderr,
                    )
        for id in all_owner_group_ids:
            try:
                if id == CORE_MAINTAINERS_ID:
                    external_groups.add(CORE_MAINTAINERS_NAME)
                else:
                    owners = self.api.get(f"/groups/{id}/members/")
                    a = {
                        Account(o.get("_account_id"), o.get("name"), o.get("email"))
                        for o in owners
                    }
                    accounts.update(a)
            except requests.HTTPError:
                print(
                    f"Failed to read owner group {id} of plugin {name}", file=sys.stderr
                )
        csv = ", ".join(sorted({a.name for a in accounts} | external_groups))
        return accounts, csv

    def _is_project_empty(self, pluginName):
        gitiles_uri = f"{GITILES}/{pluginName}"
        try:
            response = requests.get(gitiles_uri)
            if response.status_code == 200:
                if response.text.find("Empty Repository") > -1:
                    return True
        except requests.HTTPError as e:
            print(f"Failed to browse {pluginName} in gitiles:\n{e}", file=sys.stderr)
        return False

    def _render_maintainers(self, output):
        header = "|Maintainer|Plugins|"
        dashes = "|----------|-------|"
        output.write("\n\n## Plugin Maintainers")
        output.write(f"\n\n{header}|\n{dashes}|\n")
        for m in sorted(self.maintainers):
            plugins = set()
            for (_, p) in self.maintainers.get(m):
                plugins.add(p)
            output.write(f"|{m}|{', '.join(sorted(plugins))}|\n")

    def render_maintainers_email(self, output):
        output.write(f"\nAll {len(self.maintainers)} plugin maintainers:\n")
        for m in sorted(self.maintainers):
            if m == CORE_MAINTAINERS_NAME:
                continue
            emails = set()
            output.write(f"{m}: ")
            for (accounts, _) in self.maintainers.get(m):
                if accounts.email:
                    emails.add(accounts.email)
            output.write(f"{', '.join(sorted(emails))}\n")

    def render_page(self, output):
        output.writelines(self._render_template())
        (header, dashes, spacer, links) = self._render_header()

        flags = (None, None)
        for p in self.plugins:
            if flags != (p.state, p.empty):
                output.write(f"\n\n### {self._get_matrix_header(p.state, p.empty)}")
                output.write(f"\n\n{header}|\n{dashes}|\n{spacer}|\n")
            branches = "|".join(
                [f"{b.render()}|{b.build.render()}" for b in p.branches]
            )
            output.write(
                f"|[{p.name}]"
                + f"|{p.state.render()}|{p.render_empty()}"
                + f"|{p.recent_changes_count}"
                + f"/[{p.all_changes_count}]({GERRIT}/q/project:plugins/{p.name})"
                + f"|{p.description}"
                + f"|{p.owner_names}"
                + f"|{branches}"
                + "|\n"
            )
            flags = (p.state, p.empty)
            links += f"[{p.name}]: {GITILES}/plugins/{p.name}\n"

        output.write(links)
        self._render_maintainers(output)


def main():
    plugins = Plugins()
    with open("pages/site/plugins/plugins.md", "w") as output:
        plugins.render_page(output)
    plugins.render_maintainers_email(sys.stdout)


if __name__ == "__main__":
    main()