subcmds/upload.py

# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import functools
import optparse
import re
import sys
from typing import List

from command import DEFAULT_LOCAL_JOBS
from command import InteractiveCommand
from editor import Editor
from error import GitError
from error import SilentRepoExitError
from error import UploadError
from git_command import GitCommand
from git_refs import R_HEADS
from hooks import RepoHook
from project import ReviewableBranch
from repo_logging import RepoLogger
from subcmds.sync import LocalSyncState


_DEFAULT_UNUSUAL_COMMIT_THRESHOLD = 5
logger = RepoLogger(__file__)


class UploadExitError(SilentRepoExitError):
    """Indicates that there is an upload command error requiring a sys exit."""


def _VerifyPendingCommits(branches: List[ReviewableBranch]) -> bool:
    """Perform basic safety checks on the given set of branches.

    Ensures that each branch does not have a "large" number of commits
    and, if so, prompts the user to confirm they want to proceed with
    the upload.

    Returns true if all branches pass the safety check or the user
    confirmed. Returns false if the upload should be aborted.
    """

    # Determine if any branch has a suspicious number of commits.
    many_commits = False
    for branch in branches:
        # Get the user's unusual threshold for the branch.
        #
        # Each branch may be configured to have a different threshold.
        remote = branch.project.GetBranch(branch.name).remote
        key = f"review.{remote.review}.uploadwarningthreshold"
        threshold = branch.project.config.GetInt(key)
        if threshold is None:
            threshold = _DEFAULT_UNUSUAL_COMMIT_THRESHOLD

        # If the branch has more commits than the threshold, show a warning.
        if len(branch.commits) > threshold:
            many_commits = True
            break

    # If any branch has many commits, prompt the user.
    if many_commits:
        if len(branches) > 1:
            logger.warning(
                "ATTENTION: One or more branches has an unusually high number "
                "of commits."
            )
        else:
            logger.warning(
                "ATTENTION: You are uploading an unusually high number of "
                "commits."
            )
        logger.warning(
            "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across "
            "branches?)"
        )
        answer = input(
            "If you are sure you intend to do this, type 'yes': "
        ).strip()
        return answer == "yes"

    return True


def _die(fmt, *args):
    msg = fmt % args
    logger.error("error: %s", msg)
    raise UploadExitError(msg)


def _SplitEmails(values):
    result = []
    for value in values:
        result.extend([s.strip() for s in value.split(",")])
    return result


class Upload(InteractiveCommand):
    COMMON = True
    helpSummary = "Upload changes for code review"
    helpUsage = """
%prog [--re --cc] [<project>]...
"""
    helpDescription = """
The '%prog' command is used to send changes to the Gerrit Code
Review system.  It searches for topic branches in local projects
that have not yet been published for review.  If multiple topic
branches are found, '%prog' opens an editor to allow the user to
select which branches to upload.

'%prog' searches for uploadable changes in all projects listed at
the command line.  Projects can be specified either by name, or by
a relative or absolute path to the project's local directory. If no
projects are specified, '%prog' will search for uploadable changes
in all projects listed in the manifest.

If the --reviewers or --cc options are passed, those emails are
added to the respective list of users, and emails are sent to any
new users.  Users passed as --reviewers must already be registered
with the code review system, or the upload will fail.

While most normal Gerrit options have dedicated command line options,
direct access to the Gerit options is available via --push-options.
This is useful when Gerrit has newer functionality that %prog doesn't
yet support, or doesn't have plans to support.  See the Push Options
documentation for more details:
https://gerrit-review.googlesource.com/Documentation/user-upload.html#push_options

# Configuration

review.URL.autoupload:

To disable the "Upload ... (y/N)?" prompt, you can set a per-project
or global Git configuration option.  If review.URL.autoupload is set
to "true" then repo will assume you always answer "y" at the prompt,
and will not prompt you further.  If it is set to "false" then repo
will assume you always answer "n", and will abort.

review.URL.autoreviewer:

To automatically append a user or mailing list to reviews, you can set
a per-project or global Git option to do so.

review.URL.autocopy:

To automatically copy a user or mailing list to all uploaded reviews,
you can set a per-project or global Git option to do so. Specifically,
review.URL.autocopy can be set to a comma separated list of reviewers
who you always want copied on all uploads with a non-empty --re
argument.

review.URL.username:

Override the username used to connect to Gerrit Code Review.
By default the local part of the email address is used.

The URL must match the review URL listed in the manifest XML file,
or in the .git/config within the project.  For example:

  [remote "origin"]
    url = git://git.example.com/project.git
    review = http://review.example.com/

  [review "http://review.example.com/"]
    autoupload = true
    autocopy = johndoe@company.com,my-team-alias@company.com

review.URL.uploadtopic:

To add a topic branch whenever uploading a commit, you can set a
per-project or global Git option to do so. If review.URL.uploadtopic
is set to "true" then repo will assume you always want the equivalent
of the -t option to the repo command. If unset or set to "false" then
repo will make use of only the command line option.

review.URL.uploadhashtags:

To add hashtags whenever uploading a commit, you can set a per-project
or global Git option to do so. The value of review.URL.uploadhashtags
will be used as comma delimited hashtags like the --hashtag option.

review.URL.uploadlabels:

To add labels whenever uploading a commit, you can set a per-project
or global Git option to do so. The value of review.URL.uploadlabels
will be used as comma delimited labels like the --label option.

review.URL.uploadnotify:

Control e-mail notifications when uploading.
https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify

review.URL.uploadwarningthreshold:

Repo will warn you if you are attempting to upload a large number
of commits in one or more branches. By default, the threshold
is five commits. This option allows you to override the warning
threshold to a different value.

# References

Gerrit Code Review:  https://www.gerritcodereview.com/

"""
    PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

    def _Options(self, p):
        p.add_option(
            "-t",
            "--topic-branch",
            dest="auto_topic",
            action="store_true",
            help="set the topic to the local branch name",
        )
        p.add_option(
            "--topic",
            help="set topic for the change",
        )
        p.add_option(
            "--hashtag",
            "--ht",
            dest="hashtags",
            action="append",
            default=[],
            help="add hashtags (comma delimited) to the review",
        )
        p.add_option(
            "--hashtag-branch",
            "--htb",
            action="store_true",
            help="add local branch name as a hashtag",
        )
        p.add_option(
            "-l",
            "--label",
            dest="labels",
            action="append",
            default=[],
            help="add a label when uploading",
        )
        p.add_option(
            "--pd",
            "--patchset-description",
            dest="patchset_description",
            help="description for patchset",
        )
        p.add_option(
            "--re",
            "--reviewers",
            type="string",
            action="append",
            dest="reviewers",
            help="request reviews from these people",
        )
        p.add_option(
            "--cc",
            type="string",
            action="append",
            dest="cc",
            help="also send email to these email addresses",
        )
        p.add_option(
            "--br",
            "--branch",
            type="string",
            action="store",
            dest="branch",
            help="(local) branch to upload",
        )
        p.add_option(
            "-c",
            "--current-branch",
            dest="current_branch",
            action="store_true",
            help="upload current git branch",
        )
        p.add_option(
            "--no-current-branch",
            dest="current_branch",
            action="store_false",
            help="upload all git branches",
        )
        # Turn this into a warning & remove this someday.
        p.add_option(
            "--cbr",
            dest="current_branch",
            action="store_true",
            help=optparse.SUPPRESS_HELP,
        )
        p.add_option(
            "--ne",
            "--no-emails",
            action="store_false",
            dest="notify",
            default=True,
            help="do not send e-mails on upload",
        )
        p.add_option(
            "-p",
            "--private",
            action="store_true",
            dest="private",
            default=False,
            help="upload as a private change (deprecated; use --wip)",
        )
        p.add_option(
            "-w",
            "--wip",
            action="store_true",
            dest="wip",
            default=False,
            help="upload as a work-in-progress change",
        )
        p.add_option(
            "-r",
            "--ready",
            action="store_true",
            default=False,
            help="mark change as ready (clears work-in-progress setting)",
        )
        p.add_option(
            "-o",
            "--push-option",
            type="string",
            action="append",
            dest="push_options",
            default=[],
            help="additional push options to transmit",
        )
        p.add_option(
            "-D",
            "--destination",
            "--dest",
            type="string",
            action="store",
            dest="dest_branch",
            metavar="BRANCH",
            help="submit for review on this target branch",
        )
        p.add_option(
            "-n",
            "--dry-run",
            dest="dryrun",
            default=False,
            action="store_true",
            help="do everything except actually upload the CL",
        )
        p.add_option(
            "-y",
            "--yes",
            default=False,
            action="store_true",
            help="answer yes to all safe prompts",
        )
        p.add_option(
            "--ignore-untracked-files",
            action="store_true",
            default=False,
            help="ignore untracked files in the working copy",
        )
        p.add_option(
            "--no-ignore-untracked-files",
            dest="ignore_untracked_files",
            action="store_false",
            help="always ask about untracked files in the working copy",
        )
        p.add_option(
            "--no-cert-checks",
            dest="validate_certs",
            action="store_false",
            default=True,
            help="disable verifying ssl certs (unsafe)",
        )
        RepoHook.AddOptionGroup(p, "pre-upload")

    def _SingleBranch(self, opt, branch, people):
        project = branch.project
        name = branch.name
        remote = project.GetBranch(name).remote

        key = "review.%s.autoupload" % remote.review
        answer = project.config.GetBoolean(key)

        if answer is False:
            _die("upload blocked by %s = false" % key)

        if answer is None:
            date = branch.date
            commit_list = branch.commits

            destination = (
                opt.dest_branch or project.dest_branch or project.revisionExpr
            )
            print(
                "Upload project %s/ to remote branch %s%s:"
                % (
                    project.RelPath(local=opt.this_manifest_only),
                    destination,
                    " (private)" if opt.private else "",
                )
            )
            print(
                "  branch %s (%2d commit%s, %s):"
                % (
                    name,
                    len(commit_list),
                    len(commit_list) != 1 and "s" or "",
                    date,
                )
            )
            for commit in commit_list:
                print("         %s" % commit)

            print("to %s (y/N)? " % remote.review, end="", flush=True)
            if opt.yes:
                print("<--yes>")
                answer = True
            else:
                answer = sys.stdin.readline().strip().lower()
                answer = answer in ("y", "yes", "1", "true", "t")
            if not answer:
                _die("upload aborted by user")

        # Perform some basic safety checks prior to uploading.
        if not opt.yes and not _VerifyPendingCommits([branch]):
            _die("upload aborted by user")

        self._UploadAndReport(opt, [branch], people)

    def _MultipleBranches(self, opt, pending, people):
        projects = {}
        branches = {}

        script = []
        script.append("# Uncomment the branches to upload:")
        for project, avail in pending:
            project_path = project.RelPath(local=opt.this_manifest_only)
            script.append("#")
            script.append(f"# project {project_path}/:")

            b = {}
            for branch in avail:
                if branch is None:
                    continue
                name = branch.name
                date = branch.date
                commit_list = branch.commits

                if b:
                    script.append("#")
                destination = (
                    opt.dest_branch
                    or project.dest_branch
                    or project.revisionExpr
                )
                script.append(
                    "#  branch %s (%2d commit%s, %s) to remote branch %s:"
                    % (
                        name,
                        len(commit_list),
                        len(commit_list) != 1 and "s" or "",
                        date,
                        destination,
                    )
                )
                for commit in commit_list:
                    script.append("#         %s" % commit)
                b[name] = branch

            projects[project_path] = project
            branches[project_path] = b
        script.append("")

        script = Editor.EditString("\n".join(script)).split("\n")

        project_re = re.compile(r"^#?\s*project\s*([^\s]+)/:$")
        branch_re = re.compile(r"^\s*branch\s*([^\s(]+)\s*\(.*")

        project = None
        todo = []

        for line in script:
            m = project_re.match(line)
            if m:
                name = m.group(1)
                project = projects.get(name)
                if not project:
                    _die("project %s not available for upload", name)
                continue

            m = branch_re.match(line)
            if m:
                name = m.group(1)
                if not project:
                    _die("project for branch %s not in script", name)
                project_path = project.RelPath(local=opt.this_manifest_only)
                branch = branches[project_path].get(name)
                if not branch:
                    _die("branch %s not in %s", name, project_path)
                todo.append(branch)
        if not todo:
            _die("nothing uncommented for upload")

        # Perform some basic safety checks prior to uploading.
        if not opt.yes and not _VerifyPendingCommits(todo):
            _die("upload aborted by user")

        self._UploadAndReport(opt, todo, people)

    def _AppendAutoList(self, branch, people):
        """
        Appends the list of reviewers in the git project's config.
        Appends the list of users in the CC list in the git project's config if
        a non-empty reviewer list was found.
        """
        name = branch.name
        project = branch.project

        key = "review.%s.autoreviewer" % project.GetBranch(name).remote.review
        raw_list = project.config.GetString(key)
        if raw_list is not None:
            people[0].extend([entry.strip() for entry in raw_list.split(",")])

        key = "review.%s.autocopy" % project.GetBranch(name).remote.review
        raw_list = project.config.GetString(key)
        if raw_list is not None and len(people[0]) > 0:
            people[1].extend([entry.strip() for entry in raw_list.split(",")])

    def _FindGerritChange(self, branch):
        last_pub = branch.project.WasPublished(branch.name)
        if last_pub is None:
            return ""

        refs = branch.GetPublishedRefs()
        try:
            # refs/changes/XYZ/N --> XYZ
            return refs.get(last_pub).split("/")[-2]
        except (AttributeError, IndexError):
            return ""

    def _UploadBranch(self, opt, branch, original_people):
        """Upload Branch."""
        people = copy.deepcopy(original_people)
        self._AppendAutoList(branch, people)

        # Check if topic branches should be sent to the server during
        # upload.
        if opt.topic is None:
            if opt.auto_topic is not True:
                key = "review.%s.uploadtopic" % branch.project.remote.review
                opt.auto_topic = branch.project.config.GetBoolean(key)
            if opt.auto_topic:
                opt.topic = branch.name

        def _ExpandCommaList(value):
            """Split |value| up into comma delimited entries."""
            if not value:
                return
            for ret in value.split(","):
                ret = ret.strip()
                if ret:
                    yield ret

        # Check if hashtags should be included.
        key = "review.%s.uploadhashtags" % branch.project.remote.review
        hashtags = set(_ExpandCommaList(branch.project.config.GetString(key)))
        for tag in opt.hashtags:
            hashtags.update(_ExpandCommaList(tag))
        if opt.hashtag_branch:
            hashtags.add(branch.name)

        # Check if labels should be included.
        key = "review.%s.uploadlabels" % branch.project.remote.review
        labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
        for label in opt.labels:
            labels.update(_ExpandCommaList(label))

        # Handle e-mail notifications.
        if opt.notify is False:
            notify = "NONE"
        else:
            key = "review.%s.uploadnotify" % branch.project.remote.review
            notify = branch.project.config.GetString(key)

        destination = opt.dest_branch or branch.project.dest_branch

        if branch.project.dest_branch and not opt.dest_branch:
            merge_branch = self._GetMergeBranch(
                branch.project, local_branch=branch.name
            )

            full_dest = destination
            if not full_dest.startswith(R_HEADS):
                full_dest = R_HEADS + full_dest
            full_revision = branch.project.revisionExpr
            if not full_revision.startswith(R_HEADS):
                full_revision = R_HEADS + full_revision

            # If the merge branch of the local branch is different from
            # the project's revision AND destination, this might not be
            # intentional.
            if (
                merge_branch
                and merge_branch != full_revision
                and merge_branch != full_dest
            ):
                print(
                    f"For local branch {branch.name}: merge branch "
                    f"{merge_branch} does not match destination branch "
                    f"{destination} and revision {branch.project.revisionExpr}"
                )
                print("skipping upload.")
                print(
                    f"Please use `--destination {destination}` if this "
                    "is intentional"
                )
                branch.uploaded = False
                return

        branch.UploadForReview(
            people,
            dryrun=opt.dryrun,
            topic=opt.topic,
            hashtags=hashtags,
            labels=labels,
            private=opt.private,
            notify=notify,
            wip=opt.wip,
            ready=opt.ready,
            dest_branch=destination,
            validate_certs=opt.validate_certs,
            push_options=opt.push_options,
            patchset_description=opt.patchset_description,
        )

        branch.uploaded = True

    def _UploadAndReport(self, opt, todo, people):
        have_errors = False
        aggregate_errors = []
        for branch in todo:
            try:
                self._UploadBranch(opt, branch, people)
            except (UploadError, GitError) as e:
                self.git_event_log.ErrorEvent(f"upload error: {e}")
                branch.error = e
                aggregate_errors.append(e)
                branch.uploaded = False
                have_errors = True

        print(file=sys.stderr)
        print("-" * 70, file=sys.stderr)

        if have_errors:
            for branch in todo:
                if not branch.uploaded:
                    if len(str(branch.error)) <= 30:
                        fmt = " (%s)"
                    else:
                        fmt = "\n       (%s)"
                    print(
                        ("[FAILED] %-15s %-15s" + fmt)
                        % (
                            branch.project.RelPath(local=opt.this_manifest_only)
                            + "/",
                            branch.name,
                            str(branch.error),
                        ),
                        file=sys.stderr,
                    )
            print()

        for branch in todo:
            if branch.uploaded:
                print(
                    "[OK    ] %-15s %s"
                    % (
                        branch.project.RelPath(local=opt.this_manifest_only)
                        + "/",
                        branch.name,
                    ),
                    file=sys.stderr,
                )

        if have_errors:
            raise UploadExitError(aggregate_errors=aggregate_errors)

    def _GetMergeBranch(self, project, local_branch=None):
        if local_branch is None:
            p = GitCommand(
                project,
                ["rev-parse", "--abbrev-ref", "HEAD"],
                capture_stdout=True,
                capture_stderr=True,
            )
            p.Wait()
            local_branch = p.stdout.strip()
        p = GitCommand(
            project,
            ["config", "--get", "branch.%s.merge" % local_branch],
            capture_stdout=True,
            capture_stderr=True,
        )
        p.Wait()
        merge_branch = p.stdout.strip()
        return merge_branch

    @classmethod
    def _GatherOne(cls, opt, project_idx):
        """Figure out the upload status for |project|."""
        project = cls.get_parallel_context()["projects"][project_idx]
        if opt.current_branch:
            cbr = project.CurrentBranch
            up_branch = project.GetUploadableBranch(cbr)
            avail = [up_branch] if up_branch else None
        else:
            avail = project.GetUploadableBranches(opt.branch)
        return (project_idx, avail)

    def Execute(self, opt, args):
        projects = self.GetProjects(
            args, all_manifests=not opt.this_manifest_only
        )

        def _ProcessResults(_pool, _out, results):
            pending = []
            for result in results:
                project_idx, avail = result
                project = projects[project_idx]
                if avail is None:
                    logger.error(
                        'repo: error: %s: Unable to upload branch "%s". '
                        "You might be able to fix the branch by running:\n"
                        "  git branch --set-upstream-to m/%s",
                        project.RelPath(local=opt.this_manifest_only),
                        project.CurrentBranch,
                        project.manifest.branch,
                    )
                elif avail:
                    pending.append((project, avail))
            return pending

        with self.ParallelContext():
            self.get_parallel_context()["projects"] = projects
            pending = self.ExecuteInParallel(
                opt.jobs,
                functools.partial(self._GatherOne, opt),
                range(len(projects)),
                callback=_ProcessResults,
            )

        if not pending:
            if opt.branch is None:
                logger.error("repo: error: no branches ready for upload")
            else:
                logger.error(
                    'repo: error: no branches named "%s" ready for upload',
                    opt.branch,
                )
            return 1

        manifests = {
            project.manifest.topdir: project.manifest
            for (project, available) in pending
        }
        ret = 0
        for manifest in manifests.values():
            pending_proj_names = [
                project.name
                for (project, available) in pending
                if project.manifest.topdir == manifest.topdir
            ]
            pending_worktrees = [
                project.worktree
                for (project, available) in pending
                if project.manifest.topdir == manifest.topdir
            ]
            hook = RepoHook.FromSubcmd(
                hook_type="pre-upload",
                manifest=manifest,
                opt=opt,
                abort_if_user_denies=True,
            )
            if not hook.Run(
                project_list=pending_proj_names, worktree_list=pending_worktrees
            ):
                if LocalSyncState(manifest).IsPartiallySynced():
                    logger.error(
                        "Partially synced tree detected. Syncing all projects "
                        "may resolve issues you're seeing."
                    )
                ret = 1
        if ret:
            return ret

        reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
        cc = _SplitEmails(opt.cc) if opt.cc else []
        people = (reviewers, cc)

        if len(pending) == 1 and len(pending[0][1]) == 1:
            self._SingleBranch(opt, pending[0][1][0], people)
        else:
            self._MultipleBranches(opt, pending, people)