| # Copyright (C) 2008 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import errno |
| import functools |
| import io |
| import multiprocessing |
| import os |
| import re |
| import signal |
| import subprocess |
| import sys |
| |
| from color import Coloring |
| from command import Command |
| from command import DEFAULT_LOCAL_JOBS |
| from command import MirrorSafeCommand |
| from command import WORKER_BATCH_SIZE |
| from error import ManifestInvalidRevisionError |
| from repo_logging import RepoLogger |
| |
| |
| logger = RepoLogger(__file__) |
| _CAN_COLOR = [ |
| "branch", |
| "diff", |
| "grep", |
| "log", |
| ] |
| |
| |
| class ForallColoring(Coloring): |
| def __init__(self, config): |
| Coloring.__init__(self, config, "forall") |
| self.project = self.printer("project", attr="bold") |
| |
| |
| class Forall(Command, MirrorSafeCommand): |
| COMMON = False |
| helpSummary = "Run a shell command in each project" |
| helpUsage = """ |
| %prog [<project>...] -c <command> [<arg>...] |
| %prog -r str1 [str2] ... -c <command> [<arg>...] |
| """ |
| helpDescription = """ |
| Executes the same shell command in each project. |
| |
| The -r option allows running the command only on projects matching |
| regex or wildcard expression. |
| |
| By default, projects are processed non-interactively in parallel. If you want |
| to run interactive commands, make sure to pass --interactive to force --jobs 1. |
| While the processing order of projects is not guaranteed, the order of project |
| output is stable. |
| |
| # Output Formatting |
| |
| The -p option causes '%prog' to bind pipes to the command's stdin, |
| stdout and stderr streams, and pipe all output into a continuous |
| stream that is displayed in a single pager session. Project headings |
| are inserted before the output of each command is displayed. If the |
| command produces no output in a project, no heading is displayed. |
| |
| The formatting convention used by -p is very suitable for some |
| types of searching, e.g. `repo forall -p -c git log -SFoo` will |
| print all commits that add or remove references to Foo. |
| |
| The -v option causes '%prog' to display stderr messages if a |
| command produces output only on stderr. Normally the -p option |
| causes command output to be suppressed until the command produces |
| at least one byte of output on stdout. |
| |
| # Environment |
| |
| pwd is the project's working directory. If the current client is |
| a mirror client, then pwd is the Git repository. |
| |
| REPO_PROJECT is set to the unique name of the project. |
| |
| REPO_PATH is the path relative the the root of the client. |
| |
| REPO_OUTERPATH is the path of the sub manifest's root relative to the root of |
| the client. |
| |
| REPO_INNERPATH is the path relative to the root of the sub manifest. |
| |
| REPO_REMOTE is the name of the remote system from the manifest. |
| |
| REPO_LREV is the name of the revision from the manifest, translated |
| to a local tracking branch. If you need to pass the manifest |
| revision to a locally executed git command, use REPO_LREV. |
| |
| REPO_RREV is the name of the revision from the manifest, exactly |
| as written in the manifest. |
| |
| REPO_COUNT is the total number of projects being iterated. |
| |
| REPO_I is the current (1-based) iteration count. Can be used in |
| conjunction with REPO_COUNT to add a simple progress indicator to your |
| command. |
| |
| REPO__* are any extra environment variables, specified by the |
| "annotation" element under any project element. This can be useful |
| for differentiating trees based on user-specific criteria, or simply |
| annotating tree details. |
| |
| shell positional arguments ($1, $2, .., $#) are set to any arguments |
| following <command>. |
| |
| Example: to list projects: |
| |
| %prog -c 'echo $REPO_PROJECT' |
| |
| Notice that $REPO_PROJECT is quoted to ensure it is expanded in |
| the context of running <command> instead of in the calling shell. |
| |
| Unless -p is used, stdin, stdout, stderr are inherited from the |
| terminal and are not redirected. |
| |
| If -e is used, when a command exits unsuccessfully, '%prog' will abort |
| without iterating through the remaining projects. |
| """ |
| PARALLEL_JOBS = DEFAULT_LOCAL_JOBS |
| |
| @staticmethod |
| def _cmd_option(option, _opt_str, _value, parser): |
| setattr(parser.values, option.dest, list(parser.rargs)) |
| while parser.rargs: |
| del parser.rargs[0] |
| |
| def _Options(self, p): |
| p.add_option( |
| "-r", |
| "--regex", |
| dest="regex", |
| action="store_true", |
| help="execute the command only on projects matching regex or " |
| "wildcard expression", |
| ) |
| p.add_option( |
| "-i", |
| "--inverse-regex", |
| dest="inverse_regex", |
| action="store_true", |
| help="execute the command only on projects not matching regex or " |
| "wildcard expression", |
| ) |
| p.add_option( |
| "-g", |
| "--groups", |
| dest="groups", |
| help="execute the command only on projects matching the specified " |
| "groups", |
| ) |
| p.add_option( |
| "-c", |
| "--command", |
| help="command (and arguments) to execute", |
| dest="command", |
| action="callback", |
| callback=self._cmd_option, |
| ) |
| p.add_option( |
| "-e", |
| "--abort-on-errors", |
| dest="abort_on_errors", |
| action="store_true", |
| help="abort if a command exits unsuccessfully", |
| ) |
| p.add_option( |
| "--ignore-missing", |
| action="store_true", |
| help="silently skip & do not exit non-zero due missing " |
| "checkouts", |
| ) |
| |
| g = p.get_option_group("--quiet") |
| g.add_option( |
| "-p", |
| dest="project_header", |
| action="store_true", |
| help="show project headers before output", |
| ) |
| p.add_option( |
| "--interactive", action="store_true", help="force interactive usage" |
| ) |
| |
| def WantPager(self, opt): |
| return opt.project_header and opt.jobs == 1 |
| |
| def ValidateOptions(self, opt, args): |
| if not opt.command: |
| self.Usage() |
| |
| def Execute(self, opt, args): |
| cmd = [opt.command[0]] |
| all_trees = not opt.this_manifest_only |
| |
| shell = True |
| if re.compile(r"^[a-z0-9A-Z_/\.-]+$").match(cmd[0]): |
| shell = False |
| |
| if shell: |
| cmd.append(cmd[0]) |
| cmd.extend(opt.command[1:]) |
| |
| # Historically, forall operated interactively, and in serial. If the |
| # user has selected 1 job, then default to interacive mode. |
| if opt.jobs == 1: |
| opt.interactive = True |
| |
| if opt.project_header and not shell and cmd[0] == "git": |
| # If this is a direct git command that can enable colorized |
| # output and the user prefers coloring, add --color into the |
| # command line because we are going to wrap the command into |
| # a pipe and git won't know coloring should activate. |
| # |
| for cn in cmd[1:]: |
| if not cn.startswith("-"): |
| break |
| else: |
| cn = None |
| if cn and cn in _CAN_COLOR: |
| |
| class ColorCmd(Coloring): |
| def __init__(self, config, cmd): |
| Coloring.__init__(self, config, cmd) |
| |
| if ColorCmd(self.manifest.manifestProject.config, cn).is_on: |
| cmd.insert(cmd.index(cn) + 1, "--color") |
| |
| mirror = self.manifest.IsMirror |
| rc = 0 |
| |
| smart_sync_manifest_name = "smart_sync_override.xml" |
| smart_sync_manifest_path = os.path.join( |
| self.manifest.manifestProject.worktree, smart_sync_manifest_name |
| ) |
| |
| if os.path.isfile(smart_sync_manifest_path): |
| self.manifest.Override(smart_sync_manifest_path) |
| |
| if opt.regex: |
| projects = self.FindProjects(args, all_manifests=all_trees) |
| elif opt.inverse_regex: |
| projects = self.FindProjects( |
| args, inverse=True, all_manifests=all_trees |
| ) |
| else: |
| projects = self.GetProjects( |
| args, groups=opt.groups, all_manifests=all_trees |
| ) |
| |
| os.environ["REPO_COUNT"] = str(len(projects)) |
| |
| try: |
| config = self.manifest.manifestProject.config |
| with multiprocessing.Pool(opt.jobs, InitWorker) as pool: |
| results_it = pool.imap( |
| functools.partial( |
| DoWorkWrapper, mirror, opt, cmd, shell, config |
| ), |
| enumerate(projects), |
| chunksize=WORKER_BATCH_SIZE, |
| ) |
| first = True |
| for r, output in results_it: |
| if output: |
| if first: |
| first = False |
| elif opt.project_header: |
| print() |
| # To simplify the DoWorkWrapper, take care of automatic |
| # newlines. |
| end = "\n" |
| if output[-1] == "\n": |
| end = "" |
| print(output, end=end) |
| rc = rc or r |
| if r != 0 and opt.abort_on_errors: |
| raise Exception("Aborting due to previous error") |
| except (KeyboardInterrupt, WorkerKeyboardInterrupt): |
| # Catch KeyboardInterrupt raised inside and outside of workers |
| rc = rc or errno.EINTR |
| except Exception as e: |
| # Catch any other exceptions raised |
| logger.error( |
| "forall: unhandled error, terminating the pool: %s: %s", |
| type(e).__name__, |
| e, |
| ) |
| rc = rc or getattr(e, "errno", 1) |
| if rc != 0: |
| sys.exit(rc) |
| |
| |
| class WorkerKeyboardInterrupt(Exception): |
| """Keyboard interrupt exception for worker processes.""" |
| |
| |
| def InitWorker(): |
| signal.signal(signal.SIGINT, signal.SIG_IGN) |
| |
| |
| def DoWorkWrapper(mirror, opt, cmd, shell, config, args): |
| """A wrapper around the DoWork() method. |
| |
| Catch the KeyboardInterrupt exceptions here and re-raise them as a |
| different, ``Exception``-based exception to stop it flooding the console |
| with stacktraces and making the parent hang indefinitely. |
| |
| """ |
| cnt, project = args |
| try: |
| return DoWork(project, mirror, opt, cmd, shell, cnt, config) |
| except KeyboardInterrupt: |
| print("%s: Worker interrupted" % project.name) |
| raise WorkerKeyboardInterrupt() |
| |
| |
| def DoWork(project, mirror, opt, cmd, shell, cnt, config): |
| env = os.environ.copy() |
| |
| def setenv(name, val): |
| if val is None: |
| val = "" |
| env[name] = val |
| |
| setenv("REPO_PROJECT", project.name) |
| setenv("REPO_OUTERPATH", project.manifest.path_prefix) |
| setenv("REPO_INNERPATH", project.relpath) |
| setenv("REPO_PATH", project.RelPath(local=opt.this_manifest_only)) |
| setenv("REPO_REMOTE", project.remote.name) |
| try: |
| # If we aren't in a fully synced state and we don't have the ref the |
| # manifest wants, then this will fail. Ignore it for the purposes of |
| # this code. |
| lrev = "" if mirror else project.GetRevisionId() |
| except ManifestInvalidRevisionError: |
| lrev = "" |
| setenv("REPO_LREV", lrev) |
| setenv("REPO_RREV", project.revisionExpr) |
| setenv("REPO_UPSTREAM", project.upstream) |
| setenv("REPO_DEST_BRANCH", project.dest_branch) |
| setenv("REPO_I", str(cnt + 1)) |
| for annotation in project.annotations: |
| setenv("REPO__%s" % (annotation.name), annotation.value) |
| |
| if mirror: |
| setenv("GIT_DIR", project.gitdir) |
| cwd = project.gitdir |
| else: |
| cwd = project.worktree |
| |
| if not os.path.exists(cwd): |
| # Allow the user to silently ignore missing checkouts so they can run on |
| # partial checkouts (good for infra recovery tools). |
| if opt.ignore_missing: |
| return (0, "") |
| |
| output = "" |
| if (opt.project_header and opt.verbose) or not opt.project_header: |
| output = "skipping %s/" % project.RelPath( |
| local=opt.this_manifest_only |
| ) |
| return (1, output) |
| |
| if opt.verbose: |
| stderr = subprocess.STDOUT |
| else: |
| stderr = subprocess.DEVNULL |
| |
| stdin = None if opt.interactive else subprocess.DEVNULL |
| |
| result = subprocess.run( |
| cmd, |
| cwd=cwd, |
| shell=shell, |
| env=env, |
| check=False, |
| encoding="utf-8", |
| errors="replace", |
| stdin=stdin, |
| stdout=subprocess.PIPE, |
| stderr=stderr, |
| ) |
| |
| output = result.stdout |
| if opt.project_header: |
| if output: |
| buf = io.StringIO() |
| out = ForallColoring(config) |
| out.redirect(buf) |
| if mirror: |
| project_header_path = project.name |
| else: |
| project_header_path = project.RelPath( |
| local=opt.this_manifest_only |
| ) |
| out.project("project %s/" % project_header_path) |
| out.nl() |
| buf.write(output) |
| output = buf.getvalue() |
| return (result.returncode, output) |