forall: simplify arg passing to worker children
The ProjectArgs function can be inlined which simplifies it quite a
bit. We shouldn't need the custom exception handling here either.
This also makes the next commit easier to review.
Change-Id: If3be04f58c302c36a0f20b99de0f67e78beac141
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/298723
Reviewed-by: Michael Mortensen <mmortensen@google.com>
Tested-by: Mike Frysinger <vapier@google.com>
diff --git a/subcmds/forall.py b/subcmds/forall.py
index a2ccb7b..24fec5c 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -13,6 +13,7 @@
# limitations under the License.
import errno
+import functools
import io
import multiprocessing
import re
@@ -240,8 +241,8 @@
config = self.manifest.manifestProject.config
with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
results_it = pool.imap(
- DoWorkWrapper,
- self.ProjectArgs(projects, mirror, opt, cmd, shell, config),
+ functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config),
+ enumerate(self._SerializeProject(x) for x in projects),
chunksize=WORKER_BATCH_SIZE)
first = True
for (r, output) in results_it:
@@ -270,21 +271,6 @@
if rc != 0:
sys.exit(rc)
- def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
- for cnt, p in enumerate(projects):
- try:
- project = self._SerializeProject(p)
- except Exception as e:
- print('Project list error on project %s: %s: %s' %
- (p.name, type(e).__name__, e),
- file=sys.stderr)
- return
- except KeyboardInterrupt:
- print('Project list interrupted',
- file=sys.stderr)
- return
- yield [mirror, opt, cmd, shell, cnt, config, project]
-
class WorkerKeyboardInterrupt(Exception):
""" Keyboard interrupt exception for worker processes. """
@@ -294,7 +280,7 @@
signal.signal(signal.SIGINT, signal.SIG_IGN)
-def DoWorkWrapper(args):
+def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
""" A wrapper around the DoWork() method.
Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
@@ -302,9 +288,9 @@
and making the parent hang indefinitely.
"""
- project = args.pop()
+ cnt, project = args
try:
- return DoWork(project, *args)
+ return DoWork(project, mirror, opt, cmd, shell, cnt, config)
except KeyboardInterrupt:
print('%s: Worker interrupted' % project['name'])
raise WorkerKeyboardInterrupt()