status: Use multiprocessing for `repo status -j<num>` instead of threading
This change increases the speed of the command with parallelization with
processes. The parallelization with threads doesn't work well, and
increasing the number of jobs to many (8 threads ~) didn't increase the speed.
Possibly, the global interpreter lock of Python affects.
Bug: https://crbug.com/gerrit/12389
Change-Id: Icbe5df8ba037dd91422b96f4e43708068d7be924
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/279936
Tested-by: Kimiyuki Onaka <kimiyuki@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
diff --git a/project.py b/project.py
index 972cfa5..fdbf9e4 100644
--- a/project.py
+++ b/project.py
@@ -3208,6 +3208,13 @@
self._bare = bare
self._gitdir = gitdir
+ # __getstate__ and __setstate__ are required for pickling because __getattr__ exists.
+ def __getstate__(self):
+ return (self._project, self._bare, self._gitdir)
+
+ def __setstate__(self, state):
+ self._project, self._bare, self._gitdir = state
+
def LsOthers(self):
p = GitCommand(self._project,
['ls-files',
diff --git a/subcmds/status.py b/subcmds/status.py
index 8537e6c..c380dba 100644
--- a/subcmds/status.py
+++ b/subcmds/status.py
@@ -16,17 +16,13 @@
from __future__ import print_function
+import functools
import glob
-import itertools
+import multiprocessing
import os
from command import PagedCommand
-try:
- import threading as _threading
-except ImportError:
- import dummy_threading as _threading
-
from color import Coloring
import platform_utils
@@ -95,25 +91,20 @@
p.add_option('-q', '--quiet', action='store_true',
help="only print the name of modified projects")
- def _StatusHelper(self, project, clean_counter, sem, quiet):
+ def _StatusHelper(self, quiet, project):
"""Obtains the status for a specific project.
Obtains the status for a project, redirecting the output to
- the specified object. It will release the semaphore
- when done.
+ the specified object.
Args:
+ quiet: Where to output the status.
project: Project to get status of.
- clean_counter: Counter for clean projects.
- sem: Semaphore, will call release() when complete.
- output: Where to output the status.
+
+ Returns:
+ The status of the project.
"""
- try:
- state = project.PrintWorkTreeStatus(quiet=quiet)
- if state == 'CLEAN':
- next(clean_counter)
- finally:
- sem.release()
+ return project.PrintWorkTreeStatus(quiet=quiet)
def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
"""find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
@@ -133,27 +124,18 @@
def Execute(self, opt, args):
all_projects = self.GetProjects(args)
- counter = itertools.count()
+ counter = 0
if opt.jobs == 1:
for project in all_projects:
state = project.PrintWorkTreeStatus(quiet=opt.quiet)
if state == 'CLEAN':
- next(counter)
+ counter += 1
else:
- sem = _threading.Semaphore(opt.jobs)
- threads = []
- for project in all_projects:
- sem.acquire()
-
- t = _threading.Thread(target=self._StatusHelper,
- args=(project, counter, sem, opt.quiet))
- threads.append(t)
- t.daemon = True
- t.start()
- for t in threads:
- t.join()
- if not opt.quiet and len(all_projects) == next(counter):
+ with multiprocessing.Pool(opt.jobs) as pool:
+ states = pool.map(functools.partial(self._StatusHelper, opt.quiet), all_projects)
+ counter += states.count('CLEAN')
+ if not opt.quiet and len(all_projects) == counter:
print('nothing to commit (working directory clean)')
if opt.orphans:
