commit 49de8ef584a074982f3fd86aac8a144f48209807
author Mike Frysinger <vapier@google.com> Fri Apr 09 00:21:02 2021 -0400
committer Mike Frysinger <vapier@google.com> Fri Apr 09 15:58:56 2021 +0000
tree 025da3d2d2c2fd2f82f288b25c629ae94778c4f7
parent a1051d8baa15d785dd9d6b55caa3cd2e8b2e0fc8

sync: add separate --jobs options for different steps

The number of jobs one wants to run against the network tends to
factor differently from the number of jobs one wants to run when
checking out local projects.  The former is constrained by your
internet connection & server limits while the later is constrained
by your local computer's CPU & storage I/O.  People with beefier
computers probably want to keep the network/server jobs bounded a
bit lower than the local/checkout jobs.

Change-Id: Ia27ab682c62c09d244a8a1427b1c65acf0116c1c
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/302804
Reviewed-by: Raman Tenneti <rtenneti@google.com>
Tested-by: Mike Frysinger <vapier@google.com>

subcmds/sync.py

diff --git a/subcmds/sync.py b/subcmds/sync.py
index b41b20b..4bcd45d 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -174,6 +174,11 @@
       pass
     super()._Options(p)
 
+    p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
+                 help='number of network jobs to run in parallel (defaults to --jobs)')
+    p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
+                 help='number of local checkout jobs to run in parallel (defaults to --jobs)')
+
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help='obsolete option (to be deleted in the future)')
@@ -364,6 +369,7 @@
   def _Fetch(self, projects, opt, err_event):
     ret = True
 
+    jobs = opt.jobs_network if opt.jobs_network else self.jobs
     fetched = set()
     pm = Progress('Fetching', len(projects), delay=False)
 
@@ -391,7 +397,7 @@
       return ret
 
     # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(projects_list) == 1 or opt.jobs == 1:
+    if len(projects_list) == 1 or jobs == 1:
       if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
         ret = False
     else:
@@ -409,7 +415,7 @@
       else:
         pm.update(inc=0, msg='warming up')
         chunksize = 4
-      with multiprocessing.Pool(opt.jobs) as pool:
+      with multiprocessing.Pool(jobs) as pool:
         results = pool.imap_unordered(
             functools.partial(self._FetchProjectList, opt),
             projects_list,
@@ -463,6 +469,7 @@
       err_results: A list of strings, paths to git repos where checkout failed.
     """
     ret = True
+    jobs = opt.jobs_checkout if opt.jobs_checkout else self.jobs
 
     # Only checkout projects with worktrees.
     all_projects = [x for x in all_projects if x.worktree]
@@ -483,11 +490,11 @@
       return True
 
     # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
+    if len(all_projects) == 1 or jobs == 1:
       if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects):
         ret = False
     else:
-      with multiprocessing.Pool(opt.jobs) as pool:
+      with multiprocessing.Pool(jobs) as pool:
         results = pool.imap_unordered(
             functools.partial(self._CheckoutOne, opt),
             all_projects,