sync: refactor main fetch loop
This is a large chunk of code that is largely isolated. Move it into
a class method to make it easier to manage & reason about, and in a
follow up CL, easier to scope.
Bug: https://crbug.com/gerrit/12389
Change-Id: I0c69d95a9e03478d347b761580b2343bffa012d5
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/305484
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Chris Mcdonald <cjmcdonald@google.com>
diff --git a/subcmds/sync.py b/subcmds/sync.py
index d3c326a..381e9e7 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -439,6 +439,63 @@
return (ret, fetched)
+ def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
+ load_local_manifests):
+ """The main network fetch loop.
+
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ args: Command line args used to filter out projects.
+ all_projects: List of all projects that should be checked out.
+ err_event: Whether an error was hit while processing.
+ manifest_name: Manifest file to be reloaded.
+ load_local_manifests: Whether to load local manifests.
+ """
+ rp = self.manifest.repoProject
+
+ to_fetch = []
+ now = time.time()
+ if _ONE_DAY_S <= (now - rp.LastFetch):
+ to_fetch.append(rp)
+ to_fetch.extend(all_projects)
+ to_fetch.sort(key=self._fetch_times.Get, reverse=True)
+
+ success, fetched = self._Fetch(to_fetch, opt, err_event)
+ if not success:
+ err_event.set()
+
+ _PostRepoFetch(rp, opt.repo_verify)
+ if opt.network_only:
+ # bail out now; the rest touches the working tree
+ if err_event.is_set():
+ print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
+ sys.exit(1)
+ return
+
+ # Iteratively fetch missing and/or nested unregistered submodules
+ previously_missing_set = set()
+ while True:
+ self._ReloadManifest(manifest_name, load_local_manifests)
+ all_projects = self.GetProjects(args,
+ missing_ok=True,
+ submodules_ok=opt.fetch_submodules)
+ missing = []
+ for project in all_projects:
+ if project.gitdir not in fetched:
+ missing.append(project)
+ if not missing:
+ break
+ # Stop us from non-stopped fetching actually-missing repos: If set of
+ # missing repos has not been changed from last fetch, we break.
+ missing_set = set(p.name for p in missing)
+ if previously_missing_set == missing_set:
+ break
+ previously_missing_set = missing_set
+ success, new_fetched = self._Fetch(missing, opt, err_event)
+ if not success:
+ err_event.set()
+ fetched.update(new_fetched)
+
def _CheckoutOne(self, detach_head, force_sync, project):
"""Checkout work tree for one project
@@ -921,48 +978,8 @@
self._fetch_times = _FetchTimes(self.manifest)
if not opt.local_only:
- to_fetch = []
- now = time.time()
- if _ONE_DAY_S <= (now - rp.LastFetch):
- to_fetch.append(rp)
- to_fetch.extend(all_projects)
- to_fetch.sort(key=self._fetch_times.Get, reverse=True)
-
- success, fetched = self._Fetch(to_fetch, opt, err_event)
- if not success:
- err_event.set()
-
- _PostRepoFetch(rp, opt.repo_verify)
- if opt.network_only:
- # bail out now; the rest touches the working tree
- if err_event.is_set():
- print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
- sys.exit(1)
- return
-
- # Iteratively fetch missing and/or nested unregistered submodules
- previously_missing_set = set()
- while True:
- self._ReloadManifest(manifest_name, load_local_manifests)
- all_projects = self.GetProjects(args,
- missing_ok=True,
- submodules_ok=opt.fetch_submodules)
- missing = []
- for project in all_projects:
- if project.gitdir not in fetched:
- missing.append(project)
- if not missing:
- break
- # Stop us from non-stopped fetching actually-missing repos: If set of
- # missing repos has not been changed from last fetch, we break.
- missing_set = set(p.name for p in missing)
- if previously_missing_set == missing_set:
- break
- previously_missing_set = missing_set
- success, new_fetched = self._Fetch(missing, opt, err_event)
- if not success:
- err_event.set()
- fetched.update(new_fetched)
+ self._FetchMain(opt, args, all_projects, err_event, manifest_name,
+ load_local_manifests)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():