sync: refactor use of self.manifest
We need to iterate over multiple manifests, and generally use the
outer_client.manifest for multi-manifest support. This refactors the
use of self.manifest into a chosen manifest.
Change-Id: I992f21d610c929675e99555ece9c38df4b635839
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334699
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 3451ab6..9a66e48 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -170,9 +170,9 @@
PARALLEL_JOBS = 1
def _CommonOptions(self, p):
- if self.manifest:
+ if self.outer_client and self.outer_client.manifest:
try:
- self.PARALLEL_JOBS = self.manifest.default.sync_j
+ self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j
except ManifestParseError:
pass
super()._CommonOptions(p)
@@ -270,25 +270,32 @@
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
- def _GetBranch(self):
- """Returns the branch name for getting the approved manifest."""
- p = self.manifest.manifestProject
- b = p.GetBranch(p.CurrentBranch)
+ def _GetBranch(self, manifest_project):
+ """Returns the branch name for getting the approved smartsync manifest.
+
+ Args:
+ manifest_project: the manifestProject to query.
+ """
+ b = manifest_project.GetBranch(manifest_project.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
return branch
- def _GetCurrentBranchOnly(self, opt):
+ def _GetCurrentBranchOnly(self, opt, manifest):
"""Returns whether current-branch or use-superproject options are enabled.
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ manifest: The manifest to use.
+
Returns:
True if a superproject is requested, otherwise the value of the
current_branch option (True, False or None).
"""
- return git_superproject.UseSuperproject(opt.use_superproject, self.manifest) or opt.current_branch_only
+ return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
- def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
+ def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
"""Update revisionId of every project with the SHA from superproject.
This function updates each project's revisionId with SHA from superproject.
@@ -300,6 +307,7 @@
docstring for details.
load_local_manifests: Whether to load local manifests.
superproject_logging_data: A dictionary of superproject data that is to be logged.
+ manifest: The manifest to use.
Returns:
Returns path to the overriding manifest file instead of None.
@@ -312,7 +320,7 @@
if opt.local_only:
manifest_path = superproject.manifest_path
if manifest_path:
- self._ReloadManifest(manifest_path, load_local_manifests)
+ self._ReloadManifest(manifest_path, manifest, load_local_manifests)
return manifest_path
all_projects = self.GetProjects(args,
@@ -323,7 +331,7 @@
manifest_path = update_result.manifest_path
superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
if manifest_path:
- self._ReloadManifest(manifest_path, load_local_manifests)
+ self._ReloadManifest(manifest_path, manifest, load_local_manifests)
else:
if print_messages:
print('warning: Update of revisionId from superproject has failed, '
@@ -366,16 +374,16 @@
quiet=opt.quiet,
verbose=opt.verbose,
output_redir=buf,
- current_branch_only=self._GetCurrentBranchOnly(opt),
+ current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest),
force_sync=opt.force_sync,
clone_bundle=opt.clone_bundle,
- tags=opt.tags, archive=self.manifest.IsArchive,
+ tags=opt.tags, archive=project.manifest.IsArchive,
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
prune=opt.prune,
ssh_proxy=self.ssh_proxy,
- clone_filter=self.manifest.CloneFilter,
- partial_clone_exclude=self.manifest.PartialCloneExclude)
+ clone_filter=project.manifest.CloneFilter,
+ partial_clone_exclude=project.manifest.PartialCloneExclude)
output = buf.getvalue()
if (opt.verbose or not success) and output:
@@ -472,13 +480,13 @@
pm.end()
self._fetch_times.Save()
- if not self.manifest.IsArchive:
+ if not self.outer_client.manifest.IsArchive:
self._GCProjects(projects, opt, err_event)
return (ret, fetched)
def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
- load_local_manifests, ssh_proxy):
+ load_local_manifests, ssh_proxy, manifest):
"""The main network fetch loop.
Args:
@@ -489,11 +497,12 @@
manifest_name: Manifest file to be reloaded.
load_local_manifests: Whether to load local manifests.
ssh_proxy: SSH manager for clients & masters.
+ manifest: The manifest to use.
Returns:
List of all projects that should be checked out.
"""
- rp = self.manifest.repoProject
+ rp = manifest.repoProject
to_fetch = []
now = time.time()
@@ -517,7 +526,7 @@
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
- self._ReloadManifest(manifest_name, load_local_manifests)
+ self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
@@ -552,7 +561,7 @@
Whether the fetch was successful.
"""
start = time.time()
- syncbuf = SyncBuffer(self.manifest.manifestProject.config,
+ syncbuf = SyncBuffer(project.manifest.manifestProject.config,
detach_head=detach_head)
success = False
try:
@@ -689,28 +698,29 @@
t.join()
pm.end()
- def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
+ def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
"""Reload the manfiest from the file specified by the |manifest_name|.
It unloads the manifest if |manifest_name| is None.
Args:
manifest_name: Manifest file to be reloaded.
+ manifest: The manifest to use.
load_local_manifests: Whether to load local manifests.
"""
if manifest_name:
# Override calls Unload already
- self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+ manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
else:
- self.manifest.Unload()
+ manifest.Unload()
- def UpdateProjectList(self, opt):
+ def UpdateProjectList(self, opt, manifest):
new_project_paths = []
for project in self.GetProjects(None, missing_ok=True):
if project.relpath:
new_project_paths.append(project.relpath)
file_name = 'project.list'
- file_path = os.path.join(self.manifest.subdir, file_name)
+ file_path = os.path.join(manifest.subdir, file_name)
old_project_paths = []
if os.path.exists(file_path):
@@ -722,16 +732,16 @@
continue
if path not in new_project_paths:
# If the path has already been deleted, we don't need to do it
- gitdir = os.path.join(self.manifest.topdir, path, '.git')
+ gitdir = os.path.join(manifest.topdir, path, '.git')
if os.path.exists(gitdir):
project = Project(
- manifest=self.manifest,
+ manifest=manifest,
name=path,
remote=RemoteSpec('origin'),
gitdir=gitdir,
objdir=gitdir,
use_git_worktrees=os.path.isfile(gitdir),
- worktree=os.path.join(self.manifest.topdir, path),
+ worktree=os.path.join(manifest.topdir, path),
relpath=path,
revisionExpr='HEAD',
revisionId=None,
@@ -747,7 +757,7 @@
fd.write('\n')
return 0
- def UpdateCopyLinkfileList(self):
+ def UpdateCopyLinkfileList(self, manifest):
"""Save all dests of copyfile and linkfile, and update them if needed.
Returns:
@@ -766,7 +776,7 @@
}
copylinkfile_name = 'copy-link-files.json'
- copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name)
+ copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name)
old_copylinkfile_paths = {}
if os.path.exists(copylinkfile_path):
@@ -797,13 +807,13 @@
json.dump(new_paths, fp)
return True
- def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
- if not self.manifest.manifest_server:
+ def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest):
+ if not manifest.manifest_server:
print('error: cannot smart sync: no manifest server defined in '
'manifest', file=sys.stderr)
sys.exit(1)
- manifest_server = self.manifest.manifest_server
+ manifest_server = manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
@@ -844,7 +854,7 @@
try:
server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
- branch = self._GetBranch()
+ branch = self._GetBranch(manifest.manifestProject)
if 'SYNC_TARGET' in os.environ:
target = os.environ['SYNC_TARGET']
@@ -870,18 +880,18 @@
% (smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
- self._ReloadManifest(manifest_name)
+ self._ReloadManifest(manifest_name, manifest)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str, file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s'
- % (self.manifest.manifest_server, e), file=sys.stderr)
+ % (manifest.manifest_server, e), file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s'
- % (self.manifest.manifest_server, e.errcode, e.errmsg),
+ % (manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
@@ -892,14 +902,14 @@
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
- current_branch_only=self._GetCurrentBranchOnly(opt),
+ current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest),
force_sync=opt.force_sync,
tags=opt.tags,
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
- submodules=self.manifest.HasSubmodules,
- clone_filter=self.manifest.CloneFilter,
- partial_clone_exclude=self.manifest.PartialCloneExclude)
+ submodules=mp.manifest.HasSubmodules,
+ clone_filter=mp.manifest.CloneFilter,
+ partial_clone_exclude=mp.manifest.PartialCloneExclude)
finish = time.time()
self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
start, finish, success)
@@ -907,15 +917,15 @@
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
start = time.time()
- mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+ mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules)
clean = syncbuf.Finish()
self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
start, time.time(), clean)
if not clean:
sys.exit(1)
- self._ReloadManifest(manifest_name)
+ self._ReloadManifest(manifest_name, mp.manifest)
if opt.jobs is None:
- self.jobs = self.manifest.default.sync_j
+ self.jobs = mp.manifest.default.sync_j
def ValidateOptions(self, opt, args):
if opt.force_broken:
@@ -938,7 +948,7 @@
if opt.prune is None:
opt.prune = True
- if self.manifest.is_multimanifest and not opt.this_manifest_only and args:
+ if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
self.OptionParser.error('partial syncs must use --this-manifest-only')
def Execute(self, opt, args):
@@ -948,18 +958,22 @@
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
+ manifest = self.outer_manifest
+ if opt.this_manifest_only or not opt.outer_manifest:
+ manifest = self.manifest
+
if opt.manifest_name:
- self.manifest.Override(opt.manifest_name)
+ manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_path = os.path.join(
- self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
+ manifest.manifestProject.worktree, 'smart_sync_override.xml')
if opt.clone_bundle is None:
- opt.clone_bundle = self.manifest.CloneBundle
+ opt.clone_bundle = manifest.CloneBundle
if opt.smart_sync or opt.smart_tag:
- manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
+ manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest)
else:
if os.path.isfile(smart_sync_manifest_path):
try:
@@ -970,7 +984,7 @@
err_event = multiprocessing.Event()
- rp = self.manifest.repoProject
+ rp = manifest.repoProject
rp.PreSync()
cb = rp.CurrentBranch
if cb:
@@ -980,35 +994,35 @@
'receive updates; run `repo init --repo-rev=stable` to fix.',
file=sys.stderr)
- mp = self.manifest.manifestProject
+ mp = manifest.manifestProject
is_standalone_manifest = bool(mp.standalone_manifest_url)
if not is_standalone_manifest:
mp.PreSync()
if opt.repo_upgraded:
- _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
+ _PostRepoUpgrade(manifest, quiet=opt.quiet)
if not opt.mp_update:
print('Skipping update of local manifest project.')
elif not is_standalone_manifest:
self._UpdateManifestProject(opt, mp, manifest_name)
- load_local_manifests = not self.manifest.HasLocalManifests
- use_superproject = git_superproject.UseSuperproject(opt.use_superproject,
- self.manifest)
- if use_superproject and (self.manifest.IsMirror or self.manifest.IsArchive):
+ load_local_manifests = not manifest.HasLocalManifests
+ use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
+ if use_superproject and (manifest.IsMirror or manifest.IsArchive):
# Don't use superproject, because we have no working tree.
use_superproject = False
if opt.use_superproject is not None:
print('Defaulting to no-use-superproject because there is no working tree.')
superproject_logging_data = {
'superproject': use_superproject,
- 'haslocalmanifests': bool(self.manifest.HasLocalManifests),
- 'hassuperprojecttag': bool(self.manifest.superproject),
+ 'haslocalmanifests': bool(manifest.HasLocalManifests),
+ 'hassuperprojecttag': bool(manifest.superproject),
}
if use_superproject:
manifest_name = self._UpdateProjectsRevisionId(
- opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
+ opt, args, load_local_manifests, superproject_logging_data,
+ manifest) or opt.manifest_name
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
@@ -1031,7 +1045,7 @@
if manifest_name:
manifest.Override(manifest_name)
else:
- manifest.Override(self.manifest.manifestFile)
+ manifest.Override(manifest.manifestFile)
gitc_utils.generate_gitc_manifest(self.gitc_manifest,
manifest,
gitc_projects)
@@ -1041,7 +1055,7 @@
# generate a new args list to represent the opened projects.
# TODO: make this more reliable -- if there's a project name/path overlap,
# this may choose the wrong project.
- args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
+ args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd())
for path in opened_projects]
if not args:
return
@@ -1052,7 +1066,7 @@
err_network_sync = False
err_update_projects = False
- self._fetch_times = _FetchTimes(self.manifest)
+ self._fetch_times = _FetchTimes(manifest)
if not opt.local_only:
with multiprocessing.Manager() as manager:
with ssh.ProxyManager(manager) as ssh_proxy:
@@ -1060,7 +1074,7 @@
ssh_proxy.sock()
all_projects = self._FetchMain(opt, args, all_projects, err_event,
manifest_name, load_local_manifests,
- ssh_proxy)
+ ssh_proxy, manifest)
if opt.network_only:
return
@@ -1076,18 +1090,18 @@
file=sys.stderr)
sys.exit(1)
- if self.manifest.IsMirror or self.manifest.IsArchive:
+ if manifest.IsMirror or manifest.IsArchive:
# bail out now, we have no working tree
return
- if self.UpdateProjectList(opt):
+ if self.UpdateProjectList(opt, manifest):
err_event.set()
err_update_projects = True
if opt.fail_fast:
print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
sys.exit(1)
- err_update_linkfiles = not self.UpdateCopyLinkfileList()
+ err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
if err_update_linkfiles:
err_event.set()
if opt.fail_fast:
@@ -1102,8 +1116,8 @@
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
- if self.manifest.notice:
- print(self.manifest.notice)
+ if manifest.notice:
+ print(manifest.notice)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py
index c1d1758..aad713f 100644
--- a/tests/test_subcmds_sync.py
+++ b/tests/test_subcmds_sync.py
@@ -42,4 +42,4 @@
opts, _ = cmd.OptionParser.parse_args(cli_args)
with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
- assert cmd._GetCurrentBranchOnly(opts) == result
+ assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
