sync: add multi-manifest support
With this change, partial syncs (sync with a project list) are again
supported.
If the updated manifest includes new sub manifests, download them
inheriting options from the parent manifestProject.
Change-Id: Id952f85df2e26d34e38b251973be26434443ff56
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334819
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: LaMont Jones <lamontjones@google.com>
diff --git a/command.py b/command.py
index 12fe417..bd6d081 100644
--- a/command.py
+++ b/command.py
@@ -144,11 +144,10 @@
help=f'number of jobs to run in parallel (default: {default})')
m = p.add_option_group('Multi-manifest options')
- m.add_option('--outer-manifest', action='store_true',
+ m.add_option('--outer-manifest', action='store_true', default=None,
help='operate starting at the outermost manifest')
m.add_option('--no-outer-manifest', dest='outer_manifest',
- action='store_false', default=None,
- help='do not operate on outer manifests')
+ action='store_false', help='do not operate on outer manifests')
m.add_option('--this-manifest-only', action='store_true', default=None,
help='only operate on this (sub)manifest')
m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -186,6 +185,10 @@
"""Validate common options."""
opt.quiet = opt.output_mode is False
opt.verbose = opt.output_mode is True
+ if opt.outer_manifest is None:
+ # By default, treat multi-manifest instances as a single manifest from
+ # the user's perspective.
+ opt.outer_manifest = True
def ValidateOptions(self, opt, args):
"""Validate the user options & arguments before executing.
@@ -385,7 +388,7 @@
opt: The command options.
"""
top = self.outer_manifest
- if opt.outer_manifest is False or opt.this_manifest_only:
+ if not opt.outer_manifest or opt.this_manifest_only:
top = self.manifest
yield top
if not opt.this_manifest_only:
diff --git a/main.py b/main.py
index 34dfb77..c54f928 100755
--- a/main.py
+++ b/main.py
@@ -294,8 +294,7 @@
cmd.ValidateOptions(copts, cargs)
this_manifest_only = copts.this_manifest_only
- # If not specified, default to using the outer manifest.
- outer_manifest = copts.outer_manifest is not False
+ outer_manifest = copts.outer_manifest
if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
result = cmd.Execute(copts, cargs)
elif outer_manifest and repo_client.manifest.is_submanifest:
diff --git a/project.py b/project.py
index faa6b32..8668bae 100644
--- a/project.py
+++ b/project.py
@@ -3467,6 +3467,67 @@
"""Return the name of the platform."""
return platform.system().lower()
+ def SyncWithPossibleInit(self, submanifest, verbose=False,
+ current_branch_only=False, tags='', git_event_log=None):
+ """Sync a manifestProject, possibly for the first time.
+
+ Call Sync() with arguments from the most recent `repo init`. If this is a
+ new sub manifest, then inherit options from the parent's manifestProject.
+
+ This is used by subcmds.Sync() to do an initial download of new sub
+ manifests.
+
+ Args:
+ submanifest: an XmlSubmanifest, the submanifest to re-sync.
+ verbose: a boolean, whether to show all output, rather than only errors.
+ current_branch_only: a boolean, whether to only fetch the current manifest
+ branch from the server.
+ tags: a boolean, whether to fetch tags.
+ git_event_log: an EventLog, for git tracing.
+ """
+ # TODO(lamontjones): when refactoring sync (and init?) consider how to
+ # better get the init options that we should use when syncing uncovers a new
+ # submanifest.
+ git_event_log = git_event_log or EventLog()
+ spec = submanifest.ToSubmanifestSpec()
+ # Use the init options from the existing manifestProject, or the parent if
+ # it doesn't exist.
+ #
+ # Today, we only support changing manifest_groups on the sub-manifest, with
+ # no supported-for-the-user way to change the other arguments from those
+ # specified by the outermost manifest.
+ #
+ # TODO(lamontjones): determine which of these should come from the outermost
+ # manifest and which should come from the parent manifest.
+ mp = self if self.Exists else submanifest.parent.manifestProject
+ return self.Sync(
+ manifest_url=spec.manifestUrl,
+ manifest_branch=spec.revision,
+ standalone_manifest=mp.standalone_manifest_url,
+ groups=mp.manifest_groups,
+ platform=mp.manifest_platform,
+ mirror=mp.mirror,
+ dissociate=mp.dissociate,
+ reference=mp.reference,
+ worktree=mp.use_worktree,
+ submodules=mp.submodules,
+ archive=mp.archive,
+ partial_clone=mp.partial_clone,
+ clone_filter=mp.clone_filter,
+ partial_clone_exclude=mp.partial_clone_exclude,
+ clone_bundle=mp.clone_bundle,
+ git_lfs=mp.git_lfs,
+ use_superproject=mp.use_superproject,
+ verbose=verbose,
+ current_branch_only=current_branch_only,
+ tags=tags,
+ depth=mp.depth,
+ git_event_log=git_event_log,
+ manifest_name=spec.manifestName,
+ this_manifest_only=True,
+ outer_manifest=False,
+ )
+
def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
standalone_manifest=False, groups='', mirror=False, reference='',
dissociate=False, worktree=False, submodules=False, archive=False,
diff --git a/subcmds/init.py b/subcmds/init.py
index 6e3951c..cced44d 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -89,11 +89,10 @@
def _Options(self, p, gitc_init=False):
Wrapper().InitParser(p, gitc_init=gitc_init)
m = p.add_option_group('Multi-manifest')
- m.add_option('--outer-manifest', action='store_true',
+ m.add_option('--outer-manifest', action='store_true', default=True,
help='operate starting at the outermost manifest')
m.add_option('--no-outer-manifest', dest='outer_manifest',
- action='store_false', default=None,
- help='do not operate on outer manifests')
+ action='store_false', help='do not operate on outer manifests')
m.add_option('--this-manifest-only', action='store_true', default=None,
help='only operate on this (sub)manifest')
m.add_option('--no-this-manifest-only', '--all-manifests',
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 9a66e48..0abe23d 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import collections
import functools
import http.cookiejar as cookielib
import io
@@ -66,7 +67,7 @@
class Sync(Command, MirrorSafeCommand):
jobs = 1
COMMON = True
- MULTI_MANIFEST_SUPPORT = False
+ MULTI_MANIFEST_SUPPORT = True
helpSummary = "Update working tree to the latest revision"
helpUsage = """
%prog [<project>...]
@@ -295,52 +296,92 @@
"""
return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
- def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
- """Update revisionId of every project with the SHA from superproject.
+ def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
+ manifest):
+ """Update revisionId of projects with the commit hash from the superproject.
- This function updates each project's revisionId with SHA from superproject.
- It writes the updated manifest into a file and reloads the manifest from it.
+ This function updates each project's revisionId with the commit hash from
+ the superproject. It writes the updated manifest into a file and reloads
+ the manifest from it. When appropriate, sub manifests are also processed.
Args:
opt: Program options returned from optparse. See _Options().
args: Arguments to pass to GetProjects. See the GetProjects
docstring for details.
- load_local_manifests: Whether to load local manifests.
- superproject_logging_data: A dictionary of superproject data that is to be logged.
+ superproject_logging_data: A dictionary of superproject data to log.
manifest: The manifest to use.
-
- Returns:
- Returns path to the overriding manifest file instead of None.
"""
- superproject = self.manifest.superproject
- superproject.SetQuiet(opt.quiet)
- print_messages = git_superproject.PrintMessages(opt.use_superproject,
- self.manifest)
- superproject.SetPrintMessages(print_messages)
+ have_superproject = manifest.superproject or any(
+ m.superproject for m in manifest.all_children)
+ if not have_superproject:
+ return
+
if opt.local_only:
- manifest_path = superproject.manifest_path
+ manifest_path = manifest.superproject.manifest_path
if manifest_path:
- self._ReloadManifest(manifest_path, manifest, load_local_manifests)
- return manifest_path
+ self._ReloadManifest(manifest_path, manifest)
+ return
all_projects = self.GetProjects(args,
missing_ok=True,
- submodules_ok=opt.fetch_submodules)
- update_result = superproject.UpdateProjectsRevisionId(
- all_projects, git_event_log=self.git_event_log)
- manifest_path = update_result.manifest_path
- superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
- if manifest_path:
- self._ReloadManifest(manifest_path, manifest, load_local_manifests)
+ submodules_ok=opt.fetch_submodules,
+ manifest=manifest,
+ all_manifests=not opt.this_manifest_only)
+
+ per_manifest = collections.defaultdict(list)
+ manifest_paths = {}
+ if opt.this_manifest_only:
+ per_manifest[manifest.path_prefix] = all_projects
else:
- if print_messages:
- print('warning: Update of revisionId from superproject has failed, '
- 'repo sync will not use superproject to fetch the source. ',
- 'Please resync with the --no-use-superproject option to avoid this repo warning.',
- file=sys.stderr)
- if update_result.fatal and opt.use_superproject is not None:
- sys.exit(1)
- return manifest_path
+ for p in all_projects:
+ per_manifest[p.manifest.path_prefix].append(p)
+
+ superproject_logging_data = {}
+ need_unload = False
+ for m in self.ManifestList(opt):
+ if not m.path_prefix in per_manifest:
+ continue
+ use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
+ if superproject_logging_data:
+ superproject_logging_data['multimanifest'] = True
+ superproject_logging_data.update(
+ superproject=use_super,
+ haslocalmanifests=bool(m.HasLocalManifests),
+ hassuperprojecttag=bool(m.superproject),
+ )
+ if use_super and (m.IsMirror or m.IsArchive):
+ # Don't use superproject, because we have no working tree.
+ use_super = False
+ superproject_logging_data['superproject'] = False
+ superproject_logging_data['noworktree'] = True
+ if opt.use_superproject is not False:
+ print(f'{m.path_prefix}: not using superproject because there is no '
+ 'working tree.')
+
+ if not use_super:
+ continue
+ m.superproject.SetQuiet(opt.quiet)
+ print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
+ m.superproject.SetPrintMessages(print_messages)
+ update_result = m.superproject.UpdateProjectsRevisionId(
+ per_manifest[m.path_prefix], git_event_log=self.git_event_log)
+ manifest_path = update_result.manifest_path
+ superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
+ if manifest_path:
+ m.SetManifestOverride(manifest_path)
+ need_unload = True
+ else:
+ if print_messages:
+ print(f'{m.path_prefix}: warning: Update of revisionId from '
+ 'superproject has failed, repo sync will not use superproject '
+ 'to fetch the source. ',
+ 'Please resync with the --no-use-superproject option to avoid '
+ 'this repo warning.',
+ file=sys.stderr)
+ if update_result.fatal and opt.use_superproject is not None:
+ sys.exit(1)
+ if need_unload:
+ m.outer_client.manifest.Unload()
def _FetchProjectList(self, opt, projects):
"""Main function of the fetch worker.
@@ -485,8 +526,8 @@
return (ret, fetched)
- def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
- load_local_manifests, ssh_proxy, manifest):
+ def _FetchMain(self, opt, args, all_projects, err_event,
+ ssh_proxy, manifest):
"""The main network fetch loop.
Args:
@@ -494,8 +535,6 @@
args: Command line args used to filter out projects.
all_projects: List of all projects that should be fetched.
err_event: Whether an error was hit while processing.
- manifest_name: Manifest file to be reloaded.
- load_local_manifests: Whether to load local manifests.
ssh_proxy: SSH manager for clients & masters.
manifest: The manifest to use.
@@ -526,10 +565,12 @@
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
- self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
+ self._ReloadManifest(None, manifest)
all_projects = self.GetProjects(args,
missing_ok=True,
- submodules_ok=opt.fetch_submodules)
+ submodules_ok=opt.fetch_submodules,
+ manifest=manifest,
+ all_manifests=not opt.this_manifest_only)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
@@ -624,7 +665,7 @@
for project in projects:
# Make sure pruning never kicks in with shared projects.
if (not project.use_git_worktrees and
- len(project.manifest.GetProjectsWithName(project.name)) > 1):
+ len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
if not opt.quiet:
print('\r%s: Shared project %s found, disabling pruning.' %
(project.relpath, project.name))
@@ -698,7 +739,7 @@
t.join()
pm.end()
- def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
+ def _ReloadManifest(self, manifest_name, manifest):
"""Reload the manfiest from the file specified by the |manifest_name|.
It unloads the manifest if |manifest_name| is None.
@@ -706,17 +747,29 @@
Args:
manifest_name: Manifest file to be reloaded.
manifest: The manifest to use.
- load_local_manifests: Whether to load local manifests.
"""
if manifest_name:
# Override calls Unload already
- manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+ manifest.Override(manifest_name)
else:
manifest.Unload()
def UpdateProjectList(self, opt, manifest):
+ """Update the cached projects list for |manifest|
+
+ In a multi-manifest checkout, each manifest has its own project.list.
+
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ manifest: The manifest to use.
+
+ Returns:
+ 0: success
+ 1: failure
+ """
new_project_paths = []
- for project in self.GetProjects(None, missing_ok=True):
+ for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
+ all_manifests=False):
if project.relpath:
new_project_paths.append(project.relpath)
file_name = 'project.list'
@@ -766,7 +819,8 @@
new_paths = {}
new_linkfile_paths = []
new_copyfile_paths = []
- for project in self.GetProjects(None, missing_ok=True):
+ for project in self.GetProjects(None, missing_ok=True,
+ manifest=manifest, all_manifests=False):
new_linkfile_paths.extend(x.dest for x in project.linkfiles)
new_copyfile_paths.extend(x.dest for x in project.copyfiles)
@@ -897,8 +951,40 @@
return manifest_name
+ def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
+ """Fetch & update the local manifest project.
+
+ After syncing the manifest project, if the manifest has any sub manifests,
+ those are recursively processed.
+
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ mp: the manifestProject to query.
+ manifest_name: Manifest file to be reloaded.
+ """
+ if not mp.standalone_manifest_url:
+ self._UpdateManifestProject(opt, mp, manifest_name)
+
+ if mp.manifest.submanifests:
+ for submanifest in mp.manifest.submanifests.values():
+ child = submanifest.repo_client.manifest
+ child.manifestProject.SyncWithPossibleInit(
+ submanifest,
+ current_branch_only=self._GetCurrentBranchOnly(opt, child),
+ verbose=opt.verbose,
+ tags=opt.tags,
+ git_event_log=self.git_event_log,
+ )
+ self._UpdateAllManifestProjects(opt, child.manifestProject, None)
+
def _UpdateManifestProject(self, opt, mp, manifest_name):
- """Fetch & update the local manifest project."""
+ """Fetch & update the local manifest project.
+
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ mp: the manifestProject to query.
+ manifest_name: Manifest file to be reloaded.
+ """
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
@@ -924,6 +1010,7 @@
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name, mp.manifest)
+
if opt.jobs is None:
self.jobs = mp.manifest.default.sync_j
@@ -948,9 +1035,6 @@
if opt.prune is None:
opt.prune = True
- if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
- self.OptionParser.error('partial syncs must use --this-manifest-only')
-
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
@@ -959,7 +1043,7 @@
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
manifest = self.outer_manifest
- if opt.this_manifest_only or not opt.outer_manifest:
+ if not opt.outer_manifest:
manifest = self.manifest
if opt.manifest_name:
@@ -994,39 +1078,26 @@
'receive updates; run `repo init --repo-rev=stable` to fix.',
file=sys.stderr)
- mp = manifest.manifestProject
- is_standalone_manifest = bool(mp.standalone_manifest_url)
- if not is_standalone_manifest:
- mp.PreSync()
+ for m in self.ManifestList(opt):
+ mp = m.manifestProject
+ is_standalone_manifest = bool(mp.standalone_manifest_url)
+ if not is_standalone_manifest:
+ mp.PreSync()
- if opt.repo_upgraded:
- _PostRepoUpgrade(manifest, quiet=opt.quiet)
+ if opt.repo_upgraded:
+ _PostRepoUpgrade(m, quiet=opt.quiet)
- if not opt.mp_update:
+ if opt.mp_update:
+ self._UpdateAllManifestProjects(opt, mp, manifest_name)
+ else:
print('Skipping update of local manifest project.')
- elif not is_standalone_manifest:
- self._UpdateManifestProject(opt, mp, manifest_name)
- load_local_manifests = not manifest.HasLocalManifests
- use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
- if use_superproject and (manifest.IsMirror or manifest.IsArchive):
- # Don't use superproject, because we have no working tree.
- use_superproject = False
- if opt.use_superproject is not None:
- print('Defaulting to no-use-superproject because there is no working tree.')
- superproject_logging_data = {
- 'superproject': use_superproject,
- 'haslocalmanifests': bool(manifest.HasLocalManifests),
- 'hassuperprojecttag': bool(manifest.superproject),
- }
- if use_superproject:
- manifest_name = self._UpdateProjectsRevisionId(
- opt, args, load_local_manifests, superproject_logging_data,
- manifest) or opt.manifest_name
+ superproject_logging_data = {}
+ self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
+ manifest)
if self.gitc_manifest:
- gitc_manifest_projects = self.GetProjects(args,
- missing_ok=True)
+ gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
gitc_projects = []
opened_projects = []
for project in gitc_manifest_projects:
@@ -1059,9 +1130,12 @@
for path in opened_projects]
if not args:
return
+
all_projects = self.GetProjects(args,
missing_ok=True,
- submodules_ok=opt.fetch_submodules)
+ submodules_ok=opt.fetch_submodules,
+ manifest=manifest,
+ all_manifests=not opt.this_manifest_only)
err_network_sync = False
err_update_projects = False
@@ -1073,7 +1147,6 @@
# Initialize the socket dir once in the parent.
ssh_proxy.sock()
all_projects = self._FetchMain(opt, args, all_projects, err_event,
- manifest_name, load_local_manifests,
ssh_proxy, manifest)
if opt.network_only:
@@ -1090,23 +1163,24 @@
file=sys.stderr)
sys.exit(1)
- if manifest.IsMirror or manifest.IsArchive:
- # bail out now, we have no working tree
- return
+ for m in self.ManifestList(opt):
+ if m.IsMirror or m.IsArchive:
+ # bail out now, we have no working tree
+ continue
- if self.UpdateProjectList(opt, manifest):
- err_event.set()
- err_update_projects = True
- if opt.fail_fast:
- print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
- sys.exit(1)
+ if self.UpdateProjectList(opt, m):
+ err_event.set()
+ err_update_projects = True
+ if opt.fail_fast:
+ print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
+ sys.exit(1)
- err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
- if err_update_linkfiles:
- err_event.set()
- if opt.fail_fast:
- print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
- sys.exit(1)
+ err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
+ if err_update_linkfiles:
+ err_event.set()
+ if opt.fail_fast:
+ print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
+ sys.exit(1)
err_results = []
# NB: We don't exit here because this is the last step.
@@ -1114,10 +1188,14 @@
if err_checkout:
err_event.set()
- # If there's a notice that's supposed to print at the end of the sync, print
- # it now...
- if manifest.notice:
- print(manifest.notice)
+ printed_notices = set()
+ # If there's a notice that's supposed to print at the end of the sync,
+ # print it now... But avoid printing duplicate messages, and preserve
+ # order.
+ for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
+ if m.notice and m.notice not in printed_notices:
+ print(m.notice)
+ printed_notices.add(m.notice)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():
