repo: Add support for standalone manifests
Added --standalone_manifest to repo tool. If set, the
manifest is downloaded directly from the appropriate source
(currently, we only support GS) and used instead of creating
a manifest git checkout. The manifests.git repo is still created to
keep track of various config but is marked as being for a standalone
manifest so that the repo tool doesn't try to run networked git
commands in it.
BUG=b:192664812
TEST=existing tests (no coverage), manual runs
Change-Id: I84378cbc7f8e515eabeccdde9665efc8cd2a9d21
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/312942
Tested-by: Jack Neus <jackneus@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md
index e3be173..af6a452 100644
--- a/docs/internal-fs-layout.md
+++ b/docs/internal-fs-layout.md
@@ -157,6 +157,7 @@
| Setting | `repo init` Option | Use/Meaning |
|------------------- |---------------------------|-------------|
| manifest.groups | `--groups` & `--platform` | The manifest groups to sync |
+| manifest.standalone | `--standalone-manifest` | Download manifest as static file instead of creating checkout |
| repo.archive | `--archive` | Use `git archive` for checkouts |
| repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly |
| repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] |
diff --git a/fetch.py b/fetch.py
new file mode 100644
index 0000000..5b9997a
--- /dev/null
+++ b/fetch.py
@@ -0,0 +1,38 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module contains functions used to fetch files from various sources."""
+
+import subprocess
+import sys
+from urllib.parse import urlparse
+
+def fetch_file(url):
+ """Fetch a file from the specified source using the appropriate protocol.
+
+ Returns:
+ The contents of the file as bytes.
+ """
+ scheme = urlparse(url).scheme
+ if scheme == 'gs':
+ cmd = ['gsutil', 'cat', url]
+ try:
+ result = subprocess.run(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ return result.stdout
+ except subprocess.CalledProcessError as e:
+ print('fatal: error running "gsutil": %s' % e.output,
+ file=sys.stderr)
+ sys.exit(1)
+ raise ValueError('unsupported url %s' % url)
diff --git a/git_config.py b/git_config.py
index d882239..778e81a 100644
--- a/git_config.py
+++ b/git_config.py
@@ -104,6 +104,10 @@
os.path.dirname(self.file),
'.repo_' + os.path.basename(self.file) + '.json')
+ def ClearCache(self):
+ """Clear the in-memory cache of config."""
+ self._cache_dict = None
+
def Has(self, name, include_defaults=True):
"""Return true if this configuration file has the key.
"""
@@ -399,7 +403,7 @@
if p.Wait() == 0:
return p.stdout
else:
- GitError('git config %s: %s' % (str(args), p.stderr))
+ raise GitError('git config %s: %s' % (str(args), p.stderr))
class RepoConfig(GitConfig):
diff --git a/man/repo-gitc-init.1 b/man/repo-gitc-init.1
index 1d1b23a..9b61866 100644
--- a/man/repo-gitc-init.1
+++ b/man/repo-gitc-init.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual"
.SH NAME
repo \- repo gitc-init - manual page for repo gitc-init
.SH SYNOPSIS
@@ -31,6 +31,10 @@
\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
initial manifest file
.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
+.TP
\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
restrict manifest projects to ones with specified
group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
diff --git a/man/repo-init.1 b/man/repo-init.1
index e860f95..9957b64 100644
--- a/man/repo-init.1
+++ b/man/repo-init.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
-.TH REPO "1" "July 2021" "repo init" "Repo Manual"
+.TH REPO "1" "September 2021" "repo init" "Repo Manual"
.SH NAME
repo \- repo init - manual page for repo init
.SH SYNOPSIS
@@ -31,6 +31,10 @@
\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
initial manifest file
.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
+.TP
\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
restrict manifest projects to ones with specified
group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -137,6 +141,12 @@
The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
used. If no manifest is specified, the manifest default.xml will be used.
.PP
+If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded
+directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting
+up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be
+fully static and will not be re\-downloaded during subsesquent `repo init` and
+`repo sync` calls.
+.PP
The \fB\-\-reference\fR option can be used to point to a directory that has the content
of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as
possible from the local reference directory when fetching from the server. This
diff --git a/repo b/repo
index 3b244c1..f61639f 100755
--- a/repo
+++ b/repo
@@ -312,6 +312,10 @@
metavar='PLATFORM')
group.add_option('--submodules', action='store_true',
help='sync any submodules associated with the manifest repo')
+ group.add_option('--standalone-manifest', action='store_true',
+ help='download the manifest as a static file '
+ 'rather then create a git checkout of '
+ 'the manifest repo')
# Options that only affect manifest project, and not any of the projects
# specified in the manifest itself.
diff --git a/subcmds/init.py b/subcmds/init.py
index 5671fc2..9c6b2ad 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -15,6 +15,7 @@
import os
import platform
import re
+import subprocess
import sys
import urllib.parse
@@ -24,6 +25,7 @@
from project import SyncBuffer
from git_config import GitConfig
from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
+import fetch
import git_superproject
import platform_utils
from wrapper import Wrapper
@@ -53,6 +55,12 @@
to be used. If no manifest is specified, the manifest default.xml
will be used.
+If the --standalone-manifest argument is set, the manifest will be downloaded
+directly from the specified --manifest-url as a static file (rather than
+setting up a manifest git checkout). With --standalone-manifest, the manifest
+will be fully static and will not be re-downloaded during subsesquent
+`repo init` and `repo sync` calls.
+
The --reference option can be used to point to a directory that
has the content of a --mirror sync. This will make the working
directory use as much data as possible from the local reference
@@ -112,6 +120,22 @@
m = self.manifest.manifestProject
is_new = not m.Exists
+ # If repo has already been initialized, we take -u with the absence of
+ # --standalone-manifest to mean "transition to a standard repo set up",
+ # which necessitates starting fresh.
+ # If --standalone-manifest is set, we always tear everything down and start
+ # anew.
+ if not is_new:
+ was_standalone_manifest = m.config.GetString('manifest.standalone')
+ if opt.standalone_manifest or (
+ was_standalone_manifest and opt.manifest_url):
+ m.config.ClearCache()
+ if m.gitdir and os.path.exists(m.gitdir):
+ platform_utils.rmtree(m.gitdir)
+ if m.worktree and os.path.exists(m.worktree):
+ platform_utils.rmtree(m.worktree)
+
+ is_new = not m.Exists
if is_new:
if not opt.manifest_url:
print('fatal: manifest url is required.', file=sys.stderr)
@@ -136,6 +160,19 @@
m._InitGitDir(mirror_git=mirrored_manifest_git)
+ # If standalone_manifest is set, mark the project as "standalone" -- we'll
+ # still do much of the manifests.git set up, but will avoid actual syncs to
+ # a remote.
+ standalone_manifest = False
+ if opt.standalone_manifest:
+ standalone_manifest = True
+ elif not opt.manifest_url:
+ # If -u is set and --standalone-manifest is not, then we're not in
+ # standalone mode. Otherwise, use config to infer what we were in the last
+ # init.
+ standalone_manifest = bool(m.config.GetString('manifest.standalone'))
+ m.config.SetString('manifest.standalone', opt.manifest_url)
+
self._ConfigureDepth(opt)
# Set the remote URL before the remote branch as we might need it below.
@@ -145,22 +182,23 @@
r.ResetFetch()
r.Save()
- if opt.manifest_branch:
- if opt.manifest_branch == 'HEAD':
- opt.manifest_branch = m.ResolveRemoteHead()
- if opt.manifest_branch is None:
- print('fatal: unable to resolve HEAD', file=sys.stderr)
- sys.exit(1)
- m.revisionExpr = opt.manifest_branch
- else:
- if is_new:
- default_branch = m.ResolveRemoteHead()
- if default_branch is None:
- # If the remote doesn't have HEAD configured, default to master.
- default_branch = 'refs/heads/master'
- m.revisionExpr = default_branch
+ if not standalone_manifest:
+ if opt.manifest_branch:
+ if opt.manifest_branch == 'HEAD':
+ opt.manifest_branch = m.ResolveRemoteHead()
+ if opt.manifest_branch is None:
+ print('fatal: unable to resolve HEAD', file=sys.stderr)
+ sys.exit(1)
+ m.revisionExpr = opt.manifest_branch
else:
- m.PreSync()
+ if is_new:
+ default_branch = m.ResolveRemoteHead()
+ if default_branch is None:
+ # If the remote doesn't have HEAD configured, default to master.
+ default_branch = 'refs/heads/master'
+ m.revisionExpr = default_branch
+ else:
+ m.PreSync()
groups = re.split(r'[,\s]+', opt.groups)
all_platforms = ['linux', 'darwin', 'windows']
@@ -250,6 +288,16 @@
if opt.use_superproject is not None:
m.config.SetBoolean('repo.superproject', opt.use_superproject)
+ if standalone_manifest:
+ if is_new:
+ manifest_name = 'default.xml'
+ manifest_data = fetch.fetch_file(opt.manifest_url)
+ dest = os.path.join(m.worktree, manifest_name)
+ os.makedirs(os.path.dirname(dest), exist_ok=True)
+ with open(dest, 'wb') as f:
+ f.write(manifest_data)
+ return
+
if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
clone_bundle=opt.clone_bundle,
current_branch_only=opt.current_branch_only,
@@ -426,6 +474,11 @@
if opt.archive and opt.mirror:
self.OptionParser.error('--mirror and --archive cannot be used together.')
+ if opt.standalone_manifest and (
+ opt.manifest_branch or opt.manifest_name != 'default.xml'):
+ self.OptionParser.error('--manifest-branch and --manifest-name cannot'
+ ' be used with --standalone-manifest.')
+
if args:
if opt.manifest_url:
self.OptionParser.error(
