Add envar to replace shallow clones with partial
An investigation go/git-repo-shallow shows a number of problems
when doing a shallow git fetch/clone. This change introduces an
environment variable REPO_ALLOW_SHALLOW. When this environment variable
is set to 1 during a repo init or repo sync all shallow git fetch
commands are replaced with partial fetch commands. Any shallow
repository needing update is unshallowed. This behavior continues until
a subsequent repo sync command is run with REPO_ALLOW_SHALLOW set to 1.
Bug: b/274340522
Change-Id: I1c3188270629359e52449788897d9d4988ebf280
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/374754
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Tested-by: Jason Chang <jasonnc@google.com>
diff --git a/manifest_xml.py b/manifest_xml.py
index 14b03a3..555bf73 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -982,6 +982,12 @@
return None
@property
+ def CloneFilterForDepth(self):
+ if self.manifestProject.clone_filter_for_depth:
+ return self.manifestProject.clone_filter_for_depth
+ return None
+
+ @property
def PartialCloneExclude(self):
exclude = self.manifest.manifestProject.partial_clone_exclude or ""
return set(x.strip() for x in exclude.split(","))
diff --git a/project.py b/project.py
index ff01839..83f3eff 100644
--- a/project.py
+++ b/project.py
@@ -1186,6 +1186,7 @@
ssh_proxy=None,
clone_filter=None,
partial_clone_exclude=set(),
+ clone_filter_for_depth=None,
):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
@@ -1295,6 +1296,10 @@
else:
depth = self.manifest.manifestProject.depth
+ if depth and clone_filter_for_depth:
+ depth = None
+ clone_filter = clone_filter_for_depth
+
# See if we can skip the network fetch entirely.
remote_fetched = False
if not (
@@ -3885,6 +3890,11 @@
return self.config.GetString("repo.partialcloneexclude")
@property
+ def clone_filter_for_depth(self):
+ """Replace shallow clone with partial clone."""
+ return self.config.GetString("repo.clonefilterfordepth")
+
+ @property
def manifest_platform(self):
"""The --platform argument from `repo init`."""
return self.config.GetString("manifest.platform")
@@ -3961,6 +3971,7 @@
manifest_name=spec.manifestName,
this_manifest_only=True,
outer_manifest=False,
+ clone_filter_for_depth=mp.clone_filter_for_depth,
)
def Sync(
@@ -3991,6 +4002,7 @@
tags="",
this_manifest_only=False,
outer_manifest=True,
+ clone_filter_for_depth=None,
):
"""Sync the manifest and all submanifests.
@@ -4035,6 +4047,8 @@
current sub manifest.
outer_manifest: a boolean, whether to start at the outermost
manifest.
+ clone_filter_for_depth: a string, when specified replaces shallow
+ clones with partial.
Returns:
a boolean, whether the sync was successful.
@@ -4297,6 +4311,9 @@
file=sys.stderr,
)
+ if clone_filter_for_depth is not None:
+ self.ConfigureCloneFilterForDepth(clone_filter_for_depth)
+
if use_superproject is not None:
self.config.SetBoolean("repo.superproject", use_superproject)
@@ -4311,6 +4328,7 @@
submodules=submodules,
clone_filter=clone_filter,
partial_clone_exclude=self.manifest.PartialCloneExclude,
+ clone_filter_for_depth=self.manifest.CloneFilterForDepth,
).success
if not success:
r = self.GetRemote()
@@ -4415,6 +4433,18 @@
return True
+ def ConfigureCloneFilterForDepth(self, clone_filter_for_depth):
+ """Configure clone filter to replace shallow clones.
+
+ Args:
+ clone_filter_for_depth: a string or None, e.g. 'blob:none' will
+ disable shallow clones and replace with partial clone. None will
+ enable shallow clones.
+ """
+ self.config.SetString(
+ "repo.clonefilterfordepth", clone_filter_for_depth
+ )
+
def _ConfigureDepth(self, depth):
"""Configure the depth we'll sync down.
diff --git a/subcmds/init.py b/subcmds/init.py
index 9946466..6d7fd85 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -20,6 +20,8 @@
from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
from wrapper import Wrapper
+_REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
+
class Init(InteractiveCommand, MirrorSafeCommand):
COMMON = True
@@ -125,6 +127,9 @@
# manifest project is special and is created when instantiating the
# manifest which happens before we parse options.
self.manifest.manifestProject.clone_depth = opt.manifest_depth
+ clone_filter_for_depth = (
+ "blob:none" if (_REPO_ALLOW_SHALLOW == "0") else None
+ )
if not self.manifest.manifestProject.Sync(
manifest_url=opt.manifest_url,
manifest_branch=opt.manifest_branch,
@@ -140,6 +145,7 @@
partial_clone=opt.partial_clone,
clone_filter=opt.clone_filter,
partial_clone_exclude=opt.partial_clone_exclude,
+ clone_filter_for_depth=clone_filter_for_depth,
clone_bundle=opt.clone_bundle,
git_lfs=opt.git_lfs,
use_superproject=opt.use_superproject,
diff --git a/subcmds/sync.py b/subcmds/sync.py
index a44ed5b..9ae8a4c 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -79,6 +79,8 @@
_REPO_AUTO_GC = "REPO_AUTO_GC"
_AUTO_GC = os.environ.get(_REPO_AUTO_GC) == "1"
+_REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
+
class _FetchOneResult(NamedTuple):
"""_FetchOne return value.
@@ -638,6 +640,7 @@
ssh_proxy=self.ssh_proxy,
clone_filter=project.manifest.CloneFilter,
partial_clone_exclude=project.manifest.PartialCloneExclude,
+ clone_filter_for_depth=project.manifest.CloneFilterForDepth,
)
success = sync_result.success
remote_fetched = sync_result.remote_fetched
@@ -1440,6 +1443,7 @@
submodules=mp.manifest.HasSubmodules,
clone_filter=mp.manifest.CloneFilter,
partial_clone_exclude=mp.manifest.PartialCloneExclude,
+ clone_filter_for_depth=mp.manifest.CloneFilterForDepth,
)
finish = time.time()
self.event_log.AddSync(
@@ -1589,6 +1593,15 @@
_PostRepoUpgrade(manifest, quiet=opt.quiet)
mp = manifest.manifestProject
+
+ if _REPO_ALLOW_SHALLOW is not None:
+ if _REPO_ALLOW_SHALLOW == "1":
+ mp.ConfigureCloneFilterForDepth(None)
+ elif (
+ _REPO_ALLOW_SHALLOW == "0" and mp.clone_filter_for_depth is None
+ ):
+ mp.ConfigureCloneFilterForDepth("blob:none")
+
if opt.mp_update:
self._UpdateAllManifestProjects(opt, mp, manifest_name)
else:
