add experimental git worktree support
This provides initial support for using git worktrees internally
instead of our own ad-hoc symlink tree. It's been lightly tested
which is why it's not currently exposed via --help.
When people opt-in to worktrees in an existing repo client checkout,
no projects are migrated. Instead, only new projects will use the
worktree method. This allows for limited testing/opting in without
having to completely blow things away or get a second checkout.
Bug: https://crbug.com/gerrit/11486
Change-Id: Ic3ff891b30940a6ba497b406b2a387e0a8517ed8
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/254075
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md
index 8e62cde..f474029 100644
--- a/docs/internal-fs-layout.md
+++ b/docs/internal-fs-layout.md
@@ -102,6 +102,11 @@
respective servers ...
* `subprojects/`: Like `projects/`, but for git submodules.
* `subproject-objects/`: Like `project-objects/`, but for git submodules.
+* `worktrees/`: Bare checkouts of every project synced by the manifest. The
+ filesystem layout matches the `<project name=...` setting in the manifest
+ (i.e. the path on the remote server).
+
+ This is used when git worktrees are enabled.
### Global settings
@@ -121,6 +126,7 @@
| repo.partialclone | `--partial-clone` | Create [partial git clones] |
| repo.reference | `--reference` | Reference repo client checkout |
| repo.submodules | `--submodules` | Sync git submodules |
+| repo.worktree | `--worktree` | Use `git worktree` for checkouts |
| user.email | `--config-name` | User's e-mail address; Copied into `.git/config` when checking out a new project |
| user.name | `--config-name` | User's name; Copied into `.git/config` when checking out a new project |
diff --git a/manifest_xml.py b/manifest_xml.py
index 7f38d8c..4162800 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -146,9 +146,17 @@
gitdir=os.path.join(repodir, 'repo/.git'),
worktree=os.path.join(repodir, 'repo'))
- self.manifestProject = MetaProject(self, 'manifests',
- gitdir=os.path.join(repodir, 'manifests.git'),
- worktree=os.path.join(repodir, 'manifests'))
+ mp = MetaProject(self, 'manifests',
+ gitdir=os.path.join(repodir, 'manifests.git'),
+ worktree=os.path.join(repodir, 'manifests'))
+ self.manifestProject = mp
+
+ # This is a bit hacky, but we're in a chicken & egg situation: all the
+ # normal repo settings live in the manifestProject which we just setup
+ # above, so we couldn't easily query before that. We assume Project()
+ # init doesn't care if this changes afterwards.
+ if mp.config.GetBoolean('repo.worktree'):
+ mp.use_git_worktrees = True
self._Unload()
@@ -428,6 +436,10 @@
return self.manifestProject.config.GetBoolean('repo.mirror')
@property
+ def UseGitWorktrees(self):
+ return self.manifestProject.config.GetBoolean('repo.worktree')
+
+ @property
def IsArchive(self):
return self.manifestProject.config.GetBoolean('repo.archive')
@@ -873,8 +885,10 @@
groups = self._ParseGroups(groups)
if parent is None:
- relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
+ relpath, worktree, gitdir, objdir, use_git_worktrees = \
+ self.GetProjectPaths(name, path)
else:
+ use_git_worktrees = False
relpath, worktree, gitdir, objdir = \
self.GetSubprojectPaths(parent, name, path)
@@ -903,6 +917,7 @@
upstream=upstream,
parent=parent,
dest_branch=dest_branch,
+ use_git_worktrees=use_git_worktrees,
**extra_proj_attrs)
for n in node.childNodes:
@@ -918,6 +933,7 @@
return project
def GetProjectPaths(self, name, path):
+ use_git_worktrees = False
relpath = path
if self.IsMirror:
worktree = None
@@ -926,8 +942,15 @@
else:
worktree = os.path.join(self.topdir, path).replace('\\', '/')
gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
- objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
- return relpath, worktree, gitdir, objdir
+ # We allow people to mix git worktrees & non-git worktrees for now.
+ # This allows for in situ migration of repo clients.
+ if os.path.exists(gitdir) or not self.UseGitWorktrees:
+ objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+ else:
+ use_git_worktrees = True
+ gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
+ objdir = gitdir
+ return relpath, worktree, gitdir, objdir, use_git_worktrees
def GetProjectsWithName(self, name):
return self._projects.get(name, [])
diff --git a/project.py b/project.py
index 86c9ef0..3a7ac9e 100644
--- a/project.py
+++ b/project.py
@@ -866,6 +866,7 @@
clone_depth=None,
upstream=None,
parent=None,
+ use_git_worktrees=False,
is_derived=False,
dest_branch=None,
optimized_fetch=False,
@@ -889,6 +890,7 @@
sync_tags: The `sync-tags` attribute of manifest.xml's project element.
upstream: The `upstream` attribute of manifest.xml's project element.
parent: The parent Project object.
+ use_git_worktrees: Whether to use `git worktree` for this project.
is_derived: False if the project was explicitly defined in the manifest;
True if the project is a discovered submodule.
dest_branch: The branch to which to push changes for review by default.
@@ -923,6 +925,10 @@
self.clone_depth = clone_depth
self.upstream = upstream
self.parent = parent
+ # NB: Do not use this setting in __init__ to change behavior so that the
+ # manifest.git checkout can inspect & change it after instantiating. See
+ # the XmlManifest init code for more info.
+ self.use_git_worktrees = use_git_worktrees
self.is_derived = is_derived
self.optimized_fetch = optimized_fetch
self.subprojects = []
@@ -1872,15 +1878,19 @@
except KeyError:
head = None
if revid and head and revid == head:
- ref = os.path.join(self.gitdir, R_HEADS + name)
- try:
- os.makedirs(os.path.dirname(ref))
- except OSError:
- pass
- _lwrite(ref, '%s\n' % revid)
- _lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
- branch.Save()
- return True
+ if self.use_git_worktrees:
+ self.work_git.update_ref(HEAD, revid)
+ branch.Save()
+ else:
+ ref = os.path.join(self.gitdir, R_HEADS + name)
+ try:
+ os.makedirs(os.path.dirname(ref))
+ except OSError:
+ pass
+ _lwrite(ref, '%s\n' % revid)
+ _lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
+ branch.Save()
+ return True
if GitCommand(self,
['checkout', '-b', branch.name, revid],
@@ -2617,6 +2627,11 @@
os.makedirs(self.objdir)
self.bare_objdir.init()
+ # Enable per-worktree config file support if possible. This is more a
+ # nice-to-have feature for users rather than a hard requirement.
+ if self.use_git_worktrees and git_require((2, 19, 0)):
+ self.config.SetString('extensions.worktreeConfig', 'true')
+
# If we have a separate directory to hold refs, initialize it as well.
if self.objdir != self.gitdir:
if init_git_dir:
@@ -2651,13 +2666,15 @@
mirror_git = os.path.join(ref_dir, self.name + '.git')
repo_git = os.path.join(ref_dir, '.repo', 'projects',
self.relpath + '.git')
+ worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees',
+ self.name + '.git')
if os.path.exists(mirror_git):
ref_dir = mirror_git
-
elif os.path.exists(repo_git):
ref_dir = repo_git
-
+ elif os.path.exists(worktrees_git):
+ ref_dir = worktrees_git
else:
ref_dir = None
@@ -2765,6 +2782,10 @@
self.bare_git.symbolic_ref('-m', msg, ref, dst)
def _CheckDirReference(self, srcdir, destdir, share_refs):
+ # Git worktrees don't use symlinks to share at all.
+ if self.use_git_worktrees:
+ return
+
symlink_files = self.shareable_files[:]
symlink_dirs = self.shareable_dirs[:]
if share_refs:
@@ -2864,11 +2885,38 @@
else:
raise
+ def _InitGitWorktree(self):
+ """Init the project using git worktrees."""
+ self.bare_git.worktree('prune')
+ self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock',
+ self.worktree, self.GetRevisionId())
+
+ # Rewrite the internal state files to use relative paths between the
+ # checkouts & worktrees.
+ dotgit = os.path.join(self.worktree, '.git')
+ with open(dotgit, 'r') as fp:
+ # Figure out the checkout->worktree path.
+ setting = fp.read()
+ assert setting.startswith('gitdir:')
+ git_worktree_path = setting.split(':', 1)[1].strip()
+ # Use relative path from checkout->worktree.
+ with open(dotgit, 'w') as fp:
+ print('gitdir:', os.path.relpath(git_worktree_path, self.worktree),
+ file=fp)
+ # Use relative path from worktree->checkout.
+ with open(os.path.join(git_worktree_path, 'gitdir'), 'w') as fp:
+ print(os.path.relpath(dotgit, git_worktree_path), file=fp)
+
def _InitWorkTree(self, force_sync=False, submodules=False):
realdotgit = os.path.join(self.worktree, '.git')
tmpdotgit = realdotgit + '.tmp'
init_dotgit = not os.path.exists(realdotgit)
if init_dotgit:
+ if self.use_git_worktrees:
+ self._InitGitWorktree()
+ self._CopyAndLinkFiles()
+ return
+
dotgit = tmpdotgit
platform_utils.rmtree(tmpdotgit, ignore_errors=True)
os.makedirs(tmpdotgit)
diff --git a/repo b/repo
index 77e7fe9..743c28b 100755
--- a/repo
+++ b/repo
@@ -302,6 +302,8 @@
group.add_option('--clone-filter', action='store', default='blob:none',
help='filter for use with --partial-clone '
'[default: %default]')
+ group.add_option('--worktree', action='store_true',
+ help=optparse.SUPPRESS_HELP)
group.add_option('--archive', action='store_true',
help='checkout an archive instead of a git repository for '
'each project. See git archive.')
diff --git a/subcmds/init.py b/subcmds/init.py
index 3c68c2c..8a29321 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -15,6 +15,8 @@
# limitations under the License.
from __future__ import print_function
+
+import optparse
import os
import platform
import re
@@ -128,6 +130,10 @@
g.add_option('--clone-filter', action='store', default='blob:none',
dest='clone_filter',
help='filter for use with --partial-clone [default: %default]')
+ # TODO(vapier): Expose option with real help text once this has been in the
+ # wild for a while w/out significant bug reports. Goal is by ~Sep 2020.
+ g.add_option('--worktree', action='store_true',
+ help=optparse.SUPPRESS_HELP)
g.add_option('--archive',
dest='archive', action='store_true',
help='checkout an archive instead of a git repository for '
@@ -246,6 +252,20 @@
if opt.dissociate:
m.config.SetString('repo.dissociate', 'true')
+ if opt.worktree:
+ if opt.mirror:
+ print('fatal: --mirror and --worktree are incompatible',
+ file=sys.stderr)
+ sys.exit(1)
+ if opt.submodules:
+ print('fatal: --submodules and --worktree are incompatible',
+ file=sys.stderr)
+ sys.exit(1)
+ m.config.SetString('repo.worktree', 'true')
+ if is_new:
+ m.use_git_worktrees = True
+ print('warning: --worktree is experimental!', file=sys.stderr)
+
if opt.archive:
if is_new:
m.config.SetString('repo.archive', 'true')
@@ -459,6 +479,10 @@
% ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr)
+ if opt.worktree:
+ # Older versions of git supported worktree, but had dangerous gc bugs.
+ git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
+
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 0ac308e..49867a9 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -15,6 +15,8 @@
# limitations under the License.
from __future__ import print_function
+
+import errno
import json
import netrc
from optparse import SUPPRESS_HELP
@@ -569,7 +571,8 @@
gc_gitdirs = {}
for project in projects:
# Make sure pruning never kicks in with shared projects.
- if len(project.manifest.GetProjectsWithName(project.name)) > 1:
+ if (not project.use_git_worktrees and
+ len(project.manifest.GetProjectsWithName(project.name)) > 1):
print('%s: Shared project %s found, disabling pruning.' %
(project.relpath, project.name))
if git_require((2, 7, 0)):
@@ -637,13 +640,22 @@
# Delete the .git directory first, so we're less likely to have a partially
# working git repository around. There shouldn't be any git projects here,
# so rmtree works.
+ dotgit = os.path.join(path, '.git')
+ # Try to remove plain files first in case of git worktrees. If this fails
+ # for any reason, we'll fall back to rmtree, and that'll display errors if
+ # it can't remove things either.
try:
- platform_utils.rmtree(os.path.join(path, '.git'))
+ platform_utils.remove(dotgit)
+ except OSError:
+ pass
+ try:
+ platform_utils.rmtree(dotgit)
except OSError as e:
- print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
- print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
- print(' remove manually, then run sync again', file=sys.stderr)
- return 1
+ if e.errno != errno.ENOENT:
+ print('error: %s: %s' % (dotgit, str(e)), file=sys.stderr)
+ print('error: %s: Failed to delete obsolete path; remove manually, then '
+ 'run sync again' % (path,), file=sys.stderr)
+ return 1
# Delete everything under the worktree, except for directories that contain
# another git project
