sync: Implement fetchcmd for standard Git layouts Introduce support for `repo.fetchcmd` configuration, allowing users to specify a custom command to fetch project data instead of `_RemoteFetch`. When `repo.fetchcmd` is specified, `repo` will execute it instead of `_RemoteFetch` during the network half of sync. Note that `repo.fetchcmd` requires `repo.uselocalgitdirs` to be enabled. The command is executed in a subshell with project-context environment variables, including the new `REPO_TREV` (target revision resolved to a commit hash) and `REPO_PROJECT_FETCH_URL`. After execution, `repo` verifies that the target commit is available and that the tracking ref and FETCH_HEAD are correctly updated. Tested with: ``` > ~/git-repo/repo init -u https://android-review.googlesource.com/platform/manifest \ --repo-url file:///usr/local/google/home/gavinmak/git-repo \ --groups developers \ --no-repo-verify \ --use-local-gitdirs ... > git config --file .repo/manifests.git/config repo.fetchcmd 'mkdir -p $REPO_PATH && cd $REPO_PATH && if [ ! -d .git ]; then git init && git remote add aosp $REPO_PROJECT_FETCH_URL; fi && git fetch aosp $REPO_TREV && git reset --hard $REPO_TREV && mkdir -p .git/refs/remotes/aosp && echo $REPO_TREV > .git/refs/remotes/aosp/main && echo $REPO_TREV > .git/FETCH_HEAD' > ~/git-repo/repo sync -j32 warning: repo is not tracking a remote branch, so it will not receive updates; run `repo init --repo-rev=stable` to fix. You are currently enrolled in Git submodules experiment (go/android-submodules-quickstart). Use --no-use-superproject to override. Syncing: 100% (4/4), done in 1m15.686s Finalizing sync state... repo sync has finished successfully. ``` Bug: 513329573 Change-Id: I754d3f3c78e86fdeee1a72115297a75b571bc497 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/583883 Reviewed-by: Becky Siegel <beckysiegel@google.com> Tested-by: Gavin Mak <gavinmak@google.com> Commit-Queue: Gavin Mak <gavinmak@google.com>
diff --git a/docs/fetch-cmd.md b/docs/fetch-cmd.md new file mode 100644 index 0000000..32fa8d1 --- /dev/null +++ b/docs/fetch-cmd.md
@@ -0,0 +1,63 @@ +# Fetch Command Contract + +The `repo.fetchcmd` configuration allows specifying a custom command to be +executed during `repo sync` to fetch objects, instead of using standard +`git fetch`. This is particularly useful in environments with virtualized +filesystems or lazy checkouts where fetching metadata and downloading file +contents should be decoupled. + +## Configuration + +To use this feature, set the following in `.repo/manifests.git/config`: +```ini +[repo] + fetchcmd = "your custom command here" + uselocalgitdirs = true +``` +Setting `repo.fetchcmd` **requires** `repo.uselocalgitdirs` to be set to `true`. + +## Environment Variables + +The custom command is executed in a subshell populated with standard +project-context environment variables. For details on standard variables (such +as `REPO_PROJECT`, `REPO_PATH`, `REPO_PROJECT_FETCH_URL`, etc.), see the +Environment section in `repo help forall` or `subcmds/forall.py`. + +The following environment variable is specific to `repo.fetchcmd`: + +* `REPO_TREV`: The target revision resolved to a full commit hash. + +## Contract + +### Postconditions on exit 0 + +After the fetch command exits with status 0, `repo` expects the following +postconditions to be met: + +1. `git cat-file -e REPO_TREV` succeeds (the commit must exist in the object + store). +2. The mapped local tracking ref (e.g. `refs/remotes/REPO_REMOTE/<branch>` + for a branch revision, or the tag ref itself for a tag) must point to + `REPO_TREV`. +3. `FETCH_HEAD` must point to `REPO_TREV`. +4. The commit graph from `REPO_TREV` must be reachable far enough to compute + merge bases with local branches. + +### Invariants + +* The command should be idempotent; fetching the same `REPO_TREV` twice should + be a no-op. +* Only `FETCH_HEAD` and `refs/remotes/*` should be modified to preserve + `repo sync --network-only` semantics. `HEAD` and local branches must not be + touched by the fetch command. +* Dirty worktree state must be preserved. +* The command is **not** executed for `MetaProject`s (i.e. the internal `repo` + repository itself at `.repo/repo` and the `manifests` repository at + `.repo/manifests`). + +### Failure + +* A non-zero exit status aborts the project's sync, and the command's stderr + is surfaced to the user. +* `repo` verifies the tracking ref and target reachability after exit 0. Any + mismatch is treated as a failure.
diff --git a/project.py b/project.py index ec645bb..649196b 100644 --- a/project.py +++ b/project.py
@@ -1557,43 +1557,25 @@ depth = None clone_filter = clone_filter_for_depth - # See if we can skip the network fetch entirely. + custom_fetch_configured = ( + self.manifest.manifestProject.use_local_gitdirs + and self.manifest.manifestProject.fetch_cmd + and not isinstance(self, MetaProject) + ) + remote_fetched = False - has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow")) - if not ( - optimized_fetch - and IsId(self.revisionExpr) - and self._CheckForImmutableRevision( - use_superproject=use_superproject - ) - and ( - has_shallow - or (not depth and not self._SharingProjectHasShallow()) - ) - ): + if custom_fetch_configured: + # The custom fetch command is executed unconditionally on every sync + # because it is responsible for updating tracking refs and + # FETCH_HEAD. remote_fetched = True try: - if not self._RemoteFetch( - initial=is_new, - quiet=quiet, - verbose=verbose, - output_redir=output_redir, - alt_dir=alt_dir, - use_superproject=use_superproject, - current_branch_only=current_branch_only, - tags=tags, - prune=prune, - depth=depth, - submodules=submodules, - force_sync=force_sync, - ssh_proxy=ssh_proxy, - clone_filter=clone_filter, - retry_fetches=retry_fetches, - ): + if not self._CustomFetch(verbose=verbose): return SyncNetworkHalfResult( remote_fetched, SyncNetworkHalfError( - f"Unable to remote fetch project {self.name}", + f"Unable to fetch project {self.name} using " + "fetchcmd", project=self.name, ), ) @@ -1602,6 +1584,53 @@ remote_fetched, e, ) + else: + # See if we can skip the standard network fetch entirely. + has_shallow = os.path.exists(os.path.join(self.gitdir, "shallow")) + skip_fetch = ( + optimized_fetch + and IsId(self.revisionExpr) + and self._CheckForImmutableRevision( + use_superproject=use_superproject + ) + and ( + has_shallow + or (not depth and not self._SharingProjectHasShallow()) + ) + ) + + if not skip_fetch: + remote_fetched = True + try: + if not self._RemoteFetch( + initial=is_new, + quiet=quiet, + verbose=verbose, + output_redir=output_redir, + alt_dir=alt_dir, + use_superproject=use_superproject, + current_branch_only=current_branch_only, + tags=tags, + prune=prune, + depth=depth, + submodules=submodules, + force_sync=force_sync, + ssh_proxy=ssh_proxy, + clone_filter=clone_filter, + retry_fetches=retry_fetches, + ): + return SyncNetworkHalfResult( + remote_fetched, + SyncNetworkHalfError( + f"Unable to remote fetch project {self.name}", + project=self.name, + ), + ) + except RepoError as e: + return SyncNetworkHalfResult( + remote_fetched, + e, + ) mp = self.manifest.manifestProject dissociate = mp.dissociate @@ -2710,6 +2739,114 @@ ) command.Wait() + def _CustomFetch(self, verbose=False) -> bool: + """Fetch using a custom command specified by repo.fetchcmd. + + Populates the subshell with project-context environment variables, + including REPO_TREV (target revision resolved to a commit hash). + + Expects the target commit to be reachable and tracking refs/FETCH_HEAD + to be updated upon a successful 0 exit code. + + For a detailed contract, environment variables, and postconditions, + see docs/fetch-cmd.md. + """ + # Resolve REPO_TREV (target revision resolved to a full commit hash). + repo_trev = None + if self.revisionId and IsId(self.revisionId): + repo_trev = self.revisionId + + if not repo_trev: + output = self._LsRemote(self.upstream or self.revisionExpr) + if output: + lines = output.splitlines() + if lines: + parts = lines[0].split() + if parts: + repo_trev = parts[0] + + if not repo_trev: + logger.error("error: Cannot resolve REPO_TREV for %s", self.name) + return False + + env = os.environ.copy() + env.update(self.GetEnvVars()) + env["REPO_TREV"] = repo_trev + cmd_str = self.manifest.manifestProject.fetch_cmd + + if verbose: + print(f"Running fetchcmd: {cmd_str} for {self.name}") + + stdout = None if verbose else subprocess.PIPE + stderr = None if verbose else subprocess.PIPE + try: + p = subprocess.run( + cmd_str, + shell=True, + cwd=self.manifest.topdir, + env=env, + stdout=stdout, + stderr=stderr, + text=True, + ) + + if p.returncode != 0: + logger.error( + "error: fetchcmd failed with exit code %d", p.returncode + ) + if not verbose: + logger.error("stderr: %s", p.stderr) + return False + + except Exception as e: + logger.error("error: Failed to execute fetchcmd: %s", e) + return False + + # Verify postconditions. + try: + self.bare_git.cat_file("-e", repo_trev) + except GitError: + logger.error( + "error: Postcondition failed: %s not found in object store", + repo_trev, + ) + return False + + try: + ref_name = self.GetRemote().ToLocal(self.revisionExpr) + except GitError as e: + logger.error("error: Failed to resolve tracking ref: %s", e) + return False + try: + resolved_ref = self.bare_git.rev_parse(ref_name) + if resolved_ref != repo_trev: + logger.error( + "error: Postcondition failed: %s is %s, expected %s", + ref_name, + resolved_ref, + repo_trev, + ) + return False + except GitError: + logger.error("error: Postcondition failed: %s not found", ref_name) + return False + + try: + fetch_head = self.bare_git.rev_parse("FETCH_HEAD") + if fetch_head != repo_trev: + logger.error( + "error: Postcondition failed: FETCH_HEAD is %s, " + "expected %s", + fetch_head, + repo_trev, + ) + return False + except GitError: + logger.error("error: Postcondition failed: FETCH_HEAD not found") + return False + + return True + def _RemoteFetch( self, name=None, @@ -4637,6 +4774,11 @@ return self.config.GetBoolean("repo.uselocalgitdirs") @property + def fetch_cmd(self): + """The fetch command to use.""" + return self.config.GetString("repo.fetchcmd") + + @property def clone_bundle(self): """Whether we use clone_bundle.""" return self.config.GetBoolean("repo.clonebundle") @@ -5049,6 +5191,13 @@ self.config.SetBoolean("repo.uselocalgitdirs", use_local_gitdirs) + if self.fetch_cmd and not (use_local_gitdirs or self.use_local_gitdirs): + logger.error( + "fatal: repo.fetchcmd is set but repo.uselocalgitdirs is " + "not enabled" + ) + return False + if archive: if is_new: self.config.SetBoolean("repo.archive", archive)
diff --git a/tests/test_project.py b/tests/test_project.py index 3973bcf..ea75710 100644 --- a/tests/test_project.py +++ b/tests/test_project.py
@@ -1286,3 +1286,71 @@ env = proj.GetEnvVars() self.assertEqual(env["REPO_LREV"], "") + + +class FetchCmdTests(unittest.TestCase): + """Tests for fetch_cmd feature.""" + + def setUpManifest(self, tempdir): + repodir = os.path.join(tempdir, ".repo") + manifest_dir = os.path.join(repodir, "manifests") + manifest_file = os.path.join(repodir, manifest_xml.MANIFEST_FILE_NAME) + os.mkdir(repodir) + os.mkdir(manifest_dir) + manifest = manifest_xml.XmlManifest(repodir, manifest_file) + + return project.ManifestProject( + manifest, "test/manifest", os.path.join(tempdir, ".git"), tempdir + ) + + def _get_project(self, tempdir): + proj = _create_mock_project( + tempdir, use_local_gitdirs=True, fetch_cmd="echo hi" + ) + proj.GetRevisionId = mock.MagicMock(return_value="1234abcd") + return proj + + def test_fetch_cmd_execution(self): + """Test that fetch_cmd is executed with correct environment.""" + with utils_for_test.TempGitTree() as tempdir: + proj = self._get_project(tempdir) + + proj.bare_git.rev_parse.return_value = "1234abcd" + mock_remote = mock.MagicMock() + mock_remote.ToLocal.return_value = "refs/remotes/origin/main" + proj.GetRemote = mock.MagicMock(return_value=mock_remote) + + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stderr="") + res = proj._CustomFetch() + + self.assertTrue(res) + mock_run.assert_called_once() + args, kwargs = mock_run.call_args + self.assertEqual(args[0], "echo hi") + self.assertEqual(kwargs["shell"], True) + self.assertEqual(kwargs["cwd"], tempdir) + self.assertEqual(kwargs["env"]["REPO_TREV"], "1234abcd") + self.assertEqual( + kwargs["env"]["REPO_PROJECT_FETCH_URL"], + "http://example.com/repo", + ) + + def test_sync_fetch_cmd_requires_use_local_gitdirs(self): + """Test that fetch_cmd requires use_local_gitdirs.""" + with utils_for_test.TempGitTree() as tempdir: + fakeproj = self.setUpManifest(tempdir) + + class DummyManifest: + is_submanifest = False + + def GetDefaultGroupsStr(self, with_platform=False): + return "" + + fakeproj.manifest = DummyManifest() + + fakeproj.config.SetString("repo.fetchcmd", "echo hi") + fakeproj.config.SetBoolean("repo.uselocalgitdirs", False) + + result = fakeproj.Sync(use_local_gitdirs=False) + self.assertFalse(result)