| # Copyright (C) 2021 The Android Open Source Project | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Provide functionality to get projects and their commit ids from Superproject. | |
| For more information on superproject, check out: | |
| https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects | |
| Examples: | |
| superproject = Superproject(manifest, name, remote, revision) | |
| UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects) | |
| """ | |
| import functools | |
| import glob | |
| import hashlib | |
| import os | |
| import sys | |
| import tempfile | |
| import time | |
| from typing import NamedTuple | |
| import urllib.parse | |
| import git_command | |
| from git_config import IsId | |
| from git_config import RepoConfig | |
| import platform_utils | |
| _SUPERPROJECT_GIT_NAME = "superproject.git" | |
| _SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml" | |
| class SyncResult(NamedTuple): | |
| """Return the status of sync and whether caller should exit.""" | |
| # Whether the superproject sync was successful. | |
| success: bool | |
| # Whether the caller should exit. | |
| fatal: bool | |
| class CommitIdsResult(NamedTuple): | |
| """Return the commit ids and whether caller should exit.""" | |
| # A dictionary with the projects/commit ids on success, otherwise None. | |
| commit_ids: dict | |
| # Whether the caller should exit. | |
| fatal: bool | |
| class UpdateProjectsResult(NamedTuple): | |
| """Return the overriding manifest file and whether caller should exit.""" | |
| # Path name of the overriding manifest file if successful, otherwise None. | |
| manifest_path: str | |
| # Whether the caller should exit. | |
| fatal: bool | |
| class Superproject: | |
| """Get commit ids from superproject. | |
| Initializes a bare local copy of a superproject for the manifest. This | |
| allows lookup of commit ids for all projects. It contains | |
| _project_commit_ids which is a dictionary with project/commit id entries. | |
| """ | |
| def __init__( | |
| self, | |
| manifest, | |
| name, | |
| remote, | |
| revision, | |
| superproject_dir="exp-superproject", | |
| ): | |
| """Initializes superproject. | |
| Args: | |
| manifest: A Manifest object that is to be written to a file. | |
| name: The unique name of the superproject | |
| remote: The RemoteSpec for the remote. | |
| revision: The name of the git branch to track. | |
| superproject_dir: Relative path under |manifest.subdir| to checkout | |
| superproject. | |
| """ | |
| self._project_commit_ids = None | |
| self._manifest = manifest | |
| self.name = name | |
| self.remote = remote | |
| self.revision = revision | |
| self._repodir = manifest.repodir | |
| self._superproject_dir = superproject_dir | |
| self._superproject_path = manifest.SubmanifestInfoDir( | |
| manifest.path_prefix, superproject_dir | |
| ) | |
| self._manifest_path = os.path.join( | |
| self._superproject_path, _SUPERPROJECT_MANIFEST_NAME | |
| ) | |
| git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-" | |
| self._remote_url = remote.url | |
| self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME | |
| self._work_git = os.path.join( | |
| self._superproject_path, self._work_git_name | |
| ) | |
| # The following are command arguemnts, rather than superproject | |
| # attributes, and were included here originally. They should eventually | |
| # become arguments that are passed down from the public methods, instead | |
| # of being treated as attributes. | |
| self._git_event_log = None | |
| self._quiet = False | |
| self._print_messages = False | |
| def SetQuiet(self, value): | |
| """Set the _quiet attribute.""" | |
| self._quiet = value | |
| def SetPrintMessages(self, value): | |
| """Set the _print_messages attribute.""" | |
| self._print_messages = value | |
| def SetRevisionId(self, revision_id: str) -> None: | |
| """Set the revisionId of the superproject to sync to.""" | |
| self.revision = revision_id | |
| @property | |
| def commit_id(self): | |
| """Returns the commit ID of the superproject checkout.""" | |
| cmd = ["rev-parse", self.revision] | |
| p = git_command.GitCommand( | |
| None, # project | |
| cmd, | |
| gitdir=self._work_git, | |
| bare=True, | |
| capture_stdout=True, | |
| capture_stderr=True, | |
| ) | |
| retval = p.Wait() | |
| if retval != 0: | |
| self._LogWarning( | |
| "git rev-parse call failed, command: git {}, " | |
| "return code: {}, stderr: {}", | |
| cmd, | |
| retval, | |
| p.stderr, | |
| ) | |
| return None | |
| return p.stdout | |
| @property | |
| def project_commit_ids(self): | |
| """Returns a dictionary of projects and their commit ids.""" | |
| return self._project_commit_ids | |
| @property | |
| def manifest_path(self): | |
| """Returns the manifest path if the path exists or None.""" | |
| return ( | |
| self._manifest_path if os.path.exists(self._manifest_path) else None | |
| ) | |
| @property | |
| def repo_id(self): | |
| """Returns the repo ID for the superproject. | |
| For example, if the superproject points to: | |
| https://android-review.googlesource.com/platform/superproject/ | |
| Then the repo_id would be: | |
| android/platform/superproject | |
| """ | |
| review_url = self.remote.review | |
| if review_url: | |
| parsed_url = urllib.parse.urlparse(review_url) | |
| netloc = parsed_url.netloc | |
| if netloc: | |
| parts = netloc.split("-review", 1) | |
| host = parts[0] | |
| rev = self._GetRef("HEAD") | |
| return f"{host}/{self.name}@{rev}" | |
| return None | |
| def _LogMessage(self, fmt, *inputs): | |
| """Logs message to stderr and _git_event_log.""" | |
| message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}" | |
| if self._print_messages: | |
| print(message, file=sys.stderr) | |
| if self._git_event_log: | |
| self._git_event_log.ErrorEvent(message, fmt) | |
| def _LogMessagePrefix(self): | |
| """Returns the prefix string to be logged in each log message""" | |
| return ( | |
| f"repo superproject revision: {self.revision} " | |
| f"url: {self._remote_url}" | |
| ) | |
| def _LogError(self, fmt, *inputs): | |
| """Logs error message to stderr and _git_event_log.""" | |
| self._LogMessage(f"error: {fmt}", *inputs) | |
| def _LogWarning(self, fmt, *inputs): | |
| """Logs warning message to stderr and _git_event_log.""" | |
| self._LogMessage(f"warning: {fmt}", *inputs) | |
| def _Init(self): | |
| """Sets up a local Git repository to get a copy of a superproject. | |
| Returns: | |
| True if initialization is successful, or False. | |
| """ | |
| if not os.path.exists(self._superproject_path): | |
| os.mkdir(self._superproject_path) | |
| if os.path.exists(self._work_git): | |
| return True | |
| if not self._quiet: | |
| print( | |
| "%s: Performing initial setup for superproject; this might " | |
| "take several minutes." % self._work_git | |
| ) | |
| tmp_gitdir_prefix = ".tmp-superproject-initgitdir-" | |
| tmp_gitdir = tempfile.mkdtemp( | |
| prefix=tmp_gitdir_prefix, | |
| dir=self._superproject_path, | |
| ) | |
| tmp_git_name = os.path.basename(tmp_gitdir) | |
| try: | |
| cmd = ["init", "--bare", tmp_git_name] | |
| p = git_command.GitCommand( | |
| None, | |
| cmd, | |
| cwd=self._superproject_path, | |
| capture_stdout=True, | |
| capture_stderr=True, | |
| ) | |
| retval = p.Wait() | |
| if retval: | |
| self._LogWarning( | |
| "git init call failed, command: git {}, " | |
| "return code: {}, stderr: {}", | |
| cmd, | |
| retval, | |
| p.stderr, | |
| ) | |
| return False | |
| platform_utils.rename(tmp_gitdir, self._work_git) | |
| tmp_gitdir = None | |
| return True | |
| finally: | |
| # Clean up the temporary directory created during the process, | |
| # as well as any stale ones left over from previous attempts. | |
| if tmp_gitdir and os.path.exists(tmp_gitdir): | |
| platform_utils.rmtree(tmp_gitdir) | |
| age_threshold = 60 * 60 * 24 # 1 day in seconds | |
| now = time.time() | |
| for tmp_dir in glob.glob( | |
| os.path.join(self._superproject_path, f"{tmp_gitdir_prefix}*") | |
| ): | |
| try: | |
| mtime = os.path.getmtime(tmp_dir) | |
| if now - mtime > age_threshold: | |
| platform_utils.rmtree(tmp_dir) | |
| except OSError: | |
| pass | |
| def _Fetch(self): | |
| """Fetches a superproject for the manifest based on |_remote_url|. | |
| This runs git fetch which stores a local copy the superproject. | |
| Returns: | |
| True if fetch is successful, or False. | |
| """ | |
| if not os.path.exists(self._work_git): | |
| self._LogWarning("git fetch missing directory: {}", self._work_git) | |
| return False | |
| if not git_command.git_require((2, 28, 0)): | |
| self._LogWarning( | |
| "superproject requires a git version 2.28 or later" | |
| ) | |
| return False | |
| cmd = [ | |
| "fetch", | |
| self._remote_url, | |
| "--depth", | |
| "1", | |
| "--force", | |
| "--no-tags", | |
| "--filter", | |
| "blob:none", | |
| ] | |
| # Check if there is a local ref that we can pass to --negotiation-tip. | |
| # If this is the first fetch, it does not exist yet. | |
| # We use --negotiation-tip to speed up the fetch. Superproject branches | |
| # do not share commits. So this lets git know it only needs to send | |
| # commits reachable from the specified local refs. | |
| negotiation_ref = self.revision | |
| if negotiation_ref and not negotiation_ref.startswith("refs/"): | |
| negotiation_ref = f"refs/heads/{negotiation_ref}" | |
| rev_commit = self._GetRef(negotiation_ref) if negotiation_ref else "" | |
| if rev_commit: | |
| cmd.extend(["--negotiation-tip", rev_commit]) | |
| if self.revision: | |
| # If revision is a commit hash, fetch it directly to avoid | |
| # creating a local branch of the same name. | |
| refspec = ( | |
| self.revision | |
| if IsId(self.revision) | |
| else f"{self.revision}:{self.revision}" | |
| ) | |
| cmd.append(refspec) | |
| p = git_command.GitCommand( | |
| None, | |
| cmd, | |
| gitdir=self._work_git, | |
| bare=True, | |
| capture_stdout=True, | |
| capture_stderr=True, | |
| ) | |
| retval = p.Wait() | |
| if retval: | |
| self._LogWarning( | |
| "git fetch call failed, command: git {}, " | |
| "return code: {}, stderr: {}", | |
| cmd, | |
| retval, | |
| p.stderr, | |
| ) | |
| return False | |
| return True | |
| def _GetRef(self, ref: str) -> str: | |
| """Resolve one local ref without loading the entire ref namespace.""" | |
| p = git_command.GitCommand( | |
| None, | |
| ["rev-parse", "--verify", "--quiet", ref], | |
| gitdir=self._work_git, | |
| bare=True, | |
| capture_stdout=True, | |
| capture_stderr=True, | |
| log_as_error=False, | |
| ) | |
| if p.Wait() == 0: | |
| return p.stdout.strip() | |
| return "" | |
| def _LsTree(self): | |
| """Gets the commit ids for all projects. | |
| Works only in git repositories. | |
| Returns: | |
| data: data returned from 'git ls-tree ...'. None on error. | |
| """ | |
| if not os.path.exists(self._work_git): | |
| self._LogWarning( | |
| "git ls-tree missing directory: {}", self._work_git | |
| ) | |
| return None | |
| data = None | |
| branch = "HEAD" if not self.revision else self.revision | |
| cmd = ["ls-tree", "-z", "-r", branch] | |
| p = git_command.GitCommand( | |
| None, | |
| cmd, | |
| gitdir=self._work_git, | |
| bare=True, | |
| capture_stdout=True, | |
| capture_stderr=True, | |
| ) | |
| retval = p.Wait() | |
| if retval == 0: | |
| data = p.stdout | |
| else: | |
| self._LogWarning( | |
| "git ls-tree call failed, command: git {}, " | |
| "return code: {}, stderr: {}", | |
| cmd, | |
| retval, | |
| p.stderr, | |
| ) | |
| return None | |
| return data | |
| def Sync(self, git_event_log): | |
| """Gets a local copy of a superproject for the manifest. | |
| Args: | |
| git_event_log: an EventLog, for git tracing. | |
| Returns: | |
| SyncResult | |
| """ | |
| self._git_event_log = git_event_log | |
| if not self._manifest.superproject: | |
| self._LogWarning( | |
| "superproject tag is not defined in manifest: {}", | |
| self._manifest.manifestFile, | |
| ) | |
| return SyncResult(False, False) | |
| should_exit = True | |
| if not self._remote_url: | |
| self._LogWarning( | |
| "superproject URL is not defined in manifest: {}", | |
| self._manifest.manifestFile, | |
| ) | |
| return SyncResult(False, should_exit) | |
| if not self._Init(): | |
| return SyncResult(False, should_exit) | |
| if IsId(self.revision) and self.commit_id: | |
| return SyncResult(True, False) | |
| if not self._Fetch(): | |
| return SyncResult(False, should_exit) | |
| if not self._quiet: | |
| print( | |
| "%s: Initial setup for superproject completed." % self._work_git | |
| ) | |
| return SyncResult(True, False) | |
| def _GetAllProjectsCommitIds(self): | |
| """Get commit ids for all projects from superproject and save them. | |
| Commit ids are saved in _project_commit_ids. | |
| Returns: | |
| CommitIdsResult | |
| """ | |
| sync_result = self.Sync(self._git_event_log) | |
| if not sync_result.success: | |
| return CommitIdsResult(None, sync_result.fatal) | |
| data = self._LsTree() | |
| if not data: | |
| self._LogWarning( | |
| "git ls-tree failed to return data for manifest: {}", | |
| self._manifest.manifestFile, | |
| ) | |
| return CommitIdsResult(None, True) | |
| # Parse lines like the following to select lines starting with '160000' | |
| # and build a dictionary with project path (last element) and its commit | |
| # id (3rd element). | |
| # | |
| # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 | |
| # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 # noqa: E501 | |
| commit_ids = {} | |
| for line in data.split("\x00"): | |
| ls_data = line.split(None, 3) | |
| if not ls_data: | |
| break | |
| if ls_data[0] == "160000": | |
| commit_ids[ls_data[3]] = ls_data[2] | |
| self._project_commit_ids = commit_ids | |
| return CommitIdsResult(commit_ids, False) | |
| def _WriteManifestFile(self): | |
| """Writes manifest to a file. | |
| Returns: | |
| manifest_path: Path name of the file into which manifest is written | |
| instead of None. | |
| """ | |
| if not os.path.exists(self._superproject_path): | |
| self._LogWarning( | |
| "missing superproject directory: {}", self._superproject_path | |
| ) | |
| return None | |
| manifest_str = self._manifest.ToXml( | |
| filter_groups="all", | |
| omit_local=True, | |
| ).toxml() | |
| manifest_path = self._manifest_path | |
| try: | |
| with open(manifest_path, "w", encoding="utf-8") as fp: | |
| fp.write(manifest_str) | |
| except OSError as e: | |
| self._LogError("cannot write manifest to : {} {}", manifest_path, e) | |
| return None | |
| return manifest_path | |
| def _SkipUpdatingProjectRevisionId(self, project): | |
| """Checks if a project's revision id needs to be updated or not. | |
| Revision id for projects from local manifest will not be updated. | |
| Args: | |
| project: project whose revision id is being updated. | |
| Returns: | |
| True if a project's revision id should not be updated, or False, | |
| """ | |
| path = project.relpath | |
| if not path: | |
| return True | |
| # Skip the project with revisionId. | |
| if project.revisionId: | |
| return True | |
| # Skip the project if it comes from the local manifest. | |
| return project.manifest.IsFromLocalManifest(project) | |
| def UpdateProjectsRevisionId(self, projects, git_event_log): | |
| """Update revisionId of every project in projects with the commit id. | |
| Args: | |
| projects: a list of projects whose revisionId needs to be updated. | |
| git_event_log: an EventLog, for git tracing. | |
| Returns: | |
| UpdateProjectsResult | |
| """ | |
| self._git_event_log = git_event_log | |
| commit_ids_result = self._GetAllProjectsCommitIds() | |
| commit_ids = commit_ids_result.commit_ids | |
| if not commit_ids: | |
| return UpdateProjectsResult(None, commit_ids_result.fatal) | |
| projects_missing_commit_ids = [] | |
| for project in projects: | |
| if self._SkipUpdatingProjectRevisionId(project): | |
| continue | |
| path = project.relpath | |
| commit_id = commit_ids.get(path) | |
| if not commit_id: | |
| projects_missing_commit_ids.append(path) | |
| # If superproject doesn't have a commit id for a project, then report an | |
| # error event and continue as if do not use superproject is specified. | |
| if projects_missing_commit_ids: | |
| self._LogWarning( | |
| "please file a bug using {} to report missing " | |
| "commit_ids for: {}", | |
| self._manifest.contactinfo.bugurl, | |
| projects_missing_commit_ids, | |
| ) | |
| return UpdateProjectsResult(None, False) | |
| for project in projects: | |
| if not self._SkipUpdatingProjectRevisionId(project): | |
| project.SetRevisionId(commit_ids.get(project.relpath)) | |
| manifest_path = self._WriteManifestFile() | |
| return UpdateProjectsResult(manifest_path, False) | |
| @functools.lru_cache(maxsize=None) | |
| def _UseSuperprojectFromConfiguration(): | |
| """Returns the user choice of whether to use superproject.""" | |
| user_cfg = RepoConfig.ForUser() | |
| time_now = int(time.time()) | |
| user_value = user_cfg.GetBoolean("repo.superprojectChoice") | |
| if user_value is not None: | |
| user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire") | |
| if ( | |
| user_expiration is None | |
| or user_expiration <= 0 | |
| or user_expiration >= time_now | |
| ): | |
| # TODO(b/190688390) - Remove prompt when we are comfortable with the | |
| # new default value. | |
| if user_value: | |
| print( | |
| ( | |
| "You are currently enrolled in Git submodules " | |
| "experiment (go/android-submodules-quickstart). Use " | |
| "--no-use-superproject to override.\n" | |
| ), | |
| file=sys.stderr, | |
| ) | |
| else: | |
| print( | |
| ( | |
| "You are not currently enrolled in Git submodules " | |
| "experiment (go/android-submodules-quickstart). Use " | |
| "--use-superproject to override.\n" | |
| ), | |
| file=sys.stderr, | |
| ) | |
| return user_value | |
| # We don't have an unexpired choice, ask for one. | |
| system_cfg = RepoConfig.ForSystem() | |
| system_value = system_cfg.GetBoolean("repo.superprojectChoice") | |
| if system_value: | |
| # The system configuration is proposing that we should enable the | |
| # use of superproject. Treat the user as enrolled for two weeks. | |
| # | |
| # TODO(b/190688390) - Remove prompt when we are comfortable with the new | |
| # default value. | |
| userchoice = True | |
| time_choiceexpire = time_now + (86400 * 14) | |
| user_cfg.SetString( | |
| "repo.superprojectChoiceExpire", str(time_choiceexpire) | |
| ) | |
| user_cfg.SetBoolean("repo.superprojectChoice", userchoice) | |
| print( | |
| "You are automatically enrolled in Git submodules experiment " | |
| "(go/android-submodules-quickstart) for another two weeks.\n", | |
| file=sys.stderr, | |
| ) | |
| return True | |
| # For all other cases, we would not use superproject by default. | |
| return False | |
| def PrintMessages(use_superproject, manifest): | |
| """Returns a boolean if error/warning messages are to be printed. | |
| Args: | |
| use_superproject: option value from optparse. | |
| manifest: manifest to use. | |
| """ | |
| return use_superproject is not None or bool(manifest.superproject) | |
| def UseSuperproject(use_superproject, manifest): | |
| """Returns a boolean if use-superproject option is enabled. | |
| Args: | |
| use_superproject: option value from optparse. | |
| manifest: manifest to use. | |
| Returns: | |
| Whether the superproject should be used. | |
| """ | |
| if not manifest.superproject: | |
| # This (sub) manifest does not have a superproject definition. | |
| return False | |
| elif use_superproject is not None: | |
| return use_superproject | |
| else: | |
| client_value = manifest.manifestProject.use_superproject | |
| if client_value is not None: | |
| return client_value | |
| elif manifest.superproject: | |
| return _UseSuperprojectFromConfiguration() | |
| else: | |
| return False |