| # -*- coding:utf-8 -*- | |
| # | |
| # Copyright (C) 2008 The Android Open Source Project | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| from __future__ import print_function | |
| import os | |
| import re | |
| import sys | |
| import subprocess | |
| import tempfile | |
| from signal import SIGTERM | |
| from error import GitError | |
| from git_refs import HEAD | |
| import platform_utils | |
| from repo_trace import REPO_TRACE, IsTrace, Trace | |
| from wrapper import Wrapper | |
| GIT = 'git' | |
| # NB: These do not need to be kept in sync with the repo launcher script. | |
| # These may be much newer as it allows the repo launcher to roll between | |
| # different repo releases while source versions might require a newer git. | |
| # | |
| # The soft version is when we start warning users that the version is old and | |
| # we'll be dropping support for it. We'll refuse to work with versions older | |
| # than the hard version. | |
| # | |
| # git-1.7 is in (EOL) Ubuntu Precise. git-1.9 is in Ubuntu Trusty. | |
| MIN_GIT_VERSION_SOFT = (1, 9, 1) | |
| MIN_GIT_VERSION_HARD = (1, 7, 2) | |
| GIT_DIR = 'GIT_DIR' | |
| LAST_GITDIR = None | |
| LAST_CWD = None | |
| _ssh_proxy_path = None | |
| _ssh_sock_path = None | |
| _ssh_clients = [] | |
| _ssh_version = None | |
| def _run_ssh_version(): | |
| """run ssh -V to display the version number""" | |
| return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode() | |
| def _parse_ssh_version(ver_str=None): | |
| """parse a ssh version string into a tuple""" | |
| if ver_str is None: | |
| ver_str = _run_ssh_version() | |
| m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str) | |
| if m: | |
| return tuple(int(x) for x in m.group(1).split('.')) | |
| else: | |
| return () | |
| def ssh_version(): | |
| """return ssh version as a tuple""" | |
| global _ssh_version | |
| if _ssh_version is None: | |
| try: | |
| _ssh_version = _parse_ssh_version() | |
| except subprocess.CalledProcessError: | |
| print('fatal: unable to detect ssh version', file=sys.stderr) | |
| sys.exit(1) | |
| return _ssh_version | |
| def ssh_sock(create=True): | |
| global _ssh_sock_path | |
| if _ssh_sock_path is None: | |
| if not create: | |
| return None | |
| tmp_dir = '/tmp' | |
| if not os.path.exists(tmp_dir): | |
| tmp_dir = tempfile.gettempdir() | |
| if ssh_version() < (6, 7): | |
| tokens = '%r@%h:%p' | |
| else: | |
| tokens = '%C' # hash of %l%h%p%r | |
| _ssh_sock_path = os.path.join( | |
| tempfile.mkdtemp('', 'ssh-', tmp_dir), | |
| 'master-' + tokens) | |
| return _ssh_sock_path | |
| def _ssh_proxy(): | |
| global _ssh_proxy_path | |
| if _ssh_proxy_path is None: | |
| _ssh_proxy_path = os.path.join( | |
| os.path.dirname(__file__), | |
| 'git_ssh') | |
| return _ssh_proxy_path | |
| def _add_ssh_client(p): | |
| _ssh_clients.append(p) | |
| def _remove_ssh_client(p): | |
| try: | |
| _ssh_clients.remove(p) | |
| except ValueError: | |
| pass | |
| def terminate_ssh_clients(): | |
| global _ssh_clients | |
| for p in _ssh_clients: | |
| try: | |
| os.kill(p.pid, SIGTERM) | |
| p.wait() | |
| except OSError: | |
| pass | |
| _ssh_clients = [] | |
| _git_version = None | |
| class _GitCall(object): | |
| def version_tuple(self): | |
| global _git_version | |
| if _git_version is None: | |
| _git_version = Wrapper().ParseGitVersion() | |
| if _git_version is None: | |
| print('fatal: unable to detect git version', file=sys.stderr) | |
| sys.exit(1) | |
| return _git_version | |
| def __getattr__(self, name): | |
| name = name.replace('_', '-') | |
| def fun(*cmdv): | |
| command = [name] | |
| command.extend(cmdv) | |
| return GitCommand(None, command).Wait() == 0 | |
| return fun | |
| git = _GitCall() | |
| def RepoSourceVersion(): | |
| """Return the version of the repo.git tree.""" | |
| ver = getattr(RepoSourceVersion, 'version', None) | |
| # We avoid GitCommand so we don't run into circular deps -- GitCommand needs | |
| # to initialize version info we provide. | |
| if ver is None: | |
| env = GitCommand._GetBasicEnv() | |
| proj = os.path.dirname(os.path.abspath(__file__)) | |
| env[GIT_DIR] = os.path.join(proj, '.git') | |
| p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE, | |
| env=env) | |
| if p.wait() == 0: | |
| ver = p.stdout.read().strip().decode('utf-8') | |
| if ver.startswith('v'): | |
| ver = ver[1:] | |
| else: | |
| ver = 'unknown' | |
| setattr(RepoSourceVersion, 'version', ver) | |
| return ver | |
| class UserAgent(object): | |
| """Mange User-Agent settings when talking to external services | |
| We follow the style as documented here: | |
| https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent | |
| """ | |
| _os = None | |
| _repo_ua = None | |
| _git_ua = None | |
| @property | |
| def os(self): | |
| """The operating system name.""" | |
| if self._os is None: | |
| os_name = sys.platform | |
| if os_name.lower().startswith('linux'): | |
| os_name = 'Linux' | |
| elif os_name == 'win32': | |
| os_name = 'Win32' | |
| elif os_name == 'cygwin': | |
| os_name = 'Cygwin' | |
| elif os_name == 'darwin': | |
| os_name = 'Darwin' | |
| self._os = os_name | |
| return self._os | |
| @property | |
| def repo(self): | |
| """The UA when connecting directly from repo.""" | |
| if self._repo_ua is None: | |
| py_version = sys.version_info | |
| self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % ( | |
| RepoSourceVersion(), | |
| self.os, | |
| git.version_tuple().full, | |
| py_version.major, py_version.minor, py_version.micro) | |
| return self._repo_ua | |
| @property | |
| def git(self): | |
| """The UA when running git.""" | |
| if self._git_ua is None: | |
| self._git_ua = 'git/%s (%s) git-repo/%s' % ( | |
| git.version_tuple().full, | |
| self.os, | |
| RepoSourceVersion()) | |
| return self._git_ua | |
| user_agent = UserAgent() | |
| def git_require(min_version, fail=False, msg=''): | |
| git_version = git.version_tuple() | |
| if min_version <= git_version: | |
| return True | |
| if fail: | |
| need = '.'.join(map(str, min_version)) | |
| if msg: | |
| msg = ' for ' + msg | |
| print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr) | |
| sys.exit(1) | |
| return False | |
| class GitCommand(object): | |
| def __init__(self, | |
| project, | |
| cmdv, | |
| bare=False, | |
| provide_stdin=False, | |
| capture_stdout=False, | |
| capture_stderr=False, | |
| merge_output=False, | |
| disable_editor=False, | |
| ssh_proxy=False, | |
| cwd=None, | |
| gitdir=None): | |
| env = self._GetBasicEnv() | |
| # If we are not capturing std* then need to print it. | |
| self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr} | |
| if disable_editor: | |
| env['GIT_EDITOR'] = ':' | |
| if ssh_proxy: | |
| env['REPO_SSH_SOCK'] = ssh_sock() | |
| env['GIT_SSH'] = _ssh_proxy() | |
| env['GIT_SSH_VARIANT'] = 'ssh' | |
| if 'http_proxy' in env and 'darwin' == sys.platform: | |
| s = "'http.proxy=%s'" % (env['http_proxy'],) | |
| p = env.get('GIT_CONFIG_PARAMETERS') | |
| if p is not None: | |
| s = p + ' ' + s | |
| env['GIT_CONFIG_PARAMETERS'] = s | |
| if 'GIT_ALLOW_PROTOCOL' not in env: | |
| env['GIT_ALLOW_PROTOCOL'] = ( | |
| 'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc') | |
| env['GIT_HTTP_USER_AGENT'] = user_agent.git | |
| if project: | |
| if not cwd: | |
| cwd = project.worktree | |
| if not gitdir: | |
| gitdir = project.gitdir | |
| command = [GIT] | |
| if bare: | |
| if gitdir: | |
| env[GIT_DIR] = gitdir | |
| cwd = None | |
| command.append(cmdv[0]) | |
| # Need to use the --progress flag for fetch/clone so output will be | |
| # displayed as by default git only does progress output if stderr is a TTY. | |
| if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'): | |
| if '--progress' not in cmdv and '--quiet' not in cmdv: | |
| command.append('--progress') | |
| command.extend(cmdv[1:]) | |
| if provide_stdin: | |
| stdin = subprocess.PIPE | |
| else: | |
| stdin = None | |
| stdout = subprocess.PIPE | |
| stderr = subprocess.STDOUT if merge_output else subprocess.PIPE | |
| if IsTrace(): | |
| global LAST_CWD | |
| global LAST_GITDIR | |
| dbg = '' | |
| if cwd and LAST_CWD != cwd: | |
| if LAST_GITDIR or LAST_CWD: | |
| dbg += '\n' | |
| dbg += ': cd %s\n' % cwd | |
| LAST_CWD = cwd | |
| if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]: | |
| if LAST_GITDIR or LAST_CWD: | |
| dbg += '\n' | |
| dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR] | |
| LAST_GITDIR = env[GIT_DIR] | |
| dbg += ': ' | |
| dbg += ' '.join(command) | |
| if stdin == subprocess.PIPE: | |
| dbg += ' 0<|' | |
| if stdout == subprocess.PIPE: | |
| dbg += ' 1>|' | |
| if stderr == subprocess.PIPE: | |
| dbg += ' 2>|' | |
| elif stderr == subprocess.STDOUT: | |
| dbg += ' 2>&1' | |
| Trace('%s', dbg) | |
| try: | |
| p = subprocess.Popen(command, | |
| cwd=cwd, | |
| env=env, | |
| stdin=stdin, | |
| stdout=stdout, | |
| stderr=stderr) | |
| except Exception as e: | |
| raise GitError('%s: %s' % (command[1], e)) | |
| if ssh_proxy: | |
| _add_ssh_client(p) | |
| self.process = p | |
| self.stdin = p.stdin | |
| @staticmethod | |
| def _GetBasicEnv(): | |
| """Return a basic env for running git under. | |
| This is guaranteed to be side-effect free. | |
| """ | |
| env = os.environ.copy() | |
| for key in (REPO_TRACE, | |
| GIT_DIR, | |
| 'GIT_ALTERNATE_OBJECT_DIRECTORIES', | |
| 'GIT_OBJECT_DIRECTORY', | |
| 'GIT_WORK_TREE', | |
| 'GIT_GRAFT_FILE', | |
| 'GIT_INDEX_FILE'): | |
| env.pop(key, None) | |
| return env | |
| def Wait(self): | |
| try: | |
| p = self.process | |
| rc = self._CaptureOutput() | |
| finally: | |
| _remove_ssh_client(p) | |
| return rc | |
| def _CaptureOutput(self): | |
| p = self.process | |
| s_in = platform_utils.FileDescriptorStreams.create() | |
| s_in.add(p.stdout, sys.stdout, 'stdout') | |
| if p.stderr is not None: | |
| s_in.add(p.stderr, sys.stderr, 'stderr') | |
| self.stdout = '' | |
| self.stderr = '' | |
| while not s_in.is_done: | |
| in_ready = s_in.select() | |
| for s in in_ready: | |
| buf = s.read() | |
| if not buf: | |
| s_in.remove(s) | |
| continue | |
| if not hasattr(buf, 'encode'): | |
| buf = buf.decode() | |
| if s.std_name == 'stdout': | |
| self.stdout += buf | |
| else: | |
| self.stderr += buf | |
| if self.tee[s.std_name]: | |
| s.dest.write(buf) | |
| s.dest.flush() | |
| return p.wait() |