Merge "Ignore clone.bundle on HTTP 501, i.e. Not Implemented"
diff --git a/SUBMITTING_PATCHES b/SUBMITTING_PATCHES
index 50e2cf7..8656ee7 100644
--- a/SUBMITTING_PATCHES
+++ b/SUBMITTING_PATCHES
@@ -4,7 +4,9 @@
- Provide a meaningful commit message.
- Check for coding errors with pylint
- Make sure all code is under the Apache License, 2.0.
- - Publish your changes for review:
+ - Publish your changes for review.
+ - Make corrections if requested.
+ - Verify your changes on gerrit so they can be submitted.
git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master
@@ -75,6 +77,17 @@
https://gerrit-review.googlesource.com/new-password
+Ensure that you have the local commit hook installed to automatically
+add a ChangeId to your commits:
+
+ curl -Lo `git rev-parse --git-dir`/hooks/commit-msg https://gerrit-review.googlesource.com/tools/hooks/commit-msg
+ chmod +x `git rev-parse --git-dir`/hooks/commit-msg
+
+If you have already committed your changes you will need to amend the commit
+to get the ChangeId added.
+
+ git commit --amend
+
Push your patches over HTTPS to the review server, possibly through
a remembered remote to make this easier in the future:
@@ -85,3 +98,18 @@
You will be automatically emailed a copy of your commits, and any
comments made by the project maintainers.
+
+
+(5) Make changes if requested
+
+The project maintainer who reviews your changes might request changes to your
+commit. If you make the requested changes you will need to amend your commit
+and push it to the review server again.
+
+
+(6) Verify your changes on gerrit
+
+After you receive a Code-Review+2 from the maintainer, select the Verified
+button on the gerrit page for the change. This verifies that you have tested
+your changes and notifies the maintainer that they are ready to be submitted.
+The maintainer will then submit your changes to the repository.
diff --git a/command.py b/command.py
index 38cacd3..bc2f950 100644
--- a/command.py
+++ b/command.py
@@ -31,7 +31,7 @@
manifest = None
_optparse = None
- def WantPager(self, opt):
+ def WantPager(self, _opt):
return False
def ReadEnvironmentOptions(self, opts):
@@ -63,7 +63,7 @@
usage = self.helpUsage.strip().replace('%prog', me)
except AttributeError:
usage = 'repo %s' % self.NAME
- self._optparse = optparse.OptionParser(usage = usage)
+ self._optparse = optparse.OptionParser(usage=usage)
self._Options(self._optparse)
return self._optparse
@@ -106,13 +106,13 @@
def _UpdatePathToProjectMap(self, project):
self._by_path[project.worktree] = project
- def _GetProjectByPath(self, path):
+ def _GetProjectByPath(self, manifest, path):
project = None
if os.path.exists(path):
oldpath = None
- while path \
- and path != oldpath \
- and path != self.manifest.topdir:
+ while path and \
+ path != oldpath and \
+ path != manifest.topdir:
try:
project = self._by_path[path]
break
@@ -126,16 +126,19 @@
pass
return project
- def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
+ def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+ submodules_ok=False):
"""A list of projects that match the arguments.
"""
- all_projects_list = self.manifest.projects
+ if not manifest:
+ manifest = self.manifest
+ all_projects_list = manifest.projects
result = []
- mp = self.manifest.manifestProject
+ mp = manifest.manifestProject
if not groups:
- groups = mp.config.GetString('manifest.groups')
+ groups = mp.config.GetString('manifest.groups')
if not groups:
groups = 'default,platform-' + platform.system().lower()
groups = [x for x in re.split(r'[,\s]+', groups) if x]
@@ -148,29 +151,28 @@
for p in project.GetDerivedSubprojects())
all_projects_list.extend(derived_projects.values())
for project in all_projects_list:
- if ((missing_ok or project.Exists) and
- project.MatchesGroups(groups)):
+ if (missing_ok or project.Exists) and project.MatchesGroups(groups):
result.append(project)
else:
self._ResetPathToProjectMap(all_projects_list)
for arg in args:
- projects = self.manifest.GetProjectsWithName(arg)
+ projects = manifest.GetProjectsWithName(arg)
if not projects:
path = os.path.abspath(arg).replace('\\', '/')
- project = self._GetProjectByPath(path)
+ project = self._GetProjectByPath(manifest, path)
# If it's not a derived project, update path->project mapping and
# search again, as arg might actually point to a derived subproject.
- if (project and not project.Derived and
- (submodules_ok or project.sync_s)):
+ if (project and not project.Derived and (submodules_ok or
+ project.sync_s)):
search_again = False
for subproject in project.GetDerivedSubprojects():
self._UpdatePathToProjectMap(subproject)
search_again = True
if search_again:
- project = self._GetProjectByPath(path) or project
+ project = self._GetProjectByPath(manifest, path) or project
if project:
projects = [project]
@@ -191,17 +193,24 @@
result.sort(key=_getpath)
return result
- def FindProjects(self, args):
+ def FindProjects(self, args, inverse=False):
result = []
patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
for project in self.GetProjects(''):
for pattern in patterns:
- if pattern.search(project.name) or pattern.search(project.relpath):
+ match = pattern.search(project.name) or pattern.search(project.relpath)
+ if not inverse and match:
result.append(project)
break
+ if inverse and match:
+ break
+ else:
+ if inverse:
+ result.append(project)
result.sort(key=lambda project: project.relpath)
return result
+
# pylint: disable=W0223
# Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
# override method `Execute` which is abstract in `Command`. Since that method
@@ -211,19 +220,33 @@
"""Command which requires user interaction on the tty and
must not run within a pager, even if the user asks to.
"""
- def WantPager(self, opt):
+ def WantPager(self, _opt):
return False
+
class PagedCommand(Command):
"""Command which defaults to output in a pager, as its
display tends to be larger than one screen full.
"""
- def WantPager(self, opt):
+ def WantPager(self, _opt):
return True
# pylint: enable=W0223
+
class MirrorSafeCommand(object):
"""Command permits itself to run within a mirror,
and does not require a working directory.
"""
+
+
+class GitcAvailableCommand(object):
+ """Command that requires GITC to be available, but does
+ not require the local client to be a GITC client.
+ """
+
+
+class GitcClientCommand(object):
+ """Command that requires the local client to be a GITC
+ client.
+ """
diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt
index 1aa9396..140a782 100644
--- a/docs/manifest-format.txt
+++ b/docs/manifest-format.txt
@@ -47,10 +47,12 @@
<!ATTLIST default sync-s CDATA #IMPLIED>
<!ELEMENT manifest-server (EMPTY)>
- <!ATTLIST url CDATA #REQUIRED>
+ <!ATTLIST manifest-server url CDATA #REQUIRED>
<!ELEMENT project (annotation*,
- project*)>
+ project*,
+ copyfile*,
+ linkfile*)>
<!ATTLIST project name CDATA #REQUIRED>
<!ATTLIST project path CDATA #IMPLIED>
<!ATTLIST project remote IDREF #IMPLIED>
@@ -68,7 +70,15 @@
<!ATTLIST annotation value CDATA #REQUIRED>
<!ATTLIST annotation keep CDATA "true">
- <!ELEMENT extend-project>
+ <!ELEMENT copyfile (EMPTY)>
+ <!ATTLIST copyfile src CDATA #REQUIRED>
+ <!ATTLIST copyfile dest CDATA #REQUIRED>
+
+ <!ELEMENT linkfile (EMPTY)>
+ <!ATTLIST linkfile src CDATA #REQUIRED>
+ <!ATTLIST linkfile dest CDATA #REQUIRED>
+
+ <!ELEMENT extend-project (EMPTY)>
<!ATTLIST extend-project name CDATA #REQUIRED>
<!ATTLIST extend-project path CDATA #IMPLIED>
<!ATTLIST extend-project groups CDATA #IMPLIED>
@@ -285,6 +295,21 @@
"false". This attribute determines whether or not the annotation will
be kept when exported with the manifest subcommand.
+Element copyfile
+----------------
+
+Zero or more copyfile elements may be specified as children of a
+project element. Each element describes a src-dest pair of files;
+the "src" file will be copied to the "dest" place during 'repo sync'
+command.
+"src" is project relative, "dest" is relative to the top of the tree.
+
+Element linkfile
+----------------
+
+It's just like copyfile and runs at the same time as copyfile but
+instead of copying it creates a symlink.
+
Element remove-project
----------------------
diff --git a/git_command.py b/git_command.py
index 0893bff..9f7d293 100644
--- a/git_command.py
+++ b/git_command.py
@@ -168,6 +168,9 @@
if p is not None:
s = p + ' ' + s
_setenv(env, 'GIT_CONFIG_PARAMETERS', s)
+ if 'GIT_ALLOW_PROTOCOL' not in env:
+ _setenv(env, 'GIT_ALLOW_PROTOCOL',
+ 'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
if project:
if not cwd:
diff --git a/git_config.py b/git_config.py
index 8ded7c2..0379181 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,6 +15,8 @@
from __future__ import print_function
+import contextlib
+import errno
import json
import os
import re
@@ -502,6 +504,43 @@
return m.group(1)
return None
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+ if url.startswith('persistent-'):
+ try:
+ p = subprocess.Popen(
+ ['git-remote-persistent-https', '-print_config', url],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ cookieprefix = 'http.cookiefile='
+ proxyprefix = 'http.proxy='
+ cookiefile = None
+ proxy = None
+ for line in p.stdout:
+ line = line.strip()
+ if line.startswith(cookieprefix):
+ cookiefile = line[len(cookieprefix):]
+ if line.startswith(proxyprefix):
+ proxy = line[len(proxyprefix):]
+ # Leave subprocess open, as cookie file may be transient.
+ if cookiefile or proxy:
+ yield cookiefile, proxy
+ return
+ finally:
+ p.stdin.close()
+ if p.wait():
+ err_msg = p.stderr.read()
+ if ' -print_config' in err_msg:
+ pass # Persistent proxy doesn't support -print_config.
+ elif not quiet:
+ print(err_msg, file=sys.stderr)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ pass # No persistent proxy.
+ raise
+ yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
def _preconnect(url):
m = URI_ALL.match(url)
if m:
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 0000000..0f3e181
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,148 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def get_gitc_manifest_dir():
+ return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+ return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
+
+def _set_project_revisions(projects):
+ """Sets the revisionExpr for a list of projects.
+
+ Because of the limit of open file descriptors allowed, length of projects
+ should not be overly large. Recommend calling this function multiple times
+ with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+ @param projects: List of project objects to set the revionExpr for.
+ """
+ # Retrieve the commit id for each project based off of it's current
+ # revisionExpr and it is not already a commit id.
+ project_gitcmds = [(
+ project, git_command.GitCommand(None,
+ ['ls-remote',
+ project.remote.url,
+ project.revisionExpr],
+ capture_stdout=True, cwd='/tmp'))
+ for project in projects if not git_config.IsId(project.revisionExpr)]
+ for proj, gitcmd in project_gitcmds:
+ if gitcmd.Wait():
+ print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+ sys.exit(1)
+ proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def _manifest_groups(manifest):
+ """Returns the manifest group string that should be synced
+
+ This is the same logic used by Command.GetProjects(), which is used during
+ repo sync
+
+ @param manifest: The XmlManifest object
+ """
+ mp = manifest.manifestProject
+ groups = mp.config.GetString('manifest.groups')
+ if not groups:
+ groups = 'default,platform-' + platform.system().lower()
+ return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+ """Generate a manifest for shafsd to use for this GITC client.
+
+ @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+ @param manifest: A GitcManifest object loaded with the current repo manifest.
+ @param paths: List of project paths we want to update.
+ """
+
+ print('Generating GITC Manifest by fetching revision SHAs for each '
+ 'project.')
+ if paths is None:
+ paths = manifest.paths.keys()
+
+ groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+ # Convert the paths to projects, and filter them to the matched groups.
+ projects = [manifest.paths[p] for p in paths]
+ projects = [p for p in projects if p.MatchesGroups(groups)]
+
+ if gitc_manifest is not None:
+ for path, proj in manifest.paths.iteritems():
+ if not proj.MatchesGroups(groups):
+ continue
+
+ if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+ proj.upstream = proj.revisionExpr
+
+ if not path in gitc_manifest.paths:
+ # Any new projects need their first revision, even if we weren't asked
+ # for them.
+ projects.append(proj)
+ elif not path in paths:
+ # And copy revisions from the previous manifest if we're not updating
+ # them now.
+ gitc_proj = gitc_manifest.paths[path]
+ if gitc_proj.old_revision:
+ proj.revisionExpr = None
+ proj.old_revision = gitc_proj.old_revision
+ else:
+ proj.revisionExpr = gitc_proj.revisionExpr
+
+ index = 0
+ while index < len(projects):
+ _set_project_revisions(
+ projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+ index += NUM_BATCH_RETRIEVE_REVISIONID
+
+ if gitc_manifest is not None:
+ for path, proj in gitc_manifest.paths.iteritems():
+ if proj.old_revision and path in paths:
+ # If we updated a project that has been started, keep the old-revision
+ # updated.
+ repo_proj = manifest.paths[path]
+ repo_proj.old_revision = repo_proj.revisionExpr
+ repo_proj.revisionExpr = None
+
+ # Convert URLs from relative to absolute.
+ for name, remote in manifest.remotes.iteritems():
+ remote.fetchUrl = remote.resolvedFetchUrl
+
+ # Save the manifest.
+ save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+ """Save the manifest file in the client_dir.
+
+ @param client_dir: Client directory to save the manifest in.
+ @param manifest: Manifest object to save.
+ """
+ if not client_dir:
+ client_dir = manifest.gitc_client_dir
+ with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+ manifest.Save(f, groups=_manifest_groups(manifest))
+ # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+ # Give the GITC filesystem time to register the manifest changes.
+ time.sleep(3)
diff --git a/hooks/commit-msg b/hooks/commit-msg
index d8f009b..40ac237 100755
--- a/hooks/commit-msg
+++ b/hooks/commit-msg
@@ -1,6 +1,7 @@
#!/bin/sh
+# From Gerrit Code Review 2.12.1
#
-# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
#
# Copyright (C) 2009 The Android Open Source Project
#
@@ -19,7 +20,7 @@
unset GREP_OPTIONS
-CHANGE_ID_AFTER="Bug|Issue"
+CHANGE_ID_AFTER="Bug|Issue|Test"
MSG="$1"
# Check for, and add if missing, a unique Change-Id
@@ -38,6 +39,12 @@
return
fi
+ # Do not add Change-Id to temp commits
+ if echo "$clean_message" | head -1 | grep -q '^\(fixup\|squash\)!'
+ then
+ return
+ fi
+
if test "false" = "`git config --bool --get gerrit.createChangeId`"
then
return
@@ -57,6 +64,10 @@
AWK=/usr/xpg4/bin/awk
fi
+ # Get core.commentChar from git config or use default symbol
+ commentChar=`git config --get core.commentChar`
+ commentChar=${commentChar:-#}
+
# How this works:
# - parse the commit message as (textLine+ blankLine*)*
# - assume textLine+ to be a footer until proven otherwise
@@ -75,8 +86,8 @@
blankLines = 0
}
- # Skip lines starting with "#" without any spaces before it.
- /^#/ { next }
+ # Skip lines starting with commentChar without any spaces before it.
+ /^'"$commentChar"'/ { next }
# Skip the line starting with the diff command and everything after it,
# up to the end of the file, assuming it is only patch data.
diff --git a/main.py b/main.py
index 6736abc..4f4eb9f 100755
--- a/main.py
+++ b/main.py
@@ -42,6 +42,7 @@
from git_config import init_ssh, close_ssh
from command import InteractiveCommand
from command import MirrorSafeCommand
+from command import GitcAvailableCommand, GitcClientCommand
from subcmds.version import Version
from editor import Editor
from error import DownloadError
@@ -51,7 +52,8 @@
from error import NoManifestException
from error import NoSuchProjectError
from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
from pager import RunPager
from wrapper import WrapperPath, Wrapper
@@ -129,6 +131,12 @@
cmd.repodir = self.repodir
cmd.manifest = XmlManifest(cmd.repodir)
+ cmd.gitc_manifest = None
+ gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+ if gitc_client_name:
+ cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+ cmd.manifest.isGitcClient = True
+
Editor.globalConfig = cmd.manifest.globalConfig
if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -136,6 +144,16 @@
file=sys.stderr)
return 1
+ if isinstance(cmd, GitcAvailableCommand) and not gitc_utils.get_gitc_manifest_dir():
+ print("fatal: '%s' requires GITC to be available" % name,
+ file=sys.stderr)
+ return 1
+
+ if isinstance(cmd, GitcClientCommand) and not gitc_client_name:
+ print("fatal: '%s' requires a GITC client" % name,
+ file=sys.stderr)
+ return 1
+
try:
copts, cargs = cmd.OptionParser.parse_args(argv)
copts = cmd.ReadEnvironmentOptions(copts)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e71960..3ac607e 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -29,6 +29,7 @@
urllib = imp.new_module('urllib')
urllib.parse = urlparse
+import gitc_utils
from git_config import GitConfig
from git_refs import R_HEADS, HEAD
from project import RemoteSpec, Project, MetaProject
@@ -112,6 +113,7 @@
self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
self.globalConfig = GitConfig.ForUser()
self.localManifestWarning = False
+ self.isGitcClient = False
self.repoProject = MetaProject(self, 'repo',
gitdir = os.path.join(repodir, 'repo/.git'),
@@ -165,12 +167,13 @@
def _ParseGroups(self, groups):
return [x for x in re.split(r'[,\s]+', groups) if x]
- def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
+ def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
"""Write the current manifest out to the given file descriptor.
"""
mp = self.manifestProject
- groups = mp.config.GetString('manifest.groups')
+ if groups is None:
+ groups = mp.config.GetString('manifest.groups')
if groups:
groups = self._ParseGroups(groups)
@@ -303,6 +306,11 @@
if p.sync_s:
e.setAttribute('sync-s', 'true')
+ if p.clone_depth:
+ e.setAttribute('clone-depth', str(p.clone_depth))
+
+ self._output_manifest_project_extras(p, e)
+
if p.subprojects:
subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, list(sorted(subprojects)))
@@ -320,6 +328,10 @@
doc.writexml(fd, '', ' ', '\n', 'UTF-8')
+ def _output_manifest_project_extras(self, p, e):
+ """Manifests can modify e if they support extra project attributes."""
+ pass
+
@property
def paths(self):
self._Load()
@@ -709,7 +721,7 @@
def _UnjoinName(self, parent_name, name):
return os.path.relpath(name, parent_name)
- def _ParseProject(self, node, parent = None):
+ def _ParseProject(self, node, parent = None, **extra_proj_attrs):
"""
reads a <project> element from the manifest file
"""
@@ -804,7 +816,8 @@
clone_depth = clone_depth,
upstream = upstream,
parent = parent,
- dest_branch = dest_branch)
+ dest_branch = dest_branch,
+ **extra_proj_attrs)
for n in node.childNodes:
if n.nodeName == 'copyfile':
@@ -935,3 +948,26 @@
diff['added'].append(toProjects[proj])
return diff
+
+
+class GitcManifest(XmlManifest):
+
+ def __init__(self, repodir, gitc_client_name):
+ """Initialize the GitcManifest object."""
+ super(GitcManifest, self).__init__(repodir)
+ self.isGitcClient = True
+ self.gitc_client_name = gitc_client_name
+ self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+ gitc_client_name)
+ self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+ def _ParseProject(self, node, parent = None):
+ """Override _ParseProject and add support for GITC specific attributes."""
+ return super(GitcManifest, self)._ParseProject(
+ node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+ def _output_manifest_project_extras(self, p, e):
+ """Output GITC Specific Project attributes"""
+ if p.old_revision:
+ e.setAttribute('old-revision', str(p.old_revision))
+
diff --git a/project.py b/project.py
index a117f4d..e3c3bd5 100644
--- a/project.py
+++ b/project.py
@@ -13,7 +13,6 @@
# limitations under the License.
from __future__ import print_function
-import contextlib
import errno
import filecmp
import glob
@@ -31,7 +30,8 @@
from color import Coloring
from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
+ ID_RE
from error import GitError, HookError, UploadError, DownloadError
from error import ManifestInvalidRevisionError
from error import NoManifestException
@@ -45,6 +45,7 @@
input = raw_input
# pylint:enable=W0622
+
def _lwrite(path, content):
lock = '%s.lock' % path
@@ -60,17 +61,27 @@
os.remove(lock)
raise
+
def _error(fmt, *args):
msg = fmt % args
print('error: %s' % msg, file=sys.stderr)
+
+def _warn(fmt, *args):
+ msg = fmt % args
+ print('warn: %s' % msg, file=sys.stderr)
+
+
def not_rev(r):
return '^' + r
+
def sq(r):
return "'" + r.replace("'", "'\''") + "'"
_project_hook_list = None
+
+
def _ProjectHooks():
"""List the hooks present in the 'hooks' directory.
@@ -104,15 +115,14 @@
@property
def commits(self):
if self._commit_cache is None:
- self._commit_cache = self.project.bare_git.rev_list(
- '--abbrev=8',
- '--abbrev-commit',
- '--pretty=oneline',
- '--reverse',
- '--date-order',
- not_rev(self.base),
- self.commit,
- '--')
+ self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
+ '--abbrev-commit',
+ '--pretty=oneline',
+ '--reverse',
+ '--date-order',
+ not_rev(self.base),
+ self.commit,
+ '--')
return self._commit_cache
@@ -131,36 +141,36 @@
@property
def commits(self):
if self._commit_cache is None:
- self._commit_cache = self.project.bare_git.rev_list(
- '--abbrev=8',
- '--abbrev-commit',
- '--pretty=oneline',
- '--reverse',
- '--date-order',
- not_rev(self.base),
- R_HEADS + self.name,
- '--')
+ self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
+ '--abbrev-commit',
+ '--pretty=oneline',
+ '--reverse',
+ '--date-order',
+ not_rev(self.base),
+ R_HEADS + self.name,
+ '--')
return self._commit_cache
@property
def unabbrev_commits(self):
r = dict()
- for commit in self.project.bare_git.rev_list(
- not_rev(self.base),
- R_HEADS + self.name,
- '--'):
+ for commit in self.project.bare_git.rev_list(not_rev(self.base),
+ R_HEADS + self.name,
+ '--'):
r[commit[0:8]] = commit
return r
@property
def date(self):
- return self.project.bare_git.log(
- '--pretty=format:%cd',
- '-n', '1',
- R_HEADS + self.name,
- '--')
+ return self.project.bare_git.log('--pretty=format:%cd',
+ '-n', '1',
+ R_HEADS + self.name,
+ '--')
- def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None):
+ def UploadForReview(self, people,
+ auto_topic=False,
+ draft=False,
+ dest_branch=None):
self.project.UploadForReview(self.name,
people,
auto_topic=auto_topic,
@@ -170,8 +180,8 @@
def GetPublishedRefs(self):
refs = {}
output = self.project.bare_git.ls_remote(
- self.branch.remote.SshReviewUrl(self.project.UserEmail),
- 'refs/changes/*')
+ self.branch.remote.SshReviewUrl(self.project.UserEmail),
+ 'refs/changes/*')
for line in output.split('\n'):
try:
(sha, ref) = line.split()
@@ -181,7 +191,9 @@
return refs
+
class StatusColoring(Coloring):
+
def __init__(self, config):
Coloring.__init__(self, config, 'status')
self.project = self.printer('header', attr='bold')
@@ -195,17 +207,22 @@
class DiffColoring(Coloring):
+
def __init__(self, config):
Coloring.__init__(self, config, 'diff')
self.project = self.printer('header', attr='bold')
+
class _Annotation(object):
+
def __init__(self, name, value, keep):
self.name = name
self.value = value
self.keep = keep
+
class _CopyFile(object):
+
def __init__(self, src, dest, abssrc, absdest):
self.src = src
self.dest = dest
@@ -233,7 +250,9 @@
except IOError:
_error('Cannot copy file %s to %s', src, dest)
+
class _LinkFile(object):
+
def __init__(self, git_worktree, src, dest, relsrc, absdest):
self.git_worktree = git_worktree
self.src = src
@@ -246,7 +265,7 @@
if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
try:
# remove existing file first, since it might be read-only
- if os.path.exists(absDest):
+ if os.path.lexists(absDest):
os.remove(absDest)
else:
dest_dir = os.path.dirname(absDest)
@@ -272,7 +291,7 @@
absDestDir = self.abs_dest
if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
_error('Link error: src with wildcard, %s must be a directory',
- absDestDir)
+ absDestDir)
else:
absSrcFiles = glob.glob(absSrc)
for absSrcFile in absSrcFiles:
@@ -289,7 +308,9 @@
relSrc = os.path.join(relSrcDir, srcFile)
self.__linkIt(relSrc, absDest)
+
class RemoteSpec(object):
+
def __init__(self,
name,
url=None,
@@ -300,7 +321,9 @@
self.review = review
self.revision = revision
+
class RepoHook(object):
+
"""A RepoHook contains information about a script to run as a hook.
Hooks are used to run a python script before running an upload (for instance,
@@ -313,6 +336,7 @@
Hooks are always python. When a hook is run, we will load the hook into the
interpreter and execute its main() function.
"""
+
def __init__(self,
hook_type,
hooks_project,
@@ -427,8 +451,8 @@
' %s\n'
'\n'
'Do you want to allow this script to run '
- '(yes/yes-never-ask-again/NO)? ') % (
- self._GetMustVerb(), self._script_fullpath)
+ '(yes/yes-never-ask-again/NO)? ') % (self._GetMustVerb(),
+ self._script_fullpath)
response = input(prompt).lower()
print()
@@ -472,19 +496,18 @@
# Exec, storing global context in the context dict. We catch exceptions
# and convert to a HookError w/ just the failing traceback.
- context = {}
+ context = {'__file__': self._script_fullpath}
try:
exec(compile(open(self._script_fullpath).read(),
self._script_fullpath, 'exec'), context)
except Exception:
- raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
- traceback.format_exc(), self._hook_type))
+ raise HookError('%s\nFailed to import %s hook; see traceback above.' %
+ (traceback.format_exc(), self._hook_type))
# Running the script should have defined a main() function.
if 'main' not in context:
raise HookError('Missing main() in: "%s"' % self._script_fullpath)
-
# Add 'hook_should_take_kwargs' to the arguments to be passed to main.
# We don't actually want hooks to define their main with this argument--
# it's there to remind them that their hook should always take **kwargs.
@@ -502,8 +525,8 @@
context['main'](**kwargs)
except Exception:
raise HookError('%s\nFailed to run main() for %s hook; see traceback '
- 'above.' % (
- traceback.format_exc(), self._hook_type))
+ 'above.' % (traceback.format_exc(),
+ self._hook_type))
finally:
# Restore sys.path and CWD.
sys.path = orig_syspath
@@ -527,8 +550,8 @@
to run a required hook (from _CheckForHookApproval).
"""
# No-op if there is no hooks project or if hook is disabled.
- if ((not self._hooks_project) or
- (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
+ if ((not self._hooks_project) or (self._hook_type not in
+ self._hooks_project.enabled_repo_hooks)):
return
# Bail with a nice error if we can't find the hook.
@@ -550,6 +573,7 @@
# These objects can only be used by a single working tree.
working_tree_files = ['config', 'packed-refs', 'shallow']
working_tree_dirs = ['logs', 'refs']
+
def __init__(self,
manifest,
name,
@@ -569,7 +593,8 @@
parent=None,
is_derived=False,
dest_branch=None,
- optimized_fetch=False):
+ optimized_fetch=False,
+ old_revision=None):
"""Init a Project object.
Args:
@@ -593,6 +618,7 @@
dest_branch: The branch to which to push changes for review by default.
optimized_fetch: If True, when a project is set to a sha1 revision, only
fetch from the remote if the sha1 is not present locally.
+ old_revision: saved git commit id for open GITC projects.
"""
self.manifest = manifest
self.name = name
@@ -606,9 +632,9 @@
self.relpath = relpath
self.revisionExpr = revisionExpr
- if revisionId is None \
- and revisionExpr \
- and IsId(revisionExpr):
+ if revisionId is None \
+ and revisionExpr \
+ and IsId(revisionExpr):
self.revisionId = revisionExpr
else:
self.revisionId = revisionId
@@ -628,9 +654,8 @@
self.copyfiles = []
self.linkfiles = []
self.annotations = []
- self.config = GitConfig.ForRepository(
- gitdir=self.gitdir,
- defaults=self.manifest.globalConfig)
+ self.config = GitConfig.ForRepository(gitdir=self.gitdir,
+ defaults=self.manifest.globalConfig)
if self.worktree:
self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
@@ -640,6 +665,7 @@
self.bare_ref = GitRefs(gitdir)
self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
self.dest_branch = dest_branch
+ self.old_revision = old_revision
# This will be filled in if a project is later identified to be the
# project containing repo hooks.
@@ -767,7 +793,7 @@
"""
expanded_manifest_groups = manifest_groups or ['default']
expanded_project_groups = ['all'] + (self.groups or [])
- if not 'notdefault' in expanded_project_groups:
+ if 'notdefault' not in expanded_project_groups:
expanded_project_groups += ['default']
matched = False
@@ -779,7 +805,7 @@
return matched
-## Status Display ##
+# Status Display ##
def UncommitedFiles(self, get_all=True):
"""Returns a list of strings, uncommitted files in the git tree.
@@ -831,7 +857,7 @@
output: If specified, redirect the output to this object.
"""
if not os.path.isdir(self.worktree):
- if output_redir == None:
+ if output_redir is None:
output_redir = sys.stdout
print(file=output_redir)
print('project %s/' % self.relpath, file=output_redir)
@@ -850,7 +876,7 @@
return 'CLEAN'
out = StatusColoring(self.config)
- if not output_redir == None:
+ if output_redir is not None:
out.redirect(output_redir)
out.project('project %-40s', self.relpath + '/ ')
@@ -893,7 +919,7 @@
if i and i.src_path:
line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
- i.src_path, p, i.level)
+ i.src_path, p, i.level)
else:
line = ' %s%s\t%s' % (i_status, f_status, p)
@@ -936,7 +962,7 @@
p.Wait()
-## Publish / Upload ##
+# Publish / Upload ##
def WasPublished(self, branch, all_refs=None):
"""Was the branch published (uploaded) for code review?
@@ -1079,7 +1105,7 @@
message=msg)
-## Sync ##
+# Sync ##
def _ExtractArchive(self, tarpath, path=None):
"""Extract the given tar on its current location
@@ -1093,26 +1119,25 @@
tar.extractall(path=path)
return True
except (IOError, tarfile.TarError) as e:
- print("error: Cannot extract archive %s: "
- "%s" % (tarpath, str(e)), file=sys.stderr)
+ _error("Cannot extract archive %s: %s", tarpath, str(e))
return False
def Sync_NetworkHalf(self,
- quiet=False,
- is_new=None,
- current_branch_only=False,
- force_sync=False,
- clone_bundle=True,
- no_tags=False,
- archive=False,
- optimized_fetch=False):
+ quiet=False,
+ is_new=None,
+ current_branch_only=False,
+ force_sync=False,
+ clone_bundle=True,
+ no_tags=False,
+ archive=False,
+ optimized_fetch=False,
+ prune=False):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
"""
if archive and not isinstance(self, MetaProject):
if self.remote.url.startswith(('http://', 'https://')):
- print("error: %s: Cannot fetch archives from http/https "
- "remotes." % self.name, file=sys.stderr)
+ _error("%s: Cannot fetch archives from http/https remotes.", self.name)
return False
name = self.relpath.replace('\\', '/')
@@ -1123,7 +1148,7 @@
try:
self._FetchArchive(tarpath, cwd=topdir)
except GitError as e:
- print('error: %s' % str(e), file=sys.stderr)
+ _error('%s', e)
return False
# From now on, we only need absolute tarpath
@@ -1134,8 +1159,7 @@
try:
os.remove(tarpath)
except OSError as e:
- print("warn: Cannot remove archive %s: "
- "%s" % (tarpath, str(e)), file=sys.stderr)
+ _warn("Cannot remove archive %s: %s", tarpath, str(e))
self._CopyAndLinkFiles()
return True
if is_new is None:
@@ -1160,8 +1184,8 @@
alt_dir = None
if clone_bundle \
- and alt_dir is None \
- and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+ and alt_dir is None \
+ and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
is_new = False
if not current_branch_only:
@@ -1173,12 +1197,13 @@
elif self.manifest.default.sync_c:
current_branch_only = True
- need_to_fetch = not (optimized_fetch and \
- (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
- if (need_to_fetch
- and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
- current_branch_only=current_branch_only,
- no_tags=no_tags)):
+ need_to_fetch = not (optimized_fetch and
+ (ID_RE.match(self.revisionExpr) and
+ self._CheckForSha1()))
+ if (need_to_fetch and
+ not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+ current_branch_only=current_branch_only,
+ no_tags=no_tags, prune=prune)):
return False
if self.worktree:
@@ -1195,6 +1220,8 @@
self._InitHooks()
def _CopyAndLinkFiles(self):
+ if self.manifest.isGitcClient:
+ return
for copyfile in self.copyfiles:
copyfile._Copy()
for linkfile in self.linkfiles:
@@ -1213,9 +1240,8 @@
try:
return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
except GitError:
- raise ManifestInvalidRevisionError(
- 'revision %s in %s not found' % (self.revisionExpr,
- self.name))
+ raise ManifestInvalidRevisionError('revision %s in %s not found' %
+ (self.revisionExpr, self.name))
def GetRevisionId(self, all_refs=None):
if self.revisionId:
@@ -1230,9 +1256,8 @@
try:
return self.bare_git.rev_parse('--verify', '%s^0' % rev)
except GitError:
- raise ManifestInvalidRevisionError(
- 'revision %s in %s not found' % (self.revisionExpr,
- self.name))
+ raise ManifestInvalidRevisionError('revision %s in %s not found' %
+ (self.revisionExpr, self.name))
def Sync_LocalHalf(self, syncbuf, force_sync=False):
"""Perform only the local IO portion of the sync process.
@@ -1270,6 +1295,8 @@
# Except if the head needs to be detached
#
if not syncbuf.detach_head:
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
return
else:
lost = self._revlist(not_rev(revid), HEAD)
@@ -1287,6 +1314,8 @@
if head == revid:
# No changes; don't do anything further.
#
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
return
branch = self.GetBranch(branch)
@@ -1317,8 +1346,8 @@
# to rewrite the published commits so we punt.
#
syncbuf.fail(self,
- "branch %s is published (but not merged) and is now %d commits behind"
- % (branch.name, len(upstream_gain)))
+ "branch %s is published (but not merged) and is now "
+ "%d commits behind" % (branch.name, len(upstream_gain)))
return
elif pub == head:
# All published commits are merged, and thus we are a
@@ -1412,7 +1441,7 @@
remote = self.GetRemote(self.remote.name)
cmd = ['fetch', remote.name]
- cmd.append('refs/changes/%2.2d/%d/%d' \
+ cmd.append('refs/changes/%2.2d/%d/%d'
% (change_id % 100, change_id, patch_id))
if GitCommand(self, cmd, bare=True).Wait() != 0:
return None
@@ -1423,11 +1452,13 @@
self.bare_git.rev_parse('FETCH_HEAD'))
-## Branch Management ##
+# Branch Management ##
- def StartBranch(self, name):
+ def StartBranch(self, name, branch_merge=''):
"""Create a new branch off the manifest's revision.
"""
+ if not branch_merge:
+ branch_merge = self.revisionExpr
head = self.work_git.GetHead()
if head == (R_HEADS + name):
return True
@@ -1441,9 +1472,9 @@
branch = self.GetBranch(name)
branch.remote = self.GetRemote(self.remote.name)
- branch.merge = self.revisionExpr
- if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
- branch.merge = R_HEADS + self.revisionExpr
+ branch.merge = branch_merge
+ if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
+ branch.merge = R_HEADS + branch_merge
revid = self.GetRevisionId(all_refs)
if head.startswith(R_HEADS):
@@ -1451,7 +1482,6 @@
head = all_refs[head]
except KeyError:
head = None
-
if revid and head and revid == head:
ref = os.path.join(self.gitdir, R_HEADS + name)
try:
@@ -1572,8 +1602,6 @@
if kill:
old = self.bare_git.GetHead()
- if old is None:
- old = 'refs/heads/please_never_use_this_as_a_branch_name'
try:
self.bare_git.DetachHead(rev)
@@ -1585,7 +1613,10 @@
capture_stderr=True)
b.Wait()
finally:
- self.bare_git.SetHead(old)
+ if ID_RE.match(old):
+ self.bare_git.DetachHead(old)
+ else:
+ self.bare_git.SetHead(old)
left = self._allrefs
for branch in kill:
@@ -1608,10 +1639,11 @@
return kept
-## Submodule Management ##
+# Submodule Management ##
def GetRegisteredSubprojects(self):
result = []
+
def rec(subprojects):
if not subprojects:
return
@@ -1646,6 +1678,7 @@
re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
+
def parse_gitmodules(gitdir, rev):
cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
try:
@@ -1755,7 +1788,7 @@
return result
-## Direct Git Commands ##
+# Direct Git Commands ##
def _CheckForSha1(self):
try:
# if revision (sha or tag) is not present then following function
@@ -1779,13 +1812,13 @@
if command.Wait() != 0:
raise GitError('git archive %s: %s' % (self.name, command.stderr))
-
def _RemoteFetch(self, name=None,
current_branch_only=False,
initial=False,
quiet=False,
alt_dir=None,
- no_tags=False):
+ no_tags=False,
+ prune=False):
is_sha1 = False
tag_name = None
@@ -1898,6 +1931,9 @@
else:
cmd.append('--tags')
+ if prune:
+ cmd.append('--prune')
+
spec = []
if not current_branch_only:
# Fetch whole repo
@@ -1939,9 +1975,9 @@
break
continue
elif current_branch_only and is_sha1 and ret == 128:
- # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
- # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
- # abort the optimization attempt and do a full sync.
+ # Exit code 128 means "couldn't find the ref you asked for"; if we're
+ # in sha1 mode, we just tried sync'ing from the upstream field; it
+ # doesn't exist, thus abort the optimization attempt and do a full sync.
break
elif ret < 0:
# Git died with a signal, exit immediately
@@ -1968,20 +2004,24 @@
initial=False, quiet=quiet, alt_dir=alt_dir)
if self.clone_depth:
self.clone_depth = None
- return self._RemoteFetch(name=name, current_branch_only=current_branch_only,
+ return self._RemoteFetch(name=name,
+ current_branch_only=current_branch_only,
initial=False, quiet=quiet, alt_dir=alt_dir)
return ok
def _ApplyCloneBundle(self, initial=False, quiet=False):
- if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth):
+ if initial and \
+ (self.manifest.manifestProject.config.GetString('repo.depth') or
+ self.clone_depth):
return False
remote = self.GetRemote(self.remote.name)
bundle_url = remote.url + '/clone.bundle'
bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
- if GetSchemeFromUrl(bundle_url) not in (
- 'http', 'https', 'persistent-http', 'persistent-https'):
+ if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
+ 'persistent-http',
+ 'persistent-https'):
return False
bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
@@ -2030,7 +2070,7 @@
os.remove(tmpPath)
if 'http_proxy' in os.environ and 'darwin' == sys.platform:
cmd += ['--proxy', os.environ['http_proxy']]
- with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
+ with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
if cookiefile:
cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
if srcUrl.startswith('persistent-'):
@@ -2078,40 +2118,6 @@
except OSError:
return False
- @contextlib.contextmanager
- def _GetBundleCookieFile(self, url, quiet):
- if url.startswith('persistent-'):
- try:
- p = subprocess.Popen(
- ['git-remote-persistent-https', '-print_config', url],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- try:
- prefix = 'http.cookiefile='
- cookiefile = None
- for line in p.stdout:
- line = line.strip()
- if line.startswith(prefix):
- cookiefile = line[len(prefix):]
- break
- # Leave subprocess open, as cookie file may be transient.
- if cookiefile:
- yield cookiefile
- return
- finally:
- p.stdin.close()
- if p.wait():
- err_msg = p.stderr.read()
- if ' -print_config' in err_msg:
- pass # Persistent proxy doesn't support -print_config.
- elif not quiet:
- print(err_msg, file=sys.stderr)
- except OSError as e:
- if e.errno == errno.ENOENT:
- pass # No persistent proxy.
- raise
- yield GitConfig.ForUser().GetString('http.cookiefile')
-
def _Checkout(self, rev, quiet=False):
cmd = ['checkout']
if quiet:
@@ -2182,12 +2188,13 @@
try:
self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
except GitError as e:
- print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
if force_sync:
+ print("Retrying clone after deleting %s" %
+ self.gitdir, file=sys.stderr)
try:
shutil.rmtree(os.path.realpath(self.gitdir))
- if self.worktree and os.path.exists(
- os.path.realpath(self.worktree)):
+ if self.worktree and os.path.exists(os.path.realpath
+ (self.worktree)):
shutil.rmtree(os.path.realpath(self.worktree))
return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
except:
@@ -2246,7 +2253,7 @@
name = os.path.basename(stock_hook)
if name in ('commit-msg',) and not self.remote.review \
- and not self is self.manifest.manifestProject:
+ and self is not self.manifest.manifestProject:
# Don't install a Gerrit Code Review hook if this
# project does not appear to use it for reviews.
#
@@ -2261,7 +2268,8 @@
if filecmp.cmp(stock_hook, dst, shallow=False):
os.remove(dst)
else:
- _error("%s: Not replacing %s hook", self.relpath, name)
+ _warn("%s: Not replacing locally modified %s hook",
+ self.relpath, name)
continue
try:
os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
@@ -2320,7 +2328,10 @@
# Fail if the links are pointing to the wrong place
if src != dst:
raise GitError('--force-sync not enabled; cannot overwrite a local '
- 'work tree')
+ 'work tree. If you\'re comfortable with the '
+ 'possibility of losing the work tree\'s git metadata,'
+ ' use `repo sync --force-sync {0}` to '
+ 'proceed.'.format(self.relpath))
def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
"""Update |dotgit| to reference |gitdir|, using symlinks where possible.
@@ -2464,6 +2475,7 @@
return logs
class _GitGetByExec(object):
+
def __init__(self, project, bare, gitdir):
self._project = project
self._bare = bare
@@ -2482,8 +2494,8 @@
if p.Wait() == 0:
out = p.stdout
if out:
+ # Backslash is not anomalous
return out[:-1].split('\0') # pylint: disable=W1401
- # Backslash is not anomalous
return []
def DiffZ(self, name, *args):
@@ -2509,6 +2521,7 @@
break
class _Info(object):
+
def __init__(self, path, omode, nmode, oid, nid, state):
self.path = path
self.src_path = None
@@ -2611,10 +2624,8 @@
line = line[:-1]
r.append(line)
if p.Wait() != 0:
- raise GitError('%s rev-list %s: %s' % (
- self._project.name,
- str(args),
- p.stderr))
+ raise GitError('%s rev-list %s: %s' %
+ (self._project.name, str(args), p.stderr))
return r
def __getattr__(self, name):
@@ -2637,6 +2648,7 @@
A callable object that will try to call git with the named command.
"""
name = name.replace('_', '-')
+
def runner(*args, **kwargs):
cmdv = []
config = kwargs.pop('config', None)
@@ -2659,10 +2671,8 @@
capture_stdout=True,
capture_stderr=True)
if p.Wait() != 0:
- raise GitError('%s %s: %s' % (
- self._project.name,
- name,
- p.stderr))
+ raise GitError('%s %s: %s' %
+ (self._project.name, name, p.stderr))
r = p.stdout
try:
r = r.decode('utf-8')
@@ -2675,14 +2685,19 @@
class _PriorSyncFailedError(Exception):
+
def __str__(self):
return 'prior sync failed; rebase still in progress'
+
class _DirtyError(Exception):
+
def __str__(self):
return 'contains uncommitted changes'
+
class _InfoMessage(object):
+
def __init__(self, project, text):
self.project = project
self.text = text
@@ -2691,7 +2706,9 @@
syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
syncbuf.out.nl()
+
class _Failure(object):
+
def __init__(self, project, why):
self.project = project
self.why = why
@@ -2702,7 +2719,9 @@
str(self.why))
syncbuf.out.nl()
+
class _Later(object):
+
def __init__(self, project, action):
self.project = project
self.action = action
@@ -2719,14 +2738,18 @@
out.nl()
return False
+
class _SyncColoring(Coloring):
+
def __init__(self, config):
Coloring.__init__(self, config, 'reposync')
self.project = self.printer('header', attr='bold')
self.info = self.printer('info')
self.fail = self.printer('fail', fg='red')
+
class SyncBuffer(object):
+
def __init__(self, config, detach_head=False):
self._messages = []
self._failures = []
@@ -2782,8 +2805,10 @@
class MetaProject(Project):
+
"""A special project housed under .repo.
"""
+
def __init__(self, manifest, name, gitdir, worktree):
Project.__init__(self,
manifest=manifest,
@@ -2817,10 +2842,9 @@
syncbuf.Finish()
return GitCommand(self,
- ['update-ref', '-d', 'refs/heads/default'],
- capture_stdout=True,
- capture_stderr=True).Wait() == 0
-
+ ['update-ref', '-d', 'refs/heads/default'],
+ capture_stdout=True,
+ capture_stderr=True).Wait() == 0
@property
def LastFetch(self):
diff --git a/repo b/repo
index 7769565..e5cb890 100755
--- a/repo
+++ b/repo
@@ -1,8 +1,11 @@
#!/usr/bin/env python
-## repo default configuration
-##
-REPO_URL = 'https://gerrit.googlesource.com/git-repo'
+# repo default configuration
+#
+import os
+REPO_URL = os.environ.get('REPO_URL', None)
+if not REPO_URL:
+ REPO_URL = 'https://gerrit.googlesource.com/git-repo'
REPO_REV = 'stable'
# Copyright (C) 2008 Google Inc.
@@ -20,7 +23,7 @@
# limitations under the License.
# increment this whenever we make important changes to this script
-VERSION = (1, 21)
+VERSION = (1, 22)
# increment this if the MAINTAINER_KEYS block is modified
KEYRING_VERSION = (1, 2)
@@ -101,18 +104,19 @@
-----END PGP PUBLIC KEY BLOCK-----
"""
-GIT = 'git' # our git command
-MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version
-repodir = '.repo' # name of repo's private directory
-S_repo = 'repo' # special repo repository
-S_manifests = 'manifests' # special manifest repository
-REPO_MAIN = S_repo + '/main.py' # main script
-MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
+GIT = 'git' # our git command
+MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version
+repodir = '.repo' # name of repo's private directory
+S_repo = 'repo' # special repo repository
+S_manifests = 'manifests' # special manifest repository
+REPO_MAIN = S_repo + '/main.py' # main script
+MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
+GITC_CONFIG_FILE = '/gitc/.config'
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
import errno
import optparse
-import os
import re
import shutil
import stat
@@ -212,14 +216,69 @@
dest='config_name', action="store_true", default=False,
help='Always prompt for name/e-mail')
+
+def _GitcInitOptions(init_optparse_arg):
+ init_optparse_arg.set_usage("repo gitc-init -u url -c client [options]")
+ g = init_optparse_arg.add_option_group('GITC options')
+ g.add_option('-f', '--manifest-file',
+ dest='manifest_file',
+ help='Optional manifest file to use for this GITC client.')
+ g.add_option('-c', '--gitc-client',
+ dest='gitc_client',
+ help='The name of the gitc_client instance to create or modify.')
+
+_gitc_manifest_dir = None
+
+
+def get_gitc_manifest_dir():
+ global _gitc_manifest_dir
+ if _gitc_manifest_dir is None:
+ _gitc_manifest_dir = ''
+ try:
+ with open(GITC_CONFIG_FILE, 'r') as gitc_config:
+ for line in gitc_config:
+ match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
+ if match:
+ _gitc_manifest_dir = match.group('gitc_manifest_dir')
+ except IOError:
+ pass
+ return _gitc_manifest_dir
+
+
+def gitc_parse_clientdir(gitc_fs_path):
+ """Parse a path in the GITC FS and return its client name.
+
+ @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+ @returns: The GITC client name
+ """
+ if gitc_fs_path == GITC_FS_ROOT_DIR:
+ return None
+ if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
+ manifest_dir = get_gitc_manifest_dir()
+ if manifest_dir == '':
+ return None
+ if manifest_dir[-1] != '/':
+ manifest_dir += '/'
+ if gitc_fs_path == manifest_dir:
+ return None
+ if not gitc_fs_path.startswith(manifest_dir):
+ return None
+ return gitc_fs_path.split(manifest_dir)[1].split('/')[0]
+ return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
+
class CloneFailure(Exception):
+
"""Indicate the remote clone of repo itself failed.
"""
-def _Init(args):
+def _Init(args, gitc_init=False):
"""Installs repo by cloning it over the network.
"""
+ if gitc_init:
+ _GitcInitOptions(init_optparse)
opt, args = init_optparse.parse_args(args)
if args:
init_optparse.print_usage()
@@ -242,6 +301,26 @@
raise CloneFailure()
try:
+ if gitc_init:
+ gitc_manifest_dir = get_gitc_manifest_dir()
+ if not gitc_manifest_dir:
+ _print('fatal: GITC filesystem is not available. Exiting...',
+ file=sys.stderr)
+ sys.exit(1)
+ gitc_client = opt.gitc_client
+ if not gitc_client:
+ gitc_client = gitc_parse_clientdir(os.getcwd())
+ if not gitc_client:
+ _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+ sys.exit(1)
+ client_dir = os.path.join(gitc_manifest_dir, gitc_client)
+ if not os.path.exists(client_dir):
+ os.makedirs(client_dir)
+ os.chdir(client_dir)
+ if os.path.exists(repodir):
+ # This GITC Client has already initialized repo so continue.
+ return
+
os.mkdir(repodir)
except OSError as e:
if e.errno != errno.EEXIST:
@@ -358,8 +437,8 @@
cmd = ['gpg', '--import']
try:
proc = subprocess.Popen(cmd,
- env = env,
- stdin = subprocess.PIPE)
+ env=env,
+ stdin=subprocess.PIPE)
except OSError as e:
if not quiet:
_print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
@@ -385,7 +464,7 @@
"""Set a git configuration option to the specified value.
"""
cmd = [GIT, 'config', name, value]
- if subprocess.Popen(cmd, cwd = local).wait() != 0:
+ if subprocess.Popen(cmd, cwd=local).wait() != 0:
raise CloneFailure()
@@ -398,9 +477,9 @@
n = netrc.netrc()
for host in n.hosts:
p = n.hosts[host]
- mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
+ mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
- except:
+ except: # pylint: disable=bare-except
pass
handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
@@ -413,6 +492,7 @@
handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
urllib.request.install_opener(urllib.request.build_opener(*handlers))
+
def _Fetch(url, local, src, quiet):
if not quiet:
_print('Get %s' % url, file=sys.stderr)
@@ -427,22 +507,23 @@
cmd.append('+refs/heads/*:refs/remotes/origin/*')
cmd.append('refs/tags/*:refs/tags/*')
- proc = subprocess.Popen(cmd, cwd = local, stderr = err)
+ proc = subprocess.Popen(cmd, cwd=local, stderr=err)
if err:
proc.stderr.read()
proc.stderr.close()
if proc.wait() != 0:
raise CloneFailure()
+
def _DownloadBundle(url, local, quiet):
if not url.endswith('/'):
url += '/'
url += 'clone.bundle'
proc = subprocess.Popen(
- [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
- cwd = local,
- stdout = subprocess.PIPE)
+ [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+ cwd=local,
+ stdout=subprocess.PIPE)
for line in proc.stdout:
m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
if m:
@@ -484,6 +565,7 @@
finally:
dest.close()
+
def _ImportBundle(local):
path = os.path.join(local, '.git', 'clone.bundle')
try:
@@ -491,6 +573,7 @@
finally:
os.remove(path)
+
def _Clone(url, local, quiet):
"""Clones a git repository to a new subdirectory of repodir
"""
@@ -503,14 +586,14 @@
cmd = [GIT, 'init', '--quiet']
try:
- proc = subprocess.Popen(cmd, cwd = local)
+ proc = subprocess.Popen(cmd, cwd=local)
except OSError as e:
_print(file=sys.stderr)
_print("fatal: '%s' is not available" % GIT, file=sys.stderr)
_print('fatal: %s' % e, file=sys.stderr)
_print(file=sys.stderr)
_print('Please make sure %s is installed and in your path.' % GIT,
- file=sys.stderr)
+ file=sys.stderr)
raise CloneFailure()
if proc.wait() != 0:
_print('fatal: could not create %s' % local, file=sys.stderr)
@@ -518,12 +601,12 @@
_InitHttp()
_SetConfig(local, 'remote.origin.url', url)
- _SetConfig(local, 'remote.origin.fetch',
- '+refs/heads/*:refs/remotes/origin/*')
+ _SetConfig(local,
+ 'remote.origin.fetch',
+ '+refs/heads/*:refs/remotes/origin/*')
if _DownloadBundle(url, local, quiet):
_ImportBundle(local)
- else:
- _Fetch(url, local, 'origin', quiet)
+ _Fetch(url, local, 'origin', quiet)
def _Verify(cwd, branch, quiet):
@@ -533,7 +616,7 @@
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- cwd = cwd)
+ cwd=cwd)
cur = proc.stdout.read().strip()
proc.stdout.close()
@@ -551,7 +634,7 @@
if not quiet:
_print(file=sys.stderr)
_print("info: Ignoring branch '%s'; using tagged release '%s'"
- % (branch, cur), file=sys.stderr)
+ % (branch, cur), file=sys.stderr)
_print(file=sys.stderr)
env = os.environ.copy()
@@ -559,10 +642,10 @@
cmd = [GIT, 'tag', '-v', cur]
proc = subprocess.Popen(cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE,
- cwd = cwd,
- env = env)
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=cwd,
+ env=env)
out = proc.stdout.read()
proc.stdout.close()
@@ -582,21 +665,21 @@
"""Checkout an upstream branch into the repository and track it.
"""
cmd = [GIT, 'update-ref', 'refs/heads/default', rev]
- if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+ if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
raise CloneFailure()
_SetConfig(cwd, 'branch.default.remote', 'origin')
_SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch)
cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default']
- if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+ if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
raise CloneFailure()
cmd = [GIT, 'read-tree', '--reset', '-u']
if not quiet:
cmd.append('-v')
cmd.append('HEAD')
- if subprocess.Popen(cmd, cwd = cwd).wait() != 0:
+ if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
raise CloneFailure()
@@ -608,8 +691,8 @@
olddir = None
while curdir != '/' \
- and curdir != olddir \
- and not repo:
+ and curdir != olddir \
+ and not repo:
repo = os.path.join(curdir, repodir, REPO_MAIN)
if not os.path.isfile(repo):
repo = None
@@ -618,7 +701,7 @@
return (repo, os.path.join(curdir, repodir))
-class _Options:
+class _Options(object):
help = False
@@ -640,15 +723,20 @@
def _Usage():
+ gitc_usage = ""
+ if get_gitc_manifest_dir():
+ gitc_usage = " gitc-init Initialize a GITC Client.\n"
+
_print(
-"""usage: repo COMMAND [ARGS]
+ """usage: repo COMMAND [ARGS]
repo is not yet installed. Use "repo init" to install it here.
The most commonly used repo commands are:
init Install repo in the current working directory
- help Display detailed help on a command
+""" + gitc_usage +
+ """ help Display detailed help on a command
For access to the full online help, install repo ("repo init").
""", file=sys.stderr)
@@ -660,6 +748,10 @@
if args[0] == 'init':
init_optparse.print_help()
sys.exit(0)
+ elif args[0] == 'gitc-init':
+ _GitcInitOptions(init_optparse)
+ init_optparse.print_help()
+ sys.exit(0)
else:
_print("error: '%s' is not a bootstrap command.\n"
' For access to online help, install repo ("repo init").'
@@ -705,8 +797,8 @@
'--git-dir=%s' % gitdir,
'symbolic-ref',
'HEAD'],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
REPO_REV = proc.stdout.read().strip()
proc.stdout.close()
@@ -719,12 +811,23 @@
def main(orig_args):
- repo_main, rel_repo_dir = _FindRepo()
cmd, opt, args = _ParseArguments(orig_args)
+ repo_main, rel_repo_dir = None, None
+ # Don't use the local repo copy, make sure to switch to the gitc client first.
+ if cmd != 'gitc-init':
+ repo_main, rel_repo_dir = _FindRepo()
+
wrapper_path = os.path.abspath(__file__)
my_main, my_git = _RunSelf(wrapper_path)
+ cwd = os.getcwd()
+ if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+ _print('error: repo cannot be used in the GITC local manifest directory.'
+ '\nIf you want to work on this GITC client please rerun this '
+ 'command from the corresponding client under /gitc/',
+ file=sys.stderr)
+ sys.exit(1)
if not repo_main:
if opt.help:
_Usage()
@@ -732,11 +835,11 @@
_Help(args)
if not cmd:
_NotInstalled()
- if cmd == 'init':
+ if cmd == 'init' or cmd == 'gitc-init':
if my_git:
_SetDefaultsTo(my_git)
try:
- _Init(args)
+ _Init(args, gitc_init=(cmd == 'gitc-init'))
except CloneFailure:
shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 96dc99d..07ee8d5 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -120,6 +120,9 @@
p.add_option('-r', '--regex',
dest='regex', action='store_true',
help="Execute the command only on projects matching regex or wildcard expression")
+ p.add_option('-i', '--inverse-regex',
+ dest='inverse_regex', action='store_true',
+ help="Execute the command only on projects not matching regex or wildcard expression")
p.add_option('-g', '--groups',
dest='groups',
help="Execute the command only on projects matching the specified groups")
@@ -215,10 +218,12 @@
if os.path.isfile(smart_sync_manifest_path):
self.manifest.Override(smart_sync_manifest_path)
- if not opt.regex:
- projects = self.GetProjects(args, groups=opt.groups)
- else:
+ if opt.regex:
projects = self.FindProjects(args)
+ elif opt.inverse_regex:
+ projects = self.FindProjects(args, inverse=True)
+ else:
+ projects = self.GetProjects(args, groups=opt.groups)
os.environ['REPO_COUNT'] = str(len(projects))
@@ -240,7 +245,8 @@
rc = rc or errno.EINTR
except Exception as e:
# Catch any other exceptions raised
- print('Got an error, terminating the pool: %r' % e,
+ print('Got an error, terminating the pool: %s: %s' %
+ (type(e).__name__, e),
file=sys.stderr)
pool.terminate()
rc = rc or getattr(e, 'errno', 1)
@@ -254,7 +260,8 @@
try:
project = self._SerializeProject(p)
except Exception as e:
- print('Project list error: %r' % e,
+ print('Project list error on project %s: %s: %s' %
+ (p.name, type(e).__name__, e),
file=sys.stderr)
return
except KeyboardInterrupt:
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
new file mode 100644
index 0000000..7380c35
--- /dev/null
+++ b/subcmds/gitc_delete.py
@@ -0,0 +1,55 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+import sys
+
+from command import Command, GitcClientCommand
+import gitc_utils
+
+from pyversion import is_python3
+if not is_python3():
+ # pylint:disable=W0622
+ input = raw_input
+ # pylint:enable=W0622
+
+class GitcDelete(Command, GitcClientCommand):
+ common = True
+ visible_everywhere = False
+ helpSummary = "Delete a GITC Client."
+ helpUsage = """
+%prog
+"""
+ helpDescription = """
+This subcommand deletes the current GITC client, deleting the GITC manifest
+and all locally downloaded sources.
+"""
+
+ def _Options(self, p):
+ p.add_option('-f', '--force',
+ dest='force', action='store_true',
+ help='Force the deletion (no prompt).')
+
+ def Execute(self, opt, args):
+ if not opt.force:
+ prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' %
+ self.gitc_manifest.gitc_client_name)
+ response = input(prompt).lower()
+ if not response == 'yes':
+ print('Response was not "yes"\n Exiting...')
+ sys.exit(1)
+ shutil.rmtree(self.gitc_manifest.gitc_client_dir)
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
new file mode 100644
index 0000000..2726eae
--- /dev/null
+++ b/subcmds/gitc_init.py
@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+
+import gitc_utils
+from command import GitcAvailableCommand
+from manifest_xml import GitcManifest
+from subcmds import init
+import wrapper
+
+
+class GitcInit(init.Init, GitcAvailableCommand):
+ common = True
+ helpSummary = "Initialize a GITC Client."
+ helpUsage = """
+%prog [options] [client name]
+"""
+ helpDescription = """
+The '%prog' command is ran to initialize a new GITC client for use
+with the GITC file system.
+
+This command will setup the client directory, initialize repo, just
+like repo init does, and then downloads the manifest collection
+and installs it in the .repo/directory of the GITC client.
+
+Once this is done, a GITC manifest is generated by pulling the HEAD
+SHA for each project and generates the properly formatted XML file
+and installs it as .manifest in the GITC client directory.
+
+The -c argument is required to specify the GITC client name.
+
+The optional -f argument can be used to specify the manifest file to
+use for this GITC client.
+"""
+
+ def _Options(self, p):
+ super(GitcInit, self)._Options(p)
+ g = p.add_option_group('GITC options')
+ g.add_option('-f', '--manifest-file',
+ dest='manifest_file',
+ help='Optional manifest file to use for this GITC client.')
+ g.add_option('-c', '--gitc-client',
+ dest='gitc_client',
+ help='The name of the gitc_client instance to create or modify.')
+
+ def Execute(self, opt, args):
+ gitc_client = gitc_utils.parse_clientdir(os.getcwd())
+ if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client):
+ print('fatal: Please update your repo command. See go/gitc for instructions.', file=sys.stderr)
+ sys.exit(1)
+ self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+ gitc_client)
+ super(GitcInit, self).Execute(opt, args)
+
+ manifest_file = self.manifest.manifestFile
+ if opt.manifest_file:
+ if not os.path.exists(opt.manifest_file):
+ print('fatal: Specified manifest file %s does not exist.' %
+ opt.manifest_file)
+ sys.exit(1)
+ manifest_file = opt.manifest_file
+
+ manifest = GitcManifest(self.repodir, gitc_client)
+ manifest.Override(manifest_file)
+ gitc_utils.generate_gitc_manifest(None, manifest)
+ print('Please run `cd %s` to view your GITC client.' %
+ os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client))
diff --git a/subcmds/help.py b/subcmds/help.py
index 4aa3f86..9bb4c8c 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -19,7 +19,8 @@
from formatter import AbstractFormatter, DumbWriter
from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
+import gitc_utils
class Help(PagedCommand, MirrorSafeCommand):
common = False
@@ -54,9 +55,21 @@
def _PrintCommonCommands(self):
print('usage: repo COMMAND [ARGS]')
print('The most commonly used repo commands are:')
+
+ def gitc_supported(cmd):
+ if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand):
+ return True
+ if self.manifest.isGitcClient:
+ return True
+ if isinstance(cmd, GitcClientCommand):
+ return False
+ if gitc_utils.get_gitc_manifest_dir():
+ return True
+ return False
+
commandNames = list(sorted([name
for name, command in self.commands.items()
- if command.common]))
+ if command.common and gitc_supported(command)]))
maxlen = 0
for name in commandNames:
diff --git a/subcmds/init.py b/subcmds/init.py
index dbb6ddd..b8e3de5 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -179,7 +179,7 @@
r.Save()
groups = re.split(r'[,\s]+', opt.groups)
- all_platforms = ['linux', 'darwin']
+ all_platforms = ['linux', 'darwin', 'windows']
platformize = lambda x: 'platform-' + x
if opt.platform == 'auto':
if (not opt.mirror and
@@ -188,7 +188,7 @@
elif opt.platform == 'all':
groups.extend(map(platformize, all_platforms))
elif opt.platform in all_platforms:
- groups.extend(platformize(opt.platform))
+ groups.append(platformize(opt.platform))
elif opt.platform != 'none':
print('fatal: invalid platform flag', file=sys.stderr)
sys.exit(1)
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index 1bdc1f0..7479697 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -54,6 +54,11 @@
p.add_option('--auto-stash',
dest='auto_stash', action='store_true',
help='Stash local modifications before starting')
+ p.add_option('-m', '--onto-manifest',
+ dest='onto_manifest', action='store_true',
+ help='Rebase onto the manifest version instead of upstream '
+ 'HEAD. This helps to make sure the local tree stays '
+ 'consistent if you previously synced to a manifest.')
def Execute(self, opt, args):
all_projects = self.GetProjects(args)
@@ -106,6 +111,10 @@
if opt.interactive:
args.append("-i")
+ if opt.onto_manifest:
+ args.append('--onto')
+ args.append(project.revisionExpr)
+
args.append(upbranch.LocalMerge)
print('# %s: rebasing %s -> %s'
diff --git a/subcmds/start.py b/subcmds/start.py
index 60ad41e..d1430a9 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -14,11 +14,15 @@
# limitations under the License.
from __future__ import print_function
+import os
import sys
+
from command import Command
from git_config import IsId
from git_command import git
+import gitc_utils
from progress import Progress
+from project import SyncBuffer
class Start(Command):
common = True
@@ -53,20 +57,57 @@
print("error: at least one project must be specified", file=sys.stderr)
sys.exit(1)
- all_projects = self.GetProjects(projects)
+ all_projects = self.GetProjects(projects,
+ missing_ok=bool(self.gitc_manifest))
+
+ # This must happen after we find all_projects, since GetProjects may need
+ # the local directory, which will disappear once we save the GITC manifest.
+ if self.gitc_manifest:
+ gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+ missing_ok=True)
+ for project in gitc_projects:
+ if project.old_revision:
+ project.already_synced = True
+ else:
+ project.already_synced = False
+ project.old_revision = project.revisionExpr
+ project.revisionExpr = None
+ # Save the GITC manifest.
+ gitc_utils.save_manifest(self.gitc_manifest)
+
+ # Make sure we have a valid CWD
+ if not os.path.exists(os.getcwd()):
+ os.chdir(self.manifest.topdir)
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
+
+ if self.gitc_manifest:
+ gitc_project = self.gitc_manifest.paths[project.relpath]
+ # Sync projects that have not been opened.
+ if not gitc_project.already_synced:
+ proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+ project.relpath)
+ project.worktree = proj_localdir
+ if not os.path.exists(proj_localdir):
+ os.makedirs(proj_localdir)
+ project.Sync_NetworkHalf()
+ sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+ project.Sync_LocalHalf(sync_buf)
+ project.revisionId = gitc_project.old_revision
+
# If the current revision is a specific SHA1 then we can't push back
# to it; so substitute with dest_branch if defined, or with manifest
# default revision instead.
+ branch_merge = ''
if IsId(project.revisionExpr):
if project.dest_branch:
- project.revisionExpr = project.dest_branch
+ branch_merge = project.dest_branch
else:
- project.revisionExpr = self.manifest.default.revisionExpr
- if not project.StartBranch(nb):
+ branch_merge = self.manifest.default.revisionExpr
+
+ if not project.StartBranch(nb, branch_merge=branch_merge):
err.append(project)
pm.end()
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450b..4af411c 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@
import socket
import subprocess
import sys
+import tempfile
import time
from pyversion import is_python3
if is_python3():
+ import http.cookiejar as cookielib
+ import urllib.error
import urllib.parse
+ import urllib.request
import xmlrpc.client
else:
+ import cookielib
import imp
+ import urllib2
import urlparse
import xmlrpclib
urllib = imp.new_module('urllib')
+ urllib.error = urllib2
urllib.parse = urlparse
+ urllib.request = urllib2
xmlrpc = imp.new_module('xmlrpc')
xmlrpc.client = xmlrpclib
@@ -57,7 +65,9 @@
multiprocessing = None
from git_command import GIT, git_require
+from git_config import GetUrlCookieFile
from git_refs import R_HEADS, HEAD
+import gitc_utils
from project import Project
from project import RemoteSpec
from command import Command, MirrorSafeCommand
@@ -65,6 +75,7 @@
from project import SyncBuffer
from progress import Progress
from wrapper import Wrapper
+from manifest_xml import GitcManifest
_ONE_DAY_S = 24 * 60 * 60
@@ -140,6 +151,9 @@
are fixed to a sha1 revision if the sha1 revision does not already
exist locally.
+The --prune option can be used to remove any refs that no longer
+exist on the remote.
+
SSH Connections
---------------
@@ -223,6 +237,8 @@
p.add_option('--optimized-fetch',
dest='optimized_fetch', action='store_true',
help='only fetch projects fixed to sha1 if revision does not exist locally')
+ p.add_option('--prune', dest='prune', action='store_true',
+ help='delete refs that no longer exist on the remote')
if show_smart:
p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true',
@@ -294,7 +310,8 @@
force_sync=opt.force_sync,
clone_bundle=not opt.no_clone_bundle,
no_tags=opt.no_tags, archive=self.manifest.IsArchive,
- optimized_fetch=opt.optimized_fetch)
+ optimized_fetch=opt.optimized_fetch,
+ prune=opt.prune)
self._fetch_times.Set(project, time.time() - start)
# Lock around all the rest of the code, since printing, updating a set
@@ -303,6 +320,7 @@
did_lock = True
if not success:
+ err_event.set()
print('error: Cannot fetch %s' % project.name, file=sys.stderr)
if opt.force_broken:
print('warn: --force-broken, continuing to sync',
@@ -313,7 +331,7 @@
fetched.add(project.gitdir)
pm.update()
except _FetchError:
- err_event.set()
+ pass
except Exception as e:
print('error: Cannot fetch %s (%s: %s)' \
% (project.name, type(e).__name__, str(e)), file=sys.stderr)
@@ -554,19 +572,18 @@
try:
info = netrc.netrc()
except IOError:
- print('.netrc file does not exist or could not be opened',
- file=sys.stderr)
+ # .netrc file does not exist or could not be opened
+ pass
else:
try:
parse_result = urllib.parse.urlparse(manifest_server)
if parse_result.hostname:
- username, _account, password = \
- info.authenticators(parse_result.hostname)
- except TypeError:
- # TypeError is raised when the given hostname is not present
- # in the .netrc file.
- print('No credentials found for %s in .netrc'
- % parse_result.hostname, file=sys.stderr)
+ auth = info.authenticators(parse_result.hostname)
+ if auth:
+ username, _account, password = auth
+ else:
+ print('No credentials found for %s in .netrc'
+ % parse_result.hostname, file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e, file=sys.stderr)
@@ -575,8 +592,12 @@
(username, password),
1)
+ transport = PersistentTransport(manifest_server)
+ if manifest_server.startswith('persistent-'):
+ manifest_server = manifest_server[len('persistent-'):]
+
try:
- server = xmlrpc.client.Server(manifest_server)
+ server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
@@ -656,6 +677,42 @@
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
+
+ if self.gitc_manifest:
+ gitc_manifest_projects = self.GetProjects(args,
+ missing_ok=True)
+ gitc_projects = []
+ opened_projects = []
+ for project in gitc_manifest_projects:
+ if project.relpath in self.gitc_manifest.paths and \
+ self.gitc_manifest.paths[project.relpath].old_revision:
+ opened_projects.append(project.relpath)
+ else:
+ gitc_projects.append(project.relpath)
+
+ if not args:
+ gitc_projects = None
+
+ if gitc_projects != [] and not opt.local_only:
+ print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+ manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
+ if manifest_name:
+ manifest.Override(manifest_name)
+ else:
+ manifest.Override(self.manifest.manifestFile)
+ gitc_utils.generate_gitc_manifest(self.gitc_manifest,
+ manifest,
+ gitc_projects)
+ print('GITC client successfully synced.')
+
+ # The opened projects need to be synced as normal, therefore we
+ # generate a new args list to represent the opened projects.
+ # TODO: make this more reliable -- if there's a project name/path overlap,
+ # this may choose the wrong project.
+ args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
+ for p in opened_projects]
+ if not args:
+ return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
@@ -850,3 +907,100 @@
os.remove(self._path)
except OSError:
pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+ def __init__(self, orig_host):
+ self.orig_host = orig_host
+
+ def request(self, host, handler, request_body, verbose=False):
+ with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+ # Python doesn't understand cookies with the #HttpOnly_ prefix
+ # Since we're only using them for HTTP, copy the file temporarily,
+ # stripping those prefixes away.
+ if cookiefile:
+ tmpcookiefile = tempfile.NamedTemporaryFile()
+ tmpcookiefile.write("# HTTP Cookie File")
+ try:
+ with open(cookiefile) as f:
+ for line in f:
+ if line.startswith("#HttpOnly_"):
+ line = line[len("#HttpOnly_"):]
+ tmpcookiefile.write(line)
+ tmpcookiefile.flush()
+
+ cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+ try:
+ cookiejar.load()
+ except cookielib.LoadError:
+ cookiejar = cookielib.CookieJar()
+ finally:
+ tmpcookiefile.close()
+ else:
+ cookiejar = cookielib.CookieJar()
+
+ proxyhandler = urllib.request.ProxyHandler
+ if proxy:
+ proxyhandler = urllib.request.ProxyHandler({
+ "http": proxy,
+ "https": proxy })
+
+ opener = urllib.request.build_opener(
+ urllib.request.HTTPCookieProcessor(cookiejar),
+ proxyhandler)
+
+ url = urllib.parse.urljoin(self.orig_host, handler)
+ parse_results = urllib.parse.urlparse(url)
+
+ scheme = parse_results.scheme
+ if scheme == 'persistent-http':
+ scheme = 'http'
+ if scheme == 'persistent-https':
+ # If we're proxying through persistent-https, use http. The
+ # proxy itself will do the https.
+ if proxy:
+ scheme = 'http'
+ else:
+ scheme = 'https'
+
+ # Parse out any authentication information using the base class
+ host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+ url = urllib.parse.urlunparse((
+ scheme,
+ host,
+ parse_results.path,
+ parse_results.params,
+ parse_results.query,
+ parse_results.fragment))
+
+ request = urllib.request.Request(url, request_body)
+ if extra_headers is not None:
+ for (name, header) in extra_headers:
+ request.add_header(name, header)
+ request.add_header('Content-Type', 'text/xml')
+ try:
+ response = opener.open(request)
+ except urllib.error.HTTPError as e:
+ if e.code == 501:
+ # We may have been redirected through a login process
+ # but our POST turned into a GET. Retry.
+ response = opener.open(request)
+ else:
+ raise
+
+ p, u = xmlrpc.client.getparser()
+ while 1:
+ data = response.read(1024)
+ if not data:
+ break
+ p.feed(data)
+ p.close()
+ return u.close()
+
+ def close(self):
+ pass
+
diff --git a/tests/fixtures/gitc_config b/tests/fixtures/gitc_config
new file mode 100644
index 0000000..a7f3d1c
--- /dev/null
+++ b/tests/fixtures/gitc_config
@@ -0,0 +1 @@
+gitc_dir=/test/usr/local/google/gitc
diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py
new file mode 100644
index 0000000..fb32e38
--- /dev/null
+++ b/tests/test_wrapper.py
@@ -0,0 +1,75 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import wrapper
+
+def fixture(*paths):
+ """Return a path relative to tests/fixtures.
+ """
+ return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
+
+class RepoWrapperUnitTest(unittest.TestCase):
+ """Tests helper functions in the repo wrapper
+ """
+ def setUp(self):
+ """Load the wrapper module every time
+ """
+ wrapper._wrapper_module = None
+ self.wrapper = wrapper.Wrapper()
+
+ def test_get_gitc_manifest_dir_no_gitc(self):
+ """
+ Test reading a missing gitc config file
+ """
+ self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
+ val = self.wrapper.get_gitc_manifest_dir()
+ self.assertEqual(val, '')
+
+ def test_get_gitc_manifest_dir(self):
+ """
+ Test reading the gitc config file and parsing the directory
+ """
+ self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
+ val = self.wrapper.get_gitc_manifest_dir()
+ self.assertEqual(val, '/test/usr/local/google/gitc')
+
+ def test_gitc_parse_clientdir_no_gitc(self):
+ """
+ Test parsing the gitc clientdir without gitc running
+ """
+ self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
+
+ def test_gitc_parse_clientdir(self):
+ """
+ Test parsing the gitc clientdir
+ """
+ self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/extra'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/extra'), 'test')
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/'), None)
+ self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/'), None)
+
+if __name__ == '__main__':
+ unittest.main()
