Merge "docs: add copyfile and linkfile elements description"
diff --git a/command.py b/command.py
index 38cacd3..997acec 100644
--- a/command.py
+++ b/command.py
@@ -106,13 +106,13 @@
def _UpdatePathToProjectMap(self, project):
self._by_path[project.worktree] = project
- def _GetProjectByPath(self, path):
+ def _GetProjectByPath(self, manifest, path):
project = None
if os.path.exists(path):
oldpath = None
while path \
and path != oldpath \
- and path != self.manifest.topdir:
+ and path != manifest.topdir:
try:
project = self._by_path[path]
break
@@ -126,13 +126,16 @@
pass
return project
- def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
+ def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+ submodules_ok=False):
"""A list of projects that match the arguments.
"""
- all_projects_list = self.manifest.projects
+ if not manifest:
+ manifest = self.manifest
+ all_projects_list = manifest.projects
result = []
- mp = self.manifest.manifestProject
+ mp = manifest.manifestProject
if not groups:
groups = mp.config.GetString('manifest.groups')
@@ -155,11 +158,11 @@
self._ResetPathToProjectMap(all_projects_list)
for arg in args:
- projects = self.manifest.GetProjectsWithName(arg)
+ projects = manifest.GetProjectsWithName(arg)
if not projects:
path = os.path.abspath(arg).replace('\\', '/')
- project = self._GetProjectByPath(path)
+ project = self._GetProjectByPath(manifest, path)
# If it's not a derived project, update path->project mapping and
# search again, as arg might actually point to a derived subproject.
@@ -170,7 +173,7 @@
self._UpdatePathToProjectMap(subproject)
search_again = True
if search_again:
- project = self._GetProjectByPath(path) or project
+ project = self._GetProjectByPath(manifest, path) or project
if project:
projects = [project]
@@ -227,3 +230,8 @@
"""Command permits itself to run within a mirror,
and does not require a working directory.
"""
+
+class RequiresGitcCommand(object):
+ """Command that requires GITC to be available, but does
+ not require the local client to be a GITC client.
+ """
diff --git a/git_config.py b/git_config.py
index 8ded7c2..0379181 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,6 +15,8 @@
from __future__ import print_function
+import contextlib
+import errno
import json
import os
import re
@@ -502,6 +504,43 @@
return m.group(1)
return None
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+ if url.startswith('persistent-'):
+ try:
+ p = subprocess.Popen(
+ ['git-remote-persistent-https', '-print_config', url],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ cookieprefix = 'http.cookiefile='
+ proxyprefix = 'http.proxy='
+ cookiefile = None
+ proxy = None
+ for line in p.stdout:
+ line = line.strip()
+ if line.startswith(cookieprefix):
+ cookiefile = line[len(cookieprefix):]
+ if line.startswith(proxyprefix):
+ proxy = line[len(proxyprefix):]
+ # Leave subprocess open, as cookie file may be transient.
+ if cookiefile or proxy:
+ yield cookiefile, proxy
+ return
+ finally:
+ p.stdin.close()
+ if p.wait():
+ err_msg = p.stderr.read()
+ if ' -print_config' in err_msg:
+ pass # Persistent proxy doesn't support -print_config.
+ elif not quiet:
+ print(err_msg, file=sys.stderr)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ pass # No persistent proxy.
+ raise
+ yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
def _preconnect(url):
m = URI_ALL.match(url)
if m:
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 0000000..dd38f89
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,158 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def get_gitc_manifest_dir():
+ return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+ """Parse a path in the GITC FS and return its client name.
+
+ @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+ @returns: The GITC client name
+ """
+ if (gitc_fs_path == GITC_FS_ROOT_DIR or
+ not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)):
+ return None
+ return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
+def _set_project_revisions(projects):
+ """Sets the revisionExpr for a list of projects.
+
+ Because of the limit of open file descriptors allowed, length of projects
+ should not be overly large. Recommend calling this function multiple times
+ with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+ @param projects: List of project objects to set the revionExpr for.
+ """
+ # Retrieve the commit id for each project based off of it's current
+ # revisionExpr and it is not already a commit id.
+ project_gitcmds = [(
+ project, git_command.GitCommand(None,
+ ['ls-remote',
+ project.remote.url,
+ project.revisionExpr],
+ capture_stdout=True, cwd='/tmp'))
+ for project in projects if not git_config.IsId(project.revisionExpr)]
+ for proj, gitcmd in project_gitcmds:
+ if gitcmd.Wait():
+ print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+ sys.exit(1)
+ proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def _manifest_groups(manifest):
+ """Returns the manifest group string that should be synced
+
+ This is the same logic used by Command.GetProjects(), which is used during
+ repo sync
+
+ @param manifest: The XmlManifest object
+ """
+ mp = manifest.manifestProject
+ groups = mp.config.GetString('manifest.groups')
+ if not groups:
+ groups = 'default,platform-' + platform.system().lower()
+ return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+ """Generate a manifest for shafsd to use for this GITC client.
+
+ @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+ @param manifest: A GitcManifest object loaded with the current repo manifest.
+ @param paths: List of project paths we want to update.
+ """
+
+ print('Generating GITC Manifest by fetching revision SHAs for each '
+ 'project.')
+ if paths is None:
+ paths = manifest.paths.keys()
+
+ groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+ # Convert the paths to projects, and filter them to the matched groups.
+ projects = [manifest.paths[p] for p in paths]
+ projects = [p for p in projects if p.MatchesGroups(groups)]
+
+ if gitc_manifest is not None:
+ for path, proj in manifest.paths.iteritems():
+ if not proj.MatchesGroups(groups):
+ continue
+
+ if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+ proj.upstream = proj.revisionExpr
+
+ if not path in gitc_manifest.paths:
+ # Any new projects need their first revision, even if we weren't asked
+ # for them.
+ projects.append(proj)
+ elif not path in paths:
+ # And copy revisions from the previous manifest if we're not updating
+ # them now.
+ gitc_proj = gitc_manifest.paths[path]
+ if gitc_proj.old_revision:
+ proj.revisionExpr = None
+ proj.old_revision = gitc_proj.old_revision
+ else:
+ proj.revisionExpr = gitc_proj.revisionExpr
+
+ index = 0
+ while index < len(projects):
+ _set_project_revisions(
+ projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+ index += NUM_BATCH_RETRIEVE_REVISIONID
+
+ if gitc_manifest is not None:
+ for path, proj in gitc_manifest.paths.iteritems():
+ if proj.old_revision and path in paths:
+ # If we updated a project that has been started, keep the old-revision
+ # updated.
+ repo_proj = manifest.paths[path]
+ repo_proj.old_revision = repo_proj.revisionExpr
+ repo_proj.revisionExpr = None
+
+ # Convert URLs from relative to absolute.
+ for name, remote in manifest.remotes.iteritems():
+ remote.fetchUrl = remote.resolvedFetchUrl
+
+ # Save the manifest.
+ save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+ """Save the manifest file in the client_dir.
+
+ @param client_dir: Client directory to save the manifest in.
+ @param manifest: Manifest object to save.
+ """
+ if not client_dir:
+ client_dir = manifest.gitc_client_dir
+ with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+ manifest.Save(f, groups=_manifest_groups(manifest))
+ # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+ # Give the GITC filesystem time to register the manifest changes.
+ time.sleep(3)
diff --git a/main.py b/main.py
index 6736abc..a5979a8 100755
--- a/main.py
+++ b/main.py
@@ -42,6 +42,7 @@
from git_config import init_ssh, close_ssh
from command import InteractiveCommand
from command import MirrorSafeCommand
+from command import RequiresGitcCommand
from subcmds.version import Version
from editor import Editor
from error import DownloadError
@@ -51,7 +52,8 @@
from error import NoManifestException
from error import NoSuchProjectError
from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
from pager import RunPager
from wrapper import WrapperPath, Wrapper
@@ -129,6 +131,12 @@
cmd.repodir = self.repodir
cmd.manifest = XmlManifest(cmd.repodir)
+ cmd.gitc_manifest = None
+ gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+ if gitc_client_name:
+ cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+ cmd.manifest.isGitcClient = True
+
Editor.globalConfig = cmd.manifest.globalConfig
if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -136,6 +144,11 @@
file=sys.stderr)
return 1
+ if isinstance(cmd, RequiresGitcCommand) and not gitc_utils.get_gitc_manifest_dir():
+ print("fatal: '%s' requires GITC to be available" % name,
+ file=sys.stderr)
+ return 1
+
try:
copts, cargs = cmd.OptionParser.parse_args(argv)
copts = cmd.ReadEnvironmentOptions(copts)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e71960..3ac607e 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -29,6 +29,7 @@
urllib = imp.new_module('urllib')
urllib.parse = urlparse
+import gitc_utils
from git_config import GitConfig
from git_refs import R_HEADS, HEAD
from project import RemoteSpec, Project, MetaProject
@@ -112,6 +113,7 @@
self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
self.globalConfig = GitConfig.ForUser()
self.localManifestWarning = False
+ self.isGitcClient = False
self.repoProject = MetaProject(self, 'repo',
gitdir = os.path.join(repodir, 'repo/.git'),
@@ -165,12 +167,13 @@
def _ParseGroups(self, groups):
return [x for x in re.split(r'[,\s]+', groups) if x]
- def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
+ def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
"""Write the current manifest out to the given file descriptor.
"""
mp = self.manifestProject
- groups = mp.config.GetString('manifest.groups')
+ if groups is None:
+ groups = mp.config.GetString('manifest.groups')
if groups:
groups = self._ParseGroups(groups)
@@ -303,6 +306,11 @@
if p.sync_s:
e.setAttribute('sync-s', 'true')
+ if p.clone_depth:
+ e.setAttribute('clone-depth', str(p.clone_depth))
+
+ self._output_manifest_project_extras(p, e)
+
if p.subprojects:
subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, list(sorted(subprojects)))
@@ -320,6 +328,10 @@
doc.writexml(fd, '', ' ', '\n', 'UTF-8')
+ def _output_manifest_project_extras(self, p, e):
+ """Manifests can modify e if they support extra project attributes."""
+ pass
+
@property
def paths(self):
self._Load()
@@ -709,7 +721,7 @@
def _UnjoinName(self, parent_name, name):
return os.path.relpath(name, parent_name)
- def _ParseProject(self, node, parent = None):
+ def _ParseProject(self, node, parent = None, **extra_proj_attrs):
"""
reads a <project> element from the manifest file
"""
@@ -804,7 +816,8 @@
clone_depth = clone_depth,
upstream = upstream,
parent = parent,
- dest_branch = dest_branch)
+ dest_branch = dest_branch,
+ **extra_proj_attrs)
for n in node.childNodes:
if n.nodeName == 'copyfile':
@@ -935,3 +948,26 @@
diff['added'].append(toProjects[proj])
return diff
+
+
+class GitcManifest(XmlManifest):
+
+ def __init__(self, repodir, gitc_client_name):
+ """Initialize the GitcManifest object."""
+ super(GitcManifest, self).__init__(repodir)
+ self.isGitcClient = True
+ self.gitc_client_name = gitc_client_name
+ self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+ gitc_client_name)
+ self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+ def _ParseProject(self, node, parent = None):
+ """Override _ParseProject and add support for GITC specific attributes."""
+ return super(GitcManifest, self)._ParseProject(
+ node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+ def _output_manifest_project_extras(self, p, e):
+ """Output GITC Specific Project attributes"""
+ if p.old_revision:
+ e.setAttribute('old-revision', str(p.old_revision))
+
diff --git a/project.py b/project.py
index a117f4d..5d8f61e 100644
--- a/project.py
+++ b/project.py
@@ -13,7 +13,6 @@
# limitations under the License.
from __future__ import print_function
-import contextlib
import errno
import filecmp
import glob
@@ -31,7 +30,7 @@
from color import Coloring
from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, ID_RE
from error import GitError, HookError, UploadError, DownloadError
from error import ManifestInvalidRevisionError
from error import NoManifestException
@@ -64,6 +63,10 @@
msg = fmt % args
print('error: %s' % msg, file=sys.stderr)
+def _warn(fmt, *args):
+ msg = fmt % args
+ print('warn: %s' % msg, file=sys.stderr)
+
def not_rev(r):
return '^' + r
@@ -569,7 +572,8 @@
parent=None,
is_derived=False,
dest_branch=None,
- optimized_fetch=False):
+ optimized_fetch=False,
+ old_revision=None):
"""Init a Project object.
Args:
@@ -593,6 +597,7 @@
dest_branch: The branch to which to push changes for review by default.
optimized_fetch: If True, when a project is set to a sha1 revision, only
fetch from the remote if the sha1 is not present locally.
+ old_revision: saved git commit id for open GITC projects.
"""
self.manifest = manifest
self.name = name
@@ -640,6 +645,7 @@
self.bare_ref = GitRefs(gitdir)
self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
self.dest_branch = dest_branch
+ self.old_revision = old_revision
# This will be filled in if a project is later identified to be the
# project containing repo hooks.
@@ -1093,8 +1099,7 @@
tar.extractall(path=path)
return True
except (IOError, tarfile.TarError) as e:
- print("error: Cannot extract archive %s: "
- "%s" % (tarpath, str(e)), file=sys.stderr)
+ _error("Cannot extract archive %s: %s", tarpath, str(e))
return False
def Sync_NetworkHalf(self,
@@ -1111,8 +1116,7 @@
"""
if archive and not isinstance(self, MetaProject):
if self.remote.url.startswith(('http://', 'https://')):
- print("error: %s: Cannot fetch archives from http/https "
- "remotes." % self.name, file=sys.stderr)
+ _error("%s: Cannot fetch archives from http/https remotes.", self.name)
return False
name = self.relpath.replace('\\', '/')
@@ -1123,7 +1127,7 @@
try:
self._FetchArchive(tarpath, cwd=topdir)
except GitError as e:
- print('error: %s' % str(e), file=sys.stderr)
+ _error('%s', e)
return False
# From now on, we only need absolute tarpath
@@ -1134,8 +1138,7 @@
try:
os.remove(tarpath)
except OSError as e:
- print("warn: Cannot remove archive %s: "
- "%s" % (tarpath, str(e)), file=sys.stderr)
+ _warn("Cannot remove archive %s: %s", tarpath, str(e))
self._CopyAndLinkFiles()
return True
if is_new is None:
@@ -1195,6 +1198,8 @@
self._InitHooks()
def _CopyAndLinkFiles(self):
+ if self.manifest.isGitcClient:
+ return
for copyfile in self.copyfiles:
copyfile._Copy()
for linkfile in self.linkfiles:
@@ -1270,6 +1275,8 @@
# Except if the head needs to be detached
#
if not syncbuf.detach_head:
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
return
else:
lost = self._revlist(not_rev(revid), HEAD)
@@ -1287,6 +1294,8 @@
if head == revid:
# No changes; don't do anything further.
#
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
return
branch = self.GetBranch(branch)
@@ -1425,9 +1434,11 @@
## Branch Management ##
- def StartBranch(self, name):
+ def StartBranch(self, name, branch_merge=''):
"""Create a new branch off the manifest's revision.
"""
+ if not branch_merge:
+ branch_merge = self.revisionExpr
head = self.work_git.GetHead()
if head == (R_HEADS + name):
return True
@@ -1441,9 +1452,9 @@
branch = self.GetBranch(name)
branch.remote = self.GetRemote(self.remote.name)
- branch.merge = self.revisionExpr
- if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
- branch.merge = R_HEADS + self.revisionExpr
+ branch.merge = branch_merge
+ if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
+ branch.merge = R_HEADS + branch_merge
revid = self.GetRevisionId(all_refs)
if head.startswith(R_HEADS):
@@ -1451,7 +1462,6 @@
head = all_refs[head]
except KeyError:
head = None
-
if revid and head and revid == head:
ref = os.path.join(self.gitdir, R_HEADS + name)
try:
@@ -2030,7 +2040,7 @@
os.remove(tmpPath)
if 'http_proxy' in os.environ and 'darwin' == sys.platform:
cmd += ['--proxy', os.environ['http_proxy']]
- with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
+ with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
if cookiefile:
cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
if srcUrl.startswith('persistent-'):
@@ -2078,40 +2088,6 @@
except OSError:
return False
- @contextlib.contextmanager
- def _GetBundleCookieFile(self, url, quiet):
- if url.startswith('persistent-'):
- try:
- p = subprocess.Popen(
- ['git-remote-persistent-https', '-print_config', url],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- try:
- prefix = 'http.cookiefile='
- cookiefile = None
- for line in p.stdout:
- line = line.strip()
- if line.startswith(prefix):
- cookiefile = line[len(prefix):]
- break
- # Leave subprocess open, as cookie file may be transient.
- if cookiefile:
- yield cookiefile
- return
- finally:
- p.stdin.close()
- if p.wait():
- err_msg = p.stderr.read()
- if ' -print_config' in err_msg:
- pass # Persistent proxy doesn't support -print_config.
- elif not quiet:
- print(err_msg, file=sys.stderr)
- except OSError as e:
- if e.errno == errno.ENOENT:
- pass # No persistent proxy.
- raise
- yield GitConfig.ForUser().GetString('http.cookiefile')
-
def _Checkout(self, rev, quiet=False):
cmd = ['checkout']
if quiet:
@@ -2182,8 +2158,8 @@
try:
self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
except GitError as e:
- print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
if force_sync:
+ print("Retrying clone after deleting %s" % self.gitdir, file=sys.stderr)
try:
shutil.rmtree(os.path.realpath(self.gitdir))
if self.worktree and os.path.exists(
@@ -2261,7 +2237,7 @@
if filecmp.cmp(stock_hook, dst, shallow=False):
os.remove(dst)
else:
- _error("%s: Not replacing %s hook", self.relpath, name)
+ _warn("%s: Not replacing locally modified %s hook", self.relpath, name)
continue
try:
os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
@@ -2320,7 +2296,10 @@
# Fail if the links are pointing to the wrong place
if src != dst:
raise GitError('--force-sync not enabled; cannot overwrite a local '
- 'work tree')
+ 'work tree. If you\'re comfortable with the '
+ 'possibility of losing the work tree\'s git metadata,'
+ ' use `repo sync --force-sync {0}` to '
+ 'proceed.'.format(self.relpath))
def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
"""Update |dotgit| to reference |gitdir|, using symlinks where possible.
diff --git a/repo b/repo
index bf8fa3d..b7d8024 100755
--- a/repo
+++ b/repo
@@ -108,7 +108,7 @@
S_manifests = 'manifests' # special manifest repository
REPO_MAIN = S_repo + '/main.py' # main script
MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
+GITC_CONFIG_FILE = '/gitc/.config'
import errno
@@ -214,6 +214,7 @@
help='Always prompt for name/e-mail')
def _GitcInitOptions(init_optparse):
+ init_optparse.set_usage("repo gitc-init -u url -c client [options]")
g = init_optparse.add_option_group('GITC options')
g.add_option('-f', '--manifest-file',
dest='manifest_file',
@@ -222,6 +223,21 @@
dest='gitc_client',
help='The name for the new gitc_client instance.')
+_gitc_manifest_dir = None
+def get_gitc_manifest_dir():
+ global _gitc_manifest_dir
+ if _gitc_manifest_dir is None:
+ _gitc_manifest_dir = ''
+ try:
+ with open(GITC_CONFIG_FILE, 'r') as gitc_config:
+ for line in gitc_config:
+ match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
+ if match:
+ _gitc_manifest_dir = match.group('gitc_manifest_dir')
+ except IOError:
+ pass
+ return _gitc_manifest_dir
+
class CloneFailure(Exception):
"""Indicate the remote clone of repo itself failed.
"""
@@ -255,7 +271,15 @@
try:
if gitc_init:
- client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
+ gitc_manifest_dir = get_gitc_manifest_dir()
+ if not gitc_manifest_dir:
+ _print('fatal: GITC filesystem is not available. Exiting...',
+ file=sys.stderr)
+ sys.exit(1)
+ if not opt.gitc_client:
+ _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+ sys.exit(1)
+ client_dir = os.path.join(gitc_manifest_dir, opt.gitc_client)
if not os.path.exists(client_dir):
os.makedirs(client_dir)
os.chdir(client_dir)
@@ -661,6 +685,10 @@
def _Usage():
+ gitc_usage = ""
+ if get_gitc_manifest_dir():
+ gitc_usage = " gitc-init Initialize a GITC Client.\n"
+
_print(
"""usage: repo COMMAND [ARGS]
@@ -669,7 +697,8 @@
The most commonly used repo commands are:
init Install repo in the current working directory
- help Display detailed help on a command
+""" + gitc_usage +
+""" help Display detailed help on a command
For access to the full online help, install repo ("repo init").
""", file=sys.stderr)
@@ -681,6 +710,10 @@
if args[0] == 'init':
init_optparse.print_help()
sys.exit(0)
+ elif args[0] == 'gitc-init':
+ _GitcInitOptions(init_optparse)
+ init_optparse.print_help()
+ sys.exit(0)
else:
_print("error: '%s' is not a bootstrap command.\n"
' For access to online help, install repo ("repo init").'
@@ -746,6 +779,12 @@
wrapper_path = os.path.abspath(__file__)
my_main, my_git = _RunSelf(wrapper_path)
+ cwd = os.getcwd()
+ if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+ _print('error: repo cannot be used in the GITC local manifest directory.'
+ '\nIf you want to work on this GITC client please rerun this '
+ 'command from the corresponding client under /gitc/', file=sys.stderr)
+ sys.exit(1)
if not repo_main:
if opt.help:
_Usage()
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 9b9cefd..c243bb3 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -15,19 +15,15 @@
from __future__ import print_function
import os
-import shutil
import sys
-import git_command
+import gitc_utils
+from command import RequiresGitcCommand
+from manifest_xml import GitcManifest
from subcmds import init
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
-GITC_FS_ROOT_DIR = '/gitc/sha/rw'
-NUM_BATCH_RETRIEVE_REVISIONID = 300
-
-
-class GitcInit(init.Init):
+class GitcInit(init.Init, RequiresGitcCommand):
common = True
helpSummary = "Initialize a GITC Client."
helpUsage = """
@@ -39,7 +35,7 @@
This command will setup the client directory, initialize repo, just
like repo init does, and then downloads the manifest collection
-and installs in in the .repo/directory of the GITC client.
+and installs it in the .repo/directory of the GITC client.
Once this is done, a GITC manifest is generated by pulling the HEAD
SHA for each project and generates the properly formatted XML file
@@ -65,59 +61,24 @@
if not opt.gitc_client:
print('fatal: gitc client (-c) is required', file=sys.stderr)
sys.exit(1)
- self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
- if not os.path.exists(GITC_MANIFEST_DIR):
- os.makedirs(GITC_MANIFEST_DIR)
+ self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+ opt.gitc_client)
+ if not os.path.exists(gitc_utils.get_gitc_manifest_dir()):
+ os.makedirs(gitc_utils.get_gitc_manifest_dir())
if not os.path.exists(self.client_dir):
os.mkdir(self.client_dir)
super(GitcInit, self).Execute(opt, args)
+
+ manifest_file = self.manifest.manifestFile
if opt.manifest_file:
if not os.path.exists(opt.manifest_file):
print('fatal: Specified manifest file %s does not exist.' %
opt.manifest_file)
sys.exit(1)
- shutil.copyfile(opt.manifest_file,
- os.path.join(self.client_dir, '.manifest'))
- else:
- self._GenerateGITCManifest()
+ manifest_file = opt.manifest_file
+
+ manifest = GitcManifest(self.repodir, opt.gitc_client)
+ manifest.Override(manifest_file)
+ gitc_utils.generate_gitc_manifest(None, manifest)
print('Please run `cd %s` to view your GITC client.' %
- os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client))
-
- def _SetProjectRevisions(self, projects, branch):
- """Sets the revisionExpr for a list of projects.
-
- Because of the limit of open file descriptors allowed, length of projects
- should not be overly large. Recommend calling this function multiple times
- with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
-
- @param projects: List of project objects to set the revionExpr for.
- @param branch: The remote branch to retrieve the SHA from. If branch is
- None, 'HEAD' is used.
- """
- project_gitcmds = [(
- project, git_command.GitCommand(None,
- ['ls-remote',
- project.remote.url,
- branch], capture_stdout=True))
- for project in projects]
- for proj, gitcmd in project_gitcmds:
- if gitcmd.Wait():
- print('FATAL: Failed to retrieve revisionID for %s' % project)
- sys.exit(1)
- proj.revisionExpr = gitcmd.stdout.split('\t')[0]
-
- def _GenerateGITCManifest(self):
- """Generate a manifest for shafsd to use for this GITC client."""
- print('Generating GITC Manifest by fetching revision SHAs for each '
- 'project.')
- manifest = self.manifest
- project_gitcmd_dict = {}
- index = 0
- while index < len(manifest.projects):
- self._SetProjectRevisions(
- manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)],
- manifest.default.revisionExpr)
- index += NUM_BATCH_RETRIEVE_REVISIONID
- # Save the manifest.
- with open(os.path.join(self.client_dir, '.manifest'), 'w') as f:
- manifest.Save(f)
+ os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
diff --git a/subcmds/help.py b/subcmds/help.py
index 4aa3f86..ae5b8f0 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -19,7 +19,8 @@
from formatter import AbstractFormatter, DumbWriter
from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, RequiresGitcCommand
+import gitc_utils
class Help(PagedCommand, MirrorSafeCommand):
common = False
@@ -54,9 +55,17 @@
def _PrintCommonCommands(self):
print('usage: repo COMMAND [ARGS]')
print('The most commonly used repo commands are:')
+
+ def gitc_supported(cmd):
+ if not isinstance(cmd, RequiresGitcCommand):
+ return True
+ if gitc_utils.get_gitc_manifest_dir():
+ return True
+ return False
+
commandNames = list(sorted([name
for name, command in self.commands.items()
- if command.common]))
+ if command.common and gitc_supported(command)]))
maxlen = 0
for name in commandNames:
diff --git a/subcmds/start.py b/subcmds/start.py
index 60ad41e..940c341 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -14,11 +14,15 @@
# limitations under the License.
from __future__ import print_function
+import os
import sys
+
from command import Command
from git_config import IsId
from git_command import git
+import gitc_utils
from progress import Progress
+from project import SyncBuffer
class Start(Command):
common = True
@@ -53,20 +57,50 @@
print("error: at least one project must be specified", file=sys.stderr)
sys.exit(1)
- all_projects = self.GetProjects(projects)
+ if self.gitc_manifest:
+ all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+ missing_ok=True)
+ for project in all_projects:
+ if project.old_revision:
+ project.already_synced = True
+ else:
+ project.already_synced = False
+ project.old_revision = project.revisionExpr
+ project.revisionExpr = None
+ # Save the GITC manifest.
+ gitc_utils.save_manifest(self.gitc_manifest)
+ all_projects = self.GetProjects(projects,
+ missing_ok=bool(self.gitc_manifest))
pm = Progress('Starting %s' % nb, len(all_projects))
for project in all_projects:
pm.update()
+
+ if self.gitc_manifest:
+ gitc_project = self.gitc_manifest.paths[project.relpath]
+ # Sync projects that have not been opened.
+ if not gitc_project.already_synced:
+ proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+ project.relpath)
+ project.worktree = proj_localdir
+ if not os.path.exists(proj_localdir):
+ os.makedirs(proj_localdir)
+ project.Sync_NetworkHalf()
+ sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+ project.Sync_LocalHalf(sync_buf)
+ project.revisionId = gitc_project.old_revision
+
# If the current revision is a specific SHA1 then we can't push back
# to it; so substitute with dest_branch if defined, or with manifest
# default revision instead.
+ branch_merge = ''
if IsId(project.revisionExpr):
if project.dest_branch:
- project.revisionExpr = project.dest_branch
+ branch_merge = project.dest_branch
else:
- project.revisionExpr = self.manifest.default.revisionExpr
- if not project.StartBranch(nb):
+ branch_merge = self.manifest.default.revisionExpr
+
+ if not project.StartBranch(nb, branch_merge=branch_merge):
err.append(project)
pm.end()
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450b..a99d7e7 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@
import socket
import subprocess
import sys
+import tempfile
import time
from pyversion import is_python3
if is_python3():
+ import http.cookiejar as cookielib
+ import urllib.error
import urllib.parse
+ import urllib.request
import xmlrpc.client
else:
+ import cookielib
import imp
+ import urllib2
import urlparse
import xmlrpclib
urllib = imp.new_module('urllib')
+ urllib.error = urllib2
urllib.parse = urlparse
+ urllib.request = urllib2
xmlrpc = imp.new_module('xmlrpc')
xmlrpc.client = xmlrpclib
@@ -57,7 +65,9 @@
multiprocessing = None
from git_command import GIT, git_require
+from git_config import GetUrlCookieFile
from git_refs import R_HEADS, HEAD
+import gitc_utils
from project import Project
from project import RemoteSpec
from command import Command, MirrorSafeCommand
@@ -65,6 +75,7 @@
from project import SyncBuffer
from progress import Progress
from wrapper import Wrapper
+from manifest_xml import GitcManifest
_ONE_DAY_S = 24 * 60 * 60
@@ -554,19 +565,18 @@
try:
info = netrc.netrc()
except IOError:
- print('.netrc file does not exist or could not be opened',
- file=sys.stderr)
+ # .netrc file does not exist or could not be opened
+ pass
else:
try:
parse_result = urllib.parse.urlparse(manifest_server)
if parse_result.hostname:
- username, _account, password = \
- info.authenticators(parse_result.hostname)
- except TypeError:
- # TypeError is raised when the given hostname is not present
- # in the .netrc file.
- print('No credentials found for %s in .netrc'
- % parse_result.hostname, file=sys.stderr)
+ auth = info.authenticators(parse_result.hostname)
+ if auth:
+ username, _account, password = auth
+ else:
+ print('No credentials found for %s in .netrc'
+ % parse_result.hostname, file=sys.stderr)
except netrc.NetrcParseError as e:
print('Error parsing .netrc file: %s' % e, file=sys.stderr)
@@ -575,8 +585,12 @@
(username, password),
1)
+ transport = PersistentTransport(manifest_server)
+ if manifest_server.startswith('persistent-'):
+ manifest_server = manifest_server[len('persistent-'):]
+
try:
- server = xmlrpc.client.Server(manifest_server)
+ server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
@@ -656,6 +670,42 @@
self._ReloadManifest(manifest_name)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
+
+ if self.gitc_manifest:
+ gitc_manifest_projects = self.GetProjects(args,
+ missing_ok=True)
+ gitc_projects = []
+ opened_projects = []
+ for project in gitc_manifest_projects:
+ if project.relpath in self.gitc_manifest.paths and \
+ self.gitc_manifest.paths[project.relpath].old_revision:
+ opened_projects.append(project.relpath)
+ else:
+ gitc_projects.append(project.relpath)
+
+ if not args:
+ gitc_projects = None
+
+ if gitc_projects != [] and not opt.local_only:
+ print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+ manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
+ if manifest_name:
+ manifest.Override(manifest_name)
+ else:
+ manifest.Override(self.manifest.manifestFile)
+ gitc_utils.generate_gitc_manifest(self.gitc_manifest,
+ manifest,
+ gitc_projects)
+ print('GITC client successfully synced.')
+
+ # The opened projects need to be synced as normal, therefore we
+ # generate a new args list to represent the opened projects.
+ # TODO: make this more reliable -- if there's a project name/path overlap,
+ # this may choose the wrong project.
+ args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
+ for p in opened_projects]
+ if not args:
+ return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
@@ -850,3 +900,96 @@
os.remove(self._path)
except OSError:
pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+ def __init__(self, orig_host):
+ self.orig_host = orig_host
+
+ def request(self, host, handler, request_body, verbose=False):
+ with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+ # Python doesn't understand cookies with the #HttpOnly_ prefix
+ # Since we're only using them for HTTP, copy the file temporarily,
+ # stripping those prefixes away.
+ if cookiefile:
+ tmpcookiefile = tempfile.NamedTemporaryFile()
+ try:
+ with open(cookiefile) as f:
+ for line in f:
+ if line.startswith("#HttpOnly_"):
+ line = line[len("#HttpOnly_"):]
+ tmpcookiefile.write(line)
+ tmpcookiefile.flush()
+
+ cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+ cookiejar.load()
+ finally:
+ tmpcookiefile.close()
+ else:
+ cookiejar = cookielib.CookieJar()
+
+ proxyhandler = urllib.request.ProxyHandler
+ if proxy:
+ proxyhandler = urllib.request.ProxyHandler({
+ "http": proxy,
+ "https": proxy })
+
+ opener = urllib.request.build_opener(
+ urllib.request.HTTPCookieProcessor(cookiejar),
+ proxyhandler)
+
+ url = urllib.parse.urljoin(self.orig_host, handler)
+ parse_results = urllib.parse.urlparse(url)
+
+ scheme = parse_results.scheme
+ if scheme == 'persistent-http':
+ scheme = 'http'
+ if scheme == 'persistent-https':
+ # If we're proxying through persistent-https, use http. The
+ # proxy itself will do the https.
+ if proxy:
+ scheme = 'http'
+ else:
+ scheme = 'https'
+
+ # Parse out any authentication information using the base class
+ host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+ url = urllib.parse.urlunparse((
+ scheme,
+ host,
+ parse_results.path,
+ parse_results.params,
+ parse_results.query,
+ parse_results.fragment))
+
+ request = urllib.request.Request(url, request_body)
+ if extra_headers is not None:
+ for (name, header) in extra_headers:
+ request.add_header(name, header)
+ request.add_header('Content-Type', 'text/xml')
+ try:
+ response = opener.open(request)
+ except urllib.error.HTTPError as e:
+ if e.code == 501:
+ # We may have been redirected through a login process
+ # but our POST turned into a GET. Retry.
+ response = opener.open(request)
+ else:
+ raise
+
+ p, u = xmlrpc.client.getparser()
+ while 1:
+ data = response.read(1024)
+ if not data:
+ break
+ p.feed(data)
+ p.close()
+ return u.close()
+
+ def close(self):
+ pass
+
