Merge "Support smart-sync through persistent-http[s]"
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 0000000..bf79bd2
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+
+import git_command
+import git_config
+
+
+# TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config
+GITC_MANIFEST_DIR = '/usr/local/google/gitc/'
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def _set_project_revisions(projects):
+ """Sets the revisionExpr for a list of projects.
+
+ Because of the limit of open file descriptors allowed, length of projects
+ should not be overly large. Recommend calling this function multiple times
+ with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+ @param projects: List of project objects to set the revionExpr for.
+ """
+ # Retrieve the commit id for each project based off of it's current
+ # revisionExpr and it is not already a commit id.
+ project_gitcmds = [(
+ project, git_command.GitCommand(None,
+ ['ls-remote',
+ project.remote.url,
+ project.revisionExpr],
+ capture_stdout=True, cwd='/tmp'))
+ for project in projects if not git_config.IsId(project.revisionExpr)]
+ for proj, gitcmd in project_gitcmds:
+ if gitcmd.Wait():
+ print('FATAL: Failed to retrieve revisionExpr for %s' % project)
+ sys.exit(1)
+ proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def generate_gitc_manifest(client_dir, manifest):
+ """Generate a manifest for shafsd to use for this GITC client.
+
+ @param client_dir: GITC client directory to install the .manifest file in.
+ @param manifest: XmlManifest object representing the repo manifest.
+ """
+ print('Generating GITC Manifest by fetching revision SHAs for each '
+ 'project.')
+ project_gitcmd_dict = {}
+ index = 0
+ while index < len(manifest.projects):
+ _set_project_revisions(
+ manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+ index += NUM_BATCH_RETRIEVE_REVISIONID
+ # Save the manifest.
+ with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+ manifest.Save(f)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e71960..6dc01a4 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -303,6 +303,9 @@
if p.sync_s:
e.setAttribute('sync-s', 'true')
+ if p.clone_depth:
+ e.setAttribute('clone-depth', str(p.clone_depth))
+
if p.subprojects:
subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, list(sorted(subprojects)))
diff --git a/project.py b/project.py
index c32c1f5..24cac8b 100644
--- a/project.py
+++ b/project.py
@@ -2147,8 +2147,8 @@
try:
self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
except GitError as e:
- print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
if force_sync:
+ print("Retrying clone after deleting %s" % self.gitdir, file=sys.stderr)
try:
shutil.rmtree(os.path.realpath(self.gitdir))
if self.worktree and os.path.exists(
@@ -2285,7 +2285,10 @@
# Fail if the links are pointing to the wrong place
if src != dst:
raise GitError('--force-sync not enabled; cannot overwrite a local '
- 'work tree')
+ 'work tree. If you\'re comfortable with the '
+ 'possibility of losing the work tree\'s git metadata,'
+ ' use `repo sync --force-sync {0}` to '
+ 'proceed.'.format(self.relpath))
def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
"""Update |dotgit| to reference |gitdir|, using symlinks where possible.
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 9b9cefd..03d8cc3 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -18,15 +18,10 @@
import shutil
import sys
-import git_command
+import gitc_utils
from subcmds import init
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
-GITC_FS_ROOT_DIR = '/gitc/sha/rw'
-NUM_BATCH_RETRIEVE_REVISIONID = 300
-
-
class GitcInit(init.Init):
common = True
helpSummary = "Initialize a GITC Client."
@@ -65,59 +60,21 @@
if not opt.gitc_client:
print('fatal: gitc client (-c) is required', file=sys.stderr)
sys.exit(1)
- self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
- if not os.path.exists(GITC_MANIFEST_DIR):
- os.makedirs(GITC_MANIFEST_DIR)
+ self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+ opt.gitc_client)
+ if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR):
+ os.makedirs(gitc_utils.GITC_MANIFEST_DIR)
if not os.path.exists(self.client_dir):
os.mkdir(self.client_dir)
super(GitcInit, self).Execute(opt, args)
+ # Make the destination manifest file a symlink to repo's so both repo and
+ # GITC refer to the same manifest.
if opt.manifest_file:
if not os.path.exists(opt.manifest_file):
print('fatal: Specified manifest file %s does not exist.' %
opt.manifest_file)
sys.exit(1)
- shutil.copyfile(opt.manifest_file,
- os.path.join(self.client_dir, '.manifest'))
- else:
- self._GenerateGITCManifest()
+ self.manifest.Override(opt.manifest_file)
+ gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest)
print('Please run `cd %s` to view your GITC client.' %
- os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client))
-
- def _SetProjectRevisions(self, projects, branch):
- """Sets the revisionExpr for a list of projects.
-
- Because of the limit of open file descriptors allowed, length of projects
- should not be overly large. Recommend calling this function multiple times
- with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
-
- @param projects: List of project objects to set the revionExpr for.
- @param branch: The remote branch to retrieve the SHA from. If branch is
- None, 'HEAD' is used.
- """
- project_gitcmds = [(
- project, git_command.GitCommand(None,
- ['ls-remote',
- project.remote.url,
- branch], capture_stdout=True))
- for project in projects]
- for proj, gitcmd in project_gitcmds:
- if gitcmd.Wait():
- print('FATAL: Failed to retrieve revisionID for %s' % project)
- sys.exit(1)
- proj.revisionExpr = gitcmd.stdout.split('\t')[0]
-
- def _GenerateGITCManifest(self):
- """Generate a manifest for shafsd to use for this GITC client."""
- print('Generating GITC Manifest by fetching revision SHAs for each '
- 'project.')
- manifest = self.manifest
- project_gitcmd_dict = {}
- index = 0
- while index < len(manifest.projects):
- self._SetProjectRevisions(
- manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)],
- manifest.default.revisionExpr)
- index += NUM_BATCH_RETRIEVE_REVISIONID
- # Save the manifest.
- with open(os.path.join(self.client_dir, '.manifest'), 'w') as f:
- manifest.Save(f)
+ os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
\ No newline at end of file
diff --git a/subcmds/sync.py b/subcmds/sync.py
index ed8622c..0fc493c 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -67,6 +67,7 @@
from git_command import GIT, git_require
from git_config import GetSchemeFromUrl, GetUrlCookieFile
from git_refs import R_HEADS, HEAD
+import gitc_utils
from project import Project
from project import RemoteSpec
from command import Command, MirrorSafeCommand
@@ -193,6 +194,9 @@
help="overwrite an existing git directory if it needs to "
"point to a different object directory. WARNING: this "
"may cause loss of data")
+ p.add_option('--force-gitc',
+ dest='force_gitc', action='store_true',
+ help="actually sync sources in the gitc client directory.")
p.add_option('-l', '--local-only',
dest='local_only', action='store_true',
help="only update working tree, don't fetch")
@@ -535,6 +539,25 @@
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
+ cwd = os.getcwd()
+ if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc:
+ print('WARNING this will pull all the sources like a normal repo sync.\n'
+ '\nIf you want to update your GITC Client View please rerun this '
+ 'command in \n%s%s.\nOr if you actually want to pull the sources, '
+ 'rerun with --force-gitc.' %
+ (gitc_utils.GITC_FS_ROOT_DIR,
+ cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1]))
+ sys.exit(1)
+
+ self._gitc_sync = False
+ if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR):
+ self._gitc_sync = True
+ self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split(
+ '/')[0]
+ self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+ self._client_name)
+ print('Updating GITC client: %s' % self._client_name)
+
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
@@ -655,6 +678,12 @@
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
+ if self._gitc_sync:
+ gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest)
+ print('GITC client successfully synced.')
+ return
+
+
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
