| # |
| # Copyright (C) 2008 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import os |
| import random |
| import stat |
| import sys |
| import urllib2 |
| import StringIO |
| |
| from error import GitError, ImportError |
| from git_command import GitCommand |
| |
| class ImportExternal(object): |
| """Imports a single revision from a non-git data source. |
| Suitable for use to import a tar or zip based snapshot. |
| """ |
| def __init__(self): |
| self._marks = 0 |
| self._files = {} |
| self._tempref = 'refs/repo-external/import' |
| |
| self._urls = [] |
| self._remap = [] |
| self.parent = None |
| self._user_name = 'Upstream' |
| self._user_email = 'upstream-import@none' |
| self._user_when = 1000000 |
| |
| self.commit = None |
| |
| def Clone(self): |
| r = self.__class__() |
| |
| r.project = self.project |
| for u in self._urls: |
| r._urls.append(u) |
| for p in self._remap: |
| r._remap.append(_PathMap(r, p._old, p._new)) |
| |
| return r |
| |
| def SetProject(self, project): |
| self.project = project |
| |
| def SetVersion(self, version): |
| self.version = version |
| |
| def AddUrl(self, url): |
| self._urls.append(url) |
| |
| def SetParent(self, commit_hash): |
| self.parent = commit_hash |
| |
| def SetCommit(self, commit_hash): |
| self.commit = commit_hash |
| |
| def RemapPath(self, old, new, replace_version=True): |
| self._remap.append(_PathMap(self, old, new)) |
| |
| @property |
| def TagName(self): |
| v = '' |
| for c in self.version: |
| if c >= '0' and c <= '9': |
| v += c |
| elif c >= 'A' and c <= 'Z': |
| v += c |
| elif c >= 'a' and c <= 'z': |
| v += c |
| elif c in ('-', '_', '.', '/', '+', '@'): |
| v += c |
| return 'upstream/%s' % v |
| |
| @property |
| def PackageName(self): |
| n = self.project.name |
| if n.startswith('platform/'): |
| # This was not my finest moment... |
| # |
| n = n[len('platform/'):] |
| return n |
| |
| def Import(self): |
| self._need_graft = False |
| if self.parent: |
| try: |
| self.project.bare_git.cat_file('-e', self.parent) |
| except GitError: |
| self._need_graft = True |
| |
| gfi = GitCommand(self.project, |
| ['fast-import', '--force', '--quiet'], |
| bare = True, |
| provide_stdin = True) |
| try: |
| self._out = gfi.stdin |
| |
| try: |
| self._UnpackFiles() |
| self._MakeCommit() |
| self._out.flush() |
| finally: |
| rc = gfi.Wait() |
| if rc != 0: |
| raise ImportError('fast-import failed') |
| |
| if self._need_graft: |
| id = self._GraftCommit() |
| else: |
| id = self.project.bare_git.rev_parse('%s^0' % self._tempref) |
| |
| if self.commit and self.commit != id: |
| raise ImportError('checksum mismatch: %s expected,' |
| ' %s imported' % (self.commit, id)) |
| |
| self._MakeTag(id) |
| return id |
| finally: |
| try: |
| self.project.bare_git.DeleteRef(self._tempref) |
| except GitError: |
| pass |
| |
| def _PickUrl(self, failed): |
| u = map(lambda x: x.replace('%version%', self.version), self._urls) |
| for f in failed: |
| if f in u: |
| u.remove(f) |
| if len(u) == 0: |
| return None |
| return random.choice(u) |
| |
| def _OpenUrl(self): |
| failed = {} |
| while True: |
| url = self._PickUrl(failed.keys()) |
| if url is None: |
| why = 'Cannot download %s' % self.project.name |
| |
| if failed: |
| why += ': one or more mirrors are down\n' |
| bad_urls = list(failed.keys()) |
| bad_urls.sort() |
| for url in bad_urls: |
| why += ' %s: %s\n' % (url, failed[url]) |
| else: |
| why += ': no mirror URLs' |
| raise ImportError(why) |
| |
| print >>sys.stderr, "Getting %s ..." % url |
| try: |
| return urllib2.urlopen(url), url |
| except urllib2.HTTPError, e: |
| failed[url] = e.code |
| except urllib2.URLError, e: |
| failed[url] = e.reason[1] |
| except OSError, e: |
| failed[url] = e.strerror |
| |
| def _UnpackFiles(self): |
| raise NotImplementedError |
| |
| def _NextMark(self): |
| self._marks += 1 |
| return self._marks |
| |
| def _UnpackOneFile(self, mode, size, name, fd): |
| if stat.S_ISDIR(mode): # directory |
| return |
| else: |
| mode = self._CleanMode(mode, name) |
| |
| old_name = name |
| name = self._CleanName(name) |
| |
| if stat.S_ISLNK(mode) and self._remap: |
| # The link is relative to the old_name, and may need to |
| # be rewritten according to our remap rules if it goes |
| # up high enough in the tree structure. |
| # |
| dest = self._RewriteLink(fd.read(size), old_name, name) |
| fd = StringIO.StringIO(dest) |
| size = len(dest) |
| |
| fi = _File(mode, name, self._NextMark()) |
| |
| self._out.write('blob\n') |
| self._out.write('mark :%d\n' % fi.mark) |
| self._out.write('data %d\n' % size) |
| while size > 0: |
| n = min(2048, size) |
| self._out.write(fd.read(n)) |
| size -= n |
| self._out.write('\n') |
| self._files[fi.name] = fi |
| |
| def _SetFileMode(self, name, mode): |
| if not stat.S_ISDIR(mode): |
| mode = self._CleanMode(mode, name) |
| name = self._CleanName(name) |
| try: |
| fi = self._files[name] |
| except KeyError: |
| raise ImportError('file %s was not unpacked' % name) |
| fi.mode = mode |
| |
| def _RewriteLink(self, dest, relto_old, relto_new): |
| # Drop the last components of the symlink itself |
| # as the dest is relative to the directory its in. |
| # |
| relto_old = _TrimPath(relto_old) |
| relto_new = _TrimPath(relto_new) |
| |
| # Resolve the link to be absolute from the top of |
| # the archive, so we can remap its destination. |
| # |
| while dest.find('/./') >= 0 or dest.find('//') >= 0: |
| dest = dest.replace('/./', '/') |
| dest = dest.replace('//', '/') |
| |
| if dest.startswith('../') or dest.find('/../') > 0: |
| dest = _FoldPath('%s/%s' % (relto_old, dest)) |
| |
| for pm in self._remap: |
| if pm.Matches(dest): |
| dest = pm.Apply(dest) |
| break |
| |
| dest, relto_new = _StripCommonPrefix(dest, relto_new) |
| while relto_new: |
| i = relto_new.find('/') |
| if i > 0: |
| relto_new = relto_new[i + 1:] |
| else: |
| relto_new = '' |
| dest = '../' + dest |
| return dest |
| |
| def _CleanMode(self, mode, name): |
| if stat.S_ISREG(mode): # regular file |
| if (mode & 0111) == 0: |
| return 0644 |
| else: |
| return 0755 |
| elif stat.S_ISLNK(mode): # symlink |
| return stat.S_IFLNK |
| else: |
| raise ImportError('invalid mode %o in %s' % (mode, name)) |
| |
| def _CleanName(self, name): |
| old_name = name |
| for pm in self._remap: |
| if pm.Matches(name): |
| name = pm.Apply(name) |
| break |
| while name.startswith('/'): |
| name = name[1:] |
| if not name: |
| raise ImportError('path %s is empty after remap' % old_name) |
| if name.find('/./') >= 0 or name.find('/../') >= 0: |
| raise ImportError('path %s contains relative parts' % name) |
| return name |
| |
| def _MakeCommit(self): |
| msg = '%s %s\n' % (self.PackageName, self.version) |
| |
| self._out.write('commit %s\n' % self._tempref) |
| self._out.write('committer %s <%s> %d +0000\n' % ( |
| self._user_name, |
| self._user_email, |
| self._user_when)) |
| self._out.write('data %d\n' % len(msg)) |
| self._out.write(msg) |
| self._out.write('\n') |
| if self.parent and not self._need_graft: |
| self._out.write('from %s^0\n' % self.parent) |
| self._out.write('deleteall\n') |
| |
| for f in self._files.values(): |
| self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name)) |
| self._out.write('\n') |
| |
| def _GraftCommit(self): |
| raw = self.project.bare_git.cat_file('commit', self._tempref) |
| raw = raw.split("\n") |
| while raw[1].startswith('parent '): |
| del raw[1] |
| raw.insert(1, 'parent %s' % self.parent) |
| id = self._WriteObject('commit', "\n".join(raw)) |
| |
| graft_file = os.path.join(self.project.gitdir, 'info/grafts') |
| if os.path.exists(graft_file): |
| graft_list = open(graft_file, 'rb').read().split("\n") |
| if graft_list and graft_list[-1] == '': |
| del graft_list[-1] |
| else: |
| graft_list = [] |
| |
| exists = False |
| for line in graft_list: |
| if line == id: |
| exists = True |
| break |
| |
| if not exists: |
| graft_list.append(id) |
| graft_list.append('') |
| fd = open(graft_file, 'wb') |
| fd.write("\n".join(graft_list)) |
| fd.close() |
| |
| return id |
| |
| def _MakeTag(self, id): |
| name = self.TagName |
| |
| raw = [] |
| raw.append('object %s' % id) |
| raw.append('type commit') |
| raw.append('tag %s' % name) |
| raw.append('tagger %s <%s> %d +0000' % ( |
| self._user_name, |
| self._user_email, |
| self._user_when)) |
| raw.append('') |
| raw.append('%s %s\n' % (self.PackageName, self.version)) |
| |
| tagid = self._WriteObject('tag', "\n".join(raw)) |
| self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid) |
| |
| def _WriteObject(self, type, data): |
| wo = GitCommand(self.project, |
| ['hash-object', '-t', type, '-w', '--stdin'], |
| bare = True, |
| provide_stdin = True, |
| capture_stdout = True, |
| capture_stderr = True) |
| wo.stdin.write(data) |
| if wo.Wait() != 0: |
| raise GitError('cannot create %s from (%s)' % (type, data)) |
| return wo.stdout[:-1] |
| |
| |
| def _TrimPath(path): |
| i = path.rfind('/') |
| if i > 0: |
| path = path[0:i] |
| return '' |
| |
| def _StripCommonPrefix(a, b): |
| while True: |
| ai = a.find('/') |
| bi = b.find('/') |
| if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]: |
| a = a[ai + 1:] |
| b = b[bi + 1:] |
| else: |
| break |
| return a, b |
| |
| def _FoldPath(path): |
| while True: |
| if path.startswith('../'): |
| return path |
| |
| i = path.find('/../') |
| if i <= 0: |
| if path.startswith('/'): |
| return path[1:] |
| return path |
| |
| lhs = path[0:i] |
| rhs = path[i + 4:] |
| |
| i = lhs.rfind('/') |
| if i > 0: |
| path = lhs[0:i + 1] + rhs |
| else: |
| path = rhs |
| |
| class _File(object): |
| def __init__(self, mode, name, mark): |
| self.mode = mode |
| self.name = name |
| self.mark = mark |
| |
| |
| class _PathMap(object): |
| def __init__(self, imp, old, new): |
| self._imp = imp |
| self._old = old |
| self._new = new |
| |
| def _r(self, p): |
| return p.replace('%version%', self._imp.version) |
| |
| @property |
| def old(self): |
| return self._r(self._old) |
| |
| @property |
| def new(self): |
| return self._r(self._new) |
| |
| def Matches(self, name): |
| return name.startswith(self.old) |
| |
| def Apply(self, name): |
| return self.new + name[len(self.old):] |
