Remove import_tar, import_zip and the <snapshot> elements
Now that repo relies only on the git data stream (as it is much
faster to download through) we don't really need to be parsing the
<snapshot> elements within manifest. Its a lot of complex code to
convert the tar (or zip) through to a fast import stream, and we
just aren't calling it anymore.
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/import_ext.py b/import_ext.py
deleted file mode 100644
index 2a1ebf8..0000000
--- a/import_ext.py
+++ /dev/null
@@ -1,422 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import random
-import stat
-import sys
-import urllib2
-import StringIO
-
-from error import GitError, ImportError
-from git_command import GitCommand
-
-class ImportExternal(object):
- """Imports a single revision from a non-git data source.
- Suitable for use to import a tar or zip based snapshot.
- """
- def __init__(self):
- self._marks = 0
- self._files = {}
- self._tempref = 'refs/repo-external/import'
-
- self._urls = []
- self._remap = []
- self.parent = None
- self._user_name = 'Upstream'
- self._user_email = 'upstream-import@none'
- self._user_when = 1000000
-
- self.commit = None
-
- def Clone(self):
- r = self.__class__()
-
- r.project = self.project
- for u in self._urls:
- r._urls.append(u)
- for p in self._remap:
- r._remap.append(_PathMap(r, p._old, p._new))
-
- return r
-
- def SetProject(self, project):
- self.project = project
-
- def SetVersion(self, version):
- self.version = version
-
- def AddUrl(self, url):
- self._urls.append(url)
-
- def SetParent(self, commit_hash):
- self.parent = commit_hash
-
- def SetCommit(self, commit_hash):
- self.commit = commit_hash
-
- def RemapPath(self, old, new, replace_version=True):
- self._remap.append(_PathMap(self, old, new))
-
- @property
- def TagName(self):
- v = ''
- for c in self.version:
- if c >= '0' and c <= '9':
- v += c
- elif c >= 'A' and c <= 'Z':
- v += c
- elif c >= 'a' and c <= 'z':
- v += c
- elif c in ('-', '_', '.', '/', '+', '@'):
- v += c
- return 'upstream/%s' % v
-
- @property
- def PackageName(self):
- n = self.project.name
- if n.startswith('platform/'):
- # This was not my finest moment...
- #
- n = n[len('platform/'):]
- return n
-
- def Import(self):
- self._need_graft = False
- if self.parent:
- try:
- self.project.bare_git.cat_file('-e', self.parent)
- except GitError:
- self._need_graft = True
-
- gfi = GitCommand(self.project,
- ['fast-import', '--force', '--quiet'],
- bare = True,
- provide_stdin = True)
- try:
- self._out = gfi.stdin
-
- try:
- self._UnpackFiles()
- self._MakeCommit()
- self._out.flush()
- finally:
- rc = gfi.Wait()
- if rc != 0:
- raise ImportError('fast-import failed')
-
- if self._need_graft:
- id = self._GraftCommit()
- else:
- id = self.project.bare_git.rev_parse('%s^0' % self._tempref)
-
- if self.commit and self.commit != id:
- raise ImportError('checksum mismatch: %s expected,'
- ' %s imported' % (self.commit, id))
-
- self._MakeTag(id)
- return id
- finally:
- try:
- self.project.bare_git.DeleteRef(self._tempref)
- except GitError:
- pass
-
- def _PickUrl(self, failed):
- u = map(lambda x: x.replace('%version%', self.version), self._urls)
- for f in failed:
- if f in u:
- u.remove(f)
- if len(u) == 0:
- return None
- return random.choice(u)
-
- def _OpenUrl(self):
- failed = {}
- while True:
- url = self._PickUrl(failed.keys())
- if url is None:
- why = 'Cannot download %s' % self.project.name
-
- if failed:
- why += ': one or more mirrors are down\n'
- bad_urls = list(failed.keys())
- bad_urls.sort()
- for url in bad_urls:
- why += ' %s: %s\n' % (url, failed[url])
- else:
- why += ': no mirror URLs'
- raise ImportError(why)
-
- print >>sys.stderr, "Getting %s ..." % url
- try:
- return urllib2.urlopen(url), url
- except urllib2.HTTPError, e:
- failed[url] = e.code
- except urllib2.URLError, e:
- failed[url] = e.reason[1]
- except OSError, e:
- failed[url] = e.strerror
-
- def _UnpackFiles(self):
- raise NotImplementedError
-
- def _NextMark(self):
- self._marks += 1
- return self._marks
-
- def _UnpackOneFile(self, mode, size, name, fd):
- if stat.S_ISDIR(mode): # directory
- return
- else:
- mode = self._CleanMode(mode, name)
-
- old_name = name
- name = self._CleanName(name)
-
- if stat.S_ISLNK(mode) and self._remap:
- # The link is relative to the old_name, and may need to
- # be rewritten according to our remap rules if it goes
- # up high enough in the tree structure.
- #
- dest = self._RewriteLink(fd.read(size), old_name, name)
- fd = StringIO.StringIO(dest)
- size = len(dest)
-
- fi = _File(mode, name, self._NextMark())
-
- self._out.write('blob\n')
- self._out.write('mark :%d\n' % fi.mark)
- self._out.write('data %d\n' % size)
- while size > 0:
- n = min(2048, size)
- self._out.write(fd.read(n))
- size -= n
- self._out.write('\n')
- self._files[fi.name] = fi
-
- def _SetFileMode(self, name, mode):
- if not stat.S_ISDIR(mode):
- mode = self._CleanMode(mode, name)
- name = self._CleanName(name)
- try:
- fi = self._files[name]
- except KeyError:
- raise ImportError('file %s was not unpacked' % name)
- fi.mode = mode
-
- def _RewriteLink(self, dest, relto_old, relto_new):
- # Drop the last components of the symlink itself
- # as the dest is relative to the directory its in.
- #
- relto_old = _TrimPath(relto_old)
- relto_new = _TrimPath(relto_new)
-
- # Resolve the link to be absolute from the top of
- # the archive, so we can remap its destination.
- #
- while dest.find('/./') >= 0 or dest.find('//') >= 0:
- dest = dest.replace('/./', '/')
- dest = dest.replace('//', '/')
-
- if dest.startswith('../') or dest.find('/../') > 0:
- dest = _FoldPath('%s/%s' % (relto_old, dest))
-
- for pm in self._remap:
- if pm.Matches(dest):
- dest = pm.Apply(dest)
- break
-
- dest, relto_new = _StripCommonPrefix(dest, relto_new)
- while relto_new:
- i = relto_new.find('/')
- if i > 0:
- relto_new = relto_new[i + 1:]
- else:
- relto_new = ''
- dest = '../' + dest
- return dest
-
- def _CleanMode(self, mode, name):
- if stat.S_ISREG(mode): # regular file
- if (mode & 0111) == 0:
- return 0644
- else:
- return 0755
- elif stat.S_ISLNK(mode): # symlink
- return stat.S_IFLNK
- else:
- raise ImportError('invalid mode %o in %s' % (mode, name))
-
- def _CleanName(self, name):
- old_name = name
- for pm in self._remap:
- if pm.Matches(name):
- name = pm.Apply(name)
- break
- while name.startswith('/'):
- name = name[1:]
- if not name:
- raise ImportError('path %s is empty after remap' % old_name)
- if name.find('/./') >= 0 or name.find('/../') >= 0:
- raise ImportError('path %s contains relative parts' % name)
- return name
-
- def _MakeCommit(self):
- msg = '%s %s\n' % (self.PackageName, self.version)
-
- self._out.write('commit %s\n' % self._tempref)
- self._out.write('committer %s <%s> %d +0000\n' % (
- self._user_name,
- self._user_email,
- self._user_when))
- self._out.write('data %d\n' % len(msg))
- self._out.write(msg)
- self._out.write('\n')
- if self.parent and not self._need_graft:
- self._out.write('from %s^0\n' % self.parent)
- self._out.write('deleteall\n')
-
- for f in self._files.values():
- self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name))
- self._out.write('\n')
-
- def _GraftCommit(self):
- raw = self.project.bare_git.cat_file('commit', self._tempref)
- raw = raw.split("\n")
- while raw[1].startswith('parent '):
- del raw[1]
- raw.insert(1, 'parent %s' % self.parent)
- id = self._WriteObject('commit', "\n".join(raw))
-
- graft_file = os.path.join(self.project.gitdir, 'info/grafts')
- if os.path.exists(graft_file):
- graft_list = open(graft_file, 'rb').read().split("\n")
- if graft_list and graft_list[-1] == '':
- del graft_list[-1]
- else:
- graft_list = []
-
- exists = False
- for line in graft_list:
- if line == id:
- exists = True
- break
-
- if not exists:
- graft_list.append(id)
- graft_list.append('')
- fd = open(graft_file, 'wb')
- fd.write("\n".join(graft_list))
- fd.close()
-
- return id
-
- def _MakeTag(self, id):
- name = self.TagName
-
- raw = []
- raw.append('object %s' % id)
- raw.append('type commit')
- raw.append('tag %s' % name)
- raw.append('tagger %s <%s> %d +0000' % (
- self._user_name,
- self._user_email,
- self._user_when))
- raw.append('')
- raw.append('%s %s\n' % (self.PackageName, self.version))
-
- tagid = self._WriteObject('tag', "\n".join(raw))
- self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid)
-
- def _WriteObject(self, type, data):
- wo = GitCommand(self.project,
- ['hash-object', '-t', type, '-w', '--stdin'],
- bare = True,
- provide_stdin = True,
- capture_stdout = True,
- capture_stderr = True)
- wo.stdin.write(data)
- if wo.Wait() != 0:
- raise GitError('cannot create %s from (%s)' % (type, data))
- return wo.stdout[:-1]
-
-
-def _TrimPath(path):
- i = path.rfind('/')
- if i > 0:
- path = path[0:i]
- return ''
-
-def _StripCommonPrefix(a, b):
- while True:
- ai = a.find('/')
- bi = b.find('/')
- if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]:
- a = a[ai + 1:]
- b = b[bi + 1:]
- else:
- break
- return a, b
-
-def _FoldPath(path):
- while True:
- if path.startswith('../'):
- return path
-
- i = path.find('/../')
- if i <= 0:
- if path.startswith('/'):
- return path[1:]
- return path
-
- lhs = path[0:i]
- rhs = path[i + 4:]
-
- i = lhs.rfind('/')
- if i > 0:
- path = lhs[0:i + 1] + rhs
- else:
- path = rhs
-
-class _File(object):
- def __init__(self, mode, name, mark):
- self.mode = mode
- self.name = name
- self.mark = mark
-
-
-class _PathMap(object):
- def __init__(self, imp, old, new):
- self._imp = imp
- self._old = old
- self._new = new
-
- def _r(self, p):
- return p.replace('%version%', self._imp.version)
-
- @property
- def old(self):
- return self._r(self._old)
-
- @property
- def new(self):
- return self._r(self._new)
-
- def Matches(self, name):
- return name.startswith(self.old)
-
- def Apply(self, name):
- return self.new + name[len(self.old):]
diff --git a/import_tar.py b/import_tar.py
deleted file mode 100644
index d7ce14d..0000000
--- a/import_tar.py
+++ /dev/null
@@ -1,206 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import bz2
-import stat
-import tarfile
-import zlib
-import StringIO
-
-from import_ext import ImportExternal
-from error import ImportError
-
-class ImportTar(ImportExternal):
- """Streams a (optionally compressed) tar file from the network
- directly into a Project's Git repository.
- """
- @classmethod
- def CanAccept(cls, url):
- """Can this importer read and unpack the data stored at url?
- """
- if url.endswith('.tar.gz') or url.endswith('.tgz'):
- return True
- if url.endswith('.tar.bz2'):
- return True
- if url.endswith('.tar'):
- return True
- return False
-
- def _UnpackFiles(self):
- url_fd, url = self._OpenUrl()
- try:
- if url.endswith('.tar.gz') or url.endswith('.tgz'):
- tar_fd = _Gzip(url_fd)
- elif url.endswith('.tar.bz2'):
- tar_fd = _Bzip2(url_fd)
- elif url.endswith('.tar'):
- tar_fd = _Raw(url_fd)
- else:
- raise ImportError('non-tar file extension: %s' % url)
-
- try:
- tar = tarfile.TarFile(name = url,
- mode = 'r',
- fileobj = tar_fd)
- try:
- for entry in tar:
- mode = entry.mode
-
- if (mode & 0170000) == 0:
- if entry.isdir():
- mode |= stat.S_IFDIR
- elif entry.isfile() or entry.islnk(): # hard links as files
- mode |= stat.S_IFREG
- elif entry.issym():
- mode |= stat.S_IFLNK
-
- if stat.S_ISLNK(mode): # symlink
- data_fd = StringIO.StringIO(entry.linkname)
- data_sz = len(entry.linkname)
- elif stat.S_ISDIR(mode): # directory
- data_fd = StringIO.StringIO('')
- data_sz = 0
- else:
- data_fd = tar.extractfile(entry)
- data_sz = entry.size
-
- self._UnpackOneFile(mode, data_sz, entry.name, data_fd)
- finally:
- tar.close()
- finally:
- tar_fd.close()
- finally:
- url_fd.close()
-
-
-
-class _DecompressStream(object):
- """file like object to decompress a tar stream
- """
- def __init__(self, fd):
- self._fd = fd
- self._pos = 0
- self._buf = None
-
- def tell(self):
- return self._pos
-
- def seek(self, offset):
- d = offset - self._pos
- if d > 0:
- self.read(d)
- elif d == 0:
- pass
- else:
- raise NotImplementedError, 'seek backwards'
-
- def close(self):
- self._fd = None
-
- def read(self, size = -1):
- if not self._fd:
- raise EOFError, 'Reached EOF'
-
- r = []
- try:
- if size >= 0:
- self._ReadChunk(r, size)
- else:
- while True:
- self._ReadChunk(r, 2048)
- except EOFError:
- pass
-
- if len(r) == 1:
- r = r[0]
- else:
- r = ''.join(r)
- self._pos += len(r)
- return r
-
- def _ReadChunk(self, r, size):
- b = self._buf
- try:
- while size > 0:
- if b is None or len(b) == 0:
- b = self._Decompress(self._fd.read(2048))
- continue
-
- use = min(size, len(b))
- r.append(b[:use])
- b = b[use:]
- size -= use
- finally:
- self._buf = b
-
- def _Decompress(self, b):
- raise NotImplementedError, '_Decompress'
-
-
-class _Raw(_DecompressStream):
- """file like object for an uncompressed stream
- """
- def __init__(self, fd):
- _DecompressStream.__init__(self, fd)
-
- def _Decompress(self, b):
- return b
-
-
-class _Bzip2(_DecompressStream):
- """file like object to decompress a .bz2 stream
- """
- def __init__(self, fd):
- _DecompressStream.__init__(self, fd)
- self._bz = bz2.BZ2Decompressor()
-
- def _Decompress(self, b):
- return self._bz.decompress(b)
-
-
-_FHCRC, _FEXTRA, _FNAME, _FCOMMENT = 2, 4, 8, 16
-class _Gzip(_DecompressStream):
- """file like object to decompress a .gz stream
- """
- def __init__(self, fd):
- _DecompressStream.__init__(self, fd)
- self._z = zlib.decompressobj(-zlib.MAX_WBITS)
-
- magic = fd.read(2)
- if magic != '\037\213':
- raise IOError, 'Not a gzipped file'
-
- method = ord(fd.read(1))
- if method != 8:
- raise IOError, 'Unknown compression method'
-
- flag = ord(fd.read(1))
- fd.read(6)
-
- if flag & _FEXTRA:
- xlen = ord(fd.read(1))
- xlen += 256 * ord(fd.read(1))
- fd.read(xlen)
- if flag & _FNAME:
- while fd.read(1) != '\0':
- pass
- if flag & _FCOMMENT:
- while fd.read(1) != '\0':
- pass
- if flag & _FHCRC:
- fd.read(2)
-
- def _Decompress(self, b):
- return self._z.decompress(b)
diff --git a/import_zip.py b/import_zip.py
deleted file mode 100644
index 08aff32..0000000
--- a/import_zip.py
+++ /dev/null
@@ -1,345 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import stat
-import struct
-import zlib
-import cStringIO
-
-from import_ext import ImportExternal
-from error import ImportError
-
-class ImportZip(ImportExternal):
- """Streams a zip file from the network directly into a Project's
- Git repository.
- """
- @classmethod
- def CanAccept(cls, url):
- """Can this importer read and unpack the data stored at url?
- """
- if url.endswith('.zip') or url.endswith('.jar'):
- return True
- return False
-
- def _UnpackFiles(self):
- url_fd, url = self._OpenUrl()
- try:
- if not self.__class__.CanAccept(url):
- raise ImportError('non-zip file extension: %s' % url)
-
- zip = _ZipFile(url_fd)
- for entry in zip.FileRecords():
- data = zip.Open(entry).read()
- sz = len(data)
-
- if data and _SafeCRLF(data):
- data = data.replace('\r\n', '\n')
- sz = len(data)
-
- fd = cStringIO.StringIO(data)
- self._UnpackOneFile(entry.mode, sz, entry.name, fd)
- zip.Close(entry)
-
- for entry in zip.CentralDirectory():
- self._SetFileMode(entry.name, entry.mode)
-
- zip.CheckTail()
- finally:
- url_fd.close()
-
-
-def _SafeCRLF(data):
- """Is it reasonably safe to perform a CRLF->LF conversion?
-
- If the stream contains a NUL byte it is likely binary,
- and thus a CRLF->LF conversion may damage the stream.
-
- If the only NUL is in the last position of the stream,
- but it otherwise can do a CRLF<->LF conversion we do
- the CRLF conversion anyway. At least one source ZIP
- file has this structure in its source code.
-
- If every occurrance of a CR and LF is paired up as a
- CRLF pair then the conversion is safely bi-directional.
- s/\r\n/\n/g == s/\n/\r\\n/g can convert between them.
- """
- nul = data.find('\0')
- if 0 <= nul and nul < (len(data) - 1):
- return False
-
- n_lf = 0
- last = 0
- while True:
- lf = data.find('\n', last)
- if lf < 0:
- break
- if lf == 0 or data[lf - 1] != '\r':
- return False
- last = lf + 1
- n_lf += 1
- return n_lf > 0
-
-class _ZipFile(object):
- """Streaming iterator to parse a zip file on the fly.
- """
- def __init__(self, fd):
- self._fd = _UngetStream(fd)
-
- def FileRecords(self):
- return _FileIter(self._fd)
-
- def CentralDirectory(self):
- return _CentIter(self._fd)
-
- def CheckTail(self):
- type_buf = self._fd.read(4)
- type = struct.unpack('<I', type_buf)[0]
- if type != 0x06054b50: # end of central directory
- raise ImportError('zip record %x unsupported' % type)
-
- def Open(self, entry):
- if entry.is_compressed:
- return _InflateStream(self._fd)
- else:
- if entry.has_trailer:
- raise ImportError('unable to extract streamed zip')
- return _FixedLengthStream(self._fd, entry.uncompressed_size)
-
- def Close(self, entry):
- if entry.has_trailer:
- type = struct.unpack('<I', self._fd.read(4))[0]
- if type == 0x08074b50:
- # Not a formal type marker, but commonly seen in zips
- # as the data descriptor signature.
- #
- struct.unpack('<3I', self._fd.read(12))
- else:
- # No signature for the data descriptor, so read the
- # remaining fields out of the stream
- #
- self._fd.read(8)
-
-
-class _FileIter(object):
- def __init__(self, fd):
- self._fd = fd
-
- def __iter__(self):
- return self
-
- def next(self):
- fd = self._fd
-
- type_buf = fd.read(4)
- type = struct.unpack('<I', type_buf)[0]
-
- if type != 0x04034b50: # local file header
- fd.unread(type_buf)
- raise StopIteration()
-
- rec = _FileHeader(fd.read(26))
- rec.name = fd.read(rec.name_len)
- fd.read(rec.extra_len)
-
- if rec.name.endswith('/'):
- rec.name = rec.name[:-1]
- rec.mode = stat.S_IFDIR | 0777
- return rec
-
-
-class _FileHeader(object):
- """Information about a single file in the archive.
- 0 version needed to extract 2 bytes
- 1 general purpose bit flag 2 bytes
- 2 compression method 2 bytes
- 3 last mod file time 2 bytes
- 4 last mod file date 2 bytes
- 5 crc-32 4 bytes
- 6 compressed size 4 bytes
- 7 uncompressed size 4 bytes
- 8 file name length 2 bytes
- 9 extra field length 2 bytes
- """
- def __init__(self, raw_bin):
- rec = struct.unpack('<5H3I2H', raw_bin)
-
- if rec[2] == 8:
- self.is_compressed = True
- elif rec[2] == 0:
- self.is_compressed = False
- else:
- raise ImportError('unrecognized compression format')
-
- if rec[1] & (1 << 3):
- self.has_trailer = True
- else:
- self.has_trailer = False
-
- self.compressed_size = rec[6]
- self.uncompressed_size = rec[7]
- self.name_len = rec[8]
- self.extra_len = rec[9]
- self.mode = stat.S_IFREG | 0644
-
-
-class _CentIter(object):
- def __init__(self, fd):
- self._fd = fd
-
- def __iter__(self):
- return self
-
- def next(self):
- fd = self._fd
-
- type_buf = fd.read(4)
- type = struct.unpack('<I', type_buf)[0]
-
- if type != 0x02014b50: # central directory
- fd.unread(type_buf)
- raise StopIteration()
-
- rec = _CentHeader(fd.read(42))
- rec.name = fd.read(rec.name_len)
- fd.read(rec.extra_len)
- fd.read(rec.comment_len)
-
- if rec.name.endswith('/'):
- rec.name = rec.name[:-1]
- rec.mode = stat.S_IFDIR | 0777
- return rec
-
-
-class _CentHeader(object):
- """Information about a single file in the archive.
- 0 version made by 2 bytes
- 1 version needed to extract 2 bytes
- 2 general purpose bit flag 2 bytes
- 3 compression method 2 bytes
- 4 last mod file time 2 bytes
- 5 last mod file date 2 bytes
- 6 crc-32 4 bytes
- 7 compressed size 4 bytes
- 8 uncompressed size 4 bytes
- 9 file name length 2 bytes
- 10 extra field length 2 bytes
- 11 file comment length 2 bytes
- 12 disk number start 2 bytes
- 13 internal file attributes 2 bytes
- 14 external file attributes 4 bytes
- 15 relative offset of local header 4 bytes
- """
- def __init__(self, raw_bin):
- rec = struct.unpack('<6H3I5H2I', raw_bin)
- self.name_len = rec[9]
- self.extra_len = rec[10]
- self.comment_len = rec[11]
-
- if (rec[0] & 0xff00) == 0x0300: # UNIX
- self.mode = rec[14] >> 16
- else:
- self.mode = stat.S_IFREG | 0644
-
-
-class _UngetStream(object):
- """File like object to read and rewind a stream.
- """
- def __init__(self, fd):
- self._fd = fd
- self._buf = None
-
- def read(self, size = -1):
- r = []
- try:
- if size >= 0:
- self._ReadChunk(r, size)
- else:
- while True:
- self._ReadChunk(r, 2048)
- except EOFError:
- pass
-
- if len(r) == 1:
- return r[0]
- return ''.join(r)
-
- def unread(self, buf):
- b = self._buf
- if b is None or len(b) == 0:
- self._buf = buf
- else:
- self._buf = buf + b
-
- def _ReadChunk(self, r, size):
- b = self._buf
- try:
- while size > 0:
- if b is None or len(b) == 0:
- b = self._Inflate(self._fd.read(2048))
- if not b:
- raise EOFError()
- continue
-
- use = min(size, len(b))
- r.append(b[:use])
- b = b[use:]
- size -= use
- finally:
- self._buf = b
-
- def _Inflate(self, b):
- return b
-
-
-class _FixedLengthStream(_UngetStream):
- """File like object to read a fixed length stream.
- """
- def __init__(self, fd, have):
- _UngetStream.__init__(self, fd)
- self._have = have
-
- def _Inflate(self, b):
- n = self._have
- if n == 0:
- self._fd.unread(b)
- return None
-
- if len(b) > n:
- self._fd.unread(b[n:])
- b = b[:n]
- self._have -= len(b)
- return b
-
-
-class _InflateStream(_UngetStream):
- """Inflates the stream as it reads input.
- """
- def __init__(self, fd):
- _UngetStream.__init__(self, fd)
- self._z = zlib.decompressobj(-zlib.MAX_WBITS)
-
- def _Inflate(self, b):
- z = self._z
- if not z:
- self._fd.unread(b)
- return None
-
- b = z.decompress(b)
- if z.unconsumed_tail != '':
- self._fd.unread(z.unconsumed_tail)
- elif z.unused_data != '':
- self._fd.unread(z.unused_data)
- self._z = None
- return b
diff --git a/manifest.py b/manifest.py
index 8c5a8d6..ffff14a 100644
--- a/manifest.py
+++ b/manifest.py
@@ -18,8 +18,6 @@
import xml.dom.minidom
from git_config import GitConfig, IsId
-from import_tar import ImportTar
-from import_zip import ImportZip
from project import Project, MetaProject, R_TAGS
from remote import Remote
from error import ManifestParseError
@@ -245,78 +243,8 @@
elif n.nodeName == 'copyfile':
self._ParseCopyFile(project, n)
- to_resolve = []
- by_version = {}
-
- for n in node.childNodes:
- if n.nodeName == 'import':
- self._ParseImport(project, n, to_resolve, by_version)
-
- for pair in to_resolve:
- sn, pr = pair
- try:
- sn.SetParent(by_version[pr].commit)
- except KeyError:
- raise ManifestParseError, \
- 'snapshot %s not in project %s in %s' % \
- (pr, project.name, self.manifestFile)
-
return project
- def _ParseImport(self, project, import_node, to_resolve, by_version):
- first_url = None
- for node in import_node.childNodes:
- if node.nodeName == 'mirror':
- first_url = self._reqatt(node, 'url')
- break
- if not first_url:
- raise ManifestParseError, \
- 'mirror url required for project %s in %s' % \
- (project.name, self.manifestFile)
-
- imp = None
- for cls in [ImportTar, ImportZip]:
- if cls.CanAccept(first_url):
- imp = cls()
- break
- if not imp:
- raise ManifestParseError, \
- 'snapshot %s unsupported for project %s in %s' % \
- (first_url, project.name, self.manifestFile)
-
- imp.SetProject(project)
-
- for node in import_node.childNodes:
- if node.nodeName == 'remap':
- old = node.getAttribute('strip')
- new = node.getAttribute('insert')
- imp.RemapPath(old, new)
-
- elif node.nodeName == 'mirror':
- imp.AddUrl(self._reqatt(node, 'url'))
-
- for node in import_node.childNodes:
- if node.nodeName == 'snapshot':
- sn = imp.Clone()
- sn.SetVersion(self._reqatt(node, 'version'))
- sn.SetCommit(node.getAttribute('check'))
-
- pr = node.getAttribute('prior')
- if pr:
- if IsId(pr):
- sn.SetParent(pr)
- else:
- to_resolve.append((sn, pr))
-
- rev = R_TAGS + sn.TagName
-
- if rev in project.snapshots:
- raise ManifestParseError, \
- 'duplicate snapshot %s for project %s in %s' % \
- (sn.version, project.name, self.manifestFile)
- project.snapshots[rev] = sn
- by_version[sn.version] = sn
-
def _ParseCopyFile(self, project, node):
src = self._reqatt(node, 'src')
dest = self._reqatt(node, 'dest')
diff --git a/subcmds/compute_snapshot_check.py b/subcmds/compute_snapshot_check.py
deleted file mode 100644
index 82db359..0000000
--- a/subcmds/compute_snapshot_check.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-import tempfile
-
-from command import Command
-from error import GitError, NoSuchProjectError
-from git_config import IsId
-from import_tar import ImportTar
-from import_zip import ImportZip
-from project import Project
-from remote import Remote
-
-def _ToCommit(project, rev):
- return project.bare_git.rev_parse('--verify', '%s^0' % rev)
-
-def _Missing(project, rev):
- return project._revlist('--objects', rev, '--not', '--all')
-
-
-class ComputeSnapshotCheck(Command):
- common = False
- helpSummary = "Compute the check value for a new snapshot"
- helpUsage = """
-%prog -p NAME -v VERSION -s FILE [options]
-"""
- helpDescription = """
-%prog computes and then displays the proper check value for a
-snapshot, so it can be pasted into the manifest file for a project.
-"""
-
- def _Options(self, p):
- g = p.add_option_group('Snapshot description options')
- g.add_option('-p', '--project',
- dest='project', metavar='NAME',
- help='destination project name')
- g.add_option('-v', '--version',
- dest='version', metavar='VERSION',
- help='upstream version/revision identifier')
- g.add_option('-s', '--snapshot',
- dest='snapshot', metavar='PATH',
- help='local tarball path')
- g.add_option('--new-project',
- dest='new_project', action='store_true',
- help='destinition is a new project')
- g.add_option('--keep',
- dest='keep_git', action='store_true',
- help='keep the temporary git repository')
-
- g = p.add_option_group('Base revision grafting options')
- g.add_option('--prior',
- dest='prior', metavar='COMMIT',
- help='prior revision checksum')
-
- g = p.add_option_group('Path mangling options')
- g.add_option('--strip-prefix',
- dest='strip_prefix', metavar='PREFIX',
- help='remove prefix from all paths on import')
- g.add_option('--insert-prefix',
- dest='insert_prefix', metavar='PREFIX',
- help='insert prefix before all paths on import')
-
-
- def _Compute(self, opt):
- try:
- real_project = self.GetProjects([opt.project])[0]
- except NoSuchProjectError:
- if opt.new_project:
- print >>sys.stderr, \
- "warning: project '%s' does not exist" % opt.project
- else:
- raise NoSuchProjectError(opt.project)
-
- self._tmpdir = tempfile.mkdtemp()
- project = Project(manifest = self.manifest,
- name = opt.project,
- remote = Remote('origin'),
- gitdir = os.path.join(self._tmpdir, '.git'),
- worktree = self._tmpdir,
- relpath = opt.project,
- revision = 'refs/heads/master')
- project._InitGitDir()
-
- url = 'file://%s' % os.path.abspath(opt.snapshot)
-
- imp = None
- for cls in [ImportTar, ImportZip]:
- if cls.CanAccept(url):
- imp = cls()
- break
- if not imp:
- print >>sys.stderr, 'error: %s unsupported' % opt.snapshot
- sys.exit(1)
-
- imp.SetProject(project)
- imp.SetVersion(opt.version)
- imp.AddUrl(url)
-
- if opt.prior:
- if opt.new_project:
- if not IsId(opt.prior):
- print >>sys.stderr, 'error: --prior=%s not valid' % opt.prior
- sys.exit(1)
- else:
- try:
- opt.prior = _ToCommit(real_project, opt.prior)
- missing = _Missing(real_project, opt.prior)
- except GitError, e:
- print >>sys.stderr,\
- 'error: --prior=%s not valid\n%s' \
- % (opt.prior, e)
- sys.exit(1)
- if missing:
- print >>sys.stderr,\
- 'error: --prior=%s is valid, but is not reachable' \
- % opt.prior
- sys.exit(1)
- imp.SetParent(opt.prior)
-
- src = opt.strip_prefix
- dst = opt.insert_prefix
- if src or dst:
- if src is None:
- src = ''
- if dst is None:
- dst = ''
- imp.RemapPath(src, dst)
- commitId = imp.Import()
-
- print >>sys.stderr,"%s\t%s" % (commitId, imp.version)
- return project
-
- def Execute(self, opt, args):
- if args \
- or not opt.project \
- or not opt.version \
- or not opt.snapshot:
- self.Usage()
-
- success = False
- project = None
- try:
- self._tmpdir = None
- project = self._Compute(opt)
- finally:
- if project and opt.keep_git:
- print 'GIT_DIR = %s' % (project.gitdir)
- elif self._tmpdir:
- for root, dirs, files in os.walk(self._tmpdir, topdown=False):
- for name in files:
- os.remove(os.path.join(root, name))
- for name in dirs:
- os.rmdir(os.path.join(root, name))
- os.rmdir(self._tmpdir)
-
