Merge changes from topic "windows-support"
* changes:
Port os.rename calls to work on Windows
Workaround shutil.rmtree limitation on Windows
Add support for creating symbolic links on Windows
Make "git command" and "forall" work on Windows
diff --git a/git_command.py b/git_command.py
index 9f7d293..dfa6a92 100644
--- a/git_command.py
+++ b/git_command.py
@@ -14,14 +14,14 @@
# limitations under the License.
from __future__ import print_function
-import fcntl
import os
-import select
import sys
import subprocess
import tempfile
from signal import SIGTERM
+
from error import GitError
+import platform_utils
from trace import REPO_TRACE, IsTrace, Trace
from wrapper import Wrapper
@@ -78,16 +78,6 @@
_git_version = None
-class _sfd(object):
- """select file descriptor class"""
- def __init__(self, fd, dest, std_name):
- assert std_name in ('stdout', 'stderr')
- self.fd = fd
- self.dest = dest
- self.std_name = std_name
- def fileno(self):
- return self.fd.fileno()
-
class _GitCall(object):
def version(self):
p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -253,19 +243,16 @@
def _CaptureOutput(self):
p = self.process
- s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
- _sfd(p.stderr, sys.stderr, 'stderr')]
+ s_in = platform_utils.FileDescriptorStreams.create()
+ s_in.add(p.stdout, sys.stdout, 'stdout')
+ s_in.add(p.stderr, sys.stderr, 'stderr')
self.stdout = ''
self.stderr = ''
- for s in s_in:
- flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
- fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
- while s_in:
- in_ready, _, _ = select.select(s_in, [], [])
+ while not s_in.is_done:
+ in_ready = s_in.select()
for s in in_ready:
- buf = s.fd.read(4096)
+ buf = s.read()
if not buf:
s_in.remove(s)
continue
diff --git a/manifest_xml.py b/manifest_xml.py
index 55d25a7..05651c6 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -32,6 +32,7 @@
import gitc_utils
from git_config import GitConfig
from git_refs import R_HEADS, HEAD
+import platform_utils
from project import RemoteSpec, Project, MetaProject
from error import ManifestParseError, ManifestInvalidRevisionError
@@ -166,7 +167,7 @@
try:
if os.path.lexists(self.manifestFile):
os.remove(self.manifestFile)
- os.symlink(os.path.join('manifests', name), self.manifestFile)
+ platform_utils.symlink(os.path.join('manifests', name), self.manifestFile)
except OSError as e:
raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
diff --git a/platform_utils.py b/platform_utils.py
new file mode 100644
index 0000000..e0fa9dc
--- /dev/null
+++ b/platform_utils.py
@@ -0,0 +1,244 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+import platform
+import select
+import shutil
+import stat
+
+from Queue import Queue
+from threading import Thread
+
+
+def isWindows():
+ """ Returns True when running with the native port of Python for Windows,
+ False when running on any other platform (including the Cygwin port of
+ Python).
+ """
+ # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
+ return platform.system() == "Windows"
+
+
+class FileDescriptorStreams(object):
+ """ Platform agnostic abstraction enabling non-blocking I/O over a
+ collection of file descriptors. This abstraction is required because
+ fctnl(os.O_NONBLOCK) is not supported on Windows.
+ """
+ @classmethod
+ def create(cls):
+ """ Factory method: instantiates the concrete class according to the
+ current platform.
+ """
+ if isWindows():
+ return _FileDescriptorStreamsThreads()
+ else:
+ return _FileDescriptorStreamsNonBlocking()
+
+ def __init__(self):
+ self.streams = []
+
+ def add(self, fd, dest, std_name):
+ """ Wraps an existing file descriptor as a stream.
+ """
+ self.streams.append(self._create_stream(fd, dest, std_name))
+
+ def remove(self, stream):
+ """ Removes a stream, when done with it.
+ """
+ self.streams.remove(stream)
+
+ @property
+ def is_done(self):
+ """ Returns True when all streams have been processed.
+ """
+ return len(self.streams) == 0
+
+ def select(self):
+ """ Returns the set of streams that have data available to read.
+ The returned streams each expose a read() and a close() method.
+ When done with a stream, call the remove(stream) method.
+ """
+ raise NotImplementedError
+
+ def _create_stream(fd, dest, std_name):
+ """ Creates a new stream wrapping an existing file descriptor.
+ """
+ raise NotImplementedError
+
+
+class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
+ """ Implementation of FileDescriptorStreams for platforms that support
+ non blocking I/O.
+ """
+ class Stream(object):
+ """ Encapsulates a file descriptor """
+ def __init__(self, fd, dest, std_name):
+ self.fd = fd
+ self.dest = dest
+ self.std_name = std_name
+ self.set_non_blocking()
+
+ def set_non_blocking(self):
+ import fcntl
+ flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
+ fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+ def fileno(self):
+ return self.fd.fileno()
+
+ def read(self):
+ return self.fd.read(4096)
+
+ def close(self):
+ self.fd.close()
+
+ def _create_stream(self, fd, dest, std_name):
+ return self.Stream(fd, dest, std_name)
+
+ def select(self):
+ ready_streams, _, _ = select.select(self.streams, [], [])
+ return ready_streams
+
+
+class _FileDescriptorStreamsThreads(FileDescriptorStreams):
+ """ Implementation of FileDescriptorStreams for platforms that don't support
+ non blocking I/O. This implementation requires creating threads issuing
+ blocking read operations on file descriptors.
+ """
+ def __init__(self):
+ super(_FileDescriptorStreamsThreads, self).__init__()
+ # The queue is shared accross all threads so we can simulate the
+ # behavior of the select() function
+ self.queue = Queue(10) # Limit incoming data from streams
+
+ def _create_stream(self, fd, dest, std_name):
+ return self.Stream(fd, dest, std_name, self.queue)
+
+ def select(self):
+ # Return only one stream at a time, as it is the most straighforward
+ # thing to do and it is compatible with the select() function.
+ item = self.queue.get()
+ stream = item.stream
+ stream.data = item.data
+ return [stream]
+
+ class QueueItem(object):
+ """ Item put in the shared queue """
+ def __init__(self, stream, data):
+ self.stream = stream
+ self.data = data
+
+ class Stream(object):
+ """ Encapsulates a file descriptor """
+ def __init__(self, fd, dest, std_name, queue):
+ self.fd = fd
+ self.dest = dest
+ self.std_name = std_name
+ self.queue = queue
+ self.data = None
+ self.thread = Thread(target=self.read_to_queue)
+ self.thread.daemon = True
+ self.thread.start()
+
+ def close(self):
+ self.fd.close()
+
+ def read(self):
+ data = self.data
+ self.data = None
+ return data
+
+ def read_to_queue(self):
+ """ The thread function: reads everything from the file descriptor into
+ the shared queue and terminates when reaching EOF.
+ """
+ for line in iter(self.fd.readline, b''):
+ self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
+ self.fd.close()
+ self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
+
+
+def symlink(source, link_name):
+ """Creates a symbolic link pointing to source named link_name.
+ Note: On Windows, source must exist on disk, as the implementation needs
+ to know whether to create a "File" or a "Directory" symbolic link.
+ """
+ if isWindows():
+ import platform_utils_win32
+ source = _validate_winpath(source)
+ link_name = _validate_winpath(link_name)
+ target = os.path.join(os.path.dirname(link_name), source)
+ if os.path.isdir(target):
+ platform_utils_win32.create_dirsymlink(source, link_name)
+ else:
+ platform_utils_win32.create_filesymlink(source, link_name)
+ else:
+ return os.symlink(source, link_name)
+
+
+def _validate_winpath(path):
+ path = os.path.normpath(path)
+ if _winpath_is_valid(path):
+ return path
+ raise ValueError("Path \"%s\" must be a relative path or an absolute "
+ "path starting with a drive letter".format(path))
+
+
+def _winpath_is_valid(path):
+ """Windows only: returns True if path is relative (e.g. ".\\foo") or is
+ absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
+ is ambiguous (e.g. "x:foo" or "\\foo").
+ """
+ assert isWindows()
+ path = os.path.normpath(path)
+ drive, tail = os.path.splitdrive(path)
+ if tail:
+ if not drive:
+ return tail[0] != os.sep # "\\foo" is invalid
+ else:
+ return tail[0] == os.sep # "x:foo" is invalid
+ else:
+ return not drive # "x:" is invalid
+
+
+def rmtree(path):
+ if isWindows():
+ shutil.rmtree(path, onerror=handle_rmtree_error)
+ else:
+ shutil.rmtree(path)
+
+
+def handle_rmtree_error(function, path, excinfo):
+ # Allow deleting read-only files
+ os.chmod(path, stat.S_IWRITE)
+ function(path)
+
+
+def rename(src, dst):
+ if isWindows():
+ # On Windows, rename fails if destination exists, see
+ # https://docs.python.org/2/library/os.html#os.rename
+ try:
+ os.rename(src, dst)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ os.remove(dst)
+ os.rename(src, dst)
+ else:
+ raise
+ else:
+ os.rename(src, dst)
diff --git a/platform_utils_win32.py b/platform_utils_win32.py
new file mode 100644
index 0000000..02fb013
--- /dev/null
+++ b/platform_utils_win32.py
@@ -0,0 +1,63 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+
+from ctypes import WinDLL, get_last_error, FormatError, WinError
+from ctypes.wintypes import BOOL, LPCWSTR, DWORD
+
+kernel32 = WinDLL('kernel32', use_last_error=True)
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOL
+CreateSymbolicLinkW.argtypes = (LPCWSTR, # lpSymlinkFileName In
+ LPCWSTR, # lpTargetFileName In
+ DWORD) # dwFlags In
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+
+
+def create_filesymlink(source, link_name):
+ """Creates a Windows file symbolic link source pointing to link_name."""
+ _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+ """Creates a Windows directory symbolic link source pointing to link_name.
+ """
+ _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+ # Note: Win32 documentation for CreateSymbolicLink is incorrect.
+ # On success, the function returns "1".
+ # On error, the function returns some random value (e.g. 1280).
+ # The best bet seems to use "GetLastError" and check for error/success.
+ CreateSymbolicLinkW(link_name, source, dwFlags)
+ code = get_last_error()
+ if code != ERROR_SUCCESS:
+ error_desc = FormatError(code).strip()
+ if code == ERROR_PRIVILEGE_NOT_HELD:
+ raise OSError(errno.EPERM, error_desc, link_name)
+ error_desc = 'Error creating symbolic link %s: %s'.format(
+ link_name, error_desc)
+ raise WinError(code, error_desc)
diff --git a/project.py b/project.py
index 6b1535a..e700d16 100644
--- a/project.py
+++ b/project.py
@@ -35,6 +35,7 @@
from error import GitError, HookError, UploadError, DownloadError
from error import ManifestInvalidRevisionError
from error import NoManifestException
+import platform_utils
from trace import IsTrace, Trace
from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
@@ -62,7 +63,7 @@
fd.close()
try:
- os.rename(lock, path)
+ platform_utils.rename(lock, path)
except OSError:
os.remove(lock)
raise
@@ -281,7 +282,7 @@
dest_dir = os.path.dirname(absDest)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
- os.symlink(relSrc, absDest)
+ platform_utils.symlink(relSrc, absDest)
except IOError:
_error('Cannot link file %s to %s', relSrc, absDest)
@@ -2210,7 +2211,7 @@
if os.path.exists(tmpPath):
if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
- os.rename(tmpPath, dstPath)
+ platform_utils.rename(tmpPath, dstPath)
return True
else:
os.remove(tmpPath)
@@ -2311,10 +2312,10 @@
print("Retrying clone after deleting %s" %
self.gitdir, file=sys.stderr)
try:
- shutil.rmtree(os.path.realpath(self.gitdir))
+ platform_utils.rmtree(os.path.realpath(self.gitdir))
if self.worktree and os.path.exists(os.path.realpath
(self.worktree)):
- shutil.rmtree(os.path.realpath(self.worktree))
+ platform_utils.rmtree(os.path.realpath(self.worktree))
return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
except:
raise e
@@ -2356,9 +2357,9 @@
self.config.SetString('core.bare', None)
except Exception:
if init_obj_dir and os.path.exists(self.objdir):
- shutil.rmtree(self.objdir)
+ platform_utils.rmtree(self.objdir)
if init_git_dir and os.path.exists(self.gitdir):
- shutil.rmtree(self.gitdir)
+ platform_utils.rmtree(self.gitdir)
raise
def _UpdateHooks(self):
@@ -2392,7 +2393,8 @@
self.relpath, name)
continue
try:
- os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
+ platform_utils.symlink(
+ os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
except OSError as e:
if e.errno == errno.EPERM:
raise GitError('filesystem must support symlinks')
@@ -2491,7 +2493,8 @@
os.makedirs(src)
if name in to_symlink:
- os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+ platform_utils.symlink(
+ os.path.relpath(src, os.path.dirname(dst)), dst)
elif copy_all and not os.path.islink(dst):
if os.path.isdir(src):
shutil.copytree(src, dst)
@@ -2526,7 +2529,7 @@
except GitError as e:
if force_sync:
try:
- shutil.rmtree(dotgit)
+ platform_utils.rmtree(dotgit)
return self._InitWorkTree(force_sync=False, submodules=submodules)
except:
raise e
@@ -2546,7 +2549,7 @@
self._CopyAndLinkFiles()
except Exception:
if init_dotgit:
- shutil.rmtree(dotgit)
+ platform_utils.rmtree(dotgit)
raise
def _gitdir_path(self, path):
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 07ee8d5..2c12c55 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -15,17 +15,16 @@
from __future__ import print_function
import errno
-import fcntl
import multiprocessing
import re
import os
-import select
import signal
import sys
import subprocess
from color import Coloring
from command import Command, MirrorSafeCommand
+import platform_utils
_CAN_COLOR = [
'branch',
@@ -344,35 +343,25 @@
if opt.project_header:
out = ForallColoring(config)
out.redirect(sys.stdout)
- class sfd(object):
- def __init__(self, fd, dest):
- self.fd = fd
- self.dest = dest
- def fileno(self):
- return self.fd.fileno()
-
empty = True
errbuf = ''
p.stdin.close()
- s_in = [sfd(p.stdout, sys.stdout),
- sfd(p.stderr, sys.stderr)]
+ s_in = platform_utils.FileDescriptorStreams.create()
+ s_in.add(p.stdout, sys.stdout, 'stdout')
+ s_in.add(p.stderr, sys.stderr, 'stderr')
- for s in s_in:
- flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
- fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
- while s_in:
- in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
+ while not s_in.is_done:
+ in_ready = s_in.select()
for s in in_ready:
- buf = s.fd.read(4096)
+ buf = s.read()
if not buf:
- s.fd.close()
+ s.close()
s_in.remove(s)
continue
if not opt.verbose:
- if s.fd != p.stdout:
+ if s.std_name == 'stderr':
errbuf += buf
continue
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
index 19caac5..54f62f4 100644
--- a/subcmds/gitc_delete.py
+++ b/subcmds/gitc_delete.py
@@ -14,10 +14,10 @@
# limitations under the License.
from __future__ import print_function
-import shutil
import sys
from command import Command, GitcClientCommand
+import platform_utils
from pyversion import is_python3
if not is_python3():
@@ -50,4 +50,4 @@
if not response == 'yes':
print('Response was not "yes"\n Exiting...')
sys.exit(1)
- shutil.rmtree(self.gitc_manifest.gitc_client_dir)
+ platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)
diff --git a/subcmds/init.py b/subcmds/init.py
index 46cdd23..eeddca0 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -17,7 +17,6 @@
import os
import platform
import re
-import shutil
import sys
from pyversion import is_python3
@@ -35,6 +34,7 @@
from project import SyncBuffer
from git_config import GitConfig
from git_command import git_require, MIN_GIT_VERSION
+import platform_utils
class Init(InteractiveCommand, MirrorSafeCommand):
common = True
@@ -252,7 +252,7 @@
# Better delete the manifest git dir if we created it; otherwise next
# time (when user fixes problems) we won't go through the "is_new" logic.
if is_new:
- shutil.rmtree(m.gitdir)
+ platform_utils.rmtree(m.gitdir)
sys.exit(1)
if opt.manifest_branch:
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 8de730b..b88c596 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -19,7 +19,6 @@
from optparse import SUPPRESS_HELP
import os
import re
-import shutil
import socket
import subprocess
import sys
@@ -73,6 +72,7 @@
from project import RemoteSpec
from command import Command, MirrorSafeCommand
from error import RepoChangedException, GitError, ManifestParseError
+import platform_utils
from project import SyncBuffer
from progress import Progress
from wrapper import Wrapper
@@ -475,7 +475,7 @@
# working git repository around. There shouldn't be any git projects here,
# so rmtree works.
try:
- shutil.rmtree(os.path.join(path, '.git'))
+ platform_utils.rmtree(os.path.join(path, '.git'))
except OSError:
print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
