sync: Limit -j to file descriptors
Each worker thread requires at least 3 file descriptors to run the
forked 'git fetch' child to operate against the local repository.
Mac OS X has the RLIMIT_NOFILE set to 256 by default, which means
a sync -j128 often fails when the workers run out of pipes within
the Python parent process.
Change-Id: I2cdb14621b899424b079daf7969bc8c16b85b903
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 4689ac8..93010c5 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -28,6 +28,14 @@
except ImportError:
import dummy_threading as _threading
+try:
+ import resource
+ def _rlimit_nofile():
+ return resource.getrlimit(resource.RLIMIT_NOFILE)
+except ImportError:
+ def _rlimit_nofile():
+ return (256, 256)
+
from git_command import GIT
from git_refs import R_HEADS
from project import HEAD
@@ -312,6 +320,10 @@
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
+ if self.jobs > 1:
+ soft_limit, _ = _rlimit_nofile()
+ self.jobs = min(self.jobs, (soft_limit - 5) / 3)
+
if opt.network_only and opt.detach_head:
print >>sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
