diff --git a/TODO b/TODO new file mode 100644 index 0000000..8ea1c92 --- /dev/null +++ b/TODO @@ -0,0 +1,9 @@ + + .gitignore + jwack jobserver + make compatibility + sockety interface + sqlite .redo database + rename --ifchange/--ifcreate to separate programs + rewrite ifchange/ifcreate in C + \ No newline at end of file diff --git a/helpers.py b/helpers.py index 16ad39e..ab320ab 100644 --- a/helpers.py +++ b/helpers.py @@ -1,6 +1,13 @@ import sys, os, errno +def atoi(v): + try: + return int(v or 0) + except ValueError: + return 0 + + def unlink(f): """Delete a file at path 'f' if it currently exists. diff --git a/jwack b/jwack new file mode 120000 index 0000000..418df76 --- /dev/null +++ b/jwack @@ -0,0 +1 @@ +jwack.py \ No newline at end of file diff --git a/jwack.py b/jwack.py new file mode 100755 index 0000000..ff24066 --- /dev/null +++ b/jwack.py @@ -0,0 +1,145 @@ +#!/usr/bin/python +# +# beware the jobberwack +# +import sys, os, errno, select, subprocess, fcntl +import options +from helpers import * + +optspec = """ +jwack [-j maxjobs] -- +-- +j,jobs= maximum jobs to run at once +""" + +_fds = None +_tokens = {} +_waitfds = {} + + +def setup(maxjobs): + global _fds + if _fds: + return # already set up + flags = ' ' + os.getenv('MAKEFLAGS', '') + ' ' + FIND = ' --jobserver-fds=' + ofs = flags.find(FIND) + if ofs >= 0: + s = flags[ofs+len(FIND):] + (arg,junk) = s.split(' ', 1) + (a,b) = arg.split(',', 1) + a = atoi(a) + b = atoi(b) + if a <= 0 or b <= 0: + raise ValueError('invalid --jobserver-fds: %r' % arg) + _fds = (a,b) + if maxjobs and not _fds: + # need to start a new server + _fds = os.pipe() + os.write(_fds[1], 't' * maxjobs) + os.putenv('MAKEFLAGS', + '%s --jobserver-fds=%d,%d -j ' % (os.getenv('MAKEFLAGS'), + _fds[0], _fds[1])) + + +def wait(want_token): + rfds = _waitfds.keys() + if _fds and want_token: + rfds.append(_fds[0]) + r,w,x = select.select(rfds, [], []) + #print 'readable: %r' % r + for fd in r: + if _fds and fd == _fds[0]: + pass + else: + p = _waitfds[fd] + os.write(_fds[1], 't' * _tokens[fd]) + b = os.read(fd, 1) + #print 'read: %r' % b + if b: + #print 'giving up %d tokens for child' % _tokens[fd] + _tokens[fd] = 0 + else: + os.close(fd) + del _waitfds[fd] + del _tokens[fd] + p.wait() + + +def wait_for_token(): + pfd = atoi(os.getenv('JWACK_PARENT_FD', '')) + #print 'pfd is %d' % pfd + if pfd: + #print 'wrote to pfd' + os.write(pfd, 'j') # tell parent to give back his token + while 1: + print 'waiting for tokens...' + wait(want_token=1) + if _fds: + fcntl.fcntl(_fds[0], fcntl.F_SETFL, os.O_NONBLOCK) + try: + b = os.read(_fds[0], 1) # FIXME try: block + except OSError, e: + if e.errno == errno.EAGAIN: + b = '' + pass + else: + raise + if b: + break + print 'got a token.' + + +def wait_all(): + while _waitfds: + wait(want_token=0) + + +def force_return_tokens(): + n = sum(_tokens.values()) + print 'returning %d tokens' % n + if _fds: + os.write(_fds[1], 't' * n) + for k in _tokens.keys(): + _tokens[k] = 0 + + +def _pre_job(r,w): + os.putenv('JWACK_PARENT_FD', str(w)) + os.close(r) + + +def start_job(argv, stdout=None): + global _mytokens + setup(1) + if stdout: + argx = dict(stdout=stdout) + else: + argx = dict() + wait_for_token() + r,w = os.pipe() + p = subprocess.Popen(argv, preexec_fn=lambda: _pre_job(r,w), **argx) + os.close(w) + _waitfds[r] = p + _tokens[r] = 1 + return p + + +def main(): + o = options.Options('jwack', optspec) + (opt, flags, extra) = o.parse(sys.argv[1:]) + + if not extra: + o.fatal("no command line given") + + setup(opt.jobs) + try: + p = start_job(extra) + wait_all() + return p.wait() + finally: + force_return_tokens() + + +if __name__ == "__main__": + sys.exit(main())