670abbe305
In python3, os.read() automatically retries after EINTR, which breaks our ability to interrupt on SIGALRM. Instead, throw an exception from the SIGALRM handler, which should work on both python2 and python3. This fixes a rare deadlock during parallel builds on python3. For background: https://www.python.org/dev/peps/pep-0475/#backward-compatibility "Applications relying on the fact that system calls are interrupted with InterruptedError will hang. The authors of this PEP don't think that such applications exist [...]" Well, apparently they were mistaken :)
549 lines
20 KiB
Python
549 lines
20 KiB
Python
"""Implementation of a GNU make-compatible jobserver."""
|
|
#
|
|
# The basic idea is that both ends of a pipe (tokenfds) are shared with all
|
|
# subprocesses. At startup, we write one "token" into the pipe for each
|
|
# configured job. (So eg. redo -j20 will put 20 tokens in the pipe.) In
|
|
# order to do work, you must first obtain a token, by reading the other
|
|
# end of the pipe. When you're done working, you write the token back into
|
|
# the pipe so that someone else can grab it.
|
|
#
|
|
# The toplevel process in the hierarchy is what creates the pipes in the
|
|
# first place. Then it puts the pipe file descriptor numbers into MAKEFLAGS,
|
|
# so that subprocesses can pull them back out.
|
|
#
|
|
# As usual, edge cases make all this a bit tricky:
|
|
#
|
|
# - Every process is defined as owning a token at startup time. This makes
|
|
# sense because it's backward compatible with single-process make: if a
|
|
# subprocess neither reads nor writes the pipe, then it has exactly one
|
|
# token, so it's allowed to do one thread of work.
|
|
#
|
|
# - Thus, for symmetry, processes also must own a token at exit time.
|
|
#
|
|
# - In turn, to make *that* work, a parent process must destroy *its* token
|
|
# upon launching a subprocess. (Destroy, not release, because the
|
|
# subprocess has created its own token.) It can try to obtain another
|
|
# token, but if none are available, it has to stop work until one of its
|
|
# subprocesses finishes. When the subprocess finishes, its token is
|
|
# destroyed, so the parent creates a new one.
|
|
#
|
|
# - If our process is going to stop and wait for a lock (eg. because we
|
|
# depend on a target and someone else is already building that target),
|
|
# we must give up our token. Otherwise, we're sucking up a "thread" (a
|
|
# unit of parallelism) just to do nothing. If enough processes are waiting
|
|
# on a particular lock, then the process building that target might end up
|
|
# with only a single token, and everything gets serialized.
|
|
#
|
|
# - Unfortunately this leads to a problem: if we give up our token, we then
|
|
# have to re-acquire a token before exiting, even if we want to exit with
|
|
# an error code.
|
|
#
|
|
# - redo-log wants to linearize output so that it always prints log messages
|
|
# in the order jobs were started; but because of the above, a job being
|
|
# logged might end up with no tokens for a long time, waiting for some
|
|
# other branch of the build to complete.
|
|
#
|
|
# As a result, we extend beyond GNU make's model and make things even more
|
|
# complicated. We add a second pipe, cheatfds, which we use to "cheat" on
|
|
# tokens if our particular job is in the foreground (ie. is the one
|
|
# currently being tailed by redo-log -f). We add at most one token per
|
|
# redo-log instance. If we are the foreground task, and we need a token,
|
|
# and we don't have a token, and we don't have any subtasks (because if we
|
|
# had a subtask, then we're not in the foreground), we synthesize our own
|
|
# token by incrementing _mytokens and _cheats, but we don't read from
|
|
# tokenfds. Then, when it's time to give up our token again, we also won't
|
|
# write back to tokenfds, so the synthesized token disappears.
|
|
#
|
|
# Of course, all that then leads to *another* problem: every process must
|
|
# hold a *real* token when it exits, because its parent has given up a
|
|
# *real* token in order to start this subprocess. If we're holding a cheat
|
|
# token when it's time to exit, then we can't meet this requirement. The
|
|
# obvious thing to do would be to give up the cheat token and wait for a
|
|
# real token, but that might take a very long time, and if we're the last
|
|
# thing preventing our parent from exiting, then redo-log will sit around
|
|
# following our parent until we finally get a token so we can exit,
|
|
# defeating the whole purpose of cheating. Instead of waiting, we write our
|
|
# "cheater" token to cheatfds. Then, any task, upon noticing one of its
|
|
# subprocesses has finished, will check to see if there are any tokens on
|
|
# cheatfds; if so, it will remove one of them and *not* re-create its
|
|
# child's token, thus destroying the cheater token from earlier, and restoring
|
|
# balance.
|
|
#
|
|
# Sorry this is so complicated. I couldn't think of a way to make it
|
|
# simpler :)
|
|
#
|
|
import sys, os, errno, select, fcntl, signal
|
|
from . import env, helpers, logs, state
|
|
from .atoi import atoi
|
|
|
|
_toplevel = 0
|
|
_mytokens = 1
|
|
_cheats = 0
|
|
_tokenfds = None
|
|
_cheatfds = None
|
|
_waitfds = {}
|
|
|
|
|
|
def _debug(s):
|
|
if 0:
|
|
sys.stderr.write('job#%d: %s' % (os.getpid(), s))
|
|
|
|
|
|
def _create_tokens(n):
|
|
"""Materialize and own n tokens.
|
|
|
|
If there are any cheater tokens active, they each destroy one matching
|
|
newly-created token.
|
|
"""
|
|
global _mytokens, _cheats
|
|
assert n >= 0
|
|
assert _cheats >= 0
|
|
for _ in range(n):
|
|
if _cheats > 0:
|
|
_cheats -= 1
|
|
else:
|
|
_mytokens += 1
|
|
|
|
|
|
def _destroy_tokens(n):
|
|
"""Destroy n tokens that are currently in our posession."""
|
|
global _mytokens
|
|
assert _mytokens >= n
|
|
_mytokens -= n
|
|
|
|
|
|
def _release(n):
|
|
global _mytokens, _cheats
|
|
assert n >= 0
|
|
assert _mytokens >= n
|
|
_debug('%d,%d -> release(%d)\n' % (_mytokens, _cheats, n))
|
|
n_to_share = 0
|
|
for _ in range(n):
|
|
_mytokens -= 1
|
|
if _cheats > 0:
|
|
_cheats -= 1
|
|
else:
|
|
n_to_share += 1
|
|
assert _mytokens >= 0
|
|
assert _cheats >= 0
|
|
if n_to_share:
|
|
_debug('PUT tokenfds %d\n' % n_to_share)
|
|
os.write(_tokenfds[1], b't' * n_to_share)
|
|
|
|
|
|
def _release_except_mine():
|
|
assert _mytokens > 0
|
|
_release(_mytokens - 1)
|
|
|
|
|
|
def release_mine():
|
|
assert _mytokens >= 1
|
|
_debug('%d,%d -> release_mine()\n' % (_mytokens, _cheats))
|
|
_release(1)
|
|
|
|
|
|
class TimeoutError(Exception): pass
|
|
|
|
def _timeout(sig, frame):
|
|
raise TimeoutError()
|
|
|
|
|
|
# We make the pipes use the first available fd numbers starting at startfd.
|
|
# This makes it easier to differentiate different kinds of pipes when using
|
|
# strace.
|
|
def _make_pipe(startfd):
|
|
(a, b) = os.pipe()
|
|
fds = (fcntl.fcntl(a, fcntl.F_DUPFD, startfd),
|
|
fcntl.fcntl(b, fcntl.F_DUPFD, startfd + 1))
|
|
os.close(a)
|
|
os.close(b)
|
|
return fds
|
|
|
|
|
|
def _try_read(fd, n):
|
|
"""Try to read n bytes from fd. Returns: '' on EOF, None if EAGAIN."""
|
|
assert state.is_flushed()
|
|
|
|
# using djb's suggested way of doing non-blocking reads from a blocking
|
|
# socket: http://cr.yp.to/unix/nonblock.html
|
|
# We can't just make the socket non-blocking, because we want to be
|
|
# compatible with GNU Make, and they can't handle it.
|
|
r, w, x = select.select([fd], [], [], 0)
|
|
if not r:
|
|
return None # try again
|
|
# ok, the socket is readable - but some other process might get there
|
|
# first. We have to set an alarm() in case our read() gets stuck.
|
|
try:
|
|
oldh = signal.signal(signal.SIGALRM, _timeout)
|
|
signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01) # emergency fallback
|
|
try:
|
|
b = os.read(fd, 1)
|
|
except TimeoutError:
|
|
return None # try again
|
|
except OSError as e:
|
|
if e.errno in (errno.EAGAIN, errno.EINTR):
|
|
# interrupted or it was nonblocking
|
|
return None # try again
|
|
else:
|
|
raise
|
|
finally:
|
|
signal.setitimer(signal.ITIMER_REAL, 0, 0)
|
|
signal.signal(signal.SIGALRM, oldh)
|
|
return b
|
|
|
|
|
|
def _try_read_all(fd, n):
|
|
bb = b''
|
|
while 1:
|
|
b = _try_read(fd, n)
|
|
if not b:
|
|
break
|
|
bb += b
|
|
return bb
|
|
|
|
|
|
def setup(maxjobs):
|
|
"""Start the jobserver (if it isn't already) with the given token count.
|
|
|
|
Args:
|
|
maxjobs: if nonzero, create a new jobserver with separate tokens from
|
|
the one we inherited (if any). If zero and we inherited a jobserver,
|
|
just use that. If zero and we didn't inherit a jobserver, create
|
|
one with a default number of tokens (currently always 1).
|
|
"""
|
|
global _tokenfds, _cheatfds, _toplevel
|
|
assert maxjobs >= 0
|
|
assert not _tokenfds
|
|
_debug('setup(%d)\n' % maxjobs)
|
|
|
|
flags = ' ' + os.getenv('MAKEFLAGS', '') + ' '
|
|
FIND1 = ' --jobserver-auth=' # renamed in GNU make 4.2
|
|
FIND2 = ' --jobserver-fds=' # fallback syntax
|
|
FIND = FIND1
|
|
ofs = flags.find(FIND1)
|
|
if ofs < 0:
|
|
FIND = FIND2
|
|
ofs = flags.find(FIND2)
|
|
if ofs >= 0:
|
|
s = flags[ofs+len(FIND):]
|
|
(arg, junk) = s.split(' ', 1)
|
|
(a, b) = arg.split(',', 1)
|
|
a = atoi(a)
|
|
b = atoi(b)
|
|
if a <= 0 or b <= 0:
|
|
logs.err('invalid --jobserver-auth: %r\n' % arg)
|
|
raise helpers.ImmediateReturn(200)
|
|
if not helpers.fd_exists(a) or not helpers.fd_exists(b):
|
|
logs.err('broken --jobserver-auth from parent process:\n')
|
|
logs.err(' using GNU make? prefix your Makefile rule with "+"\n')
|
|
logs.err(
|
|
' otherwise, see ' +
|
|
'https://redo.rtfd.io/en/latest/FAQParallel/#MAKEFLAGS\n')
|
|
raise helpers.ImmediateReturn(200)
|
|
if maxjobs == 1:
|
|
# user requested exactly one token, which means they want to
|
|
# serialize us, even if the parent redo is running in parallel.
|
|
# That's pretty harmless, so allow it without a warning.
|
|
pass
|
|
elif maxjobs:
|
|
# user requested more than one token, even though we have a parent
|
|
# jobserver, which is fishy. Warn about it, like make does.
|
|
logs.warn(('warning: -j%d forced in sub-redo; ' +
|
|
'starting new jobserver.\n') % maxjobs)
|
|
else:
|
|
# user requested zero tokens, which means use the parent jobserver
|
|
# if it exists.
|
|
_tokenfds = (a, b)
|
|
|
|
cheats = os.getenv('REDO_CHEATFDS', '') if not maxjobs else ''
|
|
_cheatfds = None
|
|
if cheats:
|
|
(a, b) = cheats.split(',', 1)
|
|
a = atoi(a)
|
|
b = atoi(b)
|
|
if a <= 0 or b <= 0:
|
|
raise ValueError('invalid REDO_CHEATFDS: %r' % cheats)
|
|
_cheatfds = (a, b)
|
|
if not helpers.fd_exists(a) or not helpers.fd_exists(b):
|
|
# This can happen if we're called by a parent process who closes
|
|
# all "unknown" file descriptors (which is anti-social behaviour,
|
|
# but oh well, we'll warn about it if they close the jobserver
|
|
# fds in MAKEFLAGS, so just ignore it if it also happens here).
|
|
_cheatfds = None
|
|
if not _cheatfds:
|
|
_cheatfds = _make_pipe(102)
|
|
os.environ['REDO_CHEATFDS'] = ('%d,%d' % (_cheatfds[0], _cheatfds[1]))
|
|
|
|
if not _tokenfds:
|
|
# need to start a new server
|
|
realmax = maxjobs or 1
|
|
_toplevel = realmax
|
|
_tokenfds = _make_pipe(100)
|
|
_create_tokens(realmax - 1)
|
|
_release_except_mine()
|
|
os.environ['MAKEFLAGS'] = (
|
|
' -j --jobserver-auth=%d,%d --jobserver-fds=%d,%d' %
|
|
(_tokenfds[0], _tokenfds[1],
|
|
_tokenfds[0], _tokenfds[1]))
|
|
|
|
|
|
def _wait(want_token, max_delay):
|
|
"""Wait for a subproc to die or, if want_token, tokenfd to be readable.
|
|
|
|
Does not actually read tokenfd once it's readable (so someone else might
|
|
read it before us). This function returns after max_delay seconds or
|
|
when at least one subproc dies or (if want_token) for tokenfd to be
|
|
readable.
|
|
|
|
Args:
|
|
want_token: true if we should return when tokenfd is readable.
|
|
max_delay: max seconds to wait, or None to wait forever.
|
|
Returns:
|
|
None
|
|
"""
|
|
rfds = list(_waitfds.keys())
|
|
if want_token:
|
|
rfds.append(_tokenfds[0])
|
|
assert rfds
|
|
assert state.is_flushed()
|
|
_debug('_tokenfds=%r; jfds=%r\n' % (_tokenfds, _waitfds))
|
|
r, w, x = select.select(rfds, [], [], max_delay)
|
|
_debug('readable: %r\n' % (r,))
|
|
for fd in r:
|
|
if fd == _tokenfds[0]:
|
|
pass
|
|
else:
|
|
pd = _waitfds[fd]
|
|
_debug("done: %r\n" % pd.name)
|
|
# redo subprocesses are expected to die without releasing their
|
|
# tokens, so things are less likely to get confused if they
|
|
# die abnormally. Since a child has died, that means a token has
|
|
# 'disappeared' and we now need to recreate it.
|
|
b = _try_read(_cheatfds[0], 1)
|
|
if b:
|
|
# someone exited with _cheats > 0, so we need to compensate
|
|
# by *not* re-creating a token now.
|
|
_debug('EAT cheatfd %r\n' % b)
|
|
else:
|
|
_create_tokens(1)
|
|
if has_token():
|
|
_release_except_mine()
|
|
os.close(fd)
|
|
del _waitfds[fd]
|
|
rv = os.waitpid(pd.pid, 0)
|
|
assert rv[0] == pd.pid
|
|
_debug("done1: rv=%r\n" % (rv,))
|
|
rv = rv[1]
|
|
if os.WIFEXITED(rv):
|
|
pd.rv = os.WEXITSTATUS(rv)
|
|
else:
|
|
pd.rv = -os.WTERMSIG(rv)
|
|
_debug("done2: rv=%d\n" % pd.rv)
|
|
pd.donefunc(pd.name, pd.rv)
|
|
|
|
|
|
def has_token():
|
|
"""Returns true if this process has a job token."""
|
|
assert _mytokens >= 0
|
|
if _mytokens >= 1:
|
|
return True
|
|
|
|
|
|
def _ensure_token(reason, max_delay=None):
|
|
"""Don't return until this process has a job token.
|
|
|
|
Args:
|
|
reason: the reason (for debugging purposes) we need a token. Usually
|
|
the name of a target we want to build.
|
|
max_delay: the max time to wait for a token, or None if forever.
|
|
Returns:
|
|
None, but has_token() is now true *unless* max_delay is non-None
|
|
and we timed out.
|
|
"""
|
|
global _mytokens
|
|
assert state.is_flushed()
|
|
assert _mytokens <= 1
|
|
while 1:
|
|
if _mytokens >= 1:
|
|
_debug("_mytokens is %d\n" % _mytokens)
|
|
assert _mytokens == 1
|
|
_debug('(%r) used my own token...\n' % reason)
|
|
break
|
|
assert _mytokens < 1
|
|
_debug('(%r) waiting for tokens...\n' % reason)
|
|
_wait(want_token=1, max_delay=max_delay)
|
|
if _mytokens >= 1:
|
|
break
|
|
assert _mytokens < 1
|
|
b = _try_read(_tokenfds[0], 1)
|
|
if b == '':
|
|
raise Exception('unexpected EOF on token read')
|
|
if b:
|
|
_mytokens += 1
|
|
_debug('(%r) got a token (%r).\n' % (reason, b))
|
|
break
|
|
if max_delay != None:
|
|
break
|
|
assert _mytokens <= 1
|
|
|
|
|
|
def ensure_token_or_cheat(reason, cheatfunc):
|
|
"""Wait for a job token to become available, or cheat if possible.
|
|
|
|
If we already have a token, we return immediately. If we have any
|
|
processes running *and* we don't own any tokens, we wait for a process
|
|
to finish, and then use that token.
|
|
|
|
Otherwise, we're allowed to cheat. We call cheatfunc() occasionally
|
|
to consider cheating; if it returns n > 0, we materialize that many
|
|
cheater tokens and return.
|
|
|
|
Args:
|
|
reason: the reason (for debugging purposes) we need a token. Usually
|
|
the name of a target we want to build.
|
|
cheatfunc: a function which returns n > 0 (usually 1) if we should
|
|
cheat right now, because we're the "foreground" process that must
|
|
be allowed to continue.
|
|
"""
|
|
global _mytokens, _cheats
|
|
backoff = 0.01
|
|
while not has_token():
|
|
while running() and not has_token():
|
|
# If we already have a subproc running, then effectively we
|
|
# already have a token. Don't create a cheater token unless
|
|
# we're completely idle.
|
|
_ensure_token(reason, max_delay=None)
|
|
_ensure_token(reason, max_delay=min(1.0, backoff))
|
|
backoff *= 2
|
|
if not has_token():
|
|
assert _mytokens == 0
|
|
n = cheatfunc()
|
|
_debug('%s: %s: cheat = %d\n' % (env.v.TARGET, reason, n))
|
|
if n > 0:
|
|
_mytokens += n
|
|
_cheats += n
|
|
break
|
|
|
|
|
|
def running():
|
|
"""Returns true if we have any running jobs."""
|
|
return len(_waitfds)
|
|
|
|
|
|
def wait_all():
|
|
"""Wait for all running jobs to finish."""
|
|
_debug("%d,%d -> wait_all\n" % (_mytokens, _cheats))
|
|
assert state.is_flushed()
|
|
while 1:
|
|
while _mytokens >= 2:
|
|
_release(1)
|
|
if not running():
|
|
break
|
|
# We should only release our last token if we have remaining
|
|
# children. A terminating redo process should try to terminate while
|
|
# holding a token, and if we have no children left, we might be
|
|
# about to terminate.
|
|
if _mytokens >= 1:
|
|
release_mine()
|
|
_debug("wait_all: wait()\n")
|
|
_wait(want_token=0, max_delay=None)
|
|
_debug("wait_all: empty list\n")
|
|
if _toplevel:
|
|
# If we're the toplevel and we're sure no child processes remain,
|
|
# then we know we're totally idle. Self-test to ensure no tokens
|
|
# mysteriously got created/destroyed.
|
|
if _mytokens >= 1:
|
|
release_mine()
|
|
tokens = _try_read_all(_tokenfds[0], 8192)
|
|
cheats = _try_read_all(_cheatfds[0], 8192)
|
|
_debug('toplevel: GOT %d tokens and %d cheats\n'
|
|
% (len(tokens), len(cheats)))
|
|
if len(tokens) - len(cheats) != _toplevel:
|
|
raise Exception('on exit: expected %d tokens; found %r-%r'
|
|
% (_toplevel, len(tokens), len(cheats)))
|
|
os.write(_tokenfds[1], tokens)
|
|
# note: when we return, we may have *no* tokens, not even our own!
|
|
# If caller wants to continue, they might have to obtain one first.
|
|
|
|
|
|
def force_return_tokens():
|
|
"""Release or destroy all the tokens we own, in preparation for exit."""
|
|
n = len(_waitfds)
|
|
_debug('%d,%d -> %d jobs left in force_return_tokens\n'
|
|
% (_mytokens, _cheats, n))
|
|
for k in list(_waitfds):
|
|
del _waitfds[k]
|
|
_create_tokens(n)
|
|
if has_token():
|
|
_release_except_mine()
|
|
assert _mytokens == 1, 'mytokens=%d' % _mytokens
|
|
assert _cheats <= _mytokens, 'mytokens=%d cheats=%d' % (_mytokens, _cheats)
|
|
assert _cheats in (0, 1), 'cheats=%d' % _cheats
|
|
if _cheats:
|
|
_debug('%d,%d -> force_return_tokens: recovering final token\n'
|
|
% (_mytokens, _cheats))
|
|
_destroy_tokens(_cheats)
|
|
os.write(_cheatfds[1], b't' * _cheats)
|
|
assert state.is_flushed()
|
|
|
|
|
|
class Job(object):
|
|
"""Metadata about a running job."""
|
|
|
|
def __init__(self, name, pid, donefunc):
|
|
"""Initialize a job.
|
|
|
|
Args:
|
|
name: the name of the job (usually a target filename).
|
|
pid: the pid of the subprocess running this job.
|
|
donefunc: the function(name, return_value) to call when the
|
|
subprocess exits.
|
|
"""
|
|
self.name = name
|
|
self.pid = pid
|
|
self.rv = None
|
|
self.donefunc = donefunc
|
|
|
|
def __repr__(self):
|
|
return 'Job(%s,%d)' % (self.name, self.pid)
|
|
|
|
|
|
def start(reason, jobfunc, donefunc):
|
|
"""Start a new job. Must only be run with has_token() true.
|
|
|
|
Args:
|
|
reason: the reason the job is running. Usually the filename of a
|
|
target being built.
|
|
jobfunc: the function() to execute, **in a subprocess**, to run the job.
|
|
Usually this calls os.execve(). It is an error for this function to
|
|
ever return. If it does, we return a non-zero exit code to the parent
|
|
process (the one which ran start()).
|
|
donefunc: the function(reason, return_value) to call **in the parent**
|
|
when the subprocess exits.
|
|
"""
|
|
assert state.is_flushed()
|
|
assert _mytokens <= 1
|
|
assert _mytokens == 1
|
|
# Subprocesses always start with 1 token, so we have to destroy ours
|
|
# in order for the universe to stay in balance.
|
|
_destroy_tokens(1)
|
|
r, w = _make_pipe(50)
|
|
pid = os.fork()
|
|
if pid == 0:
|
|
# child
|
|
rv = 201
|
|
try:
|
|
os.close(r)
|
|
try:
|
|
rv = jobfunc()
|
|
assert 0, 'jobfunc returned?! (%r, %r)' % (jobfunc, rv)
|
|
except Exception: # pylint: disable=broad-except
|
|
import traceback
|
|
traceback.print_exc()
|
|
finally:
|
|
_debug('exit: %d\n' % rv)
|
|
os._exit(rv)
|
|
helpers.close_on_exec(r, True)
|
|
os.close(w)
|
|
pd = Job(reason, pid, donefunc)
|
|
_waitfds[r] = pd
|