84169c5d27
This should reduce filesystem grinding a bit, and makes the code simpler. It's also theoretically a bit more portable, since I'm guessing fifo semantics aren't the same on win32 if we ever get there. Also, a major problem with the old fifo-based system is that if a redo process died without cleaning up after itself, it wouldn't delete its lockfiles, so we had to wipe them all at the beginning of each build. Now we don't; in theory, you can now have multiple copies of redo poking at the same tree at the same time and not stepping on each other.
263 lines
9.2 KiB
Python
263 lines
9.2 KiB
Python
import sys, os, errno, stat
|
|
import vars, jwack, state
|
|
from helpers import log, log_, debug2, err, unlink, close_on_exec
|
|
|
|
|
|
def _possible_do_files(t):
|
|
t = os.path.join(vars.BASE, t)
|
|
yield "%s.do" % t, t, ''
|
|
dirname,filename = os.path.split(t)
|
|
l = filename.split('.')
|
|
l[0] = os.path.join(dirname, l[0])
|
|
for i in range(1,len(l)+1):
|
|
basename = '.'.join(l[:i])
|
|
ext = '.'.join(l[i:])
|
|
if ext: ext = '.' + ext
|
|
yield (os.path.join(dirname, "default%s.do" % ext),
|
|
os.path.join(dirname, basename), ext)
|
|
|
|
|
|
def _find_do_file(f):
|
|
for dofile,basename,ext in _possible_do_files(f.name):
|
|
debug2('%s: %s ?\n' % (f.name, dofile))
|
|
if os.path.exists(dofile):
|
|
f.add_dep('m', dofile)
|
|
return dofile,basename,ext
|
|
else:
|
|
f.add_dep('c', dofile)
|
|
return None,None,None
|
|
|
|
|
|
def _nice(t):
|
|
return state.relpath(t, vars.STARTDIR)
|
|
|
|
|
|
def _try_stat(filename):
|
|
try:
|
|
return os.stat(filename)
|
|
except OSError, e:
|
|
if e.errno == errno.ENOENT:
|
|
return None
|
|
else:
|
|
raise
|
|
|
|
|
|
class BuildJob:
|
|
def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
|
|
self.t = t # original target name, not relative to vars.BASE
|
|
self.sf = sf
|
|
self.tmpname = '%s.redo.tmp' % t
|
|
self.lock = lock
|
|
self.shouldbuildfunc = shouldbuildfunc
|
|
self.donefunc = donefunc
|
|
self.before_t = _try_stat(self.t)
|
|
|
|
def start(self):
|
|
assert(self.lock.owned)
|
|
t = self.t
|
|
sf = self.sf
|
|
tmpname = self.tmpname
|
|
if not self.shouldbuildfunc(t):
|
|
# target doesn't need to be built; skip the whole task
|
|
return self._after2(0)
|
|
if (os.path.exists(t) and not os.path.exists(t + '/.')
|
|
and not sf.is_generated):
|
|
# an existing source file that was not generated by us.
|
|
# This step is mentioned by djb in his notes.
|
|
# For example, a rule called default.c.do could be used to try
|
|
# to produce hello.c, but we don't want that to happen if
|
|
# hello.c was created by the end user.
|
|
# FIXME: always refuse to redo any file that was modified outside
|
|
# of redo? That would make it easy for someone to override a
|
|
# file temporarily, and could be undone by deleting the file.
|
|
debug2("-- static (%r)\n" % t)
|
|
sf.set_static()
|
|
sf.save()
|
|
return self._after2(0)
|
|
sf.zap_deps()
|
|
(dofile, basename, ext) = _find_do_file(sf)
|
|
if not dofile:
|
|
if os.path.exists(t):
|
|
sf.is_generated = False
|
|
sf.set_static()
|
|
sf.save()
|
|
return self._after2(0)
|
|
else:
|
|
err('no rule to make %r\n' % t)
|
|
return self._after2(1)
|
|
unlink(tmpname)
|
|
ffd = os.open(tmpname, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
|
|
close_on_exec(ffd, True)
|
|
self.f = os.fdopen(ffd, 'w+')
|
|
# this will run in the dofile's directory, so use only basenames here
|
|
argv = ['sh', '-e',
|
|
os.path.basename(dofile),
|
|
os.path.basename(basename), # target name (extension removed)
|
|
ext, # extension (if any), including leading dot
|
|
os.path.basename(tmpname) # randomized output file name
|
|
]
|
|
if vars.VERBOSE: argv[1] += 'v'
|
|
if vars.XTRACE: argv[1] += 'x'
|
|
if vars.VERBOSE or vars.XTRACE: log_('\n')
|
|
log('%s\n' % _nice(t))
|
|
self.argv = argv
|
|
sf.is_generated = True
|
|
sf.save()
|
|
dof = state.File(name=dofile)
|
|
dof.set_static()
|
|
dof.save()
|
|
state.commit()
|
|
jwack.start_job(t, self._do_subproc, self._after)
|
|
|
|
def _do_subproc(self):
|
|
# careful: REDO_PWD was the PWD relative to the STARTPATH at the time
|
|
# we *started* building the current target; but that target ran
|
|
# redo-ifchange, and it might have done it from a different directory
|
|
# than we started it in. So os.getcwd() might be != REDO_PWD right
|
|
# now.
|
|
dn = os.path.dirname(self.t)
|
|
newp = os.path.realpath(dn)
|
|
os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
|
|
os.environ['REDO_TARGET'] = os.path.basename(self.t)
|
|
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
|
|
if dn:
|
|
os.chdir(dn)
|
|
os.dup2(self.f.fileno(), 1)
|
|
os.close(self.f.fileno())
|
|
close_on_exec(1, False)
|
|
if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv))
|
|
os.execvp(self.argv[0], self.argv)
|
|
assert(0)
|
|
# returns only if there's an exception
|
|
|
|
def _after(self, t, rv):
|
|
try:
|
|
state.check_sane()
|
|
rv = self._after1(t, rv)
|
|
state.commit()
|
|
finally:
|
|
self._after2(rv)
|
|
|
|
def _after1(self, t, rv):
|
|
f = self.f
|
|
tmpname = self.tmpname
|
|
before_t = self.before_t
|
|
after_t = _try_stat(t)
|
|
before_tmp = os.fstat(f.fileno())
|
|
after_tmp = _try_stat(tmpname)
|
|
after_where = os.lseek(f.fileno(), 0, os.SEEK_CUR)
|
|
if after_t != before_t and not stat.S_ISDIR(after_t.st_mode):
|
|
err('%r modified %r directly!\n' % (self.argv[2], t))
|
|
err('...you should update $3 (a temp file) instead of $1.\n')
|
|
rv = 206
|
|
elif after_tmp and before_tmp != after_tmp and before_tmp.st_size > 0:
|
|
err('%r wrote to stdout *and* replaced $3.\n' % self.argv[2])
|
|
err('...you should write status messages to stderr, not stdout.\n')
|
|
rv = 207
|
|
elif after_where > 0 and after_tmp and after_tmp.st_size != after_where:
|
|
err('%r wrote differing data to stdout and $3.\n' % self.argv[2])
|
|
err('...you should write status messages to stderr, not stdout.\n')
|
|
rv = 208
|
|
if rv==0:
|
|
if os.path.exists(tmpname) and os.stat(tmpname).st_size:
|
|
# there's a race condition here, but if the tmpfile disappears
|
|
# at *this* point you deserve to get an error, because you're
|
|
# doing something totally scary.
|
|
os.rename(tmpname, t)
|
|
else:
|
|
unlink(tmpname)
|
|
sf = self.sf
|
|
sf.is_generated=True
|
|
sf.update_stamp()
|
|
sf.set_changed()
|
|
sf.save()
|
|
else:
|
|
unlink(tmpname)
|
|
sf = self.sf
|
|
sf.stamp = None
|
|
sf.set_changed()
|
|
sf.save()
|
|
f.close()
|
|
if rv != 0:
|
|
err('%s: exit code %d\n' % (_nice(t),rv))
|
|
else:
|
|
if vars.VERBOSE or vars.XTRACE:
|
|
log('%s (done)\n\n' % _nice(t))
|
|
return rv
|
|
|
|
def _after2(self, rv):
|
|
try:
|
|
self.donefunc(self.t, rv)
|
|
assert(self.lock.owned)
|
|
finally:
|
|
self.lock.unlock()
|
|
|
|
|
|
def main(targets, shouldbuildfunc):
|
|
retcode = [0] # a list so that it can be reassigned from done()
|
|
if vars.SHUFFLE:
|
|
import random
|
|
random.shuffle(targets)
|
|
|
|
locked = []
|
|
|
|
def done(t, rv):
|
|
if rv:
|
|
retcode[0] = 1
|
|
|
|
for i in range(len(targets)):
|
|
t = targets[i]
|
|
|
|
# In the first cycle, we just build as much as we can without worrying
|
|
# about any lock contention. If someone else has it locked, we move on.
|
|
for t in targets:
|
|
if not jwack.has_token():
|
|
state.commit()
|
|
jwack.get_token(t)
|
|
if retcode[0] and not vars.KEEP_GOING:
|
|
break
|
|
if not state.check_sane():
|
|
err('.redo directory disappeared; cannot continue.\n')
|
|
retcode[0] = 205
|
|
break
|
|
f = state.File(name=t)
|
|
lock = state.Lock(f.id)
|
|
lock.trylock()
|
|
if not lock.owned:
|
|
if vars.DEBUG_LOCKS:
|
|
log('%s (locked...)\n' % _nice(t))
|
|
locked.append((f.id,t))
|
|
else:
|
|
BuildJob(t, f, lock, shouldbuildfunc, done).start()
|
|
|
|
# Now we've built all the "easy" ones. Go back and just wait on the
|
|
# remaining ones one by one. This is non-optimal; we could go faster if
|
|
# we could wait on multiple locks at once. But it should
|
|
# be rare enough that it doesn't matter, and the logic is easier this way.
|
|
while locked or jwack.running():
|
|
state.commit()
|
|
jwack.wait_all()
|
|
# at this point, we don't have any children holding any tokens, so
|
|
# it's okay to block below.
|
|
if retcode[0] and not vars.KEEP_GOING:
|
|
break
|
|
if locked:
|
|
if not state.check_sane():
|
|
err('.redo directory disappeared; cannot continue.\n')
|
|
retcode[0] = 205
|
|
break
|
|
fid,t = locked.pop(0)
|
|
lock = state.Lock(fid)
|
|
lock.waitlock()
|
|
assert(lock.owned)
|
|
if vars.DEBUG_LOCKS:
|
|
log('%s (...unlocked!)\n' % _nice(t))
|
|
if state.File(name=t).stamp == None:
|
|
err('%s: failed in another thread\n' % _nice(t))
|
|
retcode[0] = 2
|
|
lock.unlock()
|
|
else:
|
|
BuildJob(t, state.File(id=fid), lock,
|
|
shouldbuildfunc, done).start()
|
|
state.commit()
|
|
return retcode[0]
|