Split build functions into builder.py.
This commit is contained in:
Avery Pennarun
parent
116e7e5f13
commit
f7734c3b8d
2 changed files with 151 additions and 150 deletions
148
builder.py
Normal file
148
builder.py
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
import sys, os, subprocess, random
|
||||||
|
import vars, jwack, state
|
||||||
|
from helpers import log, log_, relpath, debug2, err, unlink
|
||||||
|
|
||||||
|
|
||||||
|
class BuildError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _possible_do_files(t):
|
||||||
|
yield "%s.do" % t, t, ''
|
||||||
|
dirname,filename = os.path.split(t)
|
||||||
|
l = filename.split('.')
|
||||||
|
l[0] = os.path.join(dirname, l[0])
|
||||||
|
for i in range(1,len(l)+1):
|
||||||
|
basename = '.'.join(l[:i])
|
||||||
|
ext = '.'.join(l[i:])
|
||||||
|
if ext: ext = '.' + ext
|
||||||
|
yield (os.path.join(dirname, "default%s.do" % ext),
|
||||||
|
os.path.join(dirname, basename), ext)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_do_file(t):
|
||||||
|
for dofile,basename,ext in _possible_do_files(t):
|
||||||
|
debug2('%s: %s ?\n' % (t, dofile))
|
||||||
|
if os.path.exists(dofile):
|
||||||
|
state.add_dep(t, 'm', dofile)
|
||||||
|
return dofile,basename,ext
|
||||||
|
else:
|
||||||
|
state.add_dep(t, 'c', dofile)
|
||||||
|
return None,None,None
|
||||||
|
|
||||||
|
|
||||||
|
def _preexec(t):
|
||||||
|
os.environ['REDO_TARGET'] = os.path.basename(t)
|
||||||
|
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
|
||||||
|
dn = os.path.dirname(t)
|
||||||
|
if dn:
|
||||||
|
os.chdir(dn)
|
||||||
|
|
||||||
|
|
||||||
|
def _build(t):
|
||||||
|
if (os.path.exists(t) and not state.is_generated(t)
|
||||||
|
and not os.path.exists('%s.do' % t)):
|
||||||
|
# an existing source file that is not marked as a generated file.
|
||||||
|
# This step is mentioned by djb in his notes. It turns out to be
|
||||||
|
# important to prevent infinite recursion. For example, a rule
|
||||||
|
# called default.c.do could be used to try to produce hello.c,
|
||||||
|
# which is undesirable since hello.c existed already.
|
||||||
|
state.stamp(t)
|
||||||
|
return # success
|
||||||
|
state.start(t)
|
||||||
|
(dofile, basename, ext) = _find_do_file(t)
|
||||||
|
if not dofile:
|
||||||
|
raise BuildError('no rule to make %r' % t)
|
||||||
|
state.stamp(dofile)
|
||||||
|
tmpname = '%s.redo.tmp' % t
|
||||||
|
unlink(tmpname)
|
||||||
|
f = open(tmpname, 'w+')
|
||||||
|
|
||||||
|
# this will run in the dofile's directory, so use only basenames here
|
||||||
|
argv = ['sh', '-e',
|
||||||
|
os.path.basename(dofile),
|
||||||
|
os.path.basename(basename), # target name (extension removed)
|
||||||
|
ext, # extension (if any), including leading dot
|
||||||
|
os.path.basename(tmpname) # randomized output file name
|
||||||
|
]
|
||||||
|
if vars.VERBOSE:
|
||||||
|
argv[1] += 'v'
|
||||||
|
log_('\n')
|
||||||
|
log('%s\n' % relpath(t, vars.STARTDIR))
|
||||||
|
rv = subprocess.call(argv, preexec_fn=lambda: _preexec(t),
|
||||||
|
stdout=f.fileno())
|
||||||
|
if rv==0:
|
||||||
|
if os.path.exists(tmpname) and os.stat(tmpname).st_size:
|
||||||
|
# there's a race condition here, but if the tmpfile disappears
|
||||||
|
# at *this* point you deserve to get an error, because you're
|
||||||
|
# doing something totally scary.
|
||||||
|
os.rename(tmpname, t)
|
||||||
|
else:
|
||||||
|
unlink(tmpname)
|
||||||
|
state.stamp(t)
|
||||||
|
else:
|
||||||
|
unlink(tmpname)
|
||||||
|
state.unstamp(t)
|
||||||
|
f.close()
|
||||||
|
if rv != 0:
|
||||||
|
raise BuildError('%s: exit code %d' % (t,rv))
|
||||||
|
if vars.VERBOSE:
|
||||||
|
log('%s (done)\n\n' % relpath(t, vars.STARTDIR))
|
||||||
|
|
||||||
|
|
||||||
|
def build(t):
|
||||||
|
try:
|
||||||
|
return _build(t)
|
||||||
|
except BuildError, e:
|
||||||
|
err('%s\n' % e)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def main(targets, buildfunc):
|
||||||
|
retcode = [0] # a list so that it can be reassigned from done()
|
||||||
|
if vars.SHUFFLE:
|
||||||
|
random.shuffle(targets)
|
||||||
|
|
||||||
|
locked = []
|
||||||
|
|
||||||
|
def done(t, rv):
|
||||||
|
if rv:
|
||||||
|
err('%s: exit code was %r\n' % (t, rv))
|
||||||
|
retcode[0] = 1
|
||||||
|
|
||||||
|
for i in range(len(targets)):
|
||||||
|
t = targets[i]
|
||||||
|
if os.path.exists('%s/all.do' % t):
|
||||||
|
# t is a directory, but it has a default target
|
||||||
|
targets[i] = '%s/all' % t
|
||||||
|
|
||||||
|
for t in targets:
|
||||||
|
jwack.get_token(t)
|
||||||
|
lock = state.Lock(t)
|
||||||
|
lock.trylock()
|
||||||
|
if not lock.owned:
|
||||||
|
log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
|
||||||
|
locked.append(t)
|
||||||
|
else:
|
||||||
|
jwack.start_job(t, lock,
|
||||||
|
lambda: buildfunc(t), lambda t,rv: done(t,rv))
|
||||||
|
|
||||||
|
while locked or jwack.running():
|
||||||
|
jwack.wait_all()
|
||||||
|
if locked:
|
||||||
|
t = locked.pop(0)
|
||||||
|
lock = state.Lock(t)
|
||||||
|
while not lock.owned:
|
||||||
|
lock.wait()
|
||||||
|
lock.trylock()
|
||||||
|
assert(lock.owned)
|
||||||
|
relp = relpath(t, vars.STARTDIR)
|
||||||
|
log('%s (...unlocked!)\n' % relp)
|
||||||
|
if state.stamped(t) == None:
|
||||||
|
err('%s: failed in another thread\n' % relp)
|
||||||
|
retcode[0] = 2
|
||||||
|
lock.unlock()
|
||||||
|
else:
|
||||||
|
jwack.start_job(t, lock,
|
||||||
|
lambda: buildfunc(t), lambda t,rv: done(t,rv))
|
||||||
|
return retcode[0]
|
||||||
153
redo.py
153
redo.py
|
|
@ -1,5 +1,5 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/python
|
||||||
import sys, os, subprocess, glob, time, random
|
import sys, os, glob
|
||||||
import options, jwack, atoi
|
import options, jwack, atoi
|
||||||
|
|
||||||
optspec = """
|
optspec = """
|
||||||
|
|
@ -38,7 +38,7 @@ if not os.environ.get('REDO_BASE', ''):
|
||||||
os.environ['REDO'] = os.path.abspath(sys.argv[0])
|
os.environ['REDO'] = os.path.abspath(sys.argv[0])
|
||||||
|
|
||||||
|
|
||||||
import vars, state
|
import vars, builder
|
||||||
from helpers import *
|
from helpers import *
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -52,153 +52,6 @@ if is_root:
|
||||||
os.unlink(f)
|
os.unlink(f)
|
||||||
|
|
||||||
|
|
||||||
class BuildError(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def _possible_do_files(t):
|
|
||||||
yield "%s.do" % t, t, ''
|
|
||||||
dirname,filename = os.path.split(t)
|
|
||||||
l = filename.split('.')
|
|
||||||
l[0] = os.path.join(dirname, l[0])
|
|
||||||
for i in range(1,len(l)+1):
|
|
||||||
basename = '.'.join(l[:i])
|
|
||||||
ext = '.'.join(l[i:])
|
|
||||||
if ext: ext = '.' + ext
|
|
||||||
yield (os.path.join(dirname, "default%s.do" % ext),
|
|
||||||
os.path.join(dirname, basename), ext)
|
|
||||||
|
|
||||||
|
|
||||||
def find_do_file(t):
|
|
||||||
for dofile,basename,ext in _possible_do_files(t):
|
|
||||||
debug2('%s: %s ?\n' % (t, dofile))
|
|
||||||
if os.path.exists(dofile):
|
|
||||||
state.add_dep(t, 'm', dofile)
|
|
||||||
return dofile,basename,ext
|
|
||||||
else:
|
|
||||||
state.add_dep(t, 'c', dofile)
|
|
||||||
return None,None,None
|
|
||||||
|
|
||||||
|
|
||||||
def _preexec(t):
|
|
||||||
os.environ['REDO_TARGET'] = os.path.basename(t)
|
|
||||||
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
|
|
||||||
dn = os.path.dirname(t)
|
|
||||||
if dn:
|
|
||||||
os.chdir(dn)
|
|
||||||
|
|
||||||
|
|
||||||
def _build(t):
|
|
||||||
if (os.path.exists(t) and not state.is_generated(t)
|
|
||||||
and not os.path.exists('%s.do' % t)):
|
|
||||||
# an existing source file that is not marked as a generated file.
|
|
||||||
# This step is mentioned by djb in his notes. It turns out to be
|
|
||||||
# important to prevent infinite recursion. For example, a rule
|
|
||||||
# called default.c.do could be used to try to produce hello.c,
|
|
||||||
# which is undesirable since hello.c existed already.
|
|
||||||
state.stamp(t)
|
|
||||||
return # success
|
|
||||||
state.start(t)
|
|
||||||
(dofile, basename, ext) = find_do_file(t)
|
|
||||||
if not dofile:
|
|
||||||
raise BuildError('no rule to make %r' % t)
|
|
||||||
state.stamp(dofile)
|
|
||||||
tmpname = '%s.redo.tmp' % t
|
|
||||||
unlink(tmpname)
|
|
||||||
f = open(tmpname, 'w+')
|
|
||||||
|
|
||||||
# this will run in the dofile's directory, so use only basenames here
|
|
||||||
argv = ['sh', '-e',
|
|
||||||
os.path.basename(dofile),
|
|
||||||
os.path.basename(basename), # target name (extension removed)
|
|
||||||
ext, # extension (if any), including leading dot
|
|
||||||
os.path.basename(tmpname) # randomized output file name
|
|
||||||
]
|
|
||||||
if vars.VERBOSE:
|
|
||||||
argv[1] += 'v'
|
|
||||||
log_('\n')
|
|
||||||
log('%s\n' % relpath(t, vars.STARTDIR))
|
|
||||||
rv = subprocess.call(argv, preexec_fn=lambda: _preexec(t),
|
|
||||||
stdout=f.fileno())
|
|
||||||
if rv==0:
|
|
||||||
if os.path.exists(tmpname) and os.stat(tmpname).st_size:
|
|
||||||
# there's a race condition here, but if the tmpfile disappears
|
|
||||||
# at *this* point you deserve to get an error, because you're
|
|
||||||
# doing something totally scary.
|
|
||||||
os.rename(tmpname, t)
|
|
||||||
else:
|
|
||||||
unlink(tmpname)
|
|
||||||
state.stamp(t)
|
|
||||||
else:
|
|
||||||
unlink(tmpname)
|
|
||||||
state.unstamp(t)
|
|
||||||
f.close()
|
|
||||||
if rv != 0:
|
|
||||||
raise BuildError('%s: exit code %d' % (t,rv))
|
|
||||||
if vars.VERBOSE:
|
|
||||||
log('%s (done)\n\n' % relpath(t, vars.STARTDIR))
|
|
||||||
|
|
||||||
|
|
||||||
def build(t):
|
|
||||||
try:
|
|
||||||
return _build(t)
|
|
||||||
except BuildError, e:
|
|
||||||
err('%s\n' % e)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def main(targets, buildfunc):
|
|
||||||
retcode = [0] # a list so that it can be reassigned from done()
|
|
||||||
if vars.SHUFFLE:
|
|
||||||
random.shuffle(targets)
|
|
||||||
|
|
||||||
locked = []
|
|
||||||
|
|
||||||
def done(t, rv):
|
|
||||||
if rv == 199:
|
|
||||||
locked.append(t)
|
|
||||||
elif rv:
|
|
||||||
err('%s: exit code was %r\n' % (t, rv))
|
|
||||||
retcode[0] = 1
|
|
||||||
|
|
||||||
for i in range(len(targets)):
|
|
||||||
t = targets[i]
|
|
||||||
if os.path.exists('%s/all.do' % t):
|
|
||||||
# t is a directory, but it has a default target
|
|
||||||
targets[i] = '%s/all' % t
|
|
||||||
|
|
||||||
for t in targets:
|
|
||||||
jwack.get_token(t)
|
|
||||||
lock = state.Lock(t)
|
|
||||||
lock.trylock()
|
|
||||||
if not lock.owned:
|
|
||||||
log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
|
|
||||||
locked.append(t)
|
|
||||||
else:
|
|
||||||
jwack.start_job(t, lock,
|
|
||||||
lambda: buildfunc(t), lambda t,rv: done(t,rv))
|
|
||||||
|
|
||||||
while locked or jwack.running():
|
|
||||||
jwack.wait_all()
|
|
||||||
if locked:
|
|
||||||
t = locked.pop(0)
|
|
||||||
lock = state.Lock(t)
|
|
||||||
while not lock.owned:
|
|
||||||
lock.wait()
|
|
||||||
lock.trylock()
|
|
||||||
assert(lock.owned)
|
|
||||||
relp = relpath(t, vars.STARTDIR)
|
|
||||||
log('%s (...unlocked!)\n' % relp)
|
|
||||||
if state.stamped(t) == None:
|
|
||||||
err('%s: failed in another thread\n' % relp)
|
|
||||||
retcode[0] = 2
|
|
||||||
lock.unlock()
|
|
||||||
else:
|
|
||||||
jwack.start_job(t, lock,
|
|
||||||
lambda: buildfunc(t), lambda t,rv: done(t,rv))
|
|
||||||
return retcode[0]
|
|
||||||
|
|
||||||
|
|
||||||
if not vars.DEPTH:
|
if not vars.DEPTH:
|
||||||
# toplevel call to redo
|
# toplevel call to redo
|
||||||
exenames = [os.path.abspath(sys.argv[0]), os.path.realpath(sys.argv[0])]
|
exenames = [os.path.abspath(sys.argv[0]), os.path.realpath(sys.argv[0])]
|
||||||
|
|
@ -213,7 +66,7 @@ try:
|
||||||
err('invalid --jobs value: %r\n' % opt.jobs)
|
err('invalid --jobs value: %r\n' % opt.jobs)
|
||||||
jwack.setup(j)
|
jwack.setup(j)
|
||||||
try:
|
try:
|
||||||
retcode = main(targets, build)
|
retcode = builder.main(targets, builder.build)
|
||||||
finally:
|
finally:
|
||||||
jwack.force_return_tokens()
|
jwack.force_return_tokens()
|
||||||
if retcode:
|
if retcode:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue