forks/apenwarr-redo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

env and env_init: Eliminate weird auto-initialization of globals.

Browse source 
Merge the two files into env, and make each command explicitly call the
function that sets it up in the way that's needed for that command.

This means we can finally just import all the modules at the top of
each file, without worrying about import order.  Phew.

While we're here, remove the weird auto-appending-'all'-to-targets
feature in env.init().  Instead, do it explicitly, and only from redo and
redo-ifchange, only if is_toplevel and no other targets are given.
This commit is contained in:
Avery Pennarun 2018-12-05 01:07:16 -05:00
commit 9b6d1eeb6e
19 changed files with 267 additions and 256 deletions
Download patch file Download diff file Expand all files Collapse all files

44
redo/builder.py
View file

@ -6,7 +6,7 @@ from logs import debug2, err, warn, meta, check_tty
def _nice(t): def _nice(t):
return state.relpath(t, env.STARTDIR) return state.relpath(t, env.v.STARTDIR)
def _try_stat(filename): def _try_stat(filename):
@ -53,7 +53,7 @@ def start_stdin_log_reader(status, details, pretty, color,
os.dup2(w, 1) os.dup2(w, 1)
os.dup2(w, 2) os.dup2(w, 2)
os.close(w) os.close(w)
check_tty(sys.stderr, env.COLOR) check_tty(sys.stderr, env.v.COLOR)
else: else:
# child # child
try: try:
@ -88,7 +88,7 @@ def start_stdin_log_reader(status, details, pretty, color,
def await_log_reader(): def await_log_reader():
if not env.LOG: if not env.v.LOG:
return return
if log_reader_pid > 0: if log_reader_pid > 0:
# never actually close fd#1 or fd#2; insanity awaits. # never actually close fd#1 or fd#2; insanity awaits.
@ -110,7 +110,7 @@ class ImmediateReturn(Exception):
class BuildJob(object): class BuildJob(object):
def __init__(self, t, sf, lock, shouldbuildfunc, donefunc): def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
self.t = t # original target name, not relative to env.BASE self.t = t # original target name, not relative to env.v.BASE
self.sf = sf self.sf = sf
tmpbase = t tmpbase = t
while not os.path.isdir(os.path.dirname(tmpbase) or '.'): while not os.path.isdir(os.path.dirname(tmpbase) or '.'):
@ -140,7 +140,7 @@ class BuildJob(object):
except ImmediateReturn, e: except ImmediateReturn, e:
return self._after2(e.rv) return self._after2(e.rv)
if env.NO_OOB or dirty == True: # pylint: disable=singleton-comparison if env.v.NO_OOB or dirty == True: # pylint: disable=singleton-comparison
self._start_do() self._start_do()
else: else:
self._start_unlocked(dirty) self._start_unlocked(dirty)
@ -198,16 +198,16 @@ class BuildJob(object):
# temp output file name # temp output file name
state.relpath(os.path.abspath(self.tmpname2), dodir), state.relpath(os.path.abspath(self.tmpname2), dodir),
] ]
if env.VERBOSE: if env.v.VERBOSE:
argv[1] += 'v' argv[1] += 'v'
if env.XTRACE: if env.v.XTRACE:
argv[1] += 'x' argv[1] += 'x'
firstline = open(os.path.join(dodir, dofile)).readline().strip() firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!/'): if firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ') argv[0:2] = firstline[2:].split(' ')
# make sure to create the logfile *before* writing the meta() about it. # make sure to create the logfile *before* writing the meta() about it.
# that way redo-log won't trace into an obsolete logfile. # that way redo-log won't trace into an obsolete logfile.
if env.LOG: if env.v.LOG:
open(state.logname(self.sf.id), 'w') open(state.logname(self.sf.id), 'w')
# FIXME: put these variables somewhere else, instead of on-the-fly # FIXME: put these variables somewhere else, instead of on-the-fly
# extending this class! # extending this class!
@ -236,13 +236,13 @@ class BuildJob(object):
# grab a lock. # grab a lock.
here = os.getcwd() here = os.getcwd()
def _fix(p): def _fix(p):
return state.relpath(os.path.join(env.BASE, p), here) return state.relpath(os.path.join(env.v.BASE, p), here)
argv = (['redo-unlocked', _fix(self.sf.name)] + argv = (['redo-unlocked', _fix(self.sf.name)] +
list(set(_fix(d.name) for d in dirty))) list(set(_fix(d.name) for d in dirty)))
meta('check', state.target_relpath(self.t)) meta('check', state.target_relpath(self.t))
state.commit() state.commit()
def run(): def run():
os.environ['REDO_DEPTH'] = env.DEPTH + ' ' os.environ['REDO_DEPTH'] = env.v.DEPTH + ' '
# python ignores SIGPIPE # python ignores SIGPIPE
signal.signal(signal.SIGPIPE, signal.SIG_DFL) signal.signal(signal.SIGPIPE, signal.SIG_DFL)
os.execvp(argv[0], argv) os.execvp(argv[0], argv)
@ -261,18 +261,18 @@ class BuildJob(object):
assert state.is_flushed() assert state.is_flushed()
dn = self.dodir dn = self.dodir
newp = os.path.realpath(dn) newp = os.path.realpath(dn)
os.environ['REDO_PWD'] = state.relpath(newp, env.STARTDIR) os.environ['REDO_PWD'] = state.relpath(newp, env.v.STARTDIR)
os.environ['REDO_TARGET'] = self.basename + self.ext os.environ['REDO_TARGET'] = self.basename + self.ext
os.environ['REDO_DEPTH'] = env.DEPTH + ' ' os.environ['REDO_DEPTH'] = env.v.DEPTH + ' '
cycles.add(self.lock.fid) cycles.add(self.lock.fid)
if dn: if dn:
os.chdir(dn) os.chdir(dn)
os.dup2(self.f.fileno(), 1) os.dup2(self.f.fileno(), 1)
os.close(self.f.fileno()) os.close(self.f.fileno())
close_on_exec(1, False) close_on_exec(1, False)
if env.LOG: if env.v.LOG:
cur_inode = str(os.fstat(2).st_ino) cur_inode = str(os.fstat(2).st_ino)
if not env.LOG_INODE or cur_inode == env.LOG_INODE: if not env.v.LOG_INODE or cur_inode == env.v.LOG_INODE:
# .do script has *not* redirected stderr, which means we're # .do script has *not* redirected stderr, which means we're
# using redo-log's log saving mode. That means subprocs # using redo-log's log saving mode. That means subprocs
# should be logged to their own file. If the .do script # should be logged to their own file. If the .do script
@ -290,7 +290,7 @@ class BuildJob(object):
del os.environ['REDO_LOG_INODE'] del os.environ['REDO_LOG_INODE']
os.environ['REDO_LOG'] = '' os.environ['REDO_LOG'] = ''
signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE
if env.VERBOSE or env.XTRACE: if env.v.VERBOSE or env.v.XTRACE:
logs.write('* %s' % ' '.join(self.argv).replace('\n', ' ')) logs.write('* %s' % ' '.join(self.argv).replace('\n', ' '))
os.execvp(self.argv[0], self.argv) os.execvp(self.argv[0], self.argv)
# FIXME: it would be nice to log the exit code to logf. # FIXME: it would be nice to log the exit code to logf.
@ -387,7 +387,7 @@ class BuildJob(object):
def main(targets, shouldbuildfunc): def main(targets, shouldbuildfunc):
retcode = [0] # a list so that it can be reassigned from done() retcode = [0] # a list so that it can be reassigned from done()
if env.SHUFFLE: if env.v.SHUFFLE:
import random import random
random.shuffle(targets) random.shuffle(targets)
@ -397,9 +397,9 @@ def main(targets, shouldbuildfunc):
if rv: if rv:
retcode[0] = 1 retcode[0] = 1
if env.TARGET and not env.UNLOCKED: if env.v.TARGET and not env.v.UNLOCKED:
me = os.path.join(env.STARTDIR, me = os.path.join(env.v.STARTDIR,
os.path.join(env.PWD, env.TARGET)) os.path.join(env.v.PWD, env.v.TARGET))
myfile = state.File(name=me) myfile = state.File(name=me)
selflock = state.Lock(state.LOG_LOCK_MAGIC + myfile.id) selflock = state.Lock(state.LOG_LOCK_MAGIC + myfile.id)
else: else:
@ -435,7 +435,7 @@ def main(targets, shouldbuildfunc):
if not jobserver.has_token(): if not jobserver.has_token():
state.commit() state.commit()
jobserver.ensure_token_or_cheat(t, cheat) jobserver.ensure_token_or_cheat(t, cheat)
if retcode[0] and not env.KEEP_GOING: if retcode[0] and not env.v.KEEP_GOING:
break break
if not state.check_sane(): if not state.check_sane():
err('.redo directory disappeared; cannot continue.\n') err('.redo directory disappeared; cannot continue.\n')
@ -443,7 +443,7 @@ def main(targets, shouldbuildfunc):
break break
f = state.File(name=t) f = state.File(name=t)
lock = state.Lock(f.id) lock = state.Lock(f.id)
if env.UNLOCKED: if env.v.UNLOCKED:
lock.owned = True lock.owned = True
else: else:
lock.trylock() lock.trylock()
@ -478,7 +478,7 @@ def main(targets, shouldbuildfunc):
jobserver.ensure_token_or_cheat('self', cheat) jobserver.ensure_token_or_cheat('self', cheat)
# at this point, we don't have any children holding any tokens, so # at this point, we don't have any children holding any tokens, so
# it's okay to block below. # it's okay to block below.
if retcode[0] and not env.KEEP_GOING: if retcode[0] and not env.v.KEEP_GOING:
break break
if locked: if locked:
if not state.check_sane(): if not state.check_sane():

5
redo/cmd_always.py
View file

@ -4,8 +4,9 @@ import env, state
def main(): def main():
try: try:
me = os.path.join(env.STARTDIR, env.inherit()
os.path.join(env.PWD, env.TARGET)) me = os.path.join(env.v.STARTDIR,
os.path.join(env.v.PWD, env.v.TARGET))
f = state.File(name=me) f = state.File(name=me)
f.add_dep('m', state.ALWAYS) f.add_dep('m', state.ALWAYS)
always = state.File(name=state.ALWAYS) always = state.File(name=state.ALWAYS)

26
redo/cmd_ifchange.py
View file

@ -1,16 +1,13 @@
import os, sys, traceback import os, sys, traceback
import env_init
env_init.init(sys.argv[1:])
import env, state, builder, jobserver, deps import env, state, builder, jobserver, deps
from logs import debug2, err from logs import debug2, err
def should_build(t): def should_build(t):
f = state.File(name=t) f = state.File(name=t)
if f.is_failed(): if f.is_failed():
raise builder.ImmediateReturn(32) raise builder.ImmediateReturn(32)
dirty = deps.isdirty(f, depth='', max_changed=env.RUNID, dirty = deps.isdirty(f, depth='', max_changed=env.v.RUNID,
already_checked=[]) already_checked=[])
return f.is_generated, dirty == [f] and deps.DIRTY or dirty return f.is_generated, dirty == [f] and deps.DIRTY or dirty
@ -18,23 +15,26 @@ def should_build(t):
def main(): def main():
rv = 202 rv = 202
try: try:
if env_init.is_toplevel and env.LOG: targets = sys.argv[1:]
state.init(targets)
if env.is_toplevel and not targets:
targets = ['all']
if env.is_toplevel and env.v.LOG:
builder.close_stdin() builder.close_stdin()
builder.start_stdin_log_reader( builder.start_stdin_log_reader(
status=True, details=True, status=True, details=True,
pretty=True, color=True, debug_locks=False, debug_pids=False) pretty=True, color=True, debug_locks=False, debug_pids=False)
if env.TARGET and not env.UNLOCKED: if env.v.TARGET and not env.v.UNLOCKED:
me = os.path.join(env.STARTDIR, me = os.path.join(env.v.STARTDIR,
os.path.join(env.PWD, env.TARGET)) os.path.join(env.v.PWD, env.v.TARGET))
f = state.File(name=me) f = state.File(name=me)
debug2('TARGET: %r %r %r\n' debug2('TARGET: %r %r %r\n'
% (env.STARTDIR, env.PWD, env.TARGET)) % (env.v.STARTDIR, env.v.PWD, env.v.TARGET))
else: else:
f = me = None f = me = None
debug2('redo-ifchange: not adding depends.\n') debug2('redo-ifchange: not adding depends.\n')
jobserver.setup(1) jobserver.setup(1)
try: try:
targets = sys.argv[1:]
if f: if f:
for t in targets: for t in targets:
f.add_dep('m', t) f.add_dep('m', t)
@ -52,11 +52,11 @@ def main():
err('unexpected error: %r\n' % e) err('unexpected error: %r\n' % e)
rv = 1 rv = 1
except KeyboardInterrupt: except KeyboardInterrupt:
if env_init.is_toplevel: if env.is_toplevel:
builder.await_log_reader() builder.await_log_reader()
sys.exit(200) sys.exit(200)
state.commit() state.commit()
if env_init.is_toplevel: if env.is_toplevel:
builder.await_log_reader() builder.await_log_reader()
sys.exit(rv) sys.exit(rv)

5
redo/cmd_ifcreate.py
View file

@ -5,8 +5,9 @@ from logs import err
def main(): def main():
try: try:
me = os.path.join(env.STARTDIR, env.inherit()
os.path.join(env.PWD, env.TARGET)) me = os.path.join(env.v.STARTDIR,
os.path.join(env.v.PWD, env.v.TARGET))
f = state.File(name=me) f = state.File(name=me)
for t in sys.argv[1:]: for t in sys.argv[1:]:
if not t: if not t:

23
redo/cmd_log.py
View file

@ -1,7 +1,7 @@
import errno, fcntl, os, re, struct, sys, time import errno, fcntl, os, re, struct, sys, time
import termios import termios
from atoi import atoi from atoi import atoi
import options import env, logs, options, state
optspec = """ optspec = """
redo-log [options...] [targets...] redo-log [options...] [targets...]
@ -21,11 +21,6 @@ o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:]) (opt, flags, extra) = o.parse(sys.argv[1:])
targets = extra targets = extra
import env_init
env_init.init(list(targets))
import env, logs, state
topdir = os.getcwd() topdir = os.getcwd()
already = set() already = set()
depth = [] depth = []
@ -42,19 +37,12 @@ start_time = time.time()
REDO_LINE_RE = re.compile(r'^@@REDO:([^@]+)@@ (.*)\n$') REDO_LINE_RE = re.compile(r'^@@REDO:([^@]+)@@ (.*)\n$')
def _atoi(s):
try:
return int(s)
except TypeError:
return 0
def _tty_width(): def _tty_width():
s = struct.pack("HHHH", 0, 0, 0, 0) s = struct.pack("HHHH", 0, 0, 0, 0)
try: try:
s = fcntl.ioctl(sys.stderr.fileno(), termios.TIOCGWINSZ, s) s = fcntl.ioctl(sys.stderr.fileno(), termios.TIOCGWINSZ, s)
except (IOError, ImportError): except (IOError, ImportError):
return _atoi(os.environ.get('WIDTH')) or 70 return atoi(os.environ.get('WIDTH')) or 70
(ysize, xsize, ypix, xpix) = struct.unpack('HHHH', s) (ysize, xsize, ypix, xpix) = struct.unpack('HHHH', s)
return xsize or 70 return xsize or 70
@ -64,7 +52,7 @@ def is_locked(fid):
def _fix_depth(): def _fix_depth():
env.DEPTH = len(depth) * ' ' env.v.DEPTH = len(depth) * ' '
def _rel(top, mydir, path): def _rel(top, mydir, path):
@ -234,13 +222,14 @@ def main():
'redo-log: give at least one target; ' + 'redo-log: give at least one target; ' +
'maybe "all"?\n') 'maybe "all"?\n')
sys.exit(1) sys.exit(1)
state.init(targets)
if opt.status < 2 and not os.isatty(2): if opt.status < 2 and not os.isatty(2):
opt.status = False opt.status = False
logs.setup(tty=sys.stdout, pretty=opt.pretty, color=opt.color) logs.setup(tty=sys.stdout, pretty=opt.pretty, color=opt.color)
if opt.debug_locks: if opt.debug_locks:
env.DEBUG_LOCKS = 1 env.v.DEBUG_LOCKS = 1
if opt.debug_pids: if opt.debug_pids:
env.DEBUG_PIDS = 1 env.v.DEBUG_PIDS = 1
if opt.ack_fd: if opt.ack_fd:
# Write back to owner, to let them know we started up okay and # Write back to owner, to let them know we started up okay and
# will be able to see their error output, so it's okay to close # will be able to see their error output, so it's okay to close

18
redo/cmd_ood.py
View file

@ -1,16 +1,7 @@
import sys, os import sys, os
import env_init
env_init.init([])
import env, state, deps import env, state, deps
from logs import err from logs import err
if len(sys.argv[1:]) != 0:
err('%s: no arguments expected.\n' % sys.argv[0])
sys.exit(1)
cache = {} cache = {}
@ -27,17 +18,22 @@ def log_override(name):
def main(): def main():
if len(sys.argv[1:]) != 0:
err('%s: no arguments expected.\n' % sys.argv[0])
sys.exit(1)
state.init([])
cwd = os.getcwd() cwd = os.getcwd()
for f in state.files(): for f in state.files():
if f.is_target(): if f.is_target():
if deps.isdirty(f, if deps.isdirty(f,
depth='', depth='',
max_changed=env.RUNID, max_changed=env.v.RUNID,
already_checked=[], already_checked=[],
is_checked=is_checked, is_checked=is_checked,
set_checked=set_checked, set_checked=set_checked,
log_override=log_override): log_override=log_override):
print state.relpath(os.path.join(env.BASE, f.name), cwd) print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
if __name__ == '__main__': if __name__ == '__main__':

87
redo/cmd_redo.py
View file

@ -14,8 +14,9 @@
# limitations under the License. # limitations under the License.
# #
import sys, os, traceback import sys, os, traceback
import options import env, options, state, builder, jobserver
from atoi import atoi from atoi import atoi
from logs import warn, err
optspec = """ optspec = """
redo [targets...] redo [targets...]
@ -37,52 +38,50 @@ no-color disable ANSI color; --color to force enable (default: auto)
debug-locks print messages about file locking (useful for debugging) debug-locks print messages about file locking (useful for debugging)
debug-pids print process ids as part of log messages (useful for debugging) debug-pids print process ids as part of log messages (useful for debugging)
""" """
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
targets = extra
if opt.version:
import version
print version.TAG
sys.exit(0)
if opt.debug:
os.environ['REDO_DEBUG'] = str(opt.debug or 0)
if opt.verbose:
os.environ['REDO_VERBOSE'] = '1'
if opt.xtrace:
os.environ['REDO_XTRACE'] = '1'
if opt.keep_going:
os.environ['REDO_KEEP_GOING'] = '1'
if opt.shuffle:
os.environ['REDO_SHUFFLE'] = '1'
if opt.debug_locks:
os.environ['REDO_DEBUG_LOCKS'] = '1'
if opt.debug_pids:
os.environ['REDO_DEBUG_PIDS'] = '1'
# This is slightly tricky: the log and pretty options default to true. We
# want to inherit that 'true' value from parent processes *unless* someone
# explicitly specifies the reverse.
if opt.no_log:
os.environ['REDO_LOG'] = '0'
if opt.no_pretty:
os.environ['REDO_PRETTY'] = '0'
if opt.no_color:
os.environ['REDO_COLOR'] = '0'
import env_init
env_init.init(targets)
import env, state, builder, jobserver
from logs import warn, err
def main(): def main():
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
targets = extra
if opt.version:
import version
print version.TAG
sys.exit(0)
if opt.debug:
os.environ['REDO_DEBUG'] = str(opt.debug or 0)
if opt.verbose:
os.environ['REDO_VERBOSE'] = '1'
if opt.xtrace:
os.environ['REDO_XTRACE'] = '1'
if opt.keep_going:
os.environ['REDO_KEEP_GOING'] = '1'
if opt.shuffle:
os.environ['REDO_SHUFFLE'] = '1'
if opt.debug_locks:
os.environ['REDO_DEBUG_LOCKS'] = '1'
if opt.debug_pids:
os.environ['REDO_DEBUG_PIDS'] = '1'
# This is slightly tricky: the log and pretty options default to true. We
# want to inherit that 'true' value from parent processes *unless* someone
# explicitly specifies the reverse.
if opt.no_log:
os.environ['REDO_LOG'] = '0'
if opt.no_pretty:
os.environ['REDO_PRETTY'] = '0'
if opt.no_color:
os.environ['REDO_COLOR'] = '0'
try: try:
state.init(targets)
if env.is_toplevel and not targets:
targets = ['all']
j = atoi(opt.jobs or 1) j = atoi(opt.jobs or 1)
if env_init.is_toplevel and (env.LOG or j > 1): if env.is_toplevel and (env.v.LOG or j > 1):
builder.close_stdin() builder.close_stdin()
if env_init.is_toplevel and env.LOG: if env.is_toplevel and env.v.LOG:
builder.start_stdin_log_reader( builder.start_stdin_log_reader(
status=opt.status, details=opt.details, status=opt.status, details=opt.details,
pretty=opt.pretty, color=opt.color, pretty=opt.pretty, color=opt.color,
@ -112,11 +111,11 @@ def main():
traceback.print_exc(100, sys.stderr) traceback.print_exc(100, sys.stderr)
err('unexpected error: %r\n' % e) err('unexpected error: %r\n' % e)
retcode = 1 retcode = 1
if env_init.is_toplevel: if env.is_toplevel:
builder.await_log_reader() builder.await_log_reader()
sys.exit(retcode) sys.exit(retcode)
except KeyboardInterrupt: except KeyboardInterrupt:
if env_init.is_toplevel: if env.is_toplevel:
builder.await_log_reader() builder.await_log_reader()
sys.exit(200) sys.exit(200)

7
redo/cmd_sources.py
View file

@ -1,13 +1,10 @@
import sys, os import sys, os
import env_init
env_init.init([])
import state, env import state, env
from logs import err from logs import err
def main(): def main():
state.init([])
if len(sys.argv[1:]) != 0: if len(sys.argv[1:]) != 0:
err('%s: no arguments expected.\n' % sys.argv[0]) err('%s: no arguments expected.\n' % sys.argv[0])
sys.exit(1) sys.exit(1)
@ -15,7 +12,7 @@ def main():
cwd = os.getcwd() cwd = os.getcwd()
for f in state.files(): for f in state.files():
if f.is_source(): if f.is_source():
print state.relpath(os.path.join(env.BASE, f.name), cwd) print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
if __name__ == '__main__': if __name__ == '__main__':

7
redo/cmd_stamp.py
View file

@ -4,6 +4,7 @@ from logs import err, debug2
def main(): def main():
env.inherit()
if len(sys.argv) > 1: if len(sys.argv) > 1:
err('%s: no arguments expected.\n' % sys.argv[0]) err('%s: no arguments expected.\n' % sys.argv[0])
sys.exit(1) sys.exit(1)
@ -32,11 +33,11 @@ def main():
csum = sh.hexdigest() csum = sh.hexdigest()
if not env.TARGET: if not env.v.TARGET:
sys.exit(0) sys.exit(0)
me = os.path.join(env.STARTDIR, me = os.path.join(env.v.STARTDIR,
os.path.join(env.PWD, env.TARGET)) os.path.join(env.v.PWD, env.v.TARGET))
f = state.File(name=me) f = state.File(name=me)
changed = (csum != f.csum) changed = (csum != f.csum)
debug2('%s: old = %s\n' % (f.name, f.csum)) debug2('%s: old = %s\n' % (f.name, f.csum))

9
redo/cmd_targets.py
View file

@ -1,13 +1,10 @@
import sys, os import sys, os
import env, state
import env_init
env_init.init([])
import state, env
from logs import err from logs import err
def main(): def main():
state.init([])
if len(sys.argv[1:]) != 0: if len(sys.argv[1:]) != 0:
err('%s: no arguments expected.\n' % sys.argv[0]) err('%s: no arguments expected.\n' % sys.argv[0])
sys.exit(1) sys.exit(1)
@ -15,7 +12,7 @@ def main():
cwd = os.getcwd() cwd = os.getcwd()
for f in state.files(): for f in state.files():
if f.is_target(): if f.is_target():
print state.relpath(os.path.join(env.BASE, f.name), cwd) print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
if __name__ == '__main__': if __name__ == '__main__':

3
redo/cmd_unlocked.py
View file

@ -1,9 +1,10 @@
import sys, os import sys, os
import state import env, state
from logs import err from logs import err
def main(): def main():
env.inherit()
if len(sys.argv[1:]) < 2: if len(sys.argv[1:]) < 2:
err('%s: at least 2 arguments expected.\n' % sys.argv[0]) err('%s: at least 2 arguments expected.\n' % sys.argv[0])
sys.exit(1) sys.exit(1)

7
redo/cmd_whichdo.py
View file

@ -1,13 +1,10 @@
import sys, os import sys, os
import env, paths
import env_init
env_init.init_no_state()
import paths
from logs import err from logs import err
def main(): def main():
env.init_no_state()
if len(sys.argv[1:]) != 1: if len(sys.argv[1:]) != 1:
err('%s: exactly one argument expected.\n' % sys.argv[0]) err('%s: exactly one argument expected.\n' % sys.argv[0])
sys.exit(1) sys.exit(1)

8
redo/deps.py
View file

@ -16,7 +16,7 @@ def isdirty(f, depth, max_changed,
# is unaffected # is unaffected
already_checked = list(already_checked) + [f.id] already_checked = list(already_checked) + [f.id]
if env.DEBUG >= 1: if env.v.DEBUG >= 1:
debug('%s?%s %r,%r\n' debug('%s?%s %r,%r\n'
% (depth, f.nicename(), f.is_generated, f.is_override)) % (depth, f.nicename(), f.is_generated, f.is_override))
@ -28,10 +28,10 @@ def isdirty(f, depth, max_changed,
return DIRTY return DIRTY
if f.changed_runid > max_changed: if f.changed_runid > max_changed:
debug('%s-- DIRTY (built %d > %d; %d)\n' debug('%s-- DIRTY (built %d > %d; %d)\n'
% (depth, f.changed_runid, max_changed, env.RUNID)) % (depth, f.changed_runid, max_changed, env.v.RUNID))
return DIRTY # has been built more recently than parent return DIRTY # has been built more recently than parent
if is_checked(f): if is_checked(f):
if env.DEBUG >= 1: if env.v.DEBUG >= 1:
debug('%s-- CLEAN (checked)\n' % depth) debug('%s-- CLEAN (checked)\n' % depth)
return CLEAN # has already been checked during this session return CLEAN # has already been checked during this session
if not f.stamp: if not f.stamp:
@ -65,7 +65,7 @@ def isdirty(f, depth, max_changed,
for mode, f2 in f.deps(): for mode, f2 in f.deps():
dirty = CLEAN dirty = CLEAN
if mode == 'c': if mode == 'c':
if os.path.exists(os.path.join(env.BASE, f2.name)): if os.path.exists(os.path.join(env.v.BASE, f2.name)):
debug('%s-- DIRTY (created)\n' % depth) debug('%s-- DIRTY (created)\n' % depth)
dirty = DIRTY dirty = DIRTY
elif mode == 'm': elif mode == 'm':

141
redo/env.py
View file

@ -1,35 +1,116 @@
import os import os, sys
from atoi import atoi from atoi import atoi
if not os.environ.get('REDO'): is_toplevel = False
import sys
sys.stderr.write('%s: error: must be run from inside a .do\n'
% sys.argv[0])
sys.exit(100)
PWD = os.environ.get('REDO_PWD', '') v = None
TARGET = os.environ.get('REDO_TARGET', '')
DEPTH = os.environ.get('REDO_DEPTH', '')
DEBUG = atoi(os.environ.get('REDO_DEBUG', ''))
DEBUG_LOCKS = os.environ.get('REDO_DEBUG_LOCKS', '') and 1 or 0
DEBUG_PIDS = os.environ.get('REDO_DEBUG_PIDS', '') and 1 or 0
VERBOSE = os.environ.get('REDO_VERBOSE', '') and 1 or 0
XTRACE = os.environ.get('REDO_XTRACE', '') and 1 or 0
KEEP_GOING = os.environ.get('REDO_KEEP_GOING', '') and 1 or 0
LOG = atoi(os.environ.get('REDO_LOG', '1')) # defaults on
LOG_INODE = os.environ.get('REDO_LOG_INODE', '')
COLOR = atoi(os.environ.get('REDO_COLOR', '1')) # defaults on
# subprocesses mustn't pretty-print if a parent is running redo-log
PRETTY = (not LOG) and atoi(os.environ.get('REDO_PRETTY', '1'))
SHUFFLE = os.environ.get('REDO_SHUFFLE', '') and 1 or 0
STARTDIR = os.environ.get('REDO_STARTDIR', '')
RUNID = atoi(os.environ.get('REDO_RUNID')) or None
BASE = os.environ['REDO_BASE']
while BASE and BASE.endswith('/'):
BASE = BASE[:-1]
UNLOCKED = os.environ.get('REDO_UNLOCKED', '') and 1 or 0
os.environ['REDO_UNLOCKED'] = '' # not inheritable by subprocesses
NO_OOB = os.environ.get('REDO_NO_OOB', '') and 1 or 0 def _get(name, default):
os.environ['REDO_NO_OOB'] = '' # not inheritable by subprocesses return os.environ.get(name, default)
def _get_int(name, default):
return atoi(_get(name, str(default)))
def _get_bool(name, default):
return 1 if _get(name, default) else 0
class Env(object):
def __init__(self):
# Mandatory. If this is missing, you forgot to call init().
self.BASE = os.environ['REDO_BASE'].rstrip('/')
# Everything else, we can recover from defaults if unset.
self.PWD = _get('REDO_PWD', '')
self.TARGET = _get('REDO_TARGET', '')
self.DEPTH = _get('REDO_DEPTH', '')
self.DEBUG = atoi(_get('REDO_DEBUG', ''))
self.DEBUG_LOCKS = _get_bool('REDO_DEBUG_LOCKS', '')
self.DEBUG_PIDS = _get_bool('REDO_DEBUG_PIDS', '')
self.VERBOSE = _get_bool('REDO_VERBOSE', '')
self.XTRACE = _get_bool('REDO_XTRACE', '')
self.KEEP_GOING = _get_bool('REDO_KEEP_GOING', '')
self.LOG = _get_int('REDO_LOG', 1) # defaults on
self.LOG_INODE = _get('REDO_LOG_INODE', '')
self.COLOR = _get_int('REDO_COLOR', 1) # defaults on
# subprocesses mustn't pretty-print if a parent is running redo-log
self.PRETTY = (not self.LOG) and _get_int('REDO_PRETTY', 1)
self.SHUFFLE = _get_bool('REDO_SHUFFLE', '')
self.STARTDIR = _get('REDO_STARTDIR', '')
self.RUNID = _get_int('REDO_RUNID', '') or None
self.UNLOCKED = _get_bool('REDO_UNLOCKED', '')
self.NO_OOB = _get_bool('REDO_NO_OOB', '')
def inherit():
"""Read environment (which must already be set) to get runtime settings."""
if not os.environ.get('REDO'):
sys.stderr.write('%s: error: must be run from inside a .do\n'
% sys.argv[0])
sys.exit(100)
global v
v = Env()
# not inheritable by subprocesses
os.environ['REDO_UNLOCKED'] = ''
os.environ['REDO_NO_OOB'] = ''
def init_no_state():
"""Start a session (if needed) for a command that needs no state db."""
global is_toplevel
if not os.environ.get('REDO'):
os.environ['REDO'] = 'NOT_DEFINED'
is_toplevel = True
if not os.environ.get('REDO_BASE'):
os.environ['REDO_BASE'] = 'NOT_DEFINED'
inherit()
def init(targets):
"""Start a session (if needed) for a command that does need the state db.
Args:
targets: a list of targets we're trying to build. We use this to
help in guessing where the .redo database is located.
"""
global is_toplevel
if not os.environ.get('REDO'):
# toplevel call to redo
is_toplevel = True
exenames = [os.path.abspath(sys.argv[0]),
os.path.realpath(sys.argv[0])]
dirnames = [os.path.dirname(p) for p in exenames]
trynames = ([os.path.abspath(p+'/../lib/redo') for p in dirnames] +
[p+'/../redo' for p in dirnames] +
dirnames)
seen = {}
dirs = []
for k in trynames:
if not seen.get(k):
seen[k] = 1
dirs.append(k)
os.environ['PATH'] = ':'.join(dirs) + ':' + os.environ['PATH']
os.environ['REDO'] = os.path.abspath(sys.argv[0])
if not os.environ.get('REDO_BASE'):
if not targets:
# if no other targets given, assume the current directory
targets = ['all']
base = os.path.commonprefix([os.path.abspath(os.path.dirname(t))
for t in targets] + [os.getcwd()])
bsplit = base.split('/')
for i in range(len(bsplit)-1, 0, -1):
newbase = '/'.join(bsplit[:i])
if os.path.exists(newbase + '/.redo'):
base = newbase
break
os.environ['REDO_BASE'] = base
os.environ['REDO_STARTDIR'] = os.getcwd()
inherit()

53
redo/env_init.py
View file

@ -1,53 +0,0 @@
import sys, os
is_toplevel = False
def init_no_state():
global is_toplevel
if not os.environ.get('REDO'):
os.environ['REDO'] = 'NOT_DEFINED'
is_toplevel = True
if not os.environ.get('REDO_BASE'):
os.environ['REDO_BASE'] = 'NOT_DEFINED'
def init(targets):
global is_toplevel
if not os.environ.get('REDO'):
# toplevel call to redo
is_toplevel = True
if len(targets) == 0:
targets.append('all')
exenames = [os.path.abspath(sys.argv[0]),
os.path.realpath(sys.argv[0])]
dirnames = [os.path.dirname(p) for p in exenames]
trynames = ([os.path.abspath(p+'/../lib/redo') for p in dirnames] +
[p+'/../redo' for p in dirnames] +
dirnames)
seen = {}
dirs = []
for k in trynames:
if not seen.get(k):
seen[k] = 1
dirs.append(k)
os.environ['PATH'] = ':'.join(dirs) + ':' + os.environ['PATH']
os.environ['REDO'] = os.path.abspath(sys.argv[0])
if not os.environ.get('REDO_BASE'):
base = os.path.commonprefix([os.path.abspath(os.path.dirname(t))
for t in targets] + [os.getcwd()])
bsplit = base.split('/')
for i in range(len(bsplit)-1, 0, -1):
newbase = '/'.join(bsplit[:i])
if os.path.exists(newbase + '/.redo'):
base = newbase
break
os.environ['REDO_BASE'] = base
os.environ['REDO_STARTDIR'] = os.getcwd()
import state
state.init()
os.environ['REDO_LOCKS'] = os.environ.get('REDO_LOCKS', '')

2
redo/jobserver.py
View file

@ -343,7 +343,7 @@ def ensure_token_or_cheat(reason, cheatfunc):
if not has_token(): if not has_token():
assert _mytokens == 0 assert _mytokens == 0
n = cheatfunc() n = cheatfunc()
_debug('%s: %s: cheat = %d\n' % (env.TARGET, reason, n)) _debug('%s: %s: cheat = %d\n' % (env.v.TARGET, reason, n))
if n > 0: if n > 0:
_mytokens += n _mytokens += n
_cheats += n _cheats += n

27
redo/logs.py
View file

@ -44,12 +44,12 @@ class PrettyLog(object):
self.file = tty self.file = tty
def _pretty(self, pid, color, s): def _pretty(self, pid, color, s):
if env.DEBUG_PIDS: if env.v.DEBUG_PIDS:
redo = '%-6d redo ' % pid redo = '%-6d redo ' % pid
else: else:
redo = 'redo ' redo = 'redo '
self.file.write( self.file.write(
''.join([color, redo, env.DEPTH, ''.join([color, redo, env.v.DEPTH,
BOLD if color else '', s, PLAIN, '\n'])) BOLD if color else '', s, PLAIN, '\n']))
def write(self, s): def write(self, s):
@ -75,17 +75,17 @@ class PrettyLog(object):
rv = int(rv) rv = int(rv)
if rv: if rv:
self._pretty(pid, RED, '%s (exit %d)' % (name, rv)) self._pretty(pid, RED, '%s (exit %d)' % (name, rv))
elif env.VERBOSE or env.XTRACE or env.DEBUG: elif env.v.VERBOSE or env.v.XTRACE or env.v.DEBUG:
self._pretty(pid, GREEN, '%s (done)' % name) self._pretty(pid, GREEN, '%s (done)' % name)
self.file.write('\n') self.file.write('\n')
elif kind == 'locked': elif kind == 'locked':
if env.DEBUG_LOCKS: if env.v.DEBUG_LOCKS:
self._pretty(pid, GREEN, '%s (locked...)' % text) self._pretty(pid, GREEN, '%s (locked...)' % text)
elif kind == 'waiting': elif kind == 'waiting':
if env.DEBUG_LOCKS: if env.v.DEBUG_LOCKS:
self._pretty(pid, GREEN, '%s (WAITING)' % text) self._pretty(pid, GREEN, '%s (WAITING)' % text)
elif kind == 'unlocked': elif kind == 'unlocked':
if env.DEBUG_LOCKS: if env.v.DEBUG_LOCKS:
self._pretty(pid, GREEN, '%s (...unlocked!)' % text) self._pretty(pid, GREEN, '%s (...unlocked!)' % text)
elif kind == 'error': elif kind == 'error':
self.file.write(''.join([RED, 'redo: ', self.file.write(''.join([RED, 'redo: ',
@ -106,18 +106,21 @@ _log = None
def setup(tty, pretty, color): def setup(tty, pretty, color):
global _log global _log
if pretty or env.PRETTY: if pretty or env.v.PRETTY:
check_tty(tty, color=color) check_tty(tty, color=color)
_log = PrettyLog(tty=tty) _log = PrettyLog(tty=tty)
else: else:
_log = RawLog(tty=tty) _log = RawLog(tty=tty)
# FIXME: explicitly initialize in each program, for clarity def _maybe_setup():
setup(tty=sys.stderr, pretty=env.PRETTY, color=env.COLOR) # FIXME: explicitly initialize in each program, for clarity
if not _log:
setup(tty=sys.stderr, pretty=env.v.PRETTY, color=env.v.COLOR)
def write(s): def write(s):
_maybe_setup()
_log.write(s) _log.write(s)
@ -139,16 +142,16 @@ def warn(s):
meta('warning', s) meta('warning', s)
def debug(s): def debug(s):
if env.DEBUG >= 1: if env.v.DEBUG >= 1:
s = s.rstrip() s = s.rstrip()
meta('debug', s) meta('debug', s)
def debug2(s): def debug2(s):
if env.DEBUG >= 2: if env.v.DEBUG >= 2:
s = s.rstrip() s = s.rstrip()
meta('debug', s) meta('debug', s)
def debug3(s): def debug3(s):
if env.DEBUG >= 3: if env.v.DEBUG >= 3:
s = s.rstrip() s = s.rstrip()
meta('debug', s) meta('debug', s)

4
redo/paths.py
View file

@ -15,7 +15,7 @@ def _default_do_files(filename):
def possible_do_files(t): def possible_do_files(t):
dirname, filename = os.path.split(t) dirname, filename = os.path.split(t)
yield (os.path.join(env.BASE, dirname), "%s.do" % filename, yield (os.path.join(env.v.BASE, dirname), "%s.do" % filename,
'', filename, '') '', filename, '')
# It's important to try every possibility in a directory before resorting # It's important to try every possibility in a directory before resorting
@ -24,7 +24,7 @@ def possible_do_files(t):
# the former one might just be an artifact of someone embedding my project # the former one might just be an artifact of someone embedding my project
# into theirs as a subdir. When they do, my rules should still be used # into theirs as a subdir. When they do, my rules should still be used
# for building my project in *all* cases. # for building my project in *all* cases.
t = os.path.normpath(os.path.join(env.BASE, t)) t = os.path.normpath(os.path.join(env.v.BASE, t))
dirname, filename = os.path.split(t) dirname, filename = os.path.split(t)
dirbits = dirname.split('/') dirbits = dirname.split('/')
# since t is an absolute path, dirbits[0] is always '', so we don't # since t is an absolute path, dirbits[0] is always '', so we don't

47
redo/state.py
View file

@ -46,7 +46,7 @@ def db():
if _db: if _db:
return _db return _db
dbdir = '%s/.redo' % env.BASE dbdir = '%s/.redo' % env.v.BASE
dbfile = '%s/db.sqlite3' % dbdir dbfile = '%s/db.sqlite3' % dbdir
try: try:
os.mkdir(dbdir) os.mkdir(dbdir)
@ -56,7 +56,7 @@ def db():
else: else:
raise raise
_lockfile = os.open(os.path.join(env.BASE, '.redo/locks'), _lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'),
os.O_RDWR | os.O_CREAT, 0666) os.O_RDWR | os.O_CREAT, 0666)
close_on_exec(_lockfile, True) close_on_exec(_lockfile, True)
@ -106,17 +106,18 @@ def db():
_db.execute("insert into Runid values (1000000000)") _db.execute("insert into Runid values (1000000000)")
_db.execute("insert into Files (name) values (?)", [ALWAYS]) _db.execute("insert into Files (name) values (?)", [ALWAYS])
if not env.RUNID: if not env.v.RUNID:
_db.execute("insert into Runid values " _db.execute("insert into Runid values "
" ((select max(id)+1 from Runid))") " ((select max(id)+1 from Runid))")
env.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0] env.v.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0]
os.environ['REDO_RUNID'] = str(env.RUNID) os.environ['REDO_RUNID'] = str(env.v.RUNID)
_db.commit() _db.commit()
return _db return _db
def init(): def init(targets):
env.init(targets)
db() db()
@ -155,7 +156,7 @@ _insane = None
def check_sane(): def check_sane():
global _insane global _insane
if not _insane: if not _insane:
_insane = not os.path.exists('%s/.redo' % env.BASE) _insane = not os.path.exists('%s/.redo' % env.v.BASE)
return not _insane return not _insane
@ -179,18 +180,18 @@ def relpath(t, base):
return join('/', tparts) return join('/', tparts)
# Return a path for t, if cwd were the dirname of env.TARGET. # Return a path for t, if cwd were the dirname of env.v.TARGET.
# This is tricky! STARTDIR+PWD is the directory for the *dofile*, when # This is tricky! STARTDIR+PWD is the directory for the *dofile*, when
# the dofile was started. However, inside the dofile, someone may have done # the dofile was started. However, inside the dofile, someone may have done
# a chdir to anywhere else. env.TARGET is relative to the dofile path, so # a chdir to anywhere else. env.v.TARGET is relative to the dofile path, so
# we have to first figure out where the dofile was, then find TARGET relative # we have to first figure out where the dofile was, then find TARGET relative
# to that, then find t relative to that. # to that, then find t relative to that.
# #
# FIXME: find some cleaner terminology for all these different paths. # FIXME: find some cleaner terminology for all these different paths.
def target_relpath(t): def target_relpath(t):
dofile_dir = os.path.abspath(os.path.join(env.STARTDIR, env.PWD)) dofile_dir = os.path.abspath(os.path.join(env.v.STARTDIR, env.v.PWD))
target_dir = os.path.abspath( target_dir = os.path.abspath(
os.path.dirname(os.path.join(dofile_dir, env.TARGET))) os.path.dirname(os.path.join(dofile_dir, env.v.TARGET)))
return relpath(t, target_dir) return relpath(t, target_dir)
@ -232,7 +233,7 @@ class File(object):
q += 'where rowid=?' q += 'where rowid=?'
l = [fid] l = [fid]
elif name != None: elif name != None:
name = (name == ALWAYS) and ALWAYS or relpath(name, env.BASE) name = (name == ALWAYS) and ALWAYS or relpath(name, env.v.BASE)
q += 'where name=?' q += 'where name=?'
l = [name] l = [name]
else: else:
@ -258,8 +259,8 @@ class File(object):
(self.id, self.name, self.is_generated, self.is_override, (self.id, self.name, self.is_generated, self.is_override,
self.checked_runid, self.changed_runid, self.failed_runid, self.checked_runid, self.changed_runid, self.failed_runid,
self.stamp, self.csum) = cols self.stamp, self.csum) = cols
if self.name == ALWAYS and self.changed_runid < env.RUNID: if self.name == ALWAYS and self.changed_runid < env.v.RUNID:
self.changed_runid = env.RUNID self.changed_runid = env.v.RUNID
def __init__(self, fid=None, name=None, cols=None, allow_add=True): def __init__(self, fid=None, name=None, cols=None, allow_add=True):
if cols: if cols:
@ -284,7 +285,7 @@ class File(object):
self.id]) self.id])
def set_checked(self): def set_checked(self):
self.checked_runid = env.RUNID self.checked_runid = env.v.RUNID
def set_checked_save(self): def set_checked_save(self):
self.set_checked() self.set_checked()
@ -292,14 +293,14 @@ class File(object):
def set_changed(self): def set_changed(self):
debug2('BUILT: %r (%r)\n' % (self.name, self.stamp)) debug2('BUILT: %r (%r)\n' % (self.name, self.stamp))
self.changed_runid = env.RUNID self.changed_runid = env.v.RUNID
self.failed_runid = None self.failed_runid = None
self.is_override = False self.is_override = False
def set_failed(self): def set_failed(self):
debug2('FAILED: %r\n' % self.name) debug2('FAILED: %r\n' % self.name)
self.update_stamp() self.update_stamp()
self.failed_runid = env.RUNID self.failed_runid = env.v.RUNID
if self.stamp != STAMP_MISSING: if self.stamp != STAMP_MISSING:
# if we failed and the target file still exists, # if we failed and the target file still exists,
# then we're generated. # then we're generated.
@ -358,13 +359,13 @@ class File(object):
return True return True
def is_checked(self): def is_checked(self):
return self.checked_runid and self.checked_runid >= env.RUNID return self.checked_runid and self.checked_runid >= env.v.RUNID
def is_changed(self): def is_changed(self):
return self.changed_runid and self.changed_runid >= env.RUNID return self.changed_runid and self.changed_runid >= env.v.RUNID
def is_failed(self): def is_failed(self):
return self.failed_runid and self.failed_runid >= env.RUNID return self.failed_runid and self.failed_runid >= env.v.RUNID
def deps(self): def deps(self):
if self.is_override or not self.is_generated: if self.is_override or not self.is_generated:
@ -397,7 +398,7 @@ class File(object):
def _read_stamp_st(self, statfunc): def _read_stamp_st(self, statfunc):
try: try:
st = statfunc(os.path.join(env.BASE, self.name)) st = statfunc(os.path.join(env.v.BASE, self.name))
except OSError: except OSError:
return False, STAMP_MISSING return False, STAMP_MISSING
if stat.S_ISDIR(st.st_mode): if stat.S_ISDIR(st.st_mode):
@ -427,7 +428,7 @@ class File(object):
return pre return pre
def nicename(self): def nicename(self):
return relpath(os.path.join(env.BASE, self.name), env.STARTDIR) return relpath(os.path.join(env.v.BASE, self.name), env.v.STARTDIR)
def files(): def files():
@ -438,7 +439,7 @@ def files():
def logname(fid): def logname(fid):
"""Given the id of a File, return the filename of its build log.""" """Given the id of a File, return the filename of its build log."""
return os.path.join(env.BASE, '.redo', 'log.%d' % fid) return os.path.join(env.v.BASE, '.redo', 'log.%d' % fid)
# FIXME: I really want to use fcntl F_SETLK, F_SETLKW, etc here. But python # FIXME: I really want to use fcntl F_SETLK, F_SETLKW, etc here. But python