From 9b6d1eeb6ecbd076ad3e039a1428a76d9b4dce9a Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Wed, 5 Dec 2018 01:07:16 -0500 Subject: [PATCH] env and env_init: Eliminate weird auto-initialization of globals. Merge the two files into env, and make each command explicitly call the function that sets it up in the way that's needed for that command. This means we can finally just import all the modules at the top of each file, without worrying about import order. Phew. While we're here, remove the weird auto-appending-'all'-to-targets feature in env.init(). Instead, do it explicitly, and only from redo and redo-ifchange, only if is_toplevel and no other targets are given. --- redo/builder.py | 44 +++++++------- redo/cmd_always.py | 5 +- redo/cmd_ifchange.py | 26 ++++---- redo/cmd_ifcreate.py | 5 +- redo/cmd_log.py | 23 ++----- redo/cmd_ood.py | 18 +++--- redo/cmd_redo.py | 87 +++++++++++++------------- redo/cmd_sources.py | 7 +-- redo/cmd_stamp.py | 7 ++- redo/cmd_targets.py | 9 +-- redo/cmd_unlocked.py | 3 +- redo/cmd_whichdo.py | 7 +-- redo/deps.py | 8 +-- redo/env.py | 141 ++++++++++++++++++++++++++++++++++--------- redo/env_init.py | 53 ---------------- redo/jobserver.py | 2 +- redo/logs.py | 27 +++++---- redo/paths.py | 4 +- redo/state.py | 47 ++++++++------- 19 files changed, 267 insertions(+), 256 deletions(-) delete mode 100644 redo/env_init.py diff --git a/redo/builder.py b/redo/builder.py index 22f77c5..17a521b 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -6,7 +6,7 @@ from logs import debug2, err, warn, meta, check_tty def _nice(t): - return state.relpath(t, env.STARTDIR) + return state.relpath(t, env.v.STARTDIR) def _try_stat(filename): @@ -53,7 +53,7 @@ def start_stdin_log_reader(status, details, pretty, color, os.dup2(w, 1) os.dup2(w, 2) os.close(w) - check_tty(sys.stderr, env.COLOR) + check_tty(sys.stderr, env.v.COLOR) else: # child try: @@ -88,7 +88,7 @@ def start_stdin_log_reader(status, details, pretty, color, def await_log_reader(): - if not env.LOG: + if not env.v.LOG: return if log_reader_pid > 0: # never actually close fd#1 or fd#2; insanity awaits. @@ -110,7 +110,7 @@ class ImmediateReturn(Exception): class BuildJob(object): def __init__(self, t, sf, lock, shouldbuildfunc, donefunc): - self.t = t # original target name, not relative to env.BASE + self.t = t # original target name, not relative to env.v.BASE self.sf = sf tmpbase = t while not os.path.isdir(os.path.dirname(tmpbase) or '.'): @@ -140,7 +140,7 @@ class BuildJob(object): except ImmediateReturn, e: return self._after2(e.rv) - if env.NO_OOB or dirty == True: # pylint: disable=singleton-comparison + if env.v.NO_OOB or dirty == True: # pylint: disable=singleton-comparison self._start_do() else: self._start_unlocked(dirty) @@ -198,16 +198,16 @@ class BuildJob(object): # temp output file name state.relpath(os.path.abspath(self.tmpname2), dodir), ] - if env.VERBOSE: + if env.v.VERBOSE: argv[1] += 'v' - if env.XTRACE: + if env.v.XTRACE: argv[1] += 'x' firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') # make sure to create the logfile *before* writing the meta() about it. # that way redo-log won't trace into an obsolete logfile. - if env.LOG: + if env.v.LOG: open(state.logname(self.sf.id), 'w') # FIXME: put these variables somewhere else, instead of on-the-fly # extending this class! @@ -236,13 +236,13 @@ class BuildJob(object): # grab a lock. here = os.getcwd() def _fix(p): - return state.relpath(os.path.join(env.BASE, p), here) + return state.relpath(os.path.join(env.v.BASE, p), here) argv = (['redo-unlocked', _fix(self.sf.name)] + list(set(_fix(d.name) for d in dirty))) meta('check', state.target_relpath(self.t)) state.commit() def run(): - os.environ['REDO_DEPTH'] = env.DEPTH + ' ' + os.environ['REDO_DEPTH'] = env.v.DEPTH + ' ' # python ignores SIGPIPE signal.signal(signal.SIGPIPE, signal.SIG_DFL) os.execvp(argv[0], argv) @@ -261,18 +261,18 @@ class BuildJob(object): assert state.is_flushed() dn = self.dodir newp = os.path.realpath(dn) - os.environ['REDO_PWD'] = state.relpath(newp, env.STARTDIR) + os.environ['REDO_PWD'] = state.relpath(newp, env.v.STARTDIR) os.environ['REDO_TARGET'] = self.basename + self.ext - os.environ['REDO_DEPTH'] = env.DEPTH + ' ' + os.environ['REDO_DEPTH'] = env.v.DEPTH + ' ' cycles.add(self.lock.fid) if dn: os.chdir(dn) os.dup2(self.f.fileno(), 1) os.close(self.f.fileno()) close_on_exec(1, False) - if env.LOG: + if env.v.LOG: cur_inode = str(os.fstat(2).st_ino) - if not env.LOG_INODE or cur_inode == env.LOG_INODE: + if not env.v.LOG_INODE or cur_inode == env.v.LOG_INODE: # .do script has *not* redirected stderr, which means we're # using redo-log's log saving mode. That means subprocs # should be logged to their own file. If the .do script @@ -290,7 +290,7 @@ class BuildJob(object): del os.environ['REDO_LOG_INODE'] os.environ['REDO_LOG'] = '' signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE - if env.VERBOSE or env.XTRACE: + if env.v.VERBOSE or env.v.XTRACE: logs.write('* %s' % ' '.join(self.argv).replace('\n', ' ')) os.execvp(self.argv[0], self.argv) # FIXME: it would be nice to log the exit code to logf. @@ -387,7 +387,7 @@ class BuildJob(object): def main(targets, shouldbuildfunc): retcode = [0] # a list so that it can be reassigned from done() - if env.SHUFFLE: + if env.v.SHUFFLE: import random random.shuffle(targets) @@ -397,9 +397,9 @@ def main(targets, shouldbuildfunc): if rv: retcode[0] = 1 - if env.TARGET and not env.UNLOCKED: - me = os.path.join(env.STARTDIR, - os.path.join(env.PWD, env.TARGET)) + if env.v.TARGET and not env.v.UNLOCKED: + me = os.path.join(env.v.STARTDIR, + os.path.join(env.v.PWD, env.v.TARGET)) myfile = state.File(name=me) selflock = state.Lock(state.LOG_LOCK_MAGIC + myfile.id) else: @@ -435,7 +435,7 @@ def main(targets, shouldbuildfunc): if not jobserver.has_token(): state.commit() jobserver.ensure_token_or_cheat(t, cheat) - if retcode[0] and not env.KEEP_GOING: + if retcode[0] and not env.v.KEEP_GOING: break if not state.check_sane(): err('.redo directory disappeared; cannot continue.\n') @@ -443,7 +443,7 @@ def main(targets, shouldbuildfunc): break f = state.File(name=t) lock = state.Lock(f.id) - if env.UNLOCKED: + if env.v.UNLOCKED: lock.owned = True else: lock.trylock() @@ -478,7 +478,7 @@ def main(targets, shouldbuildfunc): jobserver.ensure_token_or_cheat('self', cheat) # at this point, we don't have any children holding any tokens, so # it's okay to block below. - if retcode[0] and not env.KEEP_GOING: + if retcode[0] and not env.v.KEEP_GOING: break if locked: if not state.check_sane(): diff --git a/redo/cmd_always.py b/redo/cmd_always.py index e388177..a9e1460 100644 --- a/redo/cmd_always.py +++ b/redo/cmd_always.py @@ -4,8 +4,9 @@ import env, state def main(): try: - me = os.path.join(env.STARTDIR, - os.path.join(env.PWD, env.TARGET)) + env.inherit() + me = os.path.join(env.v.STARTDIR, + os.path.join(env.v.PWD, env.v.TARGET)) f = state.File(name=me) f.add_dep('m', state.ALWAYS) always = state.File(name=state.ALWAYS) diff --git a/redo/cmd_ifchange.py b/redo/cmd_ifchange.py index 86bf91d..bf7d2dd 100644 --- a/redo/cmd_ifchange.py +++ b/redo/cmd_ifchange.py @@ -1,16 +1,13 @@ import os, sys, traceback - -import env_init -env_init.init(sys.argv[1:]) - import env, state, builder, jobserver, deps from logs import debug2, err + def should_build(t): f = state.File(name=t) if f.is_failed(): raise builder.ImmediateReturn(32) - dirty = deps.isdirty(f, depth='', max_changed=env.RUNID, + dirty = deps.isdirty(f, depth='', max_changed=env.v.RUNID, already_checked=[]) return f.is_generated, dirty == [f] and deps.DIRTY or dirty @@ -18,23 +15,26 @@ def should_build(t): def main(): rv = 202 try: - if env_init.is_toplevel and env.LOG: + targets = sys.argv[1:] + state.init(targets) + if env.is_toplevel and not targets: + targets = ['all'] + if env.is_toplevel and env.v.LOG: builder.close_stdin() builder.start_stdin_log_reader( status=True, details=True, pretty=True, color=True, debug_locks=False, debug_pids=False) - if env.TARGET and not env.UNLOCKED: - me = os.path.join(env.STARTDIR, - os.path.join(env.PWD, env.TARGET)) + if env.v.TARGET and not env.v.UNLOCKED: + me = os.path.join(env.v.STARTDIR, + os.path.join(env.v.PWD, env.v.TARGET)) f = state.File(name=me) debug2('TARGET: %r %r %r\n' - % (env.STARTDIR, env.PWD, env.TARGET)) + % (env.v.STARTDIR, env.v.PWD, env.v.TARGET)) else: f = me = None debug2('redo-ifchange: not adding depends.\n') jobserver.setup(1) try: - targets = sys.argv[1:] if f: for t in targets: f.add_dep('m', t) @@ -52,11 +52,11 @@ def main(): err('unexpected error: %r\n' % e) rv = 1 except KeyboardInterrupt: - if env_init.is_toplevel: + if env.is_toplevel: builder.await_log_reader() sys.exit(200) state.commit() - if env_init.is_toplevel: + if env.is_toplevel: builder.await_log_reader() sys.exit(rv) diff --git a/redo/cmd_ifcreate.py b/redo/cmd_ifcreate.py index 558e00a..5fb6dd9 100644 --- a/redo/cmd_ifcreate.py +++ b/redo/cmd_ifcreate.py @@ -5,8 +5,9 @@ from logs import err def main(): try: - me = os.path.join(env.STARTDIR, - os.path.join(env.PWD, env.TARGET)) + env.inherit() + me = os.path.join(env.v.STARTDIR, + os.path.join(env.v.PWD, env.v.TARGET)) f = state.File(name=me) for t in sys.argv[1:]: if not t: diff --git a/redo/cmd_log.py b/redo/cmd_log.py index b75413b..141e209 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -1,7 +1,7 @@ import errno, fcntl, os, re, struct, sys, time import termios from atoi import atoi -import options +import env, logs, options, state optspec = """ redo-log [options...] [targets...] @@ -21,11 +21,6 @@ o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) targets = extra -import env_init -env_init.init(list(targets)) - -import env, logs, state - topdir = os.getcwd() already = set() depth = [] @@ -42,19 +37,12 @@ start_time = time.time() REDO_LINE_RE = re.compile(r'^@@REDO:([^@]+)@@ (.*)\n$') -def _atoi(s): - try: - return int(s) - except TypeError: - return 0 - - def _tty_width(): s = struct.pack("HHHH", 0, 0, 0, 0) try: s = fcntl.ioctl(sys.stderr.fileno(), termios.TIOCGWINSZ, s) except (IOError, ImportError): - return _atoi(os.environ.get('WIDTH')) or 70 + return atoi(os.environ.get('WIDTH')) or 70 (ysize, xsize, ypix, xpix) = struct.unpack('HHHH', s) return xsize or 70 @@ -64,7 +52,7 @@ def is_locked(fid): def _fix_depth(): - env.DEPTH = len(depth) * ' ' + env.v.DEPTH = len(depth) * ' ' def _rel(top, mydir, path): @@ -234,13 +222,14 @@ def main(): 'redo-log: give at least one target; ' + 'maybe "all"?\n') sys.exit(1) + state.init(targets) if opt.status < 2 and not os.isatty(2): opt.status = False logs.setup(tty=sys.stdout, pretty=opt.pretty, color=opt.color) if opt.debug_locks: - env.DEBUG_LOCKS = 1 + env.v.DEBUG_LOCKS = 1 if opt.debug_pids: - env.DEBUG_PIDS = 1 + env.v.DEBUG_PIDS = 1 if opt.ack_fd: # Write back to owner, to let them know we started up okay and # will be able to see their error output, so it's okay to close diff --git a/redo/cmd_ood.py b/redo/cmd_ood.py index 22bc70a..54bfb1c 100644 --- a/redo/cmd_ood.py +++ b/redo/cmd_ood.py @@ -1,16 +1,7 @@ import sys, os - -import env_init -env_init.init([]) - import env, state, deps from logs import err -if len(sys.argv[1:]) != 0: - err('%s: no arguments expected.\n' % sys.argv[0]) - sys.exit(1) - - cache = {} @@ -27,17 +18,22 @@ def log_override(name): def main(): + if len(sys.argv[1:]) != 0: + err('%s: no arguments expected.\n' % sys.argv[0]) + sys.exit(1) + + state.init([]) cwd = os.getcwd() for f in state.files(): if f.is_target(): if deps.isdirty(f, depth='', - max_changed=env.RUNID, + max_changed=env.v.RUNID, already_checked=[], is_checked=is_checked, set_checked=set_checked, log_override=log_override): - print state.relpath(os.path.join(env.BASE, f.name), cwd) + print state.relpath(os.path.join(env.v.BASE, f.name), cwd) if __name__ == '__main__': diff --git a/redo/cmd_redo.py b/redo/cmd_redo.py index 18051d9..c48b1f3 100644 --- a/redo/cmd_redo.py +++ b/redo/cmd_redo.py @@ -14,8 +14,9 @@ # limitations under the License. # import sys, os, traceback -import options +import env, options, state, builder, jobserver from atoi import atoi +from logs import warn, err optspec = """ redo [targets...] @@ -37,52 +38,50 @@ no-color disable ANSI color; --color to force enable (default: auto) debug-locks print messages about file locking (useful for debugging) debug-pids print process ids as part of log messages (useful for debugging) """ -o = options.Options(optspec) -(opt, flags, extra) = o.parse(sys.argv[1:]) - -targets = extra - -if opt.version: - import version - print version.TAG - sys.exit(0) -if opt.debug: - os.environ['REDO_DEBUG'] = str(opt.debug or 0) -if opt.verbose: - os.environ['REDO_VERBOSE'] = '1' -if opt.xtrace: - os.environ['REDO_XTRACE'] = '1' -if opt.keep_going: - os.environ['REDO_KEEP_GOING'] = '1' -if opt.shuffle: - os.environ['REDO_SHUFFLE'] = '1' -if opt.debug_locks: - os.environ['REDO_DEBUG_LOCKS'] = '1' -if opt.debug_pids: - os.environ['REDO_DEBUG_PIDS'] = '1' - -# This is slightly tricky: the log and pretty options default to true. We -# want to inherit that 'true' value from parent processes *unless* someone -# explicitly specifies the reverse. -if opt.no_log: - os.environ['REDO_LOG'] = '0' - if opt.no_pretty: - os.environ['REDO_PRETTY'] = '0' - if opt.no_color: - os.environ['REDO_COLOR'] = '0' - -import env_init -env_init.init(targets) - -import env, state, builder, jobserver -from logs import warn, err def main(): + o = options.Options(optspec) + (opt, flags, extra) = o.parse(sys.argv[1:]) + + targets = extra + + if opt.version: + import version + print version.TAG + sys.exit(0) + if opt.debug: + os.environ['REDO_DEBUG'] = str(opt.debug or 0) + if opt.verbose: + os.environ['REDO_VERBOSE'] = '1' + if opt.xtrace: + os.environ['REDO_XTRACE'] = '1' + if opt.keep_going: + os.environ['REDO_KEEP_GOING'] = '1' + if opt.shuffle: + os.environ['REDO_SHUFFLE'] = '1' + if opt.debug_locks: + os.environ['REDO_DEBUG_LOCKS'] = '1' + if opt.debug_pids: + os.environ['REDO_DEBUG_PIDS'] = '1' + + # This is slightly tricky: the log and pretty options default to true. We + # want to inherit that 'true' value from parent processes *unless* someone + # explicitly specifies the reverse. + if opt.no_log: + os.environ['REDO_LOG'] = '0' + if opt.no_pretty: + os.environ['REDO_PRETTY'] = '0' + if opt.no_color: + os.environ['REDO_COLOR'] = '0' + try: + state.init(targets) + if env.is_toplevel and not targets: + targets = ['all'] j = atoi(opt.jobs or 1) - if env_init.is_toplevel and (env.LOG or j > 1): + if env.is_toplevel and (env.v.LOG or j > 1): builder.close_stdin() - if env_init.is_toplevel and env.LOG: + if env.is_toplevel and env.v.LOG: builder.start_stdin_log_reader( status=opt.status, details=opt.details, pretty=opt.pretty, color=opt.color, @@ -112,11 +111,11 @@ def main(): traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) retcode = 1 - if env_init.is_toplevel: + if env.is_toplevel: builder.await_log_reader() sys.exit(retcode) except KeyboardInterrupt: - if env_init.is_toplevel: + if env.is_toplevel: builder.await_log_reader() sys.exit(200) diff --git a/redo/cmd_sources.py b/redo/cmd_sources.py index 0297f81..407a0c6 100644 --- a/redo/cmd_sources.py +++ b/redo/cmd_sources.py @@ -1,13 +1,10 @@ import sys, os - -import env_init -env_init.init([]) - import state, env from logs import err def main(): + state.init([]) if len(sys.argv[1:]) != 0: err('%s: no arguments expected.\n' % sys.argv[0]) sys.exit(1) @@ -15,7 +12,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_source(): - print state.relpath(os.path.join(env.BASE, f.name), cwd) + print state.relpath(os.path.join(env.v.BASE, f.name), cwd) if __name__ == '__main__': diff --git a/redo/cmd_stamp.py b/redo/cmd_stamp.py index 318d631..35c2d41 100644 --- a/redo/cmd_stamp.py +++ b/redo/cmd_stamp.py @@ -4,6 +4,7 @@ from logs import err, debug2 def main(): + env.inherit() if len(sys.argv) > 1: err('%s: no arguments expected.\n' % sys.argv[0]) sys.exit(1) @@ -32,11 +33,11 @@ def main(): csum = sh.hexdigest() - if not env.TARGET: + if not env.v.TARGET: sys.exit(0) - me = os.path.join(env.STARTDIR, - os.path.join(env.PWD, env.TARGET)) + me = os.path.join(env.v.STARTDIR, + os.path.join(env.v.PWD, env.v.TARGET)) f = state.File(name=me) changed = (csum != f.csum) debug2('%s: old = %s\n' % (f.name, f.csum)) diff --git a/redo/cmd_targets.py b/redo/cmd_targets.py index a90018e..6f42982 100644 --- a/redo/cmd_targets.py +++ b/redo/cmd_targets.py @@ -1,13 +1,10 @@ import sys, os - -import env_init -env_init.init([]) - -import state, env +import env, state from logs import err def main(): + state.init([]) if len(sys.argv[1:]) != 0: err('%s: no arguments expected.\n' % sys.argv[0]) sys.exit(1) @@ -15,7 +12,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_target(): - print state.relpath(os.path.join(env.BASE, f.name), cwd) + print state.relpath(os.path.join(env.v.BASE, f.name), cwd) if __name__ == '__main__': diff --git a/redo/cmd_unlocked.py b/redo/cmd_unlocked.py index 8ff87cf..28db641 100644 --- a/redo/cmd_unlocked.py +++ b/redo/cmd_unlocked.py @@ -1,9 +1,10 @@ import sys, os -import state +import env, state from logs import err def main(): + env.inherit() if len(sys.argv[1:]) < 2: err('%s: at least 2 arguments expected.\n' % sys.argv[0]) sys.exit(1) diff --git a/redo/cmd_whichdo.py b/redo/cmd_whichdo.py index 9d1121f..1343448 100644 --- a/redo/cmd_whichdo.py +++ b/redo/cmd_whichdo.py @@ -1,13 +1,10 @@ import sys, os - -import env_init -env_init.init_no_state() - -import paths +import env, paths from logs import err def main(): + env.init_no_state() if len(sys.argv[1:]) != 1: err('%s: exactly one argument expected.\n' % sys.argv[0]) sys.exit(1) diff --git a/redo/deps.py b/redo/deps.py index d96addf..9956b9b 100644 --- a/redo/deps.py +++ b/redo/deps.py @@ -16,7 +16,7 @@ def isdirty(f, depth, max_changed, # is unaffected already_checked = list(already_checked) + [f.id] - if env.DEBUG >= 1: + if env.v.DEBUG >= 1: debug('%s?%s %r,%r\n' % (depth, f.nicename(), f.is_generated, f.is_override)) @@ -28,10 +28,10 @@ def isdirty(f, depth, max_changed, return DIRTY if f.changed_runid > max_changed: debug('%s-- DIRTY (built %d > %d; %d)\n' - % (depth, f.changed_runid, max_changed, env.RUNID)) + % (depth, f.changed_runid, max_changed, env.v.RUNID)) return DIRTY # has been built more recently than parent if is_checked(f): - if env.DEBUG >= 1: + if env.v.DEBUG >= 1: debug('%s-- CLEAN (checked)\n' % depth) return CLEAN # has already been checked during this session if not f.stamp: @@ -65,7 +65,7 @@ def isdirty(f, depth, max_changed, for mode, f2 in f.deps(): dirty = CLEAN if mode == 'c': - if os.path.exists(os.path.join(env.BASE, f2.name)): + if os.path.exists(os.path.join(env.v.BASE, f2.name)): debug('%s-- DIRTY (created)\n' % depth) dirty = DIRTY elif mode == 'm': diff --git a/redo/env.py b/redo/env.py index a0e0123..9a95e46 100644 --- a/redo/env.py +++ b/redo/env.py @@ -1,35 +1,116 @@ -import os +import os, sys from atoi import atoi -if not os.environ.get('REDO'): - import sys - sys.stderr.write('%s: error: must be run from inside a .do\n' - % sys.argv[0]) - sys.exit(100) +is_toplevel = False -PWD = os.environ.get('REDO_PWD', '') -TARGET = os.environ.get('REDO_TARGET', '') -DEPTH = os.environ.get('REDO_DEPTH', '') -DEBUG = atoi(os.environ.get('REDO_DEBUG', '')) -DEBUG_LOCKS = os.environ.get('REDO_DEBUG_LOCKS', '') and 1 or 0 -DEBUG_PIDS = os.environ.get('REDO_DEBUG_PIDS', '') and 1 or 0 -VERBOSE = os.environ.get('REDO_VERBOSE', '') and 1 or 0 -XTRACE = os.environ.get('REDO_XTRACE', '') and 1 or 0 -KEEP_GOING = os.environ.get('REDO_KEEP_GOING', '') and 1 or 0 -LOG = atoi(os.environ.get('REDO_LOG', '1')) # defaults on -LOG_INODE = os.environ.get('REDO_LOG_INODE', '') -COLOR = atoi(os.environ.get('REDO_COLOR', '1')) # defaults on -# subprocesses mustn't pretty-print if a parent is running redo-log -PRETTY = (not LOG) and atoi(os.environ.get('REDO_PRETTY', '1')) -SHUFFLE = os.environ.get('REDO_SHUFFLE', '') and 1 or 0 -STARTDIR = os.environ.get('REDO_STARTDIR', '') -RUNID = atoi(os.environ.get('REDO_RUNID')) or None -BASE = os.environ['REDO_BASE'] -while BASE and BASE.endswith('/'): - BASE = BASE[:-1] +v = None -UNLOCKED = os.environ.get('REDO_UNLOCKED', '') and 1 or 0 -os.environ['REDO_UNLOCKED'] = '' # not inheritable by subprocesses -NO_OOB = os.environ.get('REDO_NO_OOB', '') and 1 or 0 -os.environ['REDO_NO_OOB'] = '' # not inheritable by subprocesses +def _get(name, default): + return os.environ.get(name, default) + + +def _get_int(name, default): + return atoi(_get(name, str(default))) + + +def _get_bool(name, default): + return 1 if _get(name, default) else 0 + + +class Env(object): + def __init__(self): + # Mandatory. If this is missing, you forgot to call init(). + self.BASE = os.environ['REDO_BASE'].rstrip('/') + + # Everything else, we can recover from defaults if unset. + self.PWD = _get('REDO_PWD', '') + self.TARGET = _get('REDO_TARGET', '') + self.DEPTH = _get('REDO_DEPTH', '') + self.DEBUG = atoi(_get('REDO_DEBUG', '')) + self.DEBUG_LOCKS = _get_bool('REDO_DEBUG_LOCKS', '') + self.DEBUG_PIDS = _get_bool('REDO_DEBUG_PIDS', '') + self.VERBOSE = _get_bool('REDO_VERBOSE', '') + self.XTRACE = _get_bool('REDO_XTRACE', '') + self.KEEP_GOING = _get_bool('REDO_KEEP_GOING', '') + self.LOG = _get_int('REDO_LOG', 1) # defaults on + self.LOG_INODE = _get('REDO_LOG_INODE', '') + self.COLOR = _get_int('REDO_COLOR', 1) # defaults on + # subprocesses mustn't pretty-print if a parent is running redo-log + self.PRETTY = (not self.LOG) and _get_int('REDO_PRETTY', 1) + self.SHUFFLE = _get_bool('REDO_SHUFFLE', '') + self.STARTDIR = _get('REDO_STARTDIR', '') + self.RUNID = _get_int('REDO_RUNID', '') or None + self.UNLOCKED = _get_bool('REDO_UNLOCKED', '') + self.NO_OOB = _get_bool('REDO_NO_OOB', '') + + +def inherit(): + """Read environment (which must already be set) to get runtime settings.""" + + if not os.environ.get('REDO'): + sys.stderr.write('%s: error: must be run from inside a .do\n' + % sys.argv[0]) + sys.exit(100) + + global v + v = Env() + + # not inheritable by subprocesses + os.environ['REDO_UNLOCKED'] = '' + os.environ['REDO_NO_OOB'] = '' + + +def init_no_state(): + """Start a session (if needed) for a command that needs no state db.""" + global is_toplevel + if not os.environ.get('REDO'): + os.environ['REDO'] = 'NOT_DEFINED' + is_toplevel = True + if not os.environ.get('REDO_BASE'): + os.environ['REDO_BASE'] = 'NOT_DEFINED' + inherit() + + +def init(targets): + """Start a session (if needed) for a command that does need the state db. + + Args: + targets: a list of targets we're trying to build. We use this to + help in guessing where the .redo database is located. + """ + global is_toplevel + if not os.environ.get('REDO'): + # toplevel call to redo + is_toplevel = True + exenames = [os.path.abspath(sys.argv[0]), + os.path.realpath(sys.argv[0])] + dirnames = [os.path.dirname(p) for p in exenames] + trynames = ([os.path.abspath(p+'/../lib/redo') for p in dirnames] + + [p+'/../redo' for p in dirnames] + + dirnames) + seen = {} + dirs = [] + for k in trynames: + if not seen.get(k): + seen[k] = 1 + dirs.append(k) + os.environ['PATH'] = ':'.join(dirs) + ':' + os.environ['PATH'] + os.environ['REDO'] = os.path.abspath(sys.argv[0]) + + if not os.environ.get('REDO_BASE'): + if not targets: + # if no other targets given, assume the current directory + targets = ['all'] + base = os.path.commonprefix([os.path.abspath(os.path.dirname(t)) + for t in targets] + [os.getcwd()]) + bsplit = base.split('/') + for i in range(len(bsplit)-1, 0, -1): + newbase = '/'.join(bsplit[:i]) + if os.path.exists(newbase + '/.redo'): + base = newbase + break + os.environ['REDO_BASE'] = base + os.environ['REDO_STARTDIR'] = os.getcwd() + + inherit() diff --git a/redo/env_init.py b/redo/env_init.py deleted file mode 100644 index 01c7d67..0000000 --- a/redo/env_init.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys, os - - -is_toplevel = False - - -def init_no_state(): - global is_toplevel - if not os.environ.get('REDO'): - os.environ['REDO'] = 'NOT_DEFINED' - is_toplevel = True - if not os.environ.get('REDO_BASE'): - os.environ['REDO_BASE'] = 'NOT_DEFINED' - - -def init(targets): - global is_toplevel - if not os.environ.get('REDO'): - # toplevel call to redo - is_toplevel = True - if len(targets) == 0: - targets.append('all') - exenames = [os.path.abspath(sys.argv[0]), - os.path.realpath(sys.argv[0])] - dirnames = [os.path.dirname(p) for p in exenames] - trynames = ([os.path.abspath(p+'/../lib/redo') for p in dirnames] + - [p+'/../redo' for p in dirnames] + - dirnames) - seen = {} - dirs = [] - for k in trynames: - if not seen.get(k): - seen[k] = 1 - dirs.append(k) - os.environ['PATH'] = ':'.join(dirs) + ':' + os.environ['PATH'] - os.environ['REDO'] = os.path.abspath(sys.argv[0]) - - if not os.environ.get('REDO_BASE'): - base = os.path.commonprefix([os.path.abspath(os.path.dirname(t)) - for t in targets] + [os.getcwd()]) - bsplit = base.split('/') - for i in range(len(bsplit)-1, 0, -1): - newbase = '/'.join(bsplit[:i]) - if os.path.exists(newbase + '/.redo'): - base = newbase - break - os.environ['REDO_BASE'] = base - os.environ['REDO_STARTDIR'] = os.getcwd() - - import state - state.init() - - os.environ['REDO_LOCKS'] = os.environ.get('REDO_LOCKS', '') diff --git a/redo/jobserver.py b/redo/jobserver.py index 45dee9f..1b0ab11 100644 --- a/redo/jobserver.py +++ b/redo/jobserver.py @@ -343,7 +343,7 @@ def ensure_token_or_cheat(reason, cheatfunc): if not has_token(): assert _mytokens == 0 n = cheatfunc() - _debug('%s: %s: cheat = %d\n' % (env.TARGET, reason, n)) + _debug('%s: %s: cheat = %d\n' % (env.v.TARGET, reason, n)) if n > 0: _mytokens += n _cheats += n diff --git a/redo/logs.py b/redo/logs.py index 145ec2d..f8bfa39 100644 --- a/redo/logs.py +++ b/redo/logs.py @@ -44,12 +44,12 @@ class PrettyLog(object): self.file = tty def _pretty(self, pid, color, s): - if env.DEBUG_PIDS: + if env.v.DEBUG_PIDS: redo = '%-6d redo ' % pid else: redo = 'redo ' self.file.write( - ''.join([color, redo, env.DEPTH, + ''.join([color, redo, env.v.DEPTH, BOLD if color else '', s, PLAIN, '\n'])) def write(self, s): @@ -75,17 +75,17 @@ class PrettyLog(object): rv = int(rv) if rv: self._pretty(pid, RED, '%s (exit %d)' % (name, rv)) - elif env.VERBOSE or env.XTRACE or env.DEBUG: + elif env.v.VERBOSE or env.v.XTRACE or env.v.DEBUG: self._pretty(pid, GREEN, '%s (done)' % name) self.file.write('\n') elif kind == 'locked': - if env.DEBUG_LOCKS: + if env.v.DEBUG_LOCKS: self._pretty(pid, GREEN, '%s (locked...)' % text) elif kind == 'waiting': - if env.DEBUG_LOCKS: + if env.v.DEBUG_LOCKS: self._pretty(pid, GREEN, '%s (WAITING)' % text) elif kind == 'unlocked': - if env.DEBUG_LOCKS: + if env.v.DEBUG_LOCKS: self._pretty(pid, GREEN, '%s (...unlocked!)' % text) elif kind == 'error': self.file.write(''.join([RED, 'redo: ', @@ -106,18 +106,21 @@ _log = None def setup(tty, pretty, color): global _log - if pretty or env.PRETTY: + if pretty or env.v.PRETTY: check_tty(tty, color=color) _log = PrettyLog(tty=tty) else: _log = RawLog(tty=tty) -# FIXME: explicitly initialize in each program, for clarity -setup(tty=sys.stderr, pretty=env.PRETTY, color=env.COLOR) +def _maybe_setup(): + # FIXME: explicitly initialize in each program, for clarity + if not _log: + setup(tty=sys.stderr, pretty=env.v.PRETTY, color=env.v.COLOR) def write(s): + _maybe_setup() _log.write(s) @@ -139,16 +142,16 @@ def warn(s): meta('warning', s) def debug(s): - if env.DEBUG >= 1: + if env.v.DEBUG >= 1: s = s.rstrip() meta('debug', s) def debug2(s): - if env.DEBUG >= 2: + if env.v.DEBUG >= 2: s = s.rstrip() meta('debug', s) def debug3(s): - if env.DEBUG >= 3: + if env.v.DEBUG >= 3: s = s.rstrip() meta('debug', s) diff --git a/redo/paths.py b/redo/paths.py index 51cb3c4..6aba1fa 100644 --- a/redo/paths.py +++ b/redo/paths.py @@ -15,7 +15,7 @@ def _default_do_files(filename): def possible_do_files(t): dirname, filename = os.path.split(t) - yield (os.path.join(env.BASE, dirname), "%s.do" % filename, + yield (os.path.join(env.v.BASE, dirname), "%s.do" % filename, '', filename, '') # It's important to try every possibility in a directory before resorting @@ -24,7 +24,7 @@ def possible_do_files(t): # the former one might just be an artifact of someone embedding my project # into theirs as a subdir. When they do, my rules should still be used # for building my project in *all* cases. - t = os.path.normpath(os.path.join(env.BASE, t)) + t = os.path.normpath(os.path.join(env.v.BASE, t)) dirname, filename = os.path.split(t) dirbits = dirname.split('/') # since t is an absolute path, dirbits[0] is always '', so we don't diff --git a/redo/state.py b/redo/state.py index a51bae2..7fbf6ab 100644 --- a/redo/state.py +++ b/redo/state.py @@ -46,7 +46,7 @@ def db(): if _db: return _db - dbdir = '%s/.redo' % env.BASE + dbdir = '%s/.redo' % env.v.BASE dbfile = '%s/db.sqlite3' % dbdir try: os.mkdir(dbdir) @@ -56,7 +56,7 @@ def db(): else: raise - _lockfile = os.open(os.path.join(env.BASE, '.redo/locks'), + _lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'), os.O_RDWR | os.O_CREAT, 0666) close_on_exec(_lockfile, True) @@ -106,17 +106,18 @@ def db(): _db.execute("insert into Runid values (1000000000)") _db.execute("insert into Files (name) values (?)", [ALWAYS]) - if not env.RUNID: + if not env.v.RUNID: _db.execute("insert into Runid values " " ((select max(id)+1 from Runid))") - env.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0] - os.environ['REDO_RUNID'] = str(env.RUNID) + env.v.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0] + os.environ['REDO_RUNID'] = str(env.v.RUNID) _db.commit() return _db -def init(): +def init(targets): + env.init(targets) db() @@ -155,7 +156,7 @@ _insane = None def check_sane(): global _insane if not _insane: - _insane = not os.path.exists('%s/.redo' % env.BASE) + _insane = not os.path.exists('%s/.redo' % env.v.BASE) return not _insane @@ -179,18 +180,18 @@ def relpath(t, base): return join('/', tparts) -# Return a path for t, if cwd were the dirname of env.TARGET. +# Return a path for t, if cwd were the dirname of env.v.TARGET. # This is tricky! STARTDIR+PWD is the directory for the *dofile*, when # the dofile was started. However, inside the dofile, someone may have done -# a chdir to anywhere else. env.TARGET is relative to the dofile path, so +# a chdir to anywhere else. env.v.TARGET is relative to the dofile path, so # we have to first figure out where the dofile was, then find TARGET relative # to that, then find t relative to that. # # FIXME: find some cleaner terminology for all these different paths. def target_relpath(t): - dofile_dir = os.path.abspath(os.path.join(env.STARTDIR, env.PWD)) + dofile_dir = os.path.abspath(os.path.join(env.v.STARTDIR, env.v.PWD)) target_dir = os.path.abspath( - os.path.dirname(os.path.join(dofile_dir, env.TARGET))) + os.path.dirname(os.path.join(dofile_dir, env.v.TARGET))) return relpath(t, target_dir) @@ -232,7 +233,7 @@ class File(object): q += 'where rowid=?' l = [fid] elif name != None: - name = (name == ALWAYS) and ALWAYS or relpath(name, env.BASE) + name = (name == ALWAYS) and ALWAYS or relpath(name, env.v.BASE) q += 'where name=?' l = [name] else: @@ -258,8 +259,8 @@ class File(object): (self.id, self.name, self.is_generated, self.is_override, self.checked_runid, self.changed_runid, self.failed_runid, self.stamp, self.csum) = cols - if self.name == ALWAYS and self.changed_runid < env.RUNID: - self.changed_runid = env.RUNID + if self.name == ALWAYS and self.changed_runid < env.v.RUNID: + self.changed_runid = env.v.RUNID def __init__(self, fid=None, name=None, cols=None, allow_add=True): if cols: @@ -284,7 +285,7 @@ class File(object): self.id]) def set_checked(self): - self.checked_runid = env.RUNID + self.checked_runid = env.v.RUNID def set_checked_save(self): self.set_checked() @@ -292,14 +293,14 @@ class File(object): def set_changed(self): debug2('BUILT: %r (%r)\n' % (self.name, self.stamp)) - self.changed_runid = env.RUNID + self.changed_runid = env.v.RUNID self.failed_runid = None self.is_override = False def set_failed(self): debug2('FAILED: %r\n' % self.name) self.update_stamp() - self.failed_runid = env.RUNID + self.failed_runid = env.v.RUNID if self.stamp != STAMP_MISSING: # if we failed and the target file still exists, # then we're generated. @@ -358,13 +359,13 @@ class File(object): return True def is_checked(self): - return self.checked_runid and self.checked_runid >= env.RUNID + return self.checked_runid and self.checked_runid >= env.v.RUNID def is_changed(self): - return self.changed_runid and self.changed_runid >= env.RUNID + return self.changed_runid and self.changed_runid >= env.v.RUNID def is_failed(self): - return self.failed_runid and self.failed_runid >= env.RUNID + return self.failed_runid and self.failed_runid >= env.v.RUNID def deps(self): if self.is_override or not self.is_generated: @@ -397,7 +398,7 @@ class File(object): def _read_stamp_st(self, statfunc): try: - st = statfunc(os.path.join(env.BASE, self.name)) + st = statfunc(os.path.join(env.v.BASE, self.name)) except OSError: return False, STAMP_MISSING if stat.S_ISDIR(st.st_mode): @@ -427,7 +428,7 @@ class File(object): return pre def nicename(self): - return relpath(os.path.join(env.BASE, self.name), env.STARTDIR) + return relpath(os.path.join(env.v.BASE, self.name), env.v.STARTDIR) def files(): @@ -438,7 +439,7 @@ def files(): def logname(fid): """Given the id of a File, return the filename of its build log.""" - return os.path.join(env.BASE, '.redo', 'log.%d' % fid) + return os.path.join(env.v.BASE, '.redo', 'log.%d' % fid) # FIXME: I really want to use fcntl F_SETLK, F_SETLKW, etc here. But python