diff --git a/helpers.py b/helpers.py index c16a9b2..87f3a7c 100644 --- a/helpers.py +++ b/helpers.py @@ -81,17 +81,3 @@ def relpath(t, base): return '/'.join(tparts) -def sname(typ, t, fromdir=None): - # FIXME: t.replace(...) is non-reversible and non-unique here! - if fromdir: - t = os.path.join(fromdir, t) - tnew = relpath(t, vars.BASE) - v = vars.BASE + ('/.redo/%s^%s' % (typ, tnew.replace('/', '^'))) - debug2('sname: (%r) %r -> %r\n' % (os.getcwd(), t, tnew)) - return v - - -def add_dep(t, mode, dep): - debug2('add-dep(%r)\n' % t) - open(sname('dep', t), 'a').write('%s %s\n' - % (mode, relpath(dep, vars.BASE))) diff --git a/redo-ifchange.py b/redo-ifchange.py index 3b6ca54..0e7d79c 100755 --- a/redo-ifchange.py +++ b/redo-ifchange.py @@ -1,19 +1,15 @@ #!/usr/bin/python import sys, os, errno -import vars -from helpers import sname, add_dep, debug, err, mkdirp, unlink +import vars, state +from helpers import debug, err, mkdirp, unlink def _dirty_deps(t, depth, fromdir): debug('%s?%s\n' % (depth, t)) - try: - stamptime = os.stat(sname('stamp', t, fromdir)).st_mtime - except OSError, e: - if e.errno == errno.ENOENT: - debug('%s-- DIRTY (no stamp)\n' % depth) - return True - else: - raise + stamptime = state.stamped(t, fromdir) + if stamptime == None: + debug('%s-- DIRTY (no stamp)\n' % depth) + return True try: realtime = os.stat(os.path.join(fromdir or '', t)).st_mtime @@ -24,12 +20,7 @@ def _dirty_deps(t, depth, fromdir): debug('%s-- DIRTY (mtime)\n' % depth) return True - for sub in open(sname('dep', t, fromdir)).readlines(): - assert(sub[0] in ('c','m')) - assert(sub[1] == ' ') - assert(sub[-1] == '\n') - mode = sub[0] - name = sub[2:-1] + for mode,name in state.deps(t, fromdir): if mode == 'c': if os.path.exists(name): debug('%s-- DIRTY (created)\n' % depth) @@ -43,7 +34,7 @@ def _dirty_deps(t, depth, fromdir): def dirty_deps(t, depth, fromdir=None): if _dirty_deps(t, depth, fromdir): - unlink(sname('stamp', t, fromdir)) # short circuit future checks + state.unstamp(t, fromdir) return True return False @@ -56,7 +47,7 @@ try: want_build = [] for t in sys.argv[1:]: mkdirp('%s/.redo' % vars.BASE) - add_dep(vars.TARGET, 'm', t) + state.add_dep(vars.TARGET, 'm', t) if dirty_deps(t, depth = ''): want_build.append(t) diff --git a/redo-ifcreate.py b/redo-ifcreate.py index 3ddbfc8..2bd8cd1 100755 --- a/redo-ifcreate.py +++ b/redo-ifcreate.py @@ -1,7 +1,7 @@ #!/usr/bin/python import sys, os -import vars -from helpers import err, add_dep, mkdirp +import vars, state +from helpers import err, mkdirp if not vars.TARGET: @@ -15,6 +15,6 @@ try: err('redo-ifcreate: error: %r already exists\n' % t) sys.exit(1) else: - add_dep(vars.TARGET, 'c', t) + state.add_dep(vars.TARGET, 'c', t) except KeyboardInterrupt: sys.exit(200) diff --git a/redo.py b/redo.py index 35165f6..0fcdcaf 100755 --- a/redo.py +++ b/redo.py @@ -43,7 +43,7 @@ if not os.environ.get('REDO_BASE', ''): os.unlink(f) -import vars +import vars, state from helpers import * @@ -70,30 +70,13 @@ def find_do_file(t): for dofile,basename,ext in _possible_do_files(t): debug2('%s: %s ?\n' % (t, dofile)) if os.path.exists(dofile): - add_dep(t, 'm', dofile) + state.add_dep(t, 'm', dofile) return dofile,basename,ext else: - add_dep(t, 'c', dofile) + state.add_dep(t, 'c', dofile) return None,None,None -def stamp(t): - stampfile = sname('stamp', t) - newstampfile = sname('stamp' + str(os.getpid()), t) - depfile = sname('dep', t) - if not os.path.exists(vars.BASE + '/.redo'): - # .redo might not exist in a 'make clean' target - return - open(newstampfile, 'w').close() - try: - mtime = os.stat(t).st_mtime - except OSError: - mtime = 0 - os.utime(newstampfile, (mtime, mtime)) - os.rename(newstampfile, stampfile) - open(depfile, 'a').close() - - def _preexec(t): os.environ['REDO_TARGET'] = os.path.basename(t) os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' @@ -103,22 +86,21 @@ def _preexec(t): def _build(t): - if (os.path.exists(t) and not os.path.exists(sname('gen', t)) - and not os.path.exists('%s.do' % t)): + if (os.path.exists(t) and not state.is_generated(t) + and not os.path.exists('%s.do' % t)): # an existing source file that is not marked as a generated file. # This step is mentioned by djb in his notes. It turns out to be # important to prevent infinite recursion. For example, a rule # called default.c.do could be used to try to produce hello.c, # which is undesirable since hello.c existed already. - stamp(t) + state.stamp(t) return # success - unlink(sname('dep', t)) - open(sname('dep', t), 'w').close() - open(sname('gen', t), 'w').close() # it's definitely a generated file + state.unstamp(t) + state.start(t) (dofile, basename, ext) = find_do_file(t) if not dofile: raise BuildError('no rule to make %r' % t) - stamp(dofile) + state.stamp(dofile) unlink(t) tmpname = '%s.redo.tmp' % t unlink(tmpname) @@ -137,7 +119,6 @@ def _build(t): log('%s\n' % relpath(t, vars.STARTDIR)) rv = subprocess.call(argv, preexec_fn=lambda: _preexec(t), stdout=f.fileno()) - stampfile = sname('stamp', t) if rv==0: if os.path.exists(tmpname) and os.stat(tmpname).st_size: # there's a race condition here, but if the tmpfile disappears @@ -146,10 +127,10 @@ def _build(t): os.rename(tmpname, t) else: unlink(tmpname) - stamp(t) + state.stamp(t) else: unlink(tmpname) - unlink(stampfile) + state.unstamp(t) f.close() if rv != 0: raise BuildError('%s: exit code %d' % (t,rv)) @@ -159,33 +140,18 @@ def _build(t): def build(t): mkdirp('%s/.redo' % vars.BASE) - lockname = sname('lock', t) - try: - os.mkfifo(lockname, 0600) - except OSError, e: - if e.errno == errno.EEXIST: - log('%s (locked...)\n' % relpath(t, vars.STARTDIR)) - os._exit(199) - else: - raise + lock = state.Lock(t) + lock.lock() + if not lock.owned: + log('%s (locked...)\n' % relpath(t, vars.STARTDIR)) + os._exit(199) try: try: return _build(t) except BuildError, e: err('%s\n' % e) finally: - fd = None - try: - fd = os.open(lockname, os.O_WRONLY|os.O_NONBLOCK) - except OSError, e: - if e.errno == errno.ENXIO: # no readers open; that's ok - pass - elif e.errno == errno.ENOENT: # 'make clean' might do this - pass - else: - raise - unlink(lockname) - if fd != None: os.close(fd) + lock.unlock() os._exit(1) @@ -210,19 +176,11 @@ def main(): err('%s: exit code was %r\n' % (t, pd.rv)) retcode = 1 for t in locked.keys(): - lockname = sname('lock', t) - stampname = sname('stamp', t) - try: - # open() will finish only when an existing writer does close() - os.close(os.open(lockname, os.O_RDONLY)) - except OSError, e: - if e.errno == errno.ENOENT: - pass # already got unlocked - else: - raise + lock = state.Lock(t) + lock.wait() relp = relpath(t, vars.STARTDIR) log('%s (...unlocked!)\n' % relp) - if not os.path.exists(stampname): + if state.stamped(t) == None: err('%s: failed in another thread\n' % relp) retcode = 2 return retcode diff --git a/state.py b/state.py new file mode 100644 index 0000000..5ab86b4 --- /dev/null +++ b/state.py @@ -0,0 +1,126 @@ +import sys, os, errno +import vars +from helpers import unlink, relpath, debug2 + + +def _sname(typ, t, fromdir=None): + # FIXME: t.replace(...) is non-reversible and non-unique here! + if fromdir: + t = os.path.join(fromdir, t) + tnew = relpath(t, vars.BASE) + v = vars.BASE + ('/.redo/%s^%s' % (typ, tnew.replace('/', '^'))) + debug2('sname: (%r) %r -> %r\n' % (os.getcwd(), t, tnew)) + return v + + +def add_dep(t, mode, dep): + debug2('add-dep(%r)\n' % t) + open(_sname('dep', t), 'a').write('%s %s\n' + % (mode, relpath(dep, vars.BASE))) + + +def deps(t, fromdir=None): + for line in open(_sname('dep', t, fromdir)).readlines(): + assert(line[0] in ('c','m')) + assert(line[1] == ' ') + assert(line[-1] == '\n') + mode = line[0] + name = line[2:-1] + yield mode,name + + +def _stampname(t, fromdir=None): + return _sname('stamp', t, fromdir) + + +def stamp(t): + stampfile = _stampname(t) + newstampfile = _sname('stamp' + str(os.getpid()), t) + depfile = _sname('dep', t) + if not os.path.exists(vars.BASE + '/.redo'): + # .redo might not exist in a 'make clean' target + return + open(newstampfile, 'w').close() + try: + mtime = os.stat(t).st_mtime + except OSError: + mtime = 0 + os.utime(newstampfile, (mtime, mtime)) + os.rename(newstampfile, stampfile) + open(depfile, 'a').close() + + +def unstamp(t, fromdir=None): + unlink(_stampname(t, fromdir)) + + +def stamped(t, fromdir=None): + try: + stamptime = os.stat(_stampname(t, fromdir)).st_mtime + except OSError, e: + if e.errno == errno.ENOENT: + return None + else: + raise + return stamptime + + +def is_generated(t): + return os.path.exists(_sname('gen', t)) + + +def start(t): + open(_sname('dep', t), 'w').close() + open(_sname('gen', t), 'w').close() # it's definitely a generated file + + +class Lock: + def __init__(self, t): + self.lockname = _sname('lock', t) + self.owned = False + + def __del__(self): + if self.owned: + self.unlock() + + def lock(self): + try: + os.mkfifo(self.lockname, 0600) + self.owned = True + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise + + def unlock(self): + if not self.owned: + raise Exception("can't unlock %r - we don't own it" + % self.lockname) + fd = None + try: + fd = os.open(self.lockname, os.O_WRONLY|os.O_NONBLOCK) + except OSError, e: + if e.errno == errno.ENXIO: # no readers open; that's ok + pass + elif e.errno == errno.ENOENT: # 'make clean' might do this + pass + else: + raise + unlink(self.lockname) # make sure no new readers can connect + if fd != None: os.close(fd) # now unlock any existing readers + self.owned = False + + def wait(self): + if self.owned: + raise Exception("can't wait on %r - we own it" % self.lockname) + try: + # open() will finish only when a writer exists and does close() + os.close(os.open(self.lockname, os.O_RDONLY)) + #sys.stderr.write('lock %r waited ok\n' % self.lockname) + except OSError, e: + if e.errno == errno.ENOENT: + #sys.stderr.write('lock %r missing\n' % self.lockname) + pass # it's not even unlocked or was unlocked earlier + else: + raise