Extract .redo dir state management stuff into its own file.

In preparation for changing the on-disk format eventually, as well as making
the main code more readable.

helpers.py

@@ -81,17 +81,3 @@ def relpath(t, base):
     return '/'.join(tparts)
 
 
-def sname(typ, t, fromdir=None):
-    # FIXME: t.replace(...) is non-reversible and non-unique here!
-    if fromdir:
-        t = os.path.join(fromdir, t)
-    tnew = relpath(t, vars.BASE)
-    v = vars.BASE + ('/.redo/%s^%s' % (typ, tnew.replace('/', '^')))
-    debug2('sname: (%r) %r -> %r\n' % (os.getcwd(), t, tnew))
-    return v
-
-
-def add_dep(t, mode, dep):
-    debug2('add-dep(%r)\n' % t)
-    open(sname('dep', t), 'a').write('%s %s\n'
-                                     % (mode, relpath(dep, vars.BASE)))

redo-ifchange.py

@@ -1,19 +1,15 @@
 #!/usr/bin/python
 import sys, os, errno
-import vars
+import vars, state
-from helpers import sname, add_dep, debug, err, mkdirp, unlink
+from helpers import debug, err, mkdirp, unlink
 
 
 def _dirty_deps(t, depth, fromdir):
     debug('%s?%s\n' % (depth, t))
-    try:
+    stamptime = state.stamped(t, fromdir)
-        stamptime = os.stat(sname('stamp', t, fromdir)).st_mtime
+    if stamptime == None:
-    except OSError, e:
-        if e.errno == errno.ENOENT:
         debug('%s-- DIRTY (no stamp)\n' % depth)
         return True
-        else:
-            raise
 
     try:
         realtime = os.stat(os.path.join(fromdir or '', t)).st_mtime
@@ -24,12 +20,7 @@ def _dirty_deps(t, depth, fromdir):
         debug('%s-- DIRTY (mtime)\n' % depth)
         return True
     
-    for sub in open(sname('dep', t, fromdir)).readlines():
+    for mode,name in state.deps(t, fromdir):
-        assert(sub[0] in ('c','m'))
-        assert(sub[1] == ' ')
-        assert(sub[-1] == '\n')
-        mode = sub[0]
-        name = sub[2:-1]
         if mode == 'c':
             if os.path.exists(name):
                 debug('%s-- DIRTY (created)\n' % depth)
@@ -43,7 +34,7 @@ def _dirty_deps(t, depth, fromdir):
 
 def dirty_deps(t, depth, fromdir=None):
     if _dirty_deps(t, depth, fromdir):
-        unlink(sname('stamp', t, fromdir))  # short circuit future checks
+        state.unstamp(t, fromdir)
         return True
     return False
 
@@ -56,7 +47,7 @@ try:
     want_build = []
     for t in sys.argv[1:]:
         mkdirp('%s/.redo' % vars.BASE)
-        add_dep(vars.TARGET, 'm', t)
+        state.add_dep(vars.TARGET, 'm', t)
         if dirty_deps(t, depth = ''):
             want_build.append(t)
 

redo-ifcreate.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 import sys, os
-import vars
+import vars, state
-from helpers import err, add_dep, mkdirp
+from helpers import err, mkdirp
 
 
 if not vars.TARGET:
@@ -15,6 +15,6 @@ try:
             err('redo-ifcreate: error: %r already exists\n' % t)
             sys.exit(1)
         else:
-            add_dep(vars.TARGET, 'c', t)
+            state.add_dep(vars.TARGET, 'c', t)
 except KeyboardInterrupt:
     sys.exit(200)

redo.py

@@ -43,7 +43,7 @@ if not os.environ.get('REDO_BASE', ''):
         os.unlink(f)
 
 
-import vars
+import vars, state
 from helpers import *
 
 
@@ -70,30 +70,13 @@ def find_do_file(t):
     for dofile,basename,ext in _possible_do_files(t):
         debug2('%s: %s ?\n' % (t, dofile))
         if os.path.exists(dofile):
-            add_dep(t, 'm', dofile)
+            state.add_dep(t, 'm', dofile)
             return dofile,basename,ext
         else:
-            add_dep(t, 'c', dofile)
+            state.add_dep(t, 'c', dofile)
     return None,None,None
 
 
-def stamp(t):
-    stampfile = sname('stamp', t)
-    newstampfile = sname('stamp' + str(os.getpid()), t)
-    depfile = sname('dep', t)
-    if not os.path.exists(vars.BASE + '/.redo'):
-        # .redo might not exist in a 'make clean' target
-        return
-    open(newstampfile, 'w').close()
-    try:
-        mtime = os.stat(t).st_mtime
-    except OSError:
-        mtime = 0
-    os.utime(newstampfile, (mtime, mtime))
-    os.rename(newstampfile, stampfile)
-    open(depfile, 'a').close()
-
-
 def _preexec(t):
     os.environ['REDO_TARGET'] = os.path.basename(t)
     os.environ['REDO_DEPTH'] = vars.DEPTH + '  '
@@ -103,22 +86,21 @@ def _preexec(t):
 
 
 def _build(t):
-    if (os.path.exists(t) and not os.path.exists(sname('gen', t))
+    if (os.path.exists(t) and not state.is_generated(t)
           and not os.path.exists('%s.do' % t)):
         # an existing source file that is not marked as a generated file.
         # This step is mentioned by djb in his notes.  It turns out to be
         # important to prevent infinite recursion.  For example, a rule
         # called default.c.do could be used to try to produce hello.c,
         # which is undesirable since hello.c existed already.
-        stamp(t)
+        state.stamp(t)
         return  # success
-    unlink(sname('dep', t))
+    state.unstamp(t)
-    open(sname('dep', t), 'w').close()
+    state.start(t)
-    open(sname('gen', t), 'w').close()  # it's definitely a generated file
     (dofile, basename, ext) = find_do_file(t)
     if not dofile:
         raise BuildError('no rule to make %r' % t)
-    stamp(dofile)
+    state.stamp(dofile)
     unlink(t)
     tmpname = '%s.redo.tmp' % t
     unlink(tmpname)
@@ -137,7 +119,6 @@ def _build(t):
     log('%s\n' % relpath(t, vars.STARTDIR))
     rv = subprocess.call(argv, preexec_fn=lambda: _preexec(t),
                          stdout=f.fileno())
-    stampfile = sname('stamp', t)
     if rv==0:
         if os.path.exists(tmpname) and os.stat(tmpname).st_size:
             # there's a race condition here, but if the tmpfile disappears
@@ -146,10 +127,10 @@ def _build(t):
             os.rename(tmpname, t)
         else:
             unlink(tmpname)
-        stamp(t)
+        state.stamp(t)
     else:
         unlink(tmpname)
-        unlink(stampfile)
+        state.unstamp(t)
     f.close()
     if rv != 0:
         raise BuildError('%s: exit code %d' % (t,rv))
@@ -159,33 +140,18 @@ def _build(t):
 
 def build(t):
     mkdirp('%s/.redo' % vars.BASE)
-    lockname = sname('lock', t)
+    lock = state.Lock(t)
-    try:
+    lock.lock()
-        os.mkfifo(lockname, 0600)
+    if not lock.owned:
-    except OSError, e:
-        if e.errno == errno.EEXIST:
         log('%s (locked...)\n' % relpath(t, vars.STARTDIR))
         os._exit(199)
-        else:
-            raise
     try:
         try:
             return _build(t)
         except BuildError, e:
             err('%s\n' % e)
     finally:
-        fd = None
+        lock.unlock()
-        try:
-            fd = os.open(lockname, os.O_WRONLY|os.O_NONBLOCK)
-        except OSError, e:
-            if e.errno == errno.ENXIO: # no readers open; that's ok
-                pass
-            elif e.errno == errno.ENOENT: # 'make clean' might do this
-                pass
-            else:
-                raise
-        unlink(lockname)
-        if fd != None: os.close(fd)
     os._exit(1)
 
 
@@ -210,19 +176,11 @@ def main():
             err('%s: exit code was %r\n' % (t, pd.rv))
             retcode = 1
     for t in locked.keys():
-        lockname = sname('lock', t)
+        lock = state.Lock(t)
-        stampname = sname('stamp', t)
+        lock.wait()
-        try:
-            # open() will finish only when an existing writer does close()
-            os.close(os.open(lockname, os.O_RDONLY))
-        except OSError, e:
-            if e.errno == errno.ENOENT:
-                pass  # already got unlocked
-            else:
-                raise
         relp = relpath(t, vars.STARTDIR)
         log('%s (...unlocked!)\n' % relp)
-        if not os.path.exists(stampname):
+        if state.stamped(t) == None:
             err('%s: failed in another thread\n' % relp)
             retcode = 2
     return retcode

state.py

@@ -0,0 +1,126 @@
+import sys, os, errno
+import vars
+from helpers import unlink, relpath, debug2
+
+
+def _sname(typ, t, fromdir=None):
+    # FIXME: t.replace(...) is non-reversible and non-unique here!
+    if fromdir:
+        t = os.path.join(fromdir, t)
+    tnew = relpath(t, vars.BASE)
+    v = vars.BASE + ('/.redo/%s^%s' % (typ, tnew.replace('/', '^')))
+    debug2('sname: (%r) %r -> %r\n' % (os.getcwd(), t, tnew))
+    return v
+
+
+def add_dep(t, mode, dep):
+    debug2('add-dep(%r)\n' % t)
+    open(_sname('dep', t), 'a').write('%s %s\n'
+                                      % (mode, relpath(dep, vars.BASE)))
+
+
+def deps(t, fromdir=None):
+    for line in open(_sname('dep', t, fromdir)).readlines():
+        assert(line[0] in ('c','m'))
+        assert(line[1] == ' ')
+        assert(line[-1] == '\n')
+        mode = line[0]
+        name = line[2:-1]
+        yield mode,name
+
+
+def _stampname(t, fromdir=None):
+    return _sname('stamp', t, fromdir)
+    
+
+def stamp(t):
+    stampfile = _stampname(t)
+    newstampfile = _sname('stamp' + str(os.getpid()), t)
+    depfile = _sname('dep', t)
+    if not os.path.exists(vars.BASE + '/.redo'):
+        # .redo might not exist in a 'make clean' target
+        return
+    open(newstampfile, 'w').close()
+    try:
+        mtime = os.stat(t).st_mtime
+    except OSError:
+        mtime = 0
+    os.utime(newstampfile, (mtime, mtime))
+    os.rename(newstampfile, stampfile)
+    open(depfile, 'a').close()
+
+
+def unstamp(t, fromdir=None):
+    unlink(_stampname(t, fromdir))
+
+
+def stamped(t, fromdir=None):
+    try:
+        stamptime = os.stat(_stampname(t, fromdir)).st_mtime
+    except OSError, e:
+        if e.errno == errno.ENOENT:
+            return None
+        else:
+            raise
+    return stamptime
+
+
+def is_generated(t):
+    return os.path.exists(_sname('gen', t))
+
+
+def start(t):
+    open(_sname('dep', t), 'w').close()
+    open(_sname('gen', t), 'w').close()  # it's definitely a generated file
+
+
+class Lock:
+    def __init__(self, t):
+        self.lockname = _sname('lock', t)
+        self.owned = False
+
+    def __del__(self):
+        if self.owned:
+            self.unlock()
+
+    def lock(self):
+        try:
+            os.mkfifo(self.lockname, 0600)
+            self.owned = True
+        except OSError, e:
+            if e.errno == errno.EEXIST:
+                pass
+            else:
+                raise
+
+    def unlock(self):
+        if not self.owned:
+            raise Exception("can't unlock %r - we don't own it" 
+                            % self.lockname)
+        fd = None
+        try:
+            fd = os.open(self.lockname, os.O_WRONLY|os.O_NONBLOCK)
+        except OSError, e:
+            if e.errno == errno.ENXIO: # no readers open; that's ok
+                pass
+            elif e.errno == errno.ENOENT: # 'make clean' might do this
+                pass
+            else:
+                raise
+        unlink(self.lockname)  # make sure no new readers can connect
+        if fd != None: os.close(fd)  # now unlock any existing readers
+        self.owned = False
+
+    def wait(self):
+        if self.owned:
+            raise Exception("can't wait on %r - we own it" % self.lockname)
+        try:
+            # open() will finish only when a writer exists and does close()
+            os.close(os.open(self.lockname, os.O_RDONLY))
+            #sys.stderr.write('lock %r waited ok\n' % self.lockname)
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                #sys.stderr.write('lock %r missing\n' % self.lockname)
+                pass  # it's not even unlocked or was unlocked earlier
+            else:
+                raise