Merge branch 'sqlite'

This replaces the .redo state directory with an sqlite database instead,
improving correctness and sometimes performance.

builder.py

@@ -1,9 +1,10 @@
-import sys, os, random, errno, stat
+import sys, os, errno, stat
 import vars, jwack, state
-from helpers import log, log_, debug2, err, unlink, close_on_exec
+from helpers import log, log_, debug2, err, warn, unlink, close_on_exec
 
 
 def _possible_do_files(t):
+    t = os.path.join(vars.BASE, t)
     yield "%s.do" % t, t, ''
     dirname,filename = os.path.split(t)
     l = filename.split('.')
@@ -16,14 +17,14 @@ def _possible_do_files(t):
                os.path.join(dirname, basename), ext)
 
 
-def _find_do_file(t):
+def _find_do_file(f):
-    for dofile,basename,ext in _possible_do_files(t):
+    for dofile,basename,ext in _possible_do_files(f.name):
-        debug2('%s: %s ?\n' % (t, dofile))
+        debug2('%s: %s ?\n' % (f.name, dofile))
         if os.path.exists(dofile):
-            state.add_dep(t, 'm', dofile)
+            f.add_dep('m', dofile)
             return dofile,basename,ext
         else:
-            state.add_dep(t, 'c', dofile)
+            f.add_dep('c', dofile)
     return None,None,None
 
 
@@ -42,8 +43,9 @@ def _try_stat(filename):
 
 
 class BuildJob:
-    def __init__(self, t, lock, shouldbuildfunc, donefunc):
+    def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
-        self.t = t
+        self.t = t  # original target name, not relative to vars.BASE
+        self.sf = sf
         self.tmpname = '%s.redo.tmp' % t
         self.lock = lock
         self.shouldbuildfunc = shouldbuildfunc
@@ -53,12 +55,13 @@ class BuildJob:
     def start(self):
         assert(self.lock.owned)
         t = self.t
+        sf = self.sf
         tmpname = self.tmpname
         if not self.shouldbuildfunc(t):
             # target doesn't need to be built; skip the whole task
             return self._after2(0)
         if (os.path.exists(t) and not os.path.exists(t + '/.')
-             and not state.is_generated(t)):
+             and not sf.is_generated):
             # an existing source file that was not generated by us.
             # This step is mentioned by djb in his notes.
             # For example, a rule called default.c.do could be used to try
@@ -67,20 +70,21 @@ class BuildJob:
             # FIXME: always refuse to redo any file that was modified outside
             # of redo?  That would make it easy for someone to override a
             # file temporarily, and could be undone by deleting the file.
-            state.unmark_as_generated(t)
+            debug2("-- static (%r)\n" % t)
-            state.stamp_and_maybe_built(t)
+            sf.set_static()
+            sf.save()
             return self._after2(0)
-        state.start(t)
+        sf.zap_deps()
-        (dofile, basename, ext) = _find_do_file(t)
+        (dofile, basename, ext) = _find_do_file(sf)
         if not dofile:
             if os.path.exists(t):
-                state.unmark_as_generated(t)
+                sf.is_generated = False
-                state.stamp_and_maybe_built(t)
+                sf.set_static()
+                sf.save()
                 return self._after2(0)
             else:
                 err('no rule to make %r\n' % t)
                 return self._after2(1)
-        state.stamp_and_maybe_built(dofile)
         unlink(tmpname)
         ffd = os.open(tmpname, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
         close_on_exec(ffd, True)
@@ -97,13 +101,20 @@ class BuildJob:
         if vars.VERBOSE or vars.XTRACE: log_('\n')
         log('%s\n' % _nice(t))
         self.argv = argv
+        sf.is_generated = True
+        sf.save()
+        dof = state.File(name=dofile)
+        dof.set_static()
+        dof.save()
+        state.commit()
         jwack.start_job(t, self._do_subproc, self._after)
 
     def _do_subproc(self):
         # careful: REDO_PWD was the PWD relative to the STARTPATH at the time
         # we *started* building the current target; but that target ran
         # redo-ifchange, and it might have done it from a different directory
-        # than we started it in.  So os.getcwd() might be != REDO_PWD right now.
+        # than we started it in.  So os.getcwd() might be != REDO_PWD right
+        # now.
         dn = os.path.dirname(self.t)
         newp = os.path.realpath(dn)
         os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
@@ -121,7 +132,9 @@ class BuildJob:
 
     def _after(self, t, rv):
         try:
+            state.check_sane()
             rv = self._after1(t, rv)
+            state.commit()
         finally:
             self._after2(rv)
 
@@ -153,11 +166,17 @@ class BuildJob:
                 os.rename(tmpname, t)
             else:
                 unlink(tmpname)
-            state.built(t)
+            sf = self.sf
-            state.stamp(t)
+            sf.is_generated=True
+            sf.update_stamp()
+            sf.set_changed()
+            sf.save()
         else:
             unlink(tmpname)
-            state.unstamp(t)
+            sf = self.sf
+            sf.stamp = None
+            sf.set_changed()
+            sf.save()
         f.close()
         if rv != 0:
             err('%s: exit code %d\n' % (_nice(t),rv))
@@ -177,6 +196,7 @@ class BuildJob:
 def main(targets, shouldbuildfunc):
     retcode = [0]  # a list so that it can be reassigned from done()
     if vars.SHUFFLE:
+        import random
         random.shuffle(targets)
 
     locked = []
@@ -191,45 +211,60 @@ def main(targets, shouldbuildfunc):
     # In the first cycle, we just build as much as we can without worrying
     # about any lock contention.  If someone else has it locked, we move on.
     for t in targets:
+        if not jwack.has_token():
+            state.commit()
         jwack.get_token(t)
         if retcode[0] and not vars.KEEP_GOING:
             break
-        if not state.is_sane():
+        if not state.check_sane():
+            err('.redo directory disappeared; cannot continue.\n')
             retcode[0] = 205
             break
-        lock = state.Lock(t)
+        f = state.File(name=t)
+        lock = state.Lock(f.id)
         lock.trylock()
         if not lock.owned:
             if vars.DEBUG_LOCKS:
                 log('%s (locked...)\n' % _nice(t))
-            locked.append(t)
+            locked.append((f.id,t))
         else:
-            BuildJob(t, lock, shouldbuildfunc, done).start()
+            BuildJob(t, f, lock, shouldbuildfunc, done).start()
 
     # Now we've built all the "easy" ones.  Go back and just wait on the
-    # remaining ones one by one.  This is technically non-optimal; we could
+    # remaining ones one by one.  There's no reason to do it any more
-    # use select.select() to wait on more than one at a time.  But it should
+    # efficiently, because if these targets were previously locked, that
-    # be rare enough that it doesn't matter, and the logic is easier this way.
+    # means someone else was building them; thus, we probably won't need to
+    # do anything.  The only exception is if we're invoked as redo instead
+    # of redo-ifchange; then we have to redo it even if someone else already
+    # did.  But that should be rare.
     while locked or jwack.running():
+        state.commit()
         jwack.wait_all()
         # at this point, we don't have any children holding any tokens, so
         # it's okay to block below.
         if retcode[0] and not vars.KEEP_GOING:
             break
         if locked:
-            if not state.is_sane():
+            if not state.check_sane():
+                err('.redo directory disappeared; cannot continue.\n')
                 retcode[0] = 205
                 break
-            t = locked.pop(0)
+            fid,t = locked.pop(0)
-            lock = state.Lock(t)
+            lock = state.Lock(fid)
-            lock.waitlock()
+            lock.trylock()
+            if not lock.owned:
+                if vars.DEBUG_LOCKS and len(locked) >= 1:
+                    warn('%s (WAITING)\n' % _nice(t))
+                lock.waitlock()
             assert(lock.owned)
             if vars.DEBUG_LOCKS:
                 log('%s (...unlocked!)\n' % _nice(t))
-            if state.stamped(t) == None:
+            if state.File(name=t).stamp == None:
                 err('%s: failed in another thread\n' % _nice(t))
                 retcode[0] = 2
                 lock.unlock()
             else:
-                BuildJob(t, lock, shouldbuildfunc, done).start()
+                BuildJob(t, state.File(id=fid), lock,
+                         shouldbuildfunc, done).start()
+    state.commit()
     return retcode[0]

helpers.py

@@ -15,24 +15,6 @@ def unlink(f):
             pass  # it doesn't exist, that's what you asked for
 
 
-def mkdirp(d, mode=None):
-    """Recursively create directories on path 'd'.
-
-    Unlike os.makedirs(), it doesn't raise an exception if the last element of
-    the path already exists.
-    """
-    try:
-        if mode:
-            os.makedirs(d, mode)
-        else:
-            os.makedirs(d)
-    except OSError, e:
-        if e.errno == errno.EEXIST:
-            pass
-        else:
-            raise
-
-
 def log_(s):
     sys.stdout.flush()
     if vars.DEBUG_PIDS:
@@ -52,13 +34,20 @@ def _cerr(s):
 def _bwerr(s):
     log_('redo: %s%s' % (vars.DEPTH, s))
 
+def _cwarn(s):
+    log_('\x1b[33mredo: %s\x1b[1m%s\x1b[m' % (vars.DEPTH, s))
+def _bwwarn(s):
+    log_('redo: %s%s' % (vars.DEPTH, s))
+
 
 if os.isatty(2):
     log = _clog
     err = _cerr
+    warn = _cwarn
 else:
     log = _bwlog
     err = _bwerr
+    warn = _bwwarn
 
 
 def debug(s):

jwack.py

@@ -1,7 +1,7 @@
 #
 # beware the jobberwack
 #
-import sys, os, errno, select, fcntl
+import sys, os, errno, select, fcntl, signal
 import atoi
 
 _toplevel = 0
@@ -24,22 +24,35 @@ def _release(n):
         _mytokens = 1
 
 
+def _timeout(sig, frame):
+    pass
+
+
 def _try_read(fd, n):
-    # FIXME: this isn't actually safe, because GNU make can't handle it if
+    # using djb's suggested way of doing non-blocking reads from a blocking
-    # the socket is nonblocking.  Ugh.  That means we'll have to do their
+    # socket: http://cr.yp.to/unix/nonblock.html
-    # horrible SIGCHLD hack after all.
+    # We can't just make the socket non-blocking, because we want to be
-    fcntl.fcntl(_fds[0], fcntl.F_SETFL, os.O_NONBLOCK)
+    # compatible with GNU Make, and they can't handle it.
+    r,w,x = select.select([fd], [], [], 0)
+    if not r:
+        return ''  # try again
+    # ok, the socket is readable - but some other process might get there
+    # first.  We have to set an alarm() in case our read() gets stuck.
+    oldh = signal.signal(signal.SIGALRM, _timeout)
     try:
+        signal.alarm(1)  # emergency fallback
         try:
             b = os.read(_fds[0], 1)
         except OSError, e:
-            if e.errno == errno.EAGAIN:
+            if e.errno in (errno.EAGAIN, errno.EINTR):
-                return ''
+                # interrupted or it was nonblocking
+                return ''  # try again
             else:
                 raise
     finally:
-        fcntl.fcntl(_fds[0], fcntl.F_SETFL, 0)
+        signal.alarm(0)
-    return b and b or None
+        signal.signal(signal.SIGALRM, oldh)
+    return b and b or None  # None means EOF
 
 
 def setup(maxjobs):
@@ -70,7 +83,11 @@ def setup(maxjobs):
     if maxjobs and not _fds:
         # need to start a new server
         _toplevel = maxjobs
-        _fds = os.pipe()
+        _fds1 = os.pipe()
+        _fds = (fcntl.fcntl(_fds1[0], fcntl.F_DUPFD, 100),
+                fcntl.fcntl(_fds1[1], fcntl.F_DUPFD, 101))
+        os.close(_fds1[0])
+        os.close(_fds1[1])
         _release(maxjobs-1)
         os.putenv('MAKEFLAGS',
                   '%s --jobserver-fds=%d,%d -j' % (os.getenv('MAKEFLAGS'),
@@ -105,6 +122,11 @@ def wait(want_token):
             pd.donefunc(pd.name, pd.rv)
 
 
+def has_token():
+    if _mytokens >= 1:
+        return True
+
+
 def get_token(reason):
     global _mytokens
     assert(_mytokens <= 1)
@@ -149,8 +171,8 @@ def wait_all():
             bb += b
             if not b: break
         if len(bb) != _toplevel-1:
-            raise Exception('on exit: expected %d tokens; found only %d' 
+            raise Exception('on exit: expected %d tokens; found only %r' 
-                            % (_toplevel-1, len(b)))
+                            % (_toplevel-1, len(bb)))
         os.write(_fds[1], bb)
 
 

redo-ifchange.py

@@ -1,61 +1,61 @@
 #!/usr/bin/python
 import sys, os, errno, stat
 import vars, state, builder, jwack
-from helpers import debug, debug2, err, mkdirp, unlink
+from helpers import debug, debug2, err, unlink
 
 
-def dirty_deps(t, depth):
+def dirty_deps(f, depth, max_changed):
-    try:
+    if vars.DEBUG >= 1: debug('%s?%s\n' % (depth, f.name))
-        st = os.stat(t)
-        realtime = st.st_mtime
-    except OSError:
-        st = None
-        realtime = 0
 
-    debug('%s?%s\n' % (depth, t))
+    if f.changed_runid == None:
-    if state.isbuilt(t):
+        debug('%s-- DIRTY (never built)\n' % depth)
+        return True
+    if f.changed_runid > max_changed:
         debug('%s-- DIRTY (built)\n' % depth)
-        return True  # has already been built during this session
+        return True  # has been built more recently than parent
-    if state.ismarked(t):
+    if f.is_checked():
-        debug('%s-- CLEAN (marked)\n' % depth)
+        if vars.DEBUG >= 1: debug('%s-- CLEAN (checked)\n' % depth)
         return False  # has already been checked during this session
 
-    stamptime = state.stamped(t)
+    if not f.stamp:
-    if stamptime == None:
         debug('%s-- DIRTY (no stamp)\n' % depth)
         return True
 
-    if stamptime != realtime and not (st and stat.S_ISDIR(st.st_mode)):
+    if f.stamp != f.read_stamp():
         debug('%s-- DIRTY (mtime)\n' % depth)
         return True
     
-    for mode,name in state.deps(t):
+    for mode,f2 in f.deps():
         if mode == 'c':
-            if os.path.exists(name):
+            if os.path.exists(os.path.join(vars.BASE, f2.name)):
                 debug('%s-- DIRTY (created)\n' % depth)
                 return True
         elif mode == 'm':
-            if dirty_deps(os.path.join(vars.BASE, name), depth + '  '):
+            if dirty_deps(f2, depth = depth + '  ',
+                          max_changed = f.changed_runid):
                 debug('%s-- DIRTY (sub)\n' % depth)
-                state.unstamp(t)  # optimization for future callers
                 return True
-    state.mark(t)
+    f.set_checked()
+    f.save()
     return False
 
 
 def should_build(t):
-    return not state.isbuilt(t) and dirty_deps(t, depth = '')
+    f = state.File(name=t)
+    return dirty_deps(f, depth = '', max_changed = vars.RUNID)
 
 
 rv = 202
 try:
     me = os.path.join(vars.STARTDIR, 
                       os.path.join(vars.PWD, vars.TARGET))
+    f = state.File(name=me)
     debug2('TARGET: %r %r %r\n' % (vars.STARTDIR, vars.PWD, vars.TARGET))
     try:
         targets = sys.argv[1:]
         for t in targets:
-            state.add_dep(me, 'm', t)
+            f.add_dep('m', t)
+        f.save()
         rv = builder.main(targets, should_build)
     finally:
         jwack.force_return_tokens()

redo-ifcreate.py

@@ -1,15 +1,16 @@
 #!/usr/bin/python
 import sys, os
 import vars, state
-from helpers import err, mkdirp
+from helpers import err
 
 
 try:
+    me = state.File(name=vars.TARGET)
     for t in sys.argv[1:]:
         if os.path.exists(t):
             err('redo-ifcreate: error: %r already exists\n' % t)
             sys.exit(1)
         else:
-            state.add_dep(vars.TARGET, 'c', t)
+            me.add_dep('c', t)
 except KeyboardInterrupt:
     sys.exit(200)

state.py

@@ -1,29 +1,114 @@
-import sys, os, errno, glob
+import sys, os, errno, glob, stat, fcntl, sqlite3
 import vars
-from helpers import unlink, err, debug2, debug3, mkdirp, close_on_exec
+from helpers import unlink, err, debug2, debug3, close_on_exec
+import helpers
+
+SCHEMA_VER=1
+TIMEOUT=60
+
+def _connect(dbfile):
+    _db = sqlite3.connect(dbfile, timeout=TIMEOUT)
+    _db.execute("pragma synchronous = off")
+    _db.execute("pragma journal_mode = PERSIST")
+    return _db
+
+
+_db = None
+_lockfile = None
+def db():
+    global _db, _lockfile
+    if _db:
+        return _db
+        
+    dbdir = '%s/.redo' % vars.BASE
+    dbfile = '%s/db.sqlite3' % dbdir
+    try:
+        os.mkdir(dbdir)
+    except OSError, e:
+        if e.errno == errno.EEXIST:
+            pass  # if it exists, that's okay
+        else:
+            raise
+
+    _lockfile = os.open(os.path.join(vars.BASE, '.redo/locks'),
+                        os.O_RDWR | os.O_CREAT, 0666)
+    close_on_exec(_lockfile, True)
+    
+    must_create = not os.path.exists(dbfile)
+    if not must_create:
+        _db = _connect(dbfile)
+        try:
+            row = _db.cursor().execute("select version from Schema").fetchone()
+        except sqlite3.OperationalError:
+            row = None
+        ver = row and row[0] or None
+        if ver != SCHEMA_VER:
+            err("state database: discarding v%s (wanted v%s)\n"
+                % (ver, SCHEMA_VER))
+            must_create = True
+            _db = None
+    if must_create:
+        unlink(dbfile)
+        _db = _connect(dbfile)
+        _db.execute("create table Schema "
+                    "    (version int)")
+        _db.execute("create table Runid "
+                    "    (id integer primary key autoincrement)")
+        _db.execute("create table Files "
+                    "    (name not null primary key, "
+                    "     is_generated int, "
+                    "     checked_runid int, "
+                    "     changed_runid int, "
+                    "     stamp, "
+                    "     csum)")
+        _db.execute("create table Deps "
+                    "    (target int, "
+                    "     source int, "
+                    "     mode not null, "
+                    "     primary key (target,source))")
+        _db.execute("insert into Schema (version) values (?)", [SCHEMA_VER])
+        # eat the '0' runid and File id
+        _db.execute("insert into Runid default values")
+        _db.execute("insert into Files (name) values (?)", [''])
+
+    if not vars.RUNID:
+        _db.execute("insert into Runid default values")
+        vars.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0]
+        os.environ['REDO_RUNID'] = str(vars.RUNID)
+    
+    _db.commit()
+    return _db
     
 
 def init():
-    # FIXME: just wiping out all the locks is kind of cheating.  But we
+    db()
-    # only do this from the toplevel redo process, so unless the user
+
-    # deliberately starts more than one redo on the same repository, it's
+
-    # sort of ok.
+_wrote = 0
-    mkdirp('%s/.redo' % vars.BASE)
+def _write(q, l):
-    for f in glob.glob('%s/.redo/lock*' % vars.BASE):
+    if _insane:
-        os.unlink(f)
+        return
-    for f in glob.glob('%s/.redo/mark^*' % vars.BASE):
+    global _wrote
-        os.unlink(f)
+    _wrote += 1
-    for f in glob.glob('%s/.redo/built^*' % vars.BASE):
+    #helpers.log_('W: %r %r\n' % (q,l))
-        os.unlink(f)
+    db().execute(q, l)
+
+
+def commit():
+    if _insane:
+        return
+    global _wrote
+    if _wrote:
+        #helpers.log_("COMMIT (%d)\n" % _wrote)
+        db().commit()
+        _wrote = 0
 
 
 _insane = None
-def is_sane():
+def check_sane():
-    global _insane
+    global _insane, _writable
     if not _insane:
         _insane = not os.path.exists('%s/.redo' % vars.BASE)
-        if _insane:
-            err('.redo directory disappeared; cannot continue.\n')
     return not _insane
 
 
@@ -46,185 +131,154 @@ def relpath(t, base):
     return '/'.join(tparts)
 
 
-def _sname(typ, t):
+class File(object):
-    # FIXME: t.replace(...) is non-reversible and non-unique here!
+    # use this mostly to avoid accidentally assigning to typos
-    tnew = relpath(t, vars.BASE)
+    __slots__ = ['id', 'name', 'is_generated',
-    v = vars.BASE + ('/.redo/%s^%s' % (typ, tnew.replace('/', '^')))
+                 'checked_runid', 'changed_runid',
-    if vars.DEBUG >= 3:
+                 'stamp', 'csum']
-        debug3('sname: (%r) %r -> %r\n' % (os.getcwd(), t, tnew))
-    return v
 
+    def _init_from_cols(self, cols):
+        (self.id, self.name, self.is_generated,
+         self.checked_runid, self.changed_runid,
+         self.stamp, self.csum) = cols
     
-def add_dep(t, mode, dep):
+    def __init__(self, id=None, name=None, cols=None):
-    sn = _sname('dep', t)
+        if cols:
-    reldep = relpath(dep, vars.BASE)
+            return self._init_from_cols(cols)
-    debug2('add-dep: %r < %s %r\n' % (sn, mode, reldep))
+        q = ('select rowid, name, is_generated, checked_runid, changed_runid, '
-    
+             '    stamp, csum '
-    open(sn, 'a').write('%s %s\n' % (mode, reldep))
+             '  from Files ')
-
+        if id != None:
-
+            q += 'where rowid=?'
-def deps(t):
+            l = [id]
-    for line in open(_sname('dep', t)).readlines():
+        elif name != None:
-        assert(line[0] in ('c','m'))
+            name = relpath(name, vars.BASE)
-        assert(line[1] == ' ')
+            q += 'where name=?'
-        assert(line[-1] == '\n')
+            l = [name]
-        mode = line[0]
-        name = line[2:-1]
-        yield mode,name
-
-
-def _stampname(t):
-    return _sname('stamp', t)
-    
-
-def stamp(t):
-    mark(t)
-    stampfile = _stampname(t)
-    newstampfile = _sname('stamp' + str(os.getpid()), t)
-    depfile = _sname('dep', t)
-    if not os.path.exists(vars.BASE + '/.redo'):
-        # .redo might not exist in a 'make clean' target
-        return
-    open(newstampfile, 'w').close()
-    try:
-        mtime = os.stat(t).st_mtime
-    except OSError:
-        mtime = 0
-    os.utime(newstampfile, (mtime, mtime))
-    os.rename(newstampfile, stampfile)
-    open(depfile, 'a').close()
-
-
-def unstamp(t):
-    unlink(_stampname(t))
-    unlink(_sname('dep', t))
-
-
-def unmark_as_generated(t):
-    unstamp(t)
-    unlink(_sname('gen', t))
-
-
-def stamped(t):
-    try:
-        stamptime = os.stat(_stampname(t)).st_mtime
-    except OSError, e:
-        if e.errno == errno.ENOENT:
-            return None
         else:
-            raise
+            raise Exception('name or id must be set')
-    return stamptime
+        d = db()
+        row = d.execute(q, l).fetchone()
+        if not row:
+            if not name:
+                raise Exception('File with id=%r not found and '
+                                'name not given' % id)
+            try:
+                _write('insert into Files (name) values (?)', [name])
+            except sqlite3.IntegrityError:
+                # some parallel redo probably added it at the same time; no
+                # big deal.
+                pass
+            row = d.execute(q, l).fetchone()
+            assert(row)
+        self._init_from_cols(row)
 
+    def save(self):
+        _write('update Files set '
+               '    is_generated=?, checked_runid=?, changed_runid=?, '
+               '    stamp=?, csum=? '
+               '    where rowid=?',
+               [self.is_generated,
+                self.checked_runid, self.changed_runid,
+                self.stamp, self.csum,
+                self.id])
 
-def built(t):
+    def set_checked(self):
-    try:
+        self.checked_runid = vars.RUNID
-        open(_sname('built', t), 'w').close()
+        
-    except IOError, e:
+    def set_changed(self):
-        if e.errno == errno.ENOENT:
+        debug2('BUILT: %r (%r)\n' % (self.name, self.stamp))
-            pass  # may happen if someone deletes our .redo dir
+        self.changed_runid = vars.RUNID
+
+    def set_static(self):
+        self.update_stamp()
+
+    def update_stamp(self):
+        newstamp = self.read_stamp()
+        if newstamp != self.stamp:
+            debug2("STAMP: %s: %r -> %r\n" % (self.name, self.stamp, newstamp))
+            self.stamp = newstamp
+            self.set_changed()
+
+    def is_changed(self):
+        return self.changed_runid and self.changed_runid >= vars.RUNID
+
+    def is_checked(self):
+        return self.checked_runid and self.checked_runid >= vars.RUNID
+
+    def deps(self):
+        q = ('select Deps.mode, Deps.source, '
+             '    name, is_generated, checked_runid, changed_runid, '
+             '    stamp, csum '
+             '  from Files '
+             '    join Deps on Files.rowid = Deps.source '
+             '  where target=?')
+        for row in db().execute(q, [self.id]).fetchall():
+            mode = row[0]
+            cols = row[1:]
+            assert(mode in ('c', 'm'))
+            yield mode,File(cols=cols)
+
+    def zap_deps(self):
+        debug2('zap-deps: %r\n' % self.name)
+        _write('delete from Deps where target=?', [self.id])
+
+    def add_dep(self, mode, dep):
+        src = File(name=dep)
+        reldep = relpath(dep, vars.BASE)
+        debug2('add-dep: %r < %s %r\n' % (self.name, mode, reldep))
+        assert(src.name == reldep)
+        _write("insert or replace into Deps "
+               "    (target, mode, source) values (?,?,?)",
+               [self.id, mode, src.id])
+
+    def read_stamp(self):
+        try:
+            st = os.stat(os.path.join(vars.BASE, self.name))
+        except OSError:
+            return '0'  # does not exist
+        if stat.S_ISDIR(st.st_mode):
+            return 'dir'  # the timestamp of a directory is meaningless
         else:
-            raise
+            # a "unique identifier" stamp for a regular file
-
+            return str((st.st_ctime, st.st_mtime, st.st_size, st.st_ino))
-
-_builts = {}
-def isbuilt(t):
-    if _builts.get(t):
-        return True
-    if os.path.exists(_sname('built', t)):
-        _builts[t] = True
-        return True
-
-
-# stamps the given input file, but only considers it to have been "built" if its
-# mtime has changed.  This is useful for static (non-generated) files.
-def stamp_and_maybe_built(t):
-    if stamped(t) != os.stat(t).st_mtime:
-        built(t)
-    stamp(t)
-
-        
-def mark(t):
-    try:
-        open(_sname('mark', t), 'w').close()
-    except IOError, e:
-        if e.errno == errno.ENOENT:
-            pass  # may happen if someone deletes our .redo dir
-        else:
-            raise
-
-
-_marks = {}
-def ismarked(t):
-    if _marks.get(t):
-        return True
-    if os.path.exists(_sname('mark', t)):
-        _marks[t] = True
-        return True
-
-
-def is_generated(t):
-    return os.path.exists(_sname('gen', t))
-
-
-def start(t):
-    unstamp(t)
-    open(_sname('dep', t), 'w').close()
-    open(_sname('gen', t), 'w').close()  # it's definitely a generated file
 
 
+# FIXME: I really want to use fcntl F_SETLK, F_SETLKW, etc here.  But python
+# doesn't do the lockdata structure in a portable way, so we have to use
+# fcntl.lockf() instead.  Usually this is just a wrapper for fcntl, so it's
+# ok, but it doesn't have F_GETLK, so we can't report which pid owns the lock.
+# The makes debugging a bit harder.  When we someday port to C, we can do that.
 class Lock:
-    def __init__(self, t):
+    def __init__(self, fid):
+        assert(_lockfile >= 0)
         self.owned = False
-        self.rfd = self.wfd = None
+        self.fid = fid
-        self.lockname = _sname('lock', t)
 
     def __del__(self):
         if self.owned:
             self.unlock()
 
     def trylock(self):
+        assert(not self.owned)
         try:
-            os.mkfifo(self.lockname, 0600)
+            fcntl.lockf(_lockfile, fcntl.LOCK_EX|fcntl.LOCK_NB, 1, self.fid)
-            self.owned = True
+        except IOError, e:
-            self.rfd = os.open(self.lockname, os.O_RDONLY|os.O_NONBLOCK)
+            if e.errno in (errno.EAGAIN, errno.EACCES):
-            self.wfd = os.open(self.lockname, os.O_WRONLY)
+                pass  # someone else has it locked
-            close_on_exec(self.rfd, True)
-            close_on_exec(self.wfd, True)
-        except OSError, e:
-            if e.errno == errno.EEXIST:
-                pass
             else:
                 raise
+        else:
+            self.owned = True
 
     def waitlock(self):
-        while not self.owned:
+        assert(not self.owned)
-            self.wait()
+        fcntl.lockf(_lockfile, fcntl.LOCK_EX, 1, self.fid)
-            self.trylock()
+        self.owned = True
-        assert(self.owned)
             
     def unlock(self):
         if not self.owned:
             raise Exception("can't unlock %r - we don't own it" 
                             % self.lockname)
-        unlink(self.lockname)
+        fcntl.lockf(_lockfile, fcntl.LOCK_UN, 1, self.fid)
-        # ping any connected readers
-        os.close(self.rfd)
-        os.close(self.wfd)
-        self.rfd = self.wfd = None
         self.owned = False
-
-    def wait(self):
-        if self.owned:
-            raise Exception("can't wait on %r - we own it" % self.lockname)
-        try:
-            # open() will finish only when a writer exists and does close()
-            fd = os.open(self.lockname, os.O_RDONLY)
-            try:
-                os.read(fd, 1)
-            finally:
-                os.close(fd)
-        except OSError, e:
-            if e.errno == errno.ENOENT:
-                pass  # it's not even unlocked or was unlocked earlier
-            else:
-                raise

t/chdirtest.do

@@ -2,15 +2,15 @@ rm -f chdir1
 redo chdir2
 redo chdir3
 
-. ./flush-cache.sh
+./flush-cache.sh
 redo-ifchange chdir3
 
 rm -f chdir1
-. ./flush-cache.sh
+./flush-cache.sh
 redo-ifchange chdir3
 [ -e chdir1 ] || exit 77
 
 rm -f chdir1
-. ./flush-cache.sh
+./flush-cache.sh
 redo-ifchange chdir3
 [ -e chdir1 ] || exit 78

t/deps/basic/test.do

@@ -1,10 +1,10 @@
 rm -f *.out *.log
 
-. ../../flush-cache.sh
+../../flush-cache.sh
 redo-ifchange 1.out 2.out
 [ "$(cat 1.log | wc -l)" = 1 ] || exit 55
 [ "$(cat 2.log | wc -l)" = 1 ] || exit 56
-. ../../flush-cache.sh
+../../flush-cache.sh
 touch 1.in
 redo-ifchange 1.out 2.out
 [ "$(cat 1.log | wc -l)" = 2 ] || exit 57

t/deps/dirtest/test.do

@@ -1,11 +1,11 @@
 rm -f log dir1/log dir1/stinky
 touch t1.do
-. ../../flush-cache.sh
+../../flush-cache.sh
 redo t1
 touch t1.do
-. ../../flush-cache.sh
+../../flush-cache.sh
 redo t1
-. ../../flush-cache.sh
+../../flush-cache.sh
 redo-ifchange t1
 C1="$(wc -l <dir1/log)"
 C2="$(wc -l <log)"

t/deps/doublestatic.do

@@ -3,7 +3,7 @@ rm -f static.log
 redo static1 static2
 
 touch static.in
-. ../flush-cache.sh
+../flush-cache.sh
 redo-ifchange static1 static2
 
 COUNT=$(wc -l <static.log)

t/deps/gentest.do

@@ -1,19 +1,19 @@
 rm -f genfile2 genfile2.do genfile.log
 
 echo echo hello >genfile2.do
-. ../flush-cache.sh
+../flush-cache.sh
 redo genfile1
 
 # this will cause a rebuild:
 #      genfile1 depends on genfile2 depends on genfile2.do
 rm -f genfile2.do
-. ../flush-cache.sh
+../flush-cache.sh
 redo-ifchange genfile1
 
 # but genfile2.do was gone last time, so genfile2 no longer depends on it.
 # thus, it can be considered up-to-date.  Prior versions of redo had a bug
 # where the dependency on genfile2.do was never dropped.
-. ../flush-cache.sh
+../flush-cache.sh
 redo-ifchange genfile1
 
 COUNT=$(wc -l <genfile.log)

t/deps/test1.do

@@ -6,6 +6,7 @@ if [ -e t1a ]; then
 else
 	BEFORE=
 fi
+../flush-cache.sh
 redo-ifchange t1a  # it definitely had to rebuild because t1dep changed
 AFTER="$(cat t1a)"
 if [ "$BEFORE" = "$AFTER" ]; then

t/flush-cache.sh

@@ -1,3 +1,8 @@
+#!/bin/sh
 #echo "Flushing redo cache..." >&2
-find "$REDO_BASE/.redo" -name 'built^*' -o -name 'mark^*' |
+(
-	xargs rm -f >&2
+	echo ".timeout 5000"
+	echo "pragma synchronous = off;"
+	echo "update Files set checked_runid=null, " \
+	     "       changed_runid=changed_runid-1;"
+) | sqlite3  "$REDO_BASE/.redo/db.sqlite3"

t/makedir2.do

@@ -1,7 +1,7 @@
 rm -f makedir.log
 redo makedir
 touch makedir/outfile
-. ./flush-cache.sh
+./flush-cache.sh
 redo-ifchange makedir
 COUNT=$(wc -l <makedir.log)
 [ "$COUNT" = 1 ] || exit 99

vars.py

@@ -18,6 +18,7 @@ XTRACE = os.environ.get('REDO_XTRACE', '') and 1 or 0
 KEEP_GOING = os.environ.get('REDO_KEEP_GOING', '') and 1 or 0
 SHUFFLE = os.environ.get('REDO_SHUFFLE', '') and 1 or 0
 STARTDIR = os.environ['REDO_STARTDIR']
+RUNID = atoi.atoi(os.environ.get('REDO_RUNID')) or None
 BASE = os.environ['REDO_BASE']
 while BASE and BASE.endswith('/'):
     BASE = BASE[:-1]