Half-support for using file checksums instead of stamps.

A new redo-stamp program takes whatever you give it as stdin and uses it to
calculate a checksum for the current target.  If that checksum is the same
as last time, then we consider the target to be unchanged, and we set
checked_runid and stamp, but leave changed_runid alone.  That will make
future callers of redo-ifchange see this target as unmodified.

However, this is only "half" support because by the time we run the .do
script that calls redo-stamp, it's too late; the caller is a dependant of
the stamped program, which is already being rebuilt, even if redo-stamp
turns out to say that this target is unchanged.

The other half is coming up.

builder.py

@@ -1,6 +1,6 @@
 import sys, os, errno, stat
 import vars, jwack, state
-from helpers import log, log_, debug2, err, warn, unlink, close_on_exec
+from helpers import log, log_, debug, debug2, err, warn, unlink, close_on_exec
 
 
 def _possible_do_files(t):
@@ -187,9 +187,17 @@ class BuildJob:
                 unlink(self.tmpname1)
                 unlink(t)
             sf = self.sf
+            sf.refresh()
             sf.is_generated = True
-            sf.update_stamp()
+            sf.is_override = False
-            sf.set_changed()
+            if sf.is_checked() or sf.is_changed():
+                # it got checked during the run; someone ran redo-stamp.
+                # update_stamp would call set_changed(); we don't want that
+                sf.stamp = sf.read_stamp()
+            else:
+                sf.csum = None
+                sf.update_stamp()
+                sf.set_changed()
             sf.save()
         else:
             unlink(self.tmpname1)

redo-stamp

@@ -0,0 +1 @@
+redo-stamp.py

redo-stamp.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python
+import sys, os
+import vars, state
+from helpers import err, debug2
+
+if len(sys.argv) > 1:
+    err('%s: no arguments expected.\n' % sys.argv[0])
+    sys.exit(1)
+
+# hashlib is only available in python 2.5 or higher, but the 'sha' module
+# produces a DeprecationWarning in python 2.6 or higher.  We want to support
+# python 2.4 and above without any stupid warnings, so let's try using hashlib
+# first, and downgrade if it fails.
+try:
+    import hashlib
+except ImportError:
+    import sha
+    sh = sha.sha()
+else:
+    sh = hashlib.sha1()
+
+while 1:
+    b = os.read(0, 4096)
+    sh.update(b)
+    if not b: break
+
+f = state.File(name=vars.TARGET)
+csum = sh.hexdigest()
+changed = (csum != f.csum)
+debug2('%s: old = %s\n' % (f.name, f.csum))
+debug2('%s: sum = %s (%s)\n' % (f.name, csum,
+                                changed and 'changed' or 'unchanged'))
+f.is_generated = True
+f.is_override = False
+f.failed_runid = None
+if changed:
+    f.set_changed()  # update_stamp might not do this if the mtime is identical
+    f.csum = csum
+else:
+    # unchanged
+    f.set_checked()
+f.save()
+state.commit()

state.py

@@ -6,6 +6,7 @@ import helpers
 SCHEMA_VER=1
 TIMEOUT=60
 
+ALWAYS='//ALWAYS'   # an invalid filename that is always marked as dirty
 STAMP_DIR='dir'     # the stamp of a directory; mtime is unhelpful
 STAMP_MISSING='0'   # the stamp of a nonexistent file
 
@@ -75,7 +76,7 @@ def db():
         _db.execute("insert into Schema (version) values (?)", [SCHEMA_VER])
         # eat the '0' runid and File id
         _db.execute("insert into Runid default values")
-        _db.execute("insert into Files (name) values (?)", [''])
+        _db.execute("insert into Files (name) values (?)", [ALWAYS])
 
     if not vars.RUNID:
         _db.execute("insert into Runid default values")
@@ -143,14 +144,7 @@ class File(object):
                  'checked_runid', 'changed_runid', 'failed_runid',
                  'stamp', 'csum']
 
-    def _init_from_cols(self, cols):
+    def _init_from_idname(self, id, name):
-        (self.id, self.name, self.is_generated, self.is_override,
-         self.checked_runid, self.changed_runid, self.failed_runid,
-         self.stamp, self.csum) = cols
-    
-    def __init__(self, id=None, name=None, cols=None):
-        if cols:
-            return self._init_from_cols(cols)
         q = ('select rowid, name, is_generated, is_override, '
              '    checked_runid, changed_runid, failed_runid, '
              '    stamp, csum '
@@ -178,7 +172,21 @@ class File(object):
                 pass
             row = d.execute(q, l).fetchone()
             assert(row)
-        self._init_from_cols(row)
+        return self._init_from_cols(row)
+
+    def _init_from_cols(self, cols):
+        (self.id, self.name, self.is_generated, self.is_override,
+         self.checked_runid, self.changed_runid, self.failed_runid,
+         self.stamp, self.csum) = cols
+    
+    def __init__(self, id=None, name=None, cols=None):
+        if cols:
+            return self._init_from_cols(cols)
+        else:
+            return self._init_from_idname(id, name)
+
+    def refresh(self):
+        self._init_from_idname(self.id, None)
 
     def save(self):
         _write('update Files set '

t/clean.do

@@ -1,5 +1,5 @@
-redo example/clean curse/clean deps/clean "space dir/clean"
+redo example/clean curse/clean deps/clean "space dir/clean" stamp/clean
-rm -f c c.c c.c.c c.c.c.b c.c.c.b.b d mode1 makedir.log chdir1 \
+rm -f c c.c c.c.c c.c.c.b c.c.c.b.b d mode1 makedir.log chdir1 deltest2 \
 	hello [by]ellow *.o *~ .*~ CC LD passfail silence silence.do \
 	touch1 touch1.do
 rm -rf makedir

t/stamp/.gitignore

@@ -0,0 +1,4 @@
+/*.log
+/usestamp
+/stampy
+/inp

t/stamp/clean.do

@@ -0,0 +1 @@
+rm -f *.log usestamp stampy inp *~ .*~

t/stamp/stamptest.do

@@ -0,0 +1,29 @@
+rm -f stampy usestamp stampy.log usestamp.log
+echo one >inp
+
+../flush-cache.sh
+redo stampy
+[ "$(wc -l <stampy.log)" -eq 1 ] || exit 11
+
+redo-ifchange usestamp
+[ "$(wc -l <stampy.log)" -eq 1 ] || exit 21
+[ "$(wc -l <usestamp.log)" -eq 1 ] || exit 12
+
+../flush-cache.sh
+redo stampy
+[ "$(wc -l <stampy.log)" -eq 2 ] || exit 31
+[ "$(wc -l <usestamp.log)" -eq 1 ] || exit 32
+
+redo-ifchange usestamp
+[ "$(wc -l <stampy.log)" -eq 2 ] || exit 41
+[ "$(wc -l <usestamp.log)" -eq 1 ] || exit 42
+
+../flush-cache.sh
+echo two >inp
+redo stampy
+[ "$(wc -l <stampy.log)" -eq 3 ] || exit 51
+[ "$(wc -l <usestamp.log)" -eq 1 ] || exit 52
+
+redo-ifchange usestamp
+[ "$(wc -l <stampy.log)" -eq 3 ] || exit 61
+[ "$(wc -l <usestamp.log)" -eq 2 ] || exit 62

t/stamp/stampy.do

@@ -0,0 +1,3 @@
+echo $$ >>stampy.log
+cat inp
+redo-stamp <inp

t/stamp/test.do

@@ -0,0 +1 @@
+redo stamptest

t/stamp/usestamp.do

@@ -0,0 +1,3 @@
+redo-ifchange stampy
+echo $$ >>usestamp.log
+cat stampy

t/test.do

@@ -2,4 +2,4 @@ redo-ifchange all
 ./hello >&2
 redo deltest deltest2 test.args test2.args passfailtest chdirtest \
 	curse/test deps/test "space dir/test" modetest makedir2 \
-	silencetest touchtest
+	silencetest touchtest stamp/test