builder,jwack: slight cleanup to token passing.

In rare cases, one process could end up holding onto more than one token.

builder.py

@@ -125,6 +125,8 @@ def main(targets, buildfunc):
             # t is a directory, but it has a default target
             targets[i] = '%s/all' % t
 
+    # In the first cycle, we just build as much as we can without worrying
+    # about any lock contention.  If someone else has it locked, we move on.
     for t in targets:
         jwack.get_token(t)
         if retcode[0] and not vars.KEEP_GOING:
@@ -139,6 +141,10 @@ def main(targets, buildfunc):
             jwack.start_job(t, lock,
                             lambda: buildfunc(t), lambda t,rv: done(t,rv))
 
+    # Now we've built all the "easy" ones.  Go back and just wait on the
+    # remaining ones one by one.  This is technically non-optimal; we could
+    # use select.select() to wait on more than one at a time.  But it should
+    # be rare enough that it doesn't matter, and the logic is easier this way.
     while locked or jwack.running():
         jwack.wait_all()
         if retcode[0] and not vars.KEEP_GOING:
@@ -146,9 +152,7 @@ def main(targets, buildfunc):
         if locked:
             t = locked.pop(0)
             lock = state.Lock(t)
-            while not lock.owned:
-                lock.wait()
-                lock.trylock()
+            lock.waitlock()
             assert(lock.owned)
             if vars.DEBUG_LOCKS:
                 log('%s (...unlocked!)\n' % _nice(t))

jwack.py

@@ -107,21 +107,29 @@ def wait(want_token):
 
 def get_token(reason):
     global _mytokens
+    assert(_mytokens <= 1)
     setup(1)
     while 1:
         if _mytokens >= 1:
+            _debug("_mytokens is %d\n" % _mytokens)
+            assert(_mytokens == 1)
             _debug('(%r) used my own token...\n' % reason)
-            return
+            break
+        assert(_mytokens < 1)
         _debug('(%r) waiting for tokens...\n' % reason)
         wait(want_token=1)
+        if _mytokens >= 1:
+            break
+        assert(_mytokens < 1)
         if _fds:
             b = _try_read(_fds[0], 1)
             if b == None:
                 raise Exception('unexpected EOF on token read')
             if b:
+                _mytokens += 1
+                _debug('(%r) got a token (%r).\n' % (reason, b))
                 break
-    _mytokens += 1
-    _debug('(%r) got a token (%r).\n' % (reason, b))
+    assert(_mytokens <= 1)
 
 
 def running():
@@ -177,8 +185,10 @@ class Job:
 def start_job(reason, lock, jobfunc, donefunc):
     global _mytokens
     assert(lock.owned)
+    assert(_mytokens <= 1)
     get_token(reason)
     assert(_mytokens >= 1)
+    assert(_mytokens == 1)
     _mytokens -= 1
     r,w = os.pipe()
     pid = os.fork()

state.py

@@ -162,6 +162,12 @@ class Lock:
             else:
                 raise
 
+    def waitlock(self):
+        while not self.owned:
+            self.wait()
+            self.trylock()
+        assert(self.owned)
+            
     def unlock(self):
         if not self.owned:
             raise Exception("can't unlock %r - we don't own it"