From 52236a3aed11ccced369ce051623c7cb57a7761c Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Mon, 29 Oct 2018 07:21:12 +0000 Subject: [PATCH] builder.py: don't spin while fighting for a file lock. If we end up in builder phase 2, where we might need to build stuff that was previously locked by someone else, we will need to obtain a job token *and* the lock at the same time in order to continue. To prevent deadlocks, we don't wait synchronously for one lock while holding the other. If several instances are fighting over the same lock and there are insufficient job tokens for everyone, timing could cause them to fight for a long time. This seems to happen a lot in freebsd for some reason. To be a good citizen, sleep for a while after each loop iteration. This should ensure that eventually, most of the fighting instances will be asleep by the time the next one tries to grab the token, thus breaking the deadlock. --- builder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/builder.py b/builder.py index c14eb9c..6240435 100644 --- a/builder.py +++ b/builder.py @@ -1,4 +1,4 @@ -import sys, os, errno, stat, signal +import sys, os, errno, random, stat, signal, time import vars, jwack, state from helpers import unlink, close_on_exec, join from log import log, log_, debug, debug2, err, warn @@ -379,8 +379,13 @@ def main(targets, shouldbuildfunc): break fid,t = locked.pop(0) lock = state.Lock(fid) + backoff = 0.01 lock.trylock() while not lock.owned: + # Don't spin with 100% CPU while we fight for the lock. + import random + time.sleep(random.random() * min(backoff, 1.0)) + backoff *= 2 if vars.DEBUG_LOCKS: warn('%s (WAITING)\n' % _nice(t)) # this sequence looks a little silly, but the idea is to