forks/apenwarr-redo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Some speedups for doing redo-ifchange on a large number of static files.

Browse source 
Fix some wastage revealed by the (almost useless, sigh) python profiler.
This commit is contained in:
Avery Pennarun 2010-12-09 05:53:30 -08:00
commit 6e6e453908
2 changed files with 16 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

17
builder.py
View file

@ -4,6 +4,7 @@ from helpers import log, log_, debug2, err, unlink, close_on_exec
def _possible_do_files(t): def _possible_do_files(t):
t = os.path.join(vars.BASE, t)
yield "%s.do" % t, t, '' yield "%s.do" % t, t, ''
dirname,filename = os.path.split(t) dirname,filename = os.path.split(t)
l = filename.split('.') l = filename.split('.')
@ -16,14 +17,14 @@ def _possible_do_files(t):
os.path.join(dirname, basename), ext) os.path.join(dirname, basename), ext)
def _find_do_file(t): def _find_do_file(f):
for dofile,basename,ext in _possible_do_files(t): for dofile,basename,ext in _possible_do_files(f.name):
debug2('%s: %s ?\n' % (t, dofile)) debug2('%s: %s ?\n' % (f.name, dofile))
if os.path.exists(dofile): if os.path.exists(dofile):
state.File(name=t).add_dep('m', dofile) f.add_dep('m', dofile)
return dofile,basename,ext return dofile,basename,ext
else: else:
state.File(name=t).add_dep('c', dofile) f.add_dep('c', dofile)
return None,None,None return None,None,None
@ -73,7 +74,7 @@ class BuildJob:
f.save() f.save()
return self._after2(0) return self._after2(0)
f.zap_deps() f.zap_deps()
(dofile, basename, ext) = _find_do_file(t) (dofile, basename, ext) = _find_do_file(f)
if not dofile: if not dofile:
if os.path.exists(t): if os.path.exists(t):
f.is_generated = False f.is_generated = False
@ -132,6 +133,7 @@ class BuildJob:
try: try:
state.check_sane() state.check_sane()
rv = self._after1(t, rv) rv = self._after1(t, rv)
state.commit()
finally: finally:
self._after2(rv) self._after2(rv)
@ -186,7 +188,6 @@ class BuildJob:
try: try:
self.donefunc(self.t, rv) self.donefunc(self.t, rv)
assert(self.lock.owned) assert(self.lock.owned)
state.commit()
finally: finally:
self.lock.unlock() self.lock.unlock()
@ -209,6 +210,7 @@ def main(targets, shouldbuildfunc):
# In the first cycle, we just build as much as we can without worrying # In the first cycle, we just build as much as we can without worrying
# about any lock contention. If someone else has it locked, we move on. # about any lock contention. If someone else has it locked, we move on.
for t in targets: for t in targets:
if not jwack.has_token():
state.commit() state.commit()
jwack.get_token(t) jwack.get_token(t)
if retcode[0] and not vars.KEEP_GOING: if retcode[0] and not vars.KEEP_GOING:
@ -231,6 +233,7 @@ def main(targets, shouldbuildfunc):
# use select.select() to wait on more than one at a time. But it should # use select.select() to wait on more than one at a time. But it should
# be rare enough that it doesn't matter, and the logic is easier this way. # be rare enough that it doesn't matter, and the logic is easier this way.
while locked or jwack.running(): while locked or jwack.running():
state.commit()
jwack.wait_all() jwack.wait_all()
# at this point, we don't have any children holding any tokens, so # at this point, we don't have any children holding any tokens, so
# it's okay to block below. # it's okay to block below.

5
jwack.py
View file

@ -105,6 +105,11 @@ def wait(want_token):
pd.donefunc(pd.name, pd.rv) pd.donefunc(pd.name, pd.rv)
def has_token():
if _mytokens >= 1:
return True
def get_token(reason): def get_token(reason):
global _mytokens global _mytokens
assert(_mytokens <= 1) assert(_mytokens <= 1)