forks/apenwarr-redo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move into the 21st century by fixing some pylint warnings.

Browse source
This commit is contained in:
Avery Pennarun 2018-12-02 23:15:37 -05:00
commit e1327540fb
22 changed files with 797 additions and 388 deletions
Download patch file Download diff file Expand all files Collapse all files

103
builder.py
View file

@ -1,8 +1,8 @@
import sys, os, errno, random, stat, signal, time
import sys, os, errno, stat, signal, time
import vars, jwack, state, paths
from helpers import unlink, close_on_exec, join
from helpers import unlink, close_on_exec
import logs
from logs import debug, debug2, err, warn, meta, check_tty
from logs import debug2, err, warn, meta, check_tty
def _nice(t):
@ -81,15 +81,15 @@ def start_stdin_log_reader(status, details, pretty, color,
argv.append('--color' if color >= 2 else '--no-color')
argv.append('-')
os.execvp(argv[0], argv)
except Exception, e:
except Exception, e: # pylint: disable=broad-except
sys.stderr.write('redo-log: exec: %s\n' % e)
finally:
os._exit(99)
def await_log_reader():
if not vars.LOG: return
global log_reader_pid
if not vars.LOG:
return
if log_reader_pid > 0:
# never actually close fd#1 or fd#2; insanity awaits.
# replace it with something else instead.
@ -108,14 +108,14 @@ class ImmediateReturn(Exception):
self.rv = rv
class BuildJob:
class BuildJob(object):
def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
self.t = t # original target name, not relative to vars.BASE
self.sf = sf
tmpbase = t
while not os.path.isdir(os.path.dirname(tmpbase) or '.'):
ofs = tmpbase.rfind('/')
assert(ofs >= 0)
assert ofs >= 0
tmpbase = tmpbase[:ofs] + '__' + tmpbase[ofs+1:]
self.tmpname1 = '%s.redo1.tmp' % tmpbase
self.tmpname2 = '%s.redo2.tmp' % tmpbase
@ -125,7 +125,7 @@ class BuildJob:
self.before_t = _try_stat(self.t)
def start(self):
assert(self.lock.owned)
assert self.lock.owned
try:
try:
is_target, dirty = self.shouldbuildfunc(self.t)
@ -140,29 +140,29 @@ class BuildJob:
except ImmediateReturn, e:
return self._after2(e.rv)
if vars.NO_OOB or dirty == True:
if vars.NO_OOB or dirty == True: # pylint: disable=singleton-comparison
self._start_do()
else:
self._start_unlocked(dirty)
def _start_do(self):
assert(self.lock.owned)
assert self.lock.owned
t = self.t
sf = self.sf
newstamp = sf.read_stamp()
if (sf.is_generated and
newstamp != state.STAMP_MISSING and
(sf.is_override or state.detect_override(sf.stamp, newstamp))):
state.warn_override(_nice(t))
if not sf.is_override:
warn('%s - old: %r\n' % (_nice(t), sf.stamp))
warn('%s - new: %r\n' % (_nice(t), newstamp))
sf.set_override()
sf.set_checked()
sf.save()
return self._after2(0)
newstamp != state.STAMP_MISSING and
(sf.is_override or state.detect_override(sf.stamp, newstamp))):
state.warn_override(_nice(t))
if not sf.is_override:
warn('%s - old: %r\n' % (_nice(t), sf.stamp))
warn('%s - new: %r\n' % (_nice(t), newstamp))
sf.set_override()
sf.set_checked()
sf.save()
return self._after2(0)
if (os.path.exists(t) and not os.path.isdir(t + '/.')
and not sf.is_generated):
and not sf.is_generated):
# an existing source file that was not generated by us.
# This step is mentioned by djb in his notes.
# For example, a rule called default.c.do could be used to try
@ -173,7 +173,7 @@ class BuildJob:
sf.save()
return self._after2(0)
sf.zap_deps1()
(dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf)
(dodir, dofile, _, basename, ext) = paths.find_do_file(sf)
if not dofile:
if os.path.exists(t):
sf.set_static()
@ -186,6 +186,7 @@ class BuildJob:
unlink(self.tmpname2)
ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
close_on_exec(ffd, True)
# pylint: disable=attribute-defined-outside-init
self.f = os.fdopen(ffd, 'w+')
# this will run in the dofile's directory, so use only basenames here
arg1 = basename + ext # target name (including extension)
@ -196,15 +197,21 @@ class BuildJob:
arg2,
# temp output file name
state.relpath(os.path.abspath(self.tmpname2), dodir),
]
if vars.VERBOSE: argv[1] += 'v'
if vars.XTRACE: argv[1] += 'x'
]
if vars.VERBOSE:
argv[1] += 'v'
if vars.XTRACE:
argv[1] += 'x'
firstline = open(os.path.join(dodir, dofile)).readline().strip()
if firstline.startswith('#!/'):
argv[0:2] = firstline[2:].split(' ')
# make sure to create the logfile *before* writing the meta() about it.
# that way redo-log won't trace into an obsolete logfile.
if vars.LOG: open(state.logname(self.sf.id), 'w')
if vars.LOG:
open(state.logname(self.sf.id), 'w')
# FIXME: put these variables somewhere else, instead of on-the-fly
# extending this class!
# pylint: disable=attribute-defined-outside-init
self.dodir = dodir
self.basename = basename
self.ext = ext
@ -219,7 +226,7 @@ class BuildJob:
def _start_unlocked(self, dirty):
# out-of-band redo of some sub-objects. This happens when we're not
# quite sure if t needs to be built or not (because some children
# look dirty, but might turn out to be clean thanks to checksums).
# look dirty, but might turn out to be clean thanks to checksums).
# We have to call redo-unlocked to figure it all out.
#
# Note: redo-unlocked will handle all the updating of sf, so we
@ -236,9 +243,10 @@ class BuildJob:
state.commit()
def run():
os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE
# python ignores SIGPIPE
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
os.execvp(argv[0], argv)
assert(0)
assert 0
# returns only if there's an exception
def after(t, rv):
return self._after2(rv)
@ -250,7 +258,7 @@ class BuildJob:
# redo-ifchange, and it might have done it from a different directory
# than we started it in. So os.getcwd() might be != REDO_PWD right
# now.
assert(state.is_flushed())
assert state.is_flushed()
dn = self.dodir
newp = os.path.realpath(dn)
os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
@ -288,7 +296,7 @@ class BuildJob:
# FIXME: it would be nice to log the exit code to logf.
# But that would have to happen in the parent process, which doesn't
# have logf open.
assert(0)
assert 0
# returns only if there's an exception
def _after(self, t, rv):
@ -305,9 +313,9 @@ class BuildJob:
after_t = _try_stat(t)
st1 = os.fstat(f.fileno())
st2 = _try_stat(self.tmpname2)
if (after_t and
(not before_t or before_t.st_mtime != after_t.st_mtime) and
not stat.S_ISDIR(after_t.st_mode)):
if (after_t and
(not before_t or before_t.st_mtime != after_t.st_mtime) and
not stat.S_ISDIR(after_t.st_mode)):
err('%s modified %s directly!\n' % (self.argv[2], t))
err('...you should update $3 (a temp file) or stdout, not $1.\n')
rv = 206
@ -315,7 +323,7 @@ class BuildJob:
err('%s wrote to stdout *and* created $3.\n' % self.argv[2])
err('...you should write status messages to stderr, not stdout.\n')
rv = 207
if rv==0:
if rv == 0:
# FIXME: race condition here between updating stamp/is_generated
# and actually renaming the files into place. There needs to
# be some kind of two-stage commit, I guess.
@ -372,7 +380,7 @@ class BuildJob:
def _after2(self, rv):
try:
self.donefunc(self.t, rv)
assert(self.lock.owned)
assert self.lock.owned
finally:
self.lock.unlock()
@ -388,17 +396,18 @@ def main(targets, shouldbuildfunc):
def done(t, rv):
if rv:
retcode[0] = 1
if vars.TARGET and not vars.UNLOCKED:
me = os.path.join(vars.STARTDIR,
me = os.path.join(vars.STARTDIR,
os.path.join(vars.PWD, vars.TARGET))
myfile = state.File(name=me)
selflock = state.Lock(state.LOG_LOCK_MAGIC + myfile.id)
else:
selflock = myfile = me = None
def cheat():
if not selflock: return 0
if not selflock:
return 0
selflock.trylock()
if not selflock.owned:
# redo-log already owns it: let's cheat.
@ -419,7 +428,7 @@ def main(targets, shouldbuildfunc):
err('cannot build the empty target ("").\n')
retcode[0] = 204
break
assert(state.is_flushed())
assert state.is_flushed()
if t in seen:
continue
seen[t] = 1
@ -440,7 +449,7 @@ def main(targets, shouldbuildfunc):
lock.trylock()
if not lock.owned:
meta('locked', state.target_relpath(t))
locked.append((f.id,t,f.name))
locked.append((f.id, t, f.name))
else:
# We had to create f before we had a lock, because we need f.id
# to make the lock. But someone may have updated the state
@ -450,7 +459,7 @@ def main(targets, shouldbuildfunc):
f.refresh()
BuildJob(t, f, lock, shouldbuildfunc, done).start()
state.commit()
assert(state.is_flushed())
assert state.is_flushed()
lock = None
del lock
@ -465,7 +474,7 @@ def main(targets, shouldbuildfunc):
while locked or jwack.running():
state.commit()
jwack.wait_all()
assert jwack._mytokens == 0
assert jwack._mytokens == 0 # pylint: disable=protected-access
jwack.ensure_token_or_cheat('self', cheat)
# at this point, we don't have any children holding any tokens, so
# it's okay to block below.
@ -476,7 +485,7 @@ def main(targets, shouldbuildfunc):
err('.redo directory disappeared; cannot continue.\n')
retcode[0] = 205
break
fid,t,fname = locked.pop(0)
fid, t, _ = locked.pop(0)
lock = state.Lock(fid)
backoff = 0.01
lock.trylock()
@ -505,7 +514,7 @@ def main(targets, shouldbuildfunc):
lock.unlock()
jwack.ensure_token_or_cheat(t, cheat)
lock.trylock()
assert(lock.owned)
assert lock.owned
meta('unlocked', state.target_relpath(t))
if state.File(name=t).is_failed():
err('%s: failed in another thread\n' % _nice(t))