diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..7914602 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,318 @@ +[MASTER] + +# Pickle collected data for later comparisons. +persistent=no + +# Use multiple processes to speed up Pylint. +jobs=1 + + +[MESSAGES CONTROL] + +# We probably want to fix these eventually, but in the meantime, these +# ones are relatively harmless. +disable=redefined-builtin,multiple-imports,missing-docstring,wrong-import-position,locally-disabled,invalid-name,unused-argument,fixme,global-statement,redefined-variable-type,using-constant-test,unused-variable,file-ignored,simplifiable-if-statement + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=20 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format=LF + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{0,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{0,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][-a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=100 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,_exit + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=100 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=100 + +# Maximum number of return / yield for function / method body +max-returns=100 + +# Maximum number of branch for function / method body +max-branches=100 + +# Maximum number of statements in function / method body +max-statements=500 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=100 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=100 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/builder.py b/builder.py index 3f5d422..40ed870 100644 --- a/builder.py +++ b/builder.py @@ -1,8 +1,8 @@ -import sys, os, errno, random, stat, signal, time +import sys, os, errno, stat, signal, time import vars, jwack, state, paths -from helpers import unlink, close_on_exec, join +from helpers import unlink, close_on_exec import logs -from logs import debug, debug2, err, warn, meta, check_tty +from logs import debug2, err, warn, meta, check_tty def _nice(t): @@ -81,15 +81,15 @@ def start_stdin_log_reader(status, details, pretty, color, argv.append('--color' if color >= 2 else '--no-color') argv.append('-') os.execvp(argv[0], argv) - except Exception, e: + except Exception, e: # pylint: disable=broad-except sys.stderr.write('redo-log: exec: %s\n' % e) finally: os._exit(99) def await_log_reader(): - if not vars.LOG: return - global log_reader_pid + if not vars.LOG: + return if log_reader_pid > 0: # never actually close fd#1 or fd#2; insanity awaits. # replace it with something else instead. @@ -108,14 +108,14 @@ class ImmediateReturn(Exception): self.rv = rv -class BuildJob: +class BuildJob(object): def __init__(self, t, sf, lock, shouldbuildfunc, donefunc): self.t = t # original target name, not relative to vars.BASE self.sf = sf tmpbase = t while not os.path.isdir(os.path.dirname(tmpbase) or '.'): ofs = tmpbase.rfind('/') - assert(ofs >= 0) + assert ofs >= 0 tmpbase = tmpbase[:ofs] + '__' + tmpbase[ofs+1:] self.tmpname1 = '%s.redo1.tmp' % tmpbase self.tmpname2 = '%s.redo2.tmp' % tmpbase @@ -125,7 +125,7 @@ class BuildJob: self.before_t = _try_stat(self.t) def start(self): - assert(self.lock.owned) + assert self.lock.owned try: try: is_target, dirty = self.shouldbuildfunc(self.t) @@ -140,29 +140,29 @@ class BuildJob: except ImmediateReturn, e: return self._after2(e.rv) - if vars.NO_OOB or dirty == True: + if vars.NO_OOB or dirty == True: # pylint: disable=singleton-comparison self._start_do() else: self._start_unlocked(dirty) def _start_do(self): - assert(self.lock.owned) + assert self.lock.owned t = self.t sf = self.sf newstamp = sf.read_stamp() if (sf.is_generated and - newstamp != state.STAMP_MISSING and - (sf.is_override or state.detect_override(sf.stamp, newstamp))): - state.warn_override(_nice(t)) - if not sf.is_override: - warn('%s - old: %r\n' % (_nice(t), sf.stamp)) - warn('%s - new: %r\n' % (_nice(t), newstamp)) - sf.set_override() - sf.set_checked() - sf.save() - return self._after2(0) + newstamp != state.STAMP_MISSING and + (sf.is_override or state.detect_override(sf.stamp, newstamp))): + state.warn_override(_nice(t)) + if not sf.is_override: + warn('%s - old: %r\n' % (_nice(t), sf.stamp)) + warn('%s - new: %r\n' % (_nice(t), newstamp)) + sf.set_override() + sf.set_checked() + sf.save() + return self._after2(0) if (os.path.exists(t) and not os.path.isdir(t + '/.') - and not sf.is_generated): + and not sf.is_generated): # an existing source file that was not generated by us. # This step is mentioned by djb in his notes. # For example, a rule called default.c.do could be used to try @@ -173,7 +173,7 @@ class BuildJob: sf.save() return self._after2(0) sf.zap_deps1() - (dodir, dofile, basedir, basename, ext) = paths.find_do_file(sf) + (dodir, dofile, _, basename, ext) = paths.find_do_file(sf) if not dofile: if os.path.exists(t): sf.set_static() @@ -186,6 +186,7 @@ class BuildJob: unlink(self.tmpname2) ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666) close_on_exec(ffd, True) + # pylint: disable=attribute-defined-outside-init self.f = os.fdopen(ffd, 'w+') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) @@ -196,15 +197,21 @@ class BuildJob: arg2, # temp output file name state.relpath(os.path.abspath(self.tmpname2), dodir), - ] - if vars.VERBOSE: argv[1] += 'v' - if vars.XTRACE: argv[1] += 'x' + ] + if vars.VERBOSE: + argv[1] += 'v' + if vars.XTRACE: + argv[1] += 'x' firstline = open(os.path.join(dodir, dofile)).readline().strip() if firstline.startswith('#!/'): argv[0:2] = firstline[2:].split(' ') # make sure to create the logfile *before* writing the meta() about it. # that way redo-log won't trace into an obsolete logfile. - if vars.LOG: open(state.logname(self.sf.id), 'w') + if vars.LOG: + open(state.logname(self.sf.id), 'w') + # FIXME: put these variables somewhere else, instead of on-the-fly + # extending this class! + # pylint: disable=attribute-defined-outside-init self.dodir = dodir self.basename = basename self.ext = ext @@ -219,7 +226,7 @@ class BuildJob: def _start_unlocked(self, dirty): # out-of-band redo of some sub-objects. This happens when we're not # quite sure if t needs to be built or not (because some children - # look dirty, but might turn out to be clean thanks to checksums). + # look dirty, but might turn out to be clean thanks to checksums). # We have to call redo-unlocked to figure it all out. # # Note: redo-unlocked will handle all the updating of sf, so we @@ -236,9 +243,10 @@ class BuildJob: state.commit() def run(): os.environ['REDO_DEPTH'] = vars.DEPTH + ' ' - signal.signal(signal.SIGPIPE, signal.SIG_DFL) # python ignores SIGPIPE + # python ignores SIGPIPE + signal.signal(signal.SIGPIPE, signal.SIG_DFL) os.execvp(argv[0], argv) - assert(0) + assert 0 # returns only if there's an exception def after(t, rv): return self._after2(rv) @@ -250,7 +258,7 @@ class BuildJob: # redo-ifchange, and it might have done it from a different directory # than we started it in. So os.getcwd() might be != REDO_PWD right # now. - assert(state.is_flushed()) + assert state.is_flushed() dn = self.dodir newp = os.path.realpath(dn) os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR) @@ -288,7 +296,7 @@ class BuildJob: # FIXME: it would be nice to log the exit code to logf. # But that would have to happen in the parent process, which doesn't # have logf open. - assert(0) + assert 0 # returns only if there's an exception def _after(self, t, rv): @@ -305,9 +313,9 @@ class BuildJob: after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) - if (after_t and - (not before_t or before_t.st_mtime != after_t.st_mtime) and - not stat.S_ISDIR(after_t.st_mode)): + if (after_t and + (not before_t or before_t.st_mtime != after_t.st_mtime) and + not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 @@ -315,7 +323,7 @@ class BuildJob: err('%s wrote to stdout *and* created $3.\n' % self.argv[2]) err('...you should write status messages to stderr, not stdout.\n') rv = 207 - if rv==0: + if rv == 0: # FIXME: race condition here between updating stamp/is_generated # and actually renaming the files into place. There needs to # be some kind of two-stage commit, I guess. @@ -372,7 +380,7 @@ class BuildJob: def _after2(self, rv): try: self.donefunc(self.t, rv) - assert(self.lock.owned) + assert self.lock.owned finally: self.lock.unlock() @@ -388,17 +396,18 @@ def main(targets, shouldbuildfunc): def done(t, rv): if rv: retcode[0] = 1 - + if vars.TARGET and not vars.UNLOCKED: - me = os.path.join(vars.STARTDIR, + me = os.path.join(vars.STARTDIR, os.path.join(vars.PWD, vars.TARGET)) myfile = state.File(name=me) selflock = state.Lock(state.LOG_LOCK_MAGIC + myfile.id) else: selflock = myfile = me = None - + def cheat(): - if not selflock: return 0 + if not selflock: + return 0 selflock.trylock() if not selflock.owned: # redo-log already owns it: let's cheat. @@ -419,7 +428,7 @@ def main(targets, shouldbuildfunc): err('cannot build the empty target ("").\n') retcode[0] = 204 break - assert(state.is_flushed()) + assert state.is_flushed() if t in seen: continue seen[t] = 1 @@ -440,7 +449,7 @@ def main(targets, shouldbuildfunc): lock.trylock() if not lock.owned: meta('locked', state.target_relpath(t)) - locked.append((f.id,t,f.name)) + locked.append((f.id, t, f.name)) else: # We had to create f before we had a lock, because we need f.id # to make the lock. But someone may have updated the state @@ -450,7 +459,7 @@ def main(targets, shouldbuildfunc): f.refresh() BuildJob(t, f, lock, shouldbuildfunc, done).start() state.commit() - assert(state.is_flushed()) + assert state.is_flushed() lock = None del lock @@ -465,7 +474,7 @@ def main(targets, shouldbuildfunc): while locked or jwack.running(): state.commit() jwack.wait_all() - assert jwack._mytokens == 0 + assert jwack._mytokens == 0 # pylint: disable=protected-access jwack.ensure_token_or_cheat('self', cheat) # at this point, we don't have any children holding any tokens, so # it's okay to block below. @@ -476,7 +485,7 @@ def main(targets, shouldbuildfunc): err('.redo directory disappeared; cannot continue.\n') retcode[0] = 205 break - fid,t,fname = locked.pop(0) + fid, t, _ = locked.pop(0) lock = state.Lock(fid) backoff = 0.01 lock.trylock() @@ -505,7 +514,7 @@ def main(targets, shouldbuildfunc): lock.unlock() jwack.ensure_token_or_cheat(t, cheat) lock.trylock() - assert(lock.owned) + assert lock.owned meta('unlocked', state.target_relpath(t)) if state.File(name=t).is_failed(): err('%s: failed in another thread\n' % _nice(t)) diff --git a/deps.py b/deps.py index 5229996..4d75c6f 100644 --- a/deps.py +++ b/deps.py @@ -1,5 +1,5 @@ -import sys, os -import vars, state, builder +import os +import vars, state from logs import debug CLEAN = 0 @@ -17,16 +17,18 @@ def isdirty(f, depth, max_changed, already_checked = list(already_checked) + [f.id] if vars.DEBUG >= 1: - debug('%s?%s %r,%r\n' % (depth, f.nicename(), f.is_generated, f.is_override)) + debug('%s?%s %r,%r\n' + % (depth, f.nicename(), f.is_generated, f.is_override)) if f.failed_runid: debug('%s-- DIRTY (failed last time)\n' % depth) return DIRTY - if f.changed_runid == None: + if f.changed_runid is None: debug('%s-- DIRTY (never built)\n' % depth) return DIRTY if f.changed_runid > max_changed: - debug('%s-- DIRTY (built %d > %d; %d)\n' % (depth, f.changed_runid, max_changed, vars.RUNID)) + debug('%s-- DIRTY (built %d > %d; %d)\n' + % (depth, f.changed_runid, max_changed, vars.RUNID)) return DIRTY # has been built more recently than parent if is_checked(f): if vars.DEBUG >= 1: @@ -60,16 +62,16 @@ def isdirty(f, depth, max_changed, return DIRTY must_build = [] - for mode,f2 in f.deps(): + for mode, f2 in f.deps(): dirty = CLEAN if mode == 'c': if os.path.exists(os.path.join(vars.BASE, f2.name)): debug('%s-- DIRTY (created)\n' % depth) dirty = DIRTY elif mode == 'm': - sub = isdirty(f2, depth = depth + ' ', - max_changed = max(f.changed_runid, - f.checked_runid), + sub = isdirty(f2, depth=depth + ' ', + max_changed=max(f.changed_runid, + f.checked_runid), already_checked=already_checked, is_checked=is_checked, set_checked=set_checked, @@ -78,14 +80,14 @@ def isdirty(f, depth, max_changed, debug('%s-- DIRTY (sub)\n' % depth) dirty = sub else: - assert(mode in ('c','m')) + assert mode in ('c', 'm') if not f.csum: # f is a "normal" target: dirty f2 means f is instantly dirty if dirty == DIRTY: # f2 is definitely dirty, so f definitely needs to # redo. return DIRTY - elif isinstance(dirty,list): + elif isinstance(dirty, list): # our child f2 might be dirty, but it's not sure yet. It's # given us a list of targets we have to redo in order to # be sure. @@ -99,7 +101,7 @@ def isdirty(f, depth, max_changed, # redo. However, after that, f might turn out to be # unchanged. return [f] - elif isinstance(dirty,list): + elif isinstance(dirty, list): # our child f2 might be dirty, but it's not sure yet. It's # given us a list of targets we have to redo in order to # be sure. diff --git a/helpers.py b/helpers.py index c192673..64fdf44 100644 --- a/helpers.py +++ b/helpers.py @@ -1,5 +1,4 @@ import os, errno, fcntl -from atoi import atoi def join(between, l): @@ -25,5 +24,3 @@ def close_on_exec(fd, yes): if yes: fl |= fcntl.FD_CLOEXEC fcntl.fcntl(fd, fcntl.F_SETFD, fl) - - diff --git a/jwack.py b/jwack.py index 1aaf432..9d5cc6a 100644 --- a/jwack.py +++ b/jwack.py @@ -74,7 +74,8 @@ # simpler :) # import sys, os, errno, select, fcntl, signal -from helpers import atoi, close_on_exec +from atoi import atoi +from helpers import close_on_exec import state, vars _toplevel = 0 @@ -87,7 +88,7 @@ _waitfds = {} def _debug(s): if 0: - sys.stderr.write('jwack#%d: %s' % (os.getpid(),s)) + sys.stderr.write('jwack#%d: %s' % (os.getpid(), s)) def _create_tokens(n): @@ -132,7 +133,6 @@ def _release_except_mine(): def release_mine(): - global _mytokens assert _mytokens >= 1 _debug('%d,%d -> release_mine()\n' % (_mytokens, _cheats)) _release(1) @@ -146,9 +146,9 @@ def _timeout(sig, frame): # This makes it easier to differentiate different kinds of pipes when using # strace. def _make_pipe(startfd): - (a,b) = os.pipe() + (a, b) = os.pipe() fds = (fcntl.fcntl(a, fcntl.F_DUPFD, startfd), - fcntl.fcntl(b, fcntl.F_DUPFD, startfd+1)) + fcntl.fcntl(b, fcntl.F_DUPFD, startfd + 1)) os.close(a) os.close(b) return fds @@ -162,7 +162,7 @@ def _try_read(fd, n): # socket: http://cr.yp.to/unix/nonblock.html # We can't just make the socket non-blocking, because we want to be # compatible with GNU Make, and they can't handle it. - r,w,x = select.select([fd], [], [], 0) + r, w, x = select.select([fd], [], [], 0) if not r: return None # try again # ok, the socket is readable - but some other process might get there @@ -199,19 +199,19 @@ def setup(maxjobs): assert maxjobs > 0 assert not _tokenfds _debug('setup(%d)\n' % maxjobs) - + flags = ' ' + os.getenv('MAKEFLAGS', '') + ' ' FIND1 = ' --jobserver-auth=' # renamed in GNU make 4.2 FIND2 = ' --jobserver-fds=' # fallback syntax FIND = FIND1 ofs = flags.find(FIND1) if ofs < 0: - FIND = FIND2 - ofs = flags.find(FIND2) + FIND = FIND2 + ofs = flags.find(FIND2) if ofs >= 0: s = flags[ofs+len(FIND):] - (arg,junk) = s.split(' ', 1) - (a,b) = arg.split(',', 1) + (arg, junk) = s.split(' ', 1) + (a, b) = arg.split(',', 1) a = atoi(a) b = atoi(b) if a <= 0 or b <= 0: @@ -221,20 +221,21 @@ def setup(maxjobs): fcntl.fcntl(b, fcntl.F_GETFL) except IOError, e: if e.errno == errno.EBADF: - raise ValueError('broken --jobserver-auth from make; prefix your Makefile rule with a "+"') + raise ValueError('broken --jobserver-auth from make; ' + + 'prefix your Makefile rule with a "+"') else: raise - _tokenfds = (a,b) - + _tokenfds = (a, b) + cheats = os.getenv('REDO_CHEATFDS', '') if cheats: - (a,b) = cheats.split(',', 1) + (a, b) = cheats.split(',', 1) a = atoi(a) b = atoi(b) if a <= 0 or b <= 0: raise ValueError('invalid REDO_CHEATFDS: %r' % cheats) - _cheatfds = (a,b) - + _cheatfds = (a, b) + if not _tokenfds: # need to start a new server _toplevel = maxjobs @@ -256,7 +257,7 @@ def _wait(want_token, max_delay): rfds.append(_tokenfds[0]) assert rfds assert state.is_flushed() - r,w,x = select.select(rfds, [], [], max_delay) + r, w, x = select.select(rfds, [], [], max_delay) _debug('_tokenfds=%r; wfds=%r; readable: %r\n' % (_tokenfds, _waitfds, r)) for fd in r: if fd == _tokenfds[0]: @@ -270,7 +271,7 @@ def _wait(want_token, max_delay): # now need to recreate it. b = _try_read(_cheatfds[0], 1) _debug('GOT cheatfd\n') - if b == None: + if b is None: _create_tokens(1) if has_token(): _release_except_mine() @@ -383,7 +384,7 @@ def force_return_tokens(): n = len(_waitfds) _debug('%d,%d -> %d jobs left in force_return_tokens\n' % (_mytokens, _cheats, n)) - for k in _waitfds.keys(): + for k in list(_waitfds): del _waitfds[k] _create_tokens(n) if has_token(): @@ -405,26 +406,25 @@ def _pre_job(r, w, pfn): pfn() -class Job: +class Job(object): def __init__(self, name, pid, donefunc): self.name = name self.pid = pid self.rv = None self.donefunc = donefunc - + def __repr__(self): return 'Job(%s,%d)' % (self.name, self.pid) - + def start_job(reason, jobfunc, donefunc): assert state.is_flushed() - global _mytokens assert _mytokens <= 1 assert _mytokens == 1 # Subprocesses always start with 1 token, so we have to destroy ours # in order for the universe to stay in balance. _destroy_tokens(1) - r,w = _make_pipe(50) + r, w = _make_pipe(50) pid = os.fork() if pid == 0: # child @@ -433,8 +433,8 @@ def start_job(reason, jobfunc, donefunc): try: try: rv = jobfunc() or 0 - _debug('jobfunc completed (%r, %r)\n' % (jobfunc,rv)) - except Exception: + _debug('jobfunc completed (%r, %r)\n' % (jobfunc, rv)) + except Exception: # pylint: disable=broad-except import traceback traceback.print_exc() finally: diff --git a/logs.py b/logs.py index 2ef1448..a14d81e 100644 --- a/logs.py +++ b/logs.py @@ -1,12 +1,15 @@ import os, re, sys, time import vars +RED = GREEN = YELLOW = BOLD = PLAIN = None + def check_tty(file, color): global RED, GREEN, YELLOW, BOLD, PLAIN color_ok = file.isatty() and (os.environ.get('TERM') or 'dumb') != 'dumb' if (color and color_ok) or color >= 2: # ...use ANSI formatting codes. + # pylint: disable=bad-whitespace RED = "\x1b[31m" GREEN = "\x1b[32m" YELLOW = "\x1b[33m" @@ -25,7 +28,7 @@ class RawLog(object): self.file = file def write(self, s): - assert('\n' not in s) + assert '\n' not in s sys.stdout.flush() sys.stderr.flush() self.file.write(s + '\n') @@ -39,17 +42,18 @@ class PrettyLog(object): def __init__(self, file): self.topdir = os.getcwd() self.file = file - + def _pretty(self, pid, color, s): if vars.DEBUG_PIDS: redo = '%-6d redo ' % pid else: redo = 'redo ' - self.file.write(''.join([color, redo, vars.DEPTH, - BOLD if color else '', s, PLAIN, '\n'])) + self.file.write( + ''.join([color, redo, vars.DEPTH, + BOLD if color else '', s, PLAIN, '\n'])) def write(self, s): - assert('\n' not in s) + assert '\n' not in s sys.stdout.flush() sys.stderr.flush() g = REDO_RE.match(s) @@ -58,7 +62,7 @@ class PrettyLog(object): self.file.write(s[:-len(all)]) words = g.group(1).split(':') text = g.group(2) - kind, pid, when = words[0:3] + kind, pid, _ = words[0:3] pid = int(pid) if kind == 'unchanged': self._pretty(pid, '', '%s (unchanged)' % text) @@ -85,10 +89,10 @@ class PrettyLog(object): self._pretty(pid, GREEN, '%s (...unlocked!)' % text) elif kind == 'error': self.file.write(''.join([RED, 'redo: ', - BOLD, text, PLAIN, '\n'])) + BOLD, text, PLAIN, '\n'])) elif kind == 'warning': self.file.write(''.join([YELLOW, 'redo: ', - BOLD, text, PLAIN, '\n'])) + BOLD, text, PLAIN, '\n'])) elif kind == 'debug': self._pretty(pid, '', text) else: @@ -118,10 +122,11 @@ def write(s): def meta(kind, s, pid=None): - assert(':' not in kind) - assert('@' not in kind) - assert('\n' not in s) - if pid == None: pid = os.getpid() + assert ':' not in kind + assert '@' not in kind + assert '\n' not in s + if pid is None: + pid = os.getpid() write('@@REDO:%s:%d:%.4f@@ %s' % (kind, pid, time.time(), s)) diff --git a/options.py b/options.py index 30c5426..1bada98 100644 --- a/options.py +++ b/options.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# # Copyright 2011 Avery Pennarun and options.py contributors. # All rights reserved. # diff --git a/paths.py b/paths.py index c85580b..87b54a9 100644 --- a/paths.py +++ b/paths.py @@ -1,19 +1,20 @@ import os import vars -from logs import err, debug2 +from logs import debug2 def _default_do_files(filename): l = filename.split('.') - for i in range(1,len(l)+1): + for i in range(1, len(l)+1): basename = '.'.join(l[:i]) ext = '.'.join(l[i:]) - if ext: ext = '.' + ext + if ext: + ext = '.' + ext yield ("default%s.do" % ext), basename, ext def possible_do_files(t): - dirname,filename = os.path.split(t) + dirname, filename = os.path.split(t) yield (os.path.join(vars.BASE, dirname), "%s.do" % filename, '', filename, '') @@ -24,25 +25,25 @@ def possible_do_files(t): # into theirs as a subdir. When they do, my rules should still be used # for building my project in *all* cases. t = os.path.normpath(os.path.join(vars.BASE, t)) - dirname,filename = os.path.split(t) + dirname, filename = os.path.split(t) dirbits = dirname.split('/') # since t is an absolute path, dirbits[0] is always '', so we don't # need to count all the way down to i=0. for i in range(len(dirbits), 0, -1): basedir = '/'.join(dirbits[:i]) subdir = '/'.join(dirbits[i:]) - for dofile,basename,ext in _default_do_files(filename): + for dofile, basename, ext in _default_do_files(filename): yield (basedir, dofile, subdir, os.path.join(subdir, basename), ext) def find_do_file(f): - for dodir,dofile,basedir,basename,ext in possible_do_files(f.name): + for dodir, dofile, basedir, basename, ext in possible_do_files(f.name): dopath = os.path.join(dodir, dofile) debug2('%s: %s:%s ?\n' % (f.name, dodir, dofile)) if os.path.exists(dopath): f.add_dep('m', dopath) - return dodir,dofile,basedir,basename,ext + return dodir, dofile, basedir, basename, ext else: f.add_dep('c', dopath) - return None,None,None,None,None + return None, None, None, None, None diff --git a/redo-always.py b/redo-always.py index 2f2a60b..ea0ade5 100755 --- a/redo-always.py +++ b/redo-always.py @@ -1,18 +1,22 @@ #!/usr/bin/env python2 import sys, os import vars, state -from logs import err -try: - me = os.path.join(vars.STARTDIR, - os.path.join(vars.PWD, vars.TARGET)) - f = state.File(name=me) - f.add_dep('m', state.ALWAYS) - always = state.File(name=state.ALWAYS) - always.stamp = state.STAMP_MISSING - always.set_changed() - always.save() - state.commit() -except KeyboardInterrupt: - sys.exit(200) +def main(): + try: + me = os.path.join(vars.STARTDIR, + os.path.join(vars.PWD, vars.TARGET)) + f = state.File(name=me) + f.add_dep('m', state.ALWAYS) + always = state.File(name=state.ALWAYS) + always.stamp = state.STAMP_MISSING + always.set_changed() + always.save() + state.commit() + except KeyboardInterrupt: + sys.exit(200) + + +if __name__ == '__main__': + main() diff --git a/redo-ifchange.py b/redo-ifchange.py index c9db239..57875c4 100755 --- a/redo-ifchange.py +++ b/redo-ifchange.py @@ -5,56 +5,62 @@ import vars_init vars_init.init(sys.argv[1:]) import vars, state, builder, jwack, deps -from helpers import unlink -from logs import debug, debug2, err +from logs import debug2, err def should_build(t): f = state.File(name=t) if f.is_failed(): raise builder.ImmediateReturn(32) - dirty = deps.isdirty(f, depth = '', max_changed = vars.RUNID, + dirty = deps.isdirty(f, depth='', max_changed=vars.RUNID, already_checked=[]) - return f.is_generated, dirty==[f] and deps.DIRTY or dirty + return f.is_generated, dirty == [f] and deps.DIRTY or dirty -rv = 202 -try: - if vars_init.is_toplevel and vars.LOG: - builder.close_stdin() - builder.start_stdin_log_reader(status=True, details=True, - pretty=True, color=True, debug_locks=False, debug_pids=False) - if vars.TARGET and not vars.UNLOCKED: - me = os.path.join(vars.STARTDIR, - os.path.join(vars.PWD, vars.TARGET)) - f = state.File(name=me) - debug2('TARGET: %r %r %r\n' % (vars.STARTDIR, vars.PWD, vars.TARGET)) - else: - f = me = None - debug2('redo-ifchange: not adding depends.\n') - jwack.setup(1) +def main(): + rv = 202 try: - targets = sys.argv[1:] - if f: - for t in targets: - f.add_dep('m', t) - f.save() - state.commit() - rv = builder.main(targets, should_build) - finally: + if vars_init.is_toplevel and vars.LOG: + builder.close_stdin() + builder.start_stdin_log_reader( + status=True, details=True, + pretty=True, color=True, debug_locks=False, debug_pids=False) + if vars.TARGET and not vars.UNLOCKED: + me = os.path.join(vars.STARTDIR, + os.path.join(vars.PWD, vars.TARGET)) + f = state.File(name=me) + debug2('TARGET: %r %r %r\n' + % (vars.STARTDIR, vars.PWD, vars.TARGET)) + else: + f = me = None + debug2('redo-ifchange: not adding depends.\n') + jwack.setup(1) try: - state.rollback() + targets = sys.argv[1:] + if f: + for t in targets: + f.add_dep('m', t) + f.save() + state.commit() + rv = builder.main(targets, should_build) finally: try: - jwack.force_return_tokens() - except Exception, e: - traceback.print_exc(100, sys.stderr) - err('unexpected error: %r\n' % e) - rv = 1 -except KeyboardInterrupt: + state.rollback() + finally: + try: + jwack.force_return_tokens() + except Exception, e: # pylint: disable=broad-except + traceback.print_exc(100, sys.stderr) + err('unexpected error: %r\n' % e) + rv = 1 + except KeyboardInterrupt: + if vars_init.is_toplevel: + builder.await_log_reader() + sys.exit(200) + state.commit() if vars_init.is_toplevel: builder.await_log_reader() - sys.exit(200) -state.commit() -if vars_init.is_toplevel: - builder.await_log_reader() -sys.exit(rv) + sys.exit(rv) + + +if __name__ == '__main__': + main() diff --git a/redo-ifcreate.py b/redo-ifcreate.py index 16f6ddd..52013ac 100755 --- a/redo-ifcreate.py +++ b/redo-ifcreate.py @@ -4,19 +4,24 @@ import vars, state from logs import err -try: - me = os.path.join(vars.STARTDIR, - os.path.join(vars.PWD, vars.TARGET)) - f = state.File(name=me) - for t in sys.argv[1:]: - if not t: - err('cannot build the empty target ("").\n') - sys.exit(204) - if os.path.exists(t): - err('redo-ifcreate: error: %r already exists\n' % t) - sys.exit(1) - else: - f.add_dep('c', t) - state.commit() -except KeyboardInterrupt: - sys.exit(200) +def main(): + try: + me = os.path.join(vars.STARTDIR, + os.path.join(vars.PWD, vars.TARGET)) + f = state.File(name=me) + for t in sys.argv[1:]: + if not t: + err('cannot build the empty target ("").\n') + sys.exit(204) + if os.path.exists(t): + err('redo-ifcreate: error: %r already exists\n' % t) + sys.exit(1) + else: + f.add_dep('c', t) + state.commit() + except KeyboardInterrupt: + sys.exit(200) + + +if __name__ == '__main__': + main() diff --git a/redo-log.py b/redo-log.py index 6eb5f3f..827ae73 100755 --- a/redo-log.py +++ b/redo-log.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 -import errno, fcntl, os, re, struct, sys, termios, time +import errno, fcntl, os, re, struct, sys, time +import termios from atoi import atoi import options @@ -28,7 +29,6 @@ import vars, logs, state topdir = os.getcwd() already = set() -queue = [] depth = [] total_lines = 0 status = None @@ -53,11 +53,10 @@ def _atoi(s): def _tty_width(): s = struct.pack("HHHH", 0, 0, 0, 0) try: - import fcntl, termios s = fcntl.ioctl(sys.stderr.fileno(), termios.TIOCGWINSZ, s) except (IOError, ImportError): return _atoi(os.environ.get('WIDTH')) or 70 - (ysize,xsize,ypix,xpix) = struct.unpack('HHHH', s) + (ysize, xsize, ypix, xpix) = struct.unpack('HHHH', s) return xsize or 70 @@ -69,8 +68,8 @@ def _fix_depth(): vars.DEPTH = len(depth) * ' ' -def _rel(topdir, mydir, path): - return os.path.relpath(os.path.join(topdir, mydir, path), topdir) +def _rel(top, mydir, path): + return os.path.relpath(os.path.join(top, mydir, path), topdir) def catlog(t): @@ -92,7 +91,7 @@ def catlog(t): sf = state.File(name=t, allow_add=False) except KeyError: sys.stderr.write('redo-log: [%s] %r: not known to redo.\n' - % (os.getcwd(), t,)) + % (os.getcwd(), t,)) sys.exit(24) fid = sf.id del sf @@ -148,9 +147,10 @@ def catlog(t): tail = n + ' ' + tail status = head + tail if len(status) > width: - sys.stderr.write('\nOVERSIZE STATUS (%d):\n%r\n' % - (len(status), status)) - assert(len(status) <= width) + sys.stderr.write( + '\nOVERSIZE STATUS (%d):\n%r\n' + % (len(status), status)) + assert len(status) <= width sys.stdout.flush() sys.stderr.write('\r%-*.*s\r' % (width, width, status)) time.sleep(min(delay, 1.0)) @@ -184,9 +184,11 @@ def catlog(t): elif fixname not in already: logs.meta('do', relname, pid=pid) if opt.recursive: - if loglock: loglock.unlock() + if loglock: + loglock.unlock() catlog(os.path.join(mydir, text)) - if loglock: loglock.waitlock(shared=True) + if loglock: + loglock.waitlock(shared=True) already.add(fixname) elif kind in ('do', 'waiting', 'locked', 'unlocked'): if opt.debug_locks: @@ -196,9 +198,11 @@ def catlog(t): logs.meta('do', relname, pid=pid) if opt.recursive: assert text - if loglock: loglock.unlock() + if loglock: + loglock.unlock() catlog(os.path.join(mydir, text)) - if loglock: loglock.waitlock(shared=True) + if loglock: + loglock.waitlock(shared=True) already.add(fixname) elif kind == 'done': rv, name = text.split(' ', 1) @@ -218,40 +222,49 @@ def catlog(t): # partial line never got terminated print line_head if t != '-': - assert(depth[-1] == t) + assert depth[-1] == t depth.pop(-1) _fix_depth() -try: - if not targets: - sys.stderr.write('redo-log: give at least one target; maybe "all"?\n') - sys.exit(1) - if opt.status < 2 and not os.isatty(2): - opt.status = False - logs.setup(file=sys.stdout, pretty=opt.pretty, color=opt.color) - if opt.debug_locks: - vars.DEBUG_LOCKS = 1 - if opt.debug_pids: - vars.DEBUG_PIDS = 1 - if opt.ack_fd: - # Write back to owner, to let them know we started up okay and - # will be able to see their error output, so it's okay to close - # their old stderr. - ack_fd = int(opt.ack_fd) - assert(ack_fd > 2) - if os.write(ack_fd, 'REDO-OK\n') != 8: - raise Exception('write to ack_fd returned wrong length') - os.close(ack_fd) - queue += targets - while queue: - t = queue.pop(0) - if t != '-': - logs.meta('do', _rel(topdir, '.', t), pid=0) - catlog(t) -except KeyboardInterrupt: - sys.exit(200) -except IOError, e: - if e.errno == errno.EPIPE: - pass - else: - raise + +def main(): + queue = [] + try: + if not targets: + sys.stderr.write( + 'redo-log: give at least one target; ' + + 'maybe "all"?\n') + sys.exit(1) + if opt.status < 2 and not os.isatty(2): + opt.status = False + logs.setup(file=sys.stdout, pretty=opt.pretty, color=opt.color) + if opt.debug_locks: + vars.DEBUG_LOCKS = 1 + if opt.debug_pids: + vars.DEBUG_PIDS = 1 + if opt.ack_fd: + # Write back to owner, to let them know we started up okay and + # will be able to see their error output, so it's okay to close + # their old stderr. + ack_fd = int(opt.ack_fd) + assert ack_fd > 2 + if os.write(ack_fd, 'REDO-OK\n') != 8: + raise Exception('write to ack_fd returned wrong length') + os.close(ack_fd) + queue += targets + while queue: + t = queue.pop(0) + if t != '-': + logs.meta('do', _rel(topdir, '.', t), pid=0) + catlog(t) + except KeyboardInterrupt: + sys.exit(200) + except IOError, e: + if e.errno == errno.EPIPE: + pass + else: + raise + + +if __name__ == '__main__': + main() diff --git a/redo-ood.py b/redo-ood.py index 81c9c0b..a999fcf 100755 --- a/redo-ood.py +++ b/redo-ood.py @@ -27,14 +27,19 @@ def log_override(name): pass -cwd = os.getcwd() -for f in state.files(): - if f.is_target(): - if deps.isdirty(f, - depth='', - max_changed=vars.RUNID, - already_checked=[], - is_checked=is_checked, - set_checked=set_checked, - log_override=log_override): - print state.relpath(os.path.join(vars.BASE, f.name), cwd) +def main(): + cwd = os.getcwd() + for f in state.files(): + if f.is_target(): + if deps.isdirty(f, + depth='', + max_changed=vars.RUNID, + already_checked=[], + is_checked=is_checked, + set_checked=set_checked, + log_override=log_override): + print state.relpath(os.path.join(vars.BASE, f.name), cwd) + + +if __name__ == '__main__': + main() diff --git a/redo-sources.py b/redo-sources.py index 790b9c3..b550877 100755 --- a/redo-sources.py +++ b/redo-sources.py @@ -7,11 +7,17 @@ vars_init.init([]) import state, vars from logs import err -if len(sys.argv[1:]) != 0: - err('%s: no arguments expected.\n' % sys.argv[0]) - sys.exit(1) -cwd = os.getcwd() -for f in state.files(): - if f.is_source(): - print state.relpath(os.path.join(vars.BASE, f.name), cwd) +def main(): + if len(sys.argv[1:]) != 0: + err('%s: no arguments expected.\n' % sys.argv[0]) + sys.exit(1) + + cwd = os.getcwd() + for f in state.files(): + if f.is_source(): + print state.relpath(os.path.join(vars.BASE, f.name), cwd) + + +if __name__ == '__main__': + main() diff --git a/redo-stamp.py b/redo-stamp.py index a8f2dab..02b6a0b 100755 --- a/redo-stamp.py +++ b/redo-stamp.py @@ -3,51 +3,58 @@ import sys, os import vars, state from logs import err, debug2 -if len(sys.argv) > 1: - err('%s: no arguments expected.\n' % sys.argv[0]) - sys.exit(1) -if os.isatty(0): - err('%s: you must provide the data to stamp on stdin\n' % sys.argv[0]) - sys.exit(1) +def main(): + if len(sys.argv) > 1: + err('%s: no arguments expected.\n' % sys.argv[0]) + sys.exit(1) -# hashlib is only available in python 2.5 or higher, but the 'sha' module -# produces a DeprecationWarning in python 2.6 or higher. We want to support -# python 2.4 and above without any stupid warnings, so let's try using hashlib -# first, and downgrade if it fails. -try: - import hashlib -except ImportError: - import sha - sh = sha.sha() -else: - sh = hashlib.sha1() + if os.isatty(0): + err('%s: you must provide the data to stamp on stdin\n' % sys.argv[0]) + sys.exit(1) -while 1: - b = os.read(0, 4096) - sh.update(b) - if not b: break + # hashlib is only available in python 2.5 or higher, but the 'sha' + # module produces a DeprecationWarning in python 2.6 or higher. We want + # to support python 2.4 and above without any stupid warnings, so let's + # try using hashlib first, and downgrade if it fails. + try: + import hashlib + except ImportError: + import sha + sh = sha.sha() + else: + sh = hashlib.sha1() -csum = sh.hexdigest() + while 1: + b = os.read(0, 4096) + sh.update(b) + if not b: + break -if not vars.TARGET: - sys.exit(0) + csum = sh.hexdigest() -me = os.path.join(vars.STARTDIR, - os.path.join(vars.PWD, vars.TARGET)) -f = state.File(name=me) -changed = (csum != f.csum) -debug2('%s: old = %s\n' % (f.name, f.csum)) -debug2('%s: sum = %s (%s)\n' % (f.name, csum, - changed and 'changed' or 'unchanged')) -f.is_generated = True -f.is_override = False -f.failed_runid = None -if changed: - f.set_changed() # update_stamp might not do this if the mtime is identical - f.csum = csum -else: - # unchanged - f.set_checked() -f.save() -state.commit() + if not vars.TARGET: + sys.exit(0) + + me = os.path.join(vars.STARTDIR, + os.path.join(vars.PWD, vars.TARGET)) + f = state.File(name=me) + changed = (csum != f.csum) + debug2('%s: old = %s\n' % (f.name, f.csum)) + debug2('%s: sum = %s (%s)\n' % (f.name, csum, + changed and 'changed' or 'unchanged')) + f.is_generated = True + f.is_override = False + f.failed_runid = None + if changed: + f.set_changed() # update_stamp might skip this if mtime is identical + f.csum = csum + else: + # unchanged + f.set_checked() + f.save() + state.commit() + + +if __name__ == '__main__': + main() diff --git a/redo-targets.py b/redo-targets.py index f0db6e9..fd20737 100755 --- a/redo-targets.py +++ b/redo-targets.py @@ -7,11 +7,17 @@ vars_init.init([]) import state, vars from logs import err -if len(sys.argv[1:]) != 0: - err('%s: no arguments expected.\n' % sys.argv[0]) - sys.exit(1) -cwd = os.getcwd() -for f in state.files(): - if f.is_target(): - print state.relpath(os.path.join(vars.BASE, f.name), cwd) +def main(): + if len(sys.argv[1:]) != 0: + err('%s: no arguments expected.\n' % sys.argv[0]) + sys.exit(1) + + cwd = os.getcwd() + for f in state.files(): + if f.is_target(): + print state.relpath(os.path.join(vars.BASE, f.name), cwd) + + +if __name__ == '__main__': + main() diff --git a/redo-unlocked.py b/redo-unlocked.py index 126f78a..0418c60 100755 --- a/redo-unlocked.py +++ b/redo-unlocked.py @@ -3,33 +3,39 @@ import sys, os import state from logs import err -if len(sys.argv[1:]) < 2: - err('%s: at least 2 arguments expected.\n' % sys.argv[0]) - sys.exit(1) -target = sys.argv[1] -deps = sys.argv[2:] +def main(): + if len(sys.argv[1:]) < 2: + err('%s: at least 2 arguments expected.\n' % sys.argv[0]) + sys.exit(1) -for d in deps: - assert(d != target) + target = sys.argv[1] + deps = sys.argv[2:] -me = state.File(name=target) + for d in deps: + assert d != target -# Build the known dependencies of our primary target. This *does* require -# grabbing locks. -os.environ['REDO_NO_OOB'] = '1' -argv = ['redo-ifchange'] + deps -rv = os.spawnvp(os.P_WAIT, argv[0], argv) -if rv: - sys.exit(rv) + me = state.File(name=target) -# We know our caller already owns the lock on target, so we don't have to -# acquire another one; tell redo-ifchange about that. Also, REDO_NO_OOB -# persists from up above, because we don't want to do OOB now either. -# (Actually it's most important for the primary target, since it's the one -# who initiated the OOB in the first place.) -os.environ['REDO_UNLOCKED'] = '1' -argv = ['redo-ifchange', target] -rv = os.spawnvp(os.P_WAIT, argv[0], argv) -if rv: - sys.exit(rv) + # Build the known dependencies of our primary target. This *does* require + # grabbing locks. + os.environ['REDO_NO_OOB'] = '1' + argv = ['redo-ifchange'] + deps + rv = os.spawnvp(os.P_WAIT, argv[0], argv) + if rv: + sys.exit(rv) + + # We know our caller already owns the lock on target, so we don't have to + # acquire another one; tell redo-ifchange about that. Also, REDO_NO_OOB + # persists from up above, because we don't want to do OOB now either. + # (Actually it's most important for the primary target, since it's the one + # who initiated the OOB in the first place.) + os.environ['REDO_UNLOCKED'] = '1' + argv = ['redo-ifchange', target] + rv = os.spawnvp(os.P_WAIT, argv[0], argv) + if rv: + sys.exit(rv) + + +if __name__ == '__main__': + main() diff --git a/redo-whichdo.py b/redo-whichdo.py index 57a06eb..7cace82 100755 --- a/redo-whichdo.py +++ b/redo-whichdo.py @@ -7,22 +7,29 @@ vars_init.init_no_state() import paths from logs import err -if len(sys.argv[1:]) != 1: - err('%s: exactly one argument expected.\n' % sys.argv[0]) - sys.exit(1) -want = sys.argv[1] -if not want: - err('cannot build the empty target ("").\n') - sys.exit(204) +def main(): + if len(sys.argv[1:]) != 1: + err('%s: exactly one argument expected.\n' % sys.argv[0]) + sys.exit(1) -abswant = os.path.abspath(want) -for dodir,dofile,basedir,basename,ext in paths.possible_do_files(abswant): - dopath = os.path.join('/', dodir, dofile) - relpath = os.path.relpath(dopath, '.') - exists = os.path.exists(dopath) - assert('\n' not in relpath) - print relpath - if exists: - sys.exit(0) -sys.exit(1) # no appropriate dofile found + want = sys.argv[1] + if not want: + err('cannot build the empty target ("").\n') + sys.exit(204) + + abswant = os.path.abspath(want) + pdf = paths.possible_do_files(abswant) + for dodir, dofile, basedir, basename, ext in pdf: + dopath = os.path.join('/', dodir, dofile) + relpath = os.path.relpath(dopath, '.') + exists = os.path.exists(dopath) + assert '\n' not in relpath + print relpath + if exists: + sys.exit(0) + sys.exit(1) # no appropriate dofile found + + +if __name__ == '__main__': + main() diff --git a/redo.py b/redo.py index 4303df2..61c3564 100755 --- a/redo.py +++ b/redo.py @@ -1,13 +1,13 @@ #!/usr/bin/env python2 # # Copyright 2010-2018 Avery Pennarun and contributors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,7 @@ # import sys, os, traceback import options -from helpers import atoi +from atoi import atoi optspec = """ redo [targets...] @@ -83,7 +83,8 @@ try: if vars_init.is_toplevel and (vars.LOG or j > 1): builder.close_stdin() if vars_init.is_toplevel and vars.LOG: - builder.start_stdin_log_reader(status=opt.status, details=opt.details, + builder.start_stdin_log_reader( + status=opt.status, details=opt.details, pretty=opt.pretty, color=opt.color, debug_locks=opt.debug_locks, debug_pids=opt.debug_pids) for t in targets: @@ -93,21 +94,21 @@ try: warn('%s: exists and not marked as generated; not redoing.\n' % f.nicename()) state.rollback() - + if j < 1 or j > 1000: err('invalid --jobs value: %r\n' % opt.jobs) jwack.setup(j) try: - assert(state.is_flushed()) + assert state.is_flushed() retcode = builder.main(targets, lambda t: (True, True)) - assert(state.is_flushed()) + assert state.is_flushed() finally: try: state.rollback() finally: try: jwack.force_return_tokens() - except Exception, e: + except Exception, e: # pylint: disable=broad-except traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) retcode = 1 diff --git a/state.py b/state.py index e2db20c..6ccd492 100644 --- a/state.py +++ b/state.py @@ -1,7 +1,7 @@ -import sys, os, errno, glob, stat, fcntl, sqlite3 +import sys, os, errno, stat, fcntl, sqlite3 import vars from helpers import unlink, close_on_exec, join -from logs import warn, err, debug2, debug3 +from logs import warn, debug2, debug3 # When the module is imported, change the process title. # We do it here because this module is imported by all the scripts. @@ -14,17 +14,18 @@ else: cmdline[0] = os.path.splitext(os.path.basename(cmdline[0]))[0] setproctitle(" ".join(cmdline)) -SCHEMA_VER=2 -TIMEOUT=60 +SCHEMA_VER = 2 +TIMEOUT = 60 -ALWAYS='//ALWAYS' # an invalid filename that is always marked as dirty -STAMP_DIR='dir' # the stamp of a directory; mtime is unhelpful -STAMP_MISSING='0' # the stamp of a nonexistent file +ALWAYS = '//ALWAYS' # an invalid filename that is always marked as dirty +STAMP_DIR = 'dir' # the stamp of a directory; mtime is unhelpful +STAMP_MISSING = '0' # the stamp of a nonexistent file -LOG_LOCK_MAGIC=0x10000000 # fid offset for "log locks" +LOG_LOCK_MAGIC = 0x10000000 # fid offset for "log locks" -class CyclicDependencyError(Exception): pass +class CyclicDependencyError(Exception): + pass def _connect(dbfile): @@ -48,7 +49,7 @@ def db(): global _db, _lockfile if _db: return _db - + dbdir = '%s/.redo' % vars.BASE dbfile = '%s/db.sqlite3' % dbdir try: @@ -74,7 +75,8 @@ def db(): if ver != SCHEMA_VER: # Don't use err() here because this might happen before # redo-log spawns. - sys.stderr.write('redo: %s: found v%s (expected v%s)\n' + sys.stderr.write( + 'redo: %s: found v%s (expected v%s)\n' % (dbfile, ver, SCHEMA_VER)) sys.stderr.write('redo: manually delete .redo dir to start over.\n') sys.exit(1) @@ -113,10 +115,10 @@ def db(): " ((select max(id)+1 from Runid))") vars.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0] os.environ['REDO_RUNID'] = str(vars.RUNID) - + _db.commit() return _db - + def init(): db() @@ -155,7 +157,7 @@ def is_flushed(): _insane = None def check_sane(): - global _insane, _writable + global _insane if not _insane: _insane = not os.path.exists('%s/.redo' % vars.BASE) return not _insane @@ -170,7 +172,7 @@ def relpath(t, base): base = os.path.normpath(base) tparts = t.split('/') bparts = base.split('/') - for tp,bp in zip(tparts,bparts): + for tp, bp in zip(tparts, bparts): if tp != bp: break tparts.pop(0) @@ -225,13 +227,16 @@ class File(object): # use this mostly to avoid accidentally assigning to typos __slots__ = ['id'] + _file_cols[1:] + # These warnings are a result of the weird way this class is + # initialized, which we should fix, and then re-enable warning. + # pylint: disable=attribute-defined-outside-init def _init_from_idname(self, id, name, allow_add): q = ('select %s from Files ' % join(', ', _file_cols)) if id != None: q += 'where rowid=?' l = [id] elif name != None: - name = (name==ALWAYS) and ALWAYS or relpath(name, vars.BASE) + name = (name == ALWAYS) and ALWAYS or relpath(name, vars.BASE) q += 'where name=?' l = [name] else: @@ -250,7 +255,7 @@ class File(object): # big deal. pass row = d.execute(q, l).fetchone() - assert(row) + assert row return self._init_from_cols(row) def _init_from_cols(self, cols): @@ -259,12 +264,12 @@ class File(object): self.stamp, self.csum) = cols if self.name == ALWAYS and self.changed_runid < vars.RUNID: self.changed_runid = vars.RUNID - + def __init__(self, id=None, name=None, cols=None, allow_add=True): if cols: - return self._init_from_cols(cols) + self._init_from_cols(cols) else: - return self._init_from_idname(id, name, allow_add=allow_add) + self._init_from_idname(id, name, allow_add=allow_add) def __repr__(self): return "File(%r)" % (self.nicename(),) @@ -337,13 +342,13 @@ class File(object): return False # special name, ignore newstamp = self.read_stamp() if (self.is_generated and - (not self.is_failed() or newstamp != STAMP_MISSING) and - not self.is_override and - self.stamp == newstamp): + (not self.is_failed() or newstamp != STAMP_MISSING) and + not self.is_override and + self.stamp == newstamp): # target is as we left it return False if ((not self.is_generated or self.stamp != newstamp) and - newstamp == STAMP_MISSING): + newstamp == STAMP_MISSING): # target has gone missing after the last build. # It's not usefully a source *or* a target. return False @@ -375,8 +380,8 @@ class File(object): for row in db().execute(q, [self.id]).fetchall(): mode = row[0] cols = row[1:] - assert(mode in ('c', 'm')) - yield mode,File(cols=cols) + assert mode in ('c', 'm') + yield mode, File(cols=cols) def zap_deps1(self): debug2('zap-deps1: %r\n' % self.name) @@ -389,7 +394,7 @@ class File(object): def add_dep(self, mode, dep): src = File(name=dep) debug3('add-dep: "%s" < %s "%s"\n' % (self.name, mode, src.name)) - assert(self.id != src.id) + assert self.id != src.id _write("insert or replace into Deps " " (target, mode, source, delete_me) values (?,?,?,?)", [self.id, mode, src.id, False]) @@ -404,10 +409,12 @@ class File(object): return False, STAMP_DIR else: # a "unique identifier" stamp for a regular file - return (stat.S_ISLNK(st.st_mode), + return ( + stat.S_ISLNK(st.st_mode), '-'.join(str(s) for s in ('%.6f' % st.st_mtime, st.st_size, st.st_ino, - st.st_mode, st.st_uid, st.st_gid))) + st.st_mode, st.st_uid, st.st_gid)) + ) def read_stamp(self): is_link, pre = self._read_stamp_st(os.lstat) @@ -444,12 +451,12 @@ def logname(fid): # ok, but it doesn't have F_GETLK, so we can't report which pid owns the lock. # The makes debugging a bit harder. When we someday port to C, we can do that. _locks = {} -class Lock: +class Lock(object): def __init__(self, fid): self.owned = False self.fid = fid - assert(_lockfile >= 0) - assert(_locks.get(fid,0) == 0) + assert _lockfile >= 0 + assert _locks.get(fid, 0) == 0 _locks[fid] = 1 def __del__(self): @@ -458,7 +465,7 @@ class Lock: self.unlock() def check(self): - assert(not self.owned) + assert not self.owned if str(self.fid) in vars.get_locks(): # Lock already held by parent: cyclic dependence raise CyclicDependencyError() @@ -480,14 +487,15 @@ class Lock: def waitlock(self, shared=False): self.check() assert not self.owned - fcntl.lockf(_lockfile, + fcntl.lockf( + _lockfile, fcntl.LOCK_SH if shared else fcntl.LOCK_EX, 1, self.fid) self.owned = True - + def unlock(self): if not self.owned: - raise Exception("can't unlock %r - we don't own it" - % self.lockname) + raise Exception("can't unlock %r - we don't own it" + % self.fid) fcntl.lockf(_lockfile, fcntl.LOCK_UN, 1, self.fid) self.owned = False diff --git a/vars.py b/vars.py index e28235a..fa2a32c 100644 --- a/vars.py +++ b/vars.py @@ -36,11 +36,11 @@ os.environ['REDO_NO_OOB'] = '' # not inheritable by subprocesses def get_locks(): - """Get the list of held locks.""" - return os.environ.get('REDO_LOCKS', '').split(':') + """Get the list of held locks.""" + return os.environ.get('REDO_LOCKS', '').split(':') def add_lock(name): - """Add a lock to the list of held locks.""" - locks = set(get_locks()) - locks.add(name) - os.environ['REDO_LOCKS'] = ':'.join(list(locks)) + """Add a lock to the list of held locks.""" + locks = set(get_locks()) + locks.add(name) + os.environ['REDO_LOCKS'] = ':'.join(list(locks)) diff --git a/vars_init.py b/vars_init.py index 693a8dc..5ab0c53 100644 --- a/vars_init.py +++ b/vars_init.py @@ -5,6 +5,7 @@ is_toplevel = False def init_no_state(): + global is_toplevel if not os.environ.get('REDO'): os.environ['REDO'] = 'NOT_DEFINED' is_toplevel = True @@ -13,9 +14,9 @@ def init_no_state(): def init(targets): + global is_toplevel if not os.environ.get('REDO'): # toplevel call to redo - global is_toplevel is_toplevel = True if len(targets) == 0: targets.append('all')