diff --git a/docs/md2man.py b/docs/md2man.py index 035d2ac..3cabddf 100644 --- a/docs/md2man.py +++ b/docs/md2man.py @@ -1,5 +1,11 @@ +from __future__ import print_function import sys, os, markdown, re -from BeautifulSoup import BeautifulSoup +try: + from BeautifulSoup import BeautifulSoup + bsver = 3 +except ModuleNotFoundError: + from bs4 import BeautifulSoup + bsver = 4 def _split_lines(s): return re.findall(r'([^\n]*\n?)', s) @@ -179,7 +185,10 @@ def do_definition(tag): def do_list(tag): for i in tag: - name = getattr(i, 'name', '').lower() + name = getattr(i, 'name', '') + # BeautifulSoup4 sometimes results in 'tag' having attributes that have + # content 'None' + name = name.lower() if name is not None else '' if not name and not str(i).strip(): pass elif name != 'li': @@ -194,7 +203,11 @@ def do_list(tag): def do(tag): - name = getattr(tag, 'name', '').lower() + name = getattr(tag, 'name', None) + # BeautifulSoup4 sometimes results in 'tag' having attributes that have + # content 'None' + name = name.lower() if name is not None else '' + if not name: text(tag) elif name == 'h1': @@ -245,7 +258,7 @@ if len(sys.argv) != 3: infile = sys.argv[1] htmlfile = sys.argv[2] -lines += open(infile).read().decode('utf8').split('\n') +lines += open(infile, 'rb').read().decode('utf8').split('\n') # parse pandoc-style document headers (not part of markdown) g = re.match(r'^%\s+(.*?)\((.*?)\)\s+(.*)$', lines[0]) @@ -273,7 +286,12 @@ if AUTHOR: html = markdown.markdown(inp) open(htmlfile, 'w').write(html) -soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES) + +if(bsver == 3): + soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES) +elif(bsver == 4): + soup = BeautifulSoup(html, features = "html.parser") +else: assert 0 macro('.TH', PROD.upper(), SECTION, DATE, VENDOR, GROUPNAME) macro('.ad', 'l') # left justified diff --git a/redo/builder.py b/redo/builder.py index 414aaa7..1ae4a44 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -1,4 +1,5 @@ """Code for parallel-building a set of targets, if needed.""" +from __future__ import print_function import errno, os, stat, signal, sys, tempfile, time from . import cycles, env, helpers, jobserver, logs, paths, state from .logs import debug2, err, warn, meta @@ -11,12 +12,17 @@ def _nice(t): def _try_stat(filename): try: return os.lstat(filename) - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: return None else: raise +def _has_pep446(): + """Test the python version whether the PEP making file descriptors + non-inheritable applies""" + return sys.version_info >= (3,4) + log_reader_pid = None stderr_fd = None @@ -41,6 +47,7 @@ def start_stdin_log_reader(status, details, pretty, color, global stderr_fd r, w = os.pipe() # main pipe to redo-log ar, aw = os.pipe() # ack pipe from redo-log --ack-fd + if _has_pep446(): os.set_inheritable(aw, True) sys.stdout.flush() sys.stderr.flush() pid = os.fork() @@ -55,7 +62,7 @@ def start_stdin_log_reader(status, details, pretty, color, # subprocess died without sending us anything: that's bad. err('failed to start redo-log subprocess; cannot continue.\n') os._exit(99) - assert b == 'REDO-OK\n' + assert b == b'REDO-OK\n' # now we know the subproc is running and will report our errors # to stderr, so it's okay to lose our own stderr. os.close(ar) @@ -90,7 +97,7 @@ def start_stdin_log_reader(status, details, pretty, color, argv.append('--color' if color >= 2 else '--no-color') argv.append('-') os.execvp(argv[0], argv) - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except sys.stderr.write('redo-log: exec: %s\n' % e) finally: os._exit(99) @@ -138,7 +145,7 @@ class _BuildJob(object): if is_target: meta('unchanged', state.target_relpath(self.t)) return self._finalize(0) - except helpers.ImmediateReturn, e: + except helpers.ImmediateReturn as e: return self._finalize(e.rv) if env.v.NO_OOB or dirty == True: # pylint: disable=singleton-comparison @@ -211,7 +218,7 @@ class _BuildJob(object): ffd, fname = tempfile.mkstemp(prefix='redo.', suffix='.tmp') helpers.close_on_exec(ffd, True) os.unlink(fname) - self.outfile = os.fdopen(ffd, 'w+') + self.outfile = os.fdopen(ffd, 'w+b') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) @@ -397,8 +404,8 @@ class _BuildJob(object): # script wrote to stdout. Copy its contents to the tmpfile. helpers.unlink(self.tmpname) try: - newf = open(self.tmpname, 'w') - except IOError, e: + newf = open(self.tmpname, 'wb') + except IOError as e: dnt = os.path.dirname(os.path.abspath(t)) if not os.path.exists(dnt): # This could happen, so report a simple error message @@ -424,7 +431,7 @@ class _BuildJob(object): try: # Atomically replace the target file os.rename(self.tmpname, t) - except OSError, e: + except OSError as e: # This could happen for, eg. a permissions error on # the target directory. err('%s: rename %s: %s\n' % (t, self.tmpname, e)) diff --git a/redo/clean.do b/redo/clean.do index 72d16b1..3306003 100644 --- a/redo/clean.do +++ b/redo/clean.do @@ -1,2 +1,2 @@ redo version/clean -rm -f whichpython python py *.pyc */*.pyc +rm -f python py *.pyc */*.pyc diff --git a/redo/cmd_ifchange.py b/redo/cmd_ifchange.py index aaa0e41..bf593a5 100644 --- a/redo/cmd_ifchange.py +++ b/redo/cmd_ifchange.py @@ -52,7 +52,7 @@ def main(): finally: try: jobserver.force_return_tokens() - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) rv = 1 diff --git a/redo/cmd_log.py b/redo/cmd_log.py index 7d666b9..3f86511 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -1,4 +1,5 @@ """redo-log: print past build logs. """ +from __future__ import print_function import errno, fcntl, os, re, struct, sys, time import termios from .atoi import atoi @@ -102,7 +103,7 @@ def catlog(t): if not f: try: f = open(logname) - except IOError, e: + except IOError as e: if e.errno == errno.ENOENT: # ignore files without logs pass @@ -231,7 +232,7 @@ def catlog(t): status = None if line_head: # partial line never got terminated - print line_head + print(line_head) if t != '-': assert depth[-1] == t depth.pop(-1) @@ -263,7 +264,7 @@ def main(): # their old stderr. ack_fd = int(opt.ack_fd) assert ack_fd > 2 - if os.write(ack_fd, 'REDO-OK\n') != 8: + if os.write(ack_fd, b'REDO-OK\n') != 8: raise Exception('write to ack_fd returned wrong length') os.close(ack_fd) queue += targets @@ -274,9 +275,17 @@ def main(): catlog(t) except KeyboardInterrupt: sys.exit(200) - except IOError, e: + except IOError as e: if e.errno == errno.EPIPE: - pass + # this happens for example if calling `redo-log | head`, so stdout + # is piped into another program that closes the pipe before reading + # all our output. + # from https://docs.python.org/3/library/signal.html#note-on-sigpipe: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(141) # =128+13: "Terminated by SIGPIPE (signal 13)" else: raise diff --git a/redo/cmd_ood.py b/redo/cmd_ood.py index ece5441..94cd688 100644 --- a/redo/cmd_ood.py +++ b/redo/cmd_ood.py @@ -1,4 +1,5 @@ """redo-ood: list out-of-date (ood) targets.""" +from __future__ import print_function import sys, os from . import deps, env, logs, state @@ -36,7 +37,7 @@ def main(): is_checked=is_checked, set_checked=set_checked, log_override=log_override): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_redo.py b/redo/cmd_redo.py index 15f5f64..3d37502 100644 --- a/redo/cmd_redo.py +++ b/redo/cmd_redo.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import print_function import sys, os, traceback from . import builder, env, helpers, jobserver, logs, options, state from .atoi import atoi @@ -48,7 +49,7 @@ def main(): if opt.version: from . import version - print version.TAG + print(version.TAG) sys.exit(0) if opt.debug: os.environ['REDO_DEBUG'] = str(opt.debug or 0) @@ -114,7 +115,7 @@ def main(): finally: try: jobserver.force_return_tokens() - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) retcode = 1 diff --git a/redo/cmd_sources.py b/redo/cmd_sources.py index 2229452..f28cfbc 100644 --- a/redo/cmd_sources.py +++ b/redo/cmd_sources.py @@ -1,4 +1,5 @@ """redo-sources: list the known source (not target) files.""" +from __future__ import print_function import sys, os from . import env, logs, state @@ -16,7 +17,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_source(): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_targets.py b/redo/cmd_targets.py index 1a73dc5..4d10419 100644 --- a/redo/cmd_targets.py +++ b/redo/cmd_targets.py @@ -1,4 +1,5 @@ """redo-targets: list the known targets (not sources).""" +from __future__ import print_function import sys, os from . import env, logs, state @@ -16,7 +17,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_target(): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_whichdo.py b/redo/cmd_whichdo.py index 170ebe6..e926de9 100644 --- a/redo/cmd_whichdo.py +++ b/redo/cmd_whichdo.py @@ -26,7 +26,7 @@ def main(): relpath = os.path.relpath(dopath, '.') exists = os.path.exists(dopath) assert '\n' not in relpath - print relpath + print(relpath) if exists: sys.exit(0) sys.exit(1) # no appropriate dofile found diff --git a/redo/deps.py b/redo/deps.py index 0ce53d3..bc2909c 100644 --- a/redo/deps.py +++ b/redo/deps.py @@ -95,7 +95,7 @@ def isdirty(f, depth, max_changed, elif mode == 'm': sub = isdirty(f2, depth=depth + ' ', max_changed=max(f.changed_runid, - f.checked_runid), + f.checked_runid or 0), already_checked=already_checked, is_checked=is_checked, set_checked=set_checked, diff --git a/redo/helpers.py b/redo/helpers.py index 55d190c..d59e519 100644 --- a/redo/helpers.py +++ b/redo/helpers.py @@ -16,7 +16,7 @@ def unlink(f): """ try: os.unlink(f) - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: pass # it doesn't exist, that's what you asked for diff --git a/redo/jobserver.py b/redo/jobserver.py index ab6984f..9102345 100644 --- a/redo/jobserver.py +++ b/redo/jobserver.py @@ -98,7 +98,7 @@ def _create_tokens(n): global _mytokens, _cheats assert n >= 0 assert _cheats >= 0 - for _ in xrange(n): + for _ in range(n): if _cheats > 0: _cheats -= 1 else: @@ -118,7 +118,7 @@ def _release(n): assert _mytokens >= n _debug('%d,%d -> release(%d)\n' % (_mytokens, _cheats, n)) n_to_share = 0 - for _ in xrange(n): + for _ in range(n): _mytokens -= 1 if _cheats > 0: _cheats -= 1 @@ -128,7 +128,7 @@ def _release(n): assert _cheats >= 0 if n_to_share: _debug('PUT tokenfds %d\n' % n_to_share) - os.write(_tokenfds[1], 't' * n_to_share) + os.write(_tokenfds[1], b't' * n_to_share) def _release_except_mine(): @@ -176,7 +176,7 @@ def _try_read(fd, n): signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01) # emergency fallback try: b = os.read(fd, 1) - except OSError, e: + except OSError as e: if e.errno in (errno.EAGAIN, errno.EINTR): # interrupted or it was nonblocking return None # try again @@ -189,7 +189,7 @@ def _try_read(fd, n): def _try_read_all(fd, n): - bb = '' + bb = b'' while 1: b = _try_read(fd, n) if not b: @@ -297,7 +297,7 @@ def _wait(want_token, max_delay): Returns: None """ - rfds = _waitfds.keys() + rfds = list(_waitfds.keys()) if want_token: rfds.append(_tokenfds[0]) assert rfds diff --git a/redo/options.py b/redo/options.py index 1bada98..9be69e5 100644 --- a/redo/options.py +++ b/redo/options.py @@ -239,12 +239,12 @@ class Options: """ try: (flags,extra) = self.optfunc(args, self._shortopts, self._longopts) - except getopt.GetoptError, e: + except getopt.GetoptError as e: self.fatal(e) opt = OptDict() - for k,v in self._defaults.iteritems(): + for k,v in self._defaults.items(): k = self._aliases[k] opt[k] = v @@ -268,6 +268,6 @@ class Options: else: v = _intify(v) opt[k] = v - for (f1,f2) in self._aliases.iteritems(): + for (f1,f2) in self._aliases.items(): opt[f1] = opt._opts.get(f2) return (opt,flags,extra) diff --git a/redo/state.py b/redo/state.py index 4ad5f53..19458ee 100644 --- a/redo/state.py +++ b/redo/state.py @@ -46,14 +46,14 @@ def db(): dbfile = '%s/db.sqlite3' % dbdir try: os.mkdir(dbdir) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass # if it exists, that's okay else: raise _lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'), - os.O_RDWR | os.O_CREAT, 0666) + os.O_RDWR | os.O_CREAT, 0o666) close_on_exec(_lockfile, True) if env.is_toplevel and detect_broken_locks(): env.mark_locks_broken() @@ -184,7 +184,11 @@ def relpath(t, base): base = os.path.normpath(_realdirpath(base)) tparts = t.split('/') bparts = base.split('/') - for tp, bp in zip(tparts, bparts): + + # zip must not return an iterator in python 3, because the source lists of + # the iterators are changed. The iterator does not notice that and ends too + # soon. + for tp, bp in list(zip(tparts, bparts)): if tp != bp: break tparts.pop(0) @@ -278,7 +282,8 @@ class File(object): (self.id, self.name, self.is_generated, self.is_override, self.checked_runid, self.changed_runid, self.failed_runid, self.stamp, self.csum) = cols - if self.name == ALWAYS and self.changed_runid < env.v.RUNID: + if self.name == ALWAYS and ( + self.changed_runid is None or self.changed_runid < env.v.RUNID): self.changed_runid = env.v.RUNID def __init__(self, fid=None, name=None, cols=None, allow_add=True): @@ -508,7 +513,7 @@ class Lock(object): assert not self.owned try: fcntl.lockf(_lockfile, fcntl.LOCK_EX|fcntl.LOCK_NB, 1, self.fid) - except IOError, e: + except IOError as e: if e.errno in (errno.EAGAIN, errno.EACCES): pass # someone else has it locked else: diff --git a/redo/whichpython.do b/redo/whichpython.do index d5c54bb..02c5545 100644 --- a/redo/whichpython.do +++ b/redo/whichpython.do @@ -1,10 +1,8 @@ exec >&2 -for py in intentionally-missing python2.7 python2 python; do +for py in intentionally-missing python python3 python2 python2.7; do echo "Trying: $py" cmd=$(command -v "$py" || true) - # intentionally using the 'print statement' (as opposed to print - # function) here, to rule out any python3 interpreters - out=$($cmd -c 'print "success"' 2>/dev/null) || true + out=$($cmd -c 'print("success")' 2>/dev/null) || true if [ "$out" = "success" ]; then echo $cmd >$3 exit 0 diff --git a/setup.py b/setup.py index 822c45a..0b9d68a 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def read(fname): setuptools.setup( name = 'redo-tools', version = redo.version.TAG.replace('-', '+', 1), - python_requires='>=2.7, <3.0', + python_requires='>=2.7', author = 'Avery Pennarun', author_email = 'apenwarr@gmail.com', description = ('djb redo: a recursive, general purpose build system.'), diff --git a/t/130-mode/all.do b/t/130-mode/all.do index ab4532d..da5b3be 100644 --- a/t/130-mode/all.do +++ b/t/130-mode/all.do @@ -1,5 +1,5 @@ umask 0022 redo mode1 MODE=$(../../redo/py -c \ - 'import os; print oct(os.stat("mode1").st_mode & 07777)') -[ "$MODE" = "0644" ] || exit 78 + 'import os; print(oct(os.stat("mode1").st_mode & 0o7777))') +[ "$MODE" = "0644" -o "$MODE" = "0o644" ] || exit 78