From e8d4809bc57deac3714b1bc950031b3fa2483a41 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Sun, 27 Oct 2019 14:19:25 +0100 Subject: [PATCH 01/11] Remove python interpreter selection --- redo/clean.do | 2 +- redo/whichpython.do | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/redo/clean.do b/redo/clean.do index 72d16b1..3306003 100644 --- a/redo/clean.do +++ b/redo/clean.do @@ -1,2 +1,2 @@ redo version/clean -rm -f whichpython python py *.pyc */*.pyc +rm -f python py *.pyc */*.pyc diff --git a/redo/whichpython.do b/redo/whichpython.do index d5c54bb..02c5545 100644 --- a/redo/whichpython.do +++ b/redo/whichpython.do @@ -1,10 +1,8 @@ exec >&2 -for py in intentionally-missing python2.7 python2 python; do +for py in intentionally-missing python python3 python2 python2.7; do echo "Trying: $py" cmd=$(command -v "$py" || true) - # intentionally using the 'print statement' (as opposed to print - # function) here, to rule out any python3 interpreters - out=$($cmd -c 'print "success"' 2>/dev/null) || true + out=$($cmd -c 'print("success")' 2>/dev/null) || true if [ "$out" = "success" ]; then echo $cmd >$3 exit 0 From 491040ea728a5a885620982be16446ad01ff024e Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Sun, 27 Oct 2019 14:19:50 +0100 Subject: [PATCH 02/11] Run 2to3 utility --- redo/builder.py | 10 +++++----- redo/cmd_ifchange.py | 2 +- redo/cmd_log.py | 6 +++--- redo/cmd_ood.py | 2 +- redo/cmd_redo.py | 4 ++-- redo/cmd_sources.py | 2 +- redo/cmd_targets.py | 2 +- redo/cmd_whichdo.py | 2 +- redo/helpers.py | 2 +- redo/jobserver.py | 8 ++++---- redo/options.py | 6 +++--- redo/state.py | 6 +++--- 12 files changed, 26 insertions(+), 26 deletions(-) diff --git a/redo/builder.py b/redo/builder.py index 414aaa7..c07cd47 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -11,7 +11,7 @@ def _nice(t): def _try_stat(filename): try: return os.lstat(filename) - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: return None else: @@ -90,7 +90,7 @@ def start_stdin_log_reader(status, details, pretty, color, argv.append('--color' if color >= 2 else '--no-color') argv.append('-') os.execvp(argv[0], argv) - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except sys.stderr.write('redo-log: exec: %s\n' % e) finally: os._exit(99) @@ -138,7 +138,7 @@ class _BuildJob(object): if is_target: meta('unchanged', state.target_relpath(self.t)) return self._finalize(0) - except helpers.ImmediateReturn, e: + except helpers.ImmediateReturn as e: return self._finalize(e.rv) if env.v.NO_OOB or dirty == True: # pylint: disable=singleton-comparison @@ -398,7 +398,7 @@ class _BuildJob(object): helpers.unlink(self.tmpname) try: newf = open(self.tmpname, 'w') - except IOError, e: + except IOError as e: dnt = os.path.dirname(os.path.abspath(t)) if not os.path.exists(dnt): # This could happen, so report a simple error message @@ -424,7 +424,7 @@ class _BuildJob(object): try: # Atomically replace the target file os.rename(self.tmpname, t) - except OSError, e: + except OSError as e: # This could happen for, eg. a permissions error on # the target directory. err('%s: rename %s: %s\n' % (t, self.tmpname, e)) diff --git a/redo/cmd_ifchange.py b/redo/cmd_ifchange.py index aaa0e41..bf593a5 100644 --- a/redo/cmd_ifchange.py +++ b/redo/cmd_ifchange.py @@ -52,7 +52,7 @@ def main(): finally: try: jobserver.force_return_tokens() - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) rv = 1 diff --git a/redo/cmd_log.py b/redo/cmd_log.py index 7d666b9..c4ae48c 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -102,7 +102,7 @@ def catlog(t): if not f: try: f = open(logname) - except IOError, e: + except IOError as e: if e.errno == errno.ENOENT: # ignore files without logs pass @@ -231,7 +231,7 @@ def catlog(t): status = None if line_head: # partial line never got terminated - print line_head + print(line_head) if t != '-': assert depth[-1] == t depth.pop(-1) @@ -274,7 +274,7 @@ def main(): catlog(t) except KeyboardInterrupt: sys.exit(200) - except IOError, e: + except IOError as e: if e.errno == errno.EPIPE: pass else: diff --git a/redo/cmd_ood.py b/redo/cmd_ood.py index ece5441..a225034 100644 --- a/redo/cmd_ood.py +++ b/redo/cmd_ood.py @@ -36,7 +36,7 @@ def main(): is_checked=is_checked, set_checked=set_checked, log_override=log_override): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_redo.py b/redo/cmd_redo.py index 15f5f64..8790a45 100644 --- a/redo/cmd_redo.py +++ b/redo/cmd_redo.py @@ -48,7 +48,7 @@ def main(): if opt.version: from . import version - print version.TAG + print(version.TAG) sys.exit(0) if opt.debug: os.environ['REDO_DEBUG'] = str(opt.debug or 0) @@ -114,7 +114,7 @@ def main(): finally: try: jobserver.force_return_tokens() - except Exception, e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except traceback.print_exc(100, sys.stderr) err('unexpected error: %r\n' % e) retcode = 1 diff --git a/redo/cmd_sources.py b/redo/cmd_sources.py index 2229452..0e1326a 100644 --- a/redo/cmd_sources.py +++ b/redo/cmd_sources.py @@ -16,7 +16,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_source(): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_targets.py b/redo/cmd_targets.py index 1a73dc5..c1876a4 100644 --- a/redo/cmd_targets.py +++ b/redo/cmd_targets.py @@ -16,7 +16,7 @@ def main(): cwd = os.getcwd() for f in state.files(): if f.is_target(): - print state.relpath(os.path.join(env.v.BASE, f.name), cwd) + print(state.relpath(os.path.join(env.v.BASE, f.name), cwd)) if __name__ == '__main__': diff --git a/redo/cmd_whichdo.py b/redo/cmd_whichdo.py index 170ebe6..e926de9 100644 --- a/redo/cmd_whichdo.py +++ b/redo/cmd_whichdo.py @@ -26,7 +26,7 @@ def main(): relpath = os.path.relpath(dopath, '.') exists = os.path.exists(dopath) assert '\n' not in relpath - print relpath + print(relpath) if exists: sys.exit(0) sys.exit(1) # no appropriate dofile found diff --git a/redo/helpers.py b/redo/helpers.py index 55d190c..d59e519 100644 --- a/redo/helpers.py +++ b/redo/helpers.py @@ -16,7 +16,7 @@ def unlink(f): """ try: os.unlink(f) - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: pass # it doesn't exist, that's what you asked for diff --git a/redo/jobserver.py b/redo/jobserver.py index ab6984f..acd3f4f 100644 --- a/redo/jobserver.py +++ b/redo/jobserver.py @@ -98,7 +98,7 @@ def _create_tokens(n): global _mytokens, _cheats assert n >= 0 assert _cheats >= 0 - for _ in xrange(n): + for _ in range(n): if _cheats > 0: _cheats -= 1 else: @@ -118,7 +118,7 @@ def _release(n): assert _mytokens >= n _debug('%d,%d -> release(%d)\n' % (_mytokens, _cheats, n)) n_to_share = 0 - for _ in xrange(n): + for _ in range(n): _mytokens -= 1 if _cheats > 0: _cheats -= 1 @@ -176,7 +176,7 @@ def _try_read(fd, n): signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01) # emergency fallback try: b = os.read(fd, 1) - except OSError, e: + except OSError as e: if e.errno in (errno.EAGAIN, errno.EINTR): # interrupted or it was nonblocking return None # try again @@ -297,7 +297,7 @@ def _wait(want_token, max_delay): Returns: None """ - rfds = _waitfds.keys() + rfds = list(_waitfds.keys()) if want_token: rfds.append(_tokenfds[0]) assert rfds diff --git a/redo/options.py b/redo/options.py index 1bada98..9be69e5 100644 --- a/redo/options.py +++ b/redo/options.py @@ -239,12 +239,12 @@ class Options: """ try: (flags,extra) = self.optfunc(args, self._shortopts, self._longopts) - except getopt.GetoptError, e: + except getopt.GetoptError as e: self.fatal(e) opt = OptDict() - for k,v in self._defaults.iteritems(): + for k,v in self._defaults.items(): k = self._aliases[k] opt[k] = v @@ -268,6 +268,6 @@ class Options: else: v = _intify(v) opt[k] = v - for (f1,f2) in self._aliases.iteritems(): + for (f1,f2) in self._aliases.items(): opt[f1] = opt._opts.get(f2) return (opt,flags,extra) diff --git a/redo/state.py b/redo/state.py index 4ad5f53..37b1856 100644 --- a/redo/state.py +++ b/redo/state.py @@ -46,14 +46,14 @@ def db(): dbfile = '%s/db.sqlite3' % dbdir try: os.mkdir(dbdir) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass # if it exists, that's okay else: raise _lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'), - os.O_RDWR | os.O_CREAT, 0666) + os.O_RDWR | os.O_CREAT, 0o666) close_on_exec(_lockfile, True) if env.is_toplevel and detect_broken_locks(): env.mark_locks_broken() @@ -508,7 +508,7 @@ class Lock(object): assert not self.owned try: fcntl.lockf(_lockfile, fcntl.LOCK_EX|fcntl.LOCK_NB, 1, self.fid) - except IOError, e: + except IOError as e: if e.errno in (errno.EAGAIN, errno.EACCES): pass # someone else has it locked else: From 62845688e508398b4ea666ddde46ca323d7a3038 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:09:39 +0100 Subject: [PATCH 03/11] Unify print function usage for Python 2 and 3 via __future__ import --- docs/md2man.py | 1 + redo/builder.py | 1 + redo/cmd_log.py | 1 + redo/cmd_ood.py | 1 + redo/cmd_redo.py | 1 + redo/cmd_sources.py | 1 + redo/cmd_targets.py | 1 + 7 files changed, 7 insertions(+) diff --git a/docs/md2man.py b/docs/md2man.py index 035d2ac..e7f1526 100644 --- a/docs/md2man.py +++ b/docs/md2man.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys, os, markdown, re from BeautifulSoup import BeautifulSoup diff --git a/redo/builder.py b/redo/builder.py index c07cd47..4b06b15 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -1,4 +1,5 @@ """Code for parallel-building a set of targets, if needed.""" +from __future__ import print_function import errno, os, stat, signal, sys, tempfile, time from . import cycles, env, helpers, jobserver, logs, paths, state from .logs import debug2, err, warn, meta diff --git a/redo/cmd_log.py b/redo/cmd_log.py index c4ae48c..466efd3 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -1,4 +1,5 @@ """redo-log: print past build logs. """ +from __future__ import print_function import errno, fcntl, os, re, struct, sys, time import termios from .atoi import atoi diff --git a/redo/cmd_ood.py b/redo/cmd_ood.py index a225034..94cd688 100644 --- a/redo/cmd_ood.py +++ b/redo/cmd_ood.py @@ -1,4 +1,5 @@ """redo-ood: list out-of-date (ood) targets.""" +from __future__ import print_function import sys, os from . import deps, env, logs, state diff --git a/redo/cmd_redo.py b/redo/cmd_redo.py index 8790a45..3d37502 100644 --- a/redo/cmd_redo.py +++ b/redo/cmd_redo.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import print_function import sys, os, traceback from . import builder, env, helpers, jobserver, logs, options, state from .atoi import atoi diff --git a/redo/cmd_sources.py b/redo/cmd_sources.py index 0e1326a..f28cfbc 100644 --- a/redo/cmd_sources.py +++ b/redo/cmd_sources.py @@ -1,4 +1,5 @@ """redo-sources: list the known source (not target) files.""" +from __future__ import print_function import sys, os from . import env, logs, state diff --git a/redo/cmd_targets.py b/redo/cmd_targets.py index c1876a4..4d10419 100644 --- a/redo/cmd_targets.py +++ b/redo/cmd_targets.py @@ -1,4 +1,5 @@ """redo-targets: list the known targets (not sources).""" +from __future__ import print_function import sys, os from . import env, logs, state From 0d8d19437e6da1f8d6a6218a1467c83bd35dee08 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:11:00 +0100 Subject: [PATCH 04/11] Set file descriptor as inheritable for all pythons >=3.4 This is mandated by PEP 446 --- redo/builder.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/redo/builder.py b/redo/builder.py index 4b06b15..c6504fe 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -18,6 +18,11 @@ def _try_stat(filename): else: raise +def _has_pep446(): + """Test the python version whether the PEP making file descriptors + non-inheritable applies""" + return sys.version_info >= (3,4) + log_reader_pid = None stderr_fd = None @@ -42,6 +47,7 @@ def start_stdin_log_reader(status, details, pretty, color, global stderr_fd r, w = os.pipe() # main pipe to redo-log ar, aw = os.pipe() # ack pipe from redo-log --ack-fd + if _has_pep446(): os.set_inheritable(aw, True) sys.stdout.flush() sys.stderr.flush() pid = os.fork() From e239820afd1959f8c0d01ee9dac5f59bd18d0fd2 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:21:23 +0100 Subject: [PATCH 05/11] Distinguish byte (python2 str type) and unicode strings (python 3 str type) Python 3 strings are python 2 unicode strings. Therefore consistently mark strings that are sent via pipes or written/read to file as byte strings. --- docs/md2man.py | 2 +- redo/builder.py | 6 +++--- redo/cmd_log.py | 2 +- redo/jobserver.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/md2man.py b/docs/md2man.py index e7f1526..f89a459 100644 --- a/docs/md2man.py +++ b/docs/md2man.py @@ -246,7 +246,7 @@ if len(sys.argv) != 3: infile = sys.argv[1] htmlfile = sys.argv[2] -lines += open(infile).read().decode('utf8').split('\n') +lines += open(infile, 'rb').read().decode('utf8').split('\n') # parse pandoc-style document headers (not part of markdown) g = re.match(r'^%\s+(.*?)\((.*?)\)\s+(.*)$', lines[0]) diff --git a/redo/builder.py b/redo/builder.py index c6504fe..1ae4a44 100644 --- a/redo/builder.py +++ b/redo/builder.py @@ -62,7 +62,7 @@ def start_stdin_log_reader(status, details, pretty, color, # subprocess died without sending us anything: that's bad. err('failed to start redo-log subprocess; cannot continue.\n') os._exit(99) - assert b == 'REDO-OK\n' + assert b == b'REDO-OK\n' # now we know the subproc is running and will report our errors # to stderr, so it's okay to lose our own stderr. os.close(ar) @@ -218,7 +218,7 @@ class _BuildJob(object): ffd, fname = tempfile.mkstemp(prefix='redo.', suffix='.tmp') helpers.close_on_exec(ffd, True) os.unlink(fname) - self.outfile = os.fdopen(ffd, 'w+') + self.outfile = os.fdopen(ffd, 'w+b') # this will run in the dofile's directory, so use only basenames here arg1 = basename + ext # target name (including extension) arg2 = basename # target name (without extension) @@ -404,7 +404,7 @@ class _BuildJob(object): # script wrote to stdout. Copy its contents to the tmpfile. helpers.unlink(self.tmpname) try: - newf = open(self.tmpname, 'w') + newf = open(self.tmpname, 'wb') except IOError as e: dnt = os.path.dirname(os.path.abspath(t)) if not os.path.exists(dnt): diff --git a/redo/cmd_log.py b/redo/cmd_log.py index 466efd3..fdcc7b8 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -264,7 +264,7 @@ def main(): # their old stderr. ack_fd = int(opt.ack_fd) assert ack_fd > 2 - if os.write(ack_fd, 'REDO-OK\n') != 8: + if os.write(ack_fd, b'REDO-OK\n') != 8: raise Exception('write to ack_fd returned wrong length') os.close(ack_fd) queue += targets diff --git a/redo/jobserver.py b/redo/jobserver.py index acd3f4f..9102345 100644 --- a/redo/jobserver.py +++ b/redo/jobserver.py @@ -128,7 +128,7 @@ def _release(n): assert _cheats >= 0 if n_to_share: _debug('PUT tokenfds %d\n' % n_to_share) - os.write(_tokenfds[1], 't' * n_to_share) + os.write(_tokenfds[1], b't' * n_to_share) def _release_except_mine(): @@ -189,7 +189,7 @@ def _try_read(fd, n): def _try_read_all(fd, n): - bb = '' + bb = b'' while 1: b = _try_read(fd, n) if not b: From 52a8ca25b2c9bb779382771bb32621a2dce7ded1 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:28:09 +0100 Subject: [PATCH 06/11] Prevent "Exception ... ignored" in `redo-log ... | head` When STDOUT is piped to another program and that program closes the pipe, this program is supposed to terminate. However, Python 3 prints an error message because prior to exiting, STDOUT is automatically flushed which does not work due to it being closed by the other side. This commit includes code taken from the Python documentation for this case. The output stream is redirected to /dev/null --- redo/cmd_log.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/redo/cmd_log.py b/redo/cmd_log.py index fdcc7b8..3f86511 100644 --- a/redo/cmd_log.py +++ b/redo/cmd_log.py @@ -277,7 +277,15 @@ def main(): sys.exit(200) except IOError as e: if e.errno == errno.EPIPE: - pass + # this happens for example if calling `redo-log | head`, so stdout + # is piped into another program that closes the pipe before reading + # all our output. + # from https://docs.python.org/3/library/signal.html#note-on-sigpipe: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(141) # =128+13: "Terminated by SIGPIPE (signal 13)" else: raise From efab08fc9ffbecdd91ce598fadcb13abcfb6755a Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:51:17 +0100 Subject: [PATCH 07/11] Python 2/3 compatible treatment of max(n, None) Python 3 does no longer allow comparisons of different types. Therefore, explicitly convert f.checked_runid to a number that's always smaller than `f.changed_runid` --- redo/deps.py | 2 +- redo/state.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/redo/deps.py b/redo/deps.py index 0ce53d3..bc2909c 100644 --- a/redo/deps.py +++ b/redo/deps.py @@ -95,7 +95,7 @@ def isdirty(f, depth, max_changed, elif mode == 'm': sub = isdirty(f2, depth=depth + ' ', max_changed=max(f.changed_runid, - f.checked_runid), + f.checked_runid or 0), already_checked=already_checked, is_checked=is_checked, set_checked=set_checked, diff --git a/redo/state.py b/redo/state.py index 37b1856..667a3d8 100644 --- a/redo/state.py +++ b/redo/state.py @@ -278,7 +278,8 @@ class File(object): (self.id, self.name, self.is_generated, self.is_override, self.checked_runid, self.changed_runid, self.failed_runid, self.stamp, self.csum) = cols - if self.name == ALWAYS and self.changed_runid < env.v.RUNID: + if self.name == ALWAYS and ( + self.changed_runid is None or self.changed_runid < env.v.RUNID): self.changed_runid = env.v.RUNID def __init__(self, fid=None, name=None, cols=None, allow_add=True): From 7f2d04d5fffdbe7632f78898aa3f8346bb162bea Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:54:28 +0100 Subject: [PATCH 08/11] Prevent iterator being changed while iterating In Python 3, `zip()` returns an iterator that in turn contains references to `tparts` and `bparts`. Because those two variables are modified within the loop, the iterator does not advance properly. Fix this by explicitly obtaining a list. --- redo/state.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/redo/state.py b/redo/state.py index 667a3d8..19458ee 100644 --- a/redo/state.py +++ b/redo/state.py @@ -184,7 +184,11 @@ def relpath(t, base): base = os.path.normpath(_realdirpath(base)) tparts = t.split('/') bparts = base.split('/') - for tp, bp in zip(tparts, bparts): + + # zip must not return an iterator in python 3, because the source lists of + # the iterators are changed. The iterator does not notice that and ends too + # soon. + for tp, bp in list(zip(tparts, bparts)): if tp != bp: break tparts.pop(0) From 5953729a443d21eef30c6e748b50fce057e4b8bc Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 19:55:46 +0100 Subject: [PATCH 09/11] Accept octal representations of Python 2 (0nnn) and Python 3 (0onnn) The number format '0onnn' is accepted by both pythons --- t/130-mode/all.do | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/130-mode/all.do b/t/130-mode/all.do index ab4532d..da5b3be 100644 --- a/t/130-mode/all.do +++ b/t/130-mode/all.do @@ -1,5 +1,5 @@ umask 0022 redo mode1 MODE=$(../../redo/py -c \ - 'import os; print oct(os.stat("mode1").st_mode & 07777)') -[ "$MODE" = "0644" ] || exit 78 + 'import os; print(oct(os.stat("mode1").st_mode & 0o7777))') +[ "$MODE" = "0644" -o "$MODE" = "0o644" ] || exit 78 From a8fd6a123c7f9dc9d9f19ec9065bf0760c0211a2 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 21:27:46 +0100 Subject: [PATCH 10/11] Make compatible to BeautifulSoup4 --- docs/md2man.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/md2man.py b/docs/md2man.py index f89a459..3cabddf 100644 --- a/docs/md2man.py +++ b/docs/md2man.py @@ -1,6 +1,11 @@ from __future__ import print_function import sys, os, markdown, re -from BeautifulSoup import BeautifulSoup +try: + from BeautifulSoup import BeautifulSoup + bsver = 3 +except ModuleNotFoundError: + from bs4 import BeautifulSoup + bsver = 4 def _split_lines(s): return re.findall(r'([^\n]*\n?)', s) @@ -180,7 +185,10 @@ def do_definition(tag): def do_list(tag): for i in tag: - name = getattr(i, 'name', '').lower() + name = getattr(i, 'name', '') + # BeautifulSoup4 sometimes results in 'tag' having attributes that have + # content 'None' + name = name.lower() if name is not None else '' if not name and not str(i).strip(): pass elif name != 'li': @@ -195,7 +203,11 @@ def do_list(tag): def do(tag): - name = getattr(tag, 'name', '').lower() + name = getattr(tag, 'name', None) + # BeautifulSoup4 sometimes results in 'tag' having attributes that have + # content 'None' + name = name.lower() if name is not None else '' + if not name: text(tag) elif name == 'h1': @@ -274,7 +286,12 @@ if AUTHOR: html = markdown.markdown(inp) open(htmlfile, 'w').write(html) -soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES) + +if(bsver == 3): + soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES) +elif(bsver == 4): + soup = BeautifulSoup(html, features = "html.parser") +else: assert 0 macro('.TH', PROD.upper(), SECTION, DATE, VENDOR, GROUPNAME) macro('.ad', 'l') # left justified From be3bda885df90f86ec90b8b85604b7132cc54b43 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Wed, 30 Oct 2019 22:14:00 +0100 Subject: [PATCH 11/11] Remove python<3.0 restriction in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 822c45a..0b9d68a 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def read(fname): setuptools.setup( name = 'redo-tools', version = redo.version.TAG.replace('-', '+', 1), - python_requires='>=2.7, <3.0', + python_requires='>=2.7', author = 'Avery Pennarun', author_email = 'apenwarr@gmail.com', description = ('djb redo: a recursive, general purpose build system.'),