Merge: Add compatibility to Python 3 (and retain Python 2)

Merge branch 'py6' of https://github.com/mlell/redo

* 'py6' of https://github.com/mlell/redo:
  Remove python<3.0 restriction in setup.py
  Make compatible to BeautifulSoup4
  Accept octal representations of Python 2 (0nnn) and Python 3 (0onnn)
  Prevent iterator being changed while iterating
  Python 2/3 compatible treatment of max(n, None)
  Prevent "Exception ... ignored" in `redo-log ... | head`
  Distinguish byte (python2 str type) and unicode strings (python 3 str type)
  Set file descriptor as inheritable for all pythons >=3.4
  Unify print function usage for Python 2 and 3 via __future__ import
  Run 2to3 utility
  Remove python interpreter selection

docs/md2man.py

@@ -1,5 +1,11 @@
+from __future__ import print_function
 import sys, os, markdown, re
-from BeautifulSoup import BeautifulSoup
+try:
+    from BeautifulSoup import BeautifulSoup
+    bsver = 3
+except ModuleNotFoundError:
+    from bs4 import BeautifulSoup
+    bsver = 4
 
 def _split_lines(s):
     return re.findall(r'([^\n]*\n?)', s)
@@ -179,7 +185,10 @@ def do_definition(tag):
 
 def do_list(tag):
     for i in tag:
-        name = getattr(i, 'name', '').lower()
+        name = getattr(i, 'name', '')
+        # BeautifulSoup4 sometimes results in 'tag' having attributes that have
+        # content 'None'
+        name = name.lower() if name is not None else ''
         if not name and not str(i).strip():
             pass
         elif name != 'li':
@@ -194,7 +203,11 @@ def do_list(tag):
 
 
 def do(tag):
-    name = getattr(tag, 'name', '').lower()
+    name = getattr(tag, 'name', None)
+    # BeautifulSoup4 sometimes results in 'tag' having attributes that have
+    # content 'None'
+    name = name.lower() if name is not None else ''
+
     if not name:
         text(tag)
     elif name == 'h1':
@@ -245,7 +258,7 @@ if len(sys.argv) != 3:
 
 infile = sys.argv[1]
 htmlfile = sys.argv[2]
-lines += open(infile).read().decode('utf8').split('\n')
+lines += open(infile, 'rb').read().decode('utf8').split('\n')
 
 # parse pandoc-style document headers (not part of markdown)
 g = re.match(r'^%\s+(.*?)\((.*?)\)\s+(.*)$', lines[0])
@@ -273,7 +286,12 @@ if AUTHOR:
 
 html = markdown.markdown(inp)
 open(htmlfile, 'w').write(html)
-soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES)
+
+if(bsver == 3):
+    soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES)
+elif(bsver == 4):
+    soup = BeautifulSoup(html, features = "html.parser")
+else: assert 0
 
 macro('.TH', PROD.upper(), SECTION, DATE, VENDOR, GROUPNAME)
 macro('.ad', 'l')  # left justified

redo/builder.py

@@ -1,4 +1,5 @@
 """Code for parallel-building a set of targets, if needed."""
+from __future__ import print_function
 import errno, os, stat, signal, sys, tempfile, time
 from . import cycles, env, helpers, jobserver, logs, paths, state
 from .logs import debug2, err, warn, meta
@@ -11,12 +12,17 @@ def _nice(t):
 def _try_stat(filename):
     try:
         return os.lstat(filename)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.ENOENT:
             return None
         else:
             raise
 
+def _has_pep446():
+    """Test the python version whether the PEP making file descriptors 
+    non-inheritable applies"""
+    return sys.version_info >= (3,4)
+
 
 log_reader_pid = None
 stderr_fd = None
@@ -41,6 +47,7 @@ def start_stdin_log_reader(status, details, pretty, color,
     global stderr_fd
     r, w = os.pipe()    # main pipe to redo-log
     ar, aw = os.pipe()  # ack pipe from redo-log --ack-fd
+    if _has_pep446(): os.set_inheritable(aw, True)
     sys.stdout.flush()
     sys.stderr.flush()
     pid = os.fork()
@@ -55,7 +62,7 @@ def start_stdin_log_reader(status, details, pretty, color,
             # subprocess died without sending us anything: that's bad.
             err('failed to start redo-log subprocess; cannot continue.\n')
             os._exit(99)
-        assert b == 'REDO-OK\n'
+        assert b == b'REDO-OK\n'
         # now we know the subproc is running and will report our errors
         # to stderr, so it's okay to lose our own stderr.
         os.close(ar)
@@ -90,7 +97,7 @@ def start_stdin_log_reader(status, details, pretty, color,
                 argv.append('--color' if color >= 2 else '--no-color')
             argv.append('-')
             os.execvp(argv[0], argv)
-        except Exception, e:  # pylint: disable=broad-except
+        except Exception as e:  # pylint: disable=broad-except
             sys.stderr.write('redo-log: exec: %s\n' % e)
         finally:
             os._exit(99)
@@ -138,7 +145,7 @@ class _BuildJob(object):
                 if is_target:
                     meta('unchanged', state.target_relpath(self.t))
                 return self._finalize(0)
-        except helpers.ImmediateReturn, e:
+        except helpers.ImmediateReturn as e:
             return self._finalize(e.rv)
 
         if env.v.NO_OOB or dirty == True:  # pylint: disable=singleton-comparison
@@ -211,7 +218,7 @@ class _BuildJob(object):
         ffd, fname = tempfile.mkstemp(prefix='redo.', suffix='.tmp')
         helpers.close_on_exec(ffd, True)
         os.unlink(fname)
-        self.outfile = os.fdopen(ffd, 'w+')
+        self.outfile = os.fdopen(ffd, 'w+b')
         # this will run in the dofile's directory, so use only basenames here
         arg1 = basename + ext  # target name (including extension)
         arg2 = basename        # target name (without extension)
@@ -397,8 +404,8 @@ class _BuildJob(object):
                 # script wrote to stdout.  Copy its contents to the tmpfile.
                 helpers.unlink(self.tmpname)
                 try:
-                    newf = open(self.tmpname, 'w')
+                    newf = open(self.tmpname, 'wb')
-                except IOError, e:
+                except IOError as e:
                     dnt = os.path.dirname(os.path.abspath(t))
                     if not os.path.exists(dnt):
                         # This could happen, so report a simple error message
@@ -424,7 +431,7 @@ class _BuildJob(object):
                 try:
                     # Atomically replace the target file
                     os.rename(self.tmpname, t)
-                except OSError, e:
+                except OSError as e:
                     # This could happen for, eg. a permissions error on
                     # the target directory.
                     err('%s: rename %s: %s\n' % (t, self.tmpname, e))

redo/clean.do

@@ -1,2 +1,2 @@
 redo version/clean
-rm -f whichpython python py *.pyc */*.pyc
+rm -f python py *.pyc */*.pyc

redo/cmd_ifchange.py

@@ -52,7 +52,7 @@ def main():
             finally:
                 try:
                     jobserver.force_return_tokens()
-                except Exception, e:  # pylint: disable=broad-except
+                except Exception as e:  # pylint: disable=broad-except
                     traceback.print_exc(100, sys.stderr)
                     err('unexpected error: %r\n' % e)
                     rv = 1

redo/cmd_log.py

@@ -1,4 +1,5 @@
 """redo-log: print past build logs. """
+from __future__ import print_function
 import errno, fcntl, os, re, struct, sys, time
 import termios
 from .atoi import atoi
@@ -102,7 +103,7 @@ def catlog(t):
         if not f:
             try:
                 f = open(logname)
-            except IOError, e:
+            except IOError as e:
                 if e.errno == errno.ENOENT:
                     # ignore files without logs
                     pass
@@ -231,7 +232,7 @@ def catlog(t):
         status = None
     if line_head:
         # partial line never got terminated
-        print line_head
+        print(line_head)
     if t != '-':
         assert depth[-1] == t
         depth.pop(-1)
@@ -263,7 +264,7 @@ def main():
             # their old stderr.
             ack_fd = int(opt.ack_fd)
             assert ack_fd > 2
-            if os.write(ack_fd, 'REDO-OK\n') != 8:
+            if os.write(ack_fd, b'REDO-OK\n') != 8:
                 raise Exception('write to ack_fd returned wrong length')
             os.close(ack_fd)
         queue += targets
@@ -274,9 +275,17 @@ def main():
             catlog(t)
     except KeyboardInterrupt:
         sys.exit(200)
-    except IOError, e:
+    except IOError as e:
         if e.errno == errno.EPIPE:
-            pass
+            # this happens for example if calling `redo-log | head`, so stdout
+            # is piped into another program that closes the pipe before reading
+            # all our output.
+            # from https://docs.python.org/3/library/signal.html#note-on-sigpipe:
+            # Python flushes standard streams on exit; redirect remaining output
+            # to devnull to avoid another BrokenPipeError at shutdown
+            devnull = os.open(os.devnull, os.O_WRONLY)
+            os.dup2(devnull, sys.stdout.fileno())
+            sys.exit(141)  # =128+13: "Terminated by SIGPIPE (signal 13)"
         else:
             raise
 

redo/cmd_ood.py

@@ -1,4 +1,5 @@
 """redo-ood: list out-of-date (ood) targets."""
+from __future__ import print_function
 import sys, os
 from . import deps, env, logs, state
 
@@ -36,7 +37,7 @@ def main():
                             is_checked=is_checked,
                             set_checked=set_checked,
                             log_override=log_override):
-                print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
+                print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
 
 
 if __name__ == '__main__':

redo/cmd_redo.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from __future__ import print_function
 import sys, os, traceback
 from . import builder, env, helpers, jobserver, logs, options, state
 from .atoi import atoi
@@ -48,7 +49,7 @@ def main():
 
     if opt.version:
         from . import version
-        print version.TAG
+        print(version.TAG)
         sys.exit(0)
     if opt.debug:
         os.environ['REDO_DEBUG'] = str(opt.debug or 0)
@@ -114,7 +115,7 @@ def main():
             finally:
                 try:
                     jobserver.force_return_tokens()
-                except Exception, e:  # pylint: disable=broad-except
+                except Exception as e:  # pylint: disable=broad-except
                     traceback.print_exc(100, sys.stderr)
                     err('unexpected error: %r\n' % e)
                     retcode = 1

redo/cmd_sources.py

@@ -1,4 +1,5 @@
 """redo-sources: list the known source (not target) files."""
+from __future__ import print_function
 import sys, os
 from . import env, logs, state
 
@@ -16,7 +17,7 @@ def main():
     cwd = os.getcwd()
     for f in state.files():
         if f.is_source():
-            print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
+            print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
 
 
 if __name__ == '__main__':

redo/cmd_targets.py

@@ -1,4 +1,5 @@
 """redo-targets: list the known targets (not sources)."""
+from __future__ import print_function
 import sys, os
 from . import env, logs, state
 
@@ -16,7 +17,7 @@ def main():
     cwd = os.getcwd()
     for f in state.files():
         if f.is_target():
-            print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
+            print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
 
 
 if __name__ == '__main__':

redo/cmd_whichdo.py

@@ -26,7 +26,7 @@ def main():
         relpath = os.path.relpath(dopath, '.')
         exists = os.path.exists(dopath)
         assert '\n' not in relpath
-        print relpath
+        print(relpath)
         if exists:
             sys.exit(0)
     sys.exit(1)   # no appropriate dofile found

redo/deps.py

@@ -95,7 +95,7 @@ def isdirty(f, depth, max_changed,
         elif mode == 'm':
             sub = isdirty(f2, depth=depth + '  ',
                           max_changed=max(f.changed_runid,
-                                          f.checked_runid),
+                                          f.checked_runid or 0),
                           already_checked=already_checked,
                           is_checked=is_checked,
                           set_checked=set_checked,

redo/helpers.py

@@ -16,7 +16,7 @@ def unlink(f):
     """
     try:
         os.unlink(f)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.ENOENT:
             pass  # it doesn't exist, that's what you asked for
 

redo/jobserver.py

@@ -98,7 +98,7 @@ def _create_tokens(n):
     global _mytokens, _cheats
     assert n >= 0
     assert _cheats >= 0
-    for _ in xrange(n):
+    for _ in range(n):
         if _cheats > 0:
             _cheats -= 1
         else:
@@ -118,7 +118,7 @@ def _release(n):
     assert _mytokens >= n
     _debug('%d,%d -> release(%d)\n' % (_mytokens, _cheats, n))
     n_to_share = 0
-    for _ in xrange(n):
+    for _ in range(n):
         _mytokens -= 1
         if _cheats > 0:
             _cheats -= 1
@@ -128,7 +128,7 @@ def _release(n):
     assert _cheats >= 0
     if n_to_share:
         _debug('PUT tokenfds %d\n' % n_to_share)
-        os.write(_tokenfds[1], 't' * n_to_share)
+        os.write(_tokenfds[1], b't' * n_to_share)
 
 
 def _release_except_mine():
@@ -176,7 +176,7 @@ def _try_read(fd, n):
         signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01)  # emergency fallback
         try:
             b = os.read(fd, 1)
-        except OSError, e:
+        except OSError as e:
             if e.errno in (errno.EAGAIN, errno.EINTR):
                 # interrupted or it was nonblocking
                 return None  # try again
@@ -189,7 +189,7 @@ def _try_read(fd, n):
 
 
 def _try_read_all(fd, n):
-    bb = ''
+    bb = b''
     while 1:
         b = _try_read(fd, n)
         if not b:
@@ -297,7 +297,7 @@ def _wait(want_token, max_delay):
     Returns:
       None
     """
-    rfds = _waitfds.keys()
+    rfds = list(_waitfds.keys())
     if want_token:
         rfds.append(_tokenfds[0])
     assert rfds

redo/options.py

@@ -239,12 +239,12 @@ class Options:
         """
         try:
             (flags,extra) = self.optfunc(args, self._shortopts, self._longopts)
-        except getopt.GetoptError, e:
+        except getopt.GetoptError as e:
             self.fatal(e)
 
         opt = OptDict()
 
-        for k,v in self._defaults.iteritems():
+        for k,v in self._defaults.items():
             k = self._aliases[k]
             opt[k] = v
 
@@ -268,6 +268,6 @@ class Options:
                 else:
                     v = _intify(v)
             opt[k] = v
-        for (f1,f2) in self._aliases.iteritems():
+        for (f1,f2) in self._aliases.items():
             opt[f1] = opt._opts.get(f2)
         return (opt,flags,extra)

redo/state.py

@@ -46,14 +46,14 @@ def db():
     dbfile = '%s/db.sqlite3' % dbdir
     try:
         os.mkdir(dbdir)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.EEXIST:
             pass  # if it exists, that's okay
         else:
             raise
 
     _lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'),
-                        os.O_RDWR | os.O_CREAT, 0666)
+                        os.O_RDWR | os.O_CREAT, 0o666)
     close_on_exec(_lockfile, True)
     if env.is_toplevel and detect_broken_locks():
         env.mark_locks_broken()
@@ -184,7 +184,11 @@ def relpath(t, base):
     base = os.path.normpath(_realdirpath(base))
     tparts = t.split('/')
     bparts = base.split('/')
-    for tp, bp in zip(tparts, bparts):
+
+    # zip must not return an iterator in python 3, because the source lists of
+    # the iterators are changed. The iterator does not notice that and ends too
+    # soon.
+    for tp, bp in list(zip(tparts, bparts)):
         if tp != bp:
             break
         tparts.pop(0)
@@ -278,7 +282,8 @@ class File(object):
         (self.id, self.name, self.is_generated, self.is_override,
          self.checked_runid, self.changed_runid, self.failed_runid,
          self.stamp, self.csum) = cols
-        if self.name == ALWAYS and self.changed_runid < env.v.RUNID:
+        if self.name == ALWAYS and (
+            self.changed_runid is None or self.changed_runid < env.v.RUNID):
             self.changed_runid = env.v.RUNID
 
     def __init__(self, fid=None, name=None, cols=None, allow_add=True):
@@ -508,7 +513,7 @@ class Lock(object):
         assert not self.owned
         try:
             fcntl.lockf(_lockfile, fcntl.LOCK_EX|fcntl.LOCK_NB, 1, self.fid)
-        except IOError, e:
+        except IOError as e:
             if e.errno in (errno.EAGAIN, errno.EACCES):
                 pass  # someone else has it locked
             else:

redo/whichpython.do

@@ -1,10 +1,8 @@
 exec >&2
-for py in intentionally-missing python2.7 python2 python; do
+for py in intentionally-missing python python3 python2 python2.7; do
 	echo "Trying: $py"
 	cmd=$(command -v "$py" || true)
-	# intentionally using the 'print statement' (as opposed to print
+	out=$($cmd -c 'print("success")' 2>/dev/null) || true
-	# function) here, to rule out any python3 interpreters
-	out=$($cmd -c 'print "success"' 2>/dev/null) || true
 	if [ "$out" = "success" ]; then
 		echo $cmd >$3
 		exit 0

setup.py

@@ -22,7 +22,7 @@ def read(fname):
 setuptools.setup(
     name = 'redo-tools',
     version = redo.version.TAG.replace('-', '+', 1),
-    python_requires='>=2.7, <3.0',
+    python_requires='>=2.7',
     author = 'Avery Pennarun',
     author_email = 'apenwarr@gmail.com',
     description = ('djb redo: a recursive, general purpose build system.'),

t/130-mode/all.do

@@ -1,5 +1,5 @@
 umask 0022
 redo mode1
 MODE=$(../../redo/py -c \
-	'import os; print oct(os.stat("mode1").st_mode & 07777)')
+	'import os; print(oct(os.stat("mode1").st_mode & 0o7777))')
-[ "$MODE" = "0644" ] || exit 78
+[ "$MODE" = "0644" -o "$MODE" = "0o644" ] || exit 78