Merge: Add compatibility to Python 3 (and retain Python 2)
Merge branch 'py6' of https://github.com/mlell/redo * 'py6' of https://github.com/mlell/redo: Remove python<3.0 restriction in setup.py Make compatible to BeautifulSoup4 Accept octal representations of Python 2 (0nnn) and Python 3 (0onnn) Prevent iterator being changed while iterating Python 2/3 compatible treatment of max(n, None) Prevent "Exception ... ignored" in `redo-log ... | head` Distinguish byte (python2 str type) and unicode strings (python 3 str type) Set file descriptor as inheritable for all pythons >=3.4 Unify print function usage for Python 2 and 3 via __future__ import Run 2to3 utility Remove python interpreter selection
This commit is contained in:
Avery Pennarun
commit
68d355178e
18 changed files with 90 additions and 49 deletions
|
|
@ -1,5 +1,11 @@
|
|||
from __future__ import print_function
|
||||
import sys, os, markdown, re
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
try:
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
bsver = 3
|
||||
except ModuleNotFoundError:
|
||||
from bs4 import BeautifulSoup
|
||||
bsver = 4
|
||||
|
||||
def _split_lines(s):
|
||||
return re.findall(r'([^\n]*\n?)', s)
|
||||
|
|
@ -179,7 +185,10 @@ def do_definition(tag):
|
|||
|
||||
def do_list(tag):
|
||||
for i in tag:
|
||||
name = getattr(i, 'name', '').lower()
|
||||
name = getattr(i, 'name', '')
|
||||
# BeautifulSoup4 sometimes results in 'tag' having attributes that have
|
||||
# content 'None'
|
||||
name = name.lower() if name is not None else ''
|
||||
if not name and not str(i).strip():
|
||||
pass
|
||||
elif name != 'li':
|
||||
|
|
@ -194,7 +203,11 @@ def do_list(tag):
|
|||
|
||||
|
||||
def do(tag):
|
||||
name = getattr(tag, 'name', '').lower()
|
||||
name = getattr(tag, 'name', None)
|
||||
# BeautifulSoup4 sometimes results in 'tag' having attributes that have
|
||||
# content 'None'
|
||||
name = name.lower() if name is not None else ''
|
||||
|
||||
if not name:
|
||||
text(tag)
|
||||
elif name == 'h1':
|
||||
|
|
@ -245,7 +258,7 @@ if len(sys.argv) != 3:
|
|||
|
||||
infile = sys.argv[1]
|
||||
htmlfile = sys.argv[2]
|
||||
lines += open(infile).read().decode('utf8').split('\n')
|
||||
lines += open(infile, 'rb').read().decode('utf8').split('\n')
|
||||
|
||||
# parse pandoc-style document headers (not part of markdown)
|
||||
g = re.match(r'^%\s+(.*?)\((.*?)\)\s+(.*)$', lines[0])
|
||||
|
|
@ -273,7 +286,12 @@ if AUTHOR:
|
|||
|
||||
html = markdown.markdown(inp)
|
||||
open(htmlfile, 'w').write(html)
|
||||
soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES)
|
||||
|
||||
if(bsver == 3):
|
||||
soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES)
|
||||
elif(bsver == 4):
|
||||
soup = BeautifulSoup(html, features = "html.parser")
|
||||
else: assert 0
|
||||
|
||||
macro('.TH', PROD.upper(), SECTION, DATE, VENDOR, GROUPNAME)
|
||||
macro('.ad', 'l') # left justified
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Code for parallel-building a set of targets, if needed."""
|
||||
from __future__ import print_function
|
||||
import errno, os, stat, signal, sys, tempfile, time
|
||||
from . import cycles, env, helpers, jobserver, logs, paths, state
|
||||
from .logs import debug2, err, warn, meta
|
||||
|
|
@ -11,12 +12,17 @@ def _nice(t):
|
|||
def _try_stat(filename):
|
||||
try:
|
||||
return os.lstat(filename)
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
return None
|
||||
else:
|
||||
raise
|
||||
|
||||
def _has_pep446():
|
||||
"""Test the python version whether the PEP making file descriptors
|
||||
non-inheritable applies"""
|
||||
return sys.version_info >= (3,4)
|
||||
|
||||
|
||||
log_reader_pid = None
|
||||
stderr_fd = None
|
||||
|
|
@ -41,6 +47,7 @@ def start_stdin_log_reader(status, details, pretty, color,
|
|||
global stderr_fd
|
||||
r, w = os.pipe() # main pipe to redo-log
|
||||
ar, aw = os.pipe() # ack pipe from redo-log --ack-fd
|
||||
if _has_pep446(): os.set_inheritable(aw, True)
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
pid = os.fork()
|
||||
|
|
@ -55,7 +62,7 @@ def start_stdin_log_reader(status, details, pretty, color,
|
|||
# subprocess died without sending us anything: that's bad.
|
||||
err('failed to start redo-log subprocess; cannot continue.\n')
|
||||
os._exit(99)
|
||||
assert b == 'REDO-OK\n'
|
||||
assert b == b'REDO-OK\n'
|
||||
# now we know the subproc is running and will report our errors
|
||||
# to stderr, so it's okay to lose our own stderr.
|
||||
os.close(ar)
|
||||
|
|
@ -90,7 +97,7 @@ def start_stdin_log_reader(status, details, pretty, color,
|
|||
argv.append('--color' if color >= 2 else '--no-color')
|
||||
argv.append('-')
|
||||
os.execvp(argv[0], argv)
|
||||
except Exception, e: # pylint: disable=broad-except
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
sys.stderr.write('redo-log: exec: %s\n' % e)
|
||||
finally:
|
||||
os._exit(99)
|
||||
|
|
@ -138,7 +145,7 @@ class _BuildJob(object):
|
|||
if is_target:
|
||||
meta('unchanged', state.target_relpath(self.t))
|
||||
return self._finalize(0)
|
||||
except helpers.ImmediateReturn, e:
|
||||
except helpers.ImmediateReturn as e:
|
||||
return self._finalize(e.rv)
|
||||
|
||||
if env.v.NO_OOB or dirty == True: # pylint: disable=singleton-comparison
|
||||
|
|
@ -211,7 +218,7 @@ class _BuildJob(object):
|
|||
ffd, fname = tempfile.mkstemp(prefix='redo.', suffix='.tmp')
|
||||
helpers.close_on_exec(ffd, True)
|
||||
os.unlink(fname)
|
||||
self.outfile = os.fdopen(ffd, 'w+')
|
||||
self.outfile = os.fdopen(ffd, 'w+b')
|
||||
# this will run in the dofile's directory, so use only basenames here
|
||||
arg1 = basename + ext # target name (including extension)
|
||||
arg2 = basename # target name (without extension)
|
||||
|
|
@ -397,8 +404,8 @@ class _BuildJob(object):
|
|||
# script wrote to stdout. Copy its contents to the tmpfile.
|
||||
helpers.unlink(self.tmpname)
|
||||
try:
|
||||
newf = open(self.tmpname, 'w')
|
||||
except IOError, e:
|
||||
newf = open(self.tmpname, 'wb')
|
||||
except IOError as e:
|
||||
dnt = os.path.dirname(os.path.abspath(t))
|
||||
if not os.path.exists(dnt):
|
||||
# This could happen, so report a simple error message
|
||||
|
|
@ -424,7 +431,7 @@ class _BuildJob(object):
|
|||
try:
|
||||
# Atomically replace the target file
|
||||
os.rename(self.tmpname, t)
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
# This could happen for, eg. a permissions error on
|
||||
# the target directory.
|
||||
err('%s: rename %s: %s\n' % (t, self.tmpname, e))
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
redo version/clean
|
||||
rm -f whichpython python py *.pyc */*.pyc
|
||||
rm -f python py *.pyc */*.pyc
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ def main():
|
|||
finally:
|
||||
try:
|
||||
jobserver.force_return_tokens()
|
||||
except Exception, e: # pylint: disable=broad-except
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
traceback.print_exc(100, sys.stderr)
|
||||
err('unexpected error: %r\n' % e)
|
||||
rv = 1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""redo-log: print past build logs. """
|
||||
from __future__ import print_function
|
||||
import errno, fcntl, os, re, struct, sys, time
|
||||
import termios
|
||||
from .atoi import atoi
|
||||
|
|
@ -102,7 +103,7 @@ def catlog(t):
|
|||
if not f:
|
||||
try:
|
||||
f = open(logname)
|
||||
except IOError, e:
|
||||
except IOError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
# ignore files without logs
|
||||
pass
|
||||
|
|
@ -231,7 +232,7 @@ def catlog(t):
|
|||
status = None
|
||||
if line_head:
|
||||
# partial line never got terminated
|
||||
print line_head
|
||||
print(line_head)
|
||||
if t != '-':
|
||||
assert depth[-1] == t
|
||||
depth.pop(-1)
|
||||
|
|
@ -263,7 +264,7 @@ def main():
|
|||
# their old stderr.
|
||||
ack_fd = int(opt.ack_fd)
|
||||
assert ack_fd > 2
|
||||
if os.write(ack_fd, 'REDO-OK\n') != 8:
|
||||
if os.write(ack_fd, b'REDO-OK\n') != 8:
|
||||
raise Exception('write to ack_fd returned wrong length')
|
||||
os.close(ack_fd)
|
||||
queue += targets
|
||||
|
|
@ -274,9 +275,17 @@ def main():
|
|||
catlog(t)
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(200)
|
||||
except IOError, e:
|
||||
except IOError as e:
|
||||
if e.errno == errno.EPIPE:
|
||||
pass
|
||||
# this happens for example if calling `redo-log | head`, so stdout
|
||||
# is piped into another program that closes the pipe before reading
|
||||
# all our output.
|
||||
# from https://docs.python.org/3/library/signal.html#note-on-sigpipe:
|
||||
# Python flushes standard streams on exit; redirect remaining output
|
||||
# to devnull to avoid another BrokenPipeError at shutdown
|
||||
devnull = os.open(os.devnull, os.O_WRONLY)
|
||||
os.dup2(devnull, sys.stdout.fileno())
|
||||
sys.exit(141) # =128+13: "Terminated by SIGPIPE (signal 13)"
|
||||
else:
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""redo-ood: list out-of-date (ood) targets."""
|
||||
from __future__ import print_function
|
||||
import sys, os
|
||||
from . import deps, env, logs, state
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ def main():
|
|||
is_checked=is_checked,
|
||||
set_checked=set_checked,
|
||||
log_override=log_override):
|
||||
print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
|
||||
print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
from __future__ import print_function
|
||||
import sys, os, traceback
|
||||
from . import builder, env, helpers, jobserver, logs, options, state
|
||||
from .atoi import atoi
|
||||
|
|
@ -48,7 +49,7 @@ def main():
|
|||
|
||||
if opt.version:
|
||||
from . import version
|
||||
print version.TAG
|
||||
print(version.TAG)
|
||||
sys.exit(0)
|
||||
if opt.debug:
|
||||
os.environ['REDO_DEBUG'] = str(opt.debug or 0)
|
||||
|
|
@ -114,7 +115,7 @@ def main():
|
|||
finally:
|
||||
try:
|
||||
jobserver.force_return_tokens()
|
||||
except Exception, e: # pylint: disable=broad-except
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
traceback.print_exc(100, sys.stderr)
|
||||
err('unexpected error: %r\n' % e)
|
||||
retcode = 1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""redo-sources: list the known source (not target) files."""
|
||||
from __future__ import print_function
|
||||
import sys, os
|
||||
from . import env, logs, state
|
||||
|
||||
|
|
@ -16,7 +17,7 @@ def main():
|
|||
cwd = os.getcwd()
|
||||
for f in state.files():
|
||||
if f.is_source():
|
||||
print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
|
||||
print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""redo-targets: list the known targets (not sources)."""
|
||||
from __future__ import print_function
|
||||
import sys, os
|
||||
from . import env, logs, state
|
||||
|
||||
|
|
@ -16,7 +17,7 @@ def main():
|
|||
cwd = os.getcwd()
|
||||
for f in state.files():
|
||||
if f.is_target():
|
||||
print state.relpath(os.path.join(env.v.BASE, f.name), cwd)
|
||||
print(state.relpath(os.path.join(env.v.BASE, f.name), cwd))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ def main():
|
|||
relpath = os.path.relpath(dopath, '.')
|
||||
exists = os.path.exists(dopath)
|
||||
assert '\n' not in relpath
|
||||
print relpath
|
||||
print(relpath)
|
||||
if exists:
|
||||
sys.exit(0)
|
||||
sys.exit(1) # no appropriate dofile found
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ def isdirty(f, depth, max_changed,
|
|||
elif mode == 'm':
|
||||
sub = isdirty(f2, depth=depth + ' ',
|
||||
max_changed=max(f.changed_runid,
|
||||
f.checked_runid),
|
||||
f.checked_runid or 0),
|
||||
already_checked=already_checked,
|
||||
is_checked=is_checked,
|
||||
set_checked=set_checked,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def unlink(f):
|
|||
"""
|
||||
try:
|
||||
os.unlink(f)
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
pass # it doesn't exist, that's what you asked for
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ def _create_tokens(n):
|
|||
global _mytokens, _cheats
|
||||
assert n >= 0
|
||||
assert _cheats >= 0
|
||||
for _ in xrange(n):
|
||||
for _ in range(n):
|
||||
if _cheats > 0:
|
||||
_cheats -= 1
|
||||
else:
|
||||
|
|
@ -118,7 +118,7 @@ def _release(n):
|
|||
assert _mytokens >= n
|
||||
_debug('%d,%d -> release(%d)\n' % (_mytokens, _cheats, n))
|
||||
n_to_share = 0
|
||||
for _ in xrange(n):
|
||||
for _ in range(n):
|
||||
_mytokens -= 1
|
||||
if _cheats > 0:
|
||||
_cheats -= 1
|
||||
|
|
@ -128,7 +128,7 @@ def _release(n):
|
|||
assert _cheats >= 0
|
||||
if n_to_share:
|
||||
_debug('PUT tokenfds %d\n' % n_to_share)
|
||||
os.write(_tokenfds[1], 't' * n_to_share)
|
||||
os.write(_tokenfds[1], b't' * n_to_share)
|
||||
|
||||
|
||||
def _release_except_mine():
|
||||
|
|
@ -176,7 +176,7 @@ def _try_read(fd, n):
|
|||
signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01) # emergency fallback
|
||||
try:
|
||||
b = os.read(fd, 1)
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
if e.errno in (errno.EAGAIN, errno.EINTR):
|
||||
# interrupted or it was nonblocking
|
||||
return None # try again
|
||||
|
|
@ -189,7 +189,7 @@ def _try_read(fd, n):
|
|||
|
||||
|
||||
def _try_read_all(fd, n):
|
||||
bb = ''
|
||||
bb = b''
|
||||
while 1:
|
||||
b = _try_read(fd, n)
|
||||
if not b:
|
||||
|
|
@ -297,7 +297,7 @@ def _wait(want_token, max_delay):
|
|||
Returns:
|
||||
None
|
||||
"""
|
||||
rfds = _waitfds.keys()
|
||||
rfds = list(_waitfds.keys())
|
||||
if want_token:
|
||||
rfds.append(_tokenfds[0])
|
||||
assert rfds
|
||||
|
|
|
|||
|
|
@ -239,12 +239,12 @@ class Options:
|
|||
"""
|
||||
try:
|
||||
(flags,extra) = self.optfunc(args, self._shortopts, self._longopts)
|
||||
except getopt.GetoptError, e:
|
||||
except getopt.GetoptError as e:
|
||||
self.fatal(e)
|
||||
|
||||
opt = OptDict()
|
||||
|
||||
for k,v in self._defaults.iteritems():
|
||||
for k,v in self._defaults.items():
|
||||
k = self._aliases[k]
|
||||
opt[k] = v
|
||||
|
||||
|
|
@ -268,6 +268,6 @@ class Options:
|
|||
else:
|
||||
v = _intify(v)
|
||||
opt[k] = v
|
||||
for (f1,f2) in self._aliases.iteritems():
|
||||
for (f1,f2) in self._aliases.items():
|
||||
opt[f1] = opt._opts.get(f2)
|
||||
return (opt,flags,extra)
|
||||
|
|
|
|||
|
|
@ -46,14 +46,14 @@ def db():
|
|||
dbfile = '%s/db.sqlite3' % dbdir
|
||||
try:
|
||||
os.mkdir(dbdir)
|
||||
except OSError, e:
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST:
|
||||
pass # if it exists, that's okay
|
||||
else:
|
||||
raise
|
||||
|
||||
_lockfile = os.open(os.path.join(env.v.BASE, '.redo/locks'),
|
||||
os.O_RDWR | os.O_CREAT, 0666)
|
||||
os.O_RDWR | os.O_CREAT, 0o666)
|
||||
close_on_exec(_lockfile, True)
|
||||
if env.is_toplevel and detect_broken_locks():
|
||||
env.mark_locks_broken()
|
||||
|
|
@ -184,7 +184,11 @@ def relpath(t, base):
|
|||
base = os.path.normpath(_realdirpath(base))
|
||||
tparts = t.split('/')
|
||||
bparts = base.split('/')
|
||||
for tp, bp in zip(tparts, bparts):
|
||||
|
||||
# zip must not return an iterator in python 3, because the source lists of
|
||||
# the iterators are changed. The iterator does not notice that and ends too
|
||||
# soon.
|
||||
for tp, bp in list(zip(tparts, bparts)):
|
||||
if tp != bp:
|
||||
break
|
||||
tparts.pop(0)
|
||||
|
|
@ -278,7 +282,8 @@ class File(object):
|
|||
(self.id, self.name, self.is_generated, self.is_override,
|
||||
self.checked_runid, self.changed_runid, self.failed_runid,
|
||||
self.stamp, self.csum) = cols
|
||||
if self.name == ALWAYS and self.changed_runid < env.v.RUNID:
|
||||
if self.name == ALWAYS and (
|
||||
self.changed_runid is None or self.changed_runid < env.v.RUNID):
|
||||
self.changed_runid = env.v.RUNID
|
||||
|
||||
def __init__(self, fid=None, name=None, cols=None, allow_add=True):
|
||||
|
|
@ -508,7 +513,7 @@ class Lock(object):
|
|||
assert not self.owned
|
||||
try:
|
||||
fcntl.lockf(_lockfile, fcntl.LOCK_EX|fcntl.LOCK_NB, 1, self.fid)
|
||||
except IOError, e:
|
||||
except IOError as e:
|
||||
if e.errno in (errno.EAGAIN, errno.EACCES):
|
||||
pass # someone else has it locked
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
exec >&2
|
||||
for py in intentionally-missing python2.7 python2 python; do
|
||||
for py in intentionally-missing python python3 python2 python2.7; do
|
||||
echo "Trying: $py"
|
||||
cmd=$(command -v "$py" || true)
|
||||
# intentionally using the 'print statement' (as opposed to print
|
||||
# function) here, to rule out any python3 interpreters
|
||||
out=$($cmd -c 'print "success"' 2>/dev/null) || true
|
||||
out=$($cmd -c 'print("success")' 2>/dev/null) || true
|
||||
if [ "$out" = "success" ]; then
|
||||
echo $cmd >$3
|
||||
exit 0
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -22,7 +22,7 @@ def read(fname):
|
|||
setuptools.setup(
|
||||
name = 'redo-tools',
|
||||
version = redo.version.TAG.replace('-', '+', 1),
|
||||
python_requires='>=2.7, <3.0',
|
||||
python_requires='>=2.7',
|
||||
author = 'Avery Pennarun',
|
||||
author_email = 'apenwarr@gmail.com',
|
||||
description = ('djb redo: a recursive, general purpose build system.'),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
umask 0022
|
||||
redo mode1
|
||||
MODE=$(../../redo/py -c \
|
||||
'import os; print oct(os.stat("mode1").st_mode & 07777)')
|
||||
[ "$MODE" = "0644" ] || exit 78
|
||||
'import os; print(oct(os.stat("mode1").st_mode & 0o7777))')
|
||||
[ "$MODE" = "0644" -o "$MODE" = "0o644" ] || exit 78
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue