Directory reorg: move code into redo/, generate binaries in bin/.

It's time to start preparing for a version of redo that doesn't work
unless we build it first (because it will rely on C modules, and
eventually be rewritten in C altogether).

To get rolling, remove the old-style symlinks to the main programs, and
rename those programs from redo-*.py to redo/cmd_*.py.  We'll also move
all library functions into the redo/ dir, which is a more python-style
naming convention.

Previously, install.do was generating wrappers for installing in
/usr/bin, which extend sys.path and then import+run the right file.
This made "installed" redo work quite differently from running redo
inside its source tree.  Instead, let's always generate the wrappers in
bin/, and not make anything executable except those wrappers.

Since we're generating wrappers anyway, let's actually auto-detect the
right version of python for the running system; distros can't seem to
agree on what to call their python2 binaries (sigh). We'll fill in the
right #! shebang lines.  Since we're doing that, we can stop using
/usr/bin/env, which will a) make things slightly faster, and b) let us
use "python -S", which tells python not to load a bunch of extra crap
we're not using, thus improving startup times.

Annoyingly, we now have to build redo using minimal/do, then run the
tests using bin/redo.  To make this less annoying, we add a toplevel
./do script that knows the right steps, and a Makefile (whee!) for
people who are used to typing 'make' and 'make test' and 'make clean'.

redo/cmd_redo.py

@@ -0,0 +1,125 @@
+#
+# Copyright 2010-2018 Avery Pennarun and contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys, os, traceback
+import options
+from atoi import atoi
+
+optspec = """
+redo [targets...]
+--
+j,jobs=    maximum number of jobs to build at once
+d,debug    print dependency checks as they happen
+v,verbose  print commands as they are read from .do files (variables intact)
+x,xtrace   print commands as they are executed (variables expanded)
+k,keep-going  keep going as long as possible even if some targets fail
+shuffle    randomize the build order to find dependency bugs
+version    print the current version and exit
+
+ redo-log options:
+no-log     don't capture error output, just let it flow straight to stderr
+no-details only show 'redo' recursion trace (to see more later, use redo-log)
+no-status  don't display build summary line at the bottom of the screen
+no-pretty  don't pretty-print logs, show raw @@REDO output instead
+no-color   disable ANSI color; --color to force enable (default: auto)
+debug-locks  print messages about file locking (useful for debugging)
+debug-pids   print process ids as part of log messages (useful for debugging)
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+targets = extra
+
+if opt.version:
+    import version
+    print version.TAG
+    sys.exit(0)
+if opt.debug:
+    os.environ['REDO_DEBUG'] = str(opt.debug or 0)
+if opt.verbose:
+    os.environ['REDO_VERBOSE'] = '1'
+if opt.xtrace:
+    os.environ['REDO_XTRACE'] = '1'
+if opt.keep_going:
+    os.environ['REDO_KEEP_GOING'] = '1'
+if opt.shuffle:
+    os.environ['REDO_SHUFFLE'] = '1'
+if opt.debug_locks:
+    os.environ['REDO_DEBUG_LOCKS'] = '1'
+if opt.debug_pids:
+    os.environ['REDO_DEBUG_PIDS'] = '1'
+
+# This is slightly tricky: the log and pretty options default to true.  We
+# want to inherit that 'true' value from parent processes *unless* someone
+# explicitly specifies the reverse.
+if opt.no_log:
+    os.environ['REDO_LOG'] = '0'
+    if opt.no_pretty:
+        os.environ['REDO_PRETTY'] = '0'
+    if opt.no_color:
+        os.environ['REDO_COLOR'] = '0'
+
+import vars_init
+vars_init.init(targets)
+
+import vars, state, builder, jwack
+from logs import warn, err
+
+def main():
+    try:
+        j = atoi(opt.jobs or 1)
+        if vars_init.is_toplevel and (vars.LOG or j > 1):
+            builder.close_stdin()
+        if vars_init.is_toplevel and vars.LOG:
+            builder.start_stdin_log_reader(
+                status=opt.status, details=opt.details,
+                pretty=opt.pretty, color=opt.color,
+                debug_locks=opt.debug_locks, debug_pids=opt.debug_pids)
+        for t in targets:
+            if os.path.exists(t):
+                f = state.File(name=t)
+                if not f.is_generated:
+                    warn('%s: exists and not marked as generated; not redoing.\n'
+                         % f.nicename())
+        state.rollback()
+
+        if j < 1 or j > 1000:
+            err('invalid --jobs value: %r\n' % opt.jobs)
+        jwack.setup(j)
+        try:
+            assert state.is_flushed()
+            retcode = builder.main(targets, lambda t: (True, True))
+            assert state.is_flushed()
+        finally:
+            try:
+                state.rollback()
+            finally:
+                try:
+                    jwack.force_return_tokens()
+                except Exception, e:  # pylint: disable=broad-except
+                    traceback.print_exc(100, sys.stderr)
+                    err('unexpected error: %r\n' % e)
+                    retcode = 1
+        if vars_init.is_toplevel:
+            builder.await_log_reader()
+        sys.exit(retcode)
+    except KeyboardInterrupt:
+        if vars_init.is_toplevel:
+            builder.await_log_reader()
+        sys.exit(200)
+
+
+if __name__ == '__main__':
+    main()