From fb388b3ddeb84cb897673f3a22d3b314886716e8 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Tue, 21 Dec 2010 04:19:50 -0800 Subject: [PATCH 01/19] Automatically select a good shell instead of relying on /bin/sh. This includes a fairly detailed test of various known shell bugs from the autoconf docs. The idea here is that if redo works on your system, you should be able to rely on a *good* shell to run your .do files; you shouldn't have to work around zillions of bugs like autoconf does. --- .gitignore | 1 + _all.do | 2 + all.do | 2 +- builder.py | 3 +- clean.do | 2 +- install.do | 6 +- redo-sh.do | 45 +++++++++ t/.gitignore | 2 + t/clean.do | 2 +- t/shelltest.do | 8 ++ t/shelltest.od | 261 +++++++++++++++++++++++++++++++++++++++++++++++++ t/test.do | 2 +- test.do | 2 +- vars_init.py | 13 ++- 14 files changed, 341 insertions(+), 10 deletions(-) create mode 100644 _all.do create mode 100644 redo-sh.do create mode 100644 t/shelltest.do create mode 100644 t/shelltest.od diff --git a/.gitignore b/.gitignore index d0aea09..4a9ab4a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ t/LD t/[yb]ellow t/hello t/*.o +/redo-sh diff --git a/_all.do b/_all.do new file mode 100644 index 0000000..6623acb --- /dev/null +++ b/_all.do @@ -0,0 +1,2 @@ +redo-ifchange redo-sh +redo-ifchange Documentation/all diff --git a/all.do b/all.do index 340cd9f..af2187f 100644 --- a/all.do +++ b/all.do @@ -1,2 +1,2 @@ -redo-ifchange Documentation/all +redo-ifchange _all echo "Nothing much to do. Try 'redo t/all' or 'redo test'" >&2 diff --git a/builder.py b/builder.py index 3d942a7..81a6c5c 100644 --- a/builder.py +++ b/builder.py @@ -197,7 +197,8 @@ class BuildJob: after_t = _try_stat(t) st1 = os.fstat(f.fileno()) st2 = _try_stat(self.tmpname2) - if after_t != before_t and not stat.S_ISDIR(after_t.st_mode): + if (after_t and after_t != before_t and + not stat.S_ISDIR(after_t.st_mode)): err('%s modified %s directly!\n' % (self.argv[2], t)) err('...you should update $3 (a temp file) or stdout, not $1.\n') rv = 206 diff --git a/clean.do b/clean.do index 7753332..ec75b2a 100644 --- a/clean.do +++ b/clean.do @@ -6,5 +6,5 @@ fi [ -z "$DO_BUILT" ] && rm -rf .do_built .do_built.dir redo t/clean Documentation/clean rm -f *~ .*~ */*~ */.*~ *.pyc install.wrapper -rm -rf t/.redo +rm -rf t/.redo redo-sh find . -name '*.tmp' -exec rm -fv {} \; diff --git a/install.do b/install.do index 3703131..47889ae 100644 --- a/install.do +++ b/install.do @@ -1,5 +1,5 @@ exec >&2 -redo-ifchange Documentation/all +redo-ifchange _all : ${INSTALL:=install} : ${DESTDIR:=} @@ -28,6 +28,10 @@ for d in *.py; do done python -mcompileall $LIBDIR +# It's important for the file to actually be named 'sh'. Some shells (like +# bash and zsh) only go into POSIX-compatible mode if they have that name. +cp -R redo-sh/sh $LIBDIR/sh + # binaries for dd in redo*.py; do d=$(basename $dd .py) diff --git a/redo-sh.do b/redo-sh.do new file mode 100644 index 0000000..9ca1879 --- /dev/null +++ b/redo-sh.do @@ -0,0 +1,45 @@ +exec >&2 +redo-ifchange t/shelltest.od + +rm -rf $1.new $1/sh +mkdir $1.new + +GOOD= +WARN= + +for sh in dash sh ash ksh pdksh bash zsh busybox; do + printf "Testing %s... " "$sh" + FOUND=`which $sh` || { echo "missing"; continue; } + + # It's important for the file to actually be named 'sh'. Some shells (like + # bash and zsh) only go into POSIX-compatible mode if they have that name. + # If they're not in POSIX-compatible mode, they'll fail the test. + rm -f $1.new/sh + ln -s $FOUND $1.new/sh + + set +e + ( cd t && ../$1.new/sh shelltest.od >/dev/null 2>&1 ) + RV=$? + set -e + + case $RV in + 0) echo "good"; GOOD=$FOUND; break ;; + 42) echo "warnings"; [ -n "$WARN" ] || WARN=$FOUND ;; + *) echo "failed" ;; + esac +done + +rm -rf $1 $1.new $3 + +if [ -n "$GOOD" ]; then + echo "Selected perfect shell: $GOOD" + mkdir $3 + ln -s $GOOD $3/sh +elif [ -n "$WARN" ]; then + echo "Selected mostly good shell: $WARN" + mkdir $3 + ln -s $WARN $3/sh +else + echo "No good shells found! Maybe install dash, bash, or zsh." + exit 1 +fi diff --git a/t/.gitignore b/t/.gitignore index 60b3c7a..375c938 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -21,3 +21,5 @@ test2.args /ifcreate[12].log /ifcreate[12].dep /ifcreate[12] +/broken +/shellfile diff --git a/t/clean.do b/t/clean.do index 3062fb3..1b3b8b4 100644 --- a/t/clean.do +++ b/t/clean.do @@ -1,6 +1,6 @@ redo example/clean curse/clean deps/clean "space dir/clean" stamp/clean \ defaults-flat/clean -rm -f mode1 makedir.log chdir1 deltest2 \ +rm -f broken shellfile mode1 makedir.log chdir1 deltest2 \ hello [by]ellow *.o *~ .*~ *.log CC LD passfail silence silence.do \ touch1 touch1.do always1 ifcreate[12].dep ifcreate[12] rm -rf makedir diff --git a/t/shelltest.do b/t/shelltest.do new file mode 100644 index 0000000..5c65254 --- /dev/null +++ b/t/shelltest.do @@ -0,0 +1,8 @@ +set +e +( . ./shelltest.od ) +RV=$? +case $RV in + 0) exit 0 ;; + 42) exit 0 ;; + *) exit 1 ;; +esac diff --git a/t/shelltest.od b/t/shelltest.od new file mode 100644 index 0000000..e4cfc1a --- /dev/null +++ b/t/shelltest.od @@ -0,0 +1,261 @@ +# +# Most of these tests were inspired by: +# http://www.gnu.org/software/hello/manual/autoconf/Shell-Substitutions.html +# +exec >&2 +set +e +FAIL= +fail() +{ + echo " failed: $1" + FAIL=41 +} +warn() +{ + echo " warning: $1" + [ -n "$FAIL" ] || FAIL=42 +} + + +name=foo.o.o +ext=.o +[ "${name#foo.o}" = ".o" ] || fail 3 + + +spacey="this has * and spaces" +case $spacey in + *) spaceout=$name$spacey ;; +esac +[ "$spaceout" = "$name$spacey" ] || fail 4 + + +n() { echo "$#$@"; } +f=" - " +out=$(n - ""$f"" -) +[ "$out" = "5- - -" ] || warn 5 + + +n1() { echo $#; } +n2() { n1 "$@"; } +t1=$(n1) +t2=$(n2) +[ "$t1" = "0" ] || fail 6 +[ "$t2" = "0" ] || fail 7 + + +n1() { for i in "$@"; do echo $i; done; } +n2() { for i in ${1+"$@"}; do echo $i; done; } +t1=$(n1 "Hello World" "!") +t2=$(n2 "Hello World" "!") +WANT="Hello World +!" +[ "$t1" = "$WANT" ] || fail 8 +[ "$t2" = "$WANT" ] || fail 9 + + +n() { echo ${10}; } +t1=$(n 1 2 3 4 5 6 7 8 9 xx yy) +[ "$t1" = "xx" ] || fail 10 + + +chicken1=`echo " $spacey" | sed s/a/./g` +chicken2="`echo " $spacey" | sed s/a/./g`" +chicken3=$(echo " $spacey" | sed s/a/./g) +chicken4="$(echo " $spacey" | sed s/a/./g)" +[ "$chicken1" = " this h.s * .nd sp.ces" ] || fail 11 +[ "$chicken2" = " this h.s * .nd sp.ces" ] || fail 12 +[ "$chicken3" = " this h.s * .nd sp.ces" ] || fail 13 +[ "$chicken4" = " this h.s * .nd sp.ces" ] || fail 14 + + +f1= +f2=goo +g1= +g2=goo +out=$(echo ${f1:-foo} ${f2:-foo} ${g1:=foo} ${g2:=foo}) +: ${f1:-roo} ${f2:-roo} ${g1:=roo} ${g2:=roo} +[ "$out" = "foo goo foo goo" ] || fail 16 +[ "$f1$f2$g1$g2" = "gooroogoo" ] || fail 17 + + +unset a +t1=$(echo ${a-b c}) +t2=$(echo ${a-'b c'}) +t3=$(echo "${a-b c}") +t4=$(echo "${a-"b c"}") +t5=$(cat <broken +echo "`printf 'foo\r\n'`"" bar" | diff -q - broken || fail 59 + + +# +# This one is too obnoxious. dash and ash pass the test, but nothing else does, +# and this case is just too dumb to care about. Just don't do that! +# +#t=`echo $(case x in x) echo hello;; esac)` +#[ "$t" = "hello" ] || fail 60 + + +x=5 +t1=$(($x + 4)) +t2=$(echo $(( 010 + 0x10 ))) +[ "$t1" = "9" ] || fail 61 +[ "$t2" = "24" ] || fail 62 + + +t=$(echo hello ^ cat) +[ "$t" = "hello ^ cat" ] || fail 65 + + +t1=$(for d in this-glob-does-*-not-exist; do echo "$d"; done) +t2=$(for d in this-glob-does-*-not-exist; do echo "$d"; done) + + +# http://www.gnu.org/software/hello/manual/autoconf/Assignments.html +false || foo=bar; [ "$?" = 0 ] || fail 71 +foo=`exit 1`; [ "$?" != 0 ] || fail 72 + +# http://www.gnu.org/software/hello/manual/autoconf/Shell-Functions.html +f1() { echo 1; } +f2(){ echo 2;} +f3()(echo 3) +f4()if true; then echo 4; fi +f5() ( exit 5 ) +[ "$(f1)" = 1 ] || fail 81 +[ "$(f2)" = 2 ] || fail 82 +[ "$(f3)" = 3 ] || fail 83 +[ "$(f4)" = 4 ] || fail 84 +f5 && fail 85 +f6() ( + f6b() { return 1; } + set -e + f6b + fail 86 +) +f6 +f7() { :; }; f7=; f7 || fail 87 +a= +f8() { echo $a; }; +t8=$(a=1 f8) +[ "$t8" = "1" ] || fail 88 + + +# http://www.gnu.org/software/hello/manual/autoconf/Limitations-of-Builtins.html +. /dev/null || fail 90 +(! : | :) && fail 91 || true +(! { :; }) && fail 92 || true +t3=none +case frog.c in + (*.c) t3=c ;; + (*) t3=all ;; +esac +[ "$t3" = "c" ] || fail 93 +t4=$(echo '\n' | wc -l) +[ "$t4" -eq 1 ] || warn 94 +f5() { + for arg; do + echo $arg + done +} +t5=$(f5 a=5 b c) +[ "$t5" = "a=5 +b +c" ] || fail 95 +t6=$(printf -- '%d %d' 5 6) +[ "$t6" = "5 6" ] || fail 96 +echo 'word\ game stuff' >shellfile +read t7a t7b &2 -redo deltest deltest2 test.args test2.args passfailtest chdirtest \ +redo shelltest deltest deltest2 test.args test2.args passfailtest chdirtest \ curse/test deps/test "space dir/test" modetest makedir2 \ silencetest touchtest stamp/test alwaystest ifcreate-test diff --git a/test.do b/test.do index 4744d97..1c272d4 100644 --- a/test.do +++ b/test.do @@ -1,2 +1,2 @@ -redo-ifchange Documentation/all +redo-ifchange _all redo t/test diff --git a/vars_init.py b/vars_init.py index 1c78ca4..75a813b 100644 --- a/vars_init.py +++ b/vars_init.py @@ -5,10 +5,17 @@ def init(targets): # toplevel call to redo exenames = [os.path.abspath(sys.argv[0]), os.path.realpath(sys.argv[0])] - if exenames[0] == exenames[1]: - exenames = [exenames[0]] dirnames = [os.path.dirname(p) for p in exenames] - os.environ['PATH'] = ':'.join(dirnames) + ':' + os.environ['PATH'] + trynames = ([os.path.abspath(p+'/../lib/redo') for p in dirnames] + + [p+'/redo-sh' for p in dirnames] + + dirnames) + seen = {} + dirs = [] + for k in trynames: + if not seen.get(k) and os.path.exists('%s/.' % k): + seen[k] = 1 + dirs.append(k) + os.environ['PATH'] = ':'.join(dirs) + ':' + os.environ['PATH'] os.environ['REDO'] = os.path.abspath(sys.argv[0]) if not os.environ.get('REDO_BASE'): From 41ef15fde21dbdcbb228148b0b6fb8ae8cdd7d5c Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 03:14:32 -0800 Subject: [PATCH 02/19] minimal/do: use posix shell features instead of dirname/basename. This avoids a few forks, and is a good example of how to do some "modern" sh programming. Plus we now use fewer lines of code. --- minimal/do | 84 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/minimal/do b/minimal/do index 1ca7e04..9120704 100755 --- a/minimal/do +++ b/minimal/do @@ -6,18 +6,25 @@ # The author disclaims copyright to this source file and hereby places it in # the public domain. (2010 12 14) # -export REDO="$(cd "$(dirname "$0")" && echo "$PWD/$(basename "$0")")" +_dirsplit() +{ + base=${1##*/} + dir=${1%$base} +} + +_dirsplit "$0" +export REDO=$(cd "${dir:-.}" && echo "$PWD/$base") if [ -z "$DO_BUILT" ]; then - export DO_BUILT="$PWD/.do_built" + export DO_BUILT=$PWD/.do_built if [ -e "$DO_BUILT" ]; then echo "Removing previously built files..." >&2 sort -u "$DO_BUILT" | tee "$DO_BUILT.new" | - while read f; do rm -f "$f"; done + while read f; do rm -f "$f" 2>/dev/null; done mv "$DO_BUILT.new" "$DO_BUILT" fi - DO_PATH="$DO_BUILT.dir" - export PATH="$DO_PATH:$PATH" + DO_PATH=$DO_BUILT.dir + export PATH=$DO_PATH:$PATH rm -rf "$DO_PATH" mkdir "$DO_PATH" for d in redo redo-ifchange; do @@ -30,56 +37,53 @@ if [ -z "$DO_BUILT" ]; then fi -_dirsplit() +_find_dofile() { - OLDIFS="$IFS" - IFS=/ - set -- $1 - IFS="$OLDIFS" - dir="" - while [ $# -gt 1 ]; do - dir="$dir$1/" - shift + DOFILE=default.$1.do + while :; do + DOFILE=default.${DOFILE#default.*.} + [ -e "$DOFILE" -o "$DOFILE" = default.do ] && break done - base="$1" + EXT=${DOFILE#default} + EXT=${EXT%.do} + BASE=${1%$EXT} +} + + +_run_dofile() +{ + export DO_DEPTH="$DO_DEPTH " + export REDO_TARGET=$PWD/$TARGET + set -e + . "$PWD/$DOFILE" >"$TARGET.tmp" } _do() { - DIR="$1" - TARGET="$2" + DIR=$1 + TARGET=$2 if [ ! -e "$TARGET" ]; then printf '\033[32mdo %s\033[1m%s\033[m\n' \ "$DO_DEPTH" "$DIR$TARGET" >&2 echo "$PWD/$TARGET" >>"$DO_BUILT" - dof=".$TARGET" - DOFILE="$TARGET.do" - BASE="$TARGET" - EXT="" - while [ ! -e "$DOFILE" ]; do - dof2=$(echo "$dof" | sed 's/\.[^\.]*//') - [ "$dof" = "$dof2" ] && break - dof="$dof2" - DOFILE="default$dof.do" - BASE="$(basename "$TARGET" "$dof")" - EXT="$dof" - done - set "$BASE" "$EXT" "$TARGET.tmp" - RV= - ( - export DO_DEPTH="$DO_DEPTH " - export REDO_TARGET="$PWD/$TARGET" - set -e - . "$PWD/$DOFILE" >"$TARGET.tmp" - ) || RV="$?" - [ -z "$RV" ] && mv "$TARGET.tmp" "$TARGET" 2>/dev/null - : >>"$TARGET" - if [ -n "$RV" ]; then + DOFILE=$TARGET.do + BASE=$TARGET + EXT= + [ -e "$TARGET.do" ] || _find_dofile "$TARGET" + if [ ! -e "$DOFILE" ]; then + echo "do: $TARGET: no .do file" >&2 + return 1 + fi + ( _run_dofile "$BASE" "$EXT" "$TARGET.tmp" ) + RV=$? + if [ $RV != 0 ]; then printf "do: %s%s\n" "$DO_DEPTH" \ "$DIR$TARGET: got exit code $RV" >&2 return $RV fi + mv "$TARGET.tmp" "$TARGET" 2>/dev/null + : >>"$TARGET" else echo "do $DO_DEPTH$TARGET exists." >&2 fi From fb5275938d911e757d35c5084f299c039e238cd9 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 03:42:35 -0800 Subject: [PATCH 03/19] minimal/do: use ".did" stamp files instead of empty target files. If all.do runs and creates no output, we shouldn't create a file called 'all', but we should remember that 'all' has been run successfully. We do this by creating 'all.did' during the build. Since minimal/do always just wipes everything out every time it runs, we can safely remove the .did files after minimal/do terminates, so this doesn't clutter things too much in normal use. This fixes some edge cases, particularly that 'minimal/do clean' no longer leaves stupid files named "clean" lying around, and the redo-sh directory can now be rebuilt correctly since we rebuild it as long as redo-sh.did doesn't exist. (We don't want to "rm -rf redo-sh" because it makes me nervous.) --- clean.do | 2 +- minimal/do | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/clean.do b/clean.do index ec75b2a..938703b 100644 --- a/clean.do +++ b/clean.do @@ -1,3 +1,4 @@ +rm -rf t/.redo redo-sh if [ -e .do_built ]; then while read x; do rm -f "$x" @@ -6,5 +7,4 @@ fi [ -z "$DO_BUILT" ] && rm -rf .do_built .do_built.dir redo t/clean Documentation/clean rm -f *~ .*~ */*~ */.*~ *.pyc install.wrapper -rm -rf t/.redo redo-sh find . -name '*.tmp' -exec rm -fv {} \; diff --git a/minimal/do b/minimal/do index 9120704..44045ef 100755 --- a/minimal/do +++ b/minimal/do @@ -15,12 +15,14 @@ _dirsplit() _dirsplit "$0" export REDO=$(cd "${dir:-.}" && echo "$PWD/$base") +DO_TOP= if [ -z "$DO_BUILT" ]; then + DO_TOP=1 export DO_BUILT=$PWD/.do_built if [ -e "$DO_BUILT" ]; then echo "Removing previously built files..." >&2 sort -u "$DO_BUILT" | tee "$DO_BUILT.new" | - while read f; do rm -f "$f" 2>/dev/null; done + while read f; do rm -f "$f" "$f.did" 2>/dev/null; done mv "$DO_BUILT.new" "$DO_BUILT" fi DO_PATH=$DO_BUILT.dir @@ -55,7 +57,7 @@ _run_dofile() export DO_DEPTH="$DO_DEPTH " export REDO_TARGET=$PWD/$TARGET set -e - . "$PWD/$DOFILE" >"$TARGET.tmp" + . "$PWD/$DOFILE" >"$TARGET.tmp2" } @@ -63,7 +65,7 @@ _do() { DIR=$1 TARGET=$2 - if [ ! -e "$TARGET" ]; then + if [ ! -e "$TARGET" ] || [ -e "$TARGET/." -a ! -e "$TARGET.did" ]; then printf '\033[32mdo %s\033[1m%s\033[m\n' \ "$DO_DEPTH" "$DIR$TARGET" >&2 echo "$PWD/$TARGET" >>"$DO_BUILT" @@ -75,6 +77,7 @@ _do() echo "do: $TARGET: no .do file" >&2 return 1 fi + : >>"$TARGET.did" ( _run_dofile "$BASE" "$EXT" "$TARGET.tmp" ) RV=$? if [ $RV != 0 ]; then @@ -82,8 +85,10 @@ _do() "$DIR$TARGET: got exit code $RV" >&2 return $RV fi - mv "$TARGET.tmp" "$TARGET" 2>/dev/null - : >>"$TARGET" + mv "$TARGET.tmp" "$TARGET" 2>/dev/null || + ! test -s "$TARGET.tmp2" || + mv "$TARGET.tmp2" "$TARGET" 2>/dev/null + rm -f "$TARGET.tmp2" else echo "do $DO_DEPTH$TARGET exists." >&2 fi @@ -105,3 +110,8 @@ if [ -n "$*" ]; then else redo all fi + +if [ -n "$DO_TOP" ]; then + echo "Removing stamp files..." >&2 + while read f; do rm -f "$f.did" 2>/dev/null; done <"$DO_BUILT" +fi From 8aa27d050b56956abb12426f91ace69526fefcb6 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 04:07:45 -0800 Subject: [PATCH 04/19] minimal/do: delete .tmp files if a build fails. --- minimal/do | 1 + 1 file changed, 1 insertion(+) diff --git a/minimal/do b/minimal/do index 44045ef..f22c7f8 100755 --- a/minimal/do +++ b/minimal/do @@ -83,6 +83,7 @@ _do() if [ $RV != 0 ]; then printf "do: %s%s\n" "$DO_DEPTH" \ "$DIR$TARGET: got exit code $RV" >&2 + rm -f "$TARGET.tmp" "$TARGET.tmp2" return $RV fi mv "$TARGET.tmp" "$TARGET" 2>/dev/null || From 7142c342aed30f8b7f2e9e218f5d70cd5688e04a Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 04:07:57 -0800 Subject: [PATCH 05/19] minimal/do: faster deletion of stamp files. "while/read/printf | xargs -0" is much faster than while/read/rm, because it doesn't fork so many times. --- minimal/do | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/minimal/do b/minimal/do index f22c7f8..c5302bd 100755 --- a/minimal/do +++ b/minimal/do @@ -22,7 +22,8 @@ if [ -z "$DO_BUILT" ]; then if [ -e "$DO_BUILT" ]; then echo "Removing previously built files..." >&2 sort -u "$DO_BUILT" | tee "$DO_BUILT.new" | - while read f; do rm -f "$f" "$f.did" 2>/dev/null; done + while read f; do printf "%s\0%s.did\0" "$f" "$f"; done | + xargs -0 rm -f 2>/dev/null mv "$DO_BUILT.new" "$DO_BUILT" fi DO_PATH=$DO_BUILT.dir @@ -114,5 +115,6 @@ fi if [ -n "$DO_TOP" ]; then echo "Removing stamp files..." >&2 - while read f; do rm -f "$f.did" 2>/dev/null; done <"$DO_BUILT" + while read f; do printf "%s.did\0" "$f"; done <"$DO_BUILT" | + xargs -0 rm -f 2>/dev/null fi From fa4b64285d02ac12cafbb753b608321dbfcd961e Mon Sep 17 00:00:00 2001 From: Zoran Zaric Date: Sat, 1 Jan 2011 17:15:16 -0800 Subject: [PATCH 06/19] Sample bash completion rules for redo targets. --- contrib/bash_completion.d/redo | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 contrib/bash_completion.d/redo diff --git a/contrib/bash_completion.d/redo b/contrib/bash_completion.d/redo new file mode 100644 index 0000000..796e7b8 --- /dev/null +++ b/contrib/bash_completion.d/redo @@ -0,0 +1,12 @@ +_redo() +{ + local cur + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + + local targets=$(for t in `ls *.do | sed 's/\.do$//'`; do echo ${t}; done) + + COMPREPLY=($(compgen -W "${targets}" $cur)) +} + +complete -F _redo redo From d5beda0ffe622d6529a5d2082993c50b15799e5f Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 17:23:51 -0800 Subject: [PATCH 07/19] bash completions: work correctly with subdirs, ie. 'redo t/' --- contrib/bash_completion.d/redo | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/bash_completion.d/redo b/contrib/bash_completion.d/redo index 796e7b8..c3c7612 100644 --- a/contrib/bash_completion.d/redo +++ b/contrib/bash_completion.d/redo @@ -1,12 +1,12 @@ -_redo() +_redo_completions() { - local cur - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - - local targets=$(for t in `ls *.do | sed 's/\.do$//'`; do echo ${t}; done) - - COMPREPLY=($(compgen -W "${targets}" $cur)) + local cur="${COMP_WORDS[COMP_CWORD]}" + local targets=$( + for t in $cur*.do; do + echo "${t%.do}" + done + ) + COMPREPLY=($(compgen -W "$targets" $cur)) } -complete -F _redo redo +complete -F _redo_completions redo From 70923c5f303600d7fcf033330a3f72a77878a08e Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 21:11:29 -0800 Subject: [PATCH 08/19] bash completions: call redo-targets for a more complete list. Also fix handling of filenames needing quoting (like "t/space dir"), directories, and targets named after directories. --- contrib/bash_completion.d/redo | 38 +++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/contrib/bash_completion.d/redo b/contrib/bash_completion.d/redo index c3c7612..038f586 100644 --- a/contrib/bash_completion.d/redo +++ b/contrib/bash_completion.d/redo @@ -1,12 +1,40 @@ -_redo_completions() +__find_redo_targets() +{ + local IFS=$'\n:' + for d in . ../.. $PATH; do + if [ -x "$d/redo-targets" ]; then + ( cd "$d" && echo "$PWD/redo-targets" ) + break + fi + done +} +__redo_targets=$(__find_redo_targets) + + +__redo_completions() { local cur="${COMP_WORDS[COMP_CWORD]}" + local IFS=$'\n' local targets=$( - for t in $cur*.do; do - echo "${t%.do}" + # targets already known to redo + [ -x "$__redo_targets" ] && + "$__redo_targets" | + while read name; do + rest=${name#$cur} + [ "$rest" = "$name" ] && continue + name2="$cur${rest%%/*}" + [ -e "$name2/." ] || echo "$name2" + done + + # targets named explicitly by .do files + compgen -o default "$cur" | + while read name; do + local don=${name%.do} def=${name#default.} + [ "$don" = "$name" -o "$def" != "$name" ] && continue + echo "${name%.do}" done ) - COMPREPLY=($(compgen -W "$targets" $cur)) + COMPREPLY=($(compgen -W "$targets" "$cur")) } -complete -F _redo_completions redo +complete -F __redo_completions -o plusdirs -o filenames redo From 3c256dee9bc5c5ce75a5606b20420f48266940a8 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 21:57:29 -0800 Subject: [PATCH 09/19] bash completions: work correctly when $cur is an empty string. Otherwise 'cd t/defaults-flat' then 'redo ' doesn't show all the possible targets. --- contrib/bash_completion.d/redo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bash_completion.d/redo b/contrib/bash_completion.d/redo index 038f586..b39ae22 100644 --- a/contrib/bash_completion.d/redo +++ b/contrib/bash_completion.d/redo @@ -21,7 +21,7 @@ __redo_completions() "$__redo_targets" | while read name; do rest=${name#$cur} - [ "$rest" = "$name" ] && continue + [ "$cur$rest" != "$name" ] && continue name2="$cur${rest%%/*}" [ -e "$name2/." ] || echo "$name2" done From 96347f3d8358afe71dbb98ddde15f02285ccb95b Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 21:59:24 -0800 Subject: [PATCH 10/19] bash completions: also mark 'do' as a completable command. (ie. minimal/do) --- contrib/bash_completion.d/redo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bash_completion.d/redo b/contrib/bash_completion.d/redo index b39ae22..46c3621 100644 --- a/contrib/bash_completion.d/redo +++ b/contrib/bash_completion.d/redo @@ -37,4 +37,4 @@ __redo_completions() COMPREPLY=($(compgen -W "$targets" "$cur")) } -complete -F __redo_completions -o plusdirs -o filenames redo +complete -F __redo_completions -o plusdirs -o filenames redo do From f24d4f142b3845fd939f5895882c8ec7934b14f2 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 22:01:50 -0800 Subject: [PATCH 11/19] minimal/do: don't print an error on exit if we don't build anything. --- minimal/do | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/minimal/do b/minimal/do index c5302bd..198e8fe 100755 --- a/minimal/do +++ b/minimal/do @@ -19,13 +19,12 @@ DO_TOP= if [ -z "$DO_BUILT" ]; then DO_TOP=1 export DO_BUILT=$PWD/.do_built - if [ -e "$DO_BUILT" ]; then - echo "Removing previously built files..." >&2 - sort -u "$DO_BUILT" | tee "$DO_BUILT.new" | - while read f; do printf "%s\0%s.did\0" "$f" "$f"; done | - xargs -0 rm -f 2>/dev/null - mv "$DO_BUILT.new" "$DO_BUILT" - fi + : >>"$DO_BUILT" + echo "Removing previously built files..." >&2 + sort -u "$DO_BUILT" | tee "$DO_BUILT.new" | + while read f; do printf "%s\0%s.did\0" "$f" "$f"; done | + xargs -0 rm -f 2>/dev/null + mv "$DO_BUILT.new" "$DO_BUILT" DO_PATH=$DO_BUILT.dir export PATH=$DO_PATH:$PATH rm -rf "$DO_PATH" From f641e52e3ba76c190bc19f1cc2373420c3c030af Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sat, 1 Jan 2011 22:00:14 -0800 Subject: [PATCH 12/19] Handle .do files that start with "#!/" to specify an explicit interpreter. Now you can have your .do files interpreted by whatever interpreter you want. --- README.md | 29 +++++++++++++++++++++-------- builder.py | 3 +++ minimal/do | 8 +++++++- t/.gitignore | 1 + t/clean.do | 2 +- t/nonshelltest.do | 3 +++ t/test.do | 3 ++- 7 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 t/nonshelltest.do diff --git a/README.md b/README.md index ce655c0..4c26d50 100644 --- a/README.md +++ b/README.md @@ -437,15 +437,28 @@ changed, you can run `redo-ifchange target` instead. # Can my .do files be written in a language other than sh? -FIXME: Not presently. In theory, we could support starting your .do files -with the magic "#!/" sequence (eg. #!/usr/bin/python) and then using that -shell to run your .do script. But that opens new problems, like figuring -out what is the equivalent of the `redo-ifchange` command in python. Do you -just run it in a subprocess? That might be unnecessarily slow. And so on. +Yes. If the first line of your .do file starts with the +magic "#!/" sequence (eg. `#!/usr/bin/python`), then redo +will execute your script using that particular interpreter. -Right now, `redo` explicitly runs `sh filename.do`. The main reasons for -this are to make the #!/ line optional, and so you don't have to remember to -`chmod +x` your .do files. +Note that this is slightly different from normal Unix +execution semantics: redo never execs your script directly; +it only looks for the "#!/" line. The main reason for this +is so that your .do scripts don't have to be marked +executable (chmod +x). Executable .do scripts would +suggest to users that they should run them directly, and +they shouldn't; .do scripts should always be executed +inside an instance of redo, so that dependencies can be +tracked correctly. + +WARNING: If your .do script *is* written in Unix sh, we +recommend *not* including the `#!/bin/sh` line. That's +because there are many variations of /bin/sh, and not all +of them are POSIX compliant. redo tries pretty hard to +find a good default shell that will be "as POSIXy as +possible," and if you override it using #!/bin/sh, you lose +this benefit and you'll have to worry more about +portability. # Can a single .do script generate multiple outputs? diff --git a/builder.py b/builder.py index 81a6c5c..5527693 100644 --- a/builder.py +++ b/builder.py @@ -128,6 +128,9 @@ class BuildJob: if vars.VERBOSE: argv[1] += 'v' if vars.XTRACE: argv[1] += 'x' if vars.VERBOSE or vars.XTRACE: log_('\n') + firstline = open(dofile).readline().strip() + if firstline.startswith('#!/'): + argv[0:2] = firstline[2:].split(' ') log('%s\n' % _nice(t)) self.argv = argv sf.is_generated = True diff --git a/minimal/do b/minimal/do index 198e8fe..30d740c 100755 --- a/minimal/do +++ b/minimal/do @@ -57,7 +57,13 @@ _run_dofile() export DO_DEPTH="$DO_DEPTH " export REDO_TARGET=$PWD/$TARGET set -e - . "$PWD/$DOFILE" >"$TARGET.tmp2" + read line1 <"$PWD/$DOFILE" + cmd=${line1#"#!/"} + if [ "$cmd" != "$line1" ]; then + /$cmd "$PWD/$DOFILE" "$@" >"$TARGET.tmp2" + else + . "$PWD/$DOFILE" >"$TARGET.tmp2" + fi } diff --git a/t/.gitignore b/t/.gitignore index 375c938..1ff112f 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -23,3 +23,4 @@ test2.args /ifcreate[12] /broken /shellfile +/nonshelltest diff --git a/t/clean.do b/t/clean.do index 1b3b8b4..d845e1d 100644 --- a/t/clean.do +++ b/t/clean.do @@ -1,6 +1,6 @@ redo example/clean curse/clean deps/clean "space dir/clean" stamp/clean \ defaults-flat/clean -rm -f broken shellfile mode1 makedir.log chdir1 deltest2 \ +rm -f broken nonshelltest shellfile mode1 makedir.log chdir1 deltest2 \ hello [by]ellow *.o *~ .*~ *.log CC LD passfail silence silence.do \ touch1 touch1.do always1 ifcreate[12].dep ifcreate[12] rm -rf makedir diff --git a/t/nonshelltest.do b/t/nonshelltest.do new file mode 100644 index 0000000..fc8a02d --- /dev/null +++ b/t/nonshelltest.do @@ -0,0 +1,3 @@ +#!/usr/bin/env perl +$a="perly"; +print "hello $a world\n"; diff --git a/t/test.do b/t/test.do index 187fa64..65538e3 100644 --- a/t/test.do +++ b/t/test.do @@ -1,5 +1,6 @@ redo-ifchange all ./hello >&2 -redo shelltest deltest deltest2 test.args test2.args passfailtest chdirtest \ +redo nonshelltest shelltest \ + deltest deltest2 test.args test2.args passfailtest chdirtest \ curse/test deps/test "space dir/test" modetest makedir2 \ silencetest touchtest stamp/test alwaystest ifcreate-test From f2ef945700089eef55b948b4cb77f4eae48d077e Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sun, 2 Jan 2011 11:49:34 -0800 Subject: [PATCH 13/19] redo-sh.do: wrap long lines. --- redo-sh.do | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/redo-sh.do b/redo-sh.do index 9ca1879..189a43a 100644 --- a/redo-sh.do +++ b/redo-sh.do @@ -11,9 +11,10 @@ for sh in dash sh ash ksh pdksh bash zsh busybox; do printf "Testing %s... " "$sh" FOUND=`which $sh` || { echo "missing"; continue; } - # It's important for the file to actually be named 'sh'. Some shells (like - # bash and zsh) only go into POSIX-compatible mode if they have that name. - # If they're not in POSIX-compatible mode, they'll fail the test. + # It's important for the file to actually be named 'sh'. Some + # shells (like bash and zsh) only go into POSIX-compatible mode if + # they have that name. If they're not in POSIX-compatible mode, + # they'll fail the test. rm -f $1.new/sh ln -s $FOUND $1.new/sh From d55e329018e24d2e8506382b752b10b8ada20d73 Mon Sep 17 00:00:00 2001 From: Henry Gebhardt Date: Sun, 2 Jan 2011 11:49:51 -0800 Subject: [PATCH 14/19] redo-sh.do: hide warning output from 'which' in some shells. --- redo-sh.do | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redo-sh.do b/redo-sh.do index 189a43a..fb4451b 100644 --- a/redo-sh.do +++ b/redo-sh.do @@ -9,7 +9,7 @@ WARN= for sh in dash sh ash ksh pdksh bash zsh busybox; do printf "Testing %s... " "$sh" - FOUND=`which $sh` || { echo "missing"; continue; } + FOUND=`which $sh 2>/dev/null` || { echo "missing"; continue; } # It's important for the file to actually be named 'sh'. Some # shells (like bash and zsh) only go into POSIX-compatible mode if From eea3f5446aa737879a51cceee71b678f14295c66 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Sun, 2 Jan 2011 12:00:37 -0800 Subject: [PATCH 15/19] redo-sh: keep testing even after finding a 'good' shell. Otherwise we miss out on seeing the results from additional tests. --- redo-sh.do | 4 ++-- t/shelltest.od | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/redo-sh.do b/redo-sh.do index fb4451b..91372b3 100644 --- a/redo-sh.do +++ b/redo-sh.do @@ -7,7 +7,7 @@ mkdir $1.new GOOD= WARN= -for sh in dash sh ash ksh pdksh bash zsh busybox; do +for sh in dash sh ash ksh ksh88 ksh93 pdksh bash zsh busybox; do printf "Testing %s... " "$sh" FOUND=`which $sh 2>/dev/null` || { echo "missing"; continue; } @@ -24,7 +24,7 @@ for sh in dash sh ash ksh pdksh bash zsh busybox; do set -e case $RV in - 0) echo "good"; GOOD=$FOUND; break ;; + 0) echo "good"; [ -n "$GOOD" ] || GOOD=$FOUND ;; 42) echo "warnings"; [ -n "$WARN" ] || WARN=$FOUND ;; *) echo "failed" ;; esac diff --git a/t/shelltest.od b/t/shelltest.od index e4cfc1a..c3c9f7f 100644 --- a/t/shelltest.od +++ b/t/shelltest.od @@ -168,8 +168,8 @@ echo "`printf 'foo\r\n'`"" bar" | diff -q - broken || fail 59 # -# This one is too obnoxious. dash and ash pass the test, but nothing else does, -# and this case is just too dumb to care about. Just don't do that! +# This one is too obnoxious. dash and ash pass the test, but nothing else +# does, and this case is just too dumb to care about. Just don't do that! # #t=`echo $(case x in x) echo hello;; esac)` #[ "$t" = "hello" ] || fail 60 @@ -194,6 +194,7 @@ t2=$(for d in this-glob-does-*-not-exist; do echo "$d"; done) false || foo=bar; [ "$?" = 0 ] || fail 71 foo=`exit 1`; [ "$?" != 0 ] || fail 72 + # http://www.gnu.org/software/hello/manual/autoconf/Shell-Functions.html f1() { echo 1; } f2(){ echo 2;} From cb1512b14b8c7ee6f29dc7b9be57dbb582cda2cd Mon Sep 17 00:00:00 2001 From: Tim Allen Date: Tue, 4 Jan 2011 23:39:48 +1100 Subject: [PATCH 16/19] Use named constants for terminal control codes. (apenwarr slightly changed the minimal/do tty detection.) --- log.py | 44 ++++++++++++++++++++++---------------------- minimal/do | 16 ++++++++++++++-- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/log.py b/log.py index 736f732..c217e7e 100644 --- a/log.py +++ b/log.py @@ -1,6 +1,22 @@ import sys, os import vars +# By default, no output colouring. +RED = "" +GREEN = "" +YELLOW = "" +BOLD = "" +PLAIN = "" + +if sys.stderr.isatty(): + # ...use ANSI formatting codes. + RED = "\x1b[31m" + GREEN = "\x1b[32m" + YELLOW = "\x1b[33m" + BOLD = "\x1b[1m" + PLAIN = "\x1b[m" + + def log_(s): sys.stdout.flush() if vars.DEBUG_PIDS: @@ -10,30 +26,14 @@ def log_(s): sys.stderr.flush() -def _clog(s): - log_('\x1b[32mredo %s\x1b[1m%s\x1b[m' % (vars.DEPTH, s)) -def _bwlog(s): - log_('redo %s%s' % (vars.DEPTH, s)) +def log(s): + log_(''.join([GREEN, "redo ", vars.DEPTH, BOLD, s, PLAIN])) -def _cerr(s): - log_('\x1b[31mredo: %s\x1b[1m%s\x1b[m' % (vars.DEPTH, s)) -def _bwerr(s): - log_('redo: %s%s' % (vars.DEPTH, s)) +def err(s): + log_(''.join([RED, "redo ", vars.DEPTH, BOLD, s, PLAIN])) -def _cwarn(s): - log_('\x1b[33mredo: %s\x1b[1m%s\x1b[m' % (vars.DEPTH, s)) -def _bwwarn(s): - log_('redo: %s%s' % (vars.DEPTH, s)) - - -if os.isatty(2): - log = _clog - err = _cerr - warn = _cwarn -else: - log = _bwlog - err = _bwerr - warn = _bwwarn +def warn(s): + log_(''.join([YELLOW, "redo ", vars.DEPTH, BOLD, s, PLAIN])) def debug(s): diff --git a/minimal/do b/minimal/do index 30d740c..e4b02ae 100755 --- a/minimal/do +++ b/minimal/do @@ -6,6 +6,18 @@ # The author disclaims copyright to this source file and hereby places it in # the public domain. (2010 12 14) # + +# By default, no output coloring. +GREEN="" +BOLD="" +PLAIN="" + +if tty <&2 >/dev/null 2>&1; then + GREEN="$(printf '\033[32m')" + BOLD="$(printf '\033[1m')" + PLAIN="$(printf '\033[m')" +fi + _dirsplit() { base=${1##*/} @@ -72,8 +84,8 @@ _do() DIR=$1 TARGET=$2 if [ ! -e "$TARGET" ] || [ -e "$TARGET/." -a ! -e "$TARGET.did" ]; then - printf '\033[32mdo %s\033[1m%s\033[m\n' \ - "$DO_DEPTH" "$DIR$TARGET" >&2 + printf '%sdo %s%s%s%s\n' \ + "$GREEN" "$DO_DEPTH" "$BOLD" "$DIR$TARGET" "$PLAIN" >&2 echo "$PWD/$TARGET" >>"$DO_BUILT" DOFILE=$TARGET.do BASE=$TARGET From f6ea1fd76b21d71672243c1bf5e139034dbf9120 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Tue, 4 Jan 2011 14:11:29 -0800 Subject: [PATCH 17/19] log.py, minimal/do: don't use ansi colour codes if $TERM is blank or 'dumb' Apparently emacs sets TERM=dumb in its tty simulator, so even though isatty() returns true, we shouldn't use colour codes. (emacs is therefore lame. But we knew that.) --- log.py | 2 +- minimal/do | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/log.py b/log.py index c217e7e..47ea6cb 100644 --- a/log.py +++ b/log.py @@ -8,7 +8,7 @@ YELLOW = "" BOLD = "" PLAIN = "" -if sys.stderr.isatty(): +if sys.stderr.isatty() and (os.environ.get('TERM') or 'dumb') != 'dumb': # ...use ANSI formatting codes. RED = "\x1b[31m" GREEN = "\x1b[32m" diff --git a/minimal/do b/minimal/do index e4b02ae..1d3c94a 100755 --- a/minimal/do +++ b/minimal/do @@ -12,7 +12,7 @@ GREEN="" BOLD="" PLAIN="" -if tty <&2 >/dev/null 2>&1; then +if [ -n "$TERM" -a "$TERM" != "dumb" ] && tty <&2 >/dev/null 2>&1; then GREEN="$(printf '\033[32m')" BOLD="$(printf '\033[1m')" PLAIN="$(printf '\033[m')" From 68190104018c3a81f137d6b63af68e43713994a0 Mon Sep 17 00:00:00 2001 From: Ryan Kuester Date: Mon, 10 Jan 2011 22:32:39 -0600 Subject: [PATCH 18/19] Fix use of config.sh in example --- t/example/CC.do | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/example/CC.do b/t/example/CC.do index abb528c..c45013a 100644 --- a/t/example/CC.do +++ b/t/example/CC.do @@ -3,7 +3,7 @@ redo-ifchange config.sh exec >$3 cat <<-EOF redo-ifchange \$1.c - gcc -MD -MF \$3.deps.tmp -o \$3 -c \$1.c + gcc $CFLAGS -MD -MF \$3.deps.tmp -o \$3 -c \$1.c DEPS=\$(sed -e "s/^\$3://" -e 's/\\\\//g' <\$3.deps.tmp) rm -f \$3.deps.tmp redo-ifchange \$DEPS From 4017495c124a988db3bdfd6834c130df4d9af0d2 Mon Sep 17 00:00:00 2001 From: Miles Gould Date: Tue, 11 Jan 2011 14:02:49 +0000 Subject: [PATCH 19/19] Fixed markdown errors in README - code samples now correctly formatted. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4c26d50..673d689 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ it'll be very, very fast.) The easiest way to show it is with an example. Create a file called default.o.do: + redo-ifchange $1.c gcc -MD -MF $3.deps.tmp -c -o $3 $1.c DEPS=$(sed -e "s/^$3://" -e 's/\\//g' <$3.deps.tmp) @@ -76,6 +77,7 @@ Create a file called default.o.do: redo-ifchange $DEPS Create a file called myprog.do: + DEPS="a.o b.o" redo-ifchange $DEPS gcc -o $3 $DEPS @@ -84,30 +86,36 @@ Of course, you'll also have to create `a.c` and `b.c`, the C language source files that you want to build to create your application. In a.c: + #include #include "b.h" int main() { printf(bstr); } In b.h: + extern char *bstr; In b.c: char *bstr = "hello, world!\n"; Now you simply run: + $ redo myprog And it says: + redo myprog redo a.o redo b.o Now try this: + $ touch b.h $ redo myprog Sure enough, it says: + redo myprog redo a.o