X-Git-Url: http://nitlanguage.org diff --git a/tests/tests.sh b/tests/tests.sh index 78188fd..f470498 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -19,89 +19,236 @@ # Set lang do default to avoid failed tests because of locale export LANG=C +export NIT_TESTING=true usage() { e=`basename "$0"` cat< "out/$pattern.diff.sav.log" + if [ "$?" == 0 ]; then + return 1 + fi + [ -z "$soso" ] && return 3 + sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2" + sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2" + grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2" + grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2" + diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log" + if [ "$?" == 0 ]; then + return 2 + else + return 3 + fi +} + # As argument: the pattern used for the file function process_result() { + ((tapcount=tapcount+1)) # Result pattern=$1 + description=$2 SAV="" - FAIL="" - if [ -r "sav/$pattern.sav" ]; then - diff -u "$pattern.res" "sav/$pattern.sav" > "$pattern.diff.sav.log" - if [ "$?" == 0 ]; then - SAV=OK + NSAV="" + FIXME="" + NFIXME="" + SOSO="" + NSOSO="" + SOSOF="" + NSOSOF="" + for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + SAV="$sav" ;; + 2) + SOSO="$sav" ;; + 3) + NSAV="$sav";; + esac + done + for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + FIXME="$sav" ;; + 2) + SOSOF="$sav" ;; + 3) + NFIXME="$sav";; + esac + done + grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null + NYI="$?" + if [ -n "$SAV" ]; then + if [ -n "$tap" ]; then + echo "ok - $description" + elif [ -z "$FIXME" ]; then + echo "[ok] out/$pattern.res $SAV" else - SAV=NOK + echo "[ok] out/$pattern.res $SAV - but $FIXME remains!" fi - fi - if [ -r "sav/$pattern.fail" ]; then - diff -u "$pattern.res" "sav/$pattern.fail" > "$pattern.diff.fail.log" - if [ "$?" == 0 ]; then - FAIL=OK + ok="$ok $pattern" + elif [ -n "$FIXME" ]; then + if [ -n "$tap" ]; then + echo "not ok - $description # TODO expected failure" else - FAIL=NOK + echo "[fixme] out/$pattern.res $FIXME" fi - fi - if [ "x$SAV" = "xOK" ]; then - if [ "x$FAIL" = "x" ]; then - echo "[ok] $pattern.res" + todos="$todos $pattern" + elif [ -n "$SOSO" ]; then + if [ -n "$tap" ]; then + echo "ok - $description # SOSO" else - echo "[ok] $pattern.res - but sav/$pattern.fail remains!" + echo "[soso] out/$pattern.res $SOSO" fi ok="$ok $pattern" - elif [ "x$FAIL" = "xOK" ]; then - echo "[fail] $pattern.res" - ok="$ok $pattern" - elif [ "x$SAV" = "xNOK" ]; then - echo "[======= fail $pattern.res sav/$pattern.sav =======]" - nok="$nok $ff" + elif [ "x$NYI" = "x0" ]; then + if [ -n "$tap" ]; then + echo "not ok - $description # TODO not yet implemented" + else + echo "[todo] out/$pattern.res -> not yet implemented" + fi + todos="$todos $pattern" + elif [ -n "$SOSOF" ]; then + if [ -n "$tap" ]; then + echo "not ok - $description # TODO SOSO expected failure" + else + echo "[fixme soso] out/$pattern.res $SOSOF" + fi + todos="$todos $pattern" + elif [ -n "$NSAV" ]; then + if [ -n "$tap" ]; then + echo "not ok - $description" + else + echo "[======= fail out/$pattern.res $NSAV =======]" + fi + nok="$nok $pattern" echo "$ii" >> "$ERRLIST" - elif [ "x$FAIL" = "xNOK" ]; then - echo "[======= changed $pattern.res sav/$pattern.fail ======]" - nok="$nok $ff" + elif [ -n "$NFIXME" ]; then + if [ -n "$tap" ]; then + echo "not ok - $description" + else + echo "[======= changed out/$pattern.res $NFIXME ======]" + fi + nok="$nok $pattern" echo "$ii" >> "$ERRLIST" else - echo "[=== no sav ===] $pattern.res" + if [ -n "$tap" ]; then + echo "ok - $description # skip no sav" + else + echo "[=== no sav ===] out/$pattern.res" + fi nos="$nos $pattern" fi } +need_skip() +{ + test "$noskip" = true && return 1 + if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then + ((tapcount=tapcount+1)) + if [ -n "$tap" ]; then + echo "ok - $2 # skip" + else + echo "=> $2: [skip]" + fi + return 0 + fi + return 1 +} + find_nitc() { - recent=`ls -t ../src/nitc ../src/nitc_[0-9] ../bin/nitc ../c_src/nitc 2>/dev/null | head -1` + ((tapcount=tapcount+1)) + name="$enginebinname" + recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1` if [[ "x$recent" == "x" ]]; then - echo 'Could not find nitc, aborting' + if [ -n "$tap" ]; then + echo "not ok - find engine $name" + echo "Bail out! Could not find engine $name, aborting" + else + echo "Could not find engine $name, aborting" + fi exit 1 fi - echo 'Using nitc from: '$recent + if [ -n "$tap" ]; then + echo "ok - find engine $name: $recent" + else + echo "Using engine $name from: $recent" + fi NITC=$recent } -# The default nitc compiler -[ -z "$NITC" ] && find_nitc - verbose=false stop=false +tapcount=0 +engine=nitc +noskip= while [ $stop = false ]; do case $1 in -o) OPT="$OPT $2"; shift; shift;; -v) verbose=true; shift;; -h) usage; exit;; + --tap) tap=true; shift;; + --engine) engine="$2"; shift; shift;; + --noskip) noskip=true; shift;; + --soso) soso=true; shift;; + --nososo) nososo=true; shift;; *) stop=true esac done +enginebinname=$engine +case $engine in + nitc) ;; + nitg) OPT="--global $OPT";; + nitg-s) enginebinname=nitg; OPT="--separate $OPT";; + nitg-e) enginebinname=nitg; OPT="--erasure $OPT";; + nit) engine=niti ;; + niti) enginebinname=nit ;; +esac + +# The default nitc compiler +[ -z "$NITC" ] && find_nitc + +# Set NIT_DIR if needed +[ -z "$NIT_DIR" ] && export NIT_DIR=.. + +if sh -c "timelimit echo" 1>/dev/null 2>&1; then + TIMEOUT="timelimit -t 600" +elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then + TIMEOUT="timeout 600s" +else + echo "No timelimit or timeout command detected. Tests may hang :(" +fi # Mark to distinguish files among tests # MARK= @@ -126,110 +273,148 @@ fi ok="" nok="" +todos="" + +# CLEAN the out directory +rm -rf out/ 2>/dev/null +mkdir out 2>/dev/null for ii in "$@"; do if [ ! -f $ii ]; then echo "File '$ii' does not exist." continue fi - for alt in "" `sed -n 's/.*#!*\(alt[0-9]*\)#.*/\1/p' "$ii" | sort -u`; do - f=`basename "$ii" .nit` - d=`dirname "$ii"` - ff="$f" - i="$ii" - if [ "x$alt" != "x" ]; then - test -d alt || mkdir -p alt - i="alt/${f}_$alt.nit" - ff="${ff}_$alt" - sed "s/#$alt#//g;/#!$alt#/d" "$ii" > "$i" - fi - ff="$ff$MARK" + f=`basename "$ii" .nit` + + # Sould we skip the file for this engine? + need_skip $f $f && continue + + tmp=${ii/../AA} + if [ "x$tmp" = "x$ii" ]; then + includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt" + else + includes="-I alt" + fi + + for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do + bf=`basename $i .nit` + ff="out/$bf" - echo -n "=> $i: " + # Sould we skip the alternative for this engine? + need_skip $bf $bf && continue - rm "$ff.res" "$ff.err" "$ff.write" "$ff.bin" 2> /dev/null + test -z "$tap" && echo -n "=> $bf: " - # Compile - if [ "x$verbose" = "xtrue" ]; then - echo "" - echo $NITC $OPT -o "$ff.bin" "$i" -I . -I alt -I ../lib/standard + if [ -f "$f.inputs" ]; then + inputs="$f.inputs" + else + inputs=/dev/null fi - $NITC $OPT -o "$ff.bin" "$i" -I . -I alt -I ../lib/standard 2> "$ff.cmp.err" > "$ff.compile.log" - ERR=$? - if [ "x$verbose" = "xtrue" ]; then - cat "$ff.compile.log" - cat >&2 "$ff.cmp.err" + + if [ "$engine" = "niti" ]; then + cat > "./$ff.bin" < "$ff.cmp.err" + > "$ff.compile.log" + ERR=0 + else + # Compile + if [ "x$verbose" = "xtrue" ]; then + echo "" + echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes" + fi + NIT_NO_STACK=1 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log" + ERR=$? + if [ "x$verbose" = "xtrue" ]; then + cat "$ff.compile.log" + cat >&2 "$ff.cmp.err" + fi fi - egrep '^[A-Z0-9_]*$' "$ff.compile.log" > "$ff.res" if [ "$ERR" != 0 ]; then - echo -n "! " - cp "$ff.cmp.err" "$ff.res" - process_result $ff + test -z "$tap" && echo -n "! " + cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res" + process_result $bf $bf elif [ -x "./$ff.bin" ]; then - echo -n ". " + test -z "$tap" && echo -n ". " # Execute args="" if [ "x$verbose" = "xtrue" ]; then echo "" - echo "./$ff.bin" $args - fi - if [ -f "$f.inputs" ]; then - "./$ff.bin" $args < "$f.inputs" > "$ff.res" 2>"$ff.err" - else - "./$ff.bin" $args > "$ff.res" 2>"$ff.err" + echo "NIT_NO_STACK=1 ./$ff.bin" $args fi + NIT_NO_STACK=1 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err" if [ "x$verbose" = "xtrue" ]; then cat "$ff.res" cat >&2 "$ff.err" fi if [ -f "$ff.write" ]; then cat "$ff.write" >> "$ff.res" + elif [ -d "$ff.write" ]; then + LANG=C /bin/ls -F $ff.write >> "$ff.res" fi - if [ -s "$ff.err" ]; then - cat "$ff.err" >> "$ff.res" - fi - process_result $ff + cp "$ff.res" "$ff.res2" + cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res" + process_result $bf $bf if [ -f "$f.args" ]; then fargs=$f.args cptr=0 - cat $fargs | while read line; do ((cptr=cptr+1)) - args=$line + args="$line" + bff=$bf"_args"$cptr fff=$ff"_args"$cptr + name="$bf args $cptr" + + # Sould we skip the input for this engine? + need_skip $bff " $name" && continue + + rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null if [ "x$verbose" = "xtrue" ]; then echo "" - echo "./$ff.bin" $args - fi - echo -n "==> args #"$cptr " " - if [ -f "$f.inputs" ]; then - "./$ff.bin" $args < "$f.inputs" > "$fff.res" 2>"$fff.err" - else - sh -c "./$ff.bin ''$args > $fff.res 2>$fff.err" + echo "NIT_NO_STACK=1 ./$ff.bin" $args fi + test -z "$tap" && echo -n "==> $name " + echo "./$ff.bin $args" > "./$fff.bin" + chmod +x "./$fff.bin" + sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $inputs > $fff.res 2>$fff.err" if [ "x$verbose" = "xtrue" ]; then cat "$fff.res" cat >&2 "$fff.err" fi if [ -f "$fff.write" ]; then cat "$fff.write" >> "$fff.res" + elif [ -d "$fff.write" ]; then + LANG=C /bin/ls -F $fff.write >> "$fff.res" fi if [ -s "$fff.err" ]; then - cat "$fff.err" >> "$fff.res" + cp "$fff.res" "$fff.res2" + cat "$fff.err" "$fff.res2" > "$fff.res" fi - process_result $fff - done + process_result $bff " $name" + done < $fargs fi else - echo -n "! " + test -z "$tap" && echo -n "! " + cat "$ff.cmp.err" > "$ff.res" echo "Compilation error" > "$ff.res" - process_result $ff + process_result $bf "$bf" fi done done -echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w` +if [ -n "$tap" ]; then + echo "1..$tapcount" + echo "# ok:" `echo $ok | wc -w` + echo "# not ok:" `echo $nok | wc -w` + echo "# no sav:" `echo $nos | wc -w` + echo "# todo/fixme:" `echo $todos | wc -w` + exit +fi + +echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w` if [ -n "$nok" ]; then echo "fail: $nok" @@ -238,6 +423,9 @@ fi if [ -n "$nos" ]; then echo "no sav: $nos" fi +if [ -n "$todos" ]; then + echo "todo/fixme: $todos" +fi # write $ERRLIST if [ "x$ERRLIST" != "x" ]; then