X-Git-Url: http://nitlanguage.org diff --git a/tests/tests.sh b/tests/tests.sh index 80e75c1..f470498 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -19,6 +19,7 @@ # Set lang do default to avoid failed tests because of locale export LANG=C +export NIT_TESTING=true usage() { @@ -30,9 +31,40 @@ Usage: $e [options] modulenames -h This help --tap Produce TAP output --engine Use a specific engine (default=nitc) +--noskip Do not skip a test even if the .skip file matches +--[no]soso Force enable (or disable) SOSO END } +# $1 is the pattern of the test +# $2 is the file to compare to +# the result is: +# 0: if the file to compare to do not exists +# 1: if the file match +# 2: if the file match with soso +# 3: if the file do not match +function compare_to_result() +{ + local pattern="$1" + local sav="$2" + if [ ! -r "$sav" ]; then return 0; fi + diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log" + if [ "$?" == 0 ]; then + return 1 + fi + [ -z "$soso" ] && return 3 + sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2" + sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2" + grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2" + grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2" + diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log" + if [ "$?" == 0 ]; then + return 2 + else + return 3 + fi +} + # As argument: the pattern used for the file function process_result() { @@ -41,68 +73,62 @@ function process_result() pattern=$1 description=$2 SAV="" - FAIL="" + NSAV="" + FIXME="" + NFIXME="" SOSO="" + NSOSO="" SOSOF="" - if [ -r "sav/$pattern.sav" ]; then - diff -u "out/$pattern.res" "sav/$pattern.sav" > "out/$pattern.diff.sav.log" - if [ "$?" == 0 ]; then - SAV=OK - else - SAV=NOK - fi - sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2" - sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.sav" > "out/$pattern.sav2" - grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2" - grep '[Ee]rror' "sav/$pattern.sav" >/dev/null && echo "Error" >> "out/$pattern.sav2" - diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log" - if [ "$?" == 0 ]; then - SOSO=OK - else - SOSO=NOK - fi - fi - if [ -r "sav/$pattern.fail" ]; then - diff -u "out/$pattern.res" "sav/$pattern.fail" > "out/$pattern.diff.fail.log" - if [ "$?" == 0 ]; then - FAIL=OK - else - FAIL=NOK - fi - sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2" - sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.fail" > "out/$pattern.fail2" - grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2" - grep '[Ee]rror' "sav/$pattern.fail" >/dev/null && echo "Error" >> "out/$pattern.fail2" - diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log" - if [ "$?" == 0 ]; then - SOSOF=OK - else - SOSOF=NOK - fi - fi + NSOSOF="" + for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + SAV="$sav" ;; + 2) + SOSO="$sav" ;; + 3) + NSAV="$sav";; + esac + done + for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + FIXME="$sav" ;; + 2) + SOSOF="$sav" ;; + 3) + NFIXME="$sav";; + esac + done grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null NYI="$?" - if [ "x$SAV" = "xOK" ]; then + if [ -n "$SAV" ]; then if [ -n "$tap" ]; then echo "ok - $description" - elif [ "x$FAIL" = "x" ]; then - echo "[ok] out/$pattern.res" + elif [ -z "$FIXME" ]; then + echo "[ok] out/$pattern.res $SAV" else - echo "[ok] out/$pattern.res - but sav/$pattern.fail remains!" + echo "[ok] out/$pattern.res $SAV - but $FIXME remains!" fi ok="$ok $pattern" - elif [ "x$FAIL" = "xOK" ]; then + elif [ -n "$FIXME" ]; then if [ -n "$tap" ]; then echo "not ok - $description # TODO expected failure" else - echo "[fail] out/$pattern.res" + echo "[fixme] out/$pattern.res $FIXME" fi - ok="$ok $pattern" - elif [ "x$SOSO" = "xOK" ]; then + todos="$todos $pattern" + elif [ -n "$SOSO" ]; then if [ -n "$tap" ]; then echo "ok - $description # SOSO" else - echo "[soso] out/$pattern.res sav/$pattern.sav" + echo "[soso] out/$pattern.res $SOSO" fi ok="$ok $pattern" elif [ "x$NYI" = "x0" ]; then @@ -111,27 +137,27 @@ function process_result() else echo "[todo] out/$pattern.res -> not yet implemented" fi - ok="$ok $pattern" - elif [ "x$SOSOF" = "xOK" ]; then + todos="$todos $pattern" + elif [ -n "$SOSOF" ]; then if [ -n "$tap" ]; then echo "not ok - $description # TODO SOSO expected failure" else - echo "[fail soso] out/$pattern.res sav/$pattern.fail" + echo "[fixme soso] out/$pattern.res $SOSOF" fi - ok="$ok $pattern" - elif [ "x$SAV" = "xNOK" ]; then + todos="$todos $pattern" + elif [ -n "$NSAV" ]; then if [ -n "$tap" ]; then echo "not ok - $description" else - echo "[======= fail out/$pattern.res sav/$pattern.sav =======]" + echo "[======= fail out/$pattern.res $NSAV =======]" fi nok="$nok $pattern" echo "$ii" >> "$ERRLIST" - elif [ "x$FAIL" = "xNOK" ]; then + elif [ -n "$NFIXME" ]; then if [ -n "$tap" ]; then echo "not ok - $description" else - echo "[======= changed out/$pattern.res sav/$pattern.fail ======]" + echo "[======= changed out/$pattern.res $NFIXME ======]" fi nok="$nok $pattern" echo "$ii" >> "$ERRLIST" @@ -145,10 +171,25 @@ function process_result() fi } +need_skip() +{ + test "$noskip" = true && return 1 + if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then + ((tapcount=tapcount+1)) + if [ -n "$tap" ]; then + echo "ok - $2 # skip" + else + echo "=> $2: [skip]" + fi + return 0 + fi + return 1 +} + find_nitc() { ((tapcount=tapcount+1)) - name="$engine" + name="$enginebinname" recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1` if [[ "x$recent" == "x" ]]; then if [ -n "$tap" ]; then @@ -171,6 +212,7 @@ verbose=false stop=false tapcount=0 engine=nitc +noskip= while [ $stop = false ]; do case $1 in -o) OPT="$OPT $2"; shift; shift;; @@ -178,9 +220,21 @@ while [ $stop = false ]; do -h) usage; exit;; --tap) tap=true; shift;; --engine) engine="$2"; shift; shift;; + --noskip) noskip=true; shift;; + --soso) soso=true; shift;; + --nososo) nososo=true; shift;; *) stop=true esac done +enginebinname=$engine +case $engine in + nitc) ;; + nitg) OPT="--global $OPT";; + nitg-s) enginebinname=nitg; OPT="--separate $OPT";; + nitg-e) enginebinname=nitg; OPT="--erasure $OPT";; + nit) engine=niti ;; + niti) enginebinname=nit ;; +esac # The default nitc compiler [ -z "$NITC" ] && find_nitc @@ -188,6 +242,13 @@ done # Set NIT_DIR if needed [ -z "$NIT_DIR" ] && export NIT_DIR=.. +if sh -c "timelimit echo" 1>/dev/null 2>&1; then + TIMEOUT="timelimit -t 600" +elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then + TIMEOUT="timeout 600s" +else + echo "No timelimit or timeout command detected. Tests may hang :(" +fi # Mark to distinguish files among tests # MARK= @@ -212,6 +273,7 @@ fi ok="" nok="" +todos="" # CLEAN the out directory rm -rf out/ 2>/dev/null @@ -222,6 +284,10 @@ for ii in "$@"; do echo "File '$ii' does not exist." continue fi + f=`basename "$ii" .nit` + + # Sould we skip the file for this engine? + need_skip $f $f && continue tmp=${ii/../AA} if [ "x$tmp" = "x$ii" ]; then @@ -230,10 +296,13 @@ for ii in "$@"; do includes="-I alt" fi - f=`basename "$ii" .nit` for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do bf=`basename $i .nit` ff="out/$bf" + + # Sould we skip the alternative for this engine? + need_skip $bf $bf && continue + test -z "$tap" && echo -n "=> $bf: " if [ -f "$f.inputs" ]; then @@ -242,7 +311,7 @@ for ii in "$@"; do inputs=/dev/null fi - if [ "$engine" = "nit" ]; then + if [ "$engine" = "niti" ]; then cat > "./$ff.bin" < "$ff.cmp.err" > "$ff.compile.log" + NIT_NO_STACK=1 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log" ERR=$? if [ "x$verbose" = "xtrue" ]; then cat "$ff.compile.log" @@ -265,10 +334,9 @@ END fi if [ "$ERR" != 0 ]; then test -z "$tap" && echo -n "! " - cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res" + cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res" process_result $bf $bf elif [ -x "./$ff.bin" ]; then - cp "$ff.cmp.err" "$ff.res" test -z "$tap" && echo -n ". " # Execute args="" @@ -276,7 +344,7 @@ END echo "" echo "NIT_NO_STACK=1 ./$ff.bin" $args fi - NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err" + NIT_NO_STACK=1 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err" if [ "x$verbose" = "xtrue" ]; then cat "$ff.res" cat >&2 "$ff.err" @@ -286,9 +354,8 @@ END elif [ -d "$ff.write" ]; then LANG=C /bin/ls -F $ff.write >> "$ff.res" fi - if [ -s "$ff.err" ]; then - cat "$ff.err" >> "$ff.res" - fi + cp "$ff.res" "$ff.res2" + cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res" process_result $bf $bf if [ -f "$f.args" ]; then @@ -299,13 +366,20 @@ END args="$line" bff=$bf"_args"$cptr fff=$ff"_args"$cptr + name="$bf args $cptr" + + # Sould we skip the input for this engine? + need_skip $bff " $name" && continue + rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null if [ "x$verbose" = "xtrue" ]; then echo "" echo "NIT_NO_STACK=1 ./$ff.bin" $args fi - test -z "$tap" && echo -n "==> args #"$cptr " " - sh -c "NIT_NO_STACK=1 ./$ff.bin ''$args < $inputs > $fff.res 2>$fff.err" + test -z "$tap" && echo -n "==> $name " + echo "./$ff.bin $args" > "./$fff.bin" + chmod +x "./$fff.bin" + sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $inputs > $fff.res 2>$fff.err" if [ "x$verbose" = "xtrue" ]; then cat "$fff.res" cat >&2 "$fff.err" @@ -316,9 +390,10 @@ END LANG=C /bin/ls -F $fff.write >> "$fff.res" fi if [ -s "$fff.err" ]; then - cat "$fff.err" >> "$fff.res" + cp "$fff.res" "$fff.res2" + cat "$fff.err" "$fff.res2" > "$fff.res" fi - process_result $bff " args #$cptr" + process_result $bff " $name" done < $fargs fi else @@ -335,10 +410,11 @@ if [ -n "$tap" ]; then echo "# ok:" `echo $ok | wc -w` echo "# not ok:" `echo $nok | wc -w` echo "# no sav:" `echo $nos | wc -w` + echo "# todo/fixme:" `echo $todos | wc -w` exit fi -echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w` +echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w` if [ -n "$nok" ]; then echo "fail: $nok" @@ -347,6 +423,9 @@ fi if [ -n "$nos" ]; then echo "no sav: $nos" fi +if [ -n "$todos" ]; then + echo "todo/fixme: $todos" +fi # write $ERRLIST if [ "x$ERRLIST" != "x" ]; then