X-Git-Url: http://nitlanguage.org diff --git a/tests/tests.sh b/tests/tests.sh index 4314ea9..6483969 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -19,79 +19,314 @@ # Set lang do default to avoid failed tests because of locale export LANG=C +export NIT_TESTING=true -# The default nitc compiler -[ -z "$NITC" ] && NITC=../bin/nitc +unset NIT_DIR + +# Get the first Java lib available +shopt -s nullglob +paths=`echo /usr/lib/jvm/*/` +paths=($paths) +JAVA_HOME=${paths[0]} + +paths=`echo $JAVA_HOME/jre/lib/*/{client,server}/` +paths=($paths) +JNI_LIB_PATH=${paths[0]} +shopt -u nullglob usage() { e=`basename "$0"` cat< "out/$pattern.diff.sav.log" + if [ "$?" == 0 ]; then + return 1 + fi + [ -z "$soso" ] && return 3 + sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2" + sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2" + grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2" + grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2" + diff -u "out/$pattern.sav2" "out/$pattern.res2" > "out/$pattern.diff.sav.log" + if [ "$?" == 0 ]; then + return 2 + else + return 3 + fi +} + # As argument: the pattern used for the file function process_result() { # Result pattern=$1 + description=$2 + pack=$3 SAV="" - FAIL="" - if [ -r "sav/$pattern.sav" ]; then - diff -u "$pattern.res" "sav/$pattern.sav" > "$pattern.diff.sav.log" - if [ "$?" == 0 ]; then - SAV=OK - else - SAV=NOK - fi - fi - if [ -r "sav/$pattern.fail" ]; then - diff -u "$pattern.res" "sav/$pattern.fail" > "$pattern.diff.fail.log" - if [ "$?" == 0 ]; then - FAIL=OK + NSAV="" + FIXME="" + NFIXME="" + SOSO="" + NSOSO="" + SOSOF="" + NSOSOF="" + OLD="" + LIST="" + FIRST="" + echo >>$xml "" + #for sav in "sav/$engine/fixme/$pattern.res" "sav/$engine/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do + for savdir in $savdirs; do + sav=$savdir/fixme/$pattern.res + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + OLD="$LIST" + FIXME="$sav" + LIST="$LIST $sav" + ;; + 2) + SOSOF="$sav" + LIST="$LIST $sav" + ;; + 3) + if [ -z "$FIRST" ]; then + NFIXME="$sav" + FIRST="$sav" + fi + LIST="$LIST $sav" + ;; + esac + + sav=$savdir/$pattern.res + compare_to_result "$pattern" "$sav" + case "$?" in + 0) + ;; # no file + 1) + OLD="$LIST" + SAV="$sav" + LIST="$LIST $sav" + ;; + 2) + SOSO="$sav" + LIST="$LIST $sav" + ;; + 3) + if [ -z "$FIRST" ]; then + NSAV="$sav" + FIRST="$sav" + fi + LIST="$LIST $sav" + ;; + esac + done + OLD=`echo "$OLD" | sed -e 's/ */ /g' -e 's/^ //' -e 's/ $//'` + grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null + NYI="$?" + if [ -n "$SAV" ]; then + if [ -n "$OLD" ]; then + echo "[*ok*] out/$pattern.res $SAV - but $OLD remains!" + echo >>$xml "" + remains="$remains $OLD" else - FAIL=NOK + echo "[ok] out/$pattern.res $SAV" fi - fi - if [ "x$SAV" = "xOK" ]; then - if [ "x$FAIL" = "x" ]; then - echo "[ok] $pattern.res" + ok="$ok $pattern" + elif [ -n "$FIXME" ]; then + if [ -n "$OLD" ]; then + echo "[*fixme*] out/$pattern.res $FIXME - but $OLD remains!" + echo >>$xml "" + remains="$remains $OLD" else - echo "[ok] $pattern.res - but sav/$pattern.fail remains!" + echo "[fixme] out/$pattern.res $FIXME" + echo >>$xml "" fi + todos="$todos $pattern" + elif [ -n "$SOSO" ]; then + echo "[soso] out/$pattern.res $SOSO" ok="$ok $pattern" - elif [ "x$FAIL" = "xOK" ]; then - echo "[fail] $pattern.res" - ok="$ok $pattern" - elif [ "x$SAV" = "xNOK" ]; then - echo "[======= fail $pattern.res sav/$pattern.sav =======]" - nok="$nok $ff" + elif [ "x$NYI" = "x0" ]; then + echo "[todo] out/$pattern.res -> not yet implemented" + echo >>$xml "" + todos="$todos $pattern" + elif [ -n "$SOSOF" ]; then + echo "[fixme soso] out/$pattern.res $SOSOF" + echo >>$xml "" + todos="$todos $pattern" + elif [ -n "$NSAV" ]; then + echo "[======= fail out/$pattern.res $NSAV =======]" + echo >>$xml "" + echo >>$xml ">$xml -n 50 + echo >>$xml "]]>" + nok="$nok $pattern" echo "$ii" >> "$ERRLIST" - elif [ "x$FAIL" = "xNOK" ]; then - echo "[======= changed $pattern.res sav/$pattern.fail ======]" - nok="$nok $ff" + elif [ -n "$NFIXME" ]; then + echo "[======= changed out/$pattern.res $NFIXME ======]" + echo >>$xml "" + echo >>$xml ">$xml -n 50 + echo >>$xml "]]>" + nok="$nok $pattern" echo "$ii" >> "$ERRLIST" - else - echo "[=== no sav ===] $pattern.res" + elif [ -s out/$pattern.res ]; then + echo "[=== no sav ===] out/$pattern.res is not empty" + echo >>$xml "" + echo >>$xml ">$xml out/$pattern.res + echo >>$xml "]]>" nos="$nos $pattern" + else + # no sav but empty res + echo "[0k] out/$pattern.res is empty" + ok="$ok $pattern" fi + if test -s out/$pattern.cmp.err; then + echo >>$xml ">$xml out/$pattern.cmp.err + echo >>$xml "]]>" + fi + echo >>$xml "" +} + +need_skip() +{ + test "$noskip" = true && return 1 + if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then + echo "=> $2: [skip]" + echo >>$xml "" + return 0 + fi + if test $engine = niti && echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then + echo "=> $2: [skip exec]" + echo >>$xml "" + return 0 + fi + return 1 } +skip_exec() +{ + test "$noskip" = true && return 1 + if echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then + echo -n "_ " + return 0 + fi + return 1 +} + +skip_cc() +{ + test "$noskip" = true && return 1 + if echo "$1" | grep -f "cc.skip" >/dev/null 2>&1; then + return 0 + fi + return 1 +} + +find_nitc() +{ + name="$enginebinname" + recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1` + if [[ "x$recent" == "x" ]]; then + echo "Could not find binary for engine $engine, aborting" + exit 1 + fi + echo "Find binary for engine $engine: $recent $OPT" + NITC=$recent +} verbose=false stop=false +engine=nitg +noskip= +savdirs= while [ $stop = false ]; do case $1 in -o) OPT="$OPT $2"; shift; shift;; -v) verbose=true; shift;; -h) usage; exit;; + --engine) engine="$2"; shift; shift;; + --noskip) noskip=true; shift;; + --soso) soso=true; shift;; + --nososo) nososo=true; shift;; *) stop=true esac done +enginebinname=$engine +case $engine in + nitg) + engine=nitg-s; + enginebinname=nitg; + OPT="--separate $OPT" + ;; + nitg-s) + enginebinname=nitg; + OPT="--separate $OPT" + ;; + nitg-e) + enginebinname=nitg; + OPT="--erasure $OPT" + ;; + nitg-g) + enginebinname=nitg; + OPT="--global $OPT" + ;; + nit) + engine=niti + ;; + niti) + enginebinname=nit + ;; + nitc) + echo "disabled engine $engine" + exit 0 + ;; + *) + echo "unknown engine $engine" + exit 1 + ;; +esac + +savdirs="sav/$engine $savdirs sav/" + +# The default nitc compiler +[ -z "$NITC" ] && find_nitc + +# Set NIT_DIR if needed +[ -z "$NIT_DIR" ] && export NIT_DIR=.. + +if sh -c "timelimit echo" 1>/dev/null 2>&1; then + TIMEOUT="timelimit -t 600" +elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then + TIMEOUT="timeout 600s" +else + echo "No timelimit or timeout command detected. Tests may hang :(" +fi # Mark to distinguish files among tests # MARK= @@ -116,106 +351,170 @@ fi ok="" nok="" +todos="" +xml="tests-$engine.xml" +echo >$xml "" + +# CLEAN the out directory +rm -rf out/ 2>/dev/null +mkdir out 2>/dev/null for ii in "$@"; do - for alt in "" `sed -n 's/.*#!*\(alt[0-9]*\)#.*/\1/p' "$ii" | sort -u`; do - f=`basename "$ii" .nit` - d=`dirname "$ii"` - ff="$f" - i="$ii" - if [ "x$alt" != "x" ]; then - test -d alt || mkdir -p alt - i="alt/${f}_$alt.nit" - ff="${ff}_$alt" - sed "s/#$alt#//g;/#!$alt#/d" "$ii" > "$i" - fi - ff="$ff$MARK" + if [ ! -f $ii ]; then + echo "File '$ii' does not exist." + continue + fi + f=`basename "$ii" .nit` + + pack=`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'` + + # Sould we skip the file for this engine? + need_skip $f $f $pack && continue - echo -n "=> $i: " + tmp=${ii/../AA} + if [ "x$tmp" = "x$ii" ]; then + includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt" + else + includes="-I alt" + fi + + for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do + bf=`basename $i .nit` + ff="out/$bf" - rm "$ff.res" "$ff.err" "$ff.write" "$ff.bin" 2> /dev/null + # Sould we skip the alternative for this engine? + need_skip $bf $bf $pack && continue - # Compile - if [ "x$verbose" = "xtrue" ]; then - echo "" - echo $NITC $OPT -o "$ff.bin" "$i" -I . -I alt -I ../lib/standard + echo -n "=> $bf: " + + if [ -f "$f.inputs" ]; then + inputs="$f.inputs" + else + inputs=/dev/null fi - $NITC $OPT -o "$ff.bin" "$i" -I . -I alt -I ../lib/standard 2> "$ff.cmp.err" > "$ff.compile.log" - ERR=$? - if [ "x$verbose" = "xtrue" ]; then - cat "$ff.compile.log" - cat >&2 "$ff.cmp.err" + + if [ "$engine" = "niti" ]; then + cat > "./$ff.bin" < "$ff.cmp.err" + > "$ff.compile.log" + ERR=0 + else + if skip_cc "$bf"; then + nocc="--no-cc" + else + nocc= + fi + # Compile + if [ "x$verbose" = "xtrue" ]; then + echo "" + echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes" $nocc + fi + NIT_NO_STACK=1 JNI_LIB_PATH=$JNI_LIB_PATH JAVA_HOME=$JAVA_HOME \ + $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes $nocc 2> "$ff.cmp.err" > "$ff.compile.log" + ERR=$? + if [ "x$verbose" = "xtrue" ]; then + cat "$ff.compile.log" + cat >&2 "$ff.cmp.err" + fi fi - egrep '^[A-Z0-9_]*$' "$ff.compile.log" > "$ff.res" if [ "$ERR" != 0 ]; then echo -n "! " - cp "$ff.cmp.err" "$ff.res" - process_result $ff + cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res" + process_result $bf $bf $pack + elif skip_exec "$bf"; then + # No exec + > "$ff.res" + process_result $bf $bf $pack + elif [ -n "$nocc" ]; then + # not compiled + echo -n "nocc " + > "$ff.res" + process_result $bf $bf $pack elif [ -x "./$ff.bin" ]; then echo -n ". " # Execute args="" if [ "x$verbose" = "xtrue" ]; then echo "" - echo "./$ff.bin" $args - fi - if [ -f "$f.inputs" ]; then - "./$ff.bin" $args < "$f.inputs" > "$ff.res" 2>"$ff.err" - else - "./$ff.bin" $args > "$ff.res" 2>"$ff.err" - fi + echo "NIT_NO_STACK=1 ./$ff.bin" $args + fi + NIT_NO_STACK=1 LD_LIBRARY_PATH=$JNI_LIB_PATH \ + $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err" if [ "x$verbose" = "xtrue" ]; then cat "$ff.res" cat >&2 "$ff.err" fi if [ -f "$ff.write" ]; then cat "$ff.write" >> "$ff.res" + elif [ -d "$ff.write" ]; then + LANG=C /bin/ls -F $ff.write >> "$ff.res" fi - if [ -s "$ff.err" ]; then - cat "$ff.err" >> "$ff.res" - fi - process_result $ff + cp "$ff.res" "$ff.res2" + cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res" + process_result $bf $bf $pack if [ -f "$f.args" ]; then fargs=$f.args cptr=0 - cat $fargs | while read line; do ((cptr=cptr+1)) - args=$line + args="$line" + bff=$bf"_args"$cptr fff=$ff"_args"$cptr + name="$bf args $cptr" + + # Sould we skip the input for this engine? + need_skip $bff " $name" $pack && continue + + # use a specific inputs file, if required + if [ -f "$bff.inputs" ]; then + ffinputs="$bff.inputs" + else + ffinputs=$inputs + fi + + rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null if [ "x$verbose" = "xtrue" ]; then echo "" - echo "./$ff.bin" $args - fi - echo -n "==> args #"$cptr " " - if [ -f "$f.inputs" ]; then - "./$ff.bin" $args < "$f.inputs" > "$fff.res" 2>"$fff.err" - else - sh -c "./$ff.bin ''$args > $fff.res 2>$fff.err" + echo "NIT_NO_STACK=1 ./$ff.bin" $args fi + echo -n "==> $name " + echo "./$ff.bin $args" > "./$fff.bin" + chmod +x "./$fff.bin" + WRITE="$fff.write" sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $ffinputs > $fff.res 2>$fff.err" if [ "x$verbose" = "xtrue" ]; then cat "$fff.res" cat >&2 "$fff.err" fi if [ -f "$fff.write" ]; then cat "$fff.write" >> "$fff.res" + elif [ -d "$fff.write" ]; then + LANG=C /bin/ls -F $fff.write >> "$fff.res" fi if [ -s "$fff.err" ]; then - cat "$fff.err" >> "$fff.res" + cp "$fff.res" "$fff.res2" + cat "$fff.err" "$fff.res2" > "$fff.res" fi - process_result $fff - done + process_result $bff " $name" $pack + done < $fargs fi + elif [ -f "./$ff.bin" ]; then + echo "Not executable (platform?)" > "$ff.res" + process_result $bf "$bf" $pack else echo -n "! " + cat "$ff.cmp.err" > "$ff.res" echo "Compilation error" > "$ff.res" - process_result $ff + process_result $bf "$bf" $pack fi done done -echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w` +echo "engine: $engine ($enginebinname $OPT)" +echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w` if [ -n "$nok" ]; then echo "fail: $nok" @@ -224,6 +523,12 @@ fi if [ -n "$nos" ]; then echo "no sav: $nos" fi +if [ -n "$todos" ]; then + echo "todo/fixme: $todos" +fi +if [ -n "$remains" ]; then + echo "sav that remains: $remains" +fi # write $ERRLIST if [ "x$ERRLIST" != "x" ]; then @@ -233,6 +538,8 @@ if [ "x$ERRLIST" != "x" ]; then mv $ERRLIST $ERRLIST_TARGET fi +echo >>$xml "" + if [ -n "$nok" ]; then exit 1 else