tests: sum the first-execution time with the compilation time

[nit.git] / tests / tests.sh

diff --git a/tests/tests.sh b/tests/tests.sh
index 7f4fb5f..32afdf0 100755 (executable)
--- a/tests/tests.sh
+++ b/tests/tests.sh
@@ -96,7 +96,7 @@ function process_result()
        OLD=""
        LIST=""
        FIRST=""
-       echo >>$xml "<testcase classname='$pack' name='$description'>"
+       echo >>$xml "<testcase classname='$pack' name='$description' time='`cat out/$pattern.time.out`' timestamp='`date -Iseconds`'>"
        #for sav in "sav/$engine/fixme/$pattern.res" "sav/$engine/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
        for savdir in $savdirs; do
                sav=$savdir/fixme/$pattern.res
@@ -216,6 +216,7 @@ function process_result()
                cat -v >>$xml out/$pattern.res
                echo >>$xml "]]></system-out>"
                nos="$nos $pattern"
+               echo "$ii" >> "$ERRLIST"
        else
                # no sav but empty res
                echo "[0k] out/$pattern.res is empty"
@@ -234,12 +235,12 @@ need_skip()
        test "$noskip" = true && return 1
        if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
                echo "=> $2: [skip]"
-               echo >>$xml "<testcase classname='$3' name='$2'><skipped/></testcase>"
+               echo >>$xml "<testcase classname='$3' name='$2' timestamp='`date -Iseconds`'><skipped/></testcase>"
                return 0
        fi
        if test -n "$isinterpret" && echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then
                echo "=> $2: [skip exec]"
-               echo >>$xml "<testcase classname='$3' name='$2'><skipped/></testcase>"
+               echo >>$xml "<testcase classname='$3' name='$2' timestamp='`date -Iseconds`'><skipped/></testcase>"
                return 0
        fi
        return 1
@@ -323,6 +324,9 @@ case $engine in
                enginebinname=nit
                isinterpret=true
                ;;
+       nitvm)
+               isinterpret=true
+               savdirs="sav/niti/"
                ;;
        emscripten)
                enginebinname=nitg
@@ -393,7 +397,7 @@ for ii in "$@"; do
        fi
        f=`basename "$ii" .nit`
 
-       pack=`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'`
+       pack="tests.${engine}".`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'`
 
        # Sould we skip the file for this engine?
        need_skip $f $f $pack && continue
@@ -435,6 +439,7 @@ END
                        > "$ff.cmp.err"
                        > "$ff.compile.log"
                        ERR=0
+                       echo 0.0 > "$ff.time.out"
                else
                        if skip_cc "$bf"; then
                                nocc="--no-cc"
@@ -447,7 +452,7 @@ END
                                echo $NITC --no-color $OPT -o "$ffout" "$i" "$includes" $nocc
                        fi
                        NIT_NO_STACK=1 JNI_LIB_PATH=$JNI_LIB_PATH JAVA_HOME=$JAVA_HOME \
-                               $TIMEOUT $NITC --no-color $OPT -o "$ffout" "$i" $includes $nocc 2> "$ff.cmp.err" > "$ff.compile.log"
+                               /usr/bin/time --quiet -f%U -o "$ff.time.out" $TIMEOUT $NITC --no-color $OPT -o "$ffout" "$i" $includes $nocc 2> "$ff.cmp.err" > "$ff.compile.log"
                        ERR=$?
                        if [ "x$verbose" = "xtrue" ]; then
                                cat "$ff.compile.log"
@@ -459,7 +464,7 @@ END
                        chmod +x "$ff.bin"
                        if grep "Fatal Error: more than one primitive class" "$ff.compile.log" > /dev/null; then
                                echo " [skip] do no not imports kernel"
-                               echo >>$xml "<testcase classname='$pack' name='$bf'><skipped/></testcase>"
+                               echo >>$xml "<testcase classname='$pack' name='$bf' timestamp='`date -Iseconds`'><skipped/></testcase>"
                                continue
                        fi
                fi
@@ -487,7 +492,10 @@ END
                                echo "NIT_NO_STACK=1 ./$ff.bin" $args
                        fi      
                        NIT_NO_STACK=1 LD_LIBRARY_PATH=$JNI_LIB_PATH \
-                               $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
+                               /usr/bin/time --quiet -f%U -a -o "$ff.time.out" $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
+                       mv $ff.time.out $ff.times.out
+                       awk '{ SUM += $1} END { print SUM }' $ff.times.out > $ff.time.out
+
                        if [ "x$verbose" = "xtrue" ]; then
                                cat "$ff.res"
                                cat >&2 "$ff.err"
@@ -529,7 +537,7 @@ END
                                        echo -n "==> $name "
                                        echo "./$ff.bin $args" > "./$fff.bin"
                                        chmod +x "./$fff.bin"
-                                       WRITE="$fff.write" sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $ffinputs > $fff.res 2>$fff.err"
+                                       WRITE="$fff.write" /usr/bin/time --quiet -f%U -o "$fff.time.out" sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $ffinputs > $fff.res 2>$fff.err"
                                        if [ "x$verbose" = "xtrue" ]; then
                                                cat "$fff.res"
                                                cat >&2 "$fff.err"
@@ -547,7 +555,8 @@ END
                                done < $fargs
                        fi
                elif [ -f "./$ff.bin" ]; then
-                       echo "Not executable (platform?)" > "$ff.res"
+                       #Not executable (platform?)"
+                       > "$ff.res"
                        process_result $bf "$bf" $pack
                else
                        echo -n "! "
@@ -577,10 +586,11 @@ fi
 
 # write $ERRLIST
 if [ "x$ERRLIST" != "x" ]; then
-       if [ -x "$ERRLIST_TARGET" ]; then
+       if [ -f "$ERRLIST_TARGET" ]; then
                mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
        fi
-       mv $ERRLIST $ERRLIST_TARGET
+       uniq $ERRLIST > $ERRLIST_TARGET
+       rm $ERRLIST
 fi
 
 echo >>$xml "</testsuite></testsuites>"