tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22 export LC_ALL=C
  23 export NIT_TESTING=true
  24 # Use the pid as a collision prevention
  25 export NIT_TESTING_ID=$$
  26 export NIT_SRAND=0
  27
  28 unset NIT_DIR
  29
  30 # Get the first Java lib available
  31 if which_java=$(which javac 2>/dev/null); then
  32         JAVA_HOME=$(dirname $(dirname $(readlink -f "$which_java")))
  33
  34         shopt -s nullglob
  35         paths=`echo $JAVA_HOME/jre/lib/*/{client,server}/libjvm.so`
  36         paths=($paths)
  37         JNI_LIB_PATH=`dirname ${paths[0]}`
  38         shopt -u nullglob
  39 fi
  40
  41 outdir="out"
  42 compdir="nit_compile"
  43
  44 # User CPU time limit (in seconds)
  45 # Is used to avoid to CPU intensive test (infinite loops). See ulimit -t
  46 usertimelimit=600 # 1 CPU minute
  47
  48 # Real-time limit (in seconds)
  49 # Is used to avoid waiting or sleeping tests.
  50 # Require timeout or timelimit, or else is not used.
  51 realtimelimit=300 # 5 min
  52
  53 # User limit for write files (in kilo-bytes)
  54 # Is used to avoid execution that loop and fill the hard drive. See ulimit -f
  55 # Note that a test might require a lot of temporary disk space (eg. nitc+gcc)
  56 filelimit=100000 # ~100MB
  57
  58 # Limit (in bytes) for generated .res file.
  59 # Larger ones are truncated and will fail tests
  60 # Is used to avoid processing huge crappy res file (diff, xml, etc)
  61 reslimit=100000 # ~100KB
  62
  63 usage()
  64 {
  65         e=`basename "$0"`
  66         cat<<END
  67 Usage: $e [options] modulenames
  68 -o option   Pass option to the engine
  69 -v          Verbose (show tests steps)
  70 -h          This help
  71 --engine    Use a specific engine (default=nitc)
  72 --noskip    Do not skip a test even if the .skip file matches
  73 --outdir    Use a specific output folder (default=out/)
  74 --compdir   Use a specific temporary compilation folder (default=$compdir)
  75 --node      Run as a node in parallel, will not output context information
  76 --autosav   Copy the .res files directly in the sav folder overriding existing .res files
  77 END
  78 }
  79
  80 # Run a command with a timeout and a time count.
  81 # Options:
  82 #   -o file    write the user time into file (REQUIRED). see `-o` in `man time`
  83 #   -a         append the time to the file (instead of overwriting it). see `-a` in `man time`
  84 saferun()
  85 {
  86         local stop=false
  87         local o=
  88         local a=
  89         while [ $stop = false ]; do
  90                 case "$1" in
  91                         -o) o="$2"; shift; shift;;
  92                         -a) a="-a"; shift;;
  93                         *) stop=true
  94                 esac
  95         done
  96         (
  97         ulimit -f "$filelimit" 2> /dev/null
  98         ulimit -t "$usertimelimit" 2> /dev/null
  99         if test -d "$1"; then
 100                 find $1 | sort
 101         elif test -n "$TIME"; then
 102                 $TIME -o "$o" $a $TIMEOUT "$@"
 103         else
 104                 if test -n "$a"; then echo 0 >> "$o"; else echo 0 > "$o"; fi
 105                 $TIMEOUT "$@"
 106         fi
 107         )
 108 }
 109
 110 # Output a timestamp attribute for XML, or an empty line
 111 timestamp()
 112 {
 113         if test -n "$TIMESTAMP"; then
 114                 echo "timestamp='`$TIMESTAMP`'"
 115         else
 116                 echo ""
 117         fi
 118
 119 }
 120
 121 # Get platform specific commands ##########################
 122
 123 # Detect a working timeout
 124 if sh -c "timelimit echo" 1>/dev/null 2>&1; then
 125         TIMEOUT="timelimit -t $realtimelimit"
 126 elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
 127         TIMEOUT="timeout ${realtimelimit}s"
 128 else
 129         echo "No timelimit or timeout command detected. Tests may hang :("
 130 fi
 131
 132 # Detect a working time command
 133 if env time --quiet -f%U true 2>/dev/null; then
 134         TIME="env time --quiet -f%U"
 135 elif env time -f%U true 2>/dev/null; then
 136         TIME="env time -f%U"
 137 elif env gtime -f%U true 2>/dev/null; then
 138         TIME="env gtime -f%U"
 139 else
 140         TIME=
 141 fi
 142
 143 # Detect a working date command
 144 if date -Iseconds >/dev/null 2>&1; then
 145         TIMESTAMP="date -Iseconds"
 146 else
 147         TIMESTAMP=
 148 fi
 149
 150 # Detect a working hostname command
 151 if hostname --version 2>&1 | grep coreutils >/dev/null 2>&1; then
 152         HOSTNAME="hostname"
 153 else
 154         HOSTNAME="hostname -s"
 155 fi
 156
 157 UNAME=`uname | sed s/-.*//`
 158
 159 # $1 is the pattern of the test
 160 # $2 is the file to compare to
 161 # the result is:
 162 #   0: if the file to compare to do not exists
 163 #   1: if the file match
 164 #   2: if the file match with soso
 165 #   3: if the file do not match
 166 function compare_to_result()
 167 {
 168         local pattern="$1"
 169         local sav="$2"
 170         if [ ! -r "$sav" ]; then return 0; fi
 171         test "`cat -- "$sav"`" = "UNDEFINED" && return 1
 172         diff -u --strip-trailing-cr -- "$sav" "$outdir/$pattern.res" > "$outdir/$pattern.diff.sav.log"
 173         if [ "$?" == 0 ]; then
 174                 return 1
 175         fi
 176         sed '/[Ww]arning/d;/[Ee]rror/d' "$outdir/$pattern.res" > "$outdir/$pattern.res2"
 177         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "$outdir/$pattern.sav2"
 178         grep '[Ee]rror' "$outdir/$pattern.res" >/dev/null && echo "Error" >> "$outdir/$pattern.res2"
 179         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "$outdir/$pattern.sav2"
 180         diff -u --strip-trailing-cr "$outdir/$pattern.sav2" "$outdir/$pattern.res2" > "$outdir/$pattern.diff.sav.log2"
 181         if [ "$?" == 0 ]; then
 182                 return 2
 183         else
 184                 return 3
 185         fi
 186 }
 187
 188 function xmlesc()
 189 {
 190         sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g; s/'"'"'/\&#39;/g'<<EOF
 191 $*
 192 EOF
 193 }
 194
 195 # As argument: the pattern used for the file
 196 function process_result()
 197 {
 198         # Result
 199         local pattern=$1
 200         local description=$2
 201         local pack=$3
 202         local SAV=""
 203         local NSAV=""
 204         local FIXME=""
 205         local NFIXME=""
 206         local SOSO=""
 207         local NSOSO=""
 208         local SOSOF=""
 209         local NSOSOF=""
 210         local OLD=""
 211         local LIST=""
 212         local FIRST=""
 213
 214         # Truncate too big res file
 215         local size=$(wc -c < "$outdir/$pattern.res")
 216         if test -n "$reslimit" -a "$size" -gt "$reslimit"; then
 217                 # The most portable way to truncate a file is with Perl
 218                 perl -e "truncate \"$outdir/$pattern.res\", $reslimit;"
 219                 echo "***TRUNCATED***" >> "$outdir/$pattern.res"
 220         fi
 221
 222         echo >>$xml "<testcase classname='`xmlesc "$pack"`' name='`xmlesc "$description"`' time='`cat -- "$outdir/$pattern.time.out"`' `timestamp`>"
 223         #for sav in "sav/$engine/fixme/$pattern.res" "sav/$engine/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
 224         for savdir in $savdirs; do
 225                 local sav=$savdir/fixme/$pattern.res
 226                 compare_to_result "$pattern" "$sav"
 227                 case "$?" in
 228                         0)
 229                                 ;; # no file
 230                         1)
 231                                 OLD="$LIST"
 232                                 FIXME="$sav"
 233                                 LIST="$LIST $sav"
 234                                 ;;
 235                         2)
 236                                 if [ -z "$FIRST" ]; then
 237                                         SOSOF="$sav"
 238                                         FIRST="$sav"
 239                                 fi
 240                                 LIST="$LIST $sav"
 241                                 ;;
 242                         3)
 243                                 if [ -z "$FIRST" ]; then
 244                                         NFIXME="$sav"
 245                                         FIRST="$sav"
 246                                 fi
 247                                 LIST="$LIST $sav"
 248                                 ;;
 249                 esac
 250
 251                 sav=$savdir/$pattern.res
 252                 compare_to_result "$pattern" "$sav"
 253                 case "$?" in
 254                         0)
 255                                 ;; # no file
 256                         1)
 257                                 OLD="$LIST"
 258                                 SAV="$sav"
 259                                 LIST="$LIST $sav"
 260                                 ;;
 261                         2)
 262                                 if [ -z "$FIRST" ]; then
 263                                         SOSO="$sav"
 264                                         FIRST="$sav"
 265                                 fi
 266                                 LIST="$LIST $sav"
 267                                 ;;
 268                         3)
 269                                 if [ -z "$FIRST" ]; then
 270                                         NSAV="$sav"
 271                                         FIRST="$sav"
 272                                 fi
 273                                 LIST="$LIST $sav"
 274                                 ;;
 275                 esac
 276         done
 277         OLD=`echo "$OLD" | sed -e 's/   */ /g' -e 's/^ //' -e 's/ $//'`
 278         istodo  "$outdir/$pattern.res"
 279         NYI="$?"
 280         if [ -n "$SAV" ]; then
 281                 if [ -n "$OLD" ]; then
 282                         echo "[*ok*] $outdir/$pattern.res $SAV - but $OLD remains!"
 283                         echo >>$xml "<error message='`xmlesc "ok $outdir/$pattern.res - but $OLD remains"`'/>"
 284                         remains="$remains $OLD"
 285                         test "$autosav" = "true" && rm "$OLD"
 286                 else
 287                         echo "[ok] $outdir/$pattern.res $SAV"
 288                 fi
 289                 ok="$ok $pattern"
 290         elif [ -n "$FIXME" ]; then
 291                 if [ -n "$OLD" ]; then
 292                         echo "[*fixme*] $outdir/$pattern.res $FIXME - but $OLD remains!"
 293                         echo >>$xml "<error message='`xmlesc "ok $outdir/$pattern.res - but $OLD remains"`'/>"
 294                         remains="$remains $OLD"
 295                         test "$autosav" = "true" && rm "$OLD"
 296                 else
 297                         echo "[fixme] $outdir/$pattern.res $FIXME"
 298                         echo >>$xml "<skipped/>"
 299                 fi
 300                 todos="$todos $pattern"
 301         elif [ "x$NYI" = "x0" ]; then
 302                 echo "[todo] $outdir/$pattern.res -> not yet implemented"
 303                 echo >>$xml "<skipped/>"
 304                 todos="$todos $pattern"
 305         elif [ -n "$SOSO" ]; then
 306                 echo "[======= soso $outdir/$pattern.res $SOSO =======]"
 307                 echo >>$xml "<error message='`xmlesc "soso $outdir/$pattern.res $SOSO"`'/>"
 308                 echo >>$xml "<system-out><![CDATA["
 309                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 310                 echo >>$xml "]]></system-out>"
 311                 nok="$nok $pattern"
 312                 echo "$ii" >> "$ERRLIST"
 313                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$SOSO"
 314         elif [ -n "$SOSOF" ]; then
 315                 echo "[======= fixme soso $outdir/$pattern.res $SOSOF =======]"
 316                 echo >>$xml "<error message='`xmlesc "soso $outdir/$pattern.res $SOSO"`'/>"
 317                 echo >>$xml "<system-out><![CDATA["
 318                 cat -v  -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 319                 echo >>$xml "]]></system-out>"
 320                 nok="$nok $pattern"
 321                 echo "$ii" >> "$ERRLIST"
 322                 test "$autosav" = "true" && cp "$outdir/$pattern.res" && "$SOSO"
 323         elif [ -n "$NSAV" ]; then
 324                 echo "[======= fail $outdir/$pattern.res $NSAV =======]"
 325                 echo >>$xml "<error message='`xmlesc "fail $outdir/$pattern.res $NSAV"`'/>"
 326                 echo >>$xml "<system-out><![CDATA["
 327                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 328                 echo >>$xml "]]></system-out>"
 329                 nok="$nok $pattern"
 330                 echo "$ii" >> "$ERRLIST"
 331                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$NSAV"
 332         elif [ -n "$NFIXME" ]; then
 333                 echo "[======= changed $outdir/$pattern.res $NFIXME ======]"
 334                 echo >>$xml "<error message='`xmlesc "changed $outdir/$pattern.res $NFIXME"`'/>"
 335                 echo >>$xml "<system-out><![CDATA["
 336                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 337                 echo >>$xml "]]></system-out>"
 338                 nok="$nok $pattern"
 339                 echo "$ii" >> "$ERRLIST"
 340                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$NFIXME"
 341         elif [ -s "$outdir/$pattern.res" ]; then
 342                 echo "[=== no sav ===] $outdir/$pattern.res is not empty"
 343                 echo >>$xml "<error message='no sav and not empty'/>"
 344                 echo >>$xml "<system-out><![CDATA["
 345                 cat -v >>$xml -- "$outdir/$pattern.res"
 346                 echo >>$xml "]]></system-out>"
 347                 nos="$nos $pattern"
 348                 echo "$ii" >> "$ERRLIST"
 349                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "sav/"
 350         else
 351                 # no sav but empty res
 352                 echo "[0k] $outdir/$pattern.res is empty"
 353                 ok="$ok $pattern"
 354         fi
 355         if test -s "$outdir/$pattern.cmp.err"; then
 356                 echo >>$xml "<system-err><![CDATA["
 357                 cat -v >>$xml -- "$outdir/$pattern.cmp.err"
 358                 echo >>$xml "]]></system-err>"
 359         fi
 360         echo >>$xml "</testcase>"
 361 }
 362
 363 need_skip()
 364 {
 365         test "$noskip" = true && return 1
 366         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 367                 echo "=> $2: [skip]"
 368                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 369                 return 0
 370         fi
 371         if test -n "$isinterpret" && echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then
 372                 echo "=> $2: [skip exec]"
 373                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 374                 return 0
 375         fi
 376
 377         # Skip by OS
 378         local os_skip_file=$UNAME.skip
 379         if test -e $os_skip_file && echo "$1" | grep -f "$os_skip_file" >/dev/null 2>&1; then
 380                 echo "=> $2: [skip os]"
 381                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 382                 return 0
 383         fi
 384
 385         # Skip by hostname
 386         local host_skip_file=`$HOSTNAME`.skip
 387         if test -e $host_skip_file && echo "$1" | grep -f "$host_skip_file" >/dev/null 2>&1; then
 388                 echo "=> $2: [skip hostname]"
 389                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 390                 return 0
 391         fi
 392         return 1
 393 }
 394
 395 skip_exec()
 396 {
 397         test "$noskip" = true && return 1
 398         for savdir in $savdirs .; do
 399                 local f="$savdir/exec.skip"
 400                 test -f "$f" || continue
 401                 if echo "$1" | grep -f "$f" >/dev/null 2>&1; then
 402                         echo -n "_ no exec by $f; "
 403                         return 0
 404                 fi
 405         done
 406         return 1
 407 }
 408
 409 skip_cc()
 410 {
 411         test "$noskip" = true && return 1
 412         for savdir in $savdirs .; do
 413                 local f="$savdir/cc.skip"
 414                 test -f "$f" || continue
 415                 if echo "$1" | grep -f "$f" >/dev/null 2>&1; then
 416                         return 0
 417                 fi
 418         done
 419         return 1
 420 }
 421
 422 # Check that the resfile ($1) matches some magic strings in `todo` files.
 423 istodo()
 424 {
 425         test "$no" = true && return 1
 426         for savdir in $savdirs .; do
 427                 local f="$savdir/todo"
 428                 test -f "$f" || continue
 429                 if grep -f "$f" "$1" >/dev/null 2>&1; then
 430                         return 0
 431                 fi
 432         done
 433         return 1
 434 }
 435
 436 find_nitc()
 437 {
 438         local name="$enginebinname"
 439         local recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 440         if [[ "x$recent" == "x" ]]; then
 441                 echo "Could not find binary for engine $engine, aborting"
 442                 exit 1
 443         fi
 444         if [ "x$isnode" = "xfalse" ]; then
 445                 echo "Found binary for engine $engine: $recent $OPT"
 446         fi
 447         NITC=$recent
 448 }
 449
 450 verbose=false
 451 isnode=false
 452 autosav=false
 453 stop=false
 454 engine=nitc
 455 noskip=
 456 savdirs=
 457 while [ $stop = false ]; do
 458         case $1 in
 459                 -o) OPT="$OPT $2"; shift; shift;;
 460                 -v) verbose=true; shift;;
 461                 -h) usage; exit;;
 462                 --engine) engine="$2"; shift; shift;;
 463                 --noskip) noskip=true; shift;;
 464                 --outdir) outdir="$2"; shift; shift;;
 465                 --compdir) compdir="$2"; shift; shift;;
 466                 --node) isnode=true; shift;;
 467                 --autosav) autosav=true; shift;;
 468                 *) stop=true
 469         esac
 470 done
 471 enginebinname=$engine
 472 isinterpret=
 473 case $engine in
 474         nitc|nitg)
 475                 engine=nitcs;
 476                 enginebinname=nitc;
 477                 OPT="--separate $OPT --compile-dir $compdir"
 478                 savdirs="sav/nitc-common/"
 479                 ;;
 480         nitcs|nitg-s)
 481                 engine=nitcs;
 482                 enginebinname=nitc;
 483                 OPT="--separate $OPT --compile-dir $compdir"
 484                 savdirs="sav/nitc-common/"
 485                 ;;
 486         nitce|nitg-e)
 487                 engine=nitce;
 488                 enginebinname=nitc;
 489                 OPT="--erasure $OPT --compile-dir $compdir"
 490                 savdirs="sav/nitc-common/"
 491                 ;;
 492         nitcsg|nitg-sg)
 493                 engine=nitcsg;
 494                 enginebinname=nitc;
 495                 OPT="--semi-global $OPT --compile-dir $compdir"
 496                 savdirs="sav/nitc-common/"
 497                 ;;
 498         nitcg|nitg-g)
 499                 engine=nitcg;
 500                 enginebinname=nitc;
 501                 OPT="--global $OPT --compile-dir $compdir"
 502                 savdirs="sav/nitc-common/"
 503                 ;;
 504         nit)
 505                 engine=niti
 506                 isinterpret=true
 507                 ;;
 508         niti)
 509                 enginebinname=nit
 510                 isinterpret=true
 511                 ;;
 512         nitvm)
 513                 isinterpret=true
 514                 enginebinname=nit
 515                 OPT="--vm $OPT"
 516                 savdirs="sav/niti/"
 517                 ;;
 518         nitj)
 519                 engine=nitj;
 520                 OPT="--compile-dir $compdir --ant"
 521                 enginebinname=nitj;
 522                 savdirs="sav/nitc-common/"
 523                 ;;
 524         emscripten)
 525                 enginebinname=nitc
 526                 OPT="-m emscripten_nodejs.nit --semi-global $OPT --compile-dir $compdir"
 527                 savdirs="sav/nitcsg/"
 528                 ;;
 529         *)
 530                 echo "unknown engine $engine"
 531                 exit 1
 532                 ;;
 533 esac
 534
 535 savdirs="sav/`$HOSTNAME` sav/$UNAME sav/$engine $savdirs sav/"
 536
 537 # The default nitc compiler
 538 [ -z "$NITC" ] && find_nitc
 539
 540 # Set NIT_DIR if needed
 541 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 542
 543 # Mark to distinguish files among tests
 544 # MARK=
 545
 546 if [ $# = 0 ]; then
 547         usage;
 548         exit
 549 fi
 550
 551 # CLEAN the out directory
 552 rm -rf "$outdir/" 2>/dev/null
 553 mkdir "$outdir" 2>/dev/null
 554
 555 # File where error tests are outputed
 556 # Old ERRLIST is backuped
 557 ERRLIST=${ERRLIST:-errlist}
 558 ERRLIST_TARGET=$ERRLIST
 559
 560 # Initiate new ERRLIST
 561 if [ "x$ERRLIST" = "x" ]; then
 562         ERRLIST=/dev/null
 563 else
 564         ERRLIST=$ERRLIST.tmp
 565         > "$ERRLIST"
 566 fi
 567
 568 ok=""
 569 nok=""
 570 todos=""
 571
 572 if [ "x$XMLDIR" = "x" ]; then
 573         xml="tests-$engine.xml"
 574 else
 575         sum=`echo $@ | md5sum | cut -f1 -d " "`
 576         xml="$XMLDIR/tests-$engine-$sum.xml"
 577         mkdir -p "$XMLDIR"
 578 fi
 579
 580 echo >$xml "<testsuites><testsuite>"
 581
 582 for ii in "$@"; do
 583         if [ ! -f "$ii" ]; then
 584                 echo "File '$ii' does not exist."
 585                 continue
 586         fi
 587         f=`basename -- "$ii" .nit`
 588
 589         pack="tests.${engine}".`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'`
 590
 591         # Sould we skip the file for this engine?
 592         need_skip "$f" "$f" "$pack" && continue
 593
 594         local_tmp=${ii/../AA}
 595         if [ "x$local_tmp" = "x$ii" ]; then
 596                 includes="-I . -I ../lib/core -I ../lib/core/collection -I alt"
 597         else
 598                 includes="-I alt"
 599         fi
 600
 601         for i in "$ii" `./alterner.pl --start '#' --altsep '_' -- "$ii"`; do
 602                 bf=`basename -- "$i" .nit`
 603                 ff="$outdir/$bf"
 604
 605                 # Sould we skip the alternative for this engine?
 606                 need_skip "$bf" "$bf" "$pack" && continue
 607
 608                 echo -n "=> $bf: "
 609
 610                 if [ -f "$f.inputs" ]; then
 611                         inputs="$f.inputs"
 612                         export MNIT_READ_INPUT="$f.inputs"
 613                 else
 614                         inputs=/dev/null
 615                         export MNIT_READ_INPUT=/dev/null
 616                 fi
 617
 618                 ffout="$ff.bin"
 619                 if [ "$engine" = "emscripten" ]; then
 620                         ffout="$ff.bin.js"
 621                 fi
 622
 623                 if [ -n "$isinterpret" ]; then
 624                         cat > "$ff.bin" <<END
 625 exec $NITC --no-color $OPT $includes -- $(printf '%q' "$i") "\$@"
 626 END
 627                         chmod +x "$ff.bin"
 628                         > "$ff.cmp.err"
 629                         > "$ff.compile.log"
 630                         ERR=0
 631                         echo 0.0 > "$ff.time.out"
 632                 else
 633                         if skip_cc "$bf"; then
 634                                 nocc="--no-cc"
 635                         else
 636                                 nocc=
 637                         fi
 638                         # Compile
 639                         if [ "x$verbose" = "xtrue" ]; then
 640                                 echo ""
 641                                 echo $NITC --no-color $OPT -o "$ffout" "$includes" $nocc "$i"
 642                         fi
 643                         NIT_NO_STACK=1 JNI_LIB_PATH=$JNI_LIB_PATH JAVA_HOME=$JAVA_HOME \
 644                                 saferun -o "$ff.time.out" $NITC --no-color $OPT -o "$ffout" $includes $nocc "$i" 2> "$ff.cmp.err" > "$ff.compile.log"
 645                         ERR=$?
 646                         if [ "x$verbose" = "xtrue" ]; then
 647                                 cat -- "$ff.compile.log"
 648                                 cat >&2 -- "$ff.cmp.err"
 649                         fi
 650                         # Clean
 651                         rm -r "$compdir" 2>/dev/null
 652                 fi
 653                 if [ "$engine" = "emscripten" ]; then
 654                         echo > "$ff.bin" "nodejs $ffout \"\$@\""
 655                         chmod +x "$ff.bin"
 656                         if grep "Fatal Error: more than one primitive class" "$ff.compile.log" > /dev/null; then
 657                                 echo " [skip] do no not imports kernel"
 658                                 echo >>$xml "<testcase classname='`xmlesc "$pack"`' name='`xmlesc "$bf"`' `timestamp`><skipped/></testcase>"
 659                                 continue
 660                         fi
 661                 fi
 662                 if [ "$ERR" != 0 ]; then
 663                         echo -n "! "
 664                         cat -- "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 665                         process_result "$bf" "$bf" "$pack"
 666                 elif [ -n "$nocc" ]; then
 667                         # not compiled
 668                         echo -n "nocc "
 669                         > "$ff.res"
 670                         process_result "$bf" "$bf" "$pack"
 671                 elif [ -x "$ff.bin" ]; then
 672                         if skip_exec "$bf"; then
 673                                 # No exec
 674                                 > "$ff.res"
 675                                 process_result "$bf" "$bf" "$pack"
 676                                 break
 677                         fi
 678                         echo -n ". "
 679                         # Execute
 680                         args=""
 681                         if [ "x$verbose" = "xtrue" ]; then
 682                                 echo ""
 683                                 echo "NIT_NO_STACK=1 $ff.bin" $args
 684                         fi
 685                         NIT_NO_STACK=1 LD_LIBRARY_PATH=$JNI_LIB_PATH WRITE="$ff.write" \
 686                                 saferun -a -o "$ff.time.out" "$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 687                         mv "$ff.time.out" "$ff.times.out"
 688                         awk '{ SUM += $1} END { print SUM }' "$ff.times.out" > "$ff.time.out"
 689
 690                         if [ "x$verbose" = "xtrue" ]; then
 691                                 cat -- "$ff.res"
 692                                 cat >&2 -- "$ff.err"
 693                         fi
 694                         if [ -f "$ff.write" ]; then
 695                                 cat -- "$ff.write" >> "$ff.res"
 696                         elif [ -d "$ff.write" ]; then
 697                                 LANG=C /bin/ls -F "$ff.write" >> "$ff.res"
 698                         fi
 699                         cp -- "$ff.res"  "$ff.res2"
 700                         cat -- "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
 701                         process_result "$bf" "$bf" "$pack"
 702
 703                         if [ -f "$f.args" ]; then
 704                                 fargs=$f.args
 705                                 cptr=0
 706                                 while read line; do
 707                                         ((cptr=cptr+1))
 708                                         args="$line"
 709                                         bff=$bf"_args"$cptr
 710                                         fff=$ff"_args"$cptr
 711                                         name="$bf args $cptr"
 712
 713                                         # Sould we skip the input for this engine?
 714                                         need_skip "$bff" "  $name" "$pack" && continue
 715
 716                                         # use a specific inputs file, if required
 717                                         if [ -f "$bff.inputs" ]; then
 718                                                 ffinputs="$bff.inputs"
 719                                         else
 720                                                 ffinputs="$inputs"
 721                                         fi
 722
 723                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 724                                         if [ "x$verbose" = "xtrue" ]; then
 725                                                 echo ""
 726                                                 echo "NIT_NO_STACK=1 $ff.bin" $args
 727                                         fi
 728                                         echo -n "==> $name "
 729                                         echo "$ff.bin $args" > "$fff.bin"
 730                                         chmod +x "$fff.bin"
 731                                         WRITE="$fff.write" saferun -o "$fff.time.out" sh -c "NIT_NO_STACK=1 $fff.bin < $ffinputs > $fff.res 2>$fff.err"
 732                                         if [ "x$verbose" = "xtrue" ]; then
 733                                                 cat -- "$fff.res"
 734                                                 cat >&2 -- "$fff.err"
 735                                         fi
 736                                         if [ -f "$fff.write" ]; then
 737                                                 cat -- "$fff.write" >> "$fff.res"
 738                                         elif [ -d "$fff.write" ]; then
 739                                                 LANG=C /bin/ls -F -- "$fff.write" >> "$fff.res"
 740                                         fi
 741                                         if [ -s "$fff.err" ]; then
 742                                                 cp -- "$fff.res"  "$fff.res2"
 743                                                 cat -- "$fff.err" "$fff.res2" > "$fff.res"
 744                                         fi
 745                                         process_result "$bff" "  $name" "$pack"
 746                                 done < "$fargs"
 747                         fi
 748                 elif [ -f "$ff.bin" ]; then
 749                         #Not executable (platform?)"
 750                         > "$ff.res"
 751                         process_result "$bf" "$bf" "$pack"
 752                 else
 753                         echo -n "! "
 754                         cat -- "$ff.cmp.err" > "$ff.res"
 755                         echo "Compilation error" > "$ff.res"
 756                         process_result "$bf" "$bf" "$pack"
 757                 fi
 758         done
 759 done
 760
 761 if [ "x$isnode" = "xfalse" ]; then
 762         echo "engine: $engine ($enginebinname $OPT)"
 763         echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 764
 765         if [ -n "$nok" ]; then
 766                 echo "fail: $nok"
 767                 echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 768         fi
 769         if [ -n "$nos" ]; then
 770                 echo "no sav: $nos"
 771         fi
 772         if [ -n "$todos" ]; then
 773                 echo "todo/fixme: $todos"
 774         fi
 775         if [ -n "$remains" ]; then
 776                 echo "sav that remains: $remains"
 777         fi
 778 fi
 779
 780 # write $ERRLIST
 781 if [ "x$ERRLIST" != "x" ]; then
 782         if [ -f "$ERRLIST_TARGET" ]; then
 783                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 784         fi
 785         uniq $ERRLIST > $ERRLIST_TARGET
 786         rm $ERRLIST
 787 fi
 788
 789 echo >>$xml "</testsuite></testsuites>"
 790
 791 if [ -n "$nok" ]; then
 792         exit 1
 793 else
 794         exit 0
 795 fi