tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22 export LC_ALL=C
  23 export NIT_TESTING=true
  24 # Use the pid as a collision prevention
  25 export NIT_TESTING_ID=$$
  26 export NIT_SRAND=0
  27
  28 unset NIT_DIR
  29
  30 # Get the first Java lib available
  31 if which_java=$(which javac 2>/dev/null); then
  32         JAVA_HOME=$(dirname $(dirname $(readlink -f "$which_java")))
  33
  34         shopt -s nullglob
  35         paths=`echo $JAVA_HOME/jre/lib/*/{client,server}/libjvm.so`
  36         paths=($paths)
  37         JNI_LIB_PATH=`dirname ${paths[0]}`
  38         shopt -u nullglob
  39 fi
  40
  41 outdir="out"
  42 compdir="nit_compile"
  43
  44 # User CPU time limit (in seconds)
  45 # Is used to avoid to CPU intensive test (infinite loops). See ulimit -t
  46 usertimelimit=600 # 1 CPU minute
  47
  48 # Real-time limit (in seconds)
  49 # Is used to avoid waiting or sleeping tests.
  50 # Require timeout or timelimit, or else is not used.
  51 realtimelimit=300 # 5 min
  52
  53 # User limit for write files (in kilo-bytes)
  54 # Is used to avoid execution that loop and fill the hard drive. See ulimit -f
  55 # Note that a test might require a lot of temporary disk space (eg. nitc+gcc)
  56 filelimit=100000 # ~100MB
  57
  58 # Limit (in bytes) for generated .res file.
  59 # Larger ones are truncated and will fail tests
  60 # Is used to avoid processing huge crappy res file (diff, xml, etc)
  61 reslimit=100000 # ~100KB
  62
  63 usage()
  64 {
  65         e=`basename "$0"`
  66         cat<<END
  67 Usage: $e [options] modulenames
  68 -o option   Pass option to the engine
  69 -v          Verbose (show tests steps)
  70 -h          This help
  71 --engine    Use a specific engine (default=nitc)
  72 --noskip    Do not skip a test even if the .skip file matches
  73 --outdir    Use a specific output folder (default=out/)
  74 --compdir   Use a specific temporary compilation folder (default=$compdir)
  75 --node      Run as a node in parallel, will not output context information
  76 --autosav   Copy the .res files directly in the sav folder overriding existing .res files
  77 END
  78 }
  79
  80 # Run a command with a timeout and a time count.
  81 # Options:
  82 #   -o file    write the user time into file (REQUIRED). see `-o` in `man time`
  83 #   -a         append the time to the file (instead of overwriting it). see `-a` in `man time`
  84 saferun()
  85 {
  86         local stop=false
  87         local o=
  88         local a=
  89         while [ $stop = false ]; do
  90                 case "$1" in
  91                         -o) o="$2"; shift; shift;;
  92                         -a) a="-a"; shift;;
  93                         *) stop=true
  94                 esac
  95         done
  96         (
  97         ulimit -f "$filelimit" 2> /dev/null
  98         ulimit -t "$usertimelimit" 2> /dev/null
  99         if test -d "$1"; then
 100                 find $1 | sort
 101         elif test -n "$TIME"; then
 102                 $TIME -o "$o" $a $TIMEOUT "$@"
 103         else
 104                 if test -n "$a"; then echo 0 >> "$o"; else echo 0 > "$o"; fi
 105                 $TIMEOUT "$@"
 106         fi
 107         )
 108 }
 109
 110 # Output a timestamp attribute for XML, or an empty line
 111 timestamp()
 112 {
 113         if test -n "$TIMESTAMP"; then
 114                 echo "timestamp='`$TIMESTAMP`'"
 115         else
 116                 echo ""
 117         fi
 118
 119 }
 120
 121 # Get platform specific commands ##########################
 122
 123 # Detect a working timeout
 124 if sh -c "timelimit echo" 1>/dev/null 2>&1; then
 125         TIMEOUT="timelimit -t $realtimelimit"
 126 elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
 127         TIMEOUT="timeout ${realtimelimit}s"
 128 else
 129         echo "No timelimit or timeout command detected. Tests may hang :("
 130 fi
 131
 132 # Detect a working time command
 133 if env time --quiet -f%U true 2>/dev/null; then
 134         TIME="env time --quiet -f%U"
 135 elif env time -f%U true 2>/dev/null; then
 136         TIME="env time -f%U"
 137 elif env gtime -f%U true 2>/dev/null; then
 138         TIME="env gtime -f%U"
 139 else
 140         TIME=
 141 fi
 142
 143 # Detect a working date command
 144 if date -Iseconds >/dev/null 2>&1; then
 145         TIMESTAMP="date -Iseconds"
 146 else
 147         TIMESTAMP=
 148 fi
 149
 150 # Detect a working hostname command
 151 if hostname --version 2>&1 | grep coreutils >/dev/null 2>&1; then
 152         HOSTNAME="hostname"
 153 else
 154         HOSTNAME="hostname -s"
 155 fi
 156
 157 # $1 is the pattern of the test
 158 # $2 is the file to compare to
 159 # the result is:
 160 #   0: if the file to compare to do not exists
 161 #   1: if the file match
 162 #   2: if the file match with soso
 163 #   3: if the file do not match
 164 function compare_to_result()
 165 {
 166         local pattern="$1"
 167         local sav="$2"
 168         if [ ! -r "$sav" ]; then return 0; fi
 169         test "`cat -- "$sav"`" = "UNDEFINED" && return 1
 170         diff -u --strip-trailing-cr -- "$sav" "$outdir/$pattern.res" > "$outdir/$pattern.diff.sav.log"
 171         if [ "$?" == 0 ]; then
 172                 return 1
 173         fi
 174         sed '/[Ww]arning/d;/[Ee]rror/d' "$outdir/$pattern.res" > "$outdir/$pattern.res2"
 175         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "$outdir/$pattern.sav2"
 176         grep '[Ee]rror' "$outdir/$pattern.res" >/dev/null && echo "Error" >> "$outdir/$pattern.res2"
 177         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "$outdir/$pattern.sav2"
 178         diff -u --strip-trailing-cr "$outdir/$pattern.sav2" "$outdir/$pattern.res2" > "$outdir/$pattern.diff.sav.log2"
 179         if [ "$?" == 0 ]; then
 180                 return 2
 181         else
 182                 return 3
 183         fi
 184 }
 185
 186 function xmlesc()
 187 {
 188         sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g; s/'"'"'/\&#39;/g'<<EOF
 189 $*
 190 EOF
 191 }
 192
 193 # As argument: the pattern used for the file
 194 function process_result()
 195 {
 196         # Result
 197         local pattern=$1
 198         local description=$2
 199         local pack=$3
 200         local SAV=""
 201         local NSAV=""
 202         local FIXME=""
 203         local NFIXME=""
 204         local SOSO=""
 205         local NSOSO=""
 206         local SOSOF=""
 207         local NSOSOF=""
 208         local OLD=""
 209         local LIST=""
 210         local FIRST=""
 211
 212         # Truncate too big res file
 213         local size=$(wc -c < "$outdir/$pattern.res")
 214         if test -n "$reslimit" -a "$size" -gt "$reslimit"; then
 215                 # The most portable way to truncate a file is with Perl
 216                 perl -e "truncate \"$outdir/$pattern.res\", $reslimit;"
 217                 echo "***TRUNCATED***" >> "$outdir/$pattern.res"
 218         fi
 219
 220         echo >>$xml "<testcase classname='`xmlesc "$pack"`' name='`xmlesc "$description"`' time='`cat -- "$outdir/$pattern.time.out"`' `timestamp`>"
 221         #for sav in "sav/$engine/fixme/$pattern.res" "sav/$engine/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
 222         for savdir in $savdirs; do
 223                 local sav=$savdir/fixme/$pattern.res
 224                 compare_to_result "$pattern" "$sav"
 225                 case "$?" in
 226                         0)
 227                                 ;; # no file
 228                         1)
 229                                 OLD="$LIST"
 230                                 FIXME="$sav"
 231                                 LIST="$LIST $sav"
 232                                 ;;
 233                         2)
 234                                 if [ -z "$FIRST" ]; then
 235                                         SOSOF="$sav"
 236                                         FIRST="$sav"
 237                                 fi
 238                                 LIST="$LIST $sav"
 239                                 ;;
 240                         3)
 241                                 if [ -z "$FIRST" ]; then
 242                                         NFIXME="$sav"
 243                                         FIRST="$sav"
 244                                 fi
 245                                 LIST="$LIST $sav"
 246                                 ;;
 247                 esac
 248
 249                 sav=$savdir/$pattern.res
 250                 compare_to_result "$pattern" "$sav"
 251                 case "$?" in
 252                         0)
 253                                 ;; # no file
 254                         1)
 255                                 OLD="$LIST"
 256                                 SAV="$sav"
 257                                 LIST="$LIST $sav"
 258                                 ;;
 259                         2)
 260                                 if [ -z "$FIRST" ]; then
 261                                         SOSO="$sav"
 262                                         FIRST="$sav"
 263                                 fi
 264                                 LIST="$LIST $sav"
 265                                 ;;
 266                         3)
 267                                 if [ -z "$FIRST" ]; then
 268                                         NSAV="$sav"
 269                                         FIRST="$sav"
 270                                 fi
 271                                 LIST="$LIST $sav"
 272                                 ;;
 273                 esac
 274         done
 275         OLD=`echo "$OLD" | sed -e 's/   */ /g' -e 's/^ //' -e 's/ $//'`
 276         istodo  "$outdir/$pattern.res"
 277         NYI="$?"
 278         if [ -n "$SAV" ]; then
 279                 if [ -n "$OLD" ]; then
 280                         echo "[*ok*] $outdir/$pattern.res $SAV - but $OLD remains!"
 281                         echo >>$xml "<error message='`xmlesc "ok $outdir/$pattern.res - but $OLD remains"`'/>"
 282                         remains="$remains $OLD"
 283                         test "$autosav" = "true" && rm "$OLD"
 284                 else
 285                         echo "[ok] $outdir/$pattern.res $SAV"
 286                 fi
 287                 ok="$ok $pattern"
 288         elif [ -n "$FIXME" ]; then
 289                 if [ -n "$OLD" ]; then
 290                         echo "[*fixme*] $outdir/$pattern.res $FIXME - but $OLD remains!"
 291                         echo >>$xml "<error message='`xmlesc "ok $outdir/$pattern.res - but $OLD remains"`'/>"
 292                         remains="$remains $OLD"
 293                         test "$autosav" = "true" && rm "$OLD"
 294                 else
 295                         echo "[fixme] $outdir/$pattern.res $FIXME"
 296                         echo >>$xml "<skipped/>"
 297                 fi
 298                 todos="$todos $pattern"
 299         elif [ "x$NYI" = "x0" ]; then
 300                 echo "[todo] $outdir/$pattern.res -> not yet implemented"
 301                 echo >>$xml "<skipped/>"
 302                 todos="$todos $pattern"
 303         elif [ -n "$SOSO" ]; then
 304                 echo "[======= soso $outdir/$pattern.res $SOSO =======]"
 305                 echo >>$xml "<error message='`xmlesc "soso $outdir/$pattern.res $SOSO"`'/>"
 306                 echo >>$xml "<system-out><![CDATA["
 307                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 308                 echo >>$xml "]]></system-out>"
 309                 nok="$nok $pattern"
 310                 echo "$ii" >> "$ERRLIST"
 311                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$SOSO"
 312         elif [ -n "$SOSOF" ]; then
 313                 echo "[======= fixme soso $outdir/$pattern.res $SOSOF =======]"
 314                 echo >>$xml "<error message='`xmlesc "soso $outdir/$pattern.res $SOSO"`'/>"
 315                 echo >>$xml "<system-out><![CDATA["
 316                 cat -v  -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 317                 echo >>$xml "]]></system-out>"
 318                 nok="$nok $pattern"
 319                 echo "$ii" >> "$ERRLIST"
 320                 test "$autosav" = "true" && cp "$outdir/$pattern.res" && "$SOSO"
 321         elif [ -n "$NSAV" ]; then
 322                 echo "[======= fail $outdir/$pattern.res $NSAV =======]"
 323                 echo >>$xml "<error message='`xmlesc "fail $outdir/$pattern.res $NSAV"`'/>"
 324                 echo >>$xml "<system-out><![CDATA["
 325                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 326                 echo >>$xml "]]></system-out>"
 327                 nok="$nok $pattern"
 328                 echo "$ii" >> "$ERRLIST"
 329                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$NSAV"
 330         elif [ -n "$NFIXME" ]; then
 331                 echo "[======= changed $outdir/$pattern.res $NFIXME ======]"
 332                 echo >>$xml "<error message='`xmlesc "changed $outdir/$pattern.res $NFIXME"`'/>"
 333                 echo >>$xml "<system-out><![CDATA["
 334                 cat -v -- "$outdir/$pattern.diff.sav.log" | head >>$xml -n 50
 335                 echo >>$xml "]]></system-out>"
 336                 nok="$nok $pattern"
 337                 echo "$ii" >> "$ERRLIST"
 338                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "$NFIXME"
 339         elif [ -s "$outdir/$pattern.res" ]; then
 340                 echo "[=== no sav ===] $outdir/$pattern.res is not empty"
 341                 echo >>$xml "<error message='no sav and not empty'/>"
 342                 echo >>$xml "<system-out><![CDATA["
 343                 cat -v >>$xml -- "$outdir/$pattern.res"
 344                 echo >>$xml "]]></system-out>"
 345                 nos="$nos $pattern"
 346                 echo "$ii" >> "$ERRLIST"
 347                 test "$autosav" = "true" && cp "$outdir/$pattern.res" "sav/"
 348         else
 349                 # no sav but empty res
 350                 echo "[0k] $outdir/$pattern.res is empty"
 351                 ok="$ok $pattern"
 352         fi
 353         if test -s "$outdir/$pattern.cmp.err"; then
 354                 echo >>$xml "<system-err><![CDATA["
 355                 cat -v >>$xml -- "$outdir/$pattern.cmp.err"
 356                 echo >>$xml "]]></system-err>"
 357         fi
 358         echo >>$xml "</testcase>"
 359 }
 360
 361 need_skip()
 362 {
 363         test "$noskip" = true && return 1
 364         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 365                 echo "=> $2: [skip]"
 366                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 367                 return 0
 368         fi
 369         if test -n "$isinterpret" && echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then
 370                 echo "=> $2: [skip exec]"
 371                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 372                 return 0
 373         fi
 374
 375         # Skip by OS
 376         local os_skip_file=`uname`.skip
 377         if test -e $os_skip_file && echo "$1" | grep -f "$os_skip_file" >/dev/null 2>&1; then
 378                 echo "=> $2: [skip os]"
 379                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 380                 return 0
 381         fi
 382
 383         # Skip by hostname
 384         local host_skip_file=`$HOSTNAME`.skip
 385         if test -e $host_skip_file && echo "$1" | grep -f "$host_skip_file" >/dev/null 2>&1; then
 386                 echo "=> $2: [skip hostname]"
 387                 echo >>$xml "<testcase classname='`xmlesc "$3"`' name='`xmlesc "$2"`' `timestamp`><skipped/></testcase>"
 388                 return 0
 389         fi
 390         return 1
 391 }
 392
 393 skip_exec()
 394 {
 395         test "$noskip" = true && return 1
 396         for savdir in $savdirs .; do
 397                 local f="$savdir/exec.skip"
 398                 test -f "$f" || continue
 399                 if echo "$1" | grep -f "$f" >/dev/null 2>&1; then
 400                         echo -n "_ no exec by $f; "
 401                         return 0
 402                 fi
 403         done
 404         return 1
 405 }
 406
 407 skip_cc()
 408 {
 409         test "$noskip" = true && return 1
 410         for savdir in $savdirs .; do
 411                 local f="$savdir/cc.skip"
 412                 test -f "$f" || continue
 413                 if echo "$1" | grep -f "$f" >/dev/null 2>&1; then
 414                         return 0
 415                 fi
 416         done
 417         return 1
 418 }
 419
 420 # Check that the resfile ($1) matches some magic strings in `todo` files.
 421 istodo()
 422 {
 423         test "$no" = true && return 1
 424         for savdir in $savdirs .; do
 425                 local f="$savdir/todo"
 426                 test -f "$f" || continue
 427                 if grep -f "$f" "$1" >/dev/null 2>&1; then
 428                         return 0
 429                 fi
 430         done
 431         return 1
 432 }
 433
 434 find_nitc()
 435 {
 436         local name="$enginebinname"
 437         local recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 438         if [[ "x$recent" == "x" ]]; then
 439                 echo "Could not find binary for engine $engine, aborting"
 440                 exit 1
 441         fi
 442         if [ "x$isnode" = "xfalse" ]; then
 443                 echo "Found binary for engine $engine: $recent $OPT"
 444         fi
 445         NITC=$recent
 446 }
 447
 448 verbose=false
 449 isnode=false
 450 autosav=false
 451 stop=false
 452 engine=nitc
 453 noskip=
 454 savdirs=
 455 while [ $stop = false ]; do
 456         case $1 in
 457                 -o) OPT="$OPT $2"; shift; shift;;
 458                 -v) verbose=true; shift;;
 459                 -h) usage; exit;;
 460                 --engine) engine="$2"; shift; shift;;
 461                 --noskip) noskip=true; shift;;
 462                 --outdir) outdir="$2"; shift; shift;;
 463                 --compdir) compdir="$2"; shift; shift;;
 464                 --node) isnode=true; shift;;
 465                 --autosav) autosav=true; shift;;
 466                 *) stop=true
 467         esac
 468 done
 469 enginebinname=$engine
 470 isinterpret=
 471 case $engine in
 472         nitc|nitg)
 473                 engine=nitcs;
 474                 enginebinname=nitc;
 475                 OPT="--separate $OPT --compile-dir $compdir"
 476                 savdirs="sav/nitc-common/"
 477                 ;;
 478         nitcs|nitg-s)
 479                 engine=nitcs;
 480                 enginebinname=nitc;
 481                 OPT="--separate $OPT --compile-dir $compdir"
 482                 savdirs="sav/nitc-common/"
 483                 ;;
 484         nitce|nitg-e)
 485                 engine=nitce;
 486                 enginebinname=nitc;
 487                 OPT="--erasure $OPT --compile-dir $compdir"
 488                 savdirs="sav/nitc-common/"
 489                 ;;
 490         nitcsg|nitg-sg)
 491                 engine=nitcsg;
 492                 enginebinname=nitc;
 493                 OPT="--semi-global $OPT --compile-dir $compdir"
 494                 savdirs="sav/nitc-common/"
 495                 ;;
 496         nitcg|nitg-g)
 497                 engine=nitcg;
 498                 enginebinname=nitc;
 499                 OPT="--global $OPT --compile-dir $compdir"
 500                 savdirs="sav/nitc-common/"
 501                 ;;
 502         nit)
 503                 engine=niti
 504                 isinterpret=true
 505                 ;;
 506         niti)
 507                 enginebinname=nit
 508                 isinterpret=true
 509                 ;;
 510         nitvm)
 511                 isinterpret=true
 512                 enginebinname=nit
 513                 OPT="--vm $OPT"
 514                 savdirs="sav/niti/"
 515                 ;;
 516         nitj)
 517                 engine=nitj;
 518                 OPT="--compile-dir $compdir --ant"
 519                 enginebinname=nitj;
 520                 savdirs="sav/nitc-common/"
 521                 ;;
 522         emscripten)
 523                 enginebinname=nitc
 524                 OPT="-m emscripten_nodejs.nit --semi-global $OPT --compile-dir $compdir"
 525                 savdirs="sav/nitcsg/"
 526                 ;;
 527         *)
 528                 echo "unknown engine $engine"
 529                 exit 1
 530                 ;;
 531 esac
 532
 533 savdirs="sav/`$HOSTNAME` sav/`uname` sav/$engine $savdirs sav/"
 534
 535 # The default nitc compiler
 536 [ -z "$NITC" ] && find_nitc
 537
 538 # Set NIT_DIR if needed
 539 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 540
 541 # Mark to distinguish files among tests
 542 # MARK=
 543
 544 if [ $# = 0 ]; then
 545         usage;
 546         exit
 547 fi
 548
 549 # CLEAN the out directory
 550 rm -rf "$outdir/" 2>/dev/null
 551 mkdir "$outdir" 2>/dev/null
 552
 553 # File where error tests are outputed
 554 # Old ERRLIST is backuped
 555 ERRLIST=${ERRLIST:-errlist}
 556 ERRLIST_TARGET=$ERRLIST
 557
 558 # Initiate new ERRLIST
 559 if [ "x$ERRLIST" = "x" ]; then
 560         ERRLIST=/dev/null
 561 else
 562         ERRLIST=$ERRLIST.tmp
 563         > "$ERRLIST"
 564 fi
 565
 566 ok=""
 567 nok=""
 568 todos=""
 569
 570 if [ "x$XMLDIR" = "x" ]; then
 571         xml="tests-$engine.xml"
 572 else
 573         sum=`echo $@ | md5sum | cut -f1 -d " "`
 574         xml="$XMLDIR/tests-$engine-$sum.xml"
 575         mkdir -p "$XMLDIR"
 576 fi
 577
 578 echo >$xml "<testsuites><testsuite>"
 579
 580 for ii in "$@"; do
 581         if [ ! -f "$ii" ]; then
 582                 echo "File '$ii' does not exist."
 583                 continue
 584         fi
 585         f=`basename -- "$ii" .nit`
 586
 587         pack="tests.${engine}".`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'`
 588
 589         # Sould we skip the file for this engine?
 590         need_skip "$f" "$f" "$pack" && continue
 591
 592         local_tmp=${ii/../AA}
 593         if [ "x$local_tmp" = "x$ii" ]; then
 594                 includes="-I . -I ../lib/core -I ../lib/core/collection -I alt"
 595         else
 596                 includes="-I alt"
 597         fi
 598
 599         for i in "$ii" `./alterner.pl --start '#' --altsep '_' -- "$ii"`; do
 600                 bf=`basename -- "$i" .nit`
 601                 ff="$outdir/$bf"
 602
 603                 # Sould we skip the alternative for this engine?
 604                 need_skip "$bf" "$bf" "$pack" && continue
 605
 606                 echo -n "=> $bf: "
 607
 608                 if [ -f "$f.inputs" ]; then
 609                         inputs="$f.inputs"
 610                         export MNIT_READ_INPUT="$f.inputs"
 611                 else
 612                         inputs=/dev/null
 613                         export MNIT_READ_INPUT=/dev/null
 614                 fi
 615
 616                 ffout="$ff.bin"
 617                 if [ "$engine" = "emscripten" ]; then
 618                         ffout="$ff.bin.js"
 619                 fi
 620
 621                 if [ -n "$isinterpret" ]; then
 622                         cat > "$ff.bin" <<END
 623 exec $NITC --no-color $OPT $includes -- $(printf '%q' "$i") "\$@"
 624 END
 625                         chmod +x "$ff.bin"
 626                         > "$ff.cmp.err"
 627                         > "$ff.compile.log"
 628                         ERR=0
 629                         echo 0.0 > "$ff.time.out"
 630                 else
 631                         if skip_cc "$bf"; then
 632                                 nocc="--no-cc"
 633                         else
 634                                 nocc=
 635                         fi
 636                         # Compile
 637                         if [ "x$verbose" = "xtrue" ]; then
 638                                 echo ""
 639                                 echo $NITC --no-color $OPT -o "$ffout" "$includes" $nocc "$i"
 640                         fi
 641                         NIT_NO_STACK=1 JNI_LIB_PATH=$JNI_LIB_PATH JAVA_HOME=$JAVA_HOME \
 642                                 saferun -o "$ff.time.out" $NITC --no-color $OPT -o "$ffout" $includes $nocc "$i" 2> "$ff.cmp.err" > "$ff.compile.log"
 643                         ERR=$?
 644                         if [ "x$verbose" = "xtrue" ]; then
 645                                 cat -- "$ff.compile.log"
 646                                 cat >&2 -- "$ff.cmp.err"
 647                         fi
 648                         # Clean
 649                         rm -r "$compdir" 2>/dev/null
 650                 fi
 651                 if [ "$engine" = "emscripten" ]; then
 652                         echo > "$ff.bin" "nodejs $ffout \"\$@\""
 653                         chmod +x "$ff.bin"
 654                         if grep "Fatal Error: more than one primitive class" "$ff.compile.log" > /dev/null; then
 655                                 echo " [skip] do no not imports kernel"
 656                                 echo >>$xml "<testcase classname='`xmlesc "$pack"`' name='`xmlesc "$bf"`' `timestamp`><skipped/></testcase>"
 657                                 continue
 658                         fi
 659                 fi
 660                 if [ "$ERR" != 0 ]; then
 661                         echo -n "! "
 662                         cat -- "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 663                         process_result "$bf" "$bf" "$pack"
 664                 elif [ -n "$nocc" ]; then
 665                         # not compiled
 666                         echo -n "nocc "
 667                         > "$ff.res"
 668                         process_result "$bf" "$bf" "$pack"
 669                 elif [ -x "$ff.bin" ]; then
 670                         if skip_exec "$bf"; then
 671                                 # No exec
 672                                 > "$ff.res"
 673                                 process_result "$bf" "$bf" "$pack"
 674                                 break
 675                         fi
 676                         echo -n ". "
 677                         # Execute
 678                         args=""
 679                         if [ "x$verbose" = "xtrue" ]; then
 680                                 echo ""
 681                                 echo "NIT_NO_STACK=1 $ff.bin" $args
 682                         fi
 683                         NIT_NO_STACK=1 LD_LIBRARY_PATH=$JNI_LIB_PATH WRITE="$ff.write" \
 684                                 saferun -a -o "$ff.time.out" "$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 685                         mv "$ff.time.out" "$ff.times.out"
 686                         awk '{ SUM += $1} END { print SUM }' "$ff.times.out" > "$ff.time.out"
 687
 688                         if [ "x$verbose" = "xtrue" ]; then
 689                                 cat -- "$ff.res"
 690                                 cat >&2 -- "$ff.err"
 691                         fi
 692                         if [ -f "$ff.write" ]; then
 693                                 cat -- "$ff.write" >> "$ff.res"
 694                         elif [ -d "$ff.write" ]; then
 695                                 LANG=C /bin/ls -F "$ff.write" >> "$ff.res"
 696                         fi
 697                         cp -- "$ff.res"  "$ff.res2"
 698                         cat -- "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
 699                         process_result "$bf" "$bf" "$pack"
 700
 701                         if [ -f "$f.args" ]; then
 702                                 fargs=$f.args
 703                                 cptr=0
 704                                 while read line; do
 705                                         ((cptr=cptr+1))
 706                                         args="$line"
 707                                         bff=$bf"_args"$cptr
 708                                         fff=$ff"_args"$cptr
 709                                         name="$bf args $cptr"
 710
 711                                         # Sould we skip the input for this engine?
 712                                         need_skip "$bff" "  $name" "$pack" && continue
 713
 714                                         # use a specific inputs file, if required
 715                                         if [ -f "$bff.inputs" ]; then
 716                                                 ffinputs="$bff.inputs"
 717                                         else
 718                                                 ffinputs="$inputs"
 719                                         fi
 720
 721                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 722                                         if [ "x$verbose" = "xtrue" ]; then
 723                                                 echo ""
 724                                                 echo "NIT_NO_STACK=1 $ff.bin" $args
 725                                         fi
 726                                         echo -n "==> $name "
 727                                         echo "$ff.bin $args" > "$fff.bin"
 728                                         chmod +x "$fff.bin"
 729                                         WRITE="$fff.write" saferun -o "$fff.time.out" sh -c "NIT_NO_STACK=1 $fff.bin < $ffinputs > $fff.res 2>$fff.err"
 730                                         if [ "x$verbose" = "xtrue" ]; then
 731                                                 cat -- "$fff.res"
 732                                                 cat >&2 -- "$fff.err"
 733                                         fi
 734                                         if [ -f "$fff.write" ]; then
 735                                                 cat -- "$fff.write" >> "$fff.res"
 736                                         elif [ -d "$fff.write" ]; then
 737                                                 LANG=C /bin/ls -F -- "$fff.write" >> "$fff.res"
 738                                         fi
 739                                         if [ -s "$fff.err" ]; then
 740                                                 cp -- "$fff.res"  "$fff.res2"
 741                                                 cat -- "$fff.err" "$fff.res2" > "$fff.res"
 742                                         fi
 743                                         process_result "$bff" "  $name" "$pack"
 744                                 done < "$fargs"
 745                         fi
 746                 elif [ -f "$ff.bin" ]; then
 747                         #Not executable (platform?)"
 748                         > "$ff.res"
 749                         process_result "$bf" "$bf" "$pack"
 750                 else
 751                         echo -n "! "
 752                         cat -- "$ff.cmp.err" > "$ff.res"
 753                         echo "Compilation error" > "$ff.res"
 754                         process_result "$bf" "$bf" "$pack"
 755                 fi
 756         done
 757 done
 758
 759 if [ "x$isnode" = "xfalse" ]; then
 760         echo "engine: $engine ($enginebinname $OPT)"
 761         echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 762
 763         if [ -n "$nok" ]; then
 764                 echo "fail: $nok"
 765                 echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 766         fi
 767         if [ -n "$nos" ]; then
 768                 echo "no sav: $nos"
 769         fi
 770         if [ -n "$todos" ]; then
 771                 echo "todo/fixme: $todos"
 772         fi
 773         if [ -n "$remains" ]; then
 774                 echo "sav that remains: $remains"
 775         fi
 776 fi
 777
 778 # write $ERRLIST
 779 if [ "x$ERRLIST" != "x" ]; then
 780         if [ -f "$ERRLIST_TARGET" ]; then
 781                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 782         fi
 783         uniq $ERRLIST > $ERRLIST_TARGET
 784         rm $ERRLIST
 785 fi
 786
 787 echo >>$xml "</testsuite></testsuites>"
 788
 789 if [ -n "$nok" ]; then
 790         exit 1
 791 else
 792         exit 0
 793 fi