tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22 export NIT_TESTING=true
  23
  24 unset NIT_DIR
  25
  26 # Get the first Java lib available
  27 shopt -s nullglob
  28 paths=`echo /usr/lib/jvm/*/`
  29 paths=($paths)
  30 JAVA_HOME=${paths[0]}
  31
  32 paths=`echo $JAVA_HOME/jre/lib/*/{client,server}/`
  33 paths=($paths)
  34 JNI_LIB_PATH=${paths[0]}
  35 shopt -u nullglob
  36
  37 usage()
  38 {
  39         e=`basename "$0"`
  40         cat<<END
  41 Usage: $e [options] modulenames
  42 -o option   Pass option to the engine
  43 -v          Verbose (show tests steps)
  44 -h          This help
  45 --engine    Use a specific engine (default=nitg)
  46 --noskip    Do not skip a test even if the .skip file matches
  47 END
  48 }
  49
  50 # $1 is the pattern of the test
  51 # $2 is the file to compare to
  52 # the result is:
  53 #   0: if the file to compare to do not exists
  54 #   1: if the file match
  55 #   2: if the file match with soso
  56 #   3: if the file do not match
  57 function compare_to_result()
  58 {
  59         local pattern="$1"
  60         local sav="$2"
  61         if [ ! -r "$sav" ]; then return 0; fi
  62         diff -u "$sav" "out/$pattern.res" > "out/$pattern.diff.sav.log"
  63         if [ "$?" == 0 ]; then
  64                 return 1
  65         fi
  66         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  67         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  68         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  69         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  70         diff -u "out/$pattern.sav2" "out/$pattern.res2" > "out/$pattern.diff.sav.log2"
  71         if [ "$?" == 0 ]; then
  72                 return 2
  73         else
  74                 return 3
  75         fi
  76 }
  77
  78 # As argument: the pattern used for the file
  79 function process_result()
  80 {
  81         # Result
  82         pattern=$1
  83         description=$2
  84         pack=$3
  85         SAV=""
  86         NSAV=""
  87         FIXME=""
  88         NFIXME=""
  89         SOSO=""
  90         NSOSO=""
  91         SOSOF=""
  92         NSOSOF=""
  93         OLD=""
  94         LIST=""
  95         FIRST=""
  96         echo >>$xml "<testcase classname='$pack' name='$description'>"
  97         #for sav in "sav/$engine/fixme/$pattern.res" "sav/$engine/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  98         for savdir in $savdirs; do
  99                 sav=$savdir/fixme/$pattern.res
 100                 compare_to_result "$pattern" "$sav"
 101                 case "$?" in
 102                         0)
 103                                 ;; # no file
 104                         1)
 105                                 OLD="$LIST"
 106                                 FIXME="$sav"
 107                                 LIST="$LIST $sav"
 108                                 ;;
 109                         2)
 110                                 if [ -z "$FIRST" ]; then
 111                                         SOSOF="$sav"
 112                                         FIRST="$sav"
 113                                 fi
 114                                 LIST="$LIST $sav"
 115                                 ;;
 116                         3)
 117                                 if [ -z "$FIRST" ]; then
 118                                         NFIXME="$sav"
 119                                         FIRST="$sav"
 120                                 fi
 121                                 LIST="$LIST $sav"
 122                                 ;;
 123                 esac
 124
 125                 sav=$savdir/$pattern.res
 126                 compare_to_result "$pattern" "$sav"
 127                 case "$?" in
 128                         0)
 129                                 ;; # no file
 130                         1)
 131                                 OLD="$LIST"
 132                                 SAV="$sav"
 133                                 LIST="$LIST $sav"
 134                                 ;;
 135                         2)
 136                                 if [ -z "$FIRST" ]; then
 137                                         SOSO="$sav"
 138                                         FIRST="$sav"
 139                                 fi
 140                                 LIST="$LIST $sav"
 141                                 ;;
 142                         3)
 143                                 if [ -z "$FIRST" ]; then
 144                                         NSAV="$sav"
 145                                         FIRST="$sav"
 146                                 fi
 147                                 LIST="$LIST $sav"
 148                                 ;;
 149                 esac
 150         done
 151         OLD=`echo "$OLD" | sed -e 's/   */ /g' -e 's/^ //' -e 's/ $//'`
 152         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
 153         NYI="$?"
 154         if [ -n "$SAV" ]; then
 155                 if [ -n "$OLD" ]; then
 156                         echo "[*ok*] out/$pattern.res $SAV - but $OLD remains!"
 157                         echo >>$xml "<error message='ok out/$pattern.res - but $OLD remains'/>"
 158                         remains="$remains $OLD"
 159                 else
 160                         echo "[ok] out/$pattern.res $SAV"
 161                 fi
 162                 ok="$ok $pattern"
 163         elif [ -n "$FIXME" ]; then
 164                 if [ -n "$OLD" ]; then
 165                         echo "[*fixme*] out/$pattern.res $FIXME - but $OLD remains!"
 166                         echo >>$xml "<error message='ok out/$pattern.res - but $OLD remains'/>"
 167                         remains="$remains $OLD"
 168                 else
 169                         echo "[fixme] out/$pattern.res $FIXME"
 170                         echo >>$xml "<skipped/>"
 171                 fi
 172                 todos="$todos $pattern"
 173         elif [ "x$NYI" = "x0" ]; then
 174                 echo "[todo] out/$pattern.res -> not yet implemented"
 175                 echo >>$xml "<skipped/>"
 176                 todos="$todos $pattern"
 177         elif [ -n "$SOSO" ]; then
 178                 echo "[======= soso out/$pattern.res $SOSO =======]"
 179                 echo >>$xml "<error message='soso out/$pattern.res $SOSO'/>"
 180                 echo >>$xml "<system-out><![CDATA["
 181                 cat -v out/$pattern.diff.sav.log | head >>$xml -n 50
 182                 echo >>$xml "]]></system-out>"
 183                 nok="$nok $pattern"
 184                 echo "$ii" >> "$ERRLIST"
 185         elif [ -n "$SOSOF" ]; then
 186                 echo "[======= fixme soso out/$pattern.res $SOSOF =======]"
 187                 echo >>$xml "<error message='soso out/$pattern.res $SOSO'/>"
 188                 echo >>$xml "<system-out><![CDATA["
 189                 cat -v out/$pattern.diff.sav.log | head >>$xml -n 50
 190                 echo >>$xml "]]></system-out>"
 191                 nok="$nok $pattern"
 192                 echo "$ii" >> "$ERRLIST"
 193         elif [ -n "$NSAV" ]; then
 194                 echo "[======= fail out/$pattern.res $NSAV =======]"
 195                 echo >>$xml "<error message='fail out/$pattern.res $NSAV'/>"
 196                 echo >>$xml "<system-out><![CDATA["
 197                 cat -v out/$pattern.diff.sav.log | head >>$xml -n 50
 198                 echo >>$xml "]]></system-out>"
 199                 nok="$nok $pattern"
 200                 echo "$ii" >> "$ERRLIST"
 201         elif [ -n "$NFIXME" ]; then
 202                 echo "[======= changed out/$pattern.res $NFIXME ======]"
 203                 echo >>$xml "<error message='changed out/$pattern.res $NFIXME'/>"
 204                 echo >>$xml "<system-out><![CDATA["
 205                 cat -v out/$pattern.diff.sav.log | head >>$xml -n 50
 206                 echo >>$xml "]]></system-out>"
 207                 nok="$nok $pattern"
 208                 echo "$ii" >> "$ERRLIST"
 209         elif [ -s out/$pattern.res ]; then
 210                 echo "[=== no sav ===] out/$pattern.res is not empty"
 211                 echo >>$xml "<error message='no sav and not empty'/>"
 212                 echo >>$xml "<system-out><![CDATA["
 213                 cat -v >>$xml out/$pattern.res
 214                 echo >>$xml "]]></system-out>"
 215                 nos="$nos $pattern"
 216         else
 217                 # no sav but empty res
 218                 echo "[0k] out/$pattern.res is empty"
 219                 ok="$ok $pattern"
 220         fi
 221         if test -s out/$pattern.cmp.err; then
 222                 echo >>$xml "<system-err><![CDATA["
 223                 cat -v >>$xml out/$pattern.cmp.err
 224                 echo >>$xml "]]></system-err>"
 225         fi
 226         echo >>$xml "</testcase>"
 227 }
 228
 229 need_skip()
 230 {
 231         test "$noskip" = true && return 1
 232         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 233                 echo "=> $2: [skip]"
 234                 echo >>$xml "<testcase classname='$3' name='$2'><skipped/></testcase>"
 235                 return 0
 236         fi
 237         if test $engine = niti && echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then
 238                 echo "=> $2: [skip exec]"
 239                 echo >>$xml "<testcase classname='$3' name='$2'><skipped/></testcase>"
 240                 return 0
 241         fi
 242         return 1
 243 }
 244
 245 skip_exec()
 246 {
 247         test "$noskip" = true && return 1
 248         if echo "$1" | grep -f "exec.skip" >/dev/null 2>&1; then
 249                 echo -n "_ "
 250                 return 0
 251         fi
 252         return 1
 253 }
 254
 255 skip_cc()
 256 {
 257         test "$noskip" = true && return 1
 258         if echo "$1" | grep -f "cc.skip" >/dev/null 2>&1; then
 259                 return 0
 260         fi
 261         return 1
 262 }
 263
 264 find_nitc()
 265 {
 266         name="$enginebinname"
 267         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 268         if [[ "x$recent" == "x" ]]; then
 269                 echo "Could not find binary for engine $engine, aborting"
 270                 exit 1
 271         fi
 272         echo "Find binary for engine $engine: $recent $OPT"
 273         NITC=$recent
 274 }
 275
 276 verbose=false
 277 stop=false
 278 engine=nitg
 279 noskip=
 280 savdirs=
 281 while [ $stop = false ]; do
 282         case $1 in
 283                 -o) OPT="$OPT $2"; shift; shift;;
 284                 -v) verbose=true; shift;;
 285                 -h) usage; exit;;
 286                 --engine) engine="$2"; shift; shift;;
 287                 --noskip) noskip=true; shift;;
 288                 *) stop=true
 289         esac
 290 done
 291 enginebinname=$engine
 292 case $engine in
 293         nitg)
 294                 engine=nitg-s;
 295                 enginebinname=nitg;
 296                 OPT="--separate $OPT"
 297                 ;;
 298         nitg-s)
 299                 enginebinname=nitg;
 300                 OPT="--separate $OPT"
 301                 ;;
 302         nitg-e)
 303                 enginebinname=nitg;
 304                 OPT="--erasure $OPT"
 305                 ;;
 306         nitg-g)
 307                 enginebinname=nitg;
 308                 OPT="--global $OPT"
 309                 ;;
 310         nit)
 311                 engine=niti
 312                 ;;
 313         niti)
 314                 enginebinname=nit
 315                 ;;
 316         nitc)
 317                 echo "disabled engine $engine"
 318                 exit 0
 319                 ;;
 320         *)
 321                 echo "unknown engine $engine"
 322                 exit 1
 323                 ;;
 324 esac
 325
 326 savdirs="sav/$engine $savdirs sav/"
 327
 328 # The default nitc compiler
 329 [ -z "$NITC" ] && find_nitc
 330
 331 # Set NIT_DIR if needed
 332 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 333
 334 if sh -c "timelimit echo" 1>/dev/null 2>&1; then
 335         TIMEOUT="timelimit -t 600"
 336 elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
 337         TIMEOUT="timeout 600s"
 338 else
 339         echo "No timelimit or timeout command detected. Tests may hang :("
 340 fi
 341
 342 # Mark to distinguish files among tests
 343 # MARK=
 344
 345 # File where error tests are outputed
 346 # Old ERRLIST is backuped
 347 ERRLIST=${ERRLIST:-errlist}
 348 ERRLIST_TARGET=$ERRLIST
 349
 350 if [ $# = 0 ]; then
 351         usage;
 352         exit
 353 fi
 354
 355 # Initiate new ERRLIST
 356 if [ "x$ERRLIST" = "x" ]; then
 357         ERRLIST=/dev=null
 358 else
 359         ERRLIST=$ERRLIST.tmp
 360         > "$ERRLIST"
 361 fi
 362
 363 ok=""
 364 nok=""
 365 todos=""
 366 xml="tests-$engine.xml"
 367 echo >$xml "<testsuites><testsuite>"
 368
 369 # CLEAN the out directory
 370 rm -rf out/ 2>/dev/null
 371 mkdir out 2>/dev/null
 372
 373 for ii in "$@"; do
 374         if [ ! -f $ii ]; then
 375                 echo "File '$ii' does not exist."
 376                 continue
 377         fi
 378         f=`basename "$ii" .nit`
 379
 380         pack=`echo $ii | perl -p -e 's|^../([^/]*)/([a-zA-Z_]*).*|\1.\2| || s|^([a-zA-Z]*)[^_]*_([a-zA-Z]*).*|\1.\2| || s|\W*([a-zA-Z_]*).*|\1|'`
 381
 382         # Sould we skip the file for this engine?
 383         need_skip $f $f $pack && continue
 384
 385         tmp=${ii/../AA}
 386         if [ "x$tmp" = "x$ii" ]; then
 387                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 388         else
 389                 includes="-I alt"
 390         fi
 391
 392         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 393                 bf=`basename $i .nit`
 394                 ff="out/$bf"
 395
 396                 # Sould we skip the alternative for this engine?
 397                 need_skip $bf $bf $pack && continue
 398
 399                 echo -n "=> $bf: "
 400
 401                 if [ -f "$f.inputs" ]; then
 402                         inputs="$f.inputs"
 403                 else
 404                         inputs=/dev/null
 405                 fi
 406
 407                 if [ "$engine" = "niti" ]; then
 408                         cat > "./$ff.bin" <<END
 409 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 410 END
 411                         chmod +x "./$ff.bin"
 412                         > "$ff.cmp.err"
 413                         > "$ff.compile.log"
 414                         ERR=0
 415                 else
 416                         if skip_cc "$bf"; then
 417                                 nocc="--no-cc"
 418                         else
 419                                 nocc=
 420                         fi
 421                         # Compile
 422                         if [ "x$verbose" = "xtrue" ]; then
 423                                 echo ""
 424                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes" $nocc
 425                         fi
 426                         NIT_NO_STACK=1 JNI_LIB_PATH=$JNI_LIB_PATH JAVA_HOME=$JAVA_HOME \
 427                                 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes $nocc 2> "$ff.cmp.err" > "$ff.compile.log"
 428                         ERR=$?
 429                         if [ "x$verbose" = "xtrue" ]; then
 430                                 cat "$ff.compile.log"
 431                                 cat >&2 "$ff.cmp.err"
 432                         fi
 433                 fi
 434                 if [ "$ERR" != 0 ]; then
 435                         echo -n "! "
 436                         cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 437                         process_result $bf $bf $pack
 438                 elif skip_exec "$bf"; then
 439                         # No exec
 440                         > "$ff.res"
 441                         process_result $bf $bf $pack
 442                 elif [ -n "$nocc" ]; then
 443                         # not compiled
 444                         echo -n "nocc "
 445                         > "$ff.res"
 446                         process_result $bf $bf $pack
 447                 elif [ -x "./$ff.bin" ]; then
 448                         echo -n ". "
 449                         # Execute
 450                         args=""
 451                         if [ "x$verbose" = "xtrue" ]; then
 452                                 echo ""
 453                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 454                         fi
 455                         NIT_NO_STACK=1 LD_LIBRARY_PATH=$JNI_LIB_PATH \
 456                                 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 457                         if [ "x$verbose" = "xtrue" ]; then
 458                                 cat "$ff.res"
 459                                 cat >&2 "$ff.err"
 460                         fi
 461                         if [ -f "$ff.write" ]; then
 462                                 cat "$ff.write" >> "$ff.res"
 463                         elif [ -d "$ff.write" ]; then
 464                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 465                         fi
 466                         cp "$ff.res"  "$ff.res2"
 467                         cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
 468                         process_result $bf $bf $pack
 469
 470                         if [ -f "$f.args" ]; then
 471                                 fargs=$f.args
 472                                 cptr=0
 473                                 while read line; do
 474                                         ((cptr=cptr+1))
 475                                         args="$line"
 476                                         bff=$bf"_args"$cptr
 477                                         fff=$ff"_args"$cptr
 478                                         name="$bf args $cptr"
 479
 480                                         # Sould we skip the input for this engine?
 481                                         need_skip $bff "  $name" $pack && continue
 482
 483                                         # use a specific inputs file, if required
 484                                         if [ -f "$bff.inputs" ]; then
 485                                                 ffinputs="$bff.inputs"
 486                                         else
 487                                                 ffinputs=$inputs
 488                                         fi
 489
 490                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 491                                         if [ "x$verbose" = "xtrue" ]; then
 492                                                 echo ""
 493                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 494                                         fi
 495                                         echo -n "==> $name "
 496                                         echo "./$ff.bin $args" > "./$fff.bin"
 497                                         chmod +x "./$fff.bin"
 498                                         WRITE="$fff.write" sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $ffinputs > $fff.res 2>$fff.err"
 499                                         if [ "x$verbose" = "xtrue" ]; then
 500                                                 cat "$fff.res"
 501                                                 cat >&2 "$fff.err"
 502                                         fi
 503                                         if [ -f "$fff.write" ]; then
 504                                                 cat "$fff.write" >> "$fff.res"
 505                                         elif [ -d "$fff.write" ]; then
 506                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 507                                         fi
 508                                         if [ -s "$fff.err" ]; then
 509                                                 cp "$fff.res"  "$fff.res2"
 510                                                 cat "$fff.err" "$fff.res2" > "$fff.res"
 511                                         fi
 512                                         process_result $bff "  $name" $pack
 513                                 done < $fargs
 514                         fi
 515                 elif [ -f "./$ff.bin" ]; then
 516                         echo "Not executable (platform?)" > "$ff.res"
 517                         process_result $bf "$bf" $pack
 518                 else
 519                         echo -n "! "
 520                         cat "$ff.cmp.err" > "$ff.res"
 521                         echo "Compilation error" > "$ff.res"
 522                         process_result $bf "$bf" $pack
 523                 fi
 524         done
 525 done
 526
 527 echo "engine: $engine ($enginebinname $OPT)"
 528 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 529
 530 if [ -n "$nok" ]; then
 531         echo "fail: $nok"
 532         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 533 fi
 534 if [ -n "$nos" ]; then
 535         echo "no sav: $nos"
 536 fi
 537 if [ -n "$todos" ]; then
 538         echo "todo/fixme: $todos"
 539 fi
 540 if [ -n "$remains" ]; then
 541         echo "sav that remains: $remains"
 542 fi
 543
 544 # write $ERRLIST
 545 if [ "x$ERRLIST" != "x" ]; then
 546         if [ -x "$ERRLIST_TARGET" ]; then
 547                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 548         fi
 549         mv $ERRLIST $ERRLIST_TARGET
 550 fi
 551
 552 echo >>$xml "</testsuite></testsuites>"
 553
 554 if [ -n "$nok" ]; then
 555         exit 1
 556 else
 557         exit 0
 558 fi