tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22 export NIT_TESTING=true
  23
  24 usage()
  25 {
  26         e=`basename "$0"`
  27         cat<<END
  28 Usage: $e [options] modulenames
  29 -o option   Pass option to the engine
  30 -v          Verbose (show tests steps)
  31 -h          This help
  32 --tap       Produce TAP output
  33 --engine    Use a specific engine (default=nitc)
  34 --noskip    Do not skip a test even if the .skip file matches
  35 --[no]soso  Force enable (or disable) SOSO
  36 END
  37 }
  38
  39 # $1 is the pattern of the test
  40 # $2 is the file to compare to
  41 # the result is:
  42 #   0: if the file to compare to do not exists
  43 #   1: if the file match
  44 #   2: if the file match with soso
  45 #   3: if the file do not match
  46 function compare_to_result()
  47 {
  48         local pattern="$1"
  49         local sav="$2"
  50         if [ ! -r "$sav" ]; then return 0; fi
  51         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
  52         if [ "$?" == 0 ]; then
  53                 return 1
  54         fi
  55         [ -z "$soso" ] && return 3
  56         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  57         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  58         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  59         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  60         diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  61         if [ "$?" == 0 ]; then
  62                 return 2
  63         else
  64                 return 3
  65         fi
  66 }
  67
  68 # As argument: the pattern used for the file
  69 function process_result()
  70 {
  71         ((tapcount=tapcount+1))
  72         # Result
  73         pattern=$1
  74         description=$2
  75         SAV=""
  76         NSAV=""
  77         FIXME=""
  78         NFIXME=""
  79         SOSO=""
  80         NSOSO=""
  81         SOSOF=""
  82         NSOSOF=""
  83         for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  84                 compare_to_result "$pattern" "$sav"
  85                 case "$?" in
  86                         0)
  87                                 ;; # no file
  88                         1)
  89                                 SAV="$sav" ;;
  90                         2)
  91                                 SOSO="$sav" ;;
  92                         3)
  93                                 NSAV="$sav";;
  94                 esac
  95         done
  96         for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
  97                 compare_to_result "$pattern" "$sav"
  98                 case "$?" in
  99                         0)
 100                                 ;; # no file
 101                         1)
 102                                 FIXME="$sav" ;;
 103                         2)
 104                                 SOSOF="$sav" ;;
 105                         3)
 106                                 NFIXME="$sav";;
 107                 esac
 108         done
 109         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
 110         NYI="$?"
 111         if [ -n "$SAV" ]; then
 112                 if [ -n "$tap" ]; then
 113                         echo "ok - $description"
 114                 elif [ -z "$FIXME" ]; then
 115                         echo "[ok] out/$pattern.res $SAV"
 116                 else
 117                         echo "[ok] out/$pattern.res $SAV - but $FIXME remains!"
 118                 fi
 119                 ok="$ok $pattern"
 120         elif [ -n "$FIXME" ]; then
 121                 if [ -n "$tap" ]; then
 122                         echo "not ok - $description # TODO expected failure"
 123                 else
 124                         echo "[fixme] out/$pattern.res $FIXME"
 125                 fi
 126                 todos="$todos $pattern"
 127         elif [ -n "$SOSO" ]; then
 128                 if [ -n "$tap" ]; then
 129                         echo "ok - $description # SOSO"
 130                 else
 131                         echo "[soso] out/$pattern.res $SOSO"
 132                 fi
 133                 ok="$ok $pattern"
 134         elif [ "x$NYI" = "x0" ]; then
 135                 if [ -n "$tap" ]; then
 136                         echo "not ok - $description # TODO not yet implemented"
 137                 else
 138                         echo "[todo] out/$pattern.res -> not yet implemented"
 139                 fi
 140                 todos="$todos $pattern"
 141         elif [ -n "$SOSOF" ]; then
 142                 if [ -n "$tap" ]; then
 143                         echo "not ok - $description # TODO SOSO expected failure"
 144                 else
 145                         echo "[fixme soso] out/$pattern.res $SOSOF"
 146                 fi
 147                 todos="$todos $pattern"
 148         elif [ -n "$NSAV" ]; then
 149                 if [ -n "$tap" ]; then
 150                         echo "not ok - $description"
 151                 else
 152                         echo "[======= fail out/$pattern.res $NSAV =======]"
 153                 fi
 154                 nok="$nok $pattern"
 155                 echo "$ii" >> "$ERRLIST"
 156         elif [ -n "$NFIXME" ]; then
 157                 if [ -n "$tap" ]; then
 158                         echo "not ok - $description"
 159                 else
 160                         echo "[======= changed out/$pattern.res $NFIXME ======]"
 161                 fi
 162                 nok="$nok $pattern"
 163                 echo "$ii" >> "$ERRLIST"
 164         else
 165                 if [ -n "$tap" ]; then
 166                         echo "ok - $description # skip no sav"
 167                 else
 168                         echo "[=== no sav ===] out/$pattern.res"
 169                 fi
 170                 nos="$nos $pattern"
 171         fi
 172 }
 173
 174 need_skip()
 175 {
 176         test "$noskip" = true && return 1
 177         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 178                 ((tapcount=tapcount+1))
 179                 if [ -n "$tap" ]; then
 180                         echo "ok - $2 # skip"
 181                 else
 182                         echo "=> $2: [skip]"
 183                 fi
 184                 return 0
 185         fi
 186         return 1
 187 }
 188
 189 find_nitc()
 190 {
 191         ((tapcount=tapcount+1))
 192         name="$enginebinname"
 193         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 194         if [[ "x$recent" == "x" ]]; then
 195                 if [ -n "$tap" ]; then
 196                         echo "not ok - find engine $name"
 197                         echo "Bail out! Could not find engine $name, aborting"
 198                 else
 199                         echo "Could not find engine $name, aborting"
 200                 fi
 201                 exit 1
 202         fi
 203         if [ -n "$tap" ]; then
 204                 echo "ok - find engine $name: $recent"
 205         else
 206                 echo "Using engine $name from: $recent"
 207         fi
 208         NITC=$recent
 209 }
 210
 211 verbose=false
 212 stop=false
 213 tapcount=0
 214 engine=nitc
 215 noskip=
 216 while [ $stop = false ]; do
 217         case $1 in
 218                 -o) OPT="$OPT $2"; shift; shift;;
 219                 -v) verbose=true; shift;;
 220                 -h) usage; exit;;
 221                 --tap) tap=true; shift;;
 222                 --engine) engine="$2"; shift; shift;;
 223                 --noskip) noskip=true; shift;;
 224                 --soso) soso=true; shift;;
 225                 --nososo) nososo=true; shift;;
 226                 *) stop=true
 227         esac
 228 done
 229 enginebinname=$engine
 230 case $engine in
 231         nitc) ;;
 232         nitg) OPT="--global $OPT";;
 233         nitg-s) enginebinname=nitg; OPT="--separate $OPT";;
 234         nitg-e) enginebinname=nitg; OPT="--erasure $OPT";;
 235         nit) engine=niti ;;
 236         niti) enginebinname=nit ;;
 237 esac
 238
 239 # The default nitc compiler
 240 [ -z "$NITC" ] && find_nitc
 241
 242 # Set NIT_DIR if needed
 243 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 244
 245 if sh -c "timelimit echo" 1>/dev/null 2>&1; then
 246         TIMEOUT="timelimit -t 600"
 247 elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
 248         TIMEOUT="timeout 600s"
 249 else
 250         echo "No timelimit or timeout command detected. Tests may hang :("
 251 fi
 252
 253 # Mark to distinguish files among tests
 254 # MARK=
 255
 256 # File where error tests are outputed
 257 # Old ERRLIST is backuped
 258 ERRLIST=${ERRLIST:-errlist}
 259 ERRLIST_TARGET=$ERRLIST
 260
 261 if [ $# = 0 ]; then
 262         usage;
 263         exit
 264 fi
 265
 266 # Initiate new ERRLIST
 267 if [ "x$ERRLIST" = "x" ]; then
 268         ERRLIST=/dev=null
 269 else
 270         ERRLIST=$ERRLIST.tmp
 271         > "$ERRLIST"
 272 fi
 273
 274 ok=""
 275 nok=""
 276 todos=""
 277
 278 # CLEAN the out directory
 279 rm -rf out/ 2>/dev/null
 280 mkdir out 2>/dev/null
 281
 282 for ii in "$@"; do
 283         if [ ! -f $ii ]; then
 284                 echo "File '$ii' does not exist."
 285                 continue
 286         fi
 287         f=`basename "$ii" .nit`
 288
 289         # Sould we skip the file for this engine?
 290         need_skip $f $f && continue
 291
 292         tmp=${ii/../AA}
 293         if [ "x$tmp" = "x$ii" ]; then
 294                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 295         else
 296                 includes="-I alt"
 297         fi
 298
 299         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 300                 bf=`basename $i .nit`
 301                 ff="out/$bf"
 302
 303                 # Sould we skip the alternative for this engine?
 304                 need_skip $bf $bf && continue
 305
 306                 test -z "$tap" && echo -n "=> $bf: "
 307
 308                 if [ -f "$f.inputs" ]; then
 309                         inputs="$f.inputs"
 310                 else
 311                         inputs=/dev/null
 312                 fi
 313
 314                 if [ "$engine" = "niti" ]; then
 315                         cat > "./$ff.bin" <<END
 316 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 317 END
 318                         chmod +x "./$ff.bin"
 319                         > "$ff.cmp.err"
 320                         > "$ff.compile.log"
 321                         ERR=0
 322                 else
 323                         # Compile
 324                         if [ "x$verbose" = "xtrue" ]; then
 325                                 echo ""
 326                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 327                         fi
 328                         NIT_NO_STACK=1 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 329                         ERR=$?
 330                         if [ "x$verbose" = "xtrue" ]; then
 331                                 cat "$ff.compile.log"
 332                                 cat >&2 "$ff.cmp.err"
 333                         fi
 334                 fi
 335                 if [ "$ERR" != 0 ]; then
 336                         test -z "$tap" && echo -n "! "
 337                         cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 338                         process_result $bf $bf
 339                 elif [ -x "./$ff.bin" ]; then
 340                         test -z "$tap" && echo -n ". "
 341                         # Execute
 342                         args=""
 343                         if [ "x$verbose" = "xtrue" ]; then
 344                                 echo ""
 345                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 346                         fi
 347                         NIT_NO_STACK=1 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 348                         if [ "x$verbose" = "xtrue" ]; then
 349                                 cat "$ff.res"
 350                                 cat >&2 "$ff.err"
 351                         fi
 352                         if [ -f "$ff.write" ]; then
 353                                 cat "$ff.write" >> "$ff.res"
 354                         elif [ -d "$ff.write" ]; then
 355                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 356                         fi
 357                         cp "$ff.res"  "$ff.res2"
 358                         cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
 359                         process_result $bf $bf
 360
 361                         if [ -f "$f.args" ]; then
 362                                 fargs=$f.args
 363                                 cptr=0
 364                                 while read line; do
 365                                         ((cptr=cptr+1))
 366                                         args="$line"
 367                                         bff=$bf"_args"$cptr
 368                                         fff=$ff"_args"$cptr
 369                                         name="$bf args $cptr"
 370
 371                                         # Sould we skip the input for this engine?
 372                                         need_skip $bff "  $name" && continue
 373
 374                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 375                                         if [ "x$verbose" = "xtrue" ]; then
 376                                                 echo ""
 377                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 378                                         fi
 379                                         test -z "$tap" && echo -n "==> $name "
 380                                         echo "./$ff.bin $args" > "./$fff.bin"
 381                                         chmod +x "./$fff.bin"
 382                                         sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $inputs > $fff.res 2>$fff.err"
 383                                         if [ "x$verbose" = "xtrue" ]; then
 384                                                 cat "$fff.res"
 385                                                 cat >&2 "$fff.err"
 386                                         fi
 387                                         if [ -f "$fff.write" ]; then
 388                                                 cat "$fff.write" >> "$fff.res"
 389                                         elif [ -d "$fff.write" ]; then
 390                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 391                                         fi
 392                                         if [ -s "$fff.err" ]; then
 393                                                 cp "$fff.res"  "$fff.res2"
 394                                                 cat "$fff.err" "$fff.res2" > "$fff.res"
 395                                         fi
 396                                         process_result $bff "  $name"
 397                                 done < $fargs
 398                         fi
 399                 else
 400                         test -z "$tap" && echo -n "! "
 401                         cat "$ff.cmp.err" > "$ff.res"
 402                         echo "Compilation error" > "$ff.res"
 403                         process_result $bf "$bf"
 404                 fi
 405         done
 406 done
 407
 408 if [ -n "$tap" ]; then
 409         echo "1..$tapcount"
 410         echo "# ok:" `echo $ok | wc -w`
 411         echo "# not ok:" `echo $nok | wc -w`
 412         echo "# no sav:" `echo $nos | wc -w`
 413         echo "# todo/fixme:" `echo $todos | wc -w`
 414         exit
 415 fi
 416
 417 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 418
 419 if [ -n "$nok" ]; then
 420         echo "fail: $nok"
 421         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 422 fi
 423 if [ -n "$nos" ]; then
 424         echo "no sav: $nos"
 425 fi
 426 if [ -n "$todos" ]; then
 427         echo "todo/fixme: $todos"
 428 fi
 429
 430 # write $ERRLIST
 431 if [ "x$ERRLIST" != "x" ]; then
 432         if [ -x "$ERRLIST_TARGET" ]; then
 433                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 434         fi
 435         mv $ERRLIST $ERRLIST_TARGET
 436 fi
 437
 438 if [ -n "$nok" ]; then
 439         exit 1
 440 else
 441         exit 0
 442 fi