tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to the engine
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 --engine    Use a specific engine (default=nitc)
  33 --noskip    Do not skip a test even if the .skip file matches
  34 --[no]soso  Force enable (or disable) SOSO
  35 END
  36 }
  37
  38 # $1 is the pattern of the test
  39 # $2 is the file to compare to
  40 # the result is:
  41 #   0: if the file to compare to do not exists
  42 #   1: if the file match
  43 #   2: if the file match with soso
  44 #   3: if the file do not match
  45 function compare_to_result()
  46 {
  47         local pattern="$1"
  48         local sav="$2"
  49         if [ ! -r "$sav" ]; then return 0; fi
  50         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
  51         if [ "$?" == 0 ]; then
  52                 return 1
  53         fi
  54         [ -z "$soso" ] && return 3
  55         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  56         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  57         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  58         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  59         diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  60         if [ "$?" == 0 ]; then
  61                 return 2
  62         else
  63                 return 3
  64         fi
  65 }
  66
  67 # As argument: the pattern used for the file
  68 function process_result()
  69 {
  70         ((tapcount=tapcount+1))
  71         # Result
  72         pattern=$1
  73         description=$2
  74         SAV=""
  75         NSAV=""
  76         FIXME=""
  77         NFIXME=""
  78         SOSO=""
  79         NSOSO=""
  80         SOSOF=""
  81         NSOSOF=""
  82         for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  83                 compare_to_result "$pattern" "$sav"
  84                 case "$?" in
  85                         0)
  86                                 ;; # no file
  87                         1)
  88                                 SAV="$sav" ;;
  89                         2)
  90                                 SOSO="$sav" ;;
  91                         3)
  92                                 NSAV="$sav";;
  93                 esac
  94         done
  95         for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
  96                 compare_to_result "$pattern" "$sav"
  97                 case "$?" in
  98                         0)
  99                                 ;; # no file
 100                         1)
 101                                 FIXME="$sav" ;;
 102                         2)
 103                                 SOSOF="$sav" ;;
 104                         3)
 105                                 NFIXME="$sav";;
 106                 esac
 107         done
 108         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
 109         NYI="$?"
 110         if [ -n "$SAV" ]; then
 111                 if [ -n "$tap" ]; then
 112                         echo "ok - $description"
 113                 elif [ -z "$FIXME" ]; then
 114                         echo "[ok] out/$pattern.res $SAV"
 115                 else
 116                         echo "[ok] out/$pattern.res $SAV - but $FIXME remains!"
 117                 fi
 118                 ok="$ok $pattern"
 119         elif [ -n "$FIXME" ]; then
 120                 if [ -n "$tap" ]; then
 121                         echo "not ok - $description # TODO expected failure"
 122                 else
 123                         echo "[fixme] out/$pattern.res $FIXME"
 124                 fi
 125                 todos="$todos $pattern"
 126         elif [ -n "$SOSO" ]; then
 127                 if [ -n "$tap" ]; then
 128                         echo "ok - $description # SOSO"
 129                 else
 130                         echo "[soso] out/$pattern.res $SOSO"
 131                 fi
 132                 ok="$ok $pattern"
 133         elif [ "x$NYI" = "x0" ]; then
 134                 if [ -n "$tap" ]; then
 135                         echo "not ok - $description # TODO not yet implemented"
 136                 else
 137                         echo "[todo] out/$pattern.res -> not yet implemented"
 138                 fi
 139                 todos="$todos $pattern"
 140         elif [ -n "$SOSOF" ]; then
 141                 if [ -n "$tap" ]; then
 142                         echo "not ok - $description # TODO SOSO expected failure"
 143                 else
 144                         echo "[fixme soso] out/$pattern.res $SOSOF"
 145                 fi
 146                 todos="$todos $pattern"
 147         elif [ -n "$NSAV" ]; then
 148                 if [ -n "$tap" ]; then
 149                         echo "not ok - $description"
 150                 else
 151                         echo "[======= fail out/$pattern.res $NSAV =======]"
 152                 fi
 153                 nok="$nok $pattern"
 154                 echo "$ii" >> "$ERRLIST"
 155         elif [ -n "$NFIXME" ]; then
 156                 if [ -n "$tap" ]; then
 157                         echo "not ok - $description"
 158                 else
 159                         echo "[======= changed out/$pattern.res $NFIXME ======]"
 160                 fi
 161                 nok="$nok $pattern"
 162                 echo "$ii" >> "$ERRLIST"
 163         else
 164                 if [ -n "$tap" ]; then
 165                         echo "ok - $description # skip no sav"
 166                 else
 167                         echo "[=== no sav ===] out/$pattern.res"
 168                 fi
 169                 nos="$nos $pattern"
 170         fi
 171 }
 172
 173 need_skip()
 174 {
 175         test "$noskip" = true && return 1
 176         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 177                 ((tapcount=tapcount+1))
 178                 if [ -n "$tap" ]; then
 179                         echo "ok - $2 # skip"
 180                 else
 181                         echo "=> $2: [skip]"
 182                 fi
 183                 return 0
 184         fi
 185         return 1
 186 }
 187
 188 find_nitc()
 189 {
 190         ((tapcount=tapcount+1))
 191         name="$enginebinname"
 192         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 193         if [[ "x$recent" == "x" ]]; then
 194                 if [ -n "$tap" ]; then
 195                         echo "not ok - find engine $name"
 196                         echo "Bail out! Could not find engine $name, aborting"
 197                 else
 198                         echo "Could not find engine $name, aborting"
 199                 fi
 200                 exit 1
 201         fi
 202         if [ -n "$tap" ]; then
 203                 echo "ok - find engine $name: $recent"
 204         else
 205                 echo "Using engine $name from: $recent"
 206         fi
 207         NITC=$recent
 208 }
 209
 210 verbose=false
 211 stop=false
 212 tapcount=0
 213 engine=nitc
 214 noskip=
 215 while [ $stop = false ]; do
 216         case $1 in
 217                 -o) OPT="$OPT $2"; shift; shift;;
 218                 -v) verbose=true; shift;;
 219                 -h) usage; exit;;
 220                 --tap) tap=true; shift;;
 221                 --engine) engine="$2"; shift; shift;;
 222                 --noskip) noskip=true; shift;;
 223                 --soso) soso=true; shift;;
 224                 --nososo) nososo=true; shift;;
 225                 *) stop=true
 226         esac
 227 done
 228 enginebinname=$engine
 229 case $engine in
 230         nitc) ;;
 231         nitg) ;;
 232         nitg-s) enginebinname=nitg; OPT="--separate $OPT";;
 233         nitg-e) enginebinname=nitg; OPT="--erasure $OPT";;
 234         nit) engine=niti ;;
 235         niti) enginebinname=nit ;;
 236 esac
 237
 238 # The default nitc compiler
 239 [ -z "$NITC" ] && find_nitc
 240
 241 # Set NIT_DIR if needed
 242 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 243
 244 if sh -c "timelimit echo" 1>/dev/null 2>&1; then
 245         TIMEOUT="timelimit -t 600"
 246 elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
 247         TIMEOUT="timeout 600s"
 248 else
 249         echo "No timelimit or timeout command detected. Tests may hang :("
 250 fi
 251
 252 # Mark to distinguish files among tests
 253 # MARK=
 254
 255 # File where error tests are outputed
 256 # Old ERRLIST is backuped
 257 ERRLIST=${ERRLIST:-errlist}
 258 ERRLIST_TARGET=$ERRLIST
 259
 260 if [ $# = 0 ]; then
 261         usage;
 262         exit
 263 fi
 264
 265 # Initiate new ERRLIST
 266 if [ "x$ERRLIST" = "x" ]; then
 267         ERRLIST=/dev=null
 268 else
 269         ERRLIST=$ERRLIST.tmp
 270         > "$ERRLIST"
 271 fi
 272
 273 ok=""
 274 nok=""
 275 todos=""
 276
 277 # CLEAN the out directory
 278 rm -rf out/ 2>/dev/null
 279 mkdir out 2>/dev/null
 280
 281 for ii in "$@"; do
 282         if [ ! -f $ii ]; then
 283                 echo "File '$ii' does not exist."
 284                 continue
 285         fi
 286         f=`basename "$ii" .nit`
 287
 288         # Sould we skip the file for this engine?
 289         need_skip $f $f && continue
 290
 291         tmp=${ii/../AA}
 292         if [ "x$tmp" = "x$ii" ]; then
 293                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 294         else
 295                 includes="-I alt"
 296         fi
 297
 298         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 299                 bf=`basename $i .nit`
 300                 ff="out/$bf"
 301
 302                 # Sould we skip the alternative for this engine?
 303                 need_skip $bf $bf && continue
 304
 305                 test -z "$tap" && echo -n "=> $bf: "
 306
 307                 if [ -f "$f.inputs" ]; then
 308                         inputs="$f.inputs"
 309                 else
 310                         inputs=/dev/null
 311                 fi
 312
 313                 if [ "$engine" = "niti" ]; then
 314                         cat > "./$ff.bin" <<END
 315 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 316 END
 317                         chmod +x "./$ff.bin"
 318                         > "$ff.cmp.err"
 319                         > "$ff.compile.log"
 320                         ERR=0
 321                 else
 322                         # Compile
 323                         if [ "x$verbose" = "xtrue" ]; then
 324                                 echo ""
 325                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 326                         fi
 327                         NIT_NO_STACK=1 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 328                         ERR=$?
 329                         if [ "x$verbose" = "xtrue" ]; then
 330                                 cat "$ff.compile.log"
 331                                 cat >&2 "$ff.cmp.err"
 332                         fi
 333                 fi
 334                 if [ "$ERR" != 0 ]; then
 335                         test -z "$tap" && echo -n "! "
 336                         cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 337                         process_result $bf $bf
 338                 elif [ -x "./$ff.bin" ]; then
 339                         test -z "$tap" && echo -n ". "
 340                         # Execute
 341                         args=""
 342                         if [ "x$verbose" = "xtrue" ]; then
 343                                 echo ""
 344                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 345                         fi
 346                         NIT_NO_STACK=1 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 347                         if [ "x$verbose" = "xtrue" ]; then
 348                                 cat "$ff.res"
 349                                 cat >&2 "$ff.err"
 350                         fi
 351                         if [ -f "$ff.write" ]; then
 352                                 cat "$ff.write" >> "$ff.res"
 353                         elif [ -d "$ff.write" ]; then
 354                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 355                         fi
 356                         cp "$ff.res"  "$ff.res2"
 357                         cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
 358                         process_result $bf $bf
 359
 360                         if [ -f "$f.args" ]; then
 361                                 fargs=$f.args
 362                                 cptr=0
 363                                 while read line; do
 364                                         ((cptr=cptr+1))
 365                                         args="$line"
 366                                         bff=$bf"_args"$cptr
 367                                         fff=$ff"_args"$cptr
 368
 369                                         # Sould we skip the input for this engine?
 370                                         need_skip $bff "  args #$cptr" && continue
 371
 372                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 373                                         if [ "x$verbose" = "xtrue" ]; then
 374                                                 echo ""
 375                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 376                                         fi
 377                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 378                                         echo "./$ff.bin $args" > "./$fff.bin"
 379                                         chmod +x "./$fff.bin"
 380                                         sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $inputs > $fff.res 2>$fff.err"
 381                                         if [ "x$verbose" = "xtrue" ]; then
 382                                                 cat "$fff.res"
 383                                                 cat >&2 "$fff.err"
 384                                         fi
 385                                         if [ -f "$fff.write" ]; then
 386                                                 cat "$fff.write" >> "$fff.res"
 387                                         elif [ -d "$fff.write" ]; then
 388                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 389                                         fi
 390                                         if [ -s "$fff.err" ]; then
 391                                                 cp "$fff.res"  "$fff.res2"
 392                                                 cat "$fff.err" "$fff.res2" > "$fff.res"
 393                                         fi
 394                                         process_result $bff "  args #$cptr"
 395                                 done < $fargs
 396                         fi
 397                 else
 398                         test -z "$tap" && echo -n "! "
 399                         cat "$ff.cmp.err" > "$ff.res"
 400                         echo "Compilation error" > "$ff.res"
 401                         process_result $bf "$bf"
 402                 fi
 403         done
 404 done
 405
 406 if [ -n "$tap" ]; then
 407         echo "1..$tapcount"
 408         echo "# ok:" `echo $ok | wc -w`
 409         echo "# not ok:" `echo $nok | wc -w`
 410         echo "# no sav:" `echo $nos | wc -w`
 411         echo "# todo/fixme:" `echo $todos | wc -w`
 412         exit
 413 fi
 414
 415 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 416
 417 if [ -n "$nok" ]; then
 418         echo "fail: $nok"
 419         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 420 fi
 421 if [ -n "$nos" ]; then
 422         echo "no sav: $nos"
 423 fi
 424 if [ -n "$todos" ]; then
 425         echo "todo/fixme: $todos"
 426 fi
 427
 428 # write $ERRLIST
 429 if [ "x$ERRLIST" != "x" ]; then
 430         if [ -x "$ERRLIST_TARGET" ]; then
 431                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 432         fi
 433         mv $ERRLIST $ERRLIST_TARGET
 434 fi
 435
 436 if [ -n "$nok" ]; then
 437         exit 1
 438 else
 439         exit 0
 440 fi