tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to the engine
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 --engine    Use a specific engine (default=nitc)
  33 --noskip    Do not skip a test even if the .skip file matches
  34 --[no]soso  Force enable (or disable) SOSO
  35 END
  36 }
  37
  38 # $1 is the pattern of the test
  39 # $2 is the file to compare to
  40 # the result is:
  41 #   0: if the file to compare to do not exists
  42 #   1: if the file match
  43 #   2: if the file match with soso
  44 #   3: if the file do not match
  45 function compare_to_result()
  46 {
  47         local pattern="$1"
  48         local sav="$2"
  49         if [ ! -r "$sav" ]; then return 0; fi
  50         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
  51         if [ "$?" == 0 ]; then
  52                 return 1
  53         fi
  54         [ -z "$soso" ] && return 3
  55         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  56         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  57         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  58         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  59         diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  60         if [ "$?" == 0 ]; then
  61                 return 2
  62         else
  63                 return 3
  64         fi
  65 }
  66
  67 # As argument: the pattern used for the file
  68 function process_result()
  69 {
  70         ((tapcount=tapcount+1))
  71         # Result
  72         pattern=$1
  73         description=$2
  74         SAV=""
  75         NSAV=""
  76         FIXME=""
  77         NFIXME=""
  78         SOSO=""
  79         NSOSO=""
  80         SOSOF=""
  81         NSOSOF=""
  82         for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  83                 compare_to_result "$pattern" "$sav"
  84                 case "$?" in
  85                         0)
  86                                 ;; # no file
  87                         1)
  88                                 SAV="$sav" ;;
  89                         2)
  90                                 SOSO="$sav" ;;
  91                         3)
  92                                 NSAV="$sav";;
  93                 esac
  94         done
  95         for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
  96                 compare_to_result "$pattern" "$sav"
  97                 case "$?" in
  98                         0)
  99                                 ;; # no file
 100                         1)
 101                                 FIXME="$sav" ;;
 102                         2)
 103                                 SOSOF="$sav" ;;
 104                         3)
 105                                 NFIXME="$sav";;
 106                 esac
 107         done
 108         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
 109         NYI="$?"
 110         if [ -n "$SAV" ]; then
 111                 if [ -n "$tap" ]; then
 112                         echo "ok - $description"
 113                 elif [ -z "$FIXME" ]; then
 114                         echo "[ok] out/$pattern.res $SAV"
 115                 else
 116                         echo "[ok] out/$pattern.res $SAV - but $FIXME remains!"
 117                 fi
 118                 ok="$ok $pattern"
 119         elif [ -n "$FIXME" ]; then
 120                 if [ -n "$tap" ]; then
 121                         echo "not ok - $description # TODO expected failure"
 122                 else
 123                         echo "[fixme] out/$pattern.res $FIXME"
 124                 fi
 125                 todos="$todos $pattern"
 126         elif [ -n "$SOSO" ]; then
 127                 if [ -n "$tap" ]; then
 128                         echo "ok - $description # SOSO"
 129                 else
 130                         echo "[soso] out/$pattern.res $SOSO"
 131                 fi
 132                 ok="$ok $pattern"
 133         elif [ "x$NYI" = "x0" ]; then
 134                 if [ -n "$tap" ]; then
 135                         echo "not ok - $description # TODO not yet implemented"
 136                 else
 137                         echo "[todo] out/$pattern.res -> not yet implemented"
 138                 fi
 139                 todos="$todos $pattern"
 140         elif [ -n "$SOSOF" ]; then
 141                 if [ -n "$tap" ]; then
 142                         echo "not ok - $description # TODO SOSO expected failure"
 143                 else
 144                         echo "[fixme soso] out/$pattern.res $SOSOF"
 145                 fi
 146                 todos="$todos $pattern"
 147         elif [ -n "$NSAV" ]; then
 148                 if [ -n "$tap" ]; then
 149                         echo "not ok - $description"
 150                 else
 151                         echo "[======= fail out/$pattern.res $NSAV =======]"
 152                 fi
 153                 nok="$nok $pattern"
 154                 echo "$ii" >> "$ERRLIST"
 155         elif [ -n "$NFIXME" ]; then
 156                 if [ -n "$tap" ]; then
 157                         echo "not ok - $description"
 158                 else
 159                         echo "[======= changed out/$pattern.res $NFIXME ======]"
 160                 fi
 161                 nok="$nok $pattern"
 162                 echo "$ii" >> "$ERRLIST"
 163         else
 164                 if [ -n "$tap" ]; then
 165                         echo "ok - $description # skip no sav"
 166                 else
 167                         echo "[=== no sav ===] out/$pattern.res"
 168                 fi
 169                 nos="$nos $pattern"
 170         fi
 171 }
 172
 173 need_skip()
 174 {
 175         test "$noskip" = true && return 1
 176         if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 177                 ((tapcount=tapcount+1))
 178                 if [ -n "$tap" ]; then
 179                         echo "ok - $2 # skip"
 180                 else
 181                         echo "=> $2: [skip]"
 182                 fi
 183                 return 0
 184         fi
 185         return 1
 186 }
 187
 188 find_nitc()
 189 {
 190         ((tapcount=tapcount+1))
 191         name="$enginebinname"
 192         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 193         if [[ "x$recent" == "x" ]]; then
 194                 if [ -n "$tap" ]; then
 195                         echo "not ok - find engine $name"
 196                         echo "Bail out! Could not find engine $name, aborting"
 197                 else
 198                         echo "Could not find engine $name, aborting"
 199                 fi
 200                 exit 1
 201         fi
 202         if [ -n "$tap" ]; then
 203                 echo "ok - find engine $name: $recent"
 204         else
 205                 echo "Using engine $name from: $recent"
 206         fi
 207         NITC=$recent
 208 }
 209
 210 verbose=false
 211 stop=false
 212 tapcount=0
 213 engine=nitc
 214 noskip=
 215 while [ $stop = false ]; do
 216         case $1 in
 217                 -o) OPT="$OPT $2"; shift; shift;;
 218                 -v) verbose=true; shift;;
 219                 -h) usage; exit;;
 220                 --tap) tap=true; shift;;
 221                 --engine) engine="$2"; shift; shift;;
 222                 --noskip) noskip=true; shift;;
 223                 --soso) soso=true; shift;;
 224                 --nososo) nososo=true; shift;;
 225                 *) stop=true
 226         esac
 227 done
 228 enginebinname=$engine
 229 case $engine in
 230         nitc) ;;
 231         nitg) ;;
 232         nit) engine=niti ;;
 233         niti) enginebinname=nit ;;
 234 esac
 235
 236 # The default nitc compiler
 237 [ -z "$NITC" ] && find_nitc
 238
 239 # Set NIT_DIR if needed
 240 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 241
 242
 243 # Mark to distinguish files among tests
 244 # MARK=
 245
 246 # File where error tests are outputed
 247 # Old ERRLIST is backuped
 248 ERRLIST=${ERRLIST:-errlist}
 249 ERRLIST_TARGET=$ERRLIST
 250
 251 if [ $# = 0 ]; then
 252         usage;
 253         exit
 254 fi
 255
 256 # Initiate new ERRLIST
 257 if [ "x$ERRLIST" = "x" ]; then
 258         ERRLIST=/dev=null
 259 else
 260         ERRLIST=$ERRLIST.tmp
 261         > "$ERRLIST"
 262 fi
 263
 264 ok=""
 265 nok=""
 266 todos=""
 267
 268 # CLEAN the out directory
 269 rm -rf out/ 2>/dev/null
 270 mkdir out 2>/dev/null
 271
 272 for ii in "$@"; do
 273         if [ ! -f $ii ]; then
 274                 echo "File '$ii' does not exist."
 275                 continue
 276         fi
 277         f=`basename "$ii" .nit`
 278
 279         # Sould we skip the file for this engine?
 280         need_skip $f $f && continue
 281
 282         tmp=${ii/../AA}
 283         if [ "x$tmp" = "x$ii" ]; then
 284                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 285         else
 286                 includes="-I alt"
 287         fi
 288
 289         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 290                 bf=`basename $i .nit`
 291                 ff="out/$bf"
 292
 293                 # Sould we skip the alternative for this engine?
 294                 need_skip $bf $bf && continue
 295
 296                 test -z "$tap" && echo -n "=> $bf: "
 297
 298                 if [ -f "$f.inputs" ]; then
 299                         inputs="$f.inputs"
 300                 else
 301                         inputs=/dev/null
 302                 fi
 303
 304                 if [ "$engine" = "niti" ]; then
 305                         cat > "./$ff.bin" <<END
 306 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 307 END
 308                         chmod +x "./$ff.bin"
 309                         > "$ff.cmp.err"
 310                         > "$ff.compile.log"
 311                         ERR=0
 312                 else
 313                         # Compile
 314                         if [ "x$verbose" = "xtrue" ]; then
 315                                 echo ""
 316                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 317                         fi
 318                         NIT_NO_STACK=1 $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 319                         ERR=$?
 320                         if [ "x$verbose" = "xtrue" ]; then
 321                                 cat "$ff.compile.log"
 322                                 cat >&2 "$ff.cmp.err"
 323                         fi
 324                 fi
 325                 if [ "$ERR" != 0 ]; then
 326                         test -z "$tap" && echo -n "! "
 327                         cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 328                         process_result $bf $bf
 329                 elif [ -x "./$ff.bin" ]; then
 330                         cp "$ff.cmp.err" "$ff.res"
 331                         test -z "$tap" && echo -n ". "
 332                         # Execute
 333                         args=""
 334                         if [ "x$verbose" = "xtrue" ]; then
 335                                 echo ""
 336                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 337                         fi
 338                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 339                         if [ "x$verbose" = "xtrue" ]; then
 340                                 cat "$ff.res"
 341                                 cat >&2 "$ff.err"
 342                         fi
 343                         if [ -f "$ff.write" ]; then
 344                                 cat "$ff.write" >> "$ff.res"
 345                         elif [ -d "$ff.write" ]; then
 346                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 347                         fi
 348                         if [ -s "$ff.err" ]; then
 349                                 cat "$ff.err" >> "$ff.res"
 350                         fi
 351                         process_result $bf $bf
 352
 353                         if [ -f "$f.args" ]; then
 354                                 fargs=$f.args
 355                                 cptr=0
 356                                 while read line; do
 357                                         ((cptr=cptr+1))
 358                                         args="$line"
 359                                         bff=$bf"_args"$cptr
 360                                         fff=$ff"_args"$cptr
 361
 362                                         # Sould we skip the input for this engine?
 363                                         need_skip $bff "  args #$cptr" && continue
 364
 365                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 366                                         if [ "x$verbose" = "xtrue" ]; then
 367                                                 echo ""
 368                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 369                                         fi
 370                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 371                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 372                                         if [ "x$verbose" = "xtrue" ]; then
 373                                                 cat "$fff.res"
 374                                                 cat >&2 "$fff.err"
 375                                         fi
 376                                         if [ -f "$fff.write" ]; then
 377                                                 cat "$fff.write" >> "$fff.res"
 378                                         elif [ -d "$fff.write" ]; then
 379                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 380                                         fi
 381                                         if [ -s "$fff.err" ]; then
 382                                                 cat "$fff.err" >> "$fff.res"
 383                                         fi
 384                                         process_result $bff "  args #$cptr"
 385                                 done < $fargs
 386                         fi
 387                 else
 388                         test -z "$tap" && echo -n "! "
 389                         cat "$ff.cmp.err" > "$ff.res"
 390                         echo "Compilation error" > "$ff.res"
 391                         process_result $bf "$bf"
 392                 fi
 393         done
 394 done
 395
 396 if [ -n "$tap" ]; then
 397         echo "1..$tapcount"
 398         echo "# ok:" `echo $ok | wc -w`
 399         echo "# not ok:" `echo $nok | wc -w`
 400         echo "# no sav:" `echo $nos | wc -w`
 401         echo "# todo/fixme:" `echo $todos | wc -w`
 402         exit
 403 fi
 404
 405 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 406
 407 if [ -n "$nok" ]; then
 408         echo "fail: $nok"
 409         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 410 fi
 411 if [ -n "$nos" ]; then
 412         echo "no sav: $nos"
 413 fi
 414 if [ -n "$todos" ]; then
 415         echo "todo/fixme: $todos"
 416 fi
 417
 418 # write $ERRLIST
 419 if [ "x$ERRLIST" != "x" ]; then
 420         if [ -x "$ERRLIST_TARGET" ]; then
 421                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 422         fi
 423         mv $ERRLIST $ERRLIST_TARGET
 424 fi
 425
 426 if [ -n "$nok" ]; then
 427         exit 1
 428 else
 429         exit 0
 430 fi