tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to the engine
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 --engine    Use a specific engine (default=nitc)
  33 --noskip    Do not skip a test even if the .skip file matches
  34 END
  35 }
  36
  37 # As argument: the pattern used for the file
  38 function process_result()
  39 {
  40         ((tapcount=tapcount+1))
  41         # Result
  42         pattern=$1
  43         description=$2
  44         SAV=""
  45         FAIL=""
  46         SOSO=""
  47         SOSOF=""
  48         if [ -r "sav/$pattern.sav" ]; then
  49                 diff -u "out/$pattern.res" "sav/$pattern.sav" > "out/$pattern.diff.sav.log"
  50                 if [ "$?" == 0 ]; then
  51                         SAV=OK
  52                 else
  53                         SAV=NOK
  54                 fi
  55                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  56                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.sav" > "out/$pattern.sav2"
  57                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  58                 grep '[Ee]rror' "sav/$pattern.sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  59                 diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  60                 if [ "$?" == 0 ]; then
  61                         SOSO=OK
  62                 else
  63                         SOSO=NOK
  64                 fi
  65         fi
  66         if [ -r "sav/$pattern.fail" ]; then
  67                 diff -u "out/$pattern.res" "sav/$pattern.fail" > "out/$pattern.diff.fail.log"
  68                 if [ "$?" == 0 ]; then
  69                         FAIL=OK
  70                 else
  71                         FAIL=NOK
  72                 fi
  73                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  74                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.fail" > "out/$pattern.fail2"
  75                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  76                 grep '[Ee]rror' "sav/$pattern.fail" >/dev/null && echo "Error" >> "out/$pattern.fail2"
  77                 diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
  78                 if [ "$?" == 0 ]; then
  79                         SOSOF=OK
  80                 else
  81                         SOSOF=NOK
  82                 fi
  83         fi
  84         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
  85         NYI="$?"
  86         if [ "x$SAV" = "xOK" ]; then
  87                 if [ -n "$tap" ]; then
  88                         echo "ok - $description"
  89                 elif [ "x$FAIL" = "x" ]; then
  90                         echo "[ok] out/$pattern.res"
  91                 else
  92                         echo "[ok] out/$pattern.res - but sav/$pattern.fail remains!"
  93                 fi
  94                 ok="$ok $pattern"
  95         elif [ "x$FAIL" = "xOK" ]; then
  96                 if [ -n "$tap" ]; then
  97                         echo "not ok - $description # TODO expected failure"
  98                 else
  99                         echo "[fail] out/$pattern.res"
 100                 fi
 101                 ok="$ok $pattern"
 102         elif [ "x$SOSO" = "xOK" ]; then
 103                 if [ -n "$tap" ]; then
 104                         echo "ok - $description # SOSO"
 105                 else
 106                         echo "[soso] out/$pattern.res sav/$pattern.sav"
 107                 fi
 108                 ok="$ok $pattern"
 109         elif [ "x$NYI" = "x0" ]; then
 110                 if [ -n "$tap" ]; then
 111                         echo "not ok - $description # TODO not yet implemented"
 112                 else
 113                         echo "[todo] out/$pattern.res -> not yet implemented"
 114                 fi
 115                 ok="$ok $pattern"
 116         elif [ "x$SOSOF" = "xOK" ]; then
 117                 if [ -n "$tap" ]; then
 118                         echo "not ok - $description # TODO SOSO expected failure"
 119                 else
 120                         echo "[fail soso] out/$pattern.res sav/$pattern.fail"
 121                 fi
 122                 ok="$ok $pattern"
 123         elif [ "x$SAV" = "xNOK" ]; then
 124                 if [ -n "$tap" ]; then
 125                         echo "not ok - $description"
 126                 else
 127                         echo "[======= fail out/$pattern.res sav/$pattern.sav =======]"
 128                 fi
 129                 nok="$nok $pattern"
 130                 echo "$ii" >> "$ERRLIST"
 131         elif [ "x$FAIL" = "xNOK" ]; then
 132                 if [ -n "$tap" ]; then
 133                         echo "not ok - $description"
 134                 else
 135                         echo "[======= changed out/$pattern.res sav/$pattern.fail ======]"
 136                 fi
 137                 nok="$nok $pattern"
 138                 echo "$ii" >> "$ERRLIST"
 139         else
 140                 if [ -n "$tap" ]; then
 141                         echo "ok - $description # skip no sav"
 142                 else
 143                         echo "[=== no sav ===] out/$pattern.res"
 144                 fi
 145                 nos="$nos $pattern"
 146         fi
 147 }
 148
 149 need_skip()
 150 {
 151         test "$noskip" = true && return 1
 152         if grep "$engine" "sav/$1.skip" >/dev/null 2>&1; then
 153                 ((tapcount=tapcount+1))
 154                 if [ -n "$tap" ]; then
 155                         echo "ok - $2 # skip"
 156                 else
 157                         echo "=> $2: [skip]"
 158                 fi
 159                 return 0
 160         fi
 161         return 1
 162 }
 163
 164 find_nitc()
 165 {
 166         ((tapcount=tapcount+1))
 167         name="$enginebinname"
 168         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 169         if [[ "x$recent" == "x" ]]; then
 170                 if [ -n "$tap" ]; then
 171                         echo "not ok - find engine $name"
 172                         echo "Bail out! Could not find engine $name, aborting"
 173                 else
 174                         echo "Could not find engine $name, aborting"
 175                 fi
 176                 exit 1
 177         fi
 178         if [ -n "$tap" ]; then
 179                 echo "ok - find engine $name: $recent"
 180         else
 181                 echo "Using engine $name from: $recent"
 182         fi
 183         NITC=$recent
 184 }
 185
 186 verbose=false
 187 stop=false
 188 tapcount=0
 189 engine=nitc
 190 noskip=
 191 while [ $stop = false ]; do
 192         case $1 in
 193                 -o) OPT="$OPT $2"; shift; shift;;
 194                 -v) verbose=true; shift;;
 195                 -h) usage; exit;;
 196                 --tap) tap=true; shift;;
 197                 --engine) engine="$2"; shift; shift;;
 198                 --noskip) noskip=true; shift;;
 199                 *) stop=true
 200         esac
 201 done
 202 enginebinname=$engine
 203 case $engine in
 204         nitc|nitg) ;;
 205         nit) engine=niti ;;
 206         niti) enginebinname=nit ;;
 207 esac
 208
 209 # The default nitc compiler
 210 [ -z "$NITC" ] && find_nitc
 211
 212 # Set NIT_DIR if needed
 213 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 214
 215
 216 # Mark to distinguish files among tests
 217 # MARK=
 218
 219 # File where error tests are outputed
 220 # Old ERRLIST is backuped
 221 ERRLIST=${ERRLIST:-errlist}
 222 ERRLIST_TARGET=$ERRLIST
 223
 224 if [ $# = 0 ]; then
 225         usage;
 226         exit
 227 fi
 228
 229 # Initiate new ERRLIST
 230 if [ "x$ERRLIST" = "x" ]; then
 231         ERRLIST=/dev=null
 232 else
 233         ERRLIST=$ERRLIST.tmp
 234         > "$ERRLIST"
 235 fi
 236
 237 ok=""
 238 nok=""
 239
 240 # CLEAN the out directory
 241 rm -rf out/ 2>/dev/null
 242 mkdir out 2>/dev/null
 243
 244 for ii in "$@"; do
 245         if [ ! -f $ii ]; then
 246                 echo "File '$ii' does not exist."
 247                 continue
 248         fi
 249         f=`basename "$ii" .nit`
 250
 251         # Sould we skip the file for this engine?
 252         need_skip $f $f && continue
 253
 254         tmp=${ii/../AA}
 255         if [ "x$tmp" = "x$ii" ]; then
 256                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 257         else
 258                 includes="-I alt"
 259         fi
 260
 261         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 262                 bf=`basename $i .nit`
 263                 ff="out/$bf"
 264
 265                 # Sould we skip the alternative for this engine?
 266                 need_skip $bf $bf && continue
 267
 268                 test -z "$tap" && echo -n "=> $bf: "
 269
 270                 if [ -f "$f.inputs" ]; then
 271                         inputs="$f.inputs"
 272                 else
 273                         inputs=/dev/null
 274                 fi
 275
 276                 if [ "$engine" = "niti" ]; then
 277                         cat > "./$ff.bin" <<END
 278 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 279 END
 280                         chmod +x "./$ff.bin"
 281                         > "$ff.cmp.err"
 282                         > "$ff.compile.log"
 283                         ERR=0
 284                 else
 285                         # Compile
 286                         if [ "x$verbose" = "xtrue" ]; then
 287                                 echo ""
 288                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 289                         fi
 290                         $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 291                         ERR=$?
 292                         if [ "x$verbose" = "xtrue" ]; then
 293                                 cat "$ff.compile.log"
 294                                 cat >&2 "$ff.cmp.err"
 295                         fi
 296                 fi
 297                 if [ "$ERR" != 0 ]; then
 298                         test -z "$tap" && echo -n "! "
 299                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 300                         process_result $bf $bf
 301                 elif [ -x "./$ff.bin" ]; then
 302                         cp "$ff.cmp.err" "$ff.res"
 303                         test -z "$tap" && echo -n ". "
 304                         # Execute
 305                         args=""
 306                         if [ "x$verbose" = "xtrue" ]; then
 307                                 echo ""
 308                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 309                         fi
 310                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 311                         if [ "x$verbose" = "xtrue" ]; then
 312                                 cat "$ff.res"
 313                                 cat >&2 "$ff.err"
 314                         fi
 315                         if [ -f "$ff.write" ]; then
 316                                 cat "$ff.write" >> "$ff.res"
 317                         elif [ -d "$ff.write" ]; then
 318                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 319                         fi
 320                         if [ -s "$ff.err" ]; then
 321                                 cat "$ff.err" >> "$ff.res"
 322                         fi
 323                         process_result $bf $bf
 324
 325                         if [ -f "$f.args" ]; then
 326                                 fargs=$f.args
 327                                 cptr=0
 328                                 while read line; do
 329                                         ((cptr=cptr+1))
 330                                         args="$line"
 331                                         bff=$bf"_args"$cptr
 332                                         fff=$ff"_args"$cptr
 333
 334                                         # Sould we skip the input for this engine?
 335                                         need_skip $bff "  args #$cptr" && continue
 336
 337                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 338                                         if [ "x$verbose" = "xtrue" ]; then
 339                                                 echo ""
 340                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 341                                         fi
 342                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 343                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 344                                         if [ "x$verbose" = "xtrue" ]; then
 345                                                 cat "$fff.res"
 346                                                 cat >&2 "$fff.err"
 347                                         fi
 348                                         if [ -f "$fff.write" ]; then
 349                                                 cat "$fff.write" >> "$fff.res"
 350                                         elif [ -d "$fff.write" ]; then
 351                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 352                                         fi
 353                                         if [ -s "$fff.err" ]; then
 354                                                 cat "$fff.err" >> "$fff.res"
 355                                         fi
 356                                         process_result $bff "  args #$cptr"
 357                                 done < $fargs
 358                         fi
 359                 else
 360                         test -z "$tap" && echo -n "! "
 361                         cat "$ff.cmp.err" > "$ff.res"
 362                         echo "Compilation error" > "$ff.res"
 363                         process_result $bf "$bf"
 364                 fi
 365         done
 366 done
 367
 368 if [ -n "$tap" ]; then
 369         echo "1..$tapcount"
 370         echo "# ok:" `echo $ok | wc -w`
 371         echo "# not ok:" `echo $nok | wc -w`
 372         echo "# no sav:" `echo $nos | wc -w`
 373         exit
 374 fi
 375
 376 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w`
 377
 378 if [ -n "$nok" ]; then
 379         echo "fail: $nok"
 380         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 381 fi
 382 if [ -n "$nos" ]; then
 383         echo "no sav: $nos"
 384 fi
 385
 386 # write $ERRLIST
 387 if [ "x$ERRLIST" != "x" ]; then
 388         if [ -x "$ERRLIST_TARGET" ]; then
 389                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 390         fi
 391         mv $ERRLIST $ERRLIST_TARGET
 392 fi
 393
 394 if [ -n "$nok" ]; then
 395         exit 1
 396 else
 397         exit 0
 398 fi