tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to the engine
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 --engine    Use a specific engine (default=nitc)
  33 --noskip    Do not skip a test even if the .skip file matches
  34 --[no]soso  Force enable (or disable) SOSO
  35 END
  36 }
  37
  38 # As argument: the pattern used for the file
  39 function process_result()
  40 {
  41         ((tapcount=tapcount+1))
  42         # Result
  43         pattern=$1
  44         description=$2
  45         SAV=""
  46         NSAV=""
  47         FAIL=""
  48         NFAIL=""
  49         SOSO=""
  50         NSOSO=""
  51         SOSOF=""
  52         NSOSOF=""
  53         for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  54                 if [ -r "$sav" ]; then
  55                         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
  56                         if [ "$?" == 0 ]; then
  57                                 SAV="$sav"
  58                         else
  59                                 NSAV="$sav"
  60                         fi
  61                         [ -z "$soso" ] && continue
  62                         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  63                         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  64                         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  65                         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  66                         diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  67                         if [ "$?" == 0 ]; then
  68                                 SOSO="$sav"
  69                         else
  70                                 NSOSO="$sav"
  71                         fi
  72                 fi
  73         done
  74         for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
  75                 if [ -r "$sav" ]; then
  76                         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.fail.log"
  77                         if [ "$?" == 0 ]; then
  78                                 FAIL="$sav"
  79                         else
  80                                 NFAIL="$sav"
  81                         fi
  82                         [ -z "$soso" ] && continue
  83                         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  84                         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.fail2"
  85                         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  86                         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.fail2"
  87                         diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
  88                         if [ "$?" == 0 ]; then
  89                                 SOSOF="$sav"
  90                         else
  91                                 NSOSOF="$sav"
  92                         fi
  93                 fi
  94         done
  95         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
  96         NYI="$?"
  97         if [ -n "$SAV" ]; then
  98                 if [ -n "$tap" ]; then
  99                         echo "ok - $description"
 100                 elif [ -z "$FAIL" ]; then
 101                         echo "[ok] out/$pattern.res $SAV"
 102                 else
 103                         echo "[ok] out/$pattern.res $SAV - but $FAIL remains!"
 104                 fi
 105                 ok="$ok $pattern"
 106         elif [ -n "$FAIL" ]; then
 107                 if [ -n "$tap" ]; then
 108                         echo "not ok - $description # TODO expected failure"
 109                 else
 110                         echo "[fail] out/$pattern.res $FAIL"
 111                 fi
 112                 ok="$ok $pattern"
 113         elif [ -n "$SOSO" ]; then
 114                 if [ -n "$tap" ]; then
 115                         echo "ok - $description # SOSO"
 116                 else
 117                         echo "[soso] out/$pattern.res $SOSO"
 118                 fi
 119                 ok="$ok $pattern"
 120         elif [ "x$NYI" = "x0" ]; then
 121                 if [ -n "$tap" ]; then
 122                         echo "not ok - $description # TODO not yet implemented"
 123                 else
 124                         echo "[todo] out/$pattern.res -> not yet implemented"
 125                 fi
 126                 todos="$todos $pattern"
 127         elif [ -n "$SOSOF" ]; then
 128                 if [ -n "$tap" ]; then
 129                         echo "not ok - $description # TODO SOSO expected failure"
 130                 else
 131                         echo "[fail soso] out/$pattern.res $SOSOF"
 132                 fi
 133                 ok="$ok $pattern"
 134         elif [ -n "$NSAV" ]; then
 135                 if [ -n "$tap" ]; then
 136                         echo "not ok - $description"
 137                 else
 138                         echo "[======= fail out/$pattern.res $NSAV =======]"
 139                 fi
 140                 nok="$nok $pattern"
 141                 echo "$ii" >> "$ERRLIST"
 142         elif [ -n "$NFAIL" ]; then
 143                 if [ -n "$tap" ]; then
 144                         echo "not ok - $description"
 145                 else
 146                         echo "[======= changed out/$pattern.res $NFAIL ======]"
 147                 fi
 148                 nok="$nok $pattern"
 149                 echo "$ii" >> "$ERRLIST"
 150         else
 151                 if [ -n "$tap" ]; then
 152                         echo "ok - $description # skip no sav"
 153                 else
 154                         echo "[=== no sav ===] out/$pattern.res"
 155                 fi
 156                 nos="$nos $pattern"
 157         fi
 158 }
 159
 160 need_skip()
 161 {
 162         test "$noskip" = true && return 1
 163         if grep "$engine" "sav/$1.skip" >/dev/null 2>&1 || echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 164                 ((tapcount=tapcount+1))
 165                 if [ -n "$tap" ]; then
 166                         echo "ok - $2 # skip"
 167                 else
 168                         echo "=> $2: [skip]"
 169                 fi
 170                 return 0
 171         fi
 172         return 1
 173 }
 174
 175 find_nitc()
 176 {
 177         ((tapcount=tapcount+1))
 178         name="$enginebinname"
 179         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 180         if [[ "x$recent" == "x" ]]; then
 181                 if [ -n "$tap" ]; then
 182                         echo "not ok - find engine $name"
 183                         echo "Bail out! Could not find engine $name, aborting"
 184                 else
 185                         echo "Could not find engine $name, aborting"
 186                 fi
 187                 exit 1
 188         fi
 189         if [ -n "$tap" ]; then
 190                 echo "ok - find engine $name: $recent"
 191         else
 192                 echo "Using engine $name from: $recent"
 193         fi
 194         NITC=$recent
 195 }
 196
 197 verbose=false
 198 stop=false
 199 tapcount=0
 200 engine=nitc
 201 noskip=
 202 while [ $stop = false ]; do
 203         case $1 in
 204                 -o) OPT="$OPT $2"; shift; shift;;
 205                 -v) verbose=true; shift;;
 206                 -h) usage; exit;;
 207                 --tap) tap=true; shift;;
 208                 --engine) engine="$2"; shift; shift;;
 209                 --noskip) noskip=true; shift;;
 210                 --soso) soso=true; shift;;
 211                 --nososo) nososo=true; shift;;
 212                 *) stop=true
 213         esac
 214 done
 215 enginebinname=$engine
 216 case $engine in
 217         nitc) ;;
 218         nitg) ;;
 219         nit) engine=niti ;;
 220         niti) enginebinname=nit ;;
 221 esac
 222
 223 # The default nitc compiler
 224 [ -z "$NITC" ] && find_nitc
 225
 226 # Set NIT_DIR if needed
 227 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 228
 229
 230 # Mark to distinguish files among tests
 231 # MARK=
 232
 233 # File where error tests are outputed
 234 # Old ERRLIST is backuped
 235 ERRLIST=${ERRLIST:-errlist}
 236 ERRLIST_TARGET=$ERRLIST
 237
 238 if [ $# = 0 ]; then
 239         usage;
 240         exit
 241 fi
 242
 243 # Initiate new ERRLIST
 244 if [ "x$ERRLIST" = "x" ]; then
 245         ERRLIST=/dev=null
 246 else
 247         ERRLIST=$ERRLIST.tmp
 248         > "$ERRLIST"
 249 fi
 250
 251 ok=""
 252 nok=""
 253 todos=""
 254
 255 # CLEAN the out directory
 256 rm -rf out/ 2>/dev/null
 257 mkdir out 2>/dev/null
 258
 259 for ii in "$@"; do
 260         if [ ! -f $ii ]; then
 261                 echo "File '$ii' does not exist."
 262                 continue
 263         fi
 264         f=`basename "$ii" .nit`
 265
 266         # Sould we skip the file for this engine?
 267         need_skip $f $f && continue
 268
 269         tmp=${ii/../AA}
 270         if [ "x$tmp" = "x$ii" ]; then
 271                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 272         else
 273                 includes="-I alt"
 274         fi
 275
 276         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 277                 bf=`basename $i .nit`
 278                 ff="out/$bf"
 279
 280                 # Sould we skip the alternative for this engine?
 281                 need_skip $bf $bf && continue
 282
 283                 test -z "$tap" && echo -n "=> $bf: "
 284
 285                 if [ -f "$f.inputs" ]; then
 286                         inputs="$f.inputs"
 287                 else
 288                         inputs=/dev/null
 289                 fi
 290
 291                 if [ "$engine" = "niti" ]; then
 292                         cat > "./$ff.bin" <<END
 293 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 294 END
 295                         chmod +x "./$ff.bin"
 296                         > "$ff.cmp.err"
 297                         > "$ff.compile.log"
 298                         ERR=0
 299                 else
 300                         # Compile
 301                         if [ "x$verbose" = "xtrue" ]; then
 302                                 echo ""
 303                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 304                         fi
 305                         NIT_NO_STACK=1 $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 306                         ERR=$?
 307                         if [ "x$verbose" = "xtrue" ]; then
 308                                 cat "$ff.compile.log"
 309                                 cat >&2 "$ff.cmp.err"
 310                         fi
 311                 fi
 312                 if [ "$ERR" != 0 ]; then
 313                         test -z "$tap" && echo -n "! "
 314                         cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
 315                         process_result $bf $bf
 316                 elif [ -x "./$ff.bin" ]; then
 317                         cp "$ff.cmp.err" "$ff.res"
 318                         test -z "$tap" && echo -n ". "
 319                         # Execute
 320                         args=""
 321                         if [ "x$verbose" = "xtrue" ]; then
 322                                 echo ""
 323                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 324                         fi
 325                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 326                         if [ "x$verbose" = "xtrue" ]; then
 327                                 cat "$ff.res"
 328                                 cat >&2 "$ff.err"
 329                         fi
 330                         if [ -f "$ff.write" ]; then
 331                                 cat "$ff.write" >> "$ff.res"
 332                         elif [ -d "$ff.write" ]; then
 333                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 334                         fi
 335                         if [ -s "$ff.err" ]; then
 336                                 cat "$ff.err" >> "$ff.res"
 337                         fi
 338                         process_result $bf $bf
 339
 340                         if [ -f "$f.args" ]; then
 341                                 fargs=$f.args
 342                                 cptr=0
 343                                 while read line; do
 344                                         ((cptr=cptr+1))
 345                                         args="$line"
 346                                         bff=$bf"_args"$cptr
 347                                         fff=$ff"_args"$cptr
 348
 349                                         # Sould we skip the input for this engine?
 350                                         need_skip $bff "  args #$cptr" && continue
 351
 352                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 353                                         if [ "x$verbose" = "xtrue" ]; then
 354                                                 echo ""
 355                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 356                                         fi
 357                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 358                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 359                                         if [ "x$verbose" = "xtrue" ]; then
 360                                                 cat "$fff.res"
 361                                                 cat >&2 "$fff.err"
 362                                         fi
 363                                         if [ -f "$fff.write" ]; then
 364                                                 cat "$fff.write" >> "$fff.res"
 365                                         elif [ -d "$fff.write" ]; then
 366                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 367                                         fi
 368                                         if [ -s "$fff.err" ]; then
 369                                                 cat "$fff.err" >> "$fff.res"
 370                                         fi
 371                                         process_result $bff "  args #$cptr"
 372                                 done < $fargs
 373                         fi
 374                 else
 375                         test -z "$tap" && echo -n "! "
 376                         cat "$ff.cmp.err" > "$ff.res"
 377                         echo "Compilation error" > "$ff.res"
 378                         process_result $bf "$bf"
 379                 fi
 380         done
 381 done
 382
 383 if [ -n "$tap" ]; then
 384         echo "1..$tapcount"
 385         echo "# ok:" `echo $ok | wc -w`
 386         echo "# not ok:" `echo $nok | wc -w`
 387         echo "# no sav:" `echo $nos | wc -w`
 388         echo "# todos:" `echo $todos | wc -w`
 389         exit
 390 fi
 391
 392 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 393
 394 if [ -n "$nok" ]; then
 395         echo "fail: $nok"
 396         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 397 fi
 398 if [ -n "$nos" ]; then
 399         echo "no sav: $nos"
 400 fi
 401 if [ -n "$todos" ]; then
 402         echo "todos: $todos"
 403 fi
 404
 405 # write $ERRLIST
 406 if [ "x$ERRLIST" != "x" ]; then
 407         if [ -x "$ERRLIST_TARGET" ]; then
 408                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 409         fi
 410         mv $ERRLIST $ERRLIST_TARGET
 411 fi
 412
 413 if [ -n "$nok" ]; then
 414         exit 1
 415 else
 416         exit 0
 417 fi