tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to the engine
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 --engine    Use a specific engine (default=nitc)
  33 --noskip    Do not skip a test even if the .skip file matches
  34 --[no]soso  Force enable (or disable) SOSO
  35 END
  36 }
  37
  38 # As argument: the pattern used for the file
  39 function process_result()
  40 {
  41         ((tapcount=tapcount+1))
  42         # Result
  43         pattern=$1
  44         description=$2
  45         SAV=""
  46         NSAV=""
  47         FAIL=""
  48         NFAIL=""
  49         SOSO=""
  50         NSOSO=""
  51         SOSOF=""
  52         NSOSOF=""
  53         for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
  54                 if [ -r "$sav" ]; then
  55                         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
  56                         if [ "$?" == 0 ]; then
  57                                 SAV="$sav"
  58                         else
  59                                 NSAV="$sav"
  60                         fi
  61                         [ -z "$soso" ] && continue
  62                         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  63                         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
  64                         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  65                         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  66                         diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  67                         if [ "$?" == 0 ]; then
  68                                 SOSO="$sav"
  69                         else
  70                                 NSOSO="$sav"
  71                         fi
  72                 fi
  73         done
  74         for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
  75                 if [ -r "$sav" ]; then
  76                         diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.fail.log"
  77                         if [ "$?" == 0 ]; then
  78                                 FAIL="$sav"
  79                         else
  80                                 NFAIL="$sav"
  81                         fi
  82                         [ -z "$soso" ] && continue
  83                         sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  84                         sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.fail2"
  85                         grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  86                         grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.fail2"
  87                         diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
  88                         if [ "$?" == 0 ]; then
  89                                 SOSOF="$sav"
  90                         else
  91                                 NSOSOF="$sav"
  92                         fi
  93                 fi
  94         done
  95         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
  96         NYI="$?"
  97         if [ -n "$SAV" ]; then
  98                 if [ -n "$tap" ]; then
  99                         echo "ok - $description"
 100                 elif [ -z "$FAIL" ]; then
 101                         echo "[ok] out/$pattern.res $SAV"
 102                 else
 103                         echo "[ok] out/$pattern.res $SAV - but $FAIL remains!"
 104                 fi
 105                 ok="$ok $pattern"
 106         elif [ -n "$FAIL" ]; then
 107                 if [ -n "$tap" ]; then
 108                         echo "not ok - $description # TODO expected failure"
 109                 else
 110                         echo "[fail] out/$pattern.res $FAIL"
 111                 fi
 112                 ok="$ok $pattern"
 113         elif [ -n "$SOSO" ]; then
 114                 if [ -n "$tap" ]; then
 115                         echo "ok - $description # SOSO"
 116                 else
 117                         echo "[soso] out/$pattern.res $SOSO"
 118                 fi
 119                 ok="$ok $pattern"
 120         elif [ "x$NYI" = "x0" ]; then
 121                 if [ -n "$tap" ]; then
 122                         echo "not ok - $description # TODO not yet implemented"
 123                 else
 124                         echo "[todo] out/$pattern.res -> not yet implemented"
 125                 fi
 126                 ok="$ok $pattern"
 127         elif [ -n "$SOSOF" ]; then
 128                 if [ -n "$tap" ]; then
 129                         echo "not ok - $description # TODO SOSO expected failure"
 130                 else
 131                         echo "[fail soso] out/$pattern.res $SOSOF"
 132                 fi
 133                 ok="$ok $pattern"
 134         elif [ -n "$NSAV" ]; then
 135                 if [ -n "$tap" ]; then
 136                         echo "not ok - $description"
 137                 else
 138                         echo "[======= fail out/$pattern.res $NSAV =======]"
 139                 fi
 140                 nok="$nok $pattern"
 141                 echo "$ii" >> "$ERRLIST"
 142         elif [ -n "$NFAIL" ]; then
 143                 if [ -n "$tap" ]; then
 144                         echo "not ok - $description"
 145                 else
 146                         echo "[======= changed out/$pattern.res $NFAIL ======]"
 147                 fi
 148                 nok="$nok $pattern"
 149                 echo "$ii" >> "$ERRLIST"
 150         else
 151                 if [ -n "$tap" ]; then
 152                         echo "ok - $description # skip no sav"
 153                 else
 154                         echo "[=== no sav ===] out/$pattern.res"
 155                 fi
 156                 nos="$nos $pattern"
 157         fi
 158 }
 159
 160 need_skip()
 161 {
 162         test "$noskip" = true && return 1
 163         if grep "$engine" "sav/$1.skip" >/dev/null 2>&1 || echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
 164                 ((tapcount=tapcount+1))
 165                 if [ -n "$tap" ]; then
 166                         echo "ok - $2 # skip"
 167                 else
 168                         echo "=> $2: [skip]"
 169                 fi
 170                 return 0
 171         fi
 172         return 1
 173 }
 174
 175 find_nitc()
 176 {
 177         ((tapcount=tapcount+1))
 178         name="$enginebinname"
 179         recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
 180         if [[ "x$recent" == "x" ]]; then
 181                 if [ -n "$tap" ]; then
 182                         echo "not ok - find engine $name"
 183                         echo "Bail out! Could not find engine $name, aborting"
 184                 else
 185                         echo "Could not find engine $name, aborting"
 186                 fi
 187                 exit 1
 188         fi
 189         if [ -n "$tap" ]; then
 190                 echo "ok - find engine $name: $recent"
 191         else
 192                 echo "Using engine $name from: $recent"
 193         fi
 194         NITC=$recent
 195 }
 196
 197 verbose=false
 198 stop=false
 199 tapcount=0
 200 engine=nitc
 201 noskip=
 202 while [ $stop = false ]; do
 203         case $1 in
 204                 -o) OPT="$OPT $2"; shift; shift;;
 205                 -v) verbose=true; shift;;
 206                 -h) usage; exit;;
 207                 --tap) tap=true; shift;;
 208                 --engine) engine="$2"; shift; shift;;
 209                 --noskip) noskip=true; shift;;
 210                 --soso) soso=true; shift;;
 211                 --nososo) nososo=true; shift;;
 212                 *) stop=true
 213         esac
 214 done
 215 enginebinname=$engine
 216 case $engine in
 217         nitc) ;;
 218         nitg) [ -z "$nososo" ] && soso=true ;;
 219         nit) [ -z "$nososo" ] && soso=true
 220                 engine=niti ;;
 221         niti) [ -z "$nososo" ] && soso=true
 222                 enginebinname=nit ;;
 223 esac
 224
 225 # The default nitc compiler
 226 [ -z "$NITC" ] && find_nitc
 227
 228 # Set NIT_DIR if needed
 229 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 230
 231
 232 # Mark to distinguish files among tests
 233 # MARK=
 234
 235 # File where error tests are outputed
 236 # Old ERRLIST is backuped
 237 ERRLIST=${ERRLIST:-errlist}
 238 ERRLIST_TARGET=$ERRLIST
 239
 240 if [ $# = 0 ]; then
 241         usage;
 242         exit
 243 fi
 244
 245 # Initiate new ERRLIST
 246 if [ "x$ERRLIST" = "x" ]; then
 247         ERRLIST=/dev=null
 248 else
 249         ERRLIST=$ERRLIST.tmp
 250         > "$ERRLIST"
 251 fi
 252
 253 ok=""
 254 nok=""
 255
 256 # CLEAN the out directory
 257 rm -rf out/ 2>/dev/null
 258 mkdir out 2>/dev/null
 259
 260 for ii in "$@"; do
 261         if [ ! -f $ii ]; then
 262                 echo "File '$ii' does not exist."
 263                 continue
 264         fi
 265         f=`basename "$ii" .nit`
 266
 267         # Sould we skip the file for this engine?
 268         need_skip $f $f && continue
 269
 270         tmp=${ii/../AA}
 271         if [ "x$tmp" = "x$ii" ]; then
 272                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 273         else
 274                 includes="-I alt"
 275         fi
 276
 277         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 278                 bf=`basename $i .nit`
 279                 ff="out/$bf"
 280
 281                 # Sould we skip the alternative for this engine?
 282                 need_skip $bf $bf && continue
 283
 284                 test -z "$tap" && echo -n "=> $bf: "
 285
 286                 if [ -f "$f.inputs" ]; then
 287                         inputs="$f.inputs"
 288                 else
 289                         inputs=/dev/null
 290                 fi
 291
 292                 if [ "$engine" = "niti" ]; then
 293                         cat > "./$ff.bin" <<END
 294 exec $NITC --no-color $OPT "$i" $includes -- "\$@"
 295 END
 296                         chmod +x "./$ff.bin"
 297                         > "$ff.cmp.err"
 298                         > "$ff.compile.log"
 299                         ERR=0
 300                 else
 301                         # Compile
 302                         if [ "x$verbose" = "xtrue" ]; then
 303                                 echo ""
 304                                 echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 305                         fi
 306                         $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
 307                         ERR=$?
 308                         if [ "x$verbose" = "xtrue" ]; then
 309                                 cat "$ff.compile.log"
 310                                 cat >&2 "$ff.cmp.err"
 311                         fi
 312                 fi
 313                 if [ "$ERR" != 0 ]; then
 314                         test -z "$tap" && echo -n "! "
 315                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 316                         process_result $bf $bf
 317                 elif [ -x "./$ff.bin" ]; then
 318                         cp "$ff.cmp.err" "$ff.res"
 319                         test -z "$tap" && echo -n ". "
 320                         # Execute
 321                         args=""
 322                         if [ "x$verbose" = "xtrue" ]; then
 323                                 echo ""
 324                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 325                         fi
 326                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 327                         if [ "x$verbose" = "xtrue" ]; then
 328                                 cat "$ff.res"
 329                                 cat >&2 "$ff.err"
 330                         fi
 331                         if [ -f "$ff.write" ]; then
 332                                 cat "$ff.write" >> "$ff.res"
 333                         elif [ -d "$ff.write" ]; then
 334                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 335                         fi
 336                         if [ -s "$ff.err" ]; then
 337                                 cat "$ff.err" >> "$ff.res"
 338                         fi
 339                         process_result $bf $bf
 340
 341                         if [ -f "$f.args" ]; then
 342                                 fargs=$f.args
 343                                 cptr=0
 344                                 while read line; do
 345                                         ((cptr=cptr+1))
 346                                         args="$line"
 347                                         bff=$bf"_args"$cptr
 348                                         fff=$ff"_args"$cptr
 349
 350                                         # Sould we skip the input for this engine?
 351                                         need_skip $bff "  args #$cptr" && continue
 352
 353                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 354                                         if [ "x$verbose" = "xtrue" ]; then
 355                                                 echo ""
 356                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 357                                         fi
 358                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 359                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 360                                         if [ "x$verbose" = "xtrue" ]; then
 361                                                 cat "$fff.res"
 362                                                 cat >&2 "$fff.err"
 363                                         fi
 364                                         if [ -f "$fff.write" ]; then
 365                                                 cat "$fff.write" >> "$fff.res"
 366                                         elif [ -d "$fff.write" ]; then
 367                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 368                                         fi
 369                                         if [ -s "$fff.err" ]; then
 370                                                 cat "$fff.err" >> "$fff.res"
 371                                         fi
 372                                         process_result $bff "  args #$cptr"
 373                                 done < $fargs
 374                         fi
 375                 else
 376                         test -z "$tap" && echo -n "! "
 377                         cat "$ff.cmp.err" > "$ff.res"
 378                         echo "Compilation error" > "$ff.res"
 379                         process_result $bf "$bf"
 380                 fi
 381         done
 382 done
 383
 384 if [ -n "$tap" ]; then
 385         echo "1..$tapcount"
 386         echo "# ok:" `echo $ok | wc -w`
 387         echo "# not ok:" `echo $nok | wc -w`
 388         echo "# no sav:" `echo $nos | wc -w`
 389         exit
 390 fi
 391
 392 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w`
 393
 394 if [ -n "$nok" ]; then
 395         echo "fail: $nok"
 396         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 397 fi
 398 if [ -n "$nos" ]; then
 399         echo "no sav: $nos"
 400 fi
 401
 402 # write $ERRLIST
 403 if [ "x$ERRLIST" != "x" ]; then
 404         if [ -x "$ERRLIST_TARGET" ]; then
 405                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 406         fi
 407         mv $ERRLIST $ERRLIST_TARGET
 408 fi
 409
 410 if [ -n "$nok" ]; then
 411         exit 1
 412 else
 413         exit 0
 414 fi