tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to nitc
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 --tap       Produce TAP output
  32 END
  33 }
  34
  35 # As argument: the pattern used for the file
  36 function process_result()
  37 {
  38         ((tapcount=tapcount+1))
  39         # Result
  40         pattern=$1
  41         description=$2
  42         SAV=""
  43         FAIL=""
  44         SOSO=""
  45         SOSOF=""
  46         if [ -r "sav/$pattern.sav" ]; then
  47                 diff -u "out/$pattern.res" "sav/$pattern.sav" > "out/$pattern.diff.sav.log"
  48                 if [ "$?" == 0 ]; then
  49                         SAV=OK
  50                 else
  51                         SAV=NOK
  52                 fi
  53                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  54                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.sav" > "out/$pattern.sav2"
  55                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  56                 grep '[Ee]rror' "sav/$pattern.sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  57                 diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  58                 if [ "$?" == 0 ]; then
  59                         SOSO=OK
  60                 else
  61                         SOSO=NOK
  62                 fi
  63         fi
  64         if [ -r "sav/$pattern.fail" ]; then
  65                 diff -u "out/$pattern.res" "sav/$pattern.fail" > "out/$pattern.diff.fail.log"
  66                 if [ "$?" == 0 ]; then
  67                         FAIL=OK
  68                 else
  69                         FAIL=NOK
  70                 fi
  71                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  72                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.fail" > "out/$pattern.fail2"
  73                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  74                 grep '[Ee]rror' "sav/$pattern.fail" >/dev/null && echo "Error" >> "out/$pattern.fail2"
  75                 diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
  76                 if [ "$?" == 0 ]; then
  77                         SOSOF=OK
  78                 else
  79                         SOSOF=NOK
  80                 fi
  81         fi
  82         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
  83         NYI="$?"
  84         if [ "x$SAV" = "xOK" ]; then
  85                 if [ -n "$tap" ]; then
  86                         echo "ok - $description"
  87                 elif [ "x$FAIL" = "x" ]; then
  88                         echo "[ok] out/$pattern.res"
  89                 else
  90                         echo "[ok] out/$pattern.res - but sav/$pattern.fail remains!"
  91                 fi
  92                 ok="$ok $pattern"
  93         elif [ "x$FAIL" = "xOK" ]; then
  94                 if [ -n "$tap" ]; then
  95                         echo "not ok - $description # TODO expected failure"
  96                 else
  97                         echo "[fail] out/$pattern.res"
  98                 fi
  99                 ok="$ok $pattern"
 100         elif [ "x$SOSO" = "xOK" ]; then
 101                 if [ -n "$tap" ]; then
 102                         echo "ok - $description # SOSO"
 103                 else
 104                         echo "[soso] out/$pattern.res sav/$pattern.sav"
 105                 fi
 106                 ok="$ok $pattern"
 107         elif [ "x$NYI" = "x0" ]; then
 108                 if [ -n "$tap" ]; then
 109                         echo "not ok - $description # TODO not yet implemented"
 110                 else
 111                         echo "[todo] out/$pattern.res -> not yet implemented"
 112                 fi
 113                 ok="$ok $pattern"
 114         elif [ "x$SOSOF" = "xOK" ]; then
 115                 if [ -n "$tap" ]; then
 116                         echo "not ok - $description # TODO SOSO expected failure"
 117                 else
 118                         echo "[fail soso] out/$pattern.res sav/$pattern.fail"
 119                 fi
 120                 ok="$ok $pattern"
 121         elif [ "x$SAV" = "xNOK" ]; then
 122                 if [ -n "$tap" ]; then
 123                         echo "not ok - $description"
 124                 else
 125                         echo "[======= fail out/$pattern.res sav/$pattern.sav =======]"
 126                 fi
 127                 nok="$nok $pattern"
 128                 echo "$ii" >> "$ERRLIST"
 129         elif [ "x$FAIL" = "xNOK" ]; then
 130                 if [ -n "$tap" ]; then
 131                         echo "not ok - $description"
 132                 else
 133                         echo "[======= changed out/$pattern.res sav/$pattern.fail ======]"
 134                 fi
 135                 nok="$nok $pattern"
 136                 echo "$ii" >> "$ERRLIST"
 137         else
 138                 if [ -n "$tap" ]; then
 139                         echo "ok - $description # skip no sav"
 140                 else
 141                         echo "[=== no sav ===] out/$pattern.res"
 142                 fi
 143                 nos="$nos $pattern"
 144         fi
 145 }
 146
 147 find_nitc()
 148 {
 149         ((tapcount=tapcount+1))
 150         recent=`ls -t ../src/nitc ../src/nitc_[0-9] ../bin/nitc ../c_src/nitc 2>/dev/null | head -1`
 151         if [[ "x$recent" == "x" ]]; then
 152                 if [ -n "$tap" ]; then
 153                         echo "not ok - find nitc"
 154                         echo "Bail out! Could not find nitc, aborting"
 155                 else
 156                         echo 'Could not find nitc, aborting'
 157                 fi
 158                 exit 1
 159         fi
 160         if [ -n "$tap" ]; then
 161                 echo "ok - find nitc: $recent"
 162         else
 163                 echo 'Using nitc from: '$recent
 164         fi
 165         NITC=$recent
 166 }
 167
 168 verbose=false
 169 stop=false
 170 tapcount=0
 171 while [ $stop = false ]; do
 172         case $1 in
 173                 -o) OPT="$OPT $2"; shift; shift;;
 174                 -v) verbose=true; shift;;
 175                 -h) usage; exit;;
 176                 --tap) tap=true; shift;;
 177                 *) stop=true
 178         esac
 179 done
 180
 181 # The default nitc compiler
 182 [ -z "$NITC" ] && find_nitc
 183
 184 # Set NIT_DIR if needed
 185 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 186
 187
 188 # Mark to distinguish files among tests
 189 # MARK=
 190
 191 # File where error tests are outputed
 192 # Old ERRLIST is backuped
 193 ERRLIST=${ERRLIST:-errlist}
 194 ERRLIST_TARGET=$ERRLIST
 195
 196 if [ $# = 0 ]; then
 197         usage;
 198         exit
 199 fi
 200
 201 # Initiate new ERRLIST
 202 if [ "x$ERRLIST" = "x" ]; then
 203         ERRLIST=/dev=null
 204 else
 205         ERRLIST=$ERRLIST.tmp
 206         > "$ERRLIST"
 207 fi
 208
 209 ok=""
 210 nok=""
 211
 212 # CLEAN the out directory
 213 rm -rf out/ 2>/dev/null
 214 mkdir out 2>/dev/null
 215
 216 for ii in "$@"; do
 217         if [ ! -f $ii ]; then
 218                 echo "File '$ii' does not exist."
 219                 continue
 220         fi
 221
 222         tmp=${ii/../AA}
 223         if [ "x$tmp" = "x$ii" ]; then
 224                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 225         else
 226                 includes="-I alt"
 227         fi
 228
 229         f=`basename "$ii" .nit`
 230         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 231                 bf=`basename $i .nit`
 232                 ff="out/$bf"
 233                 test -z "$tap" && echo -n "=> $bf: "
 234
 235                 if [ -f "$f.inputs" ]; then
 236                         inputs="$f.inputs"
 237                 else
 238                         inputs=/dev/null
 239                 fi
 240
 241                 # Compile
 242                 if [ "x$verbose" = "xtrue" ]; then
 243                         echo ""
 244                         echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 245                 fi
 246                 $NITC --no-color $OPT -o "$ff.bin" "$i" $includes <"$inputs" 2> "$ff.cmp.err" > "$ff.compile.log"
 247                 ERR=$?
 248                 if [ "x$verbose" = "xtrue" ]; then
 249                         cat "$ff.compile.log"
 250                         cat >&2 "$ff.cmp.err"
 251                 fi
 252                 egrep '^[A-Z0-9_]*$' "$ff.compile.log" > "$ff.res"
 253                 if [ "$ERR" != 0 ]; then
 254                         test -z "$tap" && echo -n "! "
 255                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 256                         process_result $bf $bf
 257                 elif [ -x "./$ff.bin" ]; then
 258                         cp "$ff.cmp.err" "$ff.res"
 259                         test -z "$tap" && echo -n ". "
 260                         # Execute
 261                         args=""
 262                         if [ "x$verbose" = "xtrue" ]; then
 263                                 echo ""
 264                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 265                         fi
 266                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 267                         if [ "x$verbose" = "xtrue" ]; then
 268                                 cat "$ff.res"
 269                                 cat >&2 "$ff.err"
 270                         fi
 271                         if [ -f "$ff.write" ]; then
 272                                 cat "$ff.write" >> "$ff.res"
 273                         elif [ -d "$ff.write" ]; then
 274                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 275                         fi
 276                         if [ -s "$ff.err" ]; then
 277                                 cat "$ff.err" >> "$ff.res"
 278                         fi
 279                         process_result $bf $bf
 280
 281                         if [ -f "$f.args" ]; then
 282                                 fargs=$f.args
 283                                 cptr=0
 284                                 while read line; do
 285                                         ((cptr=cptr+1))
 286                                         args="$line"
 287                                         bff=$bf"_args"$cptr
 288                                         fff=$ff"_args"$cptr
 289                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 290                                         if [ "x$verbose" = "xtrue" ]; then
 291                                                 echo ""
 292                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 293                                         fi
 294                                         test -z "$tap" && echo -n "==> args #"$cptr " "
 295                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 296                                         if [ "x$verbose" = "xtrue" ]; then
 297                                                 cat "$fff.res"
 298                                                 cat >&2 "$fff.err"
 299                                         fi
 300                                         if [ -f "$fff.write" ]; then
 301                                                 cat "$fff.write" >> "$fff.res"
 302                                         elif [ -d "$fff.write" ]; then
 303                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 304                                         fi
 305                                         if [ -s "$fff.err" ]; then
 306                                                 cat "$fff.err" >> "$fff.res"
 307                                         fi
 308                                         process_result $bff "  args #$cptr"
 309                                 done < $fargs
 310                         fi
 311                 else
 312                         test -z "$tap" && echo -n "! "
 313                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 314                         #echo "Compilation error" > "$ff.res"
 315                         process_result $bf "$bf"
 316                 fi
 317         done
 318 done
 319
 320 if [ -n "$tap" ]; then
 321         echo "1..$tapcount"
 322         echo "# ok:" `echo $ok | wc -w`
 323         echo "# not ok:" `echo $nok | wc -w`
 324         echo "# no sav:" `echo $nos | wc -w`
 325         exit
 326 fi
 327
 328 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w`
 329
 330 if [ -n "$nok" ]; then
 331         echo "fail: $nok"
 332         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 333 fi
 334 if [ -n "$nos" ]; then
 335         echo "no sav: $nos"
 336 fi
 337
 338 # write $ERRLIST
 339 if [ "x$ERRLIST" != "x" ]; then
 340         if [ -x "$ERRLIST_TARGET" ]; then
 341                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 342         fi
 343         mv $ERRLIST $ERRLIST_TARGET
 344 fi
 345
 346 if [ -n "$nok" ]; then
 347         exit 1
 348 else
 349         exit 0
 350 fi