tests/tests.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Copyright 2004-2008 Jean Privat <jean@pryen.org>
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 # This shell script compile, run and verify Nit program files
  19
  20 # Set lang do default to avoid failed tests because of locale
  21 export LANG=C
  22
  23 usage()
  24 {
  25         e=`basename "$0"`
  26         cat<<END
  27 Usage: $e [options] modulenames
  28 -o option   Pass option to nitc
  29 -v          Verbose (show tests steps)
  30 -h          This help
  31 END
  32 }
  33
  34 # As argument: the pattern used for the file
  35 function process_result()
  36 {
  37         # Result
  38         pattern=$1
  39         SAV=""
  40         FAIL=""
  41         SOSO=""
  42         SOSOF=""
  43         if [ -r "sav/$pattern.sav" ]; then
  44                 diff -u "out/$pattern.res" "sav/$pattern.sav" > "out/$pattern.diff.sav.log"
  45                 if [ "$?" == 0 ]; then
  46                         SAV=OK
  47                 else
  48                         SAV=NOK
  49                 fi
  50                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  51                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.sav" > "out/$pattern.sav2"
  52                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  53                 grep '[Ee]rror' "sav/$pattern.sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
  54                 diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
  55                 if [ "$?" == 0 ]; then
  56                         SOSO=OK
  57                 else
  58                         SOSO=NOK
  59                 fi
  60         fi
  61         if [ -r "sav/$pattern.fail" ]; then
  62                 diff -u "out/$pattern.res" "sav/$pattern.fail" > "out/$pattern.diff.fail.log"
  63                 if [ "$?" == 0 ]; then
  64                         FAIL=OK
  65                 else
  66                         FAIL=NOK
  67                 fi
  68                 sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
  69                 sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.fail" > "out/$pattern.fail2"
  70                 grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
  71                 grep '[Ee]rror' "sav/$pattern.fail" >/dev/null && echo "Error" >> "out/$pattern.fail2"
  72                 diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
  73                 if [ "$?" == 0 ]; then
  74                         SOSOF=OK
  75                 else
  76                         SOSOF=NOK
  77                 fi
  78         fi
  79         grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
  80         NYI="$?"
  81         if [ "x$SAV" = "xOK" ]; then
  82                 if [ "x$FAIL" = "x" ]; then
  83                         echo "[ok] out/$pattern.res"
  84                 else
  85                         echo "[ok] out/$pattern.res - but sav/$pattern.fail remains!"
  86                 fi
  87                 ok="$ok $pattern"
  88         elif [ "x$FAIL" = "xOK" ]; then
  89                 echo "[fail] out/$pattern.res"
  90                 ok="$ok $pattern"
  91         elif [ "x$SOSO" = "xOK" ]; then
  92                 echo "[soso] out/$pattern.res sav/$pattern.sav"
  93                 ok="$ok $pattern"
  94         elif [ "x$NYI" = "x0" ]; then
  95                 echo "[todo] out/$pattern.res -> not yet implemented"
  96                 ok="$ok $pattern"
  97         elif [ "x$SOSOF" = "xOK" ]; then
  98                 echo "[fail soso] out/$pattern.res sav/$pattern.fail"
  99                 ok="$ok $pattern"
 100         elif [ "x$SAV" = "xNOK" ]; then
 101                 echo "[======= fail out/$pattern.res sav/$pattern.sav =======]"
 102                 nok="$nok $pattern"
 103                 echo "$ii" >> "$ERRLIST"
 104         elif [ "x$FAIL" = "xNOK" ]; then
 105                 echo "[======= changed out/$pattern.res sav/$pattern.fail ======]"
 106                 nok="$nok $pattern"
 107                 echo "$ii" >> "$ERRLIST"
 108         else
 109                 echo "[=== no sav ===] out/$pattern.res"
 110                 nos="$nos $pattern"
 111         fi
 112 }
 113
 114 find_nitc()
 115 {
 116         recent=`ls -t ../src/nitc ../src/nitc_[0-9] ../bin/nitc ../c_src/nitc 2>/dev/null | head -1`
 117         if [[ "x$recent" == "x" ]]; then
 118                 echo 'Could not find nitc, aborting'
 119                 exit 1
 120         fi
 121         echo 'Using nitc from: '$recent
 122         NITC=$recent
 123 }
 124
 125 # The default nitc compiler
 126 [ -z "$NITC" ] && find_nitc
 127
 128 # Set NIT_DIR if needed
 129 [ -z "$NIT_DIR" ] && export NIT_DIR=..
 130
 131 verbose=false
 132 stop=false
 133 while [ $stop = false ]; do
 134         case $1 in
 135                 -o) OPT="$OPT $2"; shift; shift;;
 136                 -v) verbose=true; shift;;
 137                 -h) usage; exit;;
 138                 *) stop=true
 139         esac
 140 done
 141
 142 # Mark to distinguish files among tests
 143 # MARK=
 144
 145 # File where error tests are outputed
 146 # Old ERRLIST is backuped
 147 ERRLIST=${ERRLIST:-errlist}
 148 ERRLIST_TARGET=$ERRLIST
 149
 150 if [ $# = 0 ]; then
 151         usage;
 152         exit
 153 fi
 154
 155 # Initiate new ERRLIST
 156 if [ "x$ERRLIST" = "x" ]; then
 157         ERRLIST=/dev=null
 158 else
 159         ERRLIST=$ERRLIST.tmp
 160         > "$ERRLIST"
 161 fi
 162
 163 ok=""
 164 nok=""
 165
 166 # CLEAN the out directory
 167 rm -rf out/ 2>/dev/null
 168 mkdir out 2>/dev/null
 169
 170 for ii in "$@"; do
 171         if [ ! -f $ii ]; then
 172                 echo "File '$ii' does not exist."
 173                 continue
 174         fi
 175
 176         tmp=${ii/../AA}
 177         if [ "x$tmp" = "x$ii" ]; then
 178                 includes="-I . -I ../lib/standard -I ../lib/standard/collection -I alt"
 179         else
 180                 includes="-I alt"
 181         fi
 182
 183         f=`basename "$ii" .nit`
 184         for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 185                 bf=`basename $i .nit`
 186                 ff="out/$bf"
 187                 echo -n "=> $bf: "
 188
 189                 if [ -f "$f.inputs" ]; then
 190                         inputs="$f.inputs"
 191                 else
 192                         inputs=/dev/null
 193                 fi
 194
 195                 # Compile
 196                 if [ "x$verbose" = "xtrue" ]; then
 197                         echo ""
 198                         echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
 199                 fi
 200                 $NITC --no-color $OPT -o "$ff.bin" "$i" $includes <"$inputs" 2> "$ff.cmp.err" > "$ff.compile.log"
 201                 ERR=$?
 202                 if [ "x$verbose" = "xtrue" ]; then
 203                         cat "$ff.compile.log"
 204                         cat >&2 "$ff.cmp.err"
 205                 fi
 206                 egrep '^[A-Z0-9_]*$' "$ff.compile.log" > "$ff.res"
 207                 if [ "$ERR" != 0 ]; then
 208                         echo -n "! "
 209                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 210                         process_result $bf
 211                 elif [ -x "./$ff.bin" ]; then
 212                         cp "$ff.cmp.err" "$ff.res"
 213                         echo -n ". "
 214                         # Execute
 215                         args=""
 216                         if [ "x$verbose" = "xtrue" ]; then
 217                                 echo ""
 218                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 219                         fi
 220                         NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
 221                         if [ "x$verbose" = "xtrue" ]; then
 222                                 cat "$ff.res"
 223                                 cat >&2 "$ff.err"
 224                         fi
 225                         if [ -f "$ff.write" ]; then
 226                                 cat "$ff.write" >> "$ff.res"
 227                         elif [ -d "$ff.write" ]; then
 228                                 LANG=C /bin/ls -F $ff.write >> "$ff.res"
 229                         fi
 230                         if [ -s "$ff.err" ]; then
 231                                 cat "$ff.err" >> "$ff.res"
 232                         fi
 233                         process_result $bf
 234
 235                         if [ -f "$f.args" ]; then
 236                                 fargs=$f.args
 237                                 cptr=0
 238                                 while read line; do
 239                                         ((cptr=cptr+1))
 240                                         args="$line"
 241                                         bff=$bf"_args"$cptr
 242                                         fff=$ff"_args"$cptr
 243                                         rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 244                                         if [ "x$verbose" = "xtrue" ]; then
 245                                                 echo ""
 246                                                 echo "NIT_NO_STACK=1 ./$ff.bin" $args
 247                                         fi
 248                                         echo -n "==> args #"$cptr " "
 249                                         sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
 250                                         if [ "x$verbose" = "xtrue" ]; then
 251                                                 cat "$fff.res"
 252                                                 cat >&2 "$fff.err"
 253                                         fi
 254                                         if [ -f "$fff.write" ]; then
 255                                                 cat "$fff.write" >> "$fff.res"
 256                                         elif [ -d "$fff.write" ]; then
 257                                                 LANG=C /bin/ls -F $fff.write >> "$fff.res"
 258                                         fi
 259                                         if [ -s "$fff.err" ]; then
 260                                                 cat "$fff.err" >> "$fff.res"
 261                                         fi
 262                                         process_result $bff
 263                                 done < $fargs
 264                         fi
 265                 else
 266                         echo -n "! "
 267                         cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
 268                         #echo "Compilation error" > "$ff.res"
 269                         process_result $bf
 270                 fi
 271         done
 272 done
 273
 274 echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w`
 275
 276 if [ -n "$nok" ]; then
 277         echo "fail: $nok"
 278         echo "There were $(echo $nok | wc -w) errors ! (see file $ERRLIST)"
 279 fi
 280 if [ -n "$nos" ]; then
 281         echo "no sav: $nos"
 282 fi
 283
 284 # write $ERRLIST
 285 if [ "x$ERRLIST" != "x" ]; then
 286         if [ -x "$ERRLIST_TARGET" ]; then
 287                 mv "$ERRLIST_TARGET" "${ERRLIST_TARGET}.bak"
 288         fi
 289         mv $ERRLIST $ERRLIST_TARGET
 290 fi
 291
 292 if [ -n "$nok" ]; then
 293         exit 1
 294 else
 295         exit 0
 296 fi