benchmarks/bench_languages.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 ### HELPER FUNCTIONS ##
  29
  30 function die()
  31 {
  32         echo >&2 "error: $*"
  33         died=1
  34 }
  35
  36 # Run a single command multiple time and store the execution times
  37 # in the current $res file.
  38 #
  39 #  $1: title of the command
  40 #  $2: long desription of the command
  41 #  rest: the command to execute
  42 function bench_command()
  43 {
  44         if [ "$dry_run" = "true" ]; then return; fi
  45         local title="$1"
  46         local desc="$2"
  47         shift
  48         shift
  49         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  50         timeout="time.out"
  51         echo "$title" > "$timeout"
  52         echo "# $desc" >> "$timeout"
  53         echo "\$ $@" >> "$timeout"
  54         echo
  55         echo "** [$title] $desc **"
  56         echo " $ $@"
  57
  58         # Execute the commands $count times
  59         for i in `seq 1 "$count"`; do
  60                 (ulimit -t 60; /usr/bin/time -f "%U" -o "$timeout" -a "$@") > $outputopts 2>&1 || die "$1: failed"
  61                 echo -n "$i. "
  62                 tail -n 1 "$timeout"
  63         done
  64
  65         line=`compute_stats "$timeout"`
  66         echo "$line ($res)"
  67         echo $line >> "$res"
  68         rm $timeout
  69 }
  70
  71 # Run a simple command witout storing the execution time
  72 # Used to display command on verbose and skip long executions when dry_run is given
  73 # $@ command to execute
  74 function run_command()
  75 {
  76         if [ "$dry_run" = "true" ]; then return; fi
  77         echo " $ $@"
  78         (ulimit -t 180; "$@") || die "$@: failed"
  79 }
  80
  81 # Check if the test should be skiped according to its name
  82 # $1: name of the test
  83 # $2: description of the test
  84 # $NOTSKIPED: arguments
  85 function skip_test()
  86 {
  87         if test -z "$NOTSKIPED"; then
  88                 echo "* $1"
  89                 return 0
  90         fi
  91         if test "$NOTSKIPED" = "all"; then
  92                 : # Execute anyway
  93         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  94                 : # Found one to execute
  95         else
  96                 return 0
  97         fi
  98         echo "*"
  99         echo "* $1 *****"
 100         echo "*"
 101         return 1
 102 }
 103
 104 ## HANDLE OPTIONS ##
 105
 106 function usage()
 107 {
 108         echo "run_bench: [options]* benchname"
 109         echo "  -v: verbose mode"
 110         echo "  -n count: number of execution for each bar (default: $count)"
 111         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 112         echo "  --fast: Run less and faster tests"
 113         echo "  -h: this help"
 114 }
 115
 116 stop=false
 117 while [ "$stop" = false ]; do
 118         case "$1" in
 119                 -v) verbose=true; shift;;
 120                 -h) usage; exit;;
 121                 -n) count="$2"; shift; shift;;
 122                 --dry) dry_run=true; shift;;
 123                 --fast) fast=true; shift;;
 124                 *) stop=true
 125         esac
 126 done
 127
 128 NOTSKIPED="$*"
 129
 130 if test -z "$NOTSKIPED"; then
 131         usage
 132         echo "List of available benches:"
 133         echo "* all: run all the benches"
 134 fi
 135
 136 ## COMPILE ENGINES
 137 cd ../src
 138 test -f ./nitc_3 || ./ncall.sh -O
 139 cd ../benchmarks
 140 test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v
 141
 142 ## EFFECTIVE BENCHS ##
 143
 144 function bench_typetest_languages()
 145 {
 146         name="$FUNCNAME"
 147         skip_test "$name" && return
 148
 149         t=t
 150         s=20
 151         seq="w2_h2 w50_h2 w2_h25 w50_h25"
 152         for b in $seq; do
 153                 run_command ./nitg languages/gen.nit
 154                 run_command ./gen.bin "${t}_$b" "$b"
 155         done
 156
 157         prepare_res "$name-g++.dat" "g++" "g++"
 158         for b in $seq; do
 159                 run_command g++ "${t}_$b.cpp" -O2 -o "${t}_$b.g++.bin"
 160                 bench_command "$b" "" "./${t}_$b.g++.bin" $s
 161         done
 162
 163         prepare_res "$name-clang++.dat" "clang++" "clang++"
 164         for b in $seq; do
 165                 run_command clang++ "${t}_$b.cpp" -O2 -o "${t}_$b.clang++.bin"
 166                 bench_command "$b" "" "./${t}_$b.clang++.bin" $s
 167         done
 168
 169         prepare_res "$name-java.dat" "java" "java"
 170         for b in $seq; do
 171                 run_command javac ${t}_$b.java
 172                 bench_command "$b" "" java "${t}_$b" $s
 173         done
 174
 175         prepare_res "$name-scala.dat" "scala" "scala"
 176         for b in $seq; do
 177                 run_command scalac ${t}_$b.scala
 178                 bench_command "$b" "" scala "${t}_$b" $s
 179         done
 180
 181         prepare_res "$name-cs.dat" "c#" "c#"
 182         for b in $seq; do
 183                 run_command gmcs ${t}_$b.cs
 184                 bench_command "$b" "" mono "${t}_$b.exe" $s
 185         done
 186
 187         prepare_res "$name-es.dat" "es" "es"
 188         for b in $seq; do
 189                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 190                 chmod +x app${t}_$b
 191                 mv app${t}_$b  ${t}_$b.es.bin
 192                 bench_command "$b" "" "./${t}_$b.es.bin" $s
 193         done
 194
 195         prepare_res "$name-se.dat" "se" "se"
 196         for b in $seq; do
 197                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 198                 bench_command "$b" "" "./${t}_$b.se.bin" $s
 199         done
 200
 201         #too slow
 202         #prepare_res "$name-nitg.dat" "nitg" "nitg"
 203         #for b in $seq; do
 204         #       run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 205         #       bench_command "$b" "" "./${t}_$b.nitg.bin" $s
 206         #done
 207
 208         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg-s"
 209         for b in $seq; do
 210                 run_command ./nitg ${t}_$b.nit --separate -o "${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 211                 bench_command "$b" "" "./${t}_$b.nitg-s.bin" $s
 212         done
 213
 214         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg-su"
 215         for b in $seq; do
 216                 run_command ./nitg ${t}_$b.nit --separate --no-check-covariance -o "${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 217                 bench_command "$b" "" "./${t}_$b.nitg-su.bin" $s
 218         done
 219
 220
 221         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg-e"
 222         for b in $seq; do
 223                 run_command ./nitg ${t}_$b.nit --erasure -o "${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 224                 bench_command "$b" "" "./${t}_$b.nitg-e.bin" $s
 225         done
 226
 227         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
 228         for b in $seq; do
 229                 run_command ./nitg ${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 230                 bench_command "$b" "" "./${t}_$b.nitg-eu.bin" $s
 231         done
 232
 233         plot "$name.gnu"
 234 }
 235 bench_typetest_languages
 236
 237 function bench_typetest_depth()
 238 {
 239         name="$FUNCNAME"
 240         skip_test "$name" && return
 241         rootdir=`pwd`
 242         basedir="./${name}.out"
 243
 244         mkdir $basedir
 245
 246         t=t
 247         s=20
 248         seq="10 25 50 100"
 249         seq="2 4 8 16"
 250         for b in $seq; do
 251                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 252                 run_command $basedir/$name.bin $basedir "${t}_$b" "$b"
 253         done
 254
 255         prepare_res $basedir/$name-g++.dat "g++" "g++"
 256         cppdir="${basedir}/cpp"
 257         for b in $seq; do
 258                 run_command g++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.g++.bin"
 259                 bench_command "$b" "" "${cppdir}/${t}_$b.g++.bin" $s
 260         done
 261
 262         prepare_res $basedir/$name-clang++.dat "clang++" "clang++"
 263         for b in $seq; do
 264                 run_command clang++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.clang++.bin"
 265                 bench_command "$b" "" "${cppdir}/${t}_$b.clang++.bin" $s
 266         done
 267
 268         prepare_res $basedir/$name-java.dat "java" "java"
 269         javadir="${basedir}/java"
 270         for b in $seq; do
 271                 run_command javac "${javadir}/${t}_$b.java"
 272                 bench_command "$b" "" java -cp "${javadir}/" "${t}_$b" $s
 273         done
 274
 275         prepare_res $basedir/$name-gcj.dat "gcj" "gcj"
 276         for b in $seq; do
 277                 run_command gcj --main=${t}_$b -O2 "${javadir}/${t}_$b.java" -o "${javadir}/${t}_$b.gcj.bin"
 278                 bench_command "$b" "" "${javadir}/${t}_$b.gcj.bin" $s
 279         done
 280
 281         prepare_res $basedir/$name-scala.dat "scala" "scala"
 282         scaladir="${basedir}/scala"
 283         for b in $seq; do
 284                 run_command scalac "${scaladir}/${t}_$b.scala" -d "${scaladir}"
 285                 bench_command "$b" "" scala -cp "${scaladir}/" "${t}_$b" $s
 286         done
 287
 288         prepare_res $basedir/$name-cs.dat "c#" "c#"
 289         csdir="${basedir}/cs"
 290         for b in $seq; do
 291                 run_command gmcs "$csdir/${t}_$b.cs"
 292                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 293         done
 294
 295         prepare_res $basedir/$name-es.dat "es" "es"
 296         esdir="${basedir}/es"
 297         for b in $seq; do
 298                 cd $esdir
 299                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 300                 chmod +x app${t}_$b
 301                 mv app${t}_$b ${t}_$b.es.bin
 302                 cd $rootdir
 303                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 304         done
 305
 306         prepare_res $basedir/$name-se.dat "se" "se"
 307         sedir="${basedir}/se"
 308         for b in $seq; do
 309                 cd $sedir
 310                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 311                 cd $rootdir
 312                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 313         done
 314
 315         nitdir="${basedir}/nit"
 316         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 317         for b in $seq; do
 318                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 319                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 320         done
 321
 322         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 323         for b in $seq; do
 324                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 325                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 326         done
 327
 328         tg="nitg-s-bm"
 329         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 330         for b in $seq; do
 331                 run_command ./nitg $nitdir/${t}_$b.nit --separate --bm-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 332                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 333         done
 334
 335         tg="nitg-s-pha"
 336         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 337         for b in $seq; do
 338                 run_command ./nitg $nitdir/${t}_$b.nit --separate --phand-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 339                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 340         done
 341
 342         tg="nitg-s-phm"
 343         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 344         for b in $seq; do
 345                 run_command ./nitg $nitdir/${t}_$b.nit --separate --phmod-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 346                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 347         done
 348
 349         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 350         for b in $seq; do
 351                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 352                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 353         done
 354
 355         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 356         for b in $seq; do
 357                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 358                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 359         done
 360
 361         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 362         for b in $seq; do
 363                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 364                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 365         done
 366
 367         plot $basedir/$name.gnu
 368 }
 369 bench_typetest_depth
 370
 371 function bench_typetest_fts_depth()
 372 {
 373         name="$FUNCNAME"
 374         skip_test "$name" && return
 375         rootdir=`pwd`
 376         basedir="./${name}.out"
 377
 378         mkdir $basedir
 379
 380         t=t
 381         s=20
 382         seq="10 25 50 100"
 383         for b in $seq; do
 384                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 385                 run_command $basedir/$name.bin $basedir "${t}_$b" "$b"
 386         done
 387
 388         prepare_res $basedir/$name-cs.dat "c#" "c#"
 389         csdir="${basedir}/cs"
 390         for b in $seq; do
 391                 run_command gmcs "$csdir/${t}_$b.cs"
 392                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 393         done
 394
 395         prepare_res $basedir/$name-es.dat "es" "es"
 396         esdir="${basedir}/es"
 397         for b in $seq; do
 398                 cd $esdir
 399                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 400                 chmod +x app${t}_$b
 401                 mv app${t}_$b ${t}_$b.es.bin
 402                 cd $rootdir
 403                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 404         done
 405
 406         prepare_res $basedir/$name-se.dat "se" "se"
 407         sedir="${basedir}/se"
 408         for b in $seq; do
 409                 cd $sedir
 410                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 411                 cd $rootdir
 412                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 413         done
 414
 415         nitdir="${basedir}/nit"
 416         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 417         for b in $seq; do
 418                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 419                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 420         done
 421
 422         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 423         for b in $seq; do
 424                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 425                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 426         done
 427
 428         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 429         for b in $seq; do
 430                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 431                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 432         done
 433
 434         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 435         for b in $seq; do
 436                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 437                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 438         done
 439
 440         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 441         for b in $seq; do
 442                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 443                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 444         done
 445
 446         plot $basedir/$name.gnu
 447 }
 448 bench_typetest_fts_depth
 449
 450 function bench_typetest_fts_width()
 451 {
 452         name="$FUNCNAME"
 453         skip_test "$name" && return
 454         rootdir=`pwd`
 455         basedir="./${name}.out"
 456
 457         mkdir $basedir
 458
 459         t=t
 460         s=20
 461         depth=10
 462         seq="1 2 5 10"
 463         for b in $seq; do
 464                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 465                 run_command $basedir/$name.bin $basedir "${t}_$b" $depth $b
 466         done
 467
 468         prepare_res $basedir/$name-cs.dat "c#" "c#"
 469         csdir="${basedir}/cs"
 470         for b in $seq; do
 471                 run_command gmcs "$csdir/${t}_$b.cs"
 472                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 473         done
 474
 475         prepare_res $basedir/$name-es.dat "es" "es"
 476         esdir="${basedir}/es"
 477         for b in $seq; do
 478                 cd $esdir
 479                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 480                 chmod +x app${t}_$b
 481                 mv app${t}_$b ${t}_$b.es.bin
 482                 cd $rootdir
 483                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 484         done
 485
 486         prepare_res $basedir/$name-se.dat "se" "se"
 487         sedir="${basedir}/se"
 488         for b in $seq; do
 489                 cd $sedir
 490                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 491                 cd $rootdir
 492                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 493         done
 494
 495         nitdir="${basedir}/nit"
 496
 497         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 498         for b in $seq; do
 499                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 500                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 501         done
 502
 503         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 504         for b in $seq; do
 505                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 506                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 507         done
 508
 509         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 510         for b in $seq; do
 511                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 512                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 513         done
 514
 515         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 516         for b in $seq; do
 517                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 518                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 519         done
 520
 521         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 522         for b in $seq; do
 523                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 524                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 525         done
 526
 527         plot $basedir/$name.gnu
 528 }
 529 bench_typetest_fts_width
 530
 531 function bench_typetest_fts_nesting()
 532 {
 533         name="$FUNCNAME"
 534         skip_test "$name" && return
 535         rootdir=`pwd`
 536         basedir="./${name}.out"
 537
 538         mkdir $basedir
 539
 540         t=t
 541         s=20
 542         depth=5
 543         seq="1 2 5 10"
 544         for b in $seq; do
 545                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 546                 run_command $basedir/$name.bin $basedir "${t}_$b" $depth $b
 547         done
 548
 549         prepare_res $basedir/$name-cs.dat "c#" "c#"
 550         csdir="${basedir}/cs"
 551         for b in $seq; do
 552                 run_command gmcs "$csdir/${t}_$b.cs"
 553                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 554         done
 555
 556         prepare_res $basedir/$name-es.dat "es" "es"
 557         esdir="${basedir}/es"
 558         for b in $seq; do
 559                 cd $esdir
 560                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 561                 chmod +x app${t}_$b
 562                 mv app${t}_$b ${t}_$b.es.bin
 563                 cd $rootdir
 564                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 565         done
 566
 567         prepare_res $basedir/$name-se.dat "se" "se"
 568         sedir="${basedir}/se"
 569         for b in $seq; do
 570                 cd $sedir
 571                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 572                 cd $rootdir
 573                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 574         done
 575
 576         nitdir="${basedir}/nit"
 577
 578         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 579         for b in $seq; do
 580                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 581                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 582         done
 583
 584         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 585         for b in $seq; do
 586                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 587                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 588         done
 589
 590         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 591         for b in $seq; do
 592                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 593                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 594         done
 595
 596         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 597         for b in $seq; do
 598                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 599                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 600         done
 601
 602         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 603         for b in $seq; do
 604                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 605                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 606         done
 607
 608         plot $basedir/$name.gnu
 609 }
 610 bench_typetest_fts_nesting
 611
 612 if test -n "$died"; then
 613         echo "Some commands failed"
 614         exit 1
 615 fi
 616 exit 0