src/run_bench.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 ## CONFIGURATION OPTIONS ##
  21
  22 # Default number of times a command must be run with bench_command
  23 # Can be overrided with 'the option -n'
  24 count=2
  25
  26 ### HELPER FUNCTIONS ##
  27
  28 function die()
  29 {
  30         echo >&2 "error: $*"
  31         died=1
  32 }
  33
  34 # Run a single command multiple time and store the execution times
  35 # in the current $res file.
  36 #
  37 #  $1: title of the command
  38 #  $2: long desription of the command
  39 #  rest: the command to execute
  40 function bench_command()
  41 {
  42         if [ "$dry_run" = "true" ]; then return; fi
  43         local title="$1"
  44         local desc="$2"
  45         shift
  46         shift
  47         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  48         timeout="time.out"
  49         echo "$title" > "$timeout"
  50         echo "# $desc" >> "$timeout"
  51         echo "\$ $@" >> "$timeout"
  52         echo
  53         echo "** [$title] $desc **"
  54         echo " $ $@"
  55
  56         # Execute the commands $count times
  57         for i in `seq 1 "$count"`; do
  58                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  59                 echo -n "$i. "
  60                 tail -n 1 "$timeout"
  61         done
  62
  63         line=`compute_stats "$timeout"`
  64         echo "$line ($res)"
  65         echo $line >> "$res"
  66 }
  67
  68 # Run a simble command witout storing the execution time
  69 # Used to display command on verbose and skip long executions when dry_run is given
  70 # $@ command to execute
  71 function run_command()
  72 {
  73         if [ "$dry_run" = "true" ]; then return; fi
  74         echo " $ $@"
  75         "$@" || die "$@: failed"
  76 }
  77
  78 # perl function to compute min/max/avg.
  79 # used by bench_command
  80 #
  81 # $1: file
  82 # return: first min max avg label
  83 function compute_stats()
  84 {
  85         file="$1"
  86         # Compute statistics
  87         perl - "$file" <<'END'
  88         @lines = ();
  89         $first = undef;
  90         chomp($label = <>);
  91         while(<>) {
  92                 chomp;
  93                 if (/^\d/) {
  94                         if (defined $first) {
  95                                 push @lines, $_;
  96                         } else {
  97                                 $first = $_;
  98                         }
  99                 }
 100         }
 101         $len = scalar @lines;
 102         if ($len) {
 103                 @lines = sort {$a <=> $b} @lines;
 104                 $min = $lines[0];
 105                 $max = $lines[$#lines];
 106                 $avg = 0;
 107                 for $i (@lines) { $avg += $i; }
 108                 $avg = $avg / $len;
 109                 print "${first} ${min} ${max} ${avg} \"${label}\"\n";
 110         } else {
 111                 print "${first} ${first} ${first} ${first} \"${label}\"\n";
 112         }
 113 END
 114 }
 115
 116 # Create a new $res to be used in the next plot
 117 #
 118 # $1 = resfile
 119 # $2 = title
 120 # $3 = description
 121 function prepare_res()
 122 {
 123         echo
 124         echo "# [$2] $3 #"
 125         res=$1
 126         if [ "$plots" = "" ]; then
 127                 plots="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
 128         else
 129                 plots="$plots replot '$1' using 4:2:3 ti '$2';"
 130         fi
 131         if [ "$dry_run" = "true" ]; then return; fi
 132         echo "# [$2] $3 ; count=$count" > "$res"
 133         echo "# first min max avg title" >> "$res"
 134 }
 135
 136 # Plot the last $res as an histogram
 137 # $1: plot file (eg toto.gnu)
 138 function plot()
 139 {
 140         cat >"$1" <<END
 141 set auto x;
 142 set yrange [0:];
 143 set style data histogram;
 144 set style histogram cluster gap 2;
 145 set style histogram errorbars linewidth 1;
 146 set style fill solid 0.3 border -1;
 147 set bars front;
 148 set boxwidth 0.9;
 149 set xtic nomirror rotate by -45 scale 0 font ',8';
 150 set title "$1 ; avg. on $count-1 runs"
 151 set ylabel "time (s)"
 152 $plots
 153 END
 154         echo "# gnuplot -p $1"
 155         gnuplot -p "$1"
 156         plots=
 157 }
 158
 159 # Check if the test should be skiped according to its name
 160 # $1: name of the test
 161 # $2: description of the test
 162 # $NOTSKIPED: arguments
 163 function skip_test()
 164 {
 165         if test -z "$NOTSKIPED"; then
 166                 echo "* $1"
 167                 return 0
 168         fi
 169         if test "$NOTSKIPED" = "all"; then
 170                 : # Execute anyway
 171         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
 172                 : # Found one to execute
 173         else
 174                 return 0
 175         fi
 176         echo "*"
 177         echo "* $1 *****"
 178         echo "*"
 179         return 1
 180 }
 181
 182 ## GLOBAL VARIABLES ##
 183
 184 # The current $res (set by prepare_res, used by bench_command)
 185 res=
 186
 187 # The current stuff to plot (set by prepare_res, used by plot)
 188 plots=
 189
 190 # HELPER FOR NIT #
 191
 192 # Run standards benchs on a compiler command
 193 # $1: title
 194 # rest: command to run (executable + options)
 195 function run_compiler()
 196 {
 197         local title=$1
 198         shift
 199         if test -n "$fast"; then
 200                 run_command "$@" nitg.nit -o "nitg.$title.bin"
 201                 bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v --no-cc test_parser.nit
 202                 run_command "$@" nit.nit -o "nit.$title.bin"
 203                 bench_command "nit" "nit test_parser.nit location.nit" "./nit.$title.bin" -v test_parser.nit -- -n location.nit
 204                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 205                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 206                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 207                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
 208         else
 209                 run_command "$@" nitg.nit -o "nitg.$title.bin"
 210                 bench_command "nitg" "nitg --no-cc nitstats.nit" "./nitg.$title.bin" -v --no-cc nitstats.nit
 211                 bench_command "nitg-s" "nitg --separate nitg.nit" "./nitg.$title.bin" -v --no-cc --separate nitg.nit
 212                 run_command "$@" nit.nit -o "nit.$title.bin"
 213                 bench_command "nit" "nit test_parser.nit rapid_type_analysis.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
 214                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 215                 bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
 216                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 217                 bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
 218         fi
 219 }
 220
 221 ## HANDLE OPTIONS ##
 222
 223 function usage()
 224 {
 225         echo "run_bench: [options]* benchname"
 226         echo "  -v: verbose mode"
 227         echo "  -n count: number of execution for each bar (default: $count)"
 228         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 229         echo "  --fast: Run less and faster tests"
 230         echo "  -h: this help"
 231 }
 232
 233 stop=false
 234 while [ "$stop" = false ]; do
 235         case "$1" in
 236                 -v) verbose=true; shift;;
 237                 -h) usage; exit;;
 238                 -n) count="$2"; shift; shift;;
 239                 --dry) dry_run=true; shift;;
 240                 --fast) fast=true; shift;;
 241                 *) stop=true
 242         esac
 243 done
 244
 245 NOTSKIPED="$*"
 246
 247 if test -z "$NOTSKIPED"; then
 248         usage
 249         echo "List of available benches:"
 250         echo "* all: run all the benches"
 251 fi
 252
 253 ## COMPILE ENGINES
 254
 255 test -f ./nitc_3 || ./ncall.sh -O
 256 test -f ./nitg || ./nitc_3 nitg.nit -O -v
 257
 258 ## EFFECTIVE BENCHS ##
 259
 260 function bench_nitg_bootstrap()
 261 {
 262         name="$FUNCNAME"
 263         skip_test "$name" && return
 264         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 265         rm nit?_nit*
 266         cp ./nitc_3 ./nitc_nitc.bin
 267         bench_command "c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O nitc.nit -o nitc_nitc.bin
 268         bench_command "c/c g" "nitc_nitc nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O nitg.nit -o nitg_nitc.bin
 269         bench_command "g/c g" "nitg_nitc nitg.nit -> nitg_nitg" ./nitg_nitc.bin nitg.nit -o nitg_nitg.bin
 270         bench_command "g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin nitg.nit -o nitg_nitg.bin
 271
 272         plot "$name.gnu"
 273 }
 274 bench_nitg_bootstrap
 275
 276 function bench_steps()
 277 {
 278         name="$FUNCNAME"
 279         skip_test "$name" && return
 280         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 281         bench_command "parse" "" ./nitc_3 --only-parse nitg.nit
 282         bench_command "metamodel" "" ./nitc_3 --only-metamodel nitg.nit
 283         bench_command "generate c" "" ./nitc_3 --no-cc nitg.nit
 284         bench_command "full" "" ./nitc_3 -O nitg.nit -o "nitg_nitg.bin"
 285
 286         prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
 287         bench_command "parse" "" ./nitc_3 --global --only-parse nitg.nit
 288         bench_command "metamodel" "" ./nitc_3 --global --only-metamodel nitg.nit
 289         bench_command "generate c" "" ./nitc_3 --global --no-cc nitg.nit
 290         bench_command "full" "" ./nitc_3 -O --global nitg.nit -o "nitg_nitc-g.bin"
 291
 292         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 293         bench_command "parse" "" ./nitg --only-parse nitg.nit
 294         bench_command "metamodel" "" ./nitg --only-metamodel nitg.nit
 295         bench_command "generate c" "" ./nitg --no-cc nitg.nit
 296         bench_command "full" "" ./nitg nitg.nit -o "nitg_nitg.bin"
 297
 298         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 299         bench_command "parse" "" ./nitg --erasure --only-parse nitg.nit
 300         bench_command "metamodel" "" ./nitg --erasure --only-metamodel nitg.nit
 301         bench_command "generate c" "" ./nitg --erasure --no-cc nitg.nit
 302         bench_command "full" "" ./nitg --erasure nitg.nit -o "nitg_nitg-e.bin"
 303
 304         plot "$name.gnu"
 305 }
 306 bench_steps
 307
 308 # $#: options to compare
 309 function bench_nitg_options()
 310 {
 311         tag=$1
 312         shift
 313         name="$FUNCNAME-$tag"
 314         skip_test "$name" && return
 315         prepare_res "$name.dat" "no options" "nitg without options"
 316         run_compiler "nitg" ./nitg
 317
 318         if test -n "$2"; then
 319                 prepare_res "$name-all.dat" "all" "nitg with all options $@"
 320                 run_compiler "nitg-$tag" ./nitg $@
 321         fi
 322
 323         for opt in "$@"; do
 324                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 325                 run_compiler "nitg$opt" ./nitg $opt
 326         done
 327
 328         plot "$name.gnu"
 329 }
 330 bench_nitg_options "hardening" --hardening
 331 bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 332
 333 function bench_nitg-s_options()
 334 {
 335         tag=$1
 336         shift
 337         name="$FUNCNAME-$tag"
 338         skip_test "$name" && return
 339         prepare_res "$name.dat" "no options" "nitg-s without options"
 340         run_compiler "nitg-s" ./nitg --separate
 341
 342         if test -n "$2"; then
 343                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 344                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 345         fi
 346
 347         for opt in "$@"; do
 348                 prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt"
 349                 run_compiler "nitg-s$opt" ./nitg --separate $opt
 350         done
 351
 352         plot "$name.gnu"
 353 }
 354 bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree --no-union-attribute --no-shortcut-equal --no-shortcut-range
 355 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 356 bench_nitg-s_options "faster" --inline-coloring-numbers
 357 bench_nitg-s_options "typing" --bm-typing --phand-typing --phmod-typing
 358
 359 function bench_nitg-e_options()
 360 {
 361         tag=$1
 362         shift
 363         name="$FUNCNAME-$tag"
 364         skip_test "$name" && return
 365         prepare_res "$name.dat" "no options" "nitg-e without options"
 366         run_compiler "nitg-e" ./nitg --erasure
 367
 368         if test -n "$2"; then
 369                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 370                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 371         fi
 372
 373         for opt in "$@"; do
 374                 prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
 375                 run_compiler "nitg-e$opt" ./nitg --erasure $opt
 376         done
 377
 378         plot "$name.gnu"
 379 }
 380 bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
 381 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 382 bench_nitg-e_options "faster" --inline-coloring-numbers
 383 bench_nitg-e_options "typing" --bm-typing --phand-typing --phmod-typing
 384
 385 function bench_nitc_gc()
 386 {
 387         name="$FUNCNAME"
 388         skip_test "$name" && return
 389         for gc in nitgc boehm malloc large; do
 390                 prepare_res "$name-$gc.dat" "$gc" "nitc with gc=$gc"
 391                 export NIT_GC_OPTION="$gc"
 392                 run_compiler "nitc" ./nitc_3 -O
 393         done
 394
 395         plot "$name.gnu"
 396 }
 397 bench_nitc_gc
 398
 399 function bench_nitc_boost()
 400 {
 401         name="$FUNCNAME"
 402         skip_test "$name" && return
 403         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 404         run_compiler "nitc_slow" ./nitc_3
 405         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 406         run_compiler "nitc" ./nitc_3 -O
 407
 408         plot "$name.gnu"
 409 }
 410 bench_nitc_boost
 411
 412 function bench_engines()
 413 {
 414         name="$FUNCNAME"
 415         skip_test "$name" && return
 416         prepare_res "$name-nitc.dat" "nitc" "nitc"
 417         run_compiler "nitc" ./nitc_3 -O
 418         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 419         run_compiler "nitc-g" ./nitc_3 -O --global
 420         prepare_res "$name-nitg.dat" "nitg" "nitg"
 421         run_compiler "nitg" ./nitg
 422         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 423         run_compiler "nitg-s" ./nitg --separate
 424         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 425         run_compiler "nitg-e" ./nitg --erasure
 426         plot "$name.gnu"
 427 }
 428 bench_engines
 429
 430 function bench_nitc_vc_nitg-e()
 431 {
 432         name="$FUNCNAME"
 433         skip_test "$name" && return
 434         prepare_res "$name-nitc.dat" "nitc" "nitc"
 435         run_compiler "nitc" ./nitc_3 -O
 436         prepare_res "$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
 437         NIT_GC_OPTION="malloc" run_compiler "nitc" ./nitc_3 -O
 438         prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
 439         NIT_GC_OPTION="boehm" run_compiler "nitc" ./nitc_3 -O
 440         prepare_res "$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
 441         run_compiler "nitg-e-nc-malloc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast --make-flags "CFLAGS=\"-O2 -DNOBOEHM\""
 442         prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
 443         run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
 444         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 445         run_compiler "nitg-e" ./nitg --erasure
 446         plot "$name.gnu"
 447 }
 448 bench_nitc_vc_nitg-e
 449
 450 function bench_nitg-e_gc()
 451 {
 452         name="$FUNCNAME"
 453         skip_test "$name" && return
 454         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 455         run_compiler "nitg-e-malloc" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 456         prepare_res "$name-nitg-e-noatomic.dat" "nitg-e-noatomic" "nitg with --erasure and no atomic"
 457         run_compiler "nitg-e-noatomic" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM_ATOMIC\""
 458         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 459         run_compiler "nitg-e" ./nitg --erasure
 460         plot "$name.gnu"
 461 }
 462 bench_nitg-e_gc
 463
 464 function bench_cc_nitg-e()
 465 {
 466         name="$FUNCNAME"
 467         skip_test "$name" && return
 468         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 469                 f=`echo "$o" | cut -f1 -d:`
 470                 o=`echo "$o" | cut -f2 -d:`
 471                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 472                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 473         done
 474         plot "$name.gnu"
 475 }
 476 bench_cc_nitg-e
 477
 478 function bench_policy()
 479 {
 480         name="$FUNCNAME"
 481         skip_test "$name" && return
 482         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 483         run_compiler "nitg-s" ./nitg --separate
 484         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 485         run_compiler "nitg-e" ./nitg --erasure
 486         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 487         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 488         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 489         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 490         plot "$name.gnu"
 491 }
 492 bench_policy
 493
 494 function bench_compilation_time
 495 {
 496         name="$FUNCNAME"
 497         skip_test "$name" && return
 498         prepare_res "$name-nitc.dat" "nitc" "nitc"
 499         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 500                 bench_command `basename "$i" .nit` "" ./nitc_3 -O "$i" --no-cc
 501         done
 502         prepare_res "$name-nitg.dat" "nitg" "nitg"
 503         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 504                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 505         done
 506         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 507         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 508                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 509         done
 510         plot "$name.gnu"
 511 }
 512 bench_compilation_time
 513
 514 function bench_typetest_languages()
 515 {
 516         name="$FUNCNAME"
 517         skip_test "$name" && return
 518
 519         t=t
 520         s=20
 521         seq="w2_h2 w50_h2 w2_h25 w50_h25"
 522         for b in $seq; do
 523                 run_command ./nitg benchs/gen.nit
 524                 run_command ./gen.bin "${t}_$b" "$b"
 525         done
 526
 527         prepare_res "$name-g++.dat" "g++" "g++"
 528         for b in $seq; do
 529                 run_command g++ "${t}_$b.cpp" -O2 -o "${t}_$b.g++.bin"
 530                 bench_command "$b" "" "./${t}_$b.g++.bin" $s
 531         done
 532
 533         prepare_res "$name-clang++.dat" "clang++" "clang++"
 534         for b in $seq; do
 535                 run_command clang++ "${t}_$b.cpp" -O2 -o "${t}_$b.clang++.bin"
 536                 bench_command "$b" "" "./${t}_$b.clang++.bin" $s
 537         done
 538
 539         prepare_res "$name-java.dat" "java" "java"
 540         for b in $seq; do
 541                 run_command javac ${t}_$b.java
 542                 bench_command "$b" "" java "${t}_$b" $s
 543         done
 544
 545         prepare_res "$name-scala.dat" "scala" "scala"
 546         for b in $seq; do
 547                 run_command scalac ${t}_$b.scala
 548                 bench_command "$b" "" scala "${t}_$b" $s
 549         done
 550
 551         prepare_res "$name-cs.dat" "c#" "c#"
 552         for b in $seq; do
 553                 run_command gmcs ${t}_$b.cs
 554                 bench_command "$b" "" mono "${t}_$b.exe" $s
 555         done
 556
 557         prepare_res "$name-es.dat" "es" "es"
 558         for b in $seq; do
 559                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 560                 chmod +x app${t}_$b
 561                 mv app${t}_$b  ${t}_$b.es.bin
 562                 bench_command "$b" "" "./${t}_$b.es.bin" $s
 563         done
 564
 565         prepare_res "$name-se.dat" "se" "se"
 566         for b in $seq; do
 567                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 568                 bench_command "$b" "" "./${t}_$b.se.bin" $s
 569         done
 570
 571         #too slow
 572         #prepare_res "$name-nitg.dat" "nitg" "nitg"
 573         #for b in $seq; do
 574         #       run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 575         #       bench_command "$b" "" "./${t}_$b.nitg.bin" $s
 576         #done
 577
 578         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg-s"
 579         for b in $seq; do
 580                 run_command ./nitg ${t}_$b.nit --separate -o "${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 581                 bench_command "$b" "" "./${t}_$b.nitg-s.bin" $s
 582         done
 583
 584         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg-su"
 585         for b in $seq; do
 586                 run_command ./nitg ${t}_$b.nit --separate --no-check-covariance -o "${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 587                 bench_command "$b" "" "./${t}_$b.nitg-su.bin" $s
 588         done
 589
 590
 591         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg-e"
 592         for b in $seq; do
 593                 run_command ./nitg ${t}_$b.nit --erasure -o "${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 594                 bench_command "$b" "" "./${t}_$b.nitg-e.bin" $s
 595         done
 596
 597         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
 598         for b in $seq; do
 599                 run_command ./nitg ${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 600                 bench_command "$b" "" "./${t}_$b.nitg-eu.bin" $s
 601         done
 602
 603         plot "$name.gnu"
 604 }
 605 bench_typetest_languages
 606
 607 if test -n "$died"; then
 608         echo "Some commands failed"
 609         exit 1
 610 fi
 611 exit 0