benchmarks/bench_languages.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 ### HELPER FUNCTIONS ##
  29
  30 function die()
  31 {
  32         echo >&2 "error: $*"
  33         died=1
  34 }
  35
  36 # Run a single command multiple time and store the execution times
  37 # in the current $res file.
  38 #
  39 #  $1: title of the command
  40 #  $2: long desription of the command
  41 #  rest: the command to execute
  42 function bench_command()
  43 {
  44         if [ "$dry_run" = "true" ]; then return; fi
  45         local title="$1"
  46         local desc="$2"
  47         shift
  48         shift
  49         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  50         timeout="time.out"
  51         echo "$title" > "$timeout"
  52         echo "# $desc" >> "$timeout"
  53         echo "\$ $@" >> "$timeout"
  54         echo
  55         echo "** [$title] $desc **"
  56         echo " $ $@"
  57
  58         # Execute the commands $count times
  59         for i in `seq 1 "$count"`; do
  60                 (ulimit -t 60; /usr/bin/time -f "%U" -o "$timeout" -a "$@") > $outputopts 2>&1 || die "$1: failed"
  61                 echo -n "$i. "
  62                 tail -n 1 "$timeout"
  63         done
  64
  65         line=`compute_stats "$timeout"`
  66         echo "$line ($res)"
  67         echo $line >> "$res"
  68         rm $timeout
  69 }
  70
  71 # Run a simple command witout storing the execution time
  72 # Used to display command on verbose and skip long executions when dry_run is given
  73 # $@ command to execute
  74 function run_command()
  75 {
  76         if [ "$dry_run" = "true" ]; then return; fi
  77         echo " $ $@"
  78         (ulimit -t 180; "$@") || die "$@: failed"
  79 }
  80
  81 # Check if the test should be skiped according to its name
  82 # $1: name of the test
  83 # $2: description of the test
  84 # $NOTSKIPED: arguments
  85 function skip_test()
  86 {
  87         if test -z "$NOTSKIPED"; then
  88                 echo "* $1"
  89                 return 0
  90         fi
  91         if test "$NOTSKIPED" = "all"; then
  92                 : # Execute anyway
  93         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  94                 : # Found one to execute
  95         else
  96                 return 0
  97         fi
  98         echo "*"
  99         echo "* $1 *****"
 100         echo "*"
 101         return 1
 102 }
 103
 104 ## HANDLE OPTIONS ##
 105
 106 function usage()
 107 {
 108         echo "run_bench: [options]* benchname"
 109         echo "  -v: verbose mode"
 110         echo "  -n count: number of execution for each bar (default: $count)"
 111         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 112         echo "  --fast: Run less and faster tests"
 113         echo "  -h: this help"
 114 }
 115
 116 stop=false
 117 while [ "$stop" = false ]; do
 118         case "$1" in
 119                 -v) verbose=true; shift;;
 120                 -h) usage; exit;;
 121                 -n) count="$2"; shift; shift;;
 122                 --dry) dry_run=true; shift;;
 123                 --fast) fast=true; shift;;
 124                 *) stop=true
 125         esac
 126 done
 127
 128 NOTSKIPED="$*"
 129
 130 if test -z "$NOTSKIPED"; then
 131         usage
 132         echo "List of available benches:"
 133         echo "* all: run all the benches"
 134 fi
 135
 136 ## COMPILE ENGINES
 137 cd ../src
 138 test -f ./nitc_3 || ./ncall.sh -O
 139 cd ../benchmarks
 140 test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v
 141
 142 ## EFFECTIVE BENCHS ##
 143
 144 function bench_typetest_languages()
 145 {
 146         name="$FUNCNAME"
 147         skip_test "$name" && return
 148
 149         t=t
 150         s=20
 151         seq="w2_h2 w50_h2 w2_h25 w50_h25"
 152         for b in $seq; do
 153                 run_command ./nitg languages/gen.nit
 154                 run_command ./gen.bin "${t}_$b" "$b"
 155         done
 156
 157         prepare_res "$name-g++.dat" "g++" "g++"
 158         for b in $seq; do
 159                 run_command g++ "${t}_$b.cpp" -O2 -o "${t}_$b.g++.bin"
 160                 bench_command "$b" "" "./${t}_$b.g++.bin" $s
 161         done
 162
 163         prepare_res "$name-clang++.dat" "clang++" "clang++"
 164         for b in $seq; do
 165                 run_command clang++ "${t}_$b.cpp" -O2 -o "${t}_$b.clang++.bin"
 166                 bench_command "$b" "" "./${t}_$b.clang++.bin" $s
 167         done
 168
 169         prepare_res "$name-java.dat" "java" "java"
 170         for b in $seq; do
 171                 run_command javac ${t}_$b.java
 172                 bench_command "$b" "" java "${t}_$b" $s
 173         done
 174
 175         prepare_res "$name-scala.dat" "scala" "scala"
 176         for b in $seq; do
 177                 run_command scalac ${t}_$b.scala
 178                 bench_command "$b" "" scala "${t}_$b" $s
 179         done
 180
 181         prepare_res "$name-cs.dat" "c#" "c#"
 182         for b in $seq; do
 183                 run_command gmcs ${t}_$b.cs
 184                 bench_command "$b" "" mono "${t}_$b.exe" $s
 185         done
 186
 187         prepare_res "$name-es.dat" "es" "es"
 188         for b in $seq; do
 189                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 190                 chmod +x app${t}_$b
 191                 mv app${t}_$b  ${t}_$b.es.bin
 192                 bench_command "$b" "" "./${t}_$b.es.bin" $s
 193         done
 194
 195         prepare_res "$name-se.dat" "se" "se"
 196         for b in $seq; do
 197                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 198                 bench_command "$b" "" "./${t}_$b.se.bin" $s
 199         done
 200
 201         #too slow
 202         #prepare_res "$name-nitg.dat" "nitg" "nitg"
 203         #for b in $seq; do
 204         #       run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 205         #       bench_command "$b" "" "./${t}_$b.nitg.bin" $s
 206         #done
 207
 208         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg-s"
 209         for b in $seq; do
 210                 run_command ./nitg ${t}_$b.nit --separate -o "${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 211                 bench_command "$b" "" "./${t}_$b.nitg-s.bin" $s
 212         done
 213
 214         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg-su"
 215         for b in $seq; do
 216                 run_command ./nitg ${t}_$b.nit --separate --no-check-covariance -o "${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 217                 bench_command "$b" "" "./${t}_$b.nitg-su.bin" $s
 218         done
 219
 220
 221         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg-e"
 222         for b in $seq; do
 223                 run_command ./nitg ${t}_$b.nit --erasure -o "${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 224                 bench_command "$b" "" "./${t}_$b.nitg-e.bin" $s
 225         done
 226
 227         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
 228         for b in $seq; do
 229                 run_command ./nitg ${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 230                 bench_command "$b" "" "./${t}_$b.nitg-eu.bin" $s
 231         done
 232
 233         plot "$name.gnu"
 234 }
 235 bench_typetest_languages
 236
 237 function bench_typetest_depth()
 238 {
 239         name="$FUNCNAME"
 240         skip_test "$name" && return
 241         rootdir=`pwd`
 242         basedir="./${name}.out"
 243
 244         mkdir $basedir
 245
 246         t=t
 247         s=20
 248         seq="10 25 50 100"
 249         for b in $seq; do
 250                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 251                 run_command $basedir/$name.bin $basedir "${t}_$b" "$b"
 252         done
 253
 254         prepare_res $basedir/$name-g++.dat "g++" "g++"
 255         cppdir="${basedir}/cpp"
 256         for b in $seq; do
 257                 run_command g++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.g++.bin"
 258                 bench_command "$b" "" "${cppdir}/${t}_$b.g++.bin" $s
 259         done
 260
 261         prepare_res $basedir/$name-clang++.dat "clang++" "clang++"
 262         for b in $seq; do
 263                 run_command clang++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.clang++.bin"
 264                 bench_command "$b" "" "${cppdir}/${t}_$b.clang++.bin" $s
 265         done
 266
 267         prepare_res $basedir/$name-java.dat "java" "java"
 268         javadir="${basedir}/java"
 269         for b in $seq; do
 270                 run_command javac "${javadir}/${t}_$b.java"
 271                 bench_command "$b" "" java -cp "${javadir}/" "${t}_$b" $s
 272         done
 273
 274         prepare_res $basedir/$name-gcj.dat "gcj" "gcj"
 275         for b in $seq; do
 276                 run_command gcj --main=${t}_$b -O2 "${javadir}/${t}_$b.java" -o "${javadir}/${t}_$b.gcj.bin"
 277                 bench_command "$b" "" "${javadir}/${t}_$b.gcj.bin" $s
 278         done
 279
 280         prepare_res $basedir/$name-scala.dat "scala" "scala"
 281         scaladir="${basedir}/scala"
 282         for b in $seq; do
 283                 run_command scalac "${scaladir}/${t}_$b.scala" -d "${scaladir}"
 284                 bench_command "$b" "" scala -cp "${scaladir}/" "${t}_$b" $s
 285         done
 286
 287         prepare_res $basedir/$name-cs.dat "c#" "c#"
 288         csdir="${basedir}/cs"
 289         for b in $seq; do
 290                 run_command gmcs "$csdir/${t}_$b.cs"
 291                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 292         done
 293
 294         prepare_res $basedir/$name-es.dat "es" "es"
 295         esdir="${basedir}/es"
 296         for b in $seq; do
 297                 cd $esdir
 298                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 299                 chmod +x app${t}_$b
 300                 mv app${t}_$b ${t}_$b.es.bin
 301                 cd $rootdir
 302                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 303         done
 304
 305         prepare_res $basedir/$name-se.dat "se" "se"
 306         sedir="${basedir}/se"
 307         for b in $seq; do
 308                 cd $sedir
 309                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 310                 cd $rootdir
 311                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 312         done
 313
 314         nitdir="${basedir}/nit"
 315         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 316         for b in $seq; do
 317                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 318                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 319         done
 320
 321         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 322         for b in $seq; do
 323                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 324                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 325         done
 326
 327         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 328         for b in $seq; do
 329                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 330                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 331         done
 332
 333         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 334         for b in $seq; do
 335                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 336                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 337         done
 338
 339         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 340         for b in $seq; do
 341                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 342                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 343         done
 344
 345         plot $basedir/$name.gnu
 346 }
 347 bench_typetest_depth
 348
 349 function bench_typetest_fts_depth()
 350 {
 351         name="$FUNCNAME"
 352         skip_test "$name" && return
 353         rootdir=`pwd`
 354         basedir="./${name}.out"
 355
 356         mkdir $basedir
 357
 358         t=t
 359         s=20
 360         seq="10 25 50 100"
 361         for b in $seq; do
 362                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 363                 run_command $basedir/$name.bin $basedir "${t}_$b" "$b"
 364         done
 365
 366         prepare_res $basedir/$name-cs.dat "c#" "c#"
 367         csdir="${basedir}/cs"
 368         for b in $seq; do
 369                 run_command gmcs "$csdir/${t}_$b.cs"
 370                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 371         done
 372
 373         prepare_res $basedir/$name-es.dat "es" "es"
 374         esdir="${basedir}/es"
 375         for b in $seq; do
 376                 cd $esdir
 377                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 378                 chmod +x app${t}_$b
 379                 mv app${t}_$b ${t}_$b.es.bin
 380                 cd $rootdir
 381                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 382         done
 383
 384         prepare_res $basedir/$name-se.dat "se" "se"
 385         sedir="${basedir}/se"
 386         for b in $seq; do
 387                 cd $sedir
 388                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 389                 cd $rootdir
 390                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 391         done
 392
 393         nitdir="${basedir}/nit"
 394         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 395         for b in $seq; do
 396                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 397                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 398         done
 399
 400         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 401         for b in $seq; do
 402                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 403                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 404         done
 405
 406         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 407         for b in $seq; do
 408                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 409                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 410         done
 411
 412         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 413         for b in $seq; do
 414                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 415                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 416         done
 417
 418         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 419         for b in $seq; do
 420                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 421                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 422         done
 423
 424         plot $basedir/$name.gnu
 425 }
 426 bench_typetest_fts_depth
 427
 428 function bench_typetest_fts_width()
 429 {
 430         name="$FUNCNAME"
 431         skip_test "$name" && return
 432         rootdir=`pwd`
 433         basedir="./${name}.out"
 434
 435         mkdir $basedir
 436
 437         t=t
 438         s=20
 439         depth=10
 440         seq="1 2 5 10"
 441         for b in $seq; do
 442                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 443                 run_command $basedir/$name.bin $basedir "${t}_$b" $depth $b
 444         done
 445
 446         prepare_res $basedir/$name-cs.dat "c#" "c#"
 447         csdir="${basedir}/cs"
 448         for b in $seq; do
 449                 run_command gmcs "$csdir/${t}_$b.cs"
 450                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 451         done
 452
 453         prepare_res $basedir/$name-es.dat "es" "es"
 454         esdir="${basedir}/es"
 455         for b in $seq; do
 456                 cd $esdir
 457                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 458                 chmod +x app${t}_$b
 459                 mv app${t}_$b ${t}_$b.es.bin
 460                 cd $rootdir
 461                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 462         done
 463
 464         prepare_res $basedir/$name-se.dat "se" "se"
 465         sedir="${basedir}/se"
 466         for b in $seq; do
 467                 cd $sedir
 468                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 469                 cd $rootdir
 470                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 471         done
 472
 473         nitdir="${basedir}/nit"
 474
 475         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 476         for b in $seq; do
 477                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 478                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 479         done
 480
 481         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 482         for b in $seq; do
 483                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 484                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 485         done
 486
 487         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 488         for b in $seq; do
 489                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 490                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 491         done
 492
 493         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 494         for b in $seq; do
 495                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 496                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 497         done
 498
 499         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 500         for b in $seq; do
 501                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 502                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 503         done
 504
 505         plot $basedir/$name.gnu
 506 }
 507 bench_typetest_fts_width
 508
 509 function bench_typetest_fts_nesting()
 510 {
 511         name="$FUNCNAME"
 512         skip_test "$name" && return
 513         rootdir=`pwd`
 514         basedir="./${name}.out"
 515
 516         mkdir $basedir
 517
 518         t=t
 519         s=20
 520         depth=5
 521         seq="1 2 5 10"
 522         for b in $seq; do
 523                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 524                 run_command $basedir/$name.bin $basedir "${t}_$b" $depth $b
 525         done
 526
 527         prepare_res $basedir/$name-cs.dat "c#" "c#"
 528         csdir="${basedir}/cs"
 529         for b in $seq; do
 530                 run_command gmcs "$csdir/${t}_$b.cs"
 531                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 532         done
 533
 534         prepare_res $basedir/$name-es.dat "es" "es"
 535         esdir="${basedir}/es"
 536         for b in $seq; do
 537                 cd $esdir
 538                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 539                 chmod +x app${t}_$b
 540                 mv app${t}_$b ${t}_$b.es.bin
 541                 cd $rootdir
 542                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 543         done
 544
 545         prepare_res $basedir/$name-se.dat "se" "se"
 546         sedir="${basedir}/se"
 547         for b in $seq; do
 548                 cd $sedir
 549                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 550                 cd $rootdir
 551                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 552         done
 553
 554         nitdir="${basedir}/nit"
 555
 556         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 557         for b in $seq; do
 558                 run_command ./nitg $nitdir/${t}_$b.nit -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 559                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 560         done
 561
 562         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 563         for b in $seq; do
 564                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 565                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 566         done
 567
 568         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 569         for b in $seq; do
 570                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 571                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 572         done
 573
 574         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 575         for b in $seq; do
 576                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 577                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 578         done
 579
 580         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 581         for b in $seq; do
 582                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 583                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 584         done
 585
 586         plot $basedir/$name.gnu
 587 }
 588 bench_typetest_fts_nesting
 589
 590 if test -n "$died"; then
 591         echo "Some commands failed"
 592         exit 1
 593 fi
 594 exit 0