src/run_bench.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 ## CONFIGURATION OPTIONS ##
  21
  22 # Default number of times a command must be run with bench_command
  23 # Can be overrided with 'the option -n'
  24 count=2
  25
  26 ### HELPER FUNCTIONS ##
  27
  28 function die()
  29 {
  30         echo >&2 "error: $*"
  31         died=1
  32 }
  33
  34 # Run a single command multiple time and store the execution times
  35 # in the current $res file.
  36 #
  37 #  $1: title of the command
  38 #  $2: long desription of the command
  39 #  rest: the command to execute
  40 function bench_command()
  41 {
  42         if [ "$dry_run" = "true" ]; then return; fi
  43         local title="$1"
  44         local desc="$2"
  45         shift
  46         shift
  47         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  48         timeout="time.out"
  49         echo "$title" > "$timeout"
  50         echo "# $desc" >> "$timeout"
  51         echo "\$ $@" >> "$timeout"
  52         echo
  53         echo "** [$title] $desc **"
  54         echo " $ $@"
  55
  56         # Execute the commands $count times
  57         for i in `seq 1 "$count"`; do
  58                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  59                 echo -n "$i. "
  60                 tail -n 1 "$timeout"
  61         done
  62
  63         line=`compute_stats "$timeout"`
  64         echo "$line ($res)"
  65         echo $line >> "$res"
  66 }
  67
  68 # Run a simble command witout storing the execution time
  69 # Used to display command on verbose and skip long executions when dry_run is given
  70 # $@ command to execute
  71 function run_command()
  72 {
  73         if [ "$dry_run" = "true" ]; then return; fi
  74         echo " $ $@"
  75         "$@" || die "$@: failed"
  76 }
  77
  78 # perl function to compute min/max/avg.
  79 # used by bench_command
  80 #
  81 # $1: file
  82 # return: first min max avg label
  83 function compute_stats()
  84 {
  85         file="$1"
  86         # Compute statistics
  87         perl - "$file" <<'END'
  88         @lines = ();
  89         $first = undef;
  90         chomp($label = <>);
  91         while(<>) {
  92                 chomp;
  93                 if (/^\d/) {
  94                         if (defined $first) {
  95                                 push @lines, $_;
  96                         } else {
  97                                 $first = $_;
  98                         }
  99                 }
 100         }
 101         $len = scalar @lines;
 102         if ($len) {
 103                 @lines = sort {$a <=> $b} @lines;
 104                 $min = $lines[0];
 105                 $max = $lines[$#lines];
 106                 $avg = 0;
 107                 for $i (@lines) { $avg += $i; }
 108                 $avg = $avg / $len;
 109                 print "${first} ${min} ${max} ${avg} \"${label}\"\n";
 110         } else {
 111                 print "${first} ${first} ${first} ${first} \"${label}\"\n";
 112         }
 113 END
 114 }
 115
 116 # Create a new $res to be used in the next plot
 117 #
 118 # $1 = resfile
 119 # $2 = title
 120 # $3 = description
 121 function prepare_res()
 122 {
 123         echo
 124         echo "# [$2] $3 #"
 125         res=$1
 126         if [ "$plots" = "" ]; then
 127                 plots="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
 128         else
 129                 plots="$plots replot '$1' using 4:2:3 ti '$2';"
 130         fi
 131         if [ "$dry_run" = "true" ]; then return; fi
 132         echo "# [$2] $3" > "$res"
 133         echo "# first min max avg title" >> "$res"
 134 }
 135
 136 # Plot the last $res as an histogram
 137 # $1: plot file (eg toto.gnu)
 138 function plot()
 139 {
 140         cat >"$1" <<END
 141 set auto x;
 142 set yrange [0:];
 143 set style data histogram;
 144 set style histogram cluster gap 2;
 145 set style histogram errorbars linewidth 1;
 146 set style fill solid 0.3 border -1;
 147 set bars front;
 148 set boxwidth 0.9;
 149 set xtic nomirror rotate by -45 scale 0 font ',8';
 150 set title "$1"
 151 set ylabel "time (s)"
 152 $plots
 153 END
 154         echo "# gnuplot -p $1"
 155         gnuplot -p "$1"
 156         plots=
 157 }
 158
 159 # Check if the test should be skiped according to its name
 160 # $1: name of the test
 161 # $2: description of the test
 162 # $NOTSKIPED: arguments
 163 function skip_test()
 164 {
 165         if test -z "$NOTSKIPED"; then
 166                 echo "* $1"
 167                 return 0
 168         fi
 169         if test "$NOTSKIPED" = "all"; then
 170                 : # Execute anyway
 171         elif echo "$1" | grep "$NOTSKIPED" >/dev/null 2>&1; then
 172                 : # Found one to execute
 173         else
 174                 return 0
 175         fi
 176         echo "*"
 177         echo "* $1 *****"
 178         echo "*"
 179         return 1
 180 }
 181
 182 ## GLOBAL VARIABLES ##
 183
 184 # The current $res (set by prepare_res, used by bench_command)
 185 res=
 186
 187 # The current stuff to plot (set by prepare_res, used by plot)
 188 plots=
 189
 190 # HELPER FOR NIT #
 191
 192 # Run standards benchs on a compiler command
 193 # $1: title
 194 # rest: command to run (executable + options)
 195 function run_compiler()
 196 {
 197         local title=$1
 198         shift
 199         run_command "$@" nitg.nit -o "nitg.$title.bin"
 200         bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
 201         run_command "$@" nit.nit -o "nit.$title.bin"
 202         bench_command "nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
 203         run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 204         bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 205         run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 206         bench_command "bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
 207 }
 208
 209 ## HANDLE OPTIONS ##
 210
 211 function usage()
 212 {
 213         echo "run_bench: [options]* benchname"
 214         echo "  -v: verbose mode"
 215         echo "  -n count: number of execution for each bar (default: $count)"
 216         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 217         echo "  -h: this help"
 218 }
 219
 220 stop=false
 221 while [ "$stop" = false ]; do
 222         case "$1" in
 223                 -v) verbose=true; shift;;
 224                 -h) usage; exit;;
 225                 -n) count="$2"; shift; shift;;
 226                 --dry) dry_run=true; shift;;
 227                 *) stop=true
 228         esac
 229 done
 230
 231 NOTSKIPED="$*"
 232
 233 if test -z "$NOTSKIPED"; then
 234         usage
 235         echo "List of available benches:"
 236         echo "* all: run all the benches"
 237 fi
 238
 239 ## COMPILE ENGINES
 240
 241 test -f ./nitc_3 || ./ncall.sh -O
 242 test -f ./nitg || ./nitc_3 nitg.nit -O -v
 243
 244 ## EFFECTIVE BENCHS ##
 245
 246 function bench_nitg_bootstrap()
 247 {
 248         name="$FUNCNAME"
 249         skip_test "$name" && return
 250         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 251         rm nit?_nit*
 252         cp ./nitc_3 ./nitc_nitc.bin
 253         bench_command "c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O nitc.nit -o nitc_nitc.bin
 254         bench_command "c/c g" "nitc_nitc nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O nitg.nit -o nitg_nitc.bin
 255         bench_command "g/c g" "nitg_nitc nitg.nit -> nitg_nitg" ./nitg_nitc.bin nitg.nit -o nitg_nitg.bin
 256         bench_command "g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin nitg.nit -o nitg_nitg.bin
 257
 258         plot "$name.gnu"
 259 }
 260 bench_nitg_bootstrap
 261
 262 function bench_steps()
 263 {
 264         name="$FUNCNAME"
 265         skip_test "$name" && return
 266         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 267         bench_command "parse" "" ./nitc_3 --only-parse nitg.nit
 268         bench_command "metamodel" "" ./nitc_3 --only-metamodel nitg.nit
 269         bench_command "generate c" "" ./nitc_3 --no-cc nitg.nit
 270         bench_command "full" "" ./nitc_3 -O nitg.nit -o "nitg_nitg.bin"
 271
 272         prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
 273         bench_command "parse" "" ./nitc_3 --global --only-parse nitg.nit
 274         bench_command "metamodel" "" ./nitc_3 --global --only-metamodel nitg.nit
 275         bench_command "generate c" "" ./nitc_3 --global --no-cc nitg.nit
 276         bench_command "full" "" ./nitc_3 -O --global nitg.nit -o "nitg_nitc-g.bin"
 277
 278         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 279         bench_command "parse" "" ./nitg --only-parse nitg.nit
 280         bench_command "metamodel" "" ./nitg --only-metamodel nitg.nit
 281         bench_command "generate c" "" ./nitg --no-cc nitg.nit
 282         bench_command "full" "" ./nitg nitg.nit -o "nitg_nitg.bin"
 283
 284         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 285         bench_command "parse" "" ./nitg --erasure --only-parse nitg.nit
 286         bench_command "metamodel" "" ./nitg --erasure --only-metamodel nitg.nit
 287         bench_command "generate c" "" ./nitg --erasure --no-cc nitg.nit
 288         bench_command "full" "" ./nitg --erasure nitg.nit -o "nitg_nitg-e.bin"
 289
 290         plot "$name.gnu"
 291 }
 292 bench_steps
 293
 294 # $#: options to compare
 295 function bench_nitg_options()
 296 {
 297         tag=$1
 298         shift
 299         name="$FUNCNAME-$tag"
 300         skip_test "$name" && return
 301         prepare_res "$name.dat" "no options" "nitg without options"
 302         run_compiler "nitg" ./nitg
 303
 304         if test -n "$2"; then
 305                 prepare_res "$name-all.dat" "all" "nitg with all options $@"
 306                 run_compiler "nitg-$tag" ./nitg $@
 307         fi
 308
 309         for opt in "$@"; do
 310                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 311                 run_compiler "nitg$opt" ./nitg $opt
 312         done
 313
 314         plot "$name.gnu"
 315 }
 316 bench_nitg_options "hardening" --hardening
 317 bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 318
 319 function bench_nitg-s_options()
 320 {
 321         tag=$1
 322         shift
 323         name="$FUNCNAME-$tag"
 324         skip_test "$name" && return
 325         prepare_res "$name.dat" "no options" "nitg-s without options"
 326         run_compiler "nitg-s" ./nitg --separate
 327
 328         if test -n "$2"; then
 329                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 330                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 331         fi
 332
 333         for opt in "$@"; do
 334                 prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt"
 335                 run_compiler "nitg-s$opt" ./nitg --separate $opt
 336         done
 337
 338         plot "$name.gnu"
 339 }
 340 bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree
 341 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 342 bench_nitg-s_options "faster" --inline-coloring-numbers
 343 bench_nitg-s_options "typing" --bm-typing --phmod-typing --phand-typing
 344
 345 function bench_nitg-e_options()
 346 {
 347         tag=$1
 348         shift
 349         name="$FUNCNAME-$tag"
 350         skip_test "$name" && return
 351         prepare_res "$name.dat" "no options" "nitg-e without options"
 352         run_compiler "nitg-e" ./nitg --erasure
 353
 354         if test -n "$2"; then
 355                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 356                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 357         fi
 358
 359         for opt in "$@"; do
 360                 prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
 361                 run_compiler "nitg-e$opt" ./nitg --erasure $opt
 362         done
 363
 364         plot "$name.gnu"
 365 }
 366 bench_nitg-e_options "slower" --hardening --no-inline-intern
 367 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 368 bench_nitg-e_options "faster" --inline-coloring-numbers
 369 bench_nitg-e_options "typing" --bm-typing --phmod-typing --phand-typing
 370
 371 function bench_nitc_gc()
 372 {
 373         name="$FUNCNAME"
 374         skip_test "$name" && return
 375         for gc in nitgc boehm malloc large; do
 376                 prepare_res "$name-$gc".dat "$gc" "nitc with gc=$gc"
 377                 export NIT_GC_OPTION="$gc"
 378                 run_compiler "nitc" ./nitc_3 -O
 379         done
 380
 381         plot "$name.gnu"
 382 }
 383 bench_nitc_gc
 384
 385 function bench_nitc_boost()
 386 {
 387         name="$FUNCNAME"
 388         skip_test "$name" && return
 389         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 390         run_compiler "nitc_slow" ./nitc_3
 391         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 392         run_compiler "nitc" ./nitc_3 -O
 393
 394         plot "$name.gnu"
 395 }
 396 bench_nitc_boost
 397
 398 function bench_engines()
 399 {
 400         name="$FUNCNAME"
 401         skip_test "$name" && return
 402         prepare_res "$name-nitc.dat" "nitc" "nitc"
 403         run_compiler "nitc" ./nitc_3 -O
 404         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 405         run_compiler "nitc-g" ./nitc_3 -O --global
 406         prepare_res "$name-nitg.dat" "nitg" "nitg"
 407         run_compiler "nitg" ./nitg
 408         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 409         run_compiler "nitg-s" ./nitg --separate
 410         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 411         run_compiler "nitg-e" ./nitg --erasure
 412         plot "$name.gnu"
 413 }
 414 bench_engines
 415
 416 function bench_nitc_vc_nitg-e()
 417 {
 418         name="$FUNCNAME"
 419         skip_test "$name" && return
 420         prepare_res "$name-nitc.dat" "nitc" "nitc"
 421         run_compiler "nitc" ./nitc_3 -O
 422         prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
 423         NIT_GC_OPTION="boehm" run_compiler "nitc" ./nitc_3 -O
 424         prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
 425         run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
 426         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 427         run_compiler "nitg-e" ./nitg --erasure
 428         plot "$name.gnu"
 429 }
 430 bench_nitc_vc_nitg-e
 431
 432 function bench_cc_nitg-e()
 433 {
 434         name="$FUNCNAME"
 435         skip_test "$name" && return
 436         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 437                 f=`echo "$o" | cut -f1 -d:`
 438                 o=`echo "$o" | cut -f2 -d:`
 439                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 440                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 441         done
 442         plot "$name.gnu"
 443 }
 444 bench_cc_nitg-e
 445
 446 function bench_compilation_time
 447 {
 448         name="$FUNCNAME"
 449         skip_test "$name" && return
 450         prepare_res "$name-nitc.dat" "nitc" "nitc"
 451         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 452                 bench_command `basename "$i" .nit` "" ./nitc_3 -O "$i" --no-cc
 453         done
 454         prepare_res "$name-nitg.dat" "nitg" "nitg"
 455         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 456                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 457         done
 458         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 459         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 460                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 461         done
 462         plot "$name.gnu"
 463 }
 464 bench_compilation_time
 465
 466 if test -n "$died"; then
 467         echo "Some commands failed"
 468         exit 1
 469 fi
 470 exit 0