src/run_bench.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 ## CONFIGURATION OPTIONS ##
  21
  22 # Default number of times a command must be run with bench_command
  23 # Can be overrided with 'the option -n'
  24 count=2
  25
  26 ### HELPER FUNCTIONS ##
  27
  28 function die()
  29 {
  30         echo >&2 "DIE: $*"
  31         exit 1
  32 }
  33
  34 # Run a single command multiple time and store the execution times
  35 # in the current $res file.
  36 #
  37 #  $1: title of the command
  38 #  $2: long desription of the command
  39 #  rest: the command to execute
  40 function bench_command()
  41 {
  42         if [ "$dry_run" = "true" ]; then return; fi
  43         local title="$1"
  44         local desc="$2"
  45         shift
  46         shift
  47         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  48         timeout="time.out"
  49         echo "$title" > "$timeout"
  50         echo "# $desc" >> "$timeout"
  51         echo "\$ $@" >> "$timeout"
  52         echo
  53         echo "** [$title] $desc **"
  54         echo " $ $@"
  55
  56         # Execute the commands $count times
  57         for i in `seq 1 "$count"`; do
  58                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  59                 echo -n "$i. "
  60                 tail -n 1 "$timeout"
  61         done
  62
  63         line=`compute_stats "$timeout"`
  64         echo "$line ($res)"
  65         echo $line >> "$res"
  66 }
  67
  68 # Run a simble command witout storing the execution time
  69 # Used to display command on verbose and skip long executions when dry_run is given
  70 # $@ command to execute
  71 function run_command()
  72 {
  73         if [ "$dry_run" = "true" ]; then return; fi
  74         echo " $ $@"
  75         "$@"
  76 }
  77
  78 # perl function to compute min/max/avg.
  79 # used by bench_command
  80 #
  81 # $1: file
  82 # return: first min max avg label
  83 function compute_stats()
  84 {
  85         file="$1"
  86         # Compute statistics
  87         perl - "$file" <<'END'
  88         @lines = ();
  89         $first = undef;
  90         chomp($label = <>);
  91         while(<>) {
  92                 chomp;
  93                 if (/^\d/) {
  94                         if (defined $first) {
  95                                 push @lines, $_;
  96                         } else {
  97                                 $first = $_;
  98                         }
  99                 }
 100         }
 101         $len = scalar @lines;
 102         if ($len) {
 103                 @lines = sort {$a <=> $b} @lines;
 104                 $min = $lines[0];
 105                 $max = $lines[$#lines];
 106                 $avg = 0;
 107                 for $i (@lines) { $avg += $i; }
 108                 $avg = $avg / $len;
 109                 print "${first} ${min} ${max} ${avg} \"${label}\"\n";
 110         } else {
 111                 print "${first} ${first} ${first} ${first} \"${label}\"\n";
 112         }
 113 END
 114 }
 115
 116 # Create a new $res to be used in the next plot
 117 #
 118 # $1 = resfile
 119 # $2 = title
 120 # $3 = description
 121 function prepare_res()
 122 {
 123         echo
 124         echo "# [$2] $3 #"
 125         res=$1
 126         if [ "$plots" = "" ]; then
 127                 plots="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
 128         else
 129                 plots="$plots replot '$1' using 4:2:3 ti '$2';"
 130         fi
 131         if [ "$dry_run" = "true" ]; then return; fi
 132         echo "# [$2] $3" > "$res"
 133         echo "# first min max avg title" >> "$res"
 134 }
 135
 136 # Plot the last $res as an histogram
 137 # $1: plot file (eg toto.gnu)
 138 function plot()
 139 {
 140         cat >"$1" <<END
 141 set auto x;
 142 set yrange [0:];
 143 set style data histogram;
 144 set style histogram cluster gap 2;
 145 set style histogram errorbars linewidth 1;
 146 set style fill solid 0.3 border -1;
 147 set bars front;
 148 set boxwidth 0.9;
 149 set xtic nomirror rotate by -45 scale 0 font ',8';
 150 set title "$1"
 151 set ylabel "time (s)"
 152 $plots
 153 END
 154         echo "# gnuplot -p $1"
 155         gnuplot -p "$1"
 156         plots=
 157 }
 158
 159 # Check if the test should be skiped according to its name
 160 # $1: name of the test
 161 # $2: description of the test
 162 # $NOTSKIPED: arguments
 163 function skip_test()
 164 {
 165         if test -z "$NOTSKIPED"; then
 166                 echo "* $1"
 167                 return 0
 168         fi
 169         if test "$NOTSKIPED" = "all"; then
 170                 : # Execute anyway
 171         elif echo "$1" | grep "$NOTSKIPED" >/dev/null 2>&1; then
 172                 : # Found one to execute
 173         else
 174                 return 0
 175         fi
 176         echo "*"
 177         echo "* $1 *****"
 178         echo "*"
 179         return 1
 180 }
 181
 182 ## GLOBAL VARIABLES ##
 183
 184 # The current $res (set by prepare_res, used by bench_command)
 185 res=
 186
 187 # The current stuff to plot (set by prepare_res, used by plot)
 188 plots=
 189
 190 # HELPER FOR NIT #
 191
 192 # Run standards benchs on a compiler command
 193 # $1: title
 194 # rest: command to run (executable + options)
 195 function run_compiler()
 196 {
 197         local title=$1
 198         shift
 199         run_command "$@" nitg.nit -o "nitg.$title.bin"
 200         bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
 201         run_command "$@" nit.nit -o "nit.$title.bin"
 202         bench_command "nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
 203         run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 204         bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 205         run_command "$@" ../tests/shootout_binarytrees.nit -o "bintrees.$title.bin"
 206         bench_command "bintrees" "shootout_binarytrees" "./bintrees.$title.bin" 17
 207 }
 208
 209 ## HANDLE OPTIONS ##
 210
 211 function usage()
 212 {
 213         echo "run_bench: [options]* benchname"
 214         echo "  -v: verbose mode"
 215         echo "  -n count: number of execution for each bar (default: $count)"
 216         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 217         echo "  -h: this help"
 218 }
 219
 220 stop=false
 221 while [ "$stop" = false ]; do
 222         case "$1" in
 223                 -v) verbose=true; shift;;
 224                 -h) usage; exit;;
 225                 -n) count="$2"; shift; shift;;
 226                 --dry) dry_run=true; shift;;
 227                 *) stop=true
 228         esac
 229 done
 230
 231 NOTSKIPED="$*"
 232
 233 if test -z "$NOTSKIPED"; then
 234         usage
 235         echo "List of available benches:"
 236         echo "* all: run all the benches"
 237 fi
 238
 239 ## COMPILE ENGINES
 240
 241 test -f ./nitc_3 || ./ncall.sh -O
 242 test -f ./nitg || ./nitc_3 nitg.nit -O -v
 243
 244 ## EFFECTIVE BENCHS ##
 245
 246 function bench_nitg_bootstrap()
 247 {
 248         name="$FUNCNAME"
 249         skip_test "$name" && return
 250         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 251         rm nit?_nit*
 252         cp ./nitc_3 ./nitc_nitc.bin
 253         bench_command "c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O nitc.nit -o nitc_nitc.bin
 254         bench_command "c/c g" "nitc_nitc nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O nitg.nit -o nitg_nitc.bin
 255         bench_command "g/c g" "nitg_nitc nitg.nit -> nitg_nitg" ./nitg_nitc.bin nitg.nit -o nitg_nitg.bin
 256         bench_command "g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin nitg.nit -o nitg_nitg.bin
 257
 258         plot "$name.gnu"
 259 }
 260 bench_nitg_bootstrap
 261
 262 function bench_steps()
 263 {
 264         name="$FUNCNAME"
 265         skip_test "$name" && return
 266         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 267         bench_command "parse" "" ./nitc_3 --only-parse nitg.nit
 268         bench_command "metamodel" "" ./nitc_3 --only-metamodel nitg.nit
 269         bench_command "generate c" "" ./nitc_3 --no-cc nitg.nit
 270         bench_command "full" "" ./nitc_3 -O nitg.nit -o "nitg_nitg.bin"
 271
 272         prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
 273         bench_command "parse" "" ./nitc_3 --global --only-parse nitg.nit
 274         bench_command "metamodel" "" ./nitc_3 --global --only-metamodel nitg.nit
 275         bench_command "generate c" "" ./nitc_3 --global --no-cc nitg.nit
 276         bench_command "full" "" ./nitc_3 -O --global nitg.nit -o "nitg_nitc-g.bin"
 277
 278         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 279         bench_command "parse" "" ./nitg --only-parse nitg.nit
 280         bench_command "metamodel" "" ./nitg --only-metamodel nitg.nit
 281         bench_command "generate c" "" ./nitg --no-cc nitg.nit
 282         bench_command "full" "" ./nitg nitg.nit -o "nitg_nitg.bin"
 283
 284         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 285         bench_command "parse" "" ./nitg --erasure --only-parse nitg.nit
 286         bench_command "metamodel" "" ./nitg --erasure --only-metamodel nitg.nit
 287         bench_command "generate c" "" ./nitg --erasure --no-cc nitg.nit
 288         bench_command "full" "" ./nitg --erasure nitg.nit -o "nitg_nitg-e.bin"
 289
 290         plot "$name.gnu"
 291 }
 292 bench_steps
 293
 294 # $#: options to compare
 295 function bench_nitg_options()
 296 {
 297         name="$FUNCNAME"
 298         skip_test "$name" && return
 299         prepare_res "$name.dat" "no options" "nitg without options"
 300         run_compiler "nitg" ./nitg
 301
 302         for opt in "$@"; do
 303                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 304                 run_compiler "nitg$opt" ./nitg $opt
 305         done
 306
 307         plot "$name.gnu"
 308 }
 309 bench_nitg_options --hardening --no-check-covariance --no-check-initialization --no-check-assert --no-check-other
 310
 311 function bench_nitg-e_options()
 312 {
 313         name="$FUNCNAME"
 314         skip_test "$name" && return
 315         prepare_res "$name.dat" "no options" "nitg-e without options"
 316         run_compiler "nitg-e" ./nitg --erasure
 317
 318         for opt in "$@"; do
 319                 prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
 320                 run_compiler "nitg-e$opt" ./nitg --erasure $opt
 321         done
 322
 323         plot "$name.gnu"
 324 }
 325 bench_nitg-e_options --hardening --no-inline-intern --no-check-covariance --no-check-initialization --no-check-assert --no-check-other
 326
 327 function bench_nitc_gc()
 328 {
 329         name="$FUNCNAME"
 330         skip_test "$name" && return
 331         for gc in nitgc boehm malloc large; do
 332                 prepare_res "$name-$gc".dat "$gc" "nitc with gc=$gc"
 333                 export NIT_GC_OPTION="$gc"
 334                 run_compiler "nitc" ./nitc_3 -O
 335         done
 336
 337         plot "$name.gnu"
 338 }
 339 bench_nitc_gc
 340
 341 function bench_nitc_boost()
 342 {
 343         name="$FUNCNAME"
 344         skip_test "$name" && return
 345         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 346         run_compiler "nitc_slow" ./nitc_3
 347         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 348         run_compiler "nitc" ./nitc_3 -O
 349
 350         plot "$name.gnu"
 351 }
 352 bench_nitc_boost
 353
 354 function bench_engines()
 355 {
 356         name="$FUNCNAME"
 357         skip_test "$name" && return
 358         prepare_res "$name-nitc.dat" "nitc" "nitc"
 359         run_compiler "nitc" ./nitc_3 -O
 360         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 361         run_compiler "nitc-g" ./nitc_3 -O --global
 362         prepare_res "$name-nitg.dat" "nitg" "nitg"
 363         run_compiler "nitg" ./nitg
 364         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 365         run_compiler "nitg-e" ./nitg --erasure
 366         plot "$name.gnu"
 367 }
 368 bench_engines
 369
 370 function bench_compilation_time
 371 {
 372         name="$FUNCNAME"
 373         skip_test "$name" && return
 374         prepare_res "$name-nitc.dat" "nitc" "nitc"
 375         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 376                 bench_command `basename "$i" .nit` "" ./nitc_3 -O "$i" --no-cc
 377         done
 378         prepare_res "$name-nitg.dat" "nitg" "nitg"
 379         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 380                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 381         done
 382         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 383         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 384                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 385         done
 386         plot "$name.gnu"
 387 }
 388 bench_compilation_time