src/run_bench.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 ## CONFIGURATION OPTIONS ##
  21
  22 # Default number of times a command must be run with bench_command
  23 # Can be overrided with 'the option -n'
  24 count=2
  25
  26 ### HELPER FUNCTIONS ##
  27
  28 function die()
  29 {
  30         echo >&2 "DIE: $*"
  31         exit 1
  32 }
  33
  34 # Run a single command multiple time and store the execution times
  35 # in the current $res file.
  36 #
  37 #  $1: title of the command
  38 #  $2: long desription of the command
  39 #  rest: the command to execute
  40 function bench_command()
  41 {
  42         if [ "$dry_run" = "true" ]; then return; fi
  43         local title="$1"
  44         local desc="$2"
  45         shift
  46         shift
  47         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  48         timeout="time.out"
  49         echo "$title" > "$timeout"
  50         echo "# $desc" >> "$timeout"
  51         echo "\$ $@" >> "$timeout"
  52         echo
  53         echo "** [$title] $desc **"
  54         echo " $ $@"
  55
  56         # Execute the commands $count times
  57         for i in `seq 1 "$count"`; do
  58                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  59                 echo -n "$i. "
  60                 tail -n 1 "$timeout"
  61         done
  62
  63         line=`compute_stats "$timeout"`
  64         echo "$line ($res)"
  65         echo $line >> "$res"
  66 }
  67
  68 # Run a simble command witout storing the execution time
  69 # Used to display command on verbose and skip long executions when dry_run is given
  70 # $@ command to execute
  71 function run_command()
  72 {
  73         if [ "$dry_run" = "true" ]; then return; fi
  74         echo " $ $@"
  75         "$@"
  76 }
  77
  78 # perl function to compute min/max/avg.
  79 # used by bench_command
  80 #
  81 # $1: file
  82 # return: first min max avg label
  83 function compute_stats()
  84 {
  85         file="$1"
  86         # Compute statistics
  87         perl - "$file" <<'END'
  88         @lines = ();
  89         $first = undef;
  90         chomp($label = <>);
  91         while(<>) {
  92                 chomp;
  93                 if (/^\d/) {
  94                         if (defined $first) {
  95                                 push @lines, $_;
  96                         } else {
  97                                 $first = $_;
  98                         }
  99                 }
 100         }
 101         $len = scalar @lines;
 102         if ($len) {
 103                 @lines = sort {$a <=> $b} @lines;
 104                 $min = $lines[0];
 105                 $max = $lines[$#lines];
 106                 $avg = 0;
 107                 for $i (@lines) { $avg += $i; }
 108                 $avg = $avg / $len;
 109                 print "${first} ${min} ${max} ${avg} \"${label}\"\n";
 110         } else {
 111                 print "${first} ${first} ${first} ${first} \"${label}\"\n";
 112         }
 113 END
 114 }
 115
 116 # Create a new $res to be used in the next plot
 117 #
 118 # $1 = resfile
 119 # $2 = title
 120 # $3 = description
 121 function prepare_res()
 122 {
 123         echo
 124         echo "# [$2] $3 #"
 125         res=$1
 126         if [ "$plots" = "" ]; then
 127                 plots="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
 128         else
 129                 plots="$plots replot '$1' using 4:2:3 ti '$2';"
 130         fi
 131         if [ "$dry_run" = "true" ]; then return; fi
 132         echo "# [$2] $3" > "$res"
 133         echo "# first min max avg title" >> "$res"
 134 }
 135
 136 # Plot the last $res as an histogram
 137 # $1: plot file (eg toto.gnu)
 138 function plot()
 139 {
 140         cat >"$1" <<END
 141 set auto x;
 142 set yrange [0:];
 143 set style data histogram;
 144 set style histogram cluster gap 2;
 145 set style histogram errorbars linewidth 1;
 146 set style fill solid 0.3 border -1;
 147 set bars front;
 148 set boxwidth 0.9;
 149 set xtic nomirror rotate by -45 scale 0 font ',8';
 150 set title "$1"
 151 set ylabel "time (s)"
 152 $plots
 153 END
 154         echo "# gnuplot -p $1"
 155         gnuplot -p "$1"
 156         plots=
 157 }
 158
 159 # Check if the test should be skiped according to its name
 160 # $1: name of the test
 161 # $2: description of the test
 162 # $NOTSKIPED: arguments
 163 function skip_test()
 164 {
 165         if test -z "$NOTSKIPED"; then
 166                 echo "* $1"
 167                 return 0
 168         fi
 169         if test "$NOTSKIPED" = "all"; then
 170                 : # Execute anyway
 171         elif echo "$1" | grep "$NOTSKIPED" >/dev/null 2>&1; then
 172                 : # Found one to execute
 173         else
 174                 return 0
 175         fi
 176         echo "*"
 177         echo "* $1 *****"
 178         echo "*"
 179         return 1
 180 }
 181
 182 ## GLOBAL VARIABLES ##
 183
 184 # The current $res (set by prepare_res, used by bench_command)
 185 res=
 186
 187 # The current stuff to plot (set by prepare_res, used by plot)
 188 plots=
 189
 190 # HELPER FOR NIT #
 191
 192 # Run standards benchs on a compiler command
 193 # $1: title
 194 # rest: command to run (executable + options)
 195 function run_compiler()
 196 {
 197         local title=$1
 198         shift
 199         run_command "$@" nitg.nit -o "nitg.$title.bin"
 200         bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
 201         run_command "$@" nit.nit -o "nit.$title.bin"
 202         bench_command "nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
 203         run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 204         bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 205 }
 206
 207 ## HANDLE OPTIONS ##
 208
 209 function usage()
 210 {
 211         echo "run_bench: [options]* benchname"
 212         echo "  -v: verbose mode"
 213         echo "  -n count: number of execution for each bar (default: $count)"
 214         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 215         echo "  -h: this help"
 216 }
 217
 218 stop=false
 219 while [ "$stop" = false ]; do
 220         case "$1" in
 221                 -v) verbose=true; shift;;
 222                 -h) usage; exit;;
 223                 -n) count="$2"; shift; shift;;
 224                 --dry) dry_run=true; shift;;
 225                 *) stop=true
 226         esac
 227 done
 228
 229 NOTSKIPED="$*"
 230
 231 if test -z "$NOTSKIPED"; then
 232         usage
 233         echo "List of available benches:"
 234         echo "* all: run all the benches"
 235 fi
 236
 237 ## EFFECTIVE BENCHS ##
 238
 239 function bench_nitg_bootstrap()
 240 {
 241         name="$FUNCNAME"
 242         skip_test "$name" && return
 243         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 244         rm nit?_nit*
 245         cp ./nitc_3 ./nitc_nitc.bin
 246         bench_command "c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O nitc.nit -o nitc_nitc.bin
 247         bench_command "c/c g" "nitc_nitc nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O nitg.nit -o nitg_nitc.bin
 248         bench_command "g/c g" "nitg_nitc nitg.nit -> nitg_nitg" ./nitg_nitc.bin nitg.nit -o nitg_nitg.bin
 249         bench_command "g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin nitg.nit -o nitg_nitg.bin
 250
 251         plot "$name.gnu"
 252 }
 253 bench_nitg_bootstrap
 254
 255 function bench_steps()
 256 {
 257         name="$FUNCNAME"
 258         skip_test "$name" && return
 259         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 260         bench_command "parse" "" ./nitc_3 --only-parse nitg.nit
 261         bench_command "metamodel" "" ./nitc_3 --only-metamodel nitg.nit
 262         bench_command "generate c" "" ./nitc_3 --no-cc nitg.nit
 263         bench_command "full" "" ./nitc_3 -O nitg.nit
 264
 265         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 266         bench_command "parse" "" ./nitg --only-parse nitg.nit
 267         bench_command "metamodel" "" ./nitg --only-metamodel nitg.nit
 268         bench_command "generate c" "" ./nitg --no-cc nitg.nit
 269         bench_command "full" "" ./nitg nitg.nit
 270
 271         plot "$name.gnu"
 272 }
 273 bench_steps
 274
 275 # $#: options to compare
 276 function bench_nitg_options()
 277 {
 278         name="$FUNCNAME"
 279         skip_test "$name" && return
 280         prepare_res "$name.dat" "no options" "nitg without options"
 281         run_compiler "nitg" ./nitg
 282
 283         for opt in "$@"; do
 284                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 285                 run_compiler "nitg$opt" ./nitg $opt
 286         done
 287
 288         plot "$name.gnu"
 289 }
 290 bench_nitg_options --hardening
 291
 292 function bench_nitc_gc()
 293 {
 294         name="$FUNCNAME"
 295         skip_test "$name" && return
 296         for gc in nitgc boehm malloc large; do
 297                 prepare_res "$name-$gc".dat "$gc" "nitc with gc=$gc"
 298                 export NIT_GC_OPTION="$gc"
 299                 run_compiler "nitc" ./nitc_3 -O
 300         done
 301
 302         plot "$name.gnu"
 303 }
 304 bench_nitc_gc
 305
 306 function bench_nitc_boost()
 307 {
 308         name="$FUNCNAME"
 309         skip_test "$name" && return
 310         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 311         run_compiler "nitc_slow" ./nitc_3
 312         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 313         run_compiler "nitc" ./nitc_3 -O
 314
 315         plot "$name.gnu"
 316 }
 317 bench_nitc_boost
 318
 319 function bench_engines()
 320 {
 321         name="$FUNCNAME"
 322         skip_test "$name" && return
 323         prepare_res "$name-nitc.dat" "nitc" "nitc"
 324         run_compiler "nitc" ./nitc_3 -O
 325         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 326         run_compiler "nitc-g" ./nitc_3 -O --global
 327         prepare_res "$name-nitg.dat" "nitg" "nitg"
 328         run_compiler "nitg" ./nitg
 329         plot "$name.gnu"
 330 }
 331 bench_engines
 332
 333 function bench_compilation_time
 334 {
 335         name="$FUNCNAME"
 336         skip_test "$name" && return
 337         prepare_res "$name-nitc.dat" "nitc" "nitc"
 338         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 339                 bench_command `basename "$i" .nit` "" ./nitc_3 -O "$i" --no-cc
 340         done
 341         prepare_res "$name-nitg.dat" "nitg" "nitg"
 342         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 343                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 344         done
 345         prepare_res "$name-nitg_g.dat" "nitg/g" "nitg/g"
 346         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 347                 bench_command `basename "$i" .nit` "" ./nitg.bin "$i" --no-cc
 348         done
 349         plot "$name.gnu"
 350 }
 351 bench_compilation_time