src/run_bench.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 ## CONFIGURATION OPTIONS ##
  21
  22 # Number of times a command must be run with bench_command
  23 count=3
  24
  25 # FIXME: verbose mode
  26 #outputopts=">/dev/null 2>&1"
  27
  28 # Do not run commands
  29 # FIXME: buggy
  30 #dry_run=true
  31
  32 ### HELPER FUNCTIONS ##
  33
  34 function die()
  35 {
  36         echo >&2 "DIE: $*"
  37         exit 1
  38 }
  39
  40 # Run a single command multiple time and store the execution times
  41 # in the current $res file.
  42 #
  43 #  $1: title of the command
  44 #  $2: long desription of the command
  45 #  rest: the command to execute
  46 function bench_command()
  47 {
  48         if [ "$dry_run" = "true" ]; then return; fi
  49         local title="$1"
  50         local desc="$2"
  51         shift
  52         shift
  53         timeout="time.out"
  54         echo "$title" > "$timeout"
  55         echo "# $desc" >> "$timeout"
  56         echo "\$ $@" >> "$timeout"
  57         echo
  58         echo "** [$title] $desc **"
  59         echo " $ $@"
  60
  61         # Execute the commands $count times
  62         for i in `seq 1 "$count"`; do
  63                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" $outputopts # || die "$1: failed"
  64                 echo -n "$i. "
  65                 tail -n 1 "$timeout"
  66         done
  67
  68         line=`compute_stats "$timeout"`
  69         echo "$line ($res)"
  70         echo $line >> "$res"
  71 }
  72
  73 # Run a simble command witout storing the execution time
  74 # Used to display command on verbose and skip long executions when dry_run is given
  75 # $@ command to execute
  76 function run_command()
  77 {
  78         if [ "$dry_run" = "true" ]; then return; fi
  79         echo " $ $@"
  80         "$@"
  81 }
  82
  83 # perl function to compute min/max/avg.
  84 # used by bench_command
  85 #
  86 # $1: file
  87 # return: first min max avg label
  88 function compute_stats()
  89 {
  90         file="$1"
  91         # Compute statistics
  92         perl - "$file" <<'END'
  93         @lines = ();
  94         $first = undef;
  95         chomp($label = <>);
  96         while(<>) {
  97                 chomp;
  98                 if (/^\d/) {
  99                         if (defined $first) {
 100                                 push @lines, $_;
 101                         } else {
 102                                 $first = $_;
 103                         }
 104                 }
 105         }
 106         $len = scalar @lines;
 107         if ($len) {
 108                 @lines = sort {$a <=> $b} @lines;
 109                 $min = $lines[0];
 110                 $max = $lines[$#lines];
 111                 $avg = 0;
 112                 for $i (@lines) { $avg += $i; }
 113                 $avg = $avg / $len;
 114                 print "${first} ${min} ${max} ${avg} \"${label}\"\n";
 115         } else {
 116                 print "${first} ${first} ${first} ${first} \"${label}\"\n";
 117         }
 118 END
 119 }
 120
 121 # Create a new $res to be used in the next plot
 122 #
 123 # $1 = resfile
 124 # $2 = title
 125 # $3 = description
 126 function prepare_res()
 127 {
 128         echo
 129         echo "# [$2] $3 #"
 130         res=$1
 131         if [ "$plots" = "" ]; then
 132                 plots="plot '$1' using 4:xticlabels(5) ti '$2';"
 133         else
 134                 plots="$plots replot '$1' using 4 ti '$2';"
 135         fi
 136         if [ "$dry_run" = "true" ]; then return; fi
 137         echo "# [$2] $3" > "$res"
 138         echo "# first min max avg title" >> "$res"
 139 }
 140
 141 # Plot the last $res as an histogram
 142 # $1: plot file (eg toto.gnu)
 143 function plot()
 144 {
 145         cat >"$1" <<END
 146 set auto x;
 147 set yrange [0:];
 148 set style data histogram;
 149 set style histogram cluster gap 2;
 150 set style fill solid border -1;
 151 set boxwidth 0.9;
 152 set xtic nomirror rotate by -45 scale 0 font ',8';
 153 set title "$1"
 154 set ylabel "time (s)"
 155 $plots
 156 END
 157         echo "# gnuplot -p $1"
 158         gnuplot -p "$1"
 159         plots=
 160 }
 161
 162 ## GLOBAL VARIABLES ##
 163
 164 # The current $res (set by prepare_res, used by bench_command)
 165 res=
 166
 167 # The current stuff to plot (set by prepare_res, used by plot)
 168 plots=
 169
 170 # HELPER FOR NIT #
 171
 172 # Run standards benchs on a compiler command
 173 # $1: title
 174 # rest: command to run (executable + options)
 175 function run_compiler()
 176 {
 177         local title=$1
 178         shift
 179         run_command "$@" nitg.nit -o "nitg.$title.bin"
 180         bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" test_parser.nit
 181         run_command "$@" nit.nit -o "nit.$title.bin"
 182         bench_command "nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" test_parser.nit -- -n rapid_type_analysis.nit
 183 }
 184
 185 ## EFFECTIVE BENCHS ##
 186
 187 function bench_nitg_bootstrap()
 188 {
 189         name="$FUNCNAME"
 190         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 191         rm nit?_nit*
 192         cp ./nitc_3 ./nitc_nitc
 193         bench_command "c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" ./nitc_nitc -O nitc.nit -o nitc_nitc
 194         bench_command "c/c g" "nitc_nitc nitg.nit -> nitg_nitc" ./nitc_nitc -O nitg.nit -o nitg_nitc
 195         bench_command "g/c g" "nitg_nitc nitg.nit -> nitg_nitg" ./nitg_nitc nitg.nit -o nitg_nitg
 196         bench_command "g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" ./nitg_nitg nitg.nit -o nitg_nitg
 197
 198         plot "$name.gnu"
 199 }
 200 bench_nitg_bootstrap
 201
 202 function bench_steps()
 203 {
 204         name="$FUNCNAME"
 205         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 206         bench_command "parse" "" ./nitc_3 --only-parse nitg.nit
 207         bench_command "metamodel" "" ./nitc_3 --only-metamodel nitg.nit
 208         bench_command "generate c" "" ./nitc_3 --no-cc nitg.nit
 209         bench_command "full" "" ./nitc_nitc -O nitg.nit
 210
 211         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 212         bench_command "parse" "" ./nitg --only-parse nitg.nit
 213         bench_command "metamodel" "" ./nitg --only-metamodel nitg.nit
 214         bench_command "generate c" "" ./nitg --no-cc nitg.nit
 215         bench_command "full" "" ./nitg nitg.nit
 216
 217         plot "$name.gnu"
 218 }
 219 bench_steps
 220
 221 # $#: options to compare
 222 function bench_nitg_options()
 223 {
 224         name="$FUNCNAME"
 225         prepare_res "$name.dat" "no options" "nitg without options"
 226         run_compiler "nitg" ./nitg
 227
 228         for opt in "$@"; do
 229                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 230                 run_compiler "nitg$opt" ./nitg $opt
 231         done
 232
 233         plot "$name.gnu"
 234 }
 235 bench_nitg_options --hardening
 236
 237 function bench_nitc_gc()
 238 {
 239         name="$FUNCNAME"
 240         for gc in nitgc boehm malloc large; do
 241                 prepare_res "$name-$gc".dat "$gc" "nitc with gc=$gc"
 242                 export NIT_GC_OPTION="$gc"
 243                 run_compiler "nitc" ./nitc_3 -O
 244         done
 245
 246         plot "$name.gnu"
 247 }
 248 bench_nitc_gc
 249
 250 function bench_nitc_boost()
 251 {
 252         name="$FUNCNAME"
 253         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 254         run_compiler "nitc_slow" ./nitc_3
 255         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 256         run_compiler "nitc" ./nitc_3 -O
 257
 258         plot "$name.gnu"
 259 }
 260 bench_nitc_boost
 261
 262 function bench_engines()
 263 {
 264         name="$FUNCNAME"
 265         prepare_res "$name-nitc.dat" "nitc" "nitc"
 266         run_compiler "nitc" ./nitc_3 -O
 267         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 268         run_compiler "nitc-g" ./nitc_3 -O --global
 269         prepare_res "$name-nitg.dat" "nitg" "nitg"
 270         run_compiler "nitg" ./nitg
 271         plot "$name.gnu"
 272 }
 273 bench_engines
 274
 275 function bench_compilation_time
 276 {
 277         name="$FUNCNAME"
 278         prepare_res "$name-nitc.dat" "nitc" "nitc"
 279         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 280                 bench_command `basename "$i" .nit` "" ./nitc_3 -O "$i" --no-cc
 281         done
 282         prepare_res "$name-nitg.dat" "nitg" "nitg"
 283         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 284                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 285         done
 286         prepare_res "$name-nitg_g.dat" "nitg/g" "nitg/g"
 287         for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 288                 bench_command `basename "$i" .nit` "" ./nitg.bin "$i" --no-cc
 289         done
 290         plot "$name.gnu"
 291 }
 292 bench_compilation_time