benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 pep8analysis=../../pep8analysis
  29
  30 ### HELPER FUNCTIONS ##
  31
  32 function die()
  33 {
  34         echo >&2 "error: $*"
  35         died=1
  36 }
  37
  38 # Run a single command multiple time and store the execution times
  39 # in the current $res file.
  40 #
  41 #  $1: title of the command
  42 #  $2: long desription of the command
  43 #  rest: the command to execute
  44 function bench_command()
  45 {
  46         if [ "$dry_run" = "true" ]; then return; fi
  47         local title="$1"
  48         local desc="$2"
  49         shift
  50         shift
  51         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  52         timeout="time.out"
  53         echo "$title" > "$timeout"
  54         echo "# $desc" >> "$timeout"
  55         echo "\$ $@" >> "$timeout"
  56         echo
  57         echo "** [$title] $desc **"
  58         echo " $ $@"
  59
  60         # Execute the commands $count times
  61         for i in `seq 1 "$count"`; do
  62                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  63                 echo -n "$i. "
  64                 tail -n 1 "$timeout"
  65         done
  66
  67         line=`compute_stats "$timeout"`
  68         echo "$line ($res)"
  69         echo $line >> "$res"
  70 }
  71
  72 # Run a simble command witout storing the execution time
  73 # Used to display command on verbose and skip long executions when dry_run is given
  74 # $@ command to execute
  75 function run_command()
  76 {
  77         if [ "$dry_run" = "true" ]; then return; fi
  78         echo " $ $@"
  79         "$@" || die "$@: failed"
  80 }
  81
  82 # Check if the test should be skiped according to its name
  83 # $1: name of the test
  84 # $2: description of the test
  85 # $NOTSKIPED: arguments
  86 function skip_test()
  87 {
  88         if test -z "$NOTSKIPED"; then
  89                 echo "* $1"
  90                 return 0
  91         fi
  92         if test "$NOTSKIPED" = "all"; then
  93                 : # Execute anyway
  94         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  95                 : # Found one to execute
  96         else
  97                 return 0
  98         fi
  99         if test -n "$html"; then
 100                 echo >>"$html" "<h2>$1</h2>"
 101         fi
 102         echo "*"
 103         echo "* $1 *****"
 104         echo "*"
 105         return 1
 106 }
 107
 108 # HELPER FOR NIT #
 109
 110 # Run standards benchs on a compiler command
 111 # $1: title
 112 # rest: command to run (executable + options)
 113 function run_compiler()
 114 {
 115         local title=$1
 116         shift
 117         if test -n "$fast"; then
 118                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 119                 bench_command "nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit
 120                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 121                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
 122                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
 123                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 124                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 125                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
 126         else
 127                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 128                 bench_command "nitg-g" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitmetrics.nit
 129                 bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit
 130                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 131                 bench_command "nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/rapid_type_analysis.nit
 132                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
 133                 bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
 134                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 135                 bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
 136                 if test -f "$pep8analysis/src/pep8analysis.nit"; then
 137                         run_command "$@" "$pep8analysis/src/pep8analysis.nit" -I "$pep8analysis/lib" -o "pep8a.$title.bin"
 138                         bench_command "pep8analisis" "bench_bintree_gen 18" "./pep8a.$title.bin" "$pep8analysis/tests/privat/"*.pep
 139                 fi
 140         fi
 141 }
 142
 143 ## HANDLE OPTIONS ##
 144
 145 function usage()
 146 {
 147         echo "run_bench: [options]* benchname"
 148         echo "  -v: verbose mode"
 149         echo "  -n count: number of execution for each bar (default: $count)"
 150         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 151         echo "  --fast: Run less and faster tests"
 152         echo "  --html: Generate and HTML output"
 153         echo "  -h: this help"
 154 }
 155
 156 stop=false
 157 while [ "$stop" = false ]; do
 158         case "$1" in
 159                 -v) verbose=true; shift;;
 160                 -h) usage; exit;;
 161                 -n) count="$2"; shift; shift;;
 162                 --dry) dry_run=true; shift;;
 163                 --fast) fast=true; shift;;
 164                 --html) html="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
 165                 *) stop=true
 166         esac
 167 done
 168
 169 NOTSKIPED="$*"
 170
 171 if test -z "$NOTSKIPED"; then
 172         usage
 173         echo "List of available benches:"
 174         echo "* all: run all the benches"
 175 fi
 176
 177 ## COMPILE ENGINES
 178
 179 # force to use the last nitg, not the bootstraped one
 180 test -f ./nitg || ../bin/nitg ../src/nitg.nit -v
 181
 182 ## EFFECTIVE BENCHS ##
 183
 184 function bench_steps()
 185 {
 186         name="$FUNCNAME"
 187         skip_test "$name" && return
 188         prepare_res "$name-nitg.dat" "nitg-g" "Various steps of nitg --global"
 189         bench_command "parse" "" ./nitg --global --only-parse ../src/nitg.nit
 190         bench_command "metamodel" "" ./nitg --global --only-metamodel ../src/nitg.nit
 191         bench_command "generate c" "" ./nitg --global --no-cc ../src/nitg.nit
 192         bench_command "full" "" ./nitg --global ../src/nitg.nit -o "nitg_nitg.bin"
 193
 194         prepare_res "$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
 195         bench_command "parse" "" ./nitg --separate --only-parse ../src/nitg.nit
 196         bench_command "metamodel" "" ./nitg --separate --only-metamodel ../src/nitg.nit
 197         bench_command "generate c" "" ./nitg --separate --no-cc ../src/nitg.nit
 198         bench_command "full" "" ./nitg --separate ../src/nitg.nit -o "nitg_nitg-e.bin"
 199
 200         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 201         bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit
 202         bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit
 203         bench_command "generate c" "" ./nitg --erasure --no-cc ../src/nitg.nit
 204         bench_command "full" "" ./nitg --erasure ../src/nitg.nit -o "nitg_nitg-e.bin"
 205
 206         plot "$name.gnu"
 207 }
 208 bench_steps
 209
 210 # $#: options to compare
 211 function bench_nitg-g_options()
 212 {
 213         tag=$1
 214         shift
 215         name="$FUNCNAME-$tag"
 216         skip_test "$name" && return
 217         prepare_res "$name.dat" "no options" "nitg-g without options"
 218         run_compiler "nitg-g" ./nitg --global
 219
 220         if test "$1" = NOALL; then
 221                 shift
 222         elif test -n "$2"; then
 223                 prepare_res "$name-all.dat" "all" "nitg-g with all options $@"
 224                 run_compiler "nitg-g-$tag" ./nitg --global $@
 225         fi
 226
 227         for opt in "$@"; do
 228                 ot=${opt// /+}
 229                 prepare_res "$name$ot.dat" "$opt" "nitg-g with option $opt"
 230                 run_compiler "nitg-g$ot" ./nitg --global $opt
 231         done
 232
 233         plot "$name.gnu"
 234 }
 235 bench_nitg-g_options "slower" --hardening
 236 bench_nitg-g_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other
 237
 238 function bench_nitg-s_options()
 239 {
 240         tag=$1
 241         shift
 242         name="$FUNCNAME-$tag"
 243         skip_test "$name" && return
 244         prepare_res "$name.dat" "no options" "nitg-s without options"
 245         run_compiler "nitg-s" ./nitg --separate
 246
 247         if test "$1" = NOALL; then
 248                 shift
 249         elif test -n "$2"; then
 250                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 251                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 252         fi
 253
 254         for opt in "$@"; do
 255                 ot=${opt// /+}
 256                 prepare_res "$name-$ot.dat" "$opt" "nitg-s with option $opt"
 257                 run_compiler "nitg-s$ot" ./nitg --separate $opt
 258         done
 259
 260         plot "$name.gnu"
 261 }
 262 bench_nitg-s_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range "--no-gcc-directive likely" "--no-gcc-directive noreturn"
 263 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other
 264 bench_nitg-s_options "faster" --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph"
 265
 266 function bench_nitg-e_options()
 267 {
 268         tag=$1
 269         shift
 270         name="$FUNCNAME-$tag"
 271         skip_test "$name" && return
 272         prepare_res "$name.dat" "no options" "nitg-e without options"
 273         run_compiler "nitg-e" ./nitg --erasure
 274
 275         if test "$1" = NOALL; then
 276                 shift
 277         elif test -n "$2"; then
 278                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 279                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 280         fi
 281
 282         for opt in "$@"; do
 283                 ot=${opt// /+}
 284                 prepare_res "$name$ot.dat" "$opt" "nitg-e with option $opt"
 285                 run_compiler "nitg-e$ot" ./nitg --erasure $opt
 286         done
 287
 288         plot "$name.gnu"
 289 }
 290 bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
 291 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 292 bench_nitg-e_options "faster" --inline-coloring-numbers
 293
 294 function bench_engines()
 295 {
 296         name="$FUNCNAME"
 297         skip_test "$name" && return
 298         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg with --global"
 299         run_compiler "nitg-g" ./nitg --global
 300         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 301         run_compiler "nitg-s" ./nitg --separate
 302         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 303         run_compiler "nitg-e" ./nitg --erasure
 304         plot "$name.gnu"
 305 }
 306 bench_engines
 307
 308 function bench_nitg-e_gc()
 309 {
 310         name="$FUNCNAME"
 311         skip_test "$name" && return
 312         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 313         NIT_GC_OPTION="malloc" run_compiler "nitg-e-malloc" ./nitg --erasure
 314         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 315         run_compiler "nitg-e" ./nitg --erasure
 316         plot "$name.gnu"
 317 }
 318 bench_nitg-e_gc
 319
 320 function bench_cc_nitg-e()
 321 {
 322         name="$FUNCNAME"
 323         skip_test "$name" && return
 324         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 325                 f=`echo "$o" | cut -f1 -d:`
 326                 o=`echo "$o" | cut -f2 -d:`
 327                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 328                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 329         done
 330         plot "$name.gnu"
 331 }
 332 bench_cc_nitg-e
 333
 334 function bench_policy()
 335 {
 336         name="$FUNCNAME"
 337         skip_test "$name" && return
 338         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 339         run_compiler "nitg-s" ./nitg --separate
 340         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 341         run_compiler "nitg-e" ./nitg --erasure
 342         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 343         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 344         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 345         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 346         plot "$name.gnu"
 347 }
 348 bench_policy
 349
 350 function bench_nullables()
 351 {
 352         name="$FUNCNAME"
 353         skip_test "$name" && return
 354         prepare_res "$name-nitc.dat" "nitc" "nitc no options"
 355         run_compiler "nitc" ./nitg --separate
 356         prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
 357         run_compiler "nitc" ./nitg --separate --no-check-attr-isset
 358         prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
 359         run_compiler "nitc" ./nitg --separate --no-union-attribute
 360         prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
 361         run_compiler "nitc" ./nitg --separate --no-union-attribute --no-check-attr-isset
 362         plot "$name.gnu"
 363 }
 364 bench_nullables
 365
 366 function bench_compilation_time
 367 {
 368         name="$FUNCNAME"
 369         skip_test "$name" && return
 370         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg --global"
 371         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 372                 bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc
 373         done
 374         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg --separate"
 375         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 376                 bench_command `basename "$i" .nit` "" ./nitg --separate "$i" --no-cc
 377         done
 378         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 379         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 380                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 381         done
 382         plot "$name.gnu"
 383 }
 384 bench_compilation_time
 385
 386 if test -n "$html"; then
 387         echo >>"$html" "</body></html>"
 388 fi
 389
 390 if test -n "$died"; then
 391         echo "Some commands failed"
 392         exit 1
 393 fi
 394 exit 0