benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 pep8analysis=../../pep8analysis
  29
  30 ### HELPER FUNCTIONS ##
  31
  32 function die()
  33 {
  34         echo >&2 "error: $*"
  35         died=1
  36 }
  37
  38 # Run a single command multiple time and store the execution times
  39 # in the current $res file.
  40 #
  41 #  $1: title of the command
  42 #  $2: long desription of the command
  43 #  rest: the command to execute
  44 function bench_command()
  45 {
  46         if [ "$dry_run" = "true" ]; then return; fi
  47         local title="$1"
  48         local desc="$2"
  49         shift
  50         shift
  51         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  52         timeout="time.out"
  53         echo "$title" > "$timeout"
  54         echo "# $desc" >> "$timeout"
  55         echo "\$ $@" >> "$timeout"
  56         echo
  57         echo "** [$title] $desc **"
  58         echo " $ $@"
  59
  60         # Execute the commands $count times
  61         for i in `seq 1 "$count"`; do
  62                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  63                 echo -n "$i. "
  64                 tail -n 1 "$timeout"
  65         done
  66
  67         line=`compute_stats "$timeout"`
  68         echo "$line ($res)"
  69         echo $line >> "$res"
  70 }
  71
  72 # Run a simble command witout storing the execution time
  73 # Used to display command on verbose and skip long executions when dry_run is given
  74 # $@ command to execute
  75 function run_command()
  76 {
  77         if [ "$dry_run" = "true" ]; then return; fi
  78         echo " $ $@"
  79         "$@" || die "$@: failed"
  80 }
  81
  82 # Check if the test should be skiped according to its name
  83 # $1: name of the test
  84 # $2: description of the test
  85 # $NOTSKIPED: arguments
  86 function skip_test()
  87 {
  88         if test -z "$NOTSKIPED"; then
  89                 echo "* $1"
  90                 return 0
  91         fi
  92         if test "$NOTSKIPED" = "all"; then
  93                 : # Execute anyway
  94         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  95                 : # Found one to execute
  96         else
  97                 return 0
  98         fi
  99         echo "*"
 100         echo "* $1 *****"
 101         echo "*"
 102         return 1
 103 }
 104
 105 # HELPER FOR NIT #
 106
 107 # Run standards benchs on a compiler command
 108 # $1: title
 109 # rest: command to run (executable + options)
 110 function run_compiler()
 111 {
 112         local title=$1
 113         shift
 114         if test -n "$fast"; then
 115                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 116                 bench_command "nitg" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit
 117                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 118                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
 119                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 120                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 121                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 122                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
 123         else
 124                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 125                 bench_command "nitg" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitmetrics.nit
 126                 bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit
 127                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 128                 bench_command "nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/rapid_type_analysis.nit
 129                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 130                 bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
 131                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 132                 bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
 133                 run_command "$@" "$pep8analysis/src/pep8analysis.nit" -I "$pep8analysis/lib" -o "pep8a.$title.bin"
 134                 bench_command "pep8analisis" "bench_bintree_gen 18" "./pep8a.$title.bin" "$pep8analysis/tests/privat/"*.pep
 135         fi
 136 }
 137
 138 ## HANDLE OPTIONS ##
 139
 140 function usage()
 141 {
 142         echo "run_bench: [options]* benchname"
 143         echo "  -v: verbose mode"
 144         echo "  -n count: number of execution for each bar (default: $count)"
 145         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 146         echo "  --fast: Run less and faster tests"
 147         echo "  -h: this help"
 148 }
 149
 150 stop=false
 151 while [ "$stop" = false ]; do
 152         case "$1" in
 153                 -v) verbose=true; shift;;
 154                 -h) usage; exit;;
 155                 -n) count="$2"; shift; shift;;
 156                 --dry) dry_run=true; shift;;
 157                 --fast) fast=true; shift;;
 158                 *) stop=true
 159         esac
 160 done
 161
 162 NOTSKIPED="$*"
 163
 164 if test -z "$NOTSKIPED"; then
 165         usage
 166         echo "List of available benches:"
 167         echo "* all: run all the benches"
 168 fi
 169
 170 ## COMPILE ENGINES
 171
 172 test -f ../src/nitc_3 || ../src/ncall.sh -O
 173 test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v
 174
 175 ## EFFECTIVE BENCHS ##
 176
 177 function bench_nitg_bootstrap()
 178 {
 179         name="$FUNCNAME"
 180         skip_test "$name" && return
 181         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 182         rm nit?_nit*
 183         cp ../src/nitc_3 ./nitc_nitc.bin
 184         bench_command "c/c c" "nitc_nitc ../src/nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O ../src/nitc.nit -o nitc_nitc.bin
 185         bench_command "c/c g" "nitc_nitc ../src/nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O ../src/nitg.nit -o nitg_nitc.bin
 186         bench_command "g/c g" "nitg_nitc ../src/nitg.nit -> nitg_nitg" ./nitg_nitc.bin ../src/nitg.nit -o nitg_nitg.bin
 187         bench_command "g/g g" "nitg_nitg ../src/nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin ../src/nitg.nit -o nitg_nitg.bin
 188
 189         plot "$name.gnu"
 190 }
 191 bench_nitg_bootstrap
 192
 193 function bench_steps()
 194 {
 195         name="$FUNCNAME"
 196         skip_test "$name" && return
 197         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 198         bench_command "parse" "" ../src/nitc_3 --only-parse ../src/nitg.nit
 199         bench_command "metamodel" "" ../src/nitc_3 --only-metamodel ../src/nitg.nit
 200         bench_command "generate c" "" ../src/nitc_3 --no-cc ../src/nitg.nit
 201         bench_command "full" "" ../src/nitc_3 -O ../src/nitg.nit -o "nitg_nitg.bin"
 202
 203         prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
 204         bench_command "parse" "" ../src/nitc_3 --global --only-parse ../src/nitg.nit
 205         bench_command "metamodel" "" ../src/nitc_3 --global --only-metamodel ../src/nitg.nit
 206         bench_command "generate c" "" ../src/nitc_3 --global --no-cc ../src/nitg.nit
 207         bench_command "full" "" ../src/nitc_3 -O --global ../src/nitg.nit -o "nitg_nitc-g.bin"
 208
 209         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg --global"
 210         bench_command "parse" "" ./nitg --global --only-parse ../src/nitg.nit
 211         bench_command "metamodel" "" ./nitg --global --only-metamodel ../src/nitg.nit
 212         bench_command "generate c" "" ./nitg --global --no-cc ../src/nitg.nit
 213         bench_command "full" "" ./nitg --global ../src/nitg.nit -o "nitg_nitg.bin"
 214
 215         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 216         bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit
 217         bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit
 218         bench_command "generate c" "" ./nitg --erasure --no-cc ../src/nitg.nit
 219         bench_command "full" "" ./nitg --erasure ../src/nitg.nit -o "nitg_nitg-e.bin"
 220
 221         plot "$name.gnu"
 222 }
 223 bench_steps
 224
 225 # $#: options to compare
 226 function bench_nitg_options()
 227 {
 228         tag=$1
 229         shift
 230         name="$FUNCNAME-$tag"
 231         skip_test "$name" && return
 232         prepare_res "$name.dat" "no options" "nitg without options"
 233         run_compiler "nitg" ./nitg --global
 234
 235         if test -n "$2"; then
 236                 prepare_res "$name-all.dat" "all" "nitg with all options $@"
 237                 run_compiler "nitg-$tag" ./nitg --global $@
 238         fi
 239
 240         for opt in "$@"; do
 241                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 242                 run_compiler "nitg$opt" ./nitg --global $opt
 243         done
 244
 245         plot "$name.gnu"
 246 }
 247 bench_nitg_options "hardening" --hardening
 248 bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 249
 250 function bench_nitg-s_options()
 251 {
 252         tag=$1
 253         shift
 254         name="$FUNCNAME-$tag"
 255         skip_test "$name" && return
 256         prepare_res "$name.dat" "no options" "nitg-s without options"
 257         run_compiler "nitg-s" ./nitg --separate
 258
 259         if test -n "$2"; then
 260                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 261                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 262         fi
 263
 264         for opt in "$@"; do
 265                 prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt"
 266                 run_compiler "nitg-s$opt" ./nitg --separate $opt
 267         done
 268
 269         plot "$name.gnu"
 270 }
 271 bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree --no-union-attribute --no-shortcut-equal --no-shortcut-range
 272 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 273 bench_nitg-s_options "faster" --inline-coloring-numbers
 274 bench_nitg-s_options "typing" --bm-typing --phand-typing
 275
 276 function bench_nitg-e_options()
 277 {
 278         tag=$1
 279         shift
 280         name="$FUNCNAME-$tag"
 281         skip_test "$name" && return
 282         prepare_res "$name.dat" "no options" "nitg-e without options"
 283         run_compiler "nitg-e" ./nitg --erasure
 284
 285         if test -n "$2"; then
 286                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 287                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 288         fi
 289
 290         for opt in "$@"; do
 291                 prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
 292                 run_compiler "nitg-e$opt" ./nitg --erasure $opt
 293         done
 294
 295         plot "$name.gnu"
 296 }
 297 bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
 298 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 299 bench_nitg-e_options "faster" --inline-coloring-numbers
 300 bench_nitg-e_options "typing" --bm-typing --phand-typing
 301
 302 function bench_nitc_gc()
 303 {
 304         name="$FUNCNAME"
 305         skip_test "$name" && return
 306         for gc in nitgc boehm malloc large; do
 307                 prepare_res "$name-$gc.dat" "$gc" "nitc with gc=$gc"
 308                 export NIT_GC_OPTION="$gc"
 309                 run_compiler "nitc" ../src/nitc_3 -O
 310         done
 311
 312         plot "$name.gnu"
 313 }
 314 bench_nitc_gc
 315
 316 function bench_nitc_boost()
 317 {
 318         name="$FUNCNAME"
 319         skip_test "$name" && return
 320         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 321         run_compiler "nitc_slow" ../src/nitc_3
 322         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 323         run_compiler "nitc" ../src/nitc_3 -O
 324
 325         plot "$name.gnu"
 326 }
 327 bench_nitc_boost
 328
 329 function bench_engines()
 330 {
 331         name="$FUNCNAME"
 332         skip_test "$name" && return
 333         prepare_res "$name-nitc.dat" "nitc" "nitc"
 334         run_compiler "nitc" ../src/nitc_3 -O
 335         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 336         run_compiler "nitc-g" ../src/nitc_3 -O --global
 337         prepare_res "$name-nitg.dat" "nitg" "nitg with --global"
 338         run_compiler "nitg" ./nitg --global
 339         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 340         run_compiler "nitg-s" ./nitg --separate
 341         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 342         run_compiler "nitg-e" ./nitg --erasure
 343         plot "$name.gnu"
 344 }
 345 bench_engines
 346
 347 function bench_nitc_vc_nitg-e()
 348 {
 349         name="$FUNCNAME"
 350         skip_test "$name" && return
 351         prepare_res "$name-nitc.dat" "nitc" "nitc"
 352         run_compiler "nitc" ../src/nitc_3 -O
 353         prepare_res "$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
 354         NIT_GC_OPTION="malloc" run_compiler "nitc" ../src/nitc_3 -O
 355         prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
 356         NIT_GC_OPTION="boehm" run_compiler "nitc" ../src/nitc_3 -O
 357         prepare_res "$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
 358         NIT_GC_OPTION="malloc" run_compiler "nitg-e-nc-malloc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
 359         prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
 360         run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
 361         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 362         run_compiler "nitg-e" ./nitg --erasure
 363         plot "$name.gnu"
 364 }
 365 bench_nitc_vc_nitg-e
 366
 367 function bench_nitg-e_gc()
 368 {
 369         name="$FUNCNAME"
 370         skip_test "$name" && return
 371         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 372         NIT_GC_OPTION="malloc" run_compiler "nitg-e-malloc" ./nitg --erasure
 373         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 374         run_compiler "nitg-e" ./nitg --erasure
 375         plot "$name.gnu"
 376 }
 377 bench_nitg-e_gc
 378
 379 function bench_cc_nitg-e()
 380 {
 381         name="$FUNCNAME"
 382         skip_test "$name" && return
 383         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 384                 f=`echo "$o" | cut -f1 -d:`
 385                 o=`echo "$o" | cut -f2 -d:`
 386                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 387                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 388         done
 389         plot "$name.gnu"
 390 }
 391 bench_cc_nitg-e
 392
 393 function bench_policy()
 394 {
 395         name="$FUNCNAME"
 396         skip_test "$name" && return
 397         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 398         run_compiler "nitg-s" ./nitg --separate
 399         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 400         run_compiler "nitg-e" ./nitg --erasure
 401         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 402         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 403         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 404         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 405         plot "$name.gnu"
 406 }
 407 bench_policy
 408
 409 function bench_compilation_time
 410 {
 411         name="$FUNCNAME"
 412         skip_test "$name" && return
 413         prepare_res "$name-nitc.dat" "nitc" "nitc"
 414         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 415                 bench_command `basename "$i" .nit` "" ../src/nitc_3 -O "$i" --no-cc
 416         done
 417         prepare_res "$name-nitg.dat" "nitg" "nitg"
 418         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 419                 bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc
 420         done
 421         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 422         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 423                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 424         done
 425         plot "$name.gnu"
 426 }
 427 bench_compilation_time
 428
 429 if test -n "$died"; then
 430         echo "Some commands failed"
 431         exit 1
 432 fi
 433 exit 0