benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 ### HELPER FUNCTIONS ##
  29
  30 function die()
  31 {
  32         echo >&2 "error: $*"
  33         died=1
  34 }
  35
  36 # Run a single command multiple time and store the execution times
  37 # in the current $res file.
  38 #
  39 #  $1: title of the command
  40 #  $2: long desription of the command
  41 #  rest: the command to execute
  42 function bench_command()
  43 {
  44         if [ "$dry_run" = "true" ]; then return; fi
  45         local title="$1"
  46         local desc="$2"
  47         shift
  48         shift
  49         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  50         timeout="time.out"
  51         echo "$title" > "$timeout"
  52         echo "# $desc" >> "$timeout"
  53         echo "\$ $@" >> "$timeout"
  54         echo
  55         echo "** [$title] $desc **"
  56         echo " $ $@"
  57
  58         # Execute the commands $count times
  59         for i in `seq 1 "$count"`; do
  60                 /usr/bin/time -f "%U" -o "$timeout" -a "$@" > $outputopts 2>&1 || die "$1: failed"
  61                 echo -n "$i. "
  62                 tail -n 1 "$timeout"
  63         done
  64
  65         line=`compute_stats "$timeout"`
  66         echo "$line ($res)"
  67         echo $line >> "$res"
  68 }
  69
  70 # Run a simble command witout storing the execution time
  71 # Used to display command on verbose and skip long executions when dry_run is given
  72 # $@ command to execute
  73 function run_command()
  74 {
  75         if [ "$dry_run" = "true" ]; then return; fi
  76         echo " $ $@"
  77         "$@" || die "$@: failed"
  78 }
  79
  80 # Check if the test should be skiped according to its name
  81 # $1: name of the test
  82 # $2: description of the test
  83 # $NOTSKIPED: arguments
  84 function skip_test()
  85 {
  86         if test -z "$NOTSKIPED"; then
  87                 echo "* $1"
  88                 return 0
  89         fi
  90         if test "$NOTSKIPED" = "all"; then
  91                 : # Execute anyway
  92         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  93                 : # Found one to execute
  94         else
  95                 return 0
  96         fi
  97         echo "*"
  98         echo "* $1 *****"
  99         echo "*"
 100         return 1
 101 }
 102
 103 # HELPER FOR NIT #
 104
 105 # Run standards benchs on a compiler command
 106 # $1: title
 107 # rest: command to run (executable + options)
 108 function run_compiler()
 109 {
 110         local title=$1
 111         shift
 112         if test -n "$fast"; then
 113                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 114                 bench_command "nitg" "nitg ../src/test_parser.nit" "./nitg.$title.bin" -v --no-cc ../src/test_parser.nit
 115                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 116                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
 117                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 118                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
 119                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 120                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
 121         else
 122                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
 123                 bench_command "nitg" "nitg --no-cc ../src/nitstats.nit" "./nitg.$title.bin" -v --no-cc ../src/nitstats.nit
 124                 bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit
 125                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
 126                 bench_command "nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/rapid_type_analysis.nit
 127                 run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
 128                 bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
 129                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
 130                 bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
 131         fi
 132 }
 133
 134 ## HANDLE OPTIONS ##
 135
 136 function usage()
 137 {
 138         echo "run_bench: [options]* benchname"
 139         echo "  -v: verbose mode"
 140         echo "  -n count: number of execution for each bar (default: $count)"
 141         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 142         echo "  --fast: Run less and faster tests"
 143         echo "  -h: this help"
 144 }
 145
 146 stop=false
 147 while [ "$stop" = false ]; do
 148         case "$1" in
 149                 -v) verbose=true; shift;;
 150                 -h) usage; exit;;
 151                 -n) count="$2"; shift; shift;;
 152                 --dry) dry_run=true; shift;;
 153                 --fast) fast=true; shift;;
 154                 *) stop=true
 155         esac
 156 done
 157
 158 NOTSKIPED="$*"
 159
 160 if test -z "$NOTSKIPED"; then
 161         usage
 162         echo "List of available benches:"
 163         echo "* all: run all the benches"
 164 fi
 165
 166 ## COMPILE ENGINES
 167
 168 test -f ../src/nitc_3 || ../src/ncall.sh -O
 169 test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v
 170
 171 ## EFFECTIVE BENCHS ##
 172
 173 function bench_nitg_bootstrap()
 174 {
 175         name="$FUNCNAME"
 176         skip_test "$name" && return
 177         prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc"
 178         rm nit?_nit*
 179         cp ../src/nitc_3 ./nitc_nitc.bin
 180         bench_command "c/c c" "nitc_nitc ../src/nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O ../src/nitc.nit -o nitc_nitc.bin
 181         bench_command "c/c g" "nitc_nitc ../src/nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O ../src/nitg.nit -o nitg_nitc.bin
 182         bench_command "g/c g" "nitg_nitc ../src/nitg.nit -> nitg_nitg" ./nitg_nitc.bin ../src/nitg.nit -o nitg_nitg.bin
 183         bench_command "g/g g" "nitg_nitg ../src/nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin ../src/nitg.nit -o nitg_nitg.bin
 184
 185         plot "$name.gnu"
 186 }
 187 bench_nitg_bootstrap
 188
 189 function bench_steps()
 190 {
 191         name="$FUNCNAME"
 192         skip_test "$name" && return
 193         prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc"
 194         bench_command "parse" "" ../src/nitc_3 --only-parse ../src/nitg.nit
 195         bench_command "metamodel" "" ../src/nitc_3 --only-metamodel ../src/nitg.nit
 196         bench_command "generate c" "" ../src/nitc_3 --no-cc ../src/nitg.nit
 197         bench_command "full" "" ../src/nitc_3 -O ../src/nitg.nit -o "nitg_nitg.bin"
 198
 199         prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
 200         bench_command "parse" "" ../src/nitc_3 --global --only-parse ../src/nitg.nit
 201         bench_command "metamodel" "" ../src/nitc_3 --global --only-metamodel ../src/nitg.nit
 202         bench_command "generate c" "" ../src/nitc_3 --global --no-cc ../src/nitg.nit
 203         bench_command "full" "" ../src/nitc_3 -O --global ../src/nitg.nit -o "nitg_nitc-g.bin"
 204
 205         prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg"
 206         bench_command "parse" "" ./nitg --only-parse ../src/nitg.nit
 207         bench_command "metamodel" "" ./nitg --only-metamodel ../src/nitg.nit
 208         bench_command "generate c" "" ./nitg --no-cc ../src/nitg.nit
 209         bench_command "full" "" ./nitg ../src/nitg.nit -o "nitg_nitg.bin"
 210
 211         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 212         bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit
 213         bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit
 214         bench_command "generate c" "" ./nitg --erasure --no-cc ../src/nitg.nit
 215         bench_command "full" "" ./nitg --erasure ../src/nitg.nit -o "nitg_nitg-e.bin"
 216
 217         plot "$name.gnu"
 218 }
 219 bench_steps
 220
 221 # $#: options to compare
 222 function bench_nitg_options()
 223 {
 224         tag=$1
 225         shift
 226         name="$FUNCNAME-$tag"
 227         skip_test "$name" && return
 228         prepare_res "$name.dat" "no options" "nitg without options"
 229         run_compiler "nitg" ./nitg
 230
 231         if test -n "$2"; then
 232                 prepare_res "$name-all.dat" "all" "nitg with all options $@"
 233                 run_compiler "nitg-$tag" ./nitg $@
 234         fi
 235
 236         for opt in "$@"; do
 237                 prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 238                 run_compiler "nitg$opt" ./nitg $opt
 239         done
 240
 241         plot "$name.gnu"
 242 }
 243 bench_nitg_options "hardening" --hardening
 244 bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 245
 246 function bench_nitg-s_options()
 247 {
 248         tag=$1
 249         shift
 250         name="$FUNCNAME-$tag"
 251         skip_test "$name" && return
 252         prepare_res "$name.dat" "no options" "nitg-s without options"
 253         run_compiler "nitg-s" ./nitg --separate
 254
 255         if test -n "$2"; then
 256                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 257                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 258         fi
 259
 260         for opt in "$@"; do
 261                 prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt"
 262                 run_compiler "nitg-s$opt" ./nitg --separate $opt
 263         done
 264
 265         plot "$name.gnu"
 266 }
 267 bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree --no-union-attribute --no-shortcut-equal --no-shortcut-range
 268 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
 269 bench_nitg-s_options "faster" --inline-coloring-numbers
 270 bench_nitg-s_options "typing" --bm-typing --phand-typing --phmod-typing
 271
 272 function bench_nitg-e_options()
 273 {
 274         tag=$1
 275         shift
 276         name="$FUNCNAME-$tag"
 277         skip_test "$name" && return
 278         prepare_res "$name.dat" "no options" "nitg-e without options"
 279         run_compiler "nitg-e" ./nitg --erasure
 280
 281         if test -n "$2"; then
 282                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 283                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 284         fi
 285
 286         for opt in "$@"; do
 287                 prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
 288                 run_compiler "nitg-e$opt" ./nitg --erasure $opt
 289         done
 290
 291         plot "$name.gnu"
 292 }
 293 bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
 294 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 295 bench_nitg-e_options "faster" --inline-coloring-numbers
 296 bench_nitg-e_options "typing" --bm-typing --phand-typing --phmod-typing
 297
 298 function bench_nitc_gc()
 299 {
 300         name="$FUNCNAME"
 301         skip_test "$name" && return
 302         for gc in nitgc boehm malloc large; do
 303                 prepare_res "$name-$gc.dat" "$gc" "nitc with gc=$gc"
 304                 export NIT_GC_OPTION="$gc"
 305                 run_compiler "nitc" ../src/nitc_3 -O
 306         done
 307
 308         plot "$name.gnu"
 309 }
 310 bench_nitc_gc
 311
 312 function bench_nitc_boost()
 313 {
 314         name="$FUNCNAME"
 315         skip_test "$name" && return
 316         prepare_res "$name-slow.dat" "no -O" "nitc without -O"
 317         run_compiler "nitc_slow" ../src/nitc_3
 318         prepare_res "$name-fast.dat" "-O" "nitc with -O"
 319         run_compiler "nitc" ../src/nitc_3 -O
 320
 321         plot "$name.gnu"
 322 }
 323 bench_nitc_boost
 324
 325 function bench_engines()
 326 {
 327         name="$FUNCNAME"
 328         skip_test "$name" && return
 329         prepare_res "$name-nitc.dat" "nitc" "nitc"
 330         run_compiler "nitc" ../src/nitc_3 -O
 331         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 332         run_compiler "nitc-g" ../src/nitc_3 -O --global
 333         prepare_res "$name-nitg.dat" "nitg" "nitg"
 334         run_compiler "nitg" ./nitg
 335         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 336         run_compiler "nitg-s" ./nitg --separate
 337         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 338         run_compiler "nitg-e" ./nitg --erasure
 339         plot "$name.gnu"
 340 }
 341 bench_engines
 342
 343 function bench_nitc_vc_nitg-e()
 344 {
 345         name="$FUNCNAME"
 346         skip_test "$name" && return
 347         prepare_res "$name-nitc.dat" "nitc" "nitc"
 348         run_compiler "nitc" ../src/nitc_3 -O
 349         prepare_res "$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
 350         NIT_GC_OPTION="malloc" run_compiler "nitc" ../src/nitc_3 -O
 351         prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
 352         NIT_GC_OPTION="boehm" run_compiler "nitc" ../src/nitc_3 -O
 353         prepare_res "$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
 354         run_compiler "nitg-e-nc-malloc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast --make-flags "CFLAGS=\"-O2 -DNOBOEHM\""
 355         prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
 356         run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
 357         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 358         run_compiler "nitg-e" ./nitg --erasure
 359         plot "$name.gnu"
 360 }
 361 bench_nitc_vc_nitg-e
 362
 363 function bench_nitg-e_gc()
 364 {
 365         name="$FUNCNAME"
 366         skip_test "$name" && return
 367         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 368         run_compiler "nitg-e-malloc" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 369         prepare_res "$name-nitg-e-noatomic.dat" "nitg-e-noatomic" "nitg with --erasure and no atomic"
 370         run_compiler "nitg-e-noatomic" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM_ATOMIC\""
 371         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 372         run_compiler "nitg-e" ./nitg --erasure
 373         plot "$name.gnu"
 374 }
 375 bench_nitg-e_gc
 376
 377 function bench_cc_nitg-e()
 378 {
 379         name="$FUNCNAME"
 380         skip_test "$name" && return
 381         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 382                 f=`echo "$o" | cut -f1 -d:`
 383                 o=`echo "$o" | cut -f2 -d:`
 384                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 385                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 386         done
 387         plot "$name.gnu"
 388 }
 389 bench_cc_nitg-e
 390
 391 function bench_policy()
 392 {
 393         name="$FUNCNAME"
 394         skip_test "$name" && return
 395         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 396         run_compiler "nitg-s" ./nitg --separate
 397         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 398         run_compiler "nitg-e" ./nitg --erasure
 399         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 400         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 401         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 402         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 403         plot "$name.gnu"
 404 }
 405 bench_policy
 406
 407 function bench_compilation_time
 408 {
 409         name="$FUNCNAME"
 410         skip_test "$name" && return
 411         prepare_res "$name-nitc.dat" "nitc" "nitc"
 412         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 413                 bench_command `basename "$i" .nit` "" ../src/nitc_3 -O "$i" --no-cc
 414         done
 415         prepare_res "$name-nitg.dat" "nitg" "nitg"
 416         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 417                 bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 418         done
 419         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 420         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 421                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 422         done
 423         plot "$name.gnu"
 424 }
 425 bench_compilation_time
 426
 427 if test -n "$died"; then
 428         echo "Some commands failed"
 429         exit 1
 430 fi
 431 exit 0