benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_common.sh
  21 source ./bench_plot.sh
  22
  23 ## CONFIGURATION OPTIONS ##
  24
  25 # Default number of times a command must be run with bench_command
  26 # Can be overrided with 'the option -n'
  27 count=2
  28
  29 ### HELPER FUNCTIONS ##
  30
  31 function die()
  32 {
  33         echo >&2 "error: $*"
  34         died=1
  35 }
  36
  37 # HELPER FOR NIT #
  38
  39 # Run standards benchs on a compiler command
  40 # $1: title
  41 # rest: command to run (executable + options)
  42 function run_compiler()
  43 {
  44         local title=$1
  45         shift
  46         if test "$fast" = truetrue; then
  47                 run_command "$@" ../examples/hello_world.nit -o "hello.$title.bin"
  48                 bench_command "hello" "hello_world" "./hello.$title.bin"
  49         elif test -n "$fast"; then
  50                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  51                 bench_command "nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v --global --no-cc ../src/test_parser.nit
  52                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  53                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
  54                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  55                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
  56                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  57                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
  58         else
  59                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  60                 bench_command "nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v --global --no-cc ../src/nitls.nit
  61                 bench_command "nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v --no-cc --separate ../src/nitc.nit
  62                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  63                 bench_command "nit-queens" "nit queens.nit 8" "./nit.$title.bin" ../lib/ai/examples/queens.nit -q 8
  64                 bench_command "nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ../contrib/nitcc/src/nitcc.nit ../contrib/nitcc/examples/calc.sablecc
  65                 run_command "$@" ../src/nitdoc.nit -o "nitdoc.$title.bin"
  66                 rm -r out 2> /dev/null
  67                 mkdir out 2> /dev/null
  68                 bench_command "nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ../src/nitls.nit -d out
  69                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  70                 bench_command "shoot" "shoot_logic 15" "./shoot.$title.bin" 15
  71                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  72                 bench_command "bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
  73                 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
  74                 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
  75                 run_command "$@" "../lib/ai/examples/queens.nit" -o "queens.$title.bin"
  76                 bench_command "queens" "bench_queens 13" "./queens.$title.bin" 13
  77                 run_command "$@" "../lib/ai/examples/puzzle.nit" -o "puzzle.$title.bin"
  78                 bench_command "puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
  79                 run_command "$@" "markdown/engines/nitmd/nitmd.nit" -o "nitmd.$title.bin"
  80                 bench_command "nitmd" "markdown" "./nitmd.$title.bin" markdown/benches/out/mixed.md 80
  81         fi
  82
  83         rm -r *.bin .nit_compile out 2> /dev/null
  84 }
  85
  86 ## HANDLE OPTIONS ##
  87
  88 function usage()
  89 {
  90         echo "run_bench: [options]* benchname"
  91         echo "  -v: verbose mode"
  92         echo "  -n count: number of execution for each bar (default: $count)"
  93         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
  94         echo "  --fast: Run less and faster tests"
  95         echo "  --html: Generate and HTML output"
  96         echo "  -h: this help"
  97 }
  98
  99 stop=false
 100 while [ "$stop" = false ]; do
 101         case "$1" in
 102                 -v) verbose=true; shift;;
 103                 -h) usage; exit;;
 104                 -n) count="$2"; shift; shift;;
 105                 --dry) dry_run=true; shift;;
 106                 --fast) fast=true$fast; shift;;
 107                 --html) html="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
 108                 *) stop=true
 109         esac
 110 done
 111
 112 xml="bench_engines.xml"
 113 echo "<testsuites><testsuite>" > "$xml"
 114
 115 NOTSKIPED="$*"
 116
 117 if test -z "$NOTSKIPED"; then
 118         usage
 119         echo "List of available benches:"
 120         echo "* all: run all the benches"
 121 fi
 122
 123 ## COMPILE ENGINES
 124
 125 # get the bootstrapped nitc
 126 cp ../bin/nitc .
 127
 128 if test -z "$fast"; then
 129         make -C markdown/benches
 130         make -C ../contrib/nitcc
 131 fi
 132
 133 ## EFFECTIVE BENCHS ##
 134
 135 function bench_steps()
 136 {
 137         name="$FUNCNAME"
 138         skip_test "$name" && return
 139         prepare_res "$name-nitc.dat" "nitc-g" "Various steps of nitc --global"
 140         bench_command "parse" "" ./nitc --global --only-parse ../src/nitc.nit
 141         bench_command "metamodel" "" ./nitc --global --only-metamodel ../src/nitc.nit
 142         bench_command "generate c" "" ./nitc --global --no-cc ../src/nitc.nit
 143         bench_command "full" "" ./nitc --global ../src/nitc.nit -o "nitc_nitc.bin"
 144
 145         prepare_res "$name-nitc-s.dat" "nitc-s" "Various steps of nitc --separate"
 146         bench_command "parse" "" ./nitc --separate --only-parse ../src/nitc.nit
 147         bench_command "metamodel" "" ./nitc --separate --only-metamodel ../src/nitc.nit
 148         bench_command "generate c" "" ./nitc --separate --no-cc ../src/nitc.nit
 149         bench_command "full" "" ./nitc --separate ../src/nitc.nit -o "nitc_nitc-e.bin"
 150
 151         prepare_res "$name-nitc-e.dat" "nitc-e" "Various steps of nitc --erasure"
 152         bench_command "parse" "" ./nitc --erasure --only-parse ../src/nitc.nit
 153         bench_command "metamodel" "" ./nitc --erasure --only-metamodel ../src/nitc.nit
 154         bench_command "generate c" "" ./nitc --erasure --no-cc ../src/nitc.nit
 155         bench_command "full" "" ./nitc --erasure ../src/nitc.nit -o "nitc_nitc-e.bin"
 156
 157         plot "$name.gnu"
 158 }
 159 bench_steps
 160
 161 # $#: options to compare
 162 function bench_nitc-g_options()
 163 {
 164         tag=$1
 165         shift
 166         name="$FUNCNAME-$tag"
 167         skip_test "$name" && return
 168         prepare_res "$name.dat" "no options" "nitc-g without options"
 169         run_compiler "nitc-g" ./nitc --global
 170
 171         if test "$1" = NOALL; then
 172                 shift
 173         elif test -n "$2"; then
 174                 prepare_res "$name-all.dat" "all" "nitc-g with all options $@"
 175                 run_compiler "nitc-g-$tag" ./nitc --global $@
 176         fi
 177
 178         for opt in "$@"; do
 179                 ot=${opt// /+}
 180                 prepare_res "$name$ot.dat" "$opt" "nitc-g with option $opt"
 181                 run_compiler "nitc-g$ot" ./nitc --global $opt
 182         done
 183
 184         plot "$name.gnu"
 185 }
 186 bench_nitc-g_options "slower" --hardening --no-shortcut-range
 187 bench_nitc-g_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 188
 189 function bench_nitc-s_options()
 190 {
 191         tag=$1
 192         shift
 193         name="$FUNCNAME-$tag"
 194         skip_test "$name" && return
 195         prepare_res "$name.dat" "no options" "nitc-s without options"
 196         run_compiler "nitc-s" ./nitc --separate
 197
 198         if test "$1" = NOALL; then
 199                 shift
 200         elif test -n "$2"; then
 201                 prepare_res "$name-all.dat" "all" "nitc-s with all options $@"
 202                 run_compiler "nitc-s-$tag" ./nitc --separate $@
 203         fi
 204
 205         for opt in "$@"; do
 206                 ot=${opt// /+}
 207                 prepare_res "$name-$ot.dat" "$opt" "nitc-s with option $opt"
 208                 run_compiler "nitc-s$ot" ./nitc --separate $opt
 209         done
 210
 211         plot "$name.gnu"
 212 }
 213 bench_nitc-s_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn" "--no-tag-primitives"
 214 bench_nitc-s_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 215 bench_nitc-s_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" ""
 216
 217 function bench_nitc-e_options()
 218 {
 219         tag=$1
 220         shift
 221         name="$FUNCNAME-$tag"
 222         skip_test "$name" && return
 223         prepare_res "$name.dat" "no options" "nitc-e without options"
 224         run_compiler "nitc-e" ./nitc --erasure
 225
 226         if test "$1" = NOALL; then
 227                 shift
 228         elif test -n "$2"; then
 229                 prepare_res "$name-all.dat" "all" "nitc-e with all options $@"
 230                 run_compiler "nitc-e-$tag" ./nitc --erasure $@
 231         fi
 232
 233         for opt in "$@"; do
 234                 ot=${opt// /+}
 235                 prepare_res "$name$ot.dat" "$opt" "nitc-e with option $opt"
 236                 run_compiler "nitc-e$ot" ./nitc --erasure $opt
 237         done
 238
 239         plot "$name.gnu"
 240 }
 241 bench_nitc-e_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern
 242 bench_nitc-e_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast
 243 bench_nitc-e_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta
 244
 245 function bench_engines()
 246 {
 247         name="$FUNCNAME"
 248         skip_test "$name" && return
 249         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 250         run_compiler "nitc-s" ./nitc --separate
 251         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 252         run_compiler "nitc-e" ./nitc --erasure
 253         prepare_res "$name-nitc-sg.dat" "nitc-sg" "nitc with --separate --semi-global"
 254         run_compiler "nitc-sg" ./nitc --separate --semi-global
 255         prepare_res "$name-nitc-eg.dat" "nitc-eg" "nitc with --erasure --semi-global"
 256         run_compiler "nitc-eg" ./nitc --erasure --semi-global
 257         prepare_res "$name-nitc-egt.dat" "nitc-egt" "nitc with --erasure --semi-global --rta"
 258         run_compiler "nitc-egt" ./nitc --erasure --semi-global --rta
 259         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 260         run_compiler "nitc-g" ./nitc --global
 261         plot "$name.gnu"
 262 }
 263 bench_engines
 264
 265 function bench_nitc-e_gc()
 266 {
 267         name="$FUNCNAME"
 268         skip_test "$name" && return
 269         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 270         run_compiler "nitc-e" ./nitc --erasure
 271         prepare_res "$name-nitc-e-malloc.dat" "nitc-e-malloc" "nitc with --erasure and malloc"
 272         NIT_GC_OPTION="malloc" run_compiler "nitc-e-malloc" ./nitc --erasure
 273         prepare_res "$name-nitc-e-large.dat" "nitc-e-large" "nitc with --erasure and large"
 274         NIT_GC_OPTION="large" run_compiler "nitc-e-large" ./nitc --erasure
 275         plot "$name.gnu"
 276 }
 277 bench_nitc-e_gc
 278
 279 function bench_cc_nitc-e()
 280 {
 281         name="$FUNCNAME"
 282         skip_test "$name" && return
 283         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 284                 f=`echo "$o" | cut -f1 -d:`
 285                 o=`echo "$o" | cut -f2 -d:`
 286                 prepare_res "$name-nitc-e-$f.dat" "nitc-e-$f" "nitc with --erasure --make-flags $o"
 287                 run_compiler "nitc-e-$f" ./nitc --erasure --make-flags "$o"
 288         done
 289         plot "$name.gnu"
 290 }
 291 bench_cc_nitc-e
 292
 293 function bench_policy()
 294 {
 295         name="$FUNCNAME"
 296         skip_test "$name" && return
 297         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 298         run_compiler "nitc-s" ./nitc --separate
 299         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 300         run_compiler "nitc-e" ./nitc --erasure
 301         prepare_res "$name-nitc-su.dat" "nitc-su" "nitc with --separate --no-check-covariance"
 302         run_compiler "nitc-su" ./nitc --separate --no-check-covariance
 303         prepare_res "$name-nitc-eu.dat" "nitc-eu" "nitc with --erasure --no-check-covariance --no-check-erasure-cast"
 304         run_compiler "nitc-eu" ./nitc --erasure --no-check-covariance --no-check-erasure-cast
 305         plot "$name.gnu"
 306 }
 307 bench_policy
 308
 309 function bench_nullables()
 310 {
 311         name="$FUNCNAME"
 312         skip_test "$name" && return
 313         prepare_res "$name-nitc.dat" "nitc" "nitc no options"
 314         run_compiler "nitc" ./nitc --separate
 315         prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
 316         run_compiler "nitc" ./nitc --separate --no-check-attr-isset
 317         prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
 318         run_compiler "nitc" ./nitc --separate --no-union-attribute
 319         prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
 320         run_compiler "nitc" ./nitc --separate --no-union-attribute --no-check-attr-isset
 321         plot "$name.gnu"
 322 }
 323 bench_nullables
 324
 325 function bench_compilation_time
 326 {
 327         name="$FUNCNAME"
 328         skip_test "$name" && return
 329         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc --global"
 330         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 331                 bench_command `basename "$i" .nit` "" ./nitc --global "$i" --no-cc
 332         done
 333         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc --separate"
 334         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 335                 bench_command `basename "$i" .nit` "" ./nitc --separate "$i" --no-cc
 336         done
 337         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc --erasure"
 338         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 339                 bench_command `basename "$i" .nit` "" ./nitc --erasure "$i" --no-cc
 340         done
 341         plot "$name.gnu"
 342 }
 343 bench_compilation_time
 344
 345 function bench_linkboost()
 346 {
 347         name="$FUNCNAME"
 348         skip_test "$name" && return
 349         prepare_res "$name-nitc-st.dat" "nitc-st" "nitc with --separate --trampoline-call"
 350         run_compiler "nitc-st" ./nitc --separate --trampoline-call
 351         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 352         run_compiler "nitc-s" ./nitc --separate
 353         prepare_res "$name-nitc-sc.dat" "nitc-sc" "nitc with --separate --colors-are-symbols"
 354         run_compiler "nitc-sc" ./nitc --separate --colors-are-symbols
 355         prepare_res "$name-nitc-sct.dat" "nitc-sct" "nitc with --separate --colors-are-symbols --trampoline-call"
 356         run_compiler "nitc-sct" ./nitc --separate --colors-are-symbols --trampoline-call
 357         prepare_res "$name-nitc-sl.dat" "nitc-sl" "nitc with --separate --link-boost"
 358         run_compiler "nitc-scts" ./nitc --separate --link-boost
 359         prepare_res "$name-nitc-scgc.dat" "nitc-scgc" "nitc with --separate --colors-are-symbols --guard-call"
 360         run_compiler "nitc-scgc" ./nitc --separate --colors-are-symbols --guard-call
 361         prepare_res "$name-nitc-scd.dat" "nitc-scd" "nitc with --separate --colors-are-symbols --direct-call-monomorph0"
 362         run_compiler "nitc-scd" ./nitc --separate --colors-are-symbols --direct-call-monomorph0
 363         plot "$name.gnu"
 364 }
 365 bench_linkboost
 366
 367 function bench_call_monomorph()
 368 {
 369         name="$FUNCNAME"
 370         skip_test "$name" && return
 371         prepare_res "$name-nitc.dat" "nitc" "nitc with --separate"
 372         run_compiler "nitc" ./nitc
 373         prepare_res "$name-nitc-d0.dat" "nitc-d0" "nitc with --separate --direct-call-monomorph0"
 374         run_compiler "nitc-d0" ./nitc --direct-call-monomorph0
 375         prepare_res "$name-nitc-d1.dat" "nitc-d" "nitc with --separate --direct-call-monomorph"
 376         run_compiler "nitc-d1" ./nitc --direct-call-monomorph
 377         prepare_res "$name-nitc-d2.dat" "nitc-d2" "nitc with --separate --direct-call-monomorph2"
 378         run_compiler "nitc-d2" ./nitc --direct-call-monomorph --direct-call-monomorph0
 379         plot "$name.gnu"
 380 }
 381 bench_call_monomorph
 382
 383 if test -n "$html"; then
 384         echo >>"$html" "</body></html>"
 385 fi
 386
 387 echo >>"$xml" "</testsuite></testsuites>"
 388
 389 if test -n "$died"; then
 390         echo "Some commands failed"
 391         exit 1
 392 fi
 393 exit 0