benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_common.sh
  21 source ./bench_plot.sh
  22
  23 ## CONFIGURATION OPTIONS ##
  24
  25 # Default number of times a command must be run with bench_command
  26 # Can be overrided with 'the option -n'
  27 count=2
  28
  29 ### HELPER FUNCTIONS ##
  30
  31 function die()
  32 {
  33         echo >&2 "error: $*"
  34         died=1
  35 }
  36
  37 # HELPER FOR NIT #
  38
  39 # Run standards benchs on a compiler command
  40 # $1: title
  41 # rest: command to run (executable + options)
  42 function run_compiler()
  43 {
  44         local title=$1
  45         shift
  46         if test -n "$fast"; then
  47                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  48                 bench_command "nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v --global --no-cc ../src/test_parser.nit
  49                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  50                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
  51                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  52                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
  53                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  54                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
  55         else
  56                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  57                 bench_command "nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v --global --no-cc ../src/nitls.nit
  58                 bench_command "nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v --no-cc --separate ../src/nitc.nit
  59                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  60                 bench_command "nit-queens" "nit queens.nit 8" "./nit.$title.bin" ../lib/ai/examples/queens.nit -q 8
  61                 bench_command "nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ../contrib/nitcc/src/nitcc.nit ../contrib/nitcc/examples/calc.sablecc
  62                 run_command "$@" ../src/nitdoc.nit -o "nitdoc.$title.bin"
  63                 rm -r out 2> /dev/null
  64                 mkdir out 2> /dev/null
  65                 bench_command "nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ../src/nitls.nit -d out
  66                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  67                 bench_command "shoot" "shoot_logic 15" "./shoot.$title.bin" 15
  68                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  69                 bench_command "bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
  70                 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
  71                 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
  72                 run_command "$@" "../lib/ai/examples/queens.nit" -o "queens.$title.bin"
  73                 bench_command "queens" "bench_queens 13" "./queens.$title.bin" 13
  74                 run_command "$@" "../lib/ai/examples/puzzle.nit" -o "puzzle.$title.bin"
  75                 bench_command "puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
  76                 run_command "$@" "markdown/engines/nitmd/nitmd.nit" -o "nitmd.$title.bin"
  77                 bench_command "nitmd" "markdown" "./nitmd.$title.bin" markdown/benches/out/mixed.md 80
  78         fi
  79
  80         rm -r *.bin .nit_compile out
  81 }
  82
  83 ## HANDLE OPTIONS ##
  84
  85 function usage()
  86 {
  87         echo "run_bench: [options]* benchname"
  88         echo "  -v: verbose mode"
  89         echo "  -n count: number of execution for each bar (default: $count)"
  90         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
  91         echo "  --fast: Run less and faster tests"
  92         echo "  --html: Generate and HTML output"
  93         echo "  -h: this help"
  94 }
  95
  96 stop=false
  97 while [ "$stop" = false ]; do
  98         case "$1" in
  99                 -v) verbose=true; shift;;
 100                 -h) usage; exit;;
 101                 -n) count="$2"; shift; shift;;
 102                 --dry) dry_run=true; shift;;
 103                 --fast) fast=true; shift;;
 104                 --html) html="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
 105                 *) stop=true
 106         esac
 107 done
 108
 109 xml="bench_engines.xml"
 110 echo "<testsuites><testsuite>" > "$xml"
 111
 112 NOTSKIPED="$*"
 113
 114 if test -z "$NOTSKIPED"; then
 115         usage
 116         echo "List of available benches:"
 117         echo "* all: run all the benches"
 118 fi
 119
 120 ## COMPILE ENGINES
 121
 122 # get the bootstrapped nitc
 123 cp ../bin/nitc .
 124
 125 if test -z "$fast"; then
 126         make -C markdown/benches
 127         make -C ../contrib/nitcc
 128 fi
 129
 130 ## EFFECTIVE BENCHS ##
 131
 132 function bench_steps()
 133 {
 134         name="$FUNCNAME"
 135         skip_test "$name" && return
 136         prepare_res "$name-nitc.dat" "nitc-g" "Various steps of nitc --global"
 137         bench_command "parse" "" ./nitc --global --only-parse ../src/nitc.nit
 138         bench_command "metamodel" "" ./nitc --global --only-metamodel ../src/nitc.nit
 139         bench_command "generate c" "" ./nitc --global --no-cc ../src/nitc.nit
 140         bench_command "full" "" ./nitc --global ../src/nitc.nit -o "nitc_nitc.bin"
 141
 142         prepare_res "$name-nitc-s.dat" "nitc-s" "Various steps of nitc --separate"
 143         bench_command "parse" "" ./nitc --separate --only-parse ../src/nitc.nit
 144         bench_command "metamodel" "" ./nitc --separate --only-metamodel ../src/nitc.nit
 145         bench_command "generate c" "" ./nitc --separate --no-cc ../src/nitc.nit
 146         bench_command "full" "" ./nitc --separate ../src/nitc.nit -o "nitc_nitc-e.bin"
 147
 148         prepare_res "$name-nitc-e.dat" "nitc-e" "Various steps of nitc --erasure"
 149         bench_command "parse" "" ./nitc --erasure --only-parse ../src/nitc.nit
 150         bench_command "metamodel" "" ./nitc --erasure --only-metamodel ../src/nitc.nit
 151         bench_command "generate c" "" ./nitc --erasure --no-cc ../src/nitc.nit
 152         bench_command "full" "" ./nitc --erasure ../src/nitc.nit -o "nitc_nitc-e.bin"
 153
 154         plot "$name.gnu"
 155 }
 156 bench_steps
 157
 158 # $#: options to compare
 159 function bench_nitc-g_options()
 160 {
 161         tag=$1
 162         shift
 163         name="$FUNCNAME-$tag"
 164         skip_test "$name" && return
 165         prepare_res "$name.dat" "no options" "nitc-g without options"
 166         run_compiler "nitc-g" ./nitc --global
 167
 168         if test "$1" = NOALL; then
 169                 shift
 170         elif test -n "$2"; then
 171                 prepare_res "$name-all.dat" "all" "nitc-g with all options $@"
 172                 run_compiler "nitc-g-$tag" ./nitc --global $@
 173         fi
 174
 175         for opt in "$@"; do
 176                 ot=${opt// /+}
 177                 prepare_res "$name$ot.dat" "$opt" "nitc-g with option $opt"
 178                 run_compiler "nitc-g$ot" ./nitc --global $opt
 179         done
 180
 181         plot "$name.gnu"
 182 }
 183 bench_nitc-g_options "slower" --hardening --no-shortcut-range
 184 bench_nitc-g_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 185
 186 function bench_nitc-s_options()
 187 {
 188         tag=$1
 189         shift
 190         name="$FUNCNAME-$tag"
 191         skip_test "$name" && return
 192         prepare_res "$name.dat" "no options" "nitc-s without options"
 193         run_compiler "nitc-s" ./nitc --separate
 194
 195         if test "$1" = NOALL; then
 196                 shift
 197         elif test -n "$2"; then
 198                 prepare_res "$name-all.dat" "all" "nitc-s with all options $@"
 199                 run_compiler "nitc-s-$tag" ./nitc --separate $@
 200         fi
 201
 202         for opt in "$@"; do
 203                 ot=${opt// /+}
 204                 prepare_res "$name-$ot.dat" "$opt" "nitc-s with option $opt"
 205                 run_compiler "nitc-s$ot" ./nitc --separate $opt
 206         done
 207
 208         plot "$name.gnu"
 209 }
 210 bench_nitc-s_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn" "--no-tag-primitives"
 211 bench_nitc-s_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 212 bench_nitc-s_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" ""
 213
 214 function bench_nitc-e_options()
 215 {
 216         tag=$1
 217         shift
 218         name="$FUNCNAME-$tag"
 219         skip_test "$name" && return
 220         prepare_res "$name.dat" "no options" "nitc-e without options"
 221         run_compiler "nitc-e" ./nitc --erasure
 222
 223         if test "$1" = NOALL; then
 224                 shift
 225         elif test -n "$2"; then
 226                 prepare_res "$name-all.dat" "all" "nitc-e with all options $@"
 227                 run_compiler "nitc-e-$tag" ./nitc --erasure $@
 228         fi
 229
 230         for opt in "$@"; do
 231                 ot=${opt// /+}
 232                 prepare_res "$name$ot.dat" "$opt" "nitc-e with option $opt"
 233                 run_compiler "nitc-e$ot" ./nitc --erasure $opt
 234         done
 235
 236         plot "$name.gnu"
 237 }
 238 bench_nitc-e_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern
 239 bench_nitc-e_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast
 240 bench_nitc-e_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta
 241
 242 function bench_engines()
 243 {
 244         name="$FUNCNAME"
 245         skip_test "$name" && return
 246         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 247         run_compiler "nitc-s" ./nitc --separate
 248         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 249         run_compiler "nitc-e" ./nitc --erasure
 250         prepare_res "$name-nitc-sg.dat" "nitc-sg" "nitc with --separate --semi-global"
 251         run_compiler "nitc-sg" ./nitc --separate --semi-global
 252         prepare_res "$name-nitc-eg.dat" "nitc-eg" "nitc with --erasure --semi-global"
 253         run_compiler "nitc-eg" ./nitc --erasure --semi-global
 254         prepare_res "$name-nitc-egt.dat" "nitc-egt" "nitc with --erasure --semi-global --rta"
 255         run_compiler "nitc-egt" ./nitc --erasure --semi-global --rta
 256         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global"
 257         run_compiler "nitc-g" ./nitc --global
 258         plot "$name.gnu"
 259 }
 260 bench_engines
 261
 262 function bench_nitc-e_gc()
 263 {
 264         name="$FUNCNAME"
 265         skip_test "$name" && return
 266         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 267         run_compiler "nitc-e" ./nitc --erasure
 268         prepare_res "$name-nitc-e-malloc.dat" "nitc-e-malloc" "nitc with --erasure and malloc"
 269         NIT_GC_OPTION="malloc" run_compiler "nitc-e-malloc" ./nitc --erasure
 270         prepare_res "$name-nitc-e-large.dat" "nitc-e-large" "nitc with --erasure and large"
 271         NIT_GC_OPTION="large" run_compiler "nitc-e-large" ./nitc --erasure
 272         plot "$name.gnu"
 273 }
 274 bench_nitc-e_gc
 275
 276 function bench_cc_nitc-e()
 277 {
 278         name="$FUNCNAME"
 279         skip_test "$name" && return
 280         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 281                 f=`echo "$o" | cut -f1 -d:`
 282                 o=`echo "$o" | cut -f2 -d:`
 283                 prepare_res "$name-nitc-e-$f.dat" "nitc-e-$f" "nitc with --erasure --make-flags $o"
 284                 run_compiler "nitc-e-$f" ./nitc --erasure --make-flags "$o"
 285         done
 286         plot "$name.gnu"
 287 }
 288 bench_cc_nitc-e
 289
 290 function bench_policy()
 291 {
 292         name="$FUNCNAME"
 293         skip_test "$name" && return
 294         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 295         run_compiler "nitc-s" ./nitc --separate
 296         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 297         run_compiler "nitc-e" ./nitc --erasure
 298         prepare_res "$name-nitc-su.dat" "nitc-su" "nitc with --separate --no-check-covariance"
 299         run_compiler "nitc-su" ./nitc --separate --no-check-covariance
 300         prepare_res "$name-nitc-eu.dat" "nitc-eu" "nitc with --erasure --no-check-covariance --no-check-erasure-cast"
 301         run_compiler "nitc-eu" ./nitc --erasure --no-check-covariance --no-check-erasure-cast
 302         plot "$name.gnu"
 303 }
 304 bench_policy
 305
 306 function bench_nullables()
 307 {
 308         name="$FUNCNAME"
 309         skip_test "$name" && return
 310         prepare_res "$name-nitc.dat" "nitc" "nitc no options"
 311         run_compiler "nitc" ./nitc --separate
 312         prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
 313         run_compiler "nitc" ./nitc --separate --no-check-attr-isset
 314         prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
 315         run_compiler "nitc" ./nitc --separate --no-union-attribute
 316         prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
 317         run_compiler "nitc" ./nitc --separate --no-union-attribute --no-check-attr-isset
 318         plot "$name.gnu"
 319 }
 320 bench_nullables
 321
 322 function bench_compilation_time
 323 {
 324         name="$FUNCNAME"
 325         skip_test "$name" && return
 326         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc --global"
 327         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 328                 bench_command `basename "$i" .nit` "" ./nitc --global "$i" --no-cc
 329         done
 330         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc --separate"
 331         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 332                 bench_command `basename "$i" .nit` "" ./nitc --separate "$i" --no-cc
 333         done
 334         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc --erasure"
 335         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 336                 bench_command `basename "$i" .nit` "" ./nitc --erasure "$i" --no-cc
 337         done
 338         plot "$name.gnu"
 339 }
 340 bench_compilation_time
 341
 342 function bench_linkboost()
 343 {
 344         name="$FUNCNAME"
 345         skip_test "$name" && return
 346         prepare_res "$name-nitc-st.dat" "nitc-st" "nitc with --separate --trampoline-call"
 347         run_compiler "nitc-st" ./nitc --separate --trampoline-call
 348         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate"
 349         run_compiler "nitc-s" ./nitc --separate
 350         prepare_res "$name-nitc-sc.dat" "nitc-sc" "nitc with --separate --colors-are-symbols"
 351         run_compiler "nitc-sc" ./nitc --separate --colors-are-symbols
 352         prepare_res "$name-nitc-sct.dat" "nitc-sct" "nitc with --separate --colors-are-symbols --trampoline-call"
 353         run_compiler "nitc-sct" ./nitc --separate --colors-are-symbols --trampoline-call
 354         prepare_res "$name-nitc-sl.dat" "nitc-sl" "nitc with --separate --link-boost"
 355         run_compiler "nitc-scts" ./nitc --separate --link-boost
 356         prepare_res "$name-nitc-scgc.dat" "nitc-scgc" "nitc with --separate --colors-are-symbols --guard-call"
 357         run_compiler "nitc-scgc" ./nitc --separate --colors-are-symbols --guard-call
 358         prepare_res "$name-nitc-scd.dat" "nitc-scd" "nitc with --separate --colors-are-symbols --direct-call-monomorph0"
 359         run_compiler "nitc-scd" ./nitc --separate --colors-are-symbols --direct-call-monomorph0
 360         plot "$name.gnu"
 361 }
 362 bench_linkboost
 363
 364 function bench_call_monomorph()
 365 {
 366         name="$FUNCNAME"
 367         skip_test "$name" && return
 368         prepare_res "$name-nitc.dat" "nitc" "nitc with --separate"
 369         run_compiler "nitc" ./nitc
 370         prepare_res "$name-nitc-d0.dat" "nitc-d0" "nitc with --separate --direct-call-monomorph0"
 371         run_compiler "nitc-d0" ./nitc --direct-call-monomorph0
 372         prepare_res "$name-nitc-d1.dat" "nitc-d" "nitc with --separate --direct-call-monomorph"
 373         run_compiler "nitc-d1" ./nitc --direct-call-monomorph
 374         prepare_res "$name-nitc-d2.dat" "nitc-d2" "nitc with --separate --direct-call-monomorph2"
 375         run_compiler "nitc-d2" ./nitc --direct-call-monomorph --direct-call-monomorph0
 376         plot "$name.gnu"
 377 }
 378 bench_call_monomorph
 379
 380 if test -n "$html"; then
 381         echo >>"$html" "</body></html>"
 382 fi
 383
 384 echo >>"$xml" "</testsuite></testsuites>"
 385
 386 if test -n "$died"; then
 387         echo "Some commands failed"
 388         exit 1
 389 fi
 390 exit 0