benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_common.sh
  21 source ./bench_plot.sh
  22
  23 ## CONFIGURATION OPTIONS ##
  24
  25 # Default number of times a command must be run with bench_command
  26 # Can be overrided with 'the option -n'
  27 count=2
  28
  29 # HELPER FOR NIT #
  30
  31 # Run standards benchs on a compiler command
  32 # $1: title
  33 # rest: command to run (executable + options)
  34 function run_compiler()
  35 {
  36         local title=$1
  37         shift
  38         if test "$fast" = truetrue; then
  39                 run_command "$@" ../examples/hello_world.nit -o "hello.$title.bin"
  40                 bench_command "hello" "hello_world" "./hello.$title.bin"
  41         elif test -n "$fast"; then
  42                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  43                 bench_command "nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v --global --no-cc ../src/test_parser.nit
  44                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  45                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
  46                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  47                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
  48                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  49                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
  50         else
  51                 rm -r out 2> /dev/null
  52                 mkdir out 2> /dev/null
  53                 run_command "$@" ../src/nitc.nit -o "nitc.$title.bin"
  54                 bench_command "nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v --global --no-cc ../src/nitls.nit
  55                 bench_command "nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v --no-cc --separate ../src/nitc.nit
  56                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  57                 bench_command "nit-queens" "nit queens.nit 8" "./nit.$title.bin" ../lib/ai/examples/queens.nit -q 8
  58                 bench_command "nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ../contrib/nitcc/src/nitcc.nit ../contrib/nitcc/examples/calc.sablecc
  59                 rm calc* 2> /dev/null # remove generated cruft
  60                 run_command "$@" ../src/nitdoc.nit -o "nitdoc.$title.bin"
  61                 bench_command "nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ../src/nitls.nit -d out
  62                 run_command "$@" ../src/nitlight.nit -o "nitlight.$title.bin"
  63                 bench_command "nitlight" "nitlight ../lib/[a-f]*/" "./nitlight.$title.bin" ../lib/[a-f]*/ -d out
  64                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  65                 bench_command "shoot" "shoot_logic 15" "./shoot.$title.bin" 15
  66                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  67                 bench_command "bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
  68                 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
  69                 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
  70                 run_command "$@" "../lib/ai/examples/queens.nit" -o "queens.$title.bin"
  71                 bench_command "queens" "bench_queens 13" "./queens.$title.bin" 13
  72                 run_command "$@" "../lib/ai/examples/puzzle.nit" -o "puzzle.$title.bin"
  73                 bench_command "puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
  74                 run_command "$@" "markdown/engines/nitmd/nitmd.nit" -o "nitmd.$title.bin"
  75                 bench_command "nitmd" "markdown" "./nitmd.$title.bin" markdown/benches/out/mixed.md 80
  76                 run_command "$@" ../contrib/jwrapper/src/jwrapper.nit -o "jwrapper.$title.bin"
  77                 bench_command "jwrapper" "jwrapper ant.jar" "./jwrapper.$title.bin" /usr/share/java/ant.jar -o out/ant_jar.nit
  78                 rm -r tmp 2> /dev/null # remove jwrapper output directory
  79         fi
  80
  81         rm -r *.bin out 2> /dev/null
  82 }
  83
  84 ## HANDLE OPTIONS ##
  85
  86 function usage()
  87 {
  88         echo "run_bench: [options]* benchname"
  89         echo "  -v: verbose mode"
  90         echo "  -n count: number of execution for each bar (default: $count)"
  91         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
  92         echo "  --fast: Run less and faster tests"
  93         echo "  -h: this help"
  94 }
  95
  96 stop=false
  97 while [ "$stop" = false ]; do
  98         case "$1" in
  99                 -v) verbose=true; shift;;
 100                 -h) usage; exit;;
 101                 -n) count="$2"; shift; shift;;
 102                 --dry) dry_run=true; shift;;
 103                 --fast) fast=true$fast; shift;;
 104                 --html) shift;; # Deprecated
 105                 *) stop=true
 106         esac
 107 done
 108
 109 html="index.html"
 110 echo >"$html" "<html><head></head><body>"
 111
 112 xml="bench_engines.xml"
 113 echo "<testsuites><testsuite>" > "$xml"
 114
 115 NOTSKIPED="$*"
 116
 117 if test -z "$NOTSKIPED"; then
 118         usage
 119         echo "List of available benches:"
 120         echo "* all: run all the benches"
 121 fi
 122
 123 ## COMPILE ENGINES
 124
 125 # get the bootstrapped nitc
 126 cp ../bin/nitc .
 127
 128 if test -z "$fast"; then
 129         make -C markdown/benches
 130         make -C ../contrib/nitcc
 131         make pre-build -C ../contrib/jwrapper
 132 fi
 133
 134 ## EFFECTIVE BENCHS ##
 135
 136 function bench_steps()
 137 {
 138         name="$FUNCNAME"
 139         skip_test "$name" && return
 140         prepare_res "$name-nitc.dat" "nitc-g" "Various steps of nitc --global"
 141         bench_command "parse" "" ./nitc --global --only-parse ../src/nitc.nit
 142         bench_command "metamodel" "" ./nitc --global --only-metamodel ../src/nitc.nit
 143         bench_command "generate c" "" ./nitc --global --no-cc ../src/nitc.nit
 144         bench_command "full" "" ./nitc --global ../src/nitc.nit -o "nitc_nitc.bin"
 145
 146         prepare_res "$name-nitc-s.dat" "nitc-s" "Various steps of nitc --separate"
 147         bench_command "parse" "" ./nitc --separate --only-parse ../src/nitc.nit
 148         bench_command "metamodel" "" ./nitc --separate --only-metamodel ../src/nitc.nit
 149         bench_command "generate c" "" ./nitc --separate --no-cc ../src/nitc.nit
 150         bench_command "full" "" ./nitc --separate ../src/nitc.nit -o "nitc_nitc-e.bin"
 151
 152         prepare_res "$name-nitc-e.dat" "nitc-e" "Various steps of nitc --erasure"
 153         bench_command "parse" "" ./nitc --erasure --only-parse ../src/nitc.nit
 154         bench_command "metamodel" "" ./nitc --erasure --only-metamodel ../src/nitc.nit
 155         bench_command "generate c" "" ./nitc --erasure --no-cc ../src/nitc.nit
 156         bench_command "full" "" ./nitc --erasure ../src/nitc.nit -o "nitc_nitc-e.bin"
 157
 158         plot "$name.gnu"
 159 }
 160 bench_steps
 161
 162 # Simple script to compare various options on nitc
 163 #
 164 # usage: *name* *common* [NOALL] *options*...
 165 #
 166 # * *name*, the name of the bench
 167 # * *common*, options to use on each case
 168 # * NOALL, optional flag to avoid a last case including all additional options
 169 # * *options*, sequences of options, one for each case
 170 #
 171 # Example: `bench_nitc_options "foo" "-a -b" -c "-d -e"` generates 4 cases:
 172 #
 173 # * only *common*: `nitc -a -b`
 174 # * *common* and first *options*: `nitc -a -b -c`
 175 # * *common* and second *options*: `nitc -a -b -d -e`
 176 # * all: *common* and all *options*: `nitc -a -b -c -d -e`
 177 function bench_nitc_options()
 178 {
 179         tag=$1
 180         shift
 181         common=$1
 182         shift
 183         name="$FUNCNAME-$tag$common"
 184         name=${name// /}
 185         skip_test "$name" && return
 186         prepare_res "$name.dat" "no options" "nitc $common without more options"
 187         run_compiler "nitc-$name" ./nitc $common
 188
 189         if test "$1" = NOALL; then
 190                 withall=
 191                 shift
 192         else
 193                 withall=true
 194         fi
 195
 196         for opt in "$@"; do
 197                 ot=${opt// /}
 198                 prepare_res "$name$ot.dat" "$opt" "nitc with option $opt"
 199                 run_compiler "nitc-$name" ./nitc $common $opt
 200         done
 201
 202         if test -n "$2" -a -n "$withall"; then
 203                 prepare_res "$name-all.dat" "all" "nitc with all options $@"
 204                 run_compiler "nitc-$name" ./nitc $common $@
 205         fi
 206
 207         plot "$name.gnu"
 208 }
 209
 210 bench_nitc_options "slower" --global --hardening --no-shortcut-range
 211 bench_nitc_options "nocheck" --global --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 212
 213 bench_nitc_options "slower" --separate --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn" "--no-tag-primitives" "--colo-dead-methods" --type-poset
 214 bench_nitc_options "nocheck" --separate --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 215 bench_nitc_options "faster" --separate --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph"
 216
 217 bench_nitc_options "slower" --erasure --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern
 218 bench_nitc_options "nocheck" --erasure --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast --no-check-all
 219 bench_nitc_options "faster" --erasure --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta
 220
 221 bench_nitc_options "engine" "" NOALL "--separate" "--erasure" "--separate --semi-global" "--erasure --semi-global" "--erasure --semi-global --rta" "--global"
 222 bench_nitc_options "policy" "" NOALL "--separate" "--erasure" "--separate --no-check-covariance" "--erasure --no-check-covariance --no-check-erasure-cast"
 223 bench_nitc_options "nullables" "" "--no-check-attr-isset" "--no-union-attribute"
 224 bench_nitc_options "linkboost" "" NOALL --trampoline-call --colors-are-symbols "--colors-are-symbols --trampoline-call" "--separate --link-boost" "--separate --colors-are-symbols --guard-call" "--separate --colors-are-symbols --direct-call-monomorph0" "--substitute-monomorph"
 225 bench_nitc_options "monomorph" "" --direct-call-monomorph0 --direct-call-monomorph
 226
 227 bench_nitc_options "misc" "" --log --typing-test-metrics --invocation-metrics --isset-checks-metrics --tables-metrics --no-stacktrace --release --debug #FIXME add --sloppy
 228
 229 function bench_nitc-e_gc()
 230 {
 231         name="$FUNCNAME"
 232         skip_test "$name" && return
 233         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
 234         run_compiler "nitc-e" ./nitc --erasure
 235         prepare_res "$name-nitc-e-malloc.dat" "nitc-e-malloc" "nitc with --erasure and malloc"
 236         NIT_GC_OPTION="malloc" run_compiler "nitc-e-malloc" ./nitc --erasure
 237         prepare_res "$name-nitc-e-large.dat" "nitc-e-large" "nitc with --erasure and large"
 238         NIT_GC_OPTION="large" run_compiler "nitc-e-large" ./nitc --erasure
 239         plot "$name.gnu"
 240 }
 241 bench_nitc-e_gc
 242
 243 function bench_cc_nitc-e()
 244 {
 245         name="$FUNCNAME"
 246         skip_test "$name" && return
 247         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 248                 f=`echo "$o" | cut -f1 -d:`
 249                 o=`echo "$o" | cut -f2 -d:`
 250                 prepare_res "$name-nitc-e-$f.dat" "nitc-e-$f" "nitc with --erasure --make-flags $o"
 251                 run_compiler "nitc-e-$f" ./nitc --erasure --make-flags "$o"
 252         done
 253         plot "$name.gnu"
 254 }
 255 bench_cc_nitc-e
 256
 257 function bench_compilation_time
 258 {
 259         name="$FUNCNAME"
 260         skip_test "$name" && return
 261         prepare_res "$name-nitc-g.dat" "nitc-g" "nitc --global"
 262         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 263                 bench_command `basename "$i" .nit` "" ./nitc --global "$i" --no-cc
 264         done
 265         prepare_res "$name-nitc-s.dat" "nitc-s" "nitc --separate"
 266         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 267                 bench_command `basename "$i" .nit` "" ./nitc --separate "$i" --no-cc
 268         done
 269         prepare_res "$name-nitc-e.dat" "nitc-e" "nitc --erasure"
 270         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitc.nit; do
 271                 bench_command `basename "$i" .nit` "" ./nitc --erasure "$i" --no-cc
 272         done
 273         plot "$name.gnu"
 274 }
 275 bench_compilation_time
 276
 277 echo >>"$html" "</body></html>"
 278
 279 echo >>"$xml" "</testsuite></testsuites>"
 280
 281 if test -n "$died"; then
 282         echo "Some commands failed"
 283         exit 1
 284 fi
 285 exit 0