benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_common.sh
  21 source ./bench_plot.sh
  22
  23 ## CONFIGURATION OPTIONS ##
  24
  25 # Default number of times a command must be run with bench_command
  26 # Can be overrided with 'the option -n'
  27 count=2
  28
  29 pep8analysis=../../pep8analysis
  30
  31 ### HELPER FUNCTIONS ##
  32
  33 function die()
  34 {
  35         echo >&2 "error: $*"
  36         died=1
  37 }
  38
  39 # HELPER FOR NIT #
  40
  41 # Run standards benchs on a compiler command
  42 # $1: title
  43 # rest: command to run (executable + options)
  44 function run_compiler()
  45 {
  46         local title=$1
  47         shift
  48         if test -n "$fast"; then
  49                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
  50                 bench_command "nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit
  51                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  52                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
  53                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  54                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
  55                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  56                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
  57         else
  58                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
  59                 bench_command "nitg-g" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitmetrics.nit
  60                 bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit
  61                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  62                 bench_command "nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/rapid_type_analysis.nit
  63                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  64                 bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
  65                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  66                 bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
  67                 if test -f "$pep8analysis/src/pep8analysis.nit"; then
  68                         run_command "$@" "$pep8analysis/src/pep8analysis.nit" -I "$pep8analysis/lib" -o "pep8a.$title.bin"
  69                         bench_command "pep8analisis" "bench_bintree_gen 18" "./pep8a.$title.bin" "$pep8analysis/tests/privat/"*.pep
  70                 fi
  71         fi
  72 }
  73
  74 ## HANDLE OPTIONS ##
  75
  76 function usage()
  77 {
  78         echo "run_bench: [options]* benchname"
  79         echo "  -v: verbose mode"
  80         echo "  -n count: number of execution for each bar (default: $count)"
  81         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
  82         echo "  --fast: Run less and faster tests"
  83         echo "  --html: Generate and HTML output"
  84         echo "  -h: this help"
  85 }
  86
  87 stop=false
  88 while [ "$stop" = false ]; do
  89         case "$1" in
  90                 -v) verbose=true; shift;;
  91                 -h) usage; exit;;
  92                 -n) count="$2"; shift; shift;;
  93                 --dry) dry_run=true; shift;;
  94                 --fast) fast=true; shift;;
  95                 --html) html="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
  96                 *) stop=true
  97         esac
  98 done
  99
 100 NOTSKIPED="$*"
 101
 102 if test -z "$NOTSKIPED"; then
 103         usage
 104         echo "List of available benches:"
 105         echo "* all: run all the benches"
 106 fi
 107
 108 ## COMPILE ENGINES
 109
 110 # force to use the last nitg, not the bootstraped one
 111 test -f ./nitg || ../bin/nitg ../src/nitg.nit -v
 112
 113 ## EFFECTIVE BENCHS ##
 114
 115 function bench_steps()
 116 {
 117         name="$FUNCNAME"
 118         skip_test "$name" && return
 119         prepare_res "$name-nitg.dat" "nitg-g" "Various steps of nitg --global"
 120         bench_command "parse" "" ./nitg --global --only-parse ../src/nitg.nit
 121         bench_command "metamodel" "" ./nitg --global --only-metamodel ../src/nitg.nit
 122         bench_command "generate c" "" ./nitg --global --no-cc ../src/nitg.nit
 123         bench_command "full" "" ./nitg --global ../src/nitg.nit -o "nitg_nitg.bin"
 124
 125         prepare_res "$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
 126         bench_command "parse" "" ./nitg --separate --only-parse ../src/nitg.nit
 127         bench_command "metamodel" "" ./nitg --separate --only-metamodel ../src/nitg.nit
 128         bench_command "generate c" "" ./nitg --separate --no-cc ../src/nitg.nit
 129         bench_command "full" "" ./nitg --separate ../src/nitg.nit -o "nitg_nitg-e.bin"
 130
 131         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 132         bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit
 133         bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit
 134         bench_command "generate c" "" ./nitg --erasure --no-cc ../src/nitg.nit
 135         bench_command "full" "" ./nitg --erasure ../src/nitg.nit -o "nitg_nitg-e.bin"
 136
 137         plot "$name.gnu"
 138 }
 139 bench_steps
 140
 141 # $#: options to compare
 142 function bench_nitg-g_options()
 143 {
 144         tag=$1
 145         shift
 146         name="$FUNCNAME-$tag"
 147         skip_test "$name" && return
 148         prepare_res "$name.dat" "no options" "nitg-g without options"
 149         run_compiler "nitg-g" ./nitg --global
 150
 151         if test "$1" = NOALL; then
 152                 shift
 153         elif test -n "$2"; then
 154                 prepare_res "$name-all.dat" "all" "nitg-g with all options $@"
 155                 run_compiler "nitg-g-$tag" ./nitg --global $@
 156         fi
 157
 158         for opt in "$@"; do
 159                 ot=${opt// /+}
 160                 prepare_res "$name$ot.dat" "$opt" "nitg-g with option $opt"
 161                 run_compiler "nitg-g$ot" ./nitg --global $opt
 162         done
 163
 164         plot "$name.gnu"
 165 }
 166 bench_nitg-g_options "slower" --hardening
 167 bench_nitg-g_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other
 168
 169 function bench_nitg-s_options()
 170 {
 171         tag=$1
 172         shift
 173         name="$FUNCNAME-$tag"
 174         skip_test "$name" && return
 175         prepare_res "$name.dat" "no options" "nitg-s without options"
 176         run_compiler "nitg-s" ./nitg --separate
 177
 178         if test "$1" = NOALL; then
 179                 shift
 180         elif test -n "$2"; then
 181                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 182                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 183         fi
 184
 185         for opt in "$@"; do
 186                 ot=${opt// /+}
 187                 prepare_res "$name-$ot.dat" "$opt" "nitg-s with option $opt"
 188                 run_compiler "nitg-s$ot" ./nitg --separate $opt
 189         done
 190
 191         plot "$name.gnu"
 192 }
 193 bench_nitg-s_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range "--no-gcc-directive likely" "--no-gcc-directive noreturn"
 194 bench_nitg-s_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other
 195 bench_nitg-s_options "faster" --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph"
 196
 197 function bench_nitg-e_options()
 198 {
 199         tag=$1
 200         shift
 201         name="$FUNCNAME-$tag"
 202         skip_test "$name" && return
 203         prepare_res "$name.dat" "no options" "nitg-e without options"
 204         run_compiler "nitg-e" ./nitg --erasure
 205
 206         if test "$1" = NOALL; then
 207                 shift
 208         elif test -n "$2"; then
 209                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 210                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 211         fi
 212
 213         for opt in "$@"; do
 214                 ot=${opt// /+}
 215                 prepare_res "$name$ot.dat" "$opt" "nitg-e with option $opt"
 216                 run_compiler "nitg-e$ot" ./nitg --erasure $opt
 217         done
 218
 219         plot "$name.gnu"
 220 }
 221 bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
 222 bench_nitg-e_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
 223 bench_nitg-e_options "faster" --inline-coloring-numbers
 224
 225 function bench_engines()
 226 {
 227         name="$FUNCNAME"
 228         skip_test "$name" && return
 229         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg with --global"
 230         run_compiler "nitg-g" ./nitg --global
 231         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 232         run_compiler "nitg-s" ./nitg --separate
 233         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 234         run_compiler "nitg-e" ./nitg --erasure
 235         plot "$name.gnu"
 236 }
 237 bench_engines
 238
 239 function bench_nitg-e_gc()
 240 {
 241         name="$FUNCNAME"
 242         skip_test "$name" && return
 243         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 244         NIT_GC_OPTION="malloc" run_compiler "nitg-e-malloc" ./nitg --erasure
 245         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 246         run_compiler "nitg-e" ./nitg --erasure
 247         plot "$name.gnu"
 248 }
 249 bench_nitg-e_gc
 250
 251 function bench_cc_nitg-e()
 252 {
 253         name="$FUNCNAME"
 254         skip_test "$name" && return
 255         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 256                 f=`echo "$o" | cut -f1 -d:`
 257                 o=`echo "$o" | cut -f2 -d:`
 258                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 259                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 260         done
 261         plot "$name.gnu"
 262 }
 263 bench_cc_nitg-e
 264
 265 function bench_policy()
 266 {
 267         name="$FUNCNAME"
 268         skip_test "$name" && return
 269         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 270         run_compiler "nitg-s" ./nitg --separate
 271         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 272         run_compiler "nitg-e" ./nitg --erasure
 273         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 274         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 275         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 276         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 277         plot "$name.gnu"
 278 }
 279 bench_policy
 280
 281 function bench_nullables()
 282 {
 283         name="$FUNCNAME"
 284         skip_test "$name" && return
 285         prepare_res "$name-nitc.dat" "nitc" "nitc no options"
 286         run_compiler "nitc" ./nitg --separate
 287         prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
 288         run_compiler "nitc" ./nitg --separate --no-check-attr-isset
 289         prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
 290         run_compiler "nitc" ./nitg --separate --no-union-attribute
 291         prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
 292         run_compiler "nitc" ./nitg --separate --no-union-attribute --no-check-attr-isset
 293         plot "$name.gnu"
 294 }
 295 bench_nullables
 296
 297 function bench_compilation_time
 298 {
 299         name="$FUNCNAME"
 300         skip_test "$name" && return
 301         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg --global"
 302         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 303                 bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc
 304         done
 305         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg --separate"
 306         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 307                 bench_command `basename "$i" .nit` "" ./nitg --separate "$i" --no-cc
 308         done
 309         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 310         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 311                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 312         done
 313         plot "$name.gnu"
 314 }
 315 bench_compilation_time
 316
 317 if test -n "$html"; then
 318         echo >>"$html" "</body></html>"
 319 fi
 320
 321 if test -n "$died"; then
 322         echo "Some commands failed"
 323         exit 1
 324 fi
 325 exit 0