benchmarks/bench_engines.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_common.sh
  21 source ./bench_plot.sh
  22
  23 ## CONFIGURATION OPTIONS ##
  24
  25 # Default number of times a command must be run with bench_command
  26 # Can be overrided with 'the option -n'
  27 count=2
  28
  29 ### HELPER FUNCTIONS ##
  30
  31 function die()
  32 {
  33         echo >&2 "error: $*"
  34         died=1
  35 }
  36
  37 # HELPER FOR NIT #
  38
  39 # Run standards benchs on a compiler command
  40 # $1: title
  41 # rest: command to run (executable + options)
  42 function run_compiler()
  43 {
  44         local title=$1
  45         shift
  46         if test -n "$fast"; then
  47                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
  48                 bench_command "nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit
  49                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  50                 bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit
  51                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  52                 bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
  53                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  54                 bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
  55         else
  56                 run_command "$@" ../src/nitg.nit -o "nitg.$title.bin"
  57                 bench_command "nitg-g" "nitg --global --no-cc ../src/nitls.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitls.nit
  58                 bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit
  59                 run_command "$@" ../src/nit.nit -o "nit.$title.bin"
  60                 bench_command "nit" "nit ../src/test_parser.nit ../src/nitls.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/nitls.nit
  61                 run_command "$@" ../src/nitdoc.nit -o "nitdoc.$title.bin"
  62                 rm -r out 2> /dev/null
  63                 mkdir out 2> /dev/null
  64                 bench_command "nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ../src/nitls.nit -d out
  65                 run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin"
  66                 bench_command "shoot" "shoot_logic 15" "./shoot.$title.bin" 15
  67                 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
  68                 bench_command "bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
  69                 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
  70                 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
  71                 run_command "$@" "../lib/ai/examples/queens.nit" -o "queens.$title.bin"
  72                 bench_command "queens" "bench_queens 13" "./queens.$title.bin" 13
  73                 run_command "$@" "../lib/ai/examples/puzzle.nit" -o "puzzle.$title.bin"
  74                 bench_command "puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
  75         fi
  76
  77         rm -r *.bin .nit_compile out
  78 }
  79
  80 ## HANDLE OPTIONS ##
  81
  82 function usage()
  83 {
  84         echo "run_bench: [options]* benchname"
  85         echo "  -v: verbose mode"
  86         echo "  -n count: number of execution for each bar (default: $count)"
  87         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
  88         echo "  --fast: Run less and faster tests"
  89         echo "  --html: Generate and HTML output"
  90         echo "  -h: this help"
  91 }
  92
  93 stop=false
  94 while [ "$stop" = false ]; do
  95         case "$1" in
  96                 -v) verbose=true; shift;;
  97                 -h) usage; exit;;
  98                 -n) count="$2"; shift; shift;;
  99                 --dry) dry_run=true; shift;;
 100                 --fast) fast=true; shift;;
 101                 --html) html="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
 102                 *) stop=true
 103         esac
 104 done
 105
 106 NOTSKIPED="$*"
 107
 108 if test -z "$NOTSKIPED"; then
 109         usage
 110         echo "List of available benches:"
 111         echo "* all: run all the benches"
 112 fi
 113
 114 ## COMPILE ENGINES
 115
 116 # get the bootstrapped nitg
 117 cp ../bin/nitg .
 118
 119 ## EFFECTIVE BENCHS ##
 120
 121 function bench_steps()
 122 {
 123         name="$FUNCNAME"
 124         skip_test "$name" && return
 125         prepare_res "$name-nitg.dat" "nitg-g" "Various steps of nitg --global"
 126         bench_command "parse" "" ./nitg --global --only-parse ../src/nitg.nit
 127         bench_command "metamodel" "" ./nitg --global --only-metamodel ../src/nitg.nit
 128         bench_command "generate c" "" ./nitg --global --no-cc ../src/nitg.nit
 129         bench_command "full" "" ./nitg --global ../src/nitg.nit -o "nitg_nitg.bin"
 130
 131         prepare_res "$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
 132         bench_command "parse" "" ./nitg --separate --only-parse ../src/nitg.nit
 133         bench_command "metamodel" "" ./nitg --separate --only-metamodel ../src/nitg.nit
 134         bench_command "generate c" "" ./nitg --separate --no-cc ../src/nitg.nit
 135         bench_command "full" "" ./nitg --separate ../src/nitg.nit -o "nitg_nitg-e.bin"
 136
 137         prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
 138         bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit
 139         bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit
 140         bench_command "generate c" "" ./nitg --erasure --no-cc ../src/nitg.nit
 141         bench_command "full" "" ./nitg --erasure ../src/nitg.nit -o "nitg_nitg-e.bin"
 142
 143         plot "$name.gnu"
 144 }
 145 bench_steps
 146
 147 # $#: options to compare
 148 function bench_nitg-g_options()
 149 {
 150         tag=$1
 151         shift
 152         name="$FUNCNAME-$tag"
 153         skip_test "$name" && return
 154         prepare_res "$name.dat" "no options" "nitg-g without options"
 155         run_compiler "nitg-g" ./nitg --global
 156
 157         if test "$1" = NOALL; then
 158                 shift
 159         elif test -n "$2"; then
 160                 prepare_res "$name-all.dat" "all" "nitg-g with all options $@"
 161                 run_compiler "nitg-g-$tag" ./nitg --global $@
 162         fi
 163
 164         for opt in "$@"; do
 165                 ot=${opt// /+}
 166                 prepare_res "$name$ot.dat" "$opt" "nitg-g with option $opt"
 167                 run_compiler "nitg-g$ot" ./nitg --global $opt
 168         done
 169
 170         plot "$name.gnu"
 171 }
 172 bench_nitg-g_options "slower" --hardening --no-shortcut-range
 173 bench_nitg-g_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 174
 175 function bench_nitg-s_options()
 176 {
 177         tag=$1
 178         shift
 179         name="$FUNCNAME-$tag"
 180         skip_test "$name" && return
 181         prepare_res "$name.dat" "no options" "nitg-s without options"
 182         run_compiler "nitg-s" ./nitg --separate
 183
 184         if test "$1" = NOALL; then
 185                 shift
 186         elif test -n "$2"; then
 187                 prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
 188                 run_compiler "nitg-s-$tag" ./nitg --separate $@
 189         fi
 190
 191         for opt in "$@"; do
 192                 ot=${opt// /+}
 193                 prepare_res "$name-$ot.dat" "$opt" "nitg-s with option $opt"
 194                 run_compiler "nitg-s$ot" ./nitg --separate $opt
 195         done
 196
 197         plot "$name.gnu"
 198 }
 199 bench_nitg-s_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn"
 200 bench_nitg-s_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert
 201 bench_nitg-s_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" ""
 202
 203 function bench_nitg-e_options()
 204 {
 205         tag=$1
 206         shift
 207         name="$FUNCNAME-$tag"
 208         skip_test "$name" && return
 209         prepare_res "$name.dat" "no options" "nitg-e without options"
 210         run_compiler "nitg-e" ./nitg --erasure
 211
 212         if test "$1" = NOALL; then
 213                 shift
 214         elif test -n "$2"; then
 215                 prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
 216                 run_compiler "nitg-e-$tag" ./nitg --erasure $@
 217         fi
 218
 219         for opt in "$@"; do
 220                 ot=${opt// /+}
 221                 prepare_res "$name$ot.dat" "$opt" "nitg-e with option $opt"
 222                 run_compiler "nitg-e$ot" ./nitg --erasure $opt
 223         done
 224
 225         plot "$name.gnu"
 226 }
 227 bench_nitg-e_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern
 228 bench_nitg-e_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast
 229 bench_nitg-e_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta
 230
 231 function bench_engines()
 232 {
 233         name="$FUNCNAME"
 234         skip_test "$name" && return
 235         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg with --global"
 236         run_compiler "nitg-g" ./nitg --global
 237         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 238         run_compiler "nitg-s" ./nitg --separate
 239         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 240         run_compiler "nitg-e" ./nitg --erasure
 241         plot "$name.gnu"
 242 }
 243 bench_engines
 244
 245 function bench_nitg-e_gc()
 246 {
 247         name="$FUNCNAME"
 248         skip_test "$name" && return
 249         prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
 250         NIT_GC_OPTION="malloc" run_compiler "nitg-e-malloc" ./nitg --erasure
 251         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 252         run_compiler "nitg-e" ./nitg --erasure
 253         plot "$name.gnu"
 254 }
 255 bench_nitg-e_gc
 256
 257 function bench_cc_nitg-e()
 258 {
 259         name="$FUNCNAME"
 260         skip_test "$name" && return
 261         for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
 262                 f=`echo "$o" | cut -f1 -d:`
 263                 o=`echo "$o" | cut -f2 -d:`
 264                 prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
 265                 run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
 266         done
 267         plot "$name.gnu"
 268 }
 269 bench_cc_nitg-e
 270
 271 function bench_policy()
 272 {
 273         name="$FUNCNAME"
 274         skip_test "$name" && return
 275         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
 276         run_compiler "nitg-s" ./nitg --separate
 277         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
 278         run_compiler "nitg-e" ./nitg --erasure
 279         prepare_res "$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
 280         run_compiler "nitg-su" ./nitg --separate --no-check-covariance
 281         prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
 282         run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
 283         plot "$name.gnu"
 284 }
 285 bench_policy
 286
 287 function bench_nullables()
 288 {
 289         name="$FUNCNAME"
 290         skip_test "$name" && return
 291         prepare_res "$name-nitc.dat" "nitc" "nitc no options"
 292         run_compiler "nitc" ./nitg --separate
 293         prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
 294         run_compiler "nitc" ./nitg --separate --no-check-attr-isset
 295         prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
 296         run_compiler "nitc" ./nitg --separate --no-union-attribute
 297         prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
 298         run_compiler "nitc" ./nitg --separate --no-union-attribute --no-check-attr-isset
 299         plot "$name.gnu"
 300 }
 301 bench_nullables
 302
 303 function bench_compilation_time
 304 {
 305         name="$FUNCNAME"
 306         skip_test "$name" && return
 307         prepare_res "$name-nitg-g.dat" "nitg-g" "nitg --global"
 308         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 309                 bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc
 310         done
 311         prepare_res "$name-nitg-s.dat" "nitg-s" "nitg --separate"
 312         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 313                 bench_command `basename "$i" .nit` "" ./nitg --separate "$i" --no-cc
 314         done
 315         prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 316         for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do
 317                 bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 318         done
 319         plot "$name.gnu"
 320 }
 321 bench_compilation_time
 322
 323 if test -n "$html"; then
 324         echo >>"$html" "</body></html>"
 325 fi
 326
 327 if test -n "$died"; then
 328         echo "Some commands failed"
 329         exit 1
 330 fi
 331 exit 0