2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_common.sh
21 source .
/bench_plot.sh
23 ## CONFIGURATION OPTIONS ##
25 # Default number of times a command must be run with bench_command
26 # Can be overrided with 'the option -n'
31 # Run standards benchs on a compiler command
33 # rest: command to run (executable + options)
34 function run_compiler
()
38 if test "$fast" = truetrue
; then
39 run_command
"$@" ..
/examples
/hello_world.nit
-o
"hello.$title.bin"
40 bench_command
"hello" "hello_world" "./hello.$title.bin"
41 elif test -n
"$fast"; then
42 run_command
"$@" ..
/src
/nitc.nit
-o
"nitc.$title.bin"
43 bench_command
"nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v
--global
--no
-cc ..
/src
/test_parser.nit
44 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
45 bench_command
"nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/location.nit
46 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
47 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
48 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
49 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
51 rm -r out
2> /dev
/null
52 mkdir out
2> /dev
/null
53 run_command
"$@" ..
/src
/nitc.nit
-o
"nitc.$title.bin"
54 bench_command
"nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v
--global
--no
-cc ..
/src
/nitls.nit
55 bench_command
"nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v
--no
-cc
--separate ..
/src
/nitc.nit
56 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
57 bench_command
"nit-queens" "nit queens.nit 8" "./nit.$title.bin" ..
/lib
/ai
/examples
/queens.nit
-q
8
58 bench_command
"nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ..
/contrib
/nitcc
/src
/nitcc.nit ..
/contrib
/nitcc
/examples
/calc.sablecc
59 rm calc
* 2> /dev
/null
# remove generated cruft
60 run_command
"$@" ..
/src
/nitdoc.nit
-o
"nitdoc.$title.bin"
61 bench_command
"nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ..
/src
/nitls.nit
-d out
62 run_command
"$@" ..
/src
/nitlight.nit
-o
"nitlight.$title.bin"
63 bench_command
"nitlight" "nitlight ../lib/[a-f]*/" "./nitlight.$title.bin" ..
/lib
/[a
-f
]*/ -d out
64 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
65 bench_command
"shoot" "shoot_logic 15" "./shoot.$title.bin" 15
66 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
67 bench_command
"bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
68 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
69 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
70 run_command
"$@" "../lib/ai/examples/queens.nit" -o
"queens.$title.bin"
71 bench_command
"queens" "bench_queens 13" "./queens.$title.bin" 13
72 run_command
"$@" "../lib/ai/examples/puzzle.nit" -o
"puzzle.$title.bin"
73 bench_command
"puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
74 run_command
"$@" "markdown/engines/nitmd/nitmd.nit" -o
"nitmd.$title.bin"
75 bench_command
"nitmd" "markdown" "./nitmd.$title.bin" markdown
/benches
/out
/mixed.md
80
76 run_command
"$@" ..
/contrib
/jwrapper
/src
/jwrapper.nit
-o
"jwrapper.$title.bin"
77 bench_command
"jwrapper" "jwrapper ant.jar" "./jwrapper.$title.bin" /usr
/share
/java
/ant.jar
-o out
/ant_jar.nit
78 rm -r tmp
2> /dev
/null
# remove jwrapper output directory
81 rm -r
*.bin out
2> /dev
/null
88 echo "run_bench: [options]* benchname"
89 echo " -v: verbose mode"
90 echo " -n count: number of execution for each bar (default: $count)"
91 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
92 echo " --fast: Run less and faster tests"
97 while [ "$stop" = false
]; do
99 -v
) verbose
=true
; shift;;
101 -n
) count
="$2"; shift; shift;;
102 --dry
) dry_run
=true
; shift;;
103 --fast
) fast
=true
$fast; shift;;
104 --html
) shift;; # Deprecated
110 echo >"$html" "<html><head></head><body>"
112 xml
="bench_engines.xml"
113 echo "<testsuites><testsuite>" > "$xml"
117 if test -z
"$NOTSKIPED"; then
119 echo "List of available benches:"
120 echo "* all: run all the benches"
125 # get the bootstrapped nitc
128 if test -z
"$fast"; then
129 make -C markdown
/benches
130 make -C ..
/contrib
/nitcc
131 make pre
-build
-C ..
/contrib
/jwrapper
134 ## EFFECTIVE BENCHS ##
136 function bench_steps
()
139 skip_test
"$name" && return
140 prepare_res
"$name-nitc.dat" "nitc-g" "Various steps of nitc --global"
141 bench_command
"parse" "" .
/nitc
--global
--only
-parse ..
/src
/nitc.nit
142 bench_command
"metamodel" "" .
/nitc
--global
--only
-metamodel ..
/src
/nitc.nit
143 bench_command
"generate c" "" .
/nitc
--global
--no
-cc ..
/src
/nitc.nit
144 bench_command
"full" "" .
/nitc
--global ..
/src
/nitc.nit
-o
"nitc_nitc.bin"
146 prepare_res
"$name-nitc-s.dat" "nitc-s" "Various steps of nitc --separate"
147 bench_command
"parse" "" .
/nitc
--separate
--only
-parse ..
/src
/nitc.nit
148 bench_command
"metamodel" "" .
/nitc
--separate
--only
-metamodel ..
/src
/nitc.nit
149 bench_command
"generate c" "" .
/nitc
--separate
--no
-cc ..
/src
/nitc.nit
150 bench_command
"full" "" .
/nitc
--separate ..
/src
/nitc.nit
-o
"nitc_nitc-e.bin"
152 prepare_res
"$name-nitc-e.dat" "nitc-e" "Various steps of nitc --erasure"
153 bench_command
"parse" "" .
/nitc
--erasure
--only
-parse ..
/src
/nitc.nit
154 bench_command
"metamodel" "" .
/nitc
--erasure
--only
-metamodel ..
/src
/nitc.nit
155 bench_command
"generate c" "" .
/nitc
--erasure
--no
-cc ..
/src
/nitc.nit
156 bench_command
"full" "" .
/nitc
--erasure ..
/src
/nitc.nit
-o
"nitc_nitc-e.bin"
162 # Simple script to compare various options on nitc
164 # usage: *name* *common* [NOALL] *options*...
166 # * *name*, the name of the bench
167 # * *common*, options to use on each case
168 # * NOALL, optional flag to avoid a last case including all additional options
169 # * *options*, sequences of options, one for each case
171 # Example: `bench_nitc_options "foo" "-a -b" -c "-d -e"` generates 4 cases:
173 # * only *common*: `nitc -a -b`
174 # * *common* and first *options*: `nitc -a -b -c`
175 # * *common* and second *options*: `nitc -a -b -d -e`
176 # * all: *common* and all *options*: `nitc -a -b -c -d -e`
177 function bench_nitc_options
()
183 name
="$FUNCNAME-$tag$common"
185 skip_test
"$name" && return
186 prepare_res
"$name.dat" "no options" "nitc $common without more options"
187 run_compiler
"nitc-$name" .
/nitc
$common
189 if test "$1" = NOALL
; then
198 prepare_res
"$name$ot.dat" "$opt" "nitc with option $opt"
199 run_compiler
"nitc-$name" .
/nitc
$common $opt
202 if test -n
"$2" -a
-n
"$withall"; then
203 prepare_res
"$name-all.dat" "all" "nitc with all options $@"
204 run_compiler
"nitc-$name" .
/nitc
$common $@
210 bench_nitc_options
"slower" --global
--hardening
--no
-shortcut
-range
211 bench_nitc_options
"nocheck" --global
--no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
213 bench_nitc_options
"slower" --separate
--hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
"--no-gcc-directive likely --no-gcc-directive noreturn" "--no-tag-primitives" "--colo-dead-methods" --type-poset
214 bench_nitc_options
"nocheck" --separate
--no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
215 bench_nitc_options
"faster" --separate
--skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
"--inline-some-methods --direct-call-monomorph"
217 bench_nitc_options
"slower" --erasure
--hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
218 bench_nitc_options
"nocheck" --erasure
--no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
--no
-check
-erasure
-cast
--no
-check
-all
219 bench_nitc_options
"faster" --erasure
--skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
--rta
221 bench_nitc_options
"engine" "" NOALL
"--separate" "--erasure" "--separate --semi-global" "--erasure --semi-global" "--erasure --semi-global --rta" "--global"
222 bench_nitc_options
"policy" "" NOALL
"--separate" "--erasure" "--separate --no-check-covariance" "--erasure --no-check-covariance --no-check-erasure-cast"
223 bench_nitc_options
"nullables" "" "--no-check-attr-isset" "--no-union-attribute"
224 bench_nitc_options
"linkboost" "" NOALL
--trampoline
-call
--colors
-are
-symbols
"--colors-are-symbols --trampoline-call" "--separate --link-boost" "--separate --colors-are-symbols --guard-call" "--separate --colors-are-symbols --direct-call-monomorph0" "--substitute-monomorph"
225 bench_nitc_options
"monomorph" "" --direct
-call
-monomorph0
--direct
-call
-monomorph
227 bench_nitc_options
"misc" "" --log
--typing
-test-metrics
--invocation
-metrics
--isset
-checks
-metrics
--tables
-metrics
--no
-stacktrace
--release
--debug
#FIXME add --sloppy
229 # sanitary just run the default configuration, this is used to check that `run_compiler` works.
230 bench_nitc_options
"sanitary" ""
232 function bench_nitc
-e_gc
()
235 skip_test
"$name" && return
236 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
237 run_compiler
"nitc-e" .
/nitc
--erasure
238 prepare_res
"$name-nitc-e-malloc.dat" "nitc-e-malloc" "nitc with --erasure and malloc"
239 NIT_GC_OPTION
="malloc" run_compiler
"nitc-e-malloc" .
/nitc
--erasure
240 prepare_res
"$name-nitc-e-large.dat" "nitc-e-large" "nitc with --erasure and large"
241 NIT_GC_OPTION
="large" run_compiler
"nitc-e-large" .
/nitc
--erasure
246 function bench_cc_nitc
-e
()
249 skip_test
"$name" && return
250 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
251 f
=`echo "$o" | cut -f1 -d:`
252 o
=`echo "$o" | cut -f2 -d:`
253 prepare_res
"$name-nitc-e-$f.dat" "nitc-e-$f" "nitc with --erasure --make-flags $o"
254 run_compiler
"nitc-e-$f" .
/nitc
--erasure
--make-flags
"$o"
260 function bench_compilation_time
263 skip_test
"$name" && return
264 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc --global"
265 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
266 bench_command
`basename "$i" .nit` "" .
/nitc
--global
"$i" --no
-cc
268 prepare_res
"$name-nitc-s.dat" "nitc-s" "nitc --separate"
269 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
270 bench_command
`basename "$i" .nit` "" .
/nitc
--separate
"$i" --no
-cc
272 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc --erasure"
273 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
274 bench_command
`basename "$i" .nit` "" .
/nitc
--erasure
"$i" --no
-cc
278 bench_compilation_time
280 echo >>"$html" "</body></html>"
282 echo >>"$xml" "</testsuite></testsuites>"
284 if test -n
"$died"; then
285 echo "Some commands failed"