2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_common.sh
21 source .
/bench_plot.sh
23 ## CONFIGURATION OPTIONS ##
25 # Default number of times a command must be run with bench_command
26 # Can be overrided with 'the option -n'
29 ### HELPER FUNCTIONS ##
39 # Run standards benchs on a compiler command
41 # rest: command to run (executable + options)
42 function run_compiler
()
46 if test -n
"$fast"; then
47 run_command
"$@" ..
/src
/nitc.nit
-o
"nitc.$title.bin"
48 bench_command
"nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v
--global
--no
-cc ..
/src
/test_parser.nit
49 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
50 bench_command
"nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/location.nit
51 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
52 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
53 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
54 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
56 run_command
"$@" ..
/src
/nitc.nit
-o
"nitc.$title.bin"
57 bench_command
"nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v
--global
--no
-cc ..
/src
/nitls.nit
58 bench_command
"nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v
--no
-cc
--separate ..
/src
/nitc.nit
59 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
60 bench_command
"nit-queens" "nit queens.nit 8" "./nit.$title.bin" ..
/lib
/ai
/examples
/queens.nit
-q
8
61 bench_command
"nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ..
/contrib
/nitcc
/src
/nitcc.nit ..
/contrib
/nitcc
/examples
/calc.sablecc
62 run_command
"$@" ..
/src
/nitdoc.nit
-o
"nitdoc.$title.bin"
63 rm -r out
2> /dev
/null
64 mkdir out
2> /dev
/null
65 bench_command
"nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ..
/src
/nitls.nit
-d out
66 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
67 bench_command
"shoot" "shoot_logic 15" "./shoot.$title.bin" 15
68 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
69 bench_command
"bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
70 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
71 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
72 run_command
"$@" "../lib/ai/examples/queens.nit" -o
"queens.$title.bin"
73 bench_command
"queens" "bench_queens 13" "./queens.$title.bin" 13
74 run_command
"$@" "../lib/ai/examples/puzzle.nit" -o
"puzzle.$title.bin"
75 bench_command
"puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
76 run_command
"$@" "markdown/engines/nitmd/nitmd.nit" -o
"nitmd.$title.bin"
77 bench_command
"nitmd" "markdown" "./nitmd.$title.bin" markdown
/benches
/out
/mixed.md
80
80 rm -r
*.bin .nit_compile out
87 echo "run_bench: [options]* benchname"
88 echo " -v: verbose mode"
89 echo " -n count: number of execution for each bar (default: $count)"
90 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
91 echo " --fast: Run less and faster tests"
92 echo " --html: Generate and HTML output"
97 while [ "$stop" = false
]; do
99 -v
) verbose
=true
; shift;;
101 -n
) count
="$2"; shift; shift;;
102 --dry
) dry_run
=true
; shift;;
103 --fast
) fast
=true
; shift;;
104 --html
) html
="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
109 xml
="bench_engines.xml"
110 echo "<testsuites><testsuite>" > "$xml"
114 if test -z
"$NOTSKIPED"; then
116 echo "List of available benches:"
117 echo "* all: run all the benches"
122 # get the bootstrapped nitc
125 if test -z
"$fast"; then
126 make -C markdown
/benches
127 make -C ..
/contrib
/nitcc
130 ## EFFECTIVE BENCHS ##
132 function bench_steps
()
135 skip_test
"$name" && return
136 prepare_res
"$name-nitc.dat" "nitc-g" "Various steps of nitc --global"
137 bench_command
"parse" "" .
/nitc
--global
--only
-parse ..
/src
/nitc.nit
138 bench_command
"metamodel" "" .
/nitc
--global
--only
-metamodel ..
/src
/nitc.nit
139 bench_command
"generate c" "" .
/nitc
--global
--no
-cc ..
/src
/nitc.nit
140 bench_command
"full" "" .
/nitc
--global ..
/src
/nitc.nit
-o
"nitc_nitc.bin"
142 prepare_res
"$name-nitc-s.dat" "nitc-s" "Various steps of nitc --separate"
143 bench_command
"parse" "" .
/nitc
--separate
--only
-parse ..
/src
/nitc.nit
144 bench_command
"metamodel" "" .
/nitc
--separate
--only
-metamodel ..
/src
/nitc.nit
145 bench_command
"generate c" "" .
/nitc
--separate
--no
-cc ..
/src
/nitc.nit
146 bench_command
"full" "" .
/nitc
--separate ..
/src
/nitc.nit
-o
"nitc_nitc-e.bin"
148 prepare_res
"$name-nitc-e.dat" "nitc-e" "Various steps of nitc --erasure"
149 bench_command
"parse" "" .
/nitc
--erasure
--only
-parse ..
/src
/nitc.nit
150 bench_command
"metamodel" "" .
/nitc
--erasure
--only
-metamodel ..
/src
/nitc.nit
151 bench_command
"generate c" "" .
/nitc
--erasure
--no
-cc ..
/src
/nitc.nit
152 bench_command
"full" "" .
/nitc
--erasure ..
/src
/nitc.nit
-o
"nitc_nitc-e.bin"
158 # $#: options to compare
159 function bench_nitc
-g_options
()
163 name
="$FUNCNAME-$tag"
164 skip_test
"$name" && return
165 prepare_res
"$name.dat" "no options" "nitc-g without options"
166 run_compiler
"nitc-g" .
/nitc
--global
168 if test "$1" = NOALL
; then
170 elif test -n
"$2"; then
171 prepare_res
"$name-all.dat" "all" "nitc-g with all options $@"
172 run_compiler
"nitc-g-$tag" .
/nitc
--global $@
177 prepare_res
"$name$ot.dat" "$opt" "nitc-g with option $opt"
178 run_compiler
"nitc-g$ot" .
/nitc
--global
$opt
183 bench_nitc
-g_options
"slower" --hardening
--no
-shortcut
-range
184 bench_nitc
-g_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
186 function bench_nitc
-s_options
()
190 name
="$FUNCNAME-$tag"
191 skip_test
"$name" && return
192 prepare_res
"$name.dat" "no options" "nitc-s without options"
193 run_compiler
"nitc-s" .
/nitc
--separate
195 if test "$1" = NOALL
; then
197 elif test -n
"$2"; then
198 prepare_res
"$name-all.dat" "all" "nitc-s with all options $@"
199 run_compiler
"nitc-s-$tag" .
/nitc
--separate $@
204 prepare_res
"$name-$ot.dat" "$opt" "nitc-s with option $opt"
205 run_compiler
"nitc-s$ot" .
/nitc
--separate
$opt
210 bench_nitc
-s_options
"slower" --hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
"--no-gcc-directive likely --no-gcc-directive noreturn"
211 bench_nitc
-s_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
212 bench_nitc
-s_options
"faster" --skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
"--inline-some-methods --direct-call-monomorph" ""
214 function bench_nitc
-e_options
()
218 name
="$FUNCNAME-$tag"
219 skip_test
"$name" && return
220 prepare_res
"$name.dat" "no options" "nitc-e without options"
221 run_compiler
"nitc-e" .
/nitc
--erasure
223 if test "$1" = NOALL
; then
225 elif test -n
"$2"; then
226 prepare_res
"$name-all.dat" "all" "nitc-e with all options $@"
227 run_compiler
"nitc-e-$tag" .
/nitc
--erasure $@
232 prepare_res
"$name$ot.dat" "$opt" "nitc-e with option $opt"
233 run_compiler
"nitc-e$ot" .
/nitc
--erasure
$opt
238 bench_nitc
-e_options
"slower" --hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
239 bench_nitc
-e_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
--no
-check
-erasure
-cast
240 bench_nitc
-e_options
"faster" --skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
--rta
242 function bench_engines
()
245 skip_test
"$name" && return
246 prepare_res
"$name-nitc-s.dat" "nitc-s" "nitc with --separate"
247 run_compiler
"nitc-s" .
/nitc
--separate
248 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
249 run_compiler
"nitc-e" .
/nitc
--erasure
250 prepare_res
"$name-nitc-sg.dat" "nitc-sg" "nitc with --separate --semi-global"
251 run_compiler
"nitc-sg" .
/nitc
--separate
--semi
-global
252 prepare_res
"$name-nitc-eg.dat" "nitc-eg" "nitc with --erasure --semi-global"
253 run_compiler
"nitc-eg" .
/nitc
--erasure
--semi
-global
254 prepare_res
"$name-nitc-egt.dat" "nitc-egt" "nitc with --erasure --semi-global --rta"
255 run_compiler
"nitc-egt" .
/nitc
--erasure
--semi
-global
--rta
256 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
257 run_compiler
"nitc-g" .
/nitc
--global
262 function bench_nitc
-e_gc
()
265 skip_test
"$name" && return
266 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
267 run_compiler
"nitc-e" .
/nitc
--erasure
268 prepare_res
"$name-nitc-e-malloc.dat" "nitc-e-malloc" "nitc with --erasure and malloc"
269 NIT_GC_OPTION
="malloc" run_compiler
"nitc-e-malloc" .
/nitc
--erasure
270 prepare_res
"$name-nitc-e-large.dat" "nitc-e-large" "nitc with --erasure and large"
271 NIT_GC_OPTION
="large" run_compiler
"nitc-e-large" .
/nitc
--erasure
276 function bench_cc_nitc
-e
()
279 skip_test
"$name" && return
280 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
281 f
=`echo "$o" | cut -f1 -d:`
282 o
=`echo "$o" | cut -f2 -d:`
283 prepare_res
"$name-nitc-e-$f.dat" "nitc-e-$f" "nitc with --erasure --make-flags $o"
284 run_compiler
"nitc-e-$f" .
/nitc
--erasure
--make-flags
"$o"
290 function bench_policy
()
293 skip_test
"$name" && return
294 prepare_res
"$name-nitc-s.dat" "nitc-s" "nitc with --separate"
295 run_compiler
"nitc-s" .
/nitc
--separate
296 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc with --erasure"
297 run_compiler
"nitc-e" .
/nitc
--erasure
298 prepare_res
"$name-nitc-su.dat" "nitc-su" "nitc with --separate --no-check-covariance"
299 run_compiler
"nitc-su" .
/nitc
--separate
--no
-check
-covariance
300 prepare_res
"$name-nitc-eu.dat" "nitc-eu" "nitc with --erasure --no-check-covariance --no-check-erasure-cast"
301 run_compiler
"nitc-eu" .
/nitc
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
306 function bench_nullables
()
309 skip_test
"$name" && return
310 prepare_res
"$name-nitc.dat" "nitc" "nitc no options"
311 run_compiler
"nitc" .
/nitc
--separate
312 prepare_res
"$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
313 run_compiler
"nitc" .
/nitc
--separate
--no
-check
-attr
-isset
314 prepare_res
"$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
315 run_compiler
"nitc" .
/nitc
--separate
--no
-union
-attribute
316 prepare_res
"$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
317 run_compiler
"nitc" .
/nitc
--separate
--no
-union
-attribute
--no
-check
-attr
-isset
322 function bench_compilation_time
325 skip_test
"$name" && return
326 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc --global"
327 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
328 bench_command
`basename "$i" .nit` "" .
/nitc
--global
"$i" --no
-cc
330 prepare_res
"$name-nitc-s.dat" "nitc-s" "nitc --separate"
331 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
332 bench_command
`basename "$i" .nit` "" .
/nitc
--separate
"$i" --no
-cc
334 prepare_res
"$name-nitc-e.dat" "nitc-e" "nitc --erasure"
335 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitc.nit
; do
336 bench_command
`basename "$i" .nit` "" .
/nitc
--erasure
"$i" --no
-cc
340 bench_compilation_time
342 function bench_linkboost
()
345 skip_test
"$name" && return
346 prepare_res
"$name-nitc-st.dat" "nitc-st" "nitc with --separate --trampoline-call"
347 run_compiler
"nitc-st" .
/nitc
--separate
--trampoline
-call
348 prepare_res
"$name-nitc-s.dat" "nitc-s" "nitc with --separate"
349 run_compiler
"nitc-s" .
/nitc
--separate
350 prepare_res
"$name-nitc-sc.dat" "nitc-sc" "nitc with --separate --colors-are-symbols"
351 run_compiler
"nitc-sc" .
/nitc
--separate
--colors
-are
-symbols
352 prepare_res
"$name-nitc-sct.dat" "nitc-sct" "nitc with --separate --colors-are-symbols --trampoline-call"
353 run_compiler
"nitc-sct" .
/nitc
--separate
--colors
-are
-symbols
--trampoline
-call
354 prepare_res
"$name-nitc-sl.dat" "nitc-sl" "nitc with --separate --link-boost"
355 run_compiler
"nitc-scts" .
/nitc
--separate
--link
-boost
356 prepare_res
"$name-nitc-scgc.dat" "nitc-scgc" "nitc with --separate --colors-are-symbols --guard-call"
357 run_compiler
"nitc-scgc" .
/nitc
--separate
--colors
-are
-symbols
--guard
-call
358 prepare_res
"$name-nitc-scd.dat" "nitc-scd" "nitc with --separate --colors-are-symbols --direct-call-monomorph0"
359 run_compiler
"nitc-scd" .
/nitc
--separate
--colors
-are
-symbols
--direct
-call
-monomorph0
364 function bench_call_monomorph
()
367 skip_test
"$name" && return
368 prepare_res
"$name-nitc.dat" "nitc" "nitc with --separate"
369 run_compiler
"nitc" .
/nitc
370 prepare_res
"$name-nitc-d0.dat" "nitc-d0" "nitc with --separate --direct-call-monomorph0"
371 run_compiler
"nitc-d0" .
/nitc
--direct
-call
-monomorph0
372 prepare_res
"$name-nitc-d1.dat" "nitc-d" "nitc with --separate --direct-call-monomorph"
373 run_compiler
"nitc-d1" .
/nitc
--direct
-call
-monomorph
374 prepare_res
"$name-nitc-d2.dat" "nitc-d2" "nitc with --separate --direct-call-monomorph2"
375 run_compiler
"nitc-d2" .
/nitc
--direct
-call
-monomorph
--direct
-call
-monomorph0
380 if test -n
"$html"; then
381 echo >>"$html" "</body></html>"
384 echo >>"$xml" "</testsuite></testsuites>"
386 if test -n
"$died"; then
387 echo "Some commands failed"