2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_common.sh
21 source .
/bench_plot.sh
23 ## CONFIGURATION OPTIONS ##
25 # Default number of times a command must be run with bench_command
26 # Can be overrided with 'the option -n'
29 ### HELPER FUNCTIONS ##
39 # Run standards benchs on a compiler command
41 # rest: command to run (executable + options)
42 function run_compiler
()
46 if test -n
"$fast"; then
47 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
48 bench_command
"nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v
--global
--no
-cc ..
/src
/test_parser.nit
49 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
50 bench_command
"nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/location.nit
51 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
52 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
53 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
54 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
56 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
57 bench_command
"nitg-g" "nitg --global --no-cc ../src/nitls.nit" "./nitg.$title.bin" -v
--global
--no
-cc ..
/src
/nitls.nit
58 bench_command
"nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v
--no
-cc
--separate ..
/src
/nitg.nit
59 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
60 bench_command
"nit" "nit ../src/test_parser.nit ../src/nitls.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/nitls.nit
61 run_command
"$@" ..
/src
/nitdoc.nit
-o
"nitdoc.$title.bin"
62 rm -r out
2> /dev
/null
63 mkdir out
2> /dev
/null
64 bench_command
"nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ..
/src
/nitls.nit
-d out
65 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
66 bench_command
"shoot" "shoot_logic 15" "./shoot.$title.bin" 15
67 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
68 bench_command
"bintrees" "bench_bintree_gen 17" "./bintrees.$title.bin" 17
69 #run_command "$@" "../contrib/pep8analysis/src/pep8analysis.nit" -o "pep8a.$title.bin"
70 #bench_command "pep8analisis" "bench_pep8analisis" "./pep8a.$title.bin" "../contrib/pep8analysis/tests/privat/"*.pep
71 run_command
"$@" "../lib/ai/examples/queens.nit" -o
"queens.$title.bin"
72 bench_command
"queens" "bench_queens 13" "./queens.$title.bin" 13
73 run_command
"$@" "../lib/ai/examples/puzzle.nit" -o
"puzzle.$title.bin"
74 bench_command
"puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi
77 rm -r
*.bin .nit_compile out
84 echo "run_bench: [options]* benchname"
85 echo " -v: verbose mode"
86 echo " -n count: number of execution for each bar (default: $count)"
87 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
88 echo " --fast: Run less and faster tests"
89 echo " --html: Generate and HTML output"
94 while [ "$stop" = false
]; do
96 -v
) verbose
=true
; shift;;
98 -n
) count
="$2"; shift; shift;;
99 --dry
) dry_run
=true
; shift;;
100 --fast
) fast
=true
; shift;;
101 --html
) html
="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
106 xml
="bench_engines.xml"
107 echo "<testsuites><testsuite>" > "$xml"
111 if test -z
"$NOTSKIPED"; then
113 echo "List of available benches:"
114 echo "* all: run all the benches"
119 # get the bootstrapped nitg
122 ## EFFECTIVE BENCHS ##
124 function bench_steps
()
127 skip_test
"$name" && return
128 prepare_res
"$name-nitg.dat" "nitg-g" "Various steps of nitg --global"
129 bench_command
"parse" "" .
/nitg
--global
--only
-parse ..
/src
/nitg.nit
130 bench_command
"metamodel" "" .
/nitg
--global
--only
-metamodel ..
/src
/nitg.nit
131 bench_command
"generate c" "" .
/nitg
--global
--no
-cc ..
/src
/nitg.nit
132 bench_command
"full" "" .
/nitg
--global ..
/src
/nitg.nit
-o
"nitg_nitg.bin"
134 prepare_res
"$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
135 bench_command
"parse" "" .
/nitg
--separate
--only
-parse ..
/src
/nitg.nit
136 bench_command
"metamodel" "" .
/nitg
--separate
--only
-metamodel ..
/src
/nitg.nit
137 bench_command
"generate c" "" .
/nitg
--separate
--no
-cc ..
/src
/nitg.nit
138 bench_command
"full" "" .
/nitg
--separate ..
/src
/nitg.nit
-o
"nitg_nitg-e.bin"
140 prepare_res
"$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
141 bench_command
"parse" "" .
/nitg
--erasure
--only
-parse ..
/src
/nitg.nit
142 bench_command
"metamodel" "" .
/nitg
--erasure
--only
-metamodel ..
/src
/nitg.nit
143 bench_command
"generate c" "" .
/nitg
--erasure
--no
-cc ..
/src
/nitg.nit
144 bench_command
"full" "" .
/nitg
--erasure ..
/src
/nitg.nit
-o
"nitg_nitg-e.bin"
150 # $#: options to compare
151 function bench_nitg
-g_options
()
155 name
="$FUNCNAME-$tag"
156 skip_test
"$name" && return
157 prepare_res
"$name.dat" "no options" "nitg-g without options"
158 run_compiler
"nitg-g" .
/nitg
--global
160 if test "$1" = NOALL
; then
162 elif test -n
"$2"; then
163 prepare_res
"$name-all.dat" "all" "nitg-g with all options $@"
164 run_compiler
"nitg-g-$tag" .
/nitg
--global $@
169 prepare_res
"$name$ot.dat" "$opt" "nitg-g with option $opt"
170 run_compiler
"nitg-g$ot" .
/nitg
--global
$opt
175 bench_nitg
-g_options
"slower" --hardening
--no
-shortcut
-range
176 bench_nitg
-g_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
178 function bench_nitg
-s_options
()
182 name
="$FUNCNAME-$tag"
183 skip_test
"$name" && return
184 prepare_res
"$name.dat" "no options" "nitg-s without options"
185 run_compiler
"nitg-s" .
/nitg
--separate
187 if test "$1" = NOALL
; then
189 elif test -n
"$2"; then
190 prepare_res
"$name-all.dat" "all" "nitg-s with all options $@"
191 run_compiler
"nitg-s-$tag" .
/nitg
--separate $@
196 prepare_res
"$name-$ot.dat" "$opt" "nitg-s with option $opt"
197 run_compiler
"nitg-s$ot" .
/nitg
--separate
$opt
202 bench_nitg
-s_options
"slower" --hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
"--no-gcc-directive likely --no-gcc-directive noreturn"
203 bench_nitg
-s_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
204 bench_nitg
-s_options
"faster" --skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
"--inline-some-methods --direct-call-monomorph" ""
206 function bench_nitg
-e_options
()
210 name
="$FUNCNAME-$tag"
211 skip_test
"$name" && return
212 prepare_res
"$name.dat" "no options" "nitg-e without options"
213 run_compiler
"nitg-e" .
/nitg
--erasure
215 if test "$1" = NOALL
; then
217 elif test -n
"$2"; then
218 prepare_res
"$name-all.dat" "all" "nitg-e with all options $@"
219 run_compiler
"nitg-e-$tag" .
/nitg
--erasure $@
224 prepare_res
"$name$ot.dat" "$opt" "nitg-e with option $opt"
225 run_compiler
"nitg-e$ot" .
/nitg
--erasure
$opt
230 bench_nitg
-e_options
"slower" --hardening
--no
-shortcut
-equal
--no
-union
-attribute
--no
-shortcut
-range
--no
-inline
-intern
231 bench_nitg
-e_options
"nocheck" --no
-check
-null
--no
-check
-autocast
--no
-check
-attr
-isset
--no
-check
-covariance
--no
-check
-assert
--no
-check
-erasure
-cast
232 bench_nitg
-e_options
"faster" --skip
-dead
-methods
--inline
-coloring
-numbers
--inline
-some
-methods
--direct
-call
-monomorph
--rta
234 function bench_engines
()
237 skip_test
"$name" && return
238 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
239 run_compiler
"nitg-s" .
/nitg
--separate
240 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
241 run_compiler
"nitg-e" .
/nitg
--erasure
242 prepare_res
"$name-nitg-sg.dat" "nitg-sg" "nitg with --separate --semi-global"
243 run_compiler
"nitg-sg" .
/nitg
--separate
--semi
-global
244 prepare_res
"$name-nitg-eg.dat" "nitg-eg" "nitg with --erasure --semi-global"
245 run_compiler
"nitg-eg" .
/nitg
--erasure
--semi
-global
246 prepare_res
"$name-nitg-egt.dat" "nitg-egt" "nitg with --erasure --semi-global --rta"
247 run_compiler
"nitg-egt" .
/nitg
--erasure
--semi
-global
--rta
248 prepare_res
"$name-nitg-g.dat" "nitg-g" "nitg with --global"
249 run_compiler
"nitg-g" .
/nitg
--global
254 function bench_nitg
-e_gc
()
257 skip_test
"$name" && return
258 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
259 run_compiler
"nitg-e" .
/nitg
--erasure
260 prepare_res
"$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
261 NIT_GC_OPTION
="malloc" run_compiler
"nitg-e-malloc" .
/nitg
--erasure
262 prepare_res
"$name-nitg-e-large.dat" "nitg-e-large" "nitg with --erasure and large"
263 NIT_GC_OPTION
="large" run_compiler
"nitg-e-large" .
/nitg
--erasure
268 function bench_cc_nitg
-e
()
271 skip_test
"$name" && return
272 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
273 f
=`echo "$o" | cut -f1 -d:`
274 o
=`echo "$o" | cut -f2 -d:`
275 prepare_res
"$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
276 run_compiler
"nitg-e-$f" .
/nitg
--erasure
--make-flags
"$o"
282 function bench_policy
()
285 skip_test
"$name" && return
286 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
287 run_compiler
"nitg-s" .
/nitg
--separate
288 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
289 run_compiler
"nitg-e" .
/nitg
--erasure
290 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
291 run_compiler
"nitg-su" .
/nitg
--separate
--no
-check
-covariance
292 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
293 run_compiler
"nitg-eu" .
/nitg
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
298 function bench_nullables
()
301 skip_test
"$name" && return
302 prepare_res
"$name-nitc.dat" "nitc" "nitc no options"
303 run_compiler
"nitc" .
/nitg
--separate
304 prepare_res
"$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset"
305 run_compiler
"nitc" .
/nitg
--separate
--no
-check
-attr
-isset
306 prepare_res
"$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute"
307 run_compiler
"nitc" .
/nitg
--separate
--no
-union
-attribute
308 prepare_res
"$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset"
309 run_compiler
"nitc" .
/nitg
--separate
--no
-union
-attribute
--no
-check
-attr
-isset
314 function bench_compilation_time
317 skip_test
"$name" && return
318 prepare_res
"$name-nitg-g.dat" "nitg-g" "nitg --global"
319 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
320 bench_command
`basename "$i" .nit` "" .
/nitg
--global
"$i" --no
-cc
322 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg --separate"
323 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
324 bench_command
`basename "$i" .nit` "" .
/nitg
--separate
"$i" --no
-cc
326 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --erasure"
327 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
328 bench_command
`basename "$i" .nit` "" .
/nitg
--erasure
"$i" --no
-cc
332 bench_compilation_time
334 if test -n
"$html"; then
335 echo >>"$html" "</body></html>"
338 echo >>"$xml" "</testsuite></testsuites>"
340 if test -n
"$died"; then
341 echo "Some commands failed"