2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_plot.sh
22 ## CONFIGURATION OPTIONS ##
24 # Default number of times a command must be run with bench_command
25 # Can be overrided with 'the option -n'
28 pep8analysis
=..
/..
/pep8analysis
30 ### HELPER FUNCTIONS ##
38 # Run a single command multiple time and store the execution times
39 # in the current $res file.
41 # $1: title of the command
42 # $2: long desription of the command
43 # rest: the command to execute
44 function bench_command
()
46 if [ "$dry_run" = "true" ]; then return; fi
51 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
53 echo "$title" > "$timeout"
54 echo "# $desc" >> "$timeout"
55 echo "\$ $@" >> "$timeout"
57 echo "** [$title] $desc **"
60 # Execute the commands $count times
61 for i
in `seq 1 "$count"`; do
62 /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@" > $outputopts 2>&1 || die
"$1: failed"
67 line
=`compute_stats "$timeout"`
72 # Run a simble command witout storing the execution time
73 # Used to display command on verbose and skip long executions when dry_run is given
74 # $@ command to execute
75 function run_command
()
77 if [ "$dry_run" = "true" ]; then return; fi
79 "$@" || die
"$@: failed"
82 # Check if the test should be skiped according to its name
83 # $1: name of the test
84 # $2: description of the test
85 # $NOTSKIPED: arguments
88 if test -z
"$NOTSKIPED"; then
92 if test "$NOTSKIPED" = "all"; then
94 elif echo "$1" |
egrep "$NOTSKIPED" >/dev
/null
2>&1; then
95 : # Found one to execute
99 if test -n
"$html"; then
100 echo >>"$html" "<h2>$1</h2>"
110 # Run standards benchs on a compiler command
112 # rest: command to run (executable + options)
113 function run_compiler
()
117 if test -n
"$fast"; then
118 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
119 bench_command
"nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v
--global
--no
-cc ..
/src
/test_parser.nit
120 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
121 bench_command
"nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/location.nit
122 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
123 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
124 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
125 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
127 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
128 bench_command
"nitg-g" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v
--global
--no
-cc ..
/src
/nitmetrics.nit
129 bench_command
"nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v
--no
-cc
--separate ..
/src
/nitg.nit
130 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
131 bench_command
"nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/rapid_type_analysis.nit
132 run_command
"$@" ..
/examples
/shoot
/src
/shoot_logic.nit
-o
"shoot.$title.bin"
133 bench_command
"shoot" "shoot_logic 30" "./shoot.$title.bin" 30
134 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
135 bench_command
"bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
136 if test -f
"$pep8analysis/src/pep8analysis.nit"; then
137 run_command
"$@" "$pep8analysis/src/pep8analysis.nit" -I
"$pep8analysis/lib" -o
"pep8a.$title.bin"
138 bench_command
"pep8analisis" "bench_bintree_gen 18" "./pep8a.$title.bin" "$pep8analysis/tests/privat/"*.pep
147 echo "run_bench: [options]* benchname"
148 echo " -v: verbose mode"
149 echo " -n count: number of execution for each bar (default: $count)"
150 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
151 echo " --fast: Run less and faster tests"
152 echo " --html: Generate and HTML output"
153 echo " -h: this help"
157 while [ "$stop" = false
]; do
159 -v
) verbose
=true
; shift;;
161 -n
) count
="$2"; shift; shift;;
162 --dry
) dry_run
=true
; shift;;
163 --fast
) fast
=true
; shift;;
164 --html
) html
="index.html"; echo >"$html" "<html><head></head><body>"; shift;;
171 if test -z
"$NOTSKIPED"; then
173 echo "List of available benches:"
174 echo "* all: run all the benches"
179 test -f ..
/src
/nitc_3 || ..
/src
/ncall.sh
-O
180 test -f .
/nitg || ..
/src
/nitc_3 ..
/src
/nitg.nit
-O
-v
182 ## EFFECTIVE BENCHS ##
184 function bench_nitg_bootstrap
()
187 skip_test
"$name" && return
188 prepare_res
"$name.dat" "" "Steps of the bootstrap of nitg by nitc"
190 cp ..
/src
/nitc_3 .
/nitc_nitc.bin
191 bench_command
"c/c c" "nitc_nitc ../src/nitc.nit -> nitc_nitc (stability)" .
/nitc_nitc.bin
-O ..
/src
/nitc.nit
-o nitc_nitc.bin
192 bench_command
"c/c g" "nitc_nitc ../src/nitg.nit -> nitg_nitc" .
/nitc_nitc.bin
-O ..
/src
/nitg.nit
-o nitg_nitc.bin
193 bench_command
"g/c g" "nitg_nitc ../src/nitg.nit -> nitg_nitg" .
/nitg_nitc.bin ..
/src
/nitg.nit
-o nitg_nitg.bin
194 bench_command
"g/g g" "nitg_nitg ../src/nitg.nit -> nitg_nitg (stability)" .
/nitg_nitg.bin ..
/src
/nitg.nit
-o nitg_nitg.bin
200 function bench_steps
()
203 skip_test
"$name" && return
204 prepare_res
"$name-nitc.dat" "nitc" "Various steps of nitc"
205 bench_command
"parse" "" ..
/src
/nitc_3
--only
-parse ..
/src
/nitg.nit
206 bench_command
"metamodel" "" ..
/src
/nitc_3
--only
-metamodel ..
/src
/nitg.nit
207 bench_command
"generate c" "" ..
/src
/nitc_3
--no
-cc ..
/src
/nitg.nit
208 bench_command
"full" "" ..
/src
/nitc_3
-O ..
/src
/nitg.nit
-o
"nitg_nitg.bin"
210 prepare_res
"$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
211 bench_command
"parse" "" ..
/src
/nitc_3
--global
--only
-parse ..
/src
/nitg.nit
212 bench_command
"metamodel" "" ..
/src
/nitc_3
--global
--only
-metamodel ..
/src
/nitg.nit
213 bench_command
"generate c" "" ..
/src
/nitc_3
--global
--no
-cc ..
/src
/nitg.nit
214 bench_command
"full" "" ..
/src
/nitc_3
-O
--global ..
/src
/nitg.nit
-o
"nitg_nitc-g.bin"
216 prepare_res
"$name-nitg.dat" "nitg-g" "Various steps of nitg --global"
217 bench_command
"parse" "" .
/nitg
--global
--only
-parse ..
/src
/nitg.nit
218 bench_command
"metamodel" "" .
/nitg
--global
--only
-metamodel ..
/src
/nitg.nit
219 bench_command
"generate c" "" .
/nitg
--global
--no
-cc ..
/src
/nitg.nit
220 bench_command
"full" "" .
/nitg
--global ..
/src
/nitg.nit
-o
"nitg_nitg.bin"
222 prepare_res
"$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
223 bench_command
"parse" "" .
/nitg
--separate
--only
-parse ..
/src
/nitg.nit
224 bench_command
"metamodel" "" .
/nitg
--separate
--only
-metamodel ..
/src
/nitg.nit
225 bench_command
"generate c" "" .
/nitg
--separate
--no
-cc ..
/src
/nitg.nit
226 bench_command
"full" "" .
/nitg
--separate ..
/src
/nitg.nit
-o
"nitg_nitg-e.bin"
228 prepare_res
"$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
229 bench_command
"parse" "" .
/nitg
--erasure
--only
-parse ..
/src
/nitg.nit
230 bench_command
"metamodel" "" .
/nitg
--erasure
--only
-metamodel ..
/src
/nitg.nit
231 bench_command
"generate c" "" .
/nitg
--erasure
--no
-cc ..
/src
/nitg.nit
232 bench_command
"full" "" .
/nitg
--erasure ..
/src
/nitg.nit
-o
"nitg_nitg-e.bin"
238 # $#: options to compare
239 function bench_nitg
-g_options
()
243 name
="$FUNCNAME-$tag"
244 skip_test
"$name" && return
245 prepare_res
"$name.dat" "no options" "nitg-g without options"
246 run_compiler
"nitg-g" .
/nitg
--global
248 if test "$1" = NOALL
; then
250 elif test -n
"$2"; then
251 prepare_res
"$name-all.dat" "all" "nitg-g with all options $@"
252 run_compiler
"nitg-g-$tag" .
/nitg
--global $@
256 prepare_res
"$name$opt.dat" "$opt" "nitg-g with option $opt"
257 run_compiler
"nitg-g$opt" .
/nitg
--global
$opt
262 bench_nitg
-g_options
"hardening" --hardening
263 bench_nitg
-g_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
265 function bench_nitg
-s_options
()
269 name
="$FUNCNAME-$tag"
270 skip_test
"$name" && return
271 prepare_res
"$name.dat" "no options" "nitg-s without options"
272 run_compiler
"nitg-s" .
/nitg
--separate
274 if test "$1" = NOALL
; then
276 elif test -n
"$2"; then
277 prepare_res
"$name-all.dat" "all" "nitg-s with all options $@"
278 run_compiler
"nitg-s-$tag" .
/nitg
--separate $@
282 prepare_res
"$name$opt.dat" "$opt" "nitg-s with option $opt"
283 run_compiler
"nitg-s$opt" .
/nitg
--separate
$opt
288 bench_nitg
-s_options
"slower" --hardening
--no
-inline
-intern
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
289 bench_nitg
-s_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
290 bench_nitg
-s_options
"faster" --inline
-coloring
-numbers
291 bench_nitg
-s_options
"typing" NOALL
--bm
-typing
--phand
-typing
293 function bench_nitg
-e_options
()
297 name
="$FUNCNAME-$tag"
298 skip_test
"$name" && return
299 prepare_res
"$name.dat" "no options" "nitg-e without options"
300 run_compiler
"nitg-e" .
/nitg
--erasure
302 if test "$1" = NOALL
; then
304 elif test -n
"$2"; then
305 prepare_res
"$name-all.dat" "all" "nitg-e with all options $@"
306 run_compiler
"nitg-e-$tag" .
/nitg
--erasure $@
310 prepare_res
"$name$opt.dat" "$opt" "nitg-e with option $opt"
311 run_compiler
"nitg-e$opt" .
/nitg
--erasure
$opt
316 bench_nitg
-e_options
"slower" --hardening
--no
-inline
-intern
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
317 bench_nitg
-e_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
--no
-check
-erasure
-cast
318 bench_nitg
-e_options
"faster" --inline
-coloring
-numbers
319 bench_nitg
-e_options
"typing" NOALL
--bm
-typing
# --phand-typing
321 function bench_nitc_gc
()
324 skip_test
"$name" && return
325 for gc
in nitgc boehm malloc large
; do
326 prepare_res
"$name-$gc.dat" "$gc" "nitc with gc=$gc"
327 export NIT_GC_OPTION
="$gc"
328 run_compiler
"nitc" ..
/src
/nitc_3
-O
335 function bench_nitc_boost
()
338 skip_test
"$name" && return
339 prepare_res
"$name-slow.dat" "no -O" "nitc without -O"
340 run_compiler
"nitc_slow" ..
/src
/nitc_3
341 prepare_res
"$name-fast.dat" "-O" "nitc with -O"
342 run_compiler
"nitc" ..
/src
/nitc_3
-O
348 function bench_engines
()
351 skip_test
"$name" && return
352 prepare_res
"$name-nitc.dat" "nitc" "nitc"
353 run_compiler
"nitc" ..
/src
/nitc_3
-O
354 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
355 run_compiler
"nitc-g" ..
/src
/nitc_3
-O
--global
356 prepare_res
"$name-nitg-g.dat" "nitg-g" "nitg with --global"
357 run_compiler
"nitg-g" .
/nitg
--global
358 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
359 run_compiler
"nitg-s" .
/nitg
--separate
360 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
361 run_compiler
"nitg-e" .
/nitg
--erasure
366 function bench_nitc_vc_nitg
-e
()
369 skip_test
"$name" && return
370 prepare_res
"$name-nitc.dat" "nitc" "nitc"
371 run_compiler
"nitc" ..
/src
/nitc_3
-O
372 prepare_res
"$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
373 NIT_GC_OPTION
="malloc" run_compiler
"nitc" ..
/src
/nitc_3
-O
374 prepare_res
"$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
375 NIT_GC_OPTION
="boehm" run_compiler
"nitc" ..
/src
/nitc_3
-O
376 prepare_res
"$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
377 NIT_GC_OPTION
="malloc" run_compiler
"nitg-e-nc-malloc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
378 prepare_res
"$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
379 run_compiler
"nitg-e-nc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
380 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
381 run_compiler
"nitg-e" .
/nitg
--erasure
386 function bench_nitg
-e_gc
()
389 skip_test
"$name" && return
390 prepare_res
"$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
391 NIT_GC_OPTION
="malloc" run_compiler
"nitg-e-malloc" .
/nitg
--erasure
392 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
393 run_compiler
"nitg-e" .
/nitg
--erasure
398 function bench_cc_nitg
-e
()
401 skip_test
"$name" && return
402 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
403 f
=`echo "$o" | cut -f1 -d:`
404 o
=`echo "$o" | cut -f2 -d:`
405 prepare_res
"$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
406 run_compiler
"nitg-e-$f" .
/nitg
--erasure
--make-flags
"$o"
412 function bench_policy
()
415 skip_test
"$name" && return
416 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
417 run_compiler
"nitg-s" .
/nitg
--separate
418 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
419 run_compiler
"nitg-e" .
/nitg
--erasure
420 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
421 run_compiler
"nitg-su" .
/nitg
--separate
--no
-check
-covariance
422 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
423 run_compiler
"nitg-eu" .
/nitg
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
428 function bench_compilation_time
431 skip_test
"$name" && return
432 prepare_res
"$name-nitc.dat" "nitc" "nitc"
433 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
434 bench_command
`basename "$i" .nit` "" ..
/src
/nitc_3
-O
"$i" --no
-cc
436 prepare_res
"$name-nitg.dat" "nitg-g" "nitg --global"
437 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
438 bench_command
`basename "$i" .nit` "" .
/nitg
--global
"$i" --no
-cc
440 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --separate"
441 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
442 bench_command
`basename "$i" .nit` "" .
/nitg
--separate
"$i" --no
-cc
444 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --erasure"
445 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
446 bench_command
`basename "$i" .nit` "" .
/nitg
--erasure
"$i" --no
-cc
450 bench_compilation_time
452 if test -n
"$html"; then
453 echo >>"$html" "</body></html>"
456 if test -n
"$died"; then
457 echo "Some commands failed"