2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 ## CONFIGURATION OPTIONS ##
22 # Default number of times a command must be run with bench_command
23 # Can be overrided with 'the option -n'
26 ### HELPER FUNCTIONS ##
34 # Run a single command multiple time and store the execution times
35 # in the current $res file.
37 # $1: title of the command
38 # $2: long desription of the command
39 # rest: the command to execute
40 function bench_command
()
42 if [ "$dry_run" = "true" ]; then return; fi
47 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
49 echo "$title" > "$timeout"
50 echo "# $desc" >> "$timeout"
51 echo "\$ $@" >> "$timeout"
53 echo "** [$title] $desc **"
56 # Execute the commands $count times
57 for i
in `seq 1 "$count"`; do
58 /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@" > $outputopts 2>&1 || die
"$1: failed"
63 line
=`compute_stats "$timeout"`
68 # Run a simble command witout storing the execution time
69 # Used to display command on verbose and skip long executions when dry_run is given
70 # $@ command to execute
71 function run_command
()
73 if [ "$dry_run" = "true" ]; then return; fi
75 "$@" || die
"$@: failed"
78 # perl function to compute min/max/avg.
79 # used by bench_command
82 # return: first min max avg label
83 function compute_stats
()
87 perl
- "$file" <<'END'
101 $len = scalar @lines
;
103 @lines
= sort {$a <=> $b} @lines
;
105 $max = $lines[$#lines];
107 for $i (@lines
) { $avg += $i; }
109 print
"${first} ${min} ${max} ${avg} \"${label}\"\n";
111 print
"${first} ${first} ${first} ${first} \"${label}\"\n";
116 # Create a new $res to be used in the next plot
121 function prepare_res
()
126 if [ "$plots" = "" ]; then
127 plots
="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
129 plots
="$plots replot '$1' using 4:2:3 ti '$2';"
131 if [ "$dry_run" = "true" ]; then return; fi
132 echo "# [$2] $3" > "$res"
133 echo "# first min max avg title" >> "$res"
136 # Plot the last $res as an histogram
137 # $1: plot file (eg toto.gnu)
143 set style data histogram
;
144 set style histogram cluster gap
2;
145 set style histogram errorbars linewidth
1;
146 set style fill solid
0.3 border
-1;
149 set xtic nomirror rotate by
-45 scale
0 font
',8';
151 set ylabel
"time (s)"
154 echo "# gnuplot -p $1"
159 # Check if the test should be skiped according to its name
160 # $1: name of the test
161 # $2: description of the test
162 # $NOTSKIPED: arguments
165 if test -z
"$NOTSKIPED"; then
169 if test "$NOTSKIPED" = "all"; then
171 elif echo "$1" |
grep "$NOTSKIPED" >/dev
/null
2>&1; then
172 : # Found one to execute
182 ## GLOBAL VARIABLES ##
184 # The current $res (set by prepare_res, used by bench_command)
187 # The current stuff to plot (set by prepare_res, used by plot)
192 # Run standards benchs on a compiler command
194 # rest: command to run (executable + options)
195 function run_compiler
()
199 run_command
"$@" nitg.nit
-o
"nitg.$title.bin"
200 bench_command
"nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
201 run_command
"$@" nit.nit
-o
"nit.$title.bin"
202 bench_command
"nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit
-- -n rapid_type_analysis.nit
203 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
204 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
205 run_command
"$@" ..
/tests
/shootout_binarytrees.nit
-o
"bintrees.$title.bin"
206 bench_command
"bintrees" "shootout_binarytrees" "./bintrees.$title.bin" 17
213 echo "run_bench: [options]* benchname"
214 echo " -v: verbose mode"
215 echo " -n count: number of execution for each bar (default: $count)"
216 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
217 echo " -h: this help"
221 while [ "$stop" = false
]; do
223 -v
) verbose
=true
; shift;;
225 -n
) count
="$2"; shift; shift;;
226 --dry
) dry_run
=true
; shift;;
233 if test -z
"$NOTSKIPED"; then
235 echo "List of available benches:"
236 echo "* all: run all the benches"
241 test -f .
/nitc_3 || .
/ncall.sh
-O
242 test -f .
/nitg || .
/nitc_3 nitg.nit
-O
-v
244 ## EFFECTIVE BENCHS ##
246 function bench_nitg_bootstrap
()
249 skip_test
"$name" && return
250 prepare_res
"$name.dat" "" "Steps of the bootstrap of nitg by nitc"
252 cp .
/nitc_3 .
/nitc_nitc.bin
253 bench_command
"c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" .
/nitc_nitc.bin
-O nitc.nit
-o nitc_nitc.bin
254 bench_command
"c/c g" "nitc_nitc nitg.nit -> nitg_nitc" .
/nitc_nitc.bin
-O nitg.nit
-o nitg_nitc.bin
255 bench_command
"g/c g" "nitg_nitc nitg.nit -> nitg_nitg" .
/nitg_nitc.bin nitg.nit
-o nitg_nitg.bin
256 bench_command
"g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" .
/nitg_nitg.bin nitg.nit
-o nitg_nitg.bin
262 function bench_steps
()
265 skip_test
"$name" && return
266 prepare_res
"$name-nitc.dat" "nitc" "Various steps of nitc"
267 bench_command
"parse" "" .
/nitc_3
--only
-parse nitg.nit
268 bench_command
"metamodel" "" .
/nitc_3
--only
-metamodel nitg.nit
269 bench_command
"generate c" "" .
/nitc_3
--no
-cc nitg.nit
270 bench_command
"full" "" .
/nitc_3
-O nitg.nit
-o
"nitg_nitg.bin"
272 prepare_res
"$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
273 bench_command
"parse" "" .
/nitc_3
--global
--only
-parse nitg.nit
274 bench_command
"metamodel" "" .
/nitc_3
--global
--only
-metamodel nitg.nit
275 bench_command
"generate c" "" .
/nitc_3
--global
--no
-cc nitg.nit
276 bench_command
"full" "" .
/nitc_3
-O
--global nitg.nit
-o
"nitg_nitc-g.bin"
278 prepare_res
"$name-nitg.dat" "nitg" "Various steps of nitg"
279 bench_command
"parse" "" .
/nitg
--only
-parse nitg.nit
280 bench_command
"metamodel" "" .
/nitg
--only
-metamodel nitg.nit
281 bench_command
"generate c" "" .
/nitg
--no
-cc nitg.nit
282 bench_command
"full" "" .
/nitg nitg.nit
-o
"nitg_nitg.bin"
284 prepare_res
"$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
285 bench_command
"parse" "" .
/nitg
--erasure
--only
-parse nitg.nit
286 bench_command
"metamodel" "" .
/nitg
--erasure
--only
-metamodel nitg.nit
287 bench_command
"generate c" "" .
/nitg
--erasure
--no
-cc nitg.nit
288 bench_command
"full" "" .
/nitg
--erasure nitg.nit
-o
"nitg_nitg-e.bin"
294 # $#: options to compare
295 function bench_nitg_options
()
299 name
="$FUNCNAME-$tag"
300 skip_test
"$name" && return
301 prepare_res
"$name.dat" "no options" "nitg without options"
302 run_compiler
"nitg" .
/nitg
304 if test -n
"$2"; then
305 prepare_res
"$name-all.dat" "all" "nitg with all options $@"
306 run_compiler
"nitg-$tag" .
/nitg $@
310 prepare_res
"$name$opt.dat" "$opt" "nitg with option $opt"
311 run_compiler
"nitg$opt" .
/nitg
$opt
316 bench_nitg_options
"hardening" --hardening
317 bench_nitg_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
319 function bench_nitg
-s_options
()
323 name
="$FUNCNAME-$tag"
324 skip_test
"$name" && return
325 prepare_res
"$name.dat" "no options" "nitg-s without options"
326 run_compiler
"nitg-s" .
/nitg
--separate
328 if test -n
"$2"; then
329 prepare_res
"$name-all.dat" "all" "nitg-s with all options $@"
330 run_compiler
"nitg-s-$tag" .
/nitg
--separate $@
334 prepare_res
"$name$opt.dat" "$opt" "nitg-s with option $opt"
335 run_compiler
"nitg-s$opt" .
/nitg
--separate
$opt
340 bench_nitg
-s_options
"slower" --hardening
--no
-inline
-intern
--generic
-resolution
-tree
341 bench_nitg
-s_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
342 bench_nitg
-s_options
"faster" --inline
-coloring
-numbers
343 bench_nitg
-s_options
"typing" --bm
-typing
--phmod
-typing
--phand
-typing
345 function bench_nitg
-e_options
()
349 name
="$FUNCNAME-$tag"
350 skip_test
"$name" && return
351 prepare_res
"$name.dat" "no options" "nitg-e without options"
352 run_compiler
"nitg-e" .
/nitg
--erasure
354 if test -n
"$2"; then
355 prepare_res
"$name-all.dat" "all" "nitg-e with all options $@"
356 run_compiler
"nitg-e-$tag" .
/nitg
--erasure $@
360 prepare_res
"$name$opt.dat" "$opt" "nitg-e with option $opt"
361 run_compiler
"nitg-e$opt" .
/nitg
--erasure
$opt
366 bench_nitg
-e_options
"slower" --hardening
--no
-inline
-intern
367 bench_nitg
-e_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
--no
-check
-erasure
-cast
368 bench_nitg
-e_options
"faster" --inline
-coloring
-numbers
369 bench_nitg
-e_options
"typing" --bm
-typing
--phmod
-typing
--phand
-typing
371 function bench_nitc_gc
()
374 skip_test
"$name" && return
375 for gc
in nitgc boehm malloc large
; do
376 prepare_res
"$name-$gc".dat
"$gc" "nitc with gc=$gc"
377 export NIT_GC_OPTION
="$gc"
378 run_compiler
"nitc" .
/nitc_3
-O
385 function bench_nitc_boost
()
388 skip_test
"$name" && return
389 prepare_res
"$name-slow.dat" "no -O" "nitc without -O"
390 run_compiler
"nitc_slow" .
/nitc_3
391 prepare_res
"$name-fast.dat" "-O" "nitc with -O"
392 run_compiler
"nitc" .
/nitc_3
-O
398 function bench_engines
()
401 skip_test
"$name" && return
402 prepare_res
"$name-nitc.dat" "nitc" "nitc"
403 run_compiler
"nitc" .
/nitc_3
-O
404 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
405 run_compiler
"nitc-g" .
/nitc_3
-O
--global
406 prepare_res
"$name-nitg.dat" "nitg" "nitg"
407 run_compiler
"nitg" .
/nitg
408 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
409 run_compiler
"nitg-s" .
/nitg
--separate
410 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
411 run_compiler
"nitg-e" .
/nitg
--erasure
416 function bench_compilation_time
419 skip_test
"$name" && return
420 prepare_res
"$name-nitc.dat" "nitc" "nitc"
421 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
422 bench_command
`basename "$i" .nit` "" .
/nitc_3
-O
"$i" --no
-cc
424 prepare_res
"$name-nitg.dat" "nitg" "nitg"
425 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
426 bench_command
`basename "$i" .nit` "" .
/nitg
"$i" --no
-cc
428 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --erasure"
429 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
430 bench_command
`basename "$i" .nit` "" .
/nitg
--erasure
"$i" --no
-cc
434 bench_compilation_time
436 if test -n
"$died"; then
437 echo "Some commands failed"