2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 ## CONFIGURATION OPTIONS ##
22 # Default number of times a command must be run with bench_command
23 # Can be overrided with 'the option -n'
26 ### HELPER FUNCTIONS ##
34 # Run a single command multiple time and store the execution times
35 # in the current $res file.
37 # $1: title of the command
38 # $2: long desription of the command
39 # rest: the command to execute
40 function bench_command
()
42 if [ "$dry_run" = "true" ]; then return; fi
47 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
49 echo "$title" > "$timeout"
50 echo "# $desc" >> "$timeout"
51 echo "\$ $@" >> "$timeout"
53 echo "** [$title] $desc **"
56 # Execute the commands $count times
57 for i
in `seq 1 "$count"`; do
58 /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@" > $outputopts 2>&1 || die
"$1: failed"
63 line
=`compute_stats "$timeout"`
68 # Run a simble command witout storing the execution time
69 # Used to display command on verbose and skip long executions when dry_run is given
70 # $@ command to execute
71 function run_command
()
73 if [ "$dry_run" = "true" ]; then return; fi
75 "$@" || die
"$@: failed"
78 # perl function to compute min/max/avg.
79 # used by bench_command
82 # return: first min max avg label
83 function compute_stats
()
87 perl
- "$file" <<'END'
101 $len = scalar @lines
;
103 @lines
= sort {$a <=> $b} @lines
;
105 $max = $lines[$#lines];
107 for $i (@lines
) { $avg += $i; }
109 print
"${first} ${min} ${max} ${avg} \"${label}\"\n";
111 print
"${first} ${first} ${first} ${first} \"${label}\"\n";
116 # Create a new $res to be used in the next plot
121 function prepare_res
()
126 if [ "$plots" = "" ]; then
127 plots
="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
129 plots
="$plots replot '$1' using 4:2:3 ti '$2';"
131 if [ "$dry_run" = "true" ]; then return; fi
132 echo "# [$2] $3 ; count=$count" > "$res"
133 echo "# first min max avg title" >> "$res"
136 # Plot the last $res as an histogram
137 # $1: plot file (eg toto.gnu)
143 set style data histogram
;
144 set style histogram cluster gap
2;
145 set style histogram errorbars linewidth
1;
146 set style fill solid
0.3 border
-1;
149 set xtic nomirror rotate by
-45 scale
0 font
',8';
150 set title
"$1 ; avg. on $count-1 runs"
151 set ylabel
"time (s)"
154 echo "# gnuplot -p $1"
159 # Check if the test should be skiped according to its name
160 # $1: name of the test
161 # $2: description of the test
162 # $NOTSKIPED: arguments
165 if test -z
"$NOTSKIPED"; then
169 if test "$NOTSKIPED" = "all"; then
171 elif echo "$1" |
egrep "$NOTSKIPED" >/dev
/null
2>&1; then
172 : # Found one to execute
182 ## GLOBAL VARIABLES ##
184 # The current $res (set by prepare_res, used by bench_command)
187 # The current stuff to plot (set by prepare_res, used by plot)
192 # Run standards benchs on a compiler command
194 # rest: command to run (executable + options)
195 function run_compiler
()
199 if test -n
"$fast"; then
200 run_command
"$@" nitg.nit
-o
"nitg.$title.bin"
201 bench_command
"nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v
--no
-cc test_parser.nit
202 run_command
"$@" nit.nit
-o
"nit.$title.bin"
203 bench_command
"nit" "nit test_parser.nit location.nit" "./nit.$title.bin" -v test_parser.nit
-- -n location.nit
204 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
205 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
206 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
207 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
209 run_command
"$@" nitg.nit
-o
"nitg.$title.bin"
210 bench_command
"nitg" "nitg --no-cc nitstats.nit" "./nitg.$title.bin" -v
--no
-cc nitstats.nit
211 bench_command
"nitg-s" "nitg --separate nitg.nit" "./nitg.$title.bin" -v
--no
-cc
--separate nitg.nit
212 run_command
"$@" nit.nit
-o
"nit.$title.bin"
213 bench_command
"nit" "nit test_parser.nit rapid_type_analysis.nit" "./nit.$title.bin" -v test_parser.nit
-- -n rapid_type_analysis.nit
214 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
215 bench_command
"shoot" "shoot_logic 30" "./shoot.$title.bin" 30
216 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
217 bench_command
"bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
225 echo "run_bench: [options]* benchname"
226 echo " -v: verbose mode"
227 echo " -n count: number of execution for each bar (default: $count)"
228 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
229 echo " --fast: Run less and faster tests"
230 echo " -h: this help"
234 while [ "$stop" = false
]; do
236 -v
) verbose
=true
; shift;;
238 -n
) count
="$2"; shift; shift;;
239 --dry
) dry_run
=true
; shift;;
240 --fast
) fast
=true
; shift;;
247 if test -z
"$NOTSKIPED"; then
249 echo "List of available benches:"
250 echo "* all: run all the benches"
255 test -f .
/nitc_3 || .
/ncall.sh
-O
256 test -f .
/nitg || .
/nitc_3 nitg.nit
-O
-v
258 ## EFFECTIVE BENCHS ##
260 function bench_nitg_bootstrap
()
263 skip_test
"$name" && return
264 prepare_res
"$name.dat" "" "Steps of the bootstrap of nitg by nitc"
266 cp .
/nitc_3 .
/nitc_nitc.bin
267 bench_command
"c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" .
/nitc_nitc.bin
-O nitc.nit
-o nitc_nitc.bin
268 bench_command
"c/c g" "nitc_nitc nitg.nit -> nitg_nitc" .
/nitc_nitc.bin
-O nitg.nit
-o nitg_nitc.bin
269 bench_command
"g/c g" "nitg_nitc nitg.nit -> nitg_nitg" .
/nitg_nitc.bin nitg.nit
-o nitg_nitg.bin
270 bench_command
"g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" .
/nitg_nitg.bin nitg.nit
-o nitg_nitg.bin
276 function bench_steps
()
279 skip_test
"$name" && return
280 prepare_res
"$name-nitc.dat" "nitc" "Various steps of nitc"
281 bench_command
"parse" "" .
/nitc_3
--only
-parse nitg.nit
282 bench_command
"metamodel" "" .
/nitc_3
--only
-metamodel nitg.nit
283 bench_command
"generate c" "" .
/nitc_3
--no
-cc nitg.nit
284 bench_command
"full" "" .
/nitc_3
-O nitg.nit
-o
"nitg_nitg.bin"
286 prepare_res
"$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
287 bench_command
"parse" "" .
/nitc_3
--global
--only
-parse nitg.nit
288 bench_command
"metamodel" "" .
/nitc_3
--global
--only
-metamodel nitg.nit
289 bench_command
"generate c" "" .
/nitc_3
--global
--no
-cc nitg.nit
290 bench_command
"full" "" .
/nitc_3
-O
--global nitg.nit
-o
"nitg_nitc-g.bin"
292 prepare_res
"$name-nitg.dat" "nitg" "Various steps of nitg"
293 bench_command
"parse" "" .
/nitg
--only
-parse nitg.nit
294 bench_command
"metamodel" "" .
/nitg
--only
-metamodel nitg.nit
295 bench_command
"generate c" "" .
/nitg
--no
-cc nitg.nit
296 bench_command
"full" "" .
/nitg nitg.nit
-o
"nitg_nitg.bin"
298 prepare_res
"$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
299 bench_command
"parse" "" .
/nitg
--erasure
--only
-parse nitg.nit
300 bench_command
"metamodel" "" .
/nitg
--erasure
--only
-metamodel nitg.nit
301 bench_command
"generate c" "" .
/nitg
--erasure
--no
-cc nitg.nit
302 bench_command
"full" "" .
/nitg
--erasure nitg.nit
-o
"nitg_nitg-e.bin"
308 # $#: options to compare
309 function bench_nitg_options
()
313 name
="$FUNCNAME-$tag"
314 skip_test
"$name" && return
315 prepare_res
"$name.dat" "no options" "nitg without options"
316 run_compiler
"nitg" .
/nitg
318 if test -n
"$2"; then
319 prepare_res
"$name-all.dat" "all" "nitg with all options $@"
320 run_compiler
"nitg-$tag" .
/nitg $@
324 prepare_res
"$name$opt.dat" "$opt" "nitg with option $opt"
325 run_compiler
"nitg$opt" .
/nitg
$opt
330 bench_nitg_options
"hardening" --hardening
331 bench_nitg_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
333 function bench_nitg
-s_options
()
337 name
="$FUNCNAME-$tag"
338 skip_test
"$name" && return
339 prepare_res
"$name.dat" "no options" "nitg-s without options"
340 run_compiler
"nitg-s" .
/nitg
--separate
342 if test -n
"$2"; then
343 prepare_res
"$name-all.dat" "all" "nitg-s with all options $@"
344 run_compiler
"nitg-s-$tag" .
/nitg
--separate $@
348 prepare_res
"$name$opt.dat" "$opt" "nitg-s with option $opt"
349 run_compiler
"nitg-s$opt" .
/nitg
--separate
$opt
354 bench_nitg
-s_options
"slower" --hardening
--no
-inline
-intern
--generic
-resolution
-tree
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
355 bench_nitg
-s_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
356 bench_nitg
-s_options
"faster" --inline
-coloring
-numbers
357 bench_nitg
-s_options
"typing" --bm
-typing
--phand
-typing
--phmod
-typing
359 function bench_nitg
-e_options
()
363 name
="$FUNCNAME-$tag"
364 skip_test
"$name" && return
365 prepare_res
"$name.dat" "no options" "nitg-e without options"
366 run_compiler
"nitg-e" .
/nitg
--erasure
368 if test -n
"$2"; then
369 prepare_res
"$name-all.dat" "all" "nitg-e with all options $@"
370 run_compiler
"nitg-e-$tag" .
/nitg
--erasure $@
374 prepare_res
"$name$opt.dat" "$opt" "nitg-e with option $opt"
375 run_compiler
"nitg-e$opt" .
/nitg
--erasure
$opt
380 bench_nitg
-e_options
"slower" --hardening
--no
-inline
-intern
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
381 bench_nitg
-e_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
--no
-check
-erasure
-cast
382 bench_nitg
-e_options
"faster" --inline
-coloring
-numbers
383 bench_nitg
-e_options
"typing" --bm
-typing
--phand
-typing
--phmod
-typing
385 function bench_nitc_gc
()
388 skip_test
"$name" && return
389 for gc
in nitgc boehm malloc large
; do
390 prepare_res
"$name-$gc.dat" "$gc" "nitc with gc=$gc"
391 export NIT_GC_OPTION
="$gc"
392 run_compiler
"nitc" .
/nitc_3
-O
399 function bench_nitc_boost
()
402 skip_test
"$name" && return
403 prepare_res
"$name-slow.dat" "no -O" "nitc without -O"
404 run_compiler
"nitc_slow" .
/nitc_3
405 prepare_res
"$name-fast.dat" "-O" "nitc with -O"
406 run_compiler
"nitc" .
/nitc_3
-O
412 function bench_engines
()
415 skip_test
"$name" && return
416 prepare_res
"$name-nitc.dat" "nitc" "nitc"
417 run_compiler
"nitc" .
/nitc_3
-O
418 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
419 run_compiler
"nitc-g" .
/nitc_3
-O
--global
420 prepare_res
"$name-nitg.dat" "nitg" "nitg"
421 run_compiler
"nitg" .
/nitg
422 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
423 run_compiler
"nitg-s" .
/nitg
--separate
424 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
425 run_compiler
"nitg-e" .
/nitg
--erasure
430 function bench_nitc_vc_nitg
-e
()
433 skip_test
"$name" && return
434 prepare_res
"$name-nitc.dat" "nitc" "nitc"
435 run_compiler
"nitc" .
/nitc_3
-O
436 prepare_res
"$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
437 NIT_GC_OPTION
="malloc" run_compiler
"nitc" .
/nitc_3
-O
438 prepare_res
"$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
439 NIT_GC_OPTION
="boehm" run_compiler
"nitc" .
/nitc_3
-O
440 prepare_res
"$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
441 run_compiler
"nitg-e-nc-malloc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
--make-flags
"CFLAGS=\"-O2 -DNOBOEHM\""
442 prepare_res
"$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
443 run_compiler
"nitg-e-nc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
444 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
445 run_compiler
"nitg-e" .
/nitg
--erasure
450 function bench_nitg
-e_gc
()
453 skip_test
"$name" && return
454 prepare_res
"$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
455 run_compiler
"nitg-e-malloc" .
/nitg
--erasure
--make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
456 prepare_res
"$name-nitg-e-noatomic.dat" "nitg-e-noatomic" "nitg with --erasure and no atomic"
457 run_compiler
"nitg-e-noatomic" .
/nitg
--erasure
--make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM_ATOMIC\""
458 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
459 run_compiler
"nitg-e" .
/nitg
--erasure
464 function bench_cc_nitg
-e
()
467 skip_test
"$name" && return
468 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
469 f
=`echo "$o" | cut -f1 -d:`
470 o
=`echo "$o" | cut -f2 -d:`
471 prepare_res
"$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
472 run_compiler
"nitg-e-$f" .
/nitg
--erasure
--make-flags
"$o"
478 function bench_policy
()
481 skip_test
"$name" && return
482 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
483 run_compiler
"nitg-s" .
/nitg
--separate
484 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
485 run_compiler
"nitg-e" .
/nitg
--erasure
486 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
487 run_compiler
"nitg-su" .
/nitg
--separate
--no
-check
-covariance
488 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
489 run_compiler
"nitg-eu" .
/nitg
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
494 function bench_compilation_time
497 skip_test
"$name" && return
498 prepare_res
"$name-nitc.dat" "nitc" "nitc"
499 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
500 bench_command
`basename "$i" .nit` "" .
/nitc_3
-O
"$i" --no
-cc
502 prepare_res
"$name-nitg.dat" "nitg" "nitg"
503 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
504 bench_command
`basename "$i" .nit` "" .
/nitg
"$i" --no
-cc
506 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --erasure"
507 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
508 bench_command
`basename "$i" .nit` "" .
/nitg
--erasure
"$i" --no
-cc
512 bench_compilation_time
514 function bench_typetest_languages
()
517 skip_test
"$name" && return
521 seq="w2_h2 w50_h2 w2_h25 w50_h25"
523 run_command .
/nitg benchs
/gen.nit
524 run_command .
/gen.bin
"${t}_$b" "$b"
527 prepare_res
"$name-g++.dat" "g++" "g++"
529 run_command g
++ "${t}_$b.cpp" -O2
-o
"${t}_$b.g++.bin"
530 bench_command
"$b" "" "./${t}_$b.g++.bin" $s
533 prepare_res
"$name-clang++.dat" "clang++" "clang++"
535 run_command clang
++ "${t}_$b.cpp" -O2
-o
"${t}_$b.clang++.bin"
536 bench_command
"$b" "" "./${t}_$b.clang++.bin" $s
539 prepare_res
"$name-java.dat" "java" "java"
541 run_command javac
${t}_
$b.java
542 bench_command
"$b" "" java
"${t}_$b" $s
545 prepare_res
"$name-scala.dat" "scala" "scala"
547 run_command scalac
${t}_
$b.scala
548 bench_command
"$b" "" scala
"${t}_$b" $s
551 prepare_res
"$name-cs.dat" "c#" "c#"
553 run_command gmcs
${t}_
$b.cs
554 bench_command
"$b" "" mono
"${t}_$b.exe" $s
557 prepare_res
"$name-es.dat" "es" "es"
559 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
561 mv app
${t}_
$b ${t}_
$b.es.bin
562 bench_command
"$b" "" "./${t}_$b.es.bin" $s
565 prepare_res
"$name-se.dat" "se" "se"
567 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
568 bench_command
"$b" "" "./${t}_$b.se.bin" $s
572 #prepare_res "$name-nitg.dat" "nitg" "nitg"
574 # run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
575 # bench_command "$b" "" "./${t}_$b.nitg.bin" $s
578 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg-s"
580 run_command .
/nitg
${t}_
$b.nit
--separate
-o
"${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
581 bench_command
"$b" "" "./${t}_$b.nitg-s.bin" $s
584 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg-su"
586 run_command .
/nitg
${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
587 bench_command
"$b" "" "./${t}_$b.nitg-su.bin" $s
591 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg-e"
593 run_command .
/nitg
${t}_
$b.nit
--erasure
-o
"${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
594 bench_command
"$b" "" "./${t}_$b.nitg-e.bin" $s
597 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
599 run_command .
/nitg
${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
600 bench_command
"$b" "" "./${t}_$b.nitg-eu.bin" $s
605 bench_typetest_languages
607 if test -n
"$died"; then
608 echo "Some commands failed"