ef0a709e41d9779b2619eabd10868ac27b413165
2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_plot.sh
22 ## CONFIGURATION OPTIONS ##
24 # Default number of times a command must be run with bench_command
25 # Can be overrided with 'the option -n'
28 ### HELPER FUNCTIONS ##
36 # Run a single command multiple time and store the execution times
37 # in the current $res file.
39 # $1: title of the command
40 # $2: long desription of the command
41 # rest: the command to execute
42 function bench_command
()
44 if [ "$dry_run" = "true" ]; then return; fi
49 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
51 echo "$title" > "$timeout"
52 echo "# $desc" >> "$timeout"
53 echo "\$ $@" >> "$timeout"
55 echo "** [$title] $desc **"
58 # Execute the commands $count times
59 for i
in `seq 1 "$count"`; do
60 /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@" > $outputopts 2>&1 || die
"$1: failed"
65 line
=`compute_stats "$timeout"`
70 # Run a simble command witout storing the execution time
71 # Used to display command on verbose and skip long executions when dry_run is given
72 # $@ command to execute
73 function run_command
()
75 if [ "$dry_run" = "true" ]; then return; fi
77 "$@" || die
"$@: failed"
80 # Check if the test should be skiped according to its name
81 # $1: name of the test
82 # $2: description of the test
83 # $NOTSKIPED: arguments
86 if test -z
"$NOTSKIPED"; then
90 if test "$NOTSKIPED" = "all"; then
92 elif echo "$1" |
egrep "$NOTSKIPED" >/dev
/null
2>&1; then
93 : # Found one to execute
105 # Run standards benchs on a compiler command
107 # rest: command to run (executable + options)
108 function run_compiler
()
112 if test -n
"$fast"; then
113 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
114 bench_command
"nitg" "nitg ../src/test_parser.nit" "./nitg.$title.bin" -v
--no
-cc ..
/src
/test_parser.nit
115 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
116 bench_command
"nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/location.nit
117 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
118 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
119 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
120 bench_command
"bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
122 run_command
"$@" ..
/src
/nitg.nit
-o
"nitg.$title.bin"
123 bench_command
"nitg" "nitg --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v
--no
-cc ..
/src
/nitmetrics.nit
124 bench_command
"nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v
--no
-cc
--separate ..
/src
/nitg.nit
125 run_command
"$@" ..
/src
/nit.nit
-o
"nit.$title.bin"
126 bench_command
"nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ..
/src
/test_parser.nit
-- -n ..
/src
/rapid_type_analysis.nit
127 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
128 bench_command
"shoot" "shoot_logic 30" "./shoot.$title.bin" 30
129 run_command
"$@" ..
/tests
/bench_bintree_gen.nit
-o
"bintrees.$title.bin"
130 bench_command
"bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
138 echo "run_bench: [options]* benchname"
139 echo " -v: verbose mode"
140 echo " -n count: number of execution for each bar (default: $count)"
141 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
142 echo " --fast: Run less and faster tests"
143 echo " -h: this help"
147 while [ "$stop" = false
]; do
149 -v
) verbose
=true
; shift;;
151 -n
) count
="$2"; shift; shift;;
152 --dry
) dry_run
=true
; shift;;
153 --fast
) fast
=true
; shift;;
160 if test -z
"$NOTSKIPED"; then
162 echo "List of available benches:"
163 echo "* all: run all the benches"
168 test -f ..
/src
/nitc_3 || ..
/src
/ncall.sh
-O
169 test -f .
/nitg || ..
/src
/nitc_3 ..
/src
/nitg.nit
-O
-v
171 ## EFFECTIVE BENCHS ##
173 function bench_nitg_bootstrap
()
176 skip_test
"$name" && return
177 prepare_res
"$name.dat" "" "Steps of the bootstrap of nitg by nitc"
179 cp ..
/src
/nitc_3 .
/nitc_nitc.bin
180 bench_command
"c/c c" "nitc_nitc ../src/nitc.nit -> nitc_nitc (stability)" .
/nitc_nitc.bin
-O ..
/src
/nitc.nit
-o nitc_nitc.bin
181 bench_command
"c/c g" "nitc_nitc ../src/nitg.nit -> nitg_nitc" .
/nitc_nitc.bin
-O ..
/src
/nitg.nit
-o nitg_nitc.bin
182 bench_command
"g/c g" "nitg_nitc ../src/nitg.nit -> nitg_nitg" .
/nitg_nitc.bin ..
/src
/nitg.nit
-o nitg_nitg.bin
183 bench_command
"g/g g" "nitg_nitg ../src/nitg.nit -> nitg_nitg (stability)" .
/nitg_nitg.bin ..
/src
/nitg.nit
-o nitg_nitg.bin
189 function bench_steps
()
192 skip_test
"$name" && return
193 prepare_res
"$name-nitc.dat" "nitc" "Various steps of nitc"
194 bench_command
"parse" "" ..
/src
/nitc_3
--only
-parse ..
/src
/nitg.nit
195 bench_command
"metamodel" "" ..
/src
/nitc_3
--only
-metamodel ..
/src
/nitg.nit
196 bench_command
"generate c" "" ..
/src
/nitc_3
--no
-cc ..
/src
/nitg.nit
197 bench_command
"full" "" ..
/src
/nitc_3
-O ..
/src
/nitg.nit
-o
"nitg_nitg.bin"
199 prepare_res
"$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
200 bench_command
"parse" "" ..
/src
/nitc_3
--global
--only
-parse ..
/src
/nitg.nit
201 bench_command
"metamodel" "" ..
/src
/nitc_3
--global
--only
-metamodel ..
/src
/nitg.nit
202 bench_command
"generate c" "" ..
/src
/nitc_3
--global
--no
-cc ..
/src
/nitg.nit
203 bench_command
"full" "" ..
/src
/nitc_3
-O
--global ..
/src
/nitg.nit
-o
"nitg_nitc-g.bin"
205 prepare_res
"$name-nitg.dat" "nitg" "Various steps of nitg"
206 bench_command
"parse" "" .
/nitg
--only
-parse ..
/src
/nitg.nit
207 bench_command
"metamodel" "" .
/nitg
--only
-metamodel ..
/src
/nitg.nit
208 bench_command
"generate c" "" .
/nitg
--no
-cc ..
/src
/nitg.nit
209 bench_command
"full" "" .
/nitg ..
/src
/nitg.nit
-o
"nitg_nitg.bin"
211 prepare_res
"$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
212 bench_command
"parse" "" .
/nitg
--erasure
--only
-parse ..
/src
/nitg.nit
213 bench_command
"metamodel" "" .
/nitg
--erasure
--only
-metamodel ..
/src
/nitg.nit
214 bench_command
"generate c" "" .
/nitg
--erasure
--no
-cc ..
/src
/nitg.nit
215 bench_command
"full" "" .
/nitg
--erasure ..
/src
/nitg.nit
-o
"nitg_nitg-e.bin"
221 # $#: options to compare
222 function bench_nitg_options
()
226 name
="$FUNCNAME-$tag"
227 skip_test
"$name" && return
228 prepare_res
"$name.dat" "no options" "nitg without options"
229 run_compiler
"nitg" .
/nitg
231 if test -n
"$2"; then
232 prepare_res
"$name-all.dat" "all" "nitg with all options $@"
233 run_compiler
"nitg-$tag" .
/nitg $@
237 prepare_res
"$name$opt.dat" "$opt" "nitg with option $opt"
238 run_compiler
"nitg$opt" .
/nitg
$opt
243 bench_nitg_options
"hardening" --hardening
244 bench_nitg_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
246 function bench_nitg
-s_options
()
250 name
="$FUNCNAME-$tag"
251 skip_test
"$name" && return
252 prepare_res
"$name.dat" "no options" "nitg-s without options"
253 run_compiler
"nitg-s" .
/nitg
--separate
255 if test -n
"$2"; then
256 prepare_res
"$name-all.dat" "all" "nitg-s with all options $@"
257 run_compiler
"nitg-s-$tag" .
/nitg
--separate $@
261 prepare_res
"$name$opt.dat" "$opt" "nitg-s with option $opt"
262 run_compiler
"nitg-s$opt" .
/nitg
--separate
$opt
267 bench_nitg
-s_options
"slower" --hardening
--no
-inline
-intern
--generic
-resolution
-tree
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
268 bench_nitg
-s_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
269 bench_nitg
-s_options
"faster" --inline
-coloring
-numbers
270 bench_nitg
-s_options
"typing" --bm
-typing
--phand
-typing
272 function bench_nitg
-e_options
()
276 name
="$FUNCNAME-$tag"
277 skip_test
"$name" && return
278 prepare_res
"$name.dat" "no options" "nitg-e without options"
279 run_compiler
"nitg-e" .
/nitg
--erasure
281 if test -n
"$2"; then
282 prepare_res
"$name-all.dat" "all" "nitg-e with all options $@"
283 run_compiler
"nitg-e-$tag" .
/nitg
--erasure $@
287 prepare_res
"$name$opt.dat" "$opt" "nitg-e with option $opt"
288 run_compiler
"nitg-e$opt" .
/nitg
--erasure
$opt
293 bench_nitg
-e_options
"slower" --hardening
--no
-inline
-intern
--no
-union
-attribute
--no
-shortcut
-equal
--no
-shortcut
-range
294 bench_nitg
-e_options
"nocheck" --no
-check
-covariance
--no
-check
-initialization
--no
-check
-assert
--no
-check
-autocast
--no
-check
-other
--no
-check
-erasure
-cast
295 bench_nitg
-e_options
"faster" --inline
-coloring
-numbers
296 bench_nitg
-e_options
"typing" --bm
-typing
--phand
-typing
298 function bench_nitc_gc
()
301 skip_test
"$name" && return
302 for gc
in nitgc boehm malloc large
; do
303 prepare_res
"$name-$gc.dat" "$gc" "nitc with gc=$gc"
304 export NIT_GC_OPTION
="$gc"
305 run_compiler
"nitc" ..
/src
/nitc_3
-O
312 function bench_nitc_boost
()
315 skip_test
"$name" && return
316 prepare_res
"$name-slow.dat" "no -O" "nitc without -O"
317 run_compiler
"nitc_slow" ..
/src
/nitc_3
318 prepare_res
"$name-fast.dat" "-O" "nitc with -O"
319 run_compiler
"nitc" ..
/src
/nitc_3
-O
325 function bench_engines
()
328 skip_test
"$name" && return
329 prepare_res
"$name-nitc.dat" "nitc" "nitc"
330 run_compiler
"nitc" ..
/src
/nitc_3
-O
331 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
332 run_compiler
"nitc-g" ..
/src
/nitc_3
-O
--global
333 prepare_res
"$name-nitg.dat" "nitg" "nitg"
334 run_compiler
"nitg" .
/nitg
335 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
336 run_compiler
"nitg-s" .
/nitg
--separate
337 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
338 run_compiler
"nitg-e" .
/nitg
--erasure
343 function bench_nitc_vc_nitg
-e
()
346 skip_test
"$name" && return
347 prepare_res
"$name-nitc.dat" "nitc" "nitc"
348 run_compiler
"nitc" ..
/src
/nitc_3
-O
349 prepare_res
"$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
350 NIT_GC_OPTION
="malloc" run_compiler
"nitc" ..
/src
/nitc_3
-O
351 prepare_res
"$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
352 NIT_GC_OPTION
="boehm" run_compiler
"nitc" ..
/src
/nitc_3
-O
353 prepare_res
"$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
354 run_compiler
"nitg-e-nc-malloc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
--make-flags
"CFLAGS=\"-O2 -DNOBOEHM\""
355 prepare_res
"$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
356 run_compiler
"nitg-e-nc" .
/nitg
--erasure
--no
-check
-autocast
--no
-check
-erasure
-cast
357 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
358 run_compiler
"nitg-e" .
/nitg
--erasure
363 function bench_nitg
-e_gc
()
366 skip_test
"$name" && return
367 prepare_res
"$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
368 run_compiler
"nitg-e-malloc" .
/nitg
--erasure
--make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
369 prepare_res
"$name-nitg-e-noatomic.dat" "nitg-e-noatomic" "nitg with --erasure and no atomic"
370 run_compiler
"nitg-e-noatomic" .
/nitg
--erasure
--make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM_ATOMIC\""
371 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
372 run_compiler
"nitg-e" .
/nitg
--erasure
377 function bench_cc_nitg
-e
()
380 skip_test
"$name" && return
381 for o
in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3" "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
382 f
=`echo "$o" | cut -f1 -d:`
383 o
=`echo "$o" | cut -f2 -d:`
384 prepare_res
"$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
385 run_compiler
"nitg-e-$f" .
/nitg
--erasure
--make-flags
"$o"
391 function bench_policy
()
394 skip_test
"$name" && return
395 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg with --separate"
396 run_compiler
"nitg-s" .
/nitg
--separate
397 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
398 run_compiler
"nitg-e" .
/nitg
--erasure
399 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg with --separate --no-check-covariance"
400 run_compiler
"nitg-su" .
/nitg
--separate
--no
-check
-covariance
401 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
402 run_compiler
"nitg-eu" .
/nitg
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
407 function bench_compilation_time
410 skip_test
"$name" && return
411 prepare_res
"$name-nitc.dat" "nitc" "nitc"
412 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
413 bench_command
`basename "$i" .nit` "" ..
/src
/nitc_3
-O
"$i" --no
-cc
415 prepare_res
"$name-nitg.dat" "nitg" "nitg"
416 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
417 bench_command
`basename "$i" .nit` "" .
/nitg
"$i" --no
-cc
419 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg --erasure"
420 for i
in ..
/examples
/hello_world.nit ..
/src
/test_parser.nit ..
/src
/nitg.nit
; do
421 bench_command
`basename "$i" .nit` "" .
/nitg
--erasure
"$i" --no
-cc
425 bench_compilation_time
427 if test -n
"$died"; then
428 echo "Some commands failed"