2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 ## CONFIGURATION OPTIONS ##
22 # Default number of times a command must be run with bench_command
23 # Can be overrided with 'the option -n'
26 ### HELPER FUNCTIONS ##
34 # Run a single command multiple time and store the execution times
35 # in the current $res file.
37 # $1: title of the command
38 # $2: long desription of the command
39 # rest: the command to execute
40 function bench_command
()
42 if [ "$dry_run" = "true" ]; then return; fi
47 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
49 echo "$title" > "$timeout"
50 echo "# $desc" >> "$timeout"
51 echo "\$ $@" >> "$timeout"
53 echo "** [$title] $desc **"
56 # Execute the commands $count times
57 for i
in `seq 1 "$count"`; do
58 /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@" > $outputopts 2>&1 || die
"$1: failed"
63 line
=`compute_stats "$timeout"`
68 # Run a simble command witout storing the execution time
69 # Used to display command on verbose and skip long executions when dry_run is given
70 # $@ command to execute
71 function run_command
()
73 if [ "$dry_run" = "true" ]; then return; fi
78 # perl function to compute min/max/avg.
79 # used by bench_command
82 # return: first min max avg label
83 function compute_stats
()
87 perl
- "$file" <<'END'
101 $len = scalar @lines
;
103 @lines
= sort {$a <=> $b} @lines
;
105 $max = $lines[$#lines];
107 for $i (@lines
) { $avg += $i; }
109 print
"${first} ${min} ${max} ${avg} \"${label}\"\n";
111 print
"${first} ${first} ${first} ${first} \"${label}\"\n";
116 # Create a new $res to be used in the next plot
121 function prepare_res
()
126 if [ "$plots" = "" ]; then
127 plots
="plot '$1' using 4:2:3:xticlabels(5) ti '$2';"
129 plots
="$plots replot '$1' using 4:2:3 ti '$2';"
131 if [ "$dry_run" = "true" ]; then return; fi
132 echo "# [$2] $3" > "$res"
133 echo "# first min max avg title" >> "$res"
136 # Plot the last $res as an histogram
137 # $1: plot file (eg toto.gnu)
143 set style data histogram
;
144 set style histogram cluster gap
2;
145 set style histogram errorbars linewidth
1;
146 set style fill solid
0.3 border
-1;
149 set xtic nomirror rotate by
-45 scale
0 font
',8';
151 set ylabel
"time (s)"
154 echo "# gnuplot -p $1"
159 # Check if the test should be skiped according to its name
160 # $1: name of the test
161 # $2: description of the test
162 # $NOTSKIPED: arguments
165 if test -z
"$NOTSKIPED"; then
169 if test "$NOTSKIPED" = "all"; then
171 elif echo "$1" |
grep "$NOTSKIPED" >/dev
/null
2>&1; then
172 : # Found one to execute
182 ## GLOBAL VARIABLES ##
184 # The current $res (set by prepare_res, used by bench_command)
187 # The current stuff to plot (set by prepare_res, used by plot)
192 # Run standards benchs on a compiler command
194 # rest: command to run (executable + options)
195 function run_compiler
()
199 run_command
"$@" nitg.nit
-o
"nitg.$title.bin"
200 bench_command
"nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
201 run_command
"$@" nit.nit
-o
"nit.$title.bin"
202 bench_command
"nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit
-- -n rapid_type_analysis.nit
203 run_command
"$@" ..
/examples
/shoot
/shoot_logic.nit
-o
"shoot.$title.bin"
204 bench_command
"shoot" "shoot_logic" "./shoot.$title.bin"
205 run_command
"$@" ..
/tests
/shootout_binarytrees.nit
-o
"bintrees.$title.bin"
206 bench_command
"bintrees" "shootout_binarytrees" "./bintrees.$title.bin" 17
213 echo "run_bench: [options]* benchname"
214 echo " -v: verbose mode"
215 echo " -n count: number of execution for each bar (default: $count)"
216 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
217 echo " -h: this help"
221 while [ "$stop" = false
]; do
223 -v
) verbose
=true
; shift;;
225 -n
) count
="$2"; shift; shift;;
226 --dry
) dry_run
=true
; shift;;
233 if test -z
"$NOTSKIPED"; then
235 echo "List of available benches:"
236 echo "* all: run all the benches"
239 ## EFFECTIVE BENCHS ##
241 function bench_nitg_bootstrap
()
244 skip_test
"$name" && return
245 prepare_res
"$name.dat" "" "Steps of the bootstrap of nitg by nitc"
247 cp .
/nitc_3 .
/nitc_nitc.bin
248 bench_command
"c/c c" "nitc_nitc nitc.nit -> nitc_nitc (stability)" .
/nitc_nitc.bin
-O nitc.nit
-o nitc_nitc.bin
249 bench_command
"c/c g" "nitc_nitc nitg.nit -> nitg_nitc" .
/nitc_nitc.bin
-O nitg.nit
-o nitg_nitc.bin
250 bench_command
"g/c g" "nitg_nitc nitg.nit -> nitg_nitg" .
/nitg_nitc.bin nitg.nit
-o nitg_nitg.bin
251 bench_command
"g/g g" "nitg_nitg nitg.nit -> nitg_nitg (stability)" .
/nitg_nitg.bin nitg.nit
-o nitg_nitg.bin
257 function bench_steps
()
260 skip_test
"$name" && return
261 prepare_res
"$name-nitc.dat" "nitc" "Various steps of nitc"
262 bench_command
"parse" "" .
/nitc_3
--only
-parse nitg.nit
263 bench_command
"metamodel" "" .
/nitc_3
--only
-metamodel nitg.nit
264 bench_command
"generate c" "" .
/nitc_3
--no
-cc nitg.nit
265 bench_command
"full" "" .
/nitc_3
-O nitg.nit
-o
"nitg_nitg.bin"
267 prepare_res
"$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global"
268 bench_command
"parse" "" .
/nitc_3
--global
--only
-parse nitg.nit
269 bench_command
"metamodel" "" .
/nitc_3
--global
--only
-metamodel nitg.nit
270 bench_command
"generate c" "" .
/nitc_3
--global
--no
-cc nitg.nit
271 bench_command
"full" "" .
/nitc_3
-O
--global nitg.nit
-o
"nitg_nitc-g.bin"
273 prepare_res
"$name-nitg.dat" "nitg" "Various steps of nitg"
274 bench_command
"parse" "" .
/nitg
--only
-parse nitg.nit
275 bench_command
"metamodel" "" .
/nitg
--only
-metamodel nitg.nit
276 bench_command
"generate c" "" .
/nitg
--no
-cc nitg.nit
277 bench_command
"full" "" .
/nitg nitg.nit
-o
"nitg_nitg.bin"
279 prepare_res
"$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
280 bench_command
"parse" "" .
/nitg
--separate
--only
-parse nitg.nit
281 bench_command
"metamodel" "" .
/nitg
--separate
--only
-metamodel nitg.nit
282 bench_command
"generate c" "" .
/nitg
--separate
--no
-cc nitg.nit
283 bench_command
"full" "" .
/nitg
--separate nitg.nit
-o
"nitg_nitg-s.bin"
289 # $#: options to compare
290 function bench_nitg_options
()
293 skip_test
"$name" && return
294 prepare_res
"$name.dat" "no options" "nitg without options"
295 run_compiler
"nitg" .
/nitg
298 prepare_res
"$name$opt.dat" "$opt" "nitg with option $opt"
299 run_compiler
"nitg$opt" .
/nitg
$opt
304 bench_nitg_options
--hardening
306 function bench_nitc_gc
()
309 skip_test
"$name" && return
310 for gc
in nitgc boehm malloc large
; do
311 prepare_res
"$name-$gc".dat
"$gc" "nitc with gc=$gc"
312 export NIT_GC_OPTION
="$gc"
313 run_compiler
"nitc" .
/nitc_3
-O
320 function bench_nitc_boost
()
323 skip_test
"$name" && return
324 prepare_res
"$name-slow.dat" "no -O" "nitc without -O"
325 run_compiler
"nitc_slow" .
/nitc_3
326 prepare_res
"$name-fast.dat" "-O" "nitc with -O"
327 run_compiler
"nitc" .
/nitc_3
-O
333 function bench_engines
()
336 skip_test
"$name" && return
337 prepare_res
"$name-nitc.dat" "nitc" "nitc"
338 run_compiler
"nitc" .
/nitc_3
-O
339 prepare_res
"$name-nitc-g.dat" "nitc-g" "nitc with --global"
340 run_compiler
"nitc-g" .
/nitc_3
-O
--global
341 prepare_res
"$name-nitg.dat" "nitg" "nitg"
342 run_compiler
"nitg" .
/nitg
347 function bench_compilation_time
350 skip_test
"$name" && return
351 prepare_res
"$name-nitc.dat" "nitc" "nitc"
352 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
353 bench_command
`basename "$i" .nit` "" .
/nitc_3
-O
"$i" --no
-cc
355 prepare_res
"$name-nitg.dat" "nitg" "nitg"
356 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
357 bench_command
`basename "$i" .nit` "" .
/nitg
"$i" --no
-cc
359 prepare_res
"$name-nitg_g.dat" "nitg/g" "nitg/g"
360 for i
in ..
/examples
/hello_world.nit test_parser.nit nitg.nit
; do
361 bench_command
`basename "$i" .nit` "" .
/nitg.bin
"$i" --no
-cc
365 bench_compilation_time