2 # This file is part of NIT ( http://www.nitlanguage.org ).
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This shell script helps running benchmarks
18 # TODO: cleanup and libify the helper-parts
20 source .
/bench_plot.sh
22 ## CONFIGURATION OPTIONS ##
24 # Default number of times a command must be run with bench_command
25 # Can be overrided with 'the option -n'
28 ### HELPER FUNCTIONS ##
36 # Run a single command multiple time and store the execution times
37 # in the current $res file.
39 # $1: title of the command
40 # $2: long desription of the command
41 # rest: the command to execute
42 function bench_command
()
44 if [ "$dry_run" = "true" ]; then return; fi
49 if test "$verbose" = true
; then outputopts
="/dev/stdout"; else outputopts
="/dev/null"; fi
51 echo "$title" > "$timeout"
52 echo "# $desc" >> "$timeout"
53 echo "\$ $@" >> "$timeout"
55 echo "** [$title] $desc **"
58 # Execute the commands $count times
59 for i
in `seq 1 "$count"`; do
60 (ulimit -t
60; /usr
/bin
/time -f
"%U" -o
"$timeout" -a
"$@") > $outputopts 2>&1 || die
"$1: failed"
65 line
=`compute_stats "$timeout"`
71 # Run a simple command witout storing the execution time
72 # Used to display command on verbose and skip long executions when dry_run is given
73 # $@ command to execute
74 function run_command
()
76 if [ "$dry_run" = "true" ]; then return; fi
78 (ulimit -t
180; "$@") || die
"$@: failed"
81 # Check if the test should be skiped according to its name
82 # $1: name of the test
83 # $2: description of the test
84 # $NOTSKIPED: arguments
87 if test -z
"$NOTSKIPED"; then
91 if test "$NOTSKIPED" = "all"; then
93 elif echo "$1" |
egrep "$NOTSKIPED" >/dev
/null
2>&1; then
94 : # Found one to execute
108 echo "run_bench: [options]* benchname"
109 echo " -v: verbose mode"
110 echo " -n count: number of execution for each bar (default: $count)"
111 echo " --dry: Do not run the commands, just reuse the data and generate the graph"
112 echo " --fast: Run less and faster tests"
113 echo " -h: this help"
117 while [ "$stop" = false
]; do
119 -v
) verbose
=true
; shift;;
121 -n
) count
="$2"; shift; shift;;
122 --dry
) dry_run
=true
; shift;;
123 --fast
) fast
=true
; shift;;
130 if test -z
"$NOTSKIPED"; then
132 echo "List of available benches:"
133 echo "* all: run all the benches"
138 test -f .
/nitc_3 || .
/ncall.sh
-O
140 test -f .
/nitg || ..
/src
/nitc_3 ..
/src
/nitg.nit
-O
-v
142 ## EFFECTIVE BENCHS ##
144 function bench_typetest_languages
()
147 skip_test
"$name" && return
151 seq="w2_h2 w50_h2 w2_h25 w50_h25"
153 run_command .
/nitg languages
/gen.nit
154 run_command .
/gen.bin
"${t}_$b" "$b"
157 prepare_res
"$name-g++.dat" "g++" "g++"
159 run_command g
++ "${t}_$b.cpp" -O2
-o
"${t}_$b.g++.bin"
160 bench_command
"$b" "" "./${t}_$b.g++.bin" $s
163 prepare_res
"$name-clang++.dat" "clang++" "clang++"
165 run_command clang
++ "${t}_$b.cpp" -O2
-o
"${t}_$b.clang++.bin"
166 bench_command
"$b" "" "./${t}_$b.clang++.bin" $s
169 prepare_res
"$name-java.dat" "java" "java"
171 run_command javac
${t}_
$b.java
172 bench_command
"$b" "" java
"${t}_$b" $s
175 prepare_res
"$name-scala.dat" "scala" "scala"
177 run_command scalac
${t}_
$b.scala
178 bench_command
"$b" "" scala
"${t}_$b" $s
181 prepare_res
"$name-cs.dat" "c#" "c#"
183 run_command gmcs
${t}_
$b.cs
184 bench_command
"$b" "" mono
"${t}_$b.exe" $s
187 prepare_res
"$name-es.dat" "es" "es"
189 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
191 mv app
${t}_
$b ${t}_
$b.es.bin
192 bench_command
"$b" "" "./${t}_$b.es.bin" $s
195 prepare_res
"$name-se.dat" "se" "se"
197 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
198 bench_command
"$b" "" "./${t}_$b.se.bin" $s
202 #prepare_res "$name-nitg.dat" "nitg" "nitg"
204 # run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
205 # bench_command "$b" "" "./${t}_$b.nitg.bin" $s
208 prepare_res
"$name-nitg-s.dat" "nitg-s" "nitg-s"
210 run_command .
/nitg
${t}_
$b.nit
--separate
-o
"${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
211 bench_command
"$b" "" "./${t}_$b.nitg-s.bin" $s
214 prepare_res
"$name-nitg-su.dat" "nitg-su" "nitg-su"
216 run_command .
/nitg
${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
217 bench_command
"$b" "" "./${t}_$b.nitg-su.bin" $s
221 prepare_res
"$name-nitg-e.dat" "nitg-e" "nitg-e"
223 run_command .
/nitg
${t}_
$b.nit
--erasure
-o
"${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
224 bench_command
"$b" "" "./${t}_$b.nitg-e.bin" $s
227 prepare_res
"$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
229 run_command .
/nitg
${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
230 bench_command
"$b" "" "./${t}_$b.nitg-eu.bin" $s
235 bench_typetest_languages
237 function bench_typetest_depth
()
240 skip_test
"$name" && return
242 basedir
="./${name}.out"
251 run_command .
/nitg languages
/$name.nit
-o
$basedir/$name.bin
252 run_command
$basedir/$name.bin
$basedir "${t}_$b" "$b"
255 prepare_res
$basedir/$name-g
++.dat
"g++" "g++"
256 cppdir
="${basedir}/cpp"
258 run_command g
++ "${cppdir}/${t}_$b.cpp" -O2
-o
"${cppdir}/${t}_$b.g++.bin"
259 bench_command
"$b" "" "${cppdir}/${t}_$b.g++.bin" $s
262 prepare_res
$basedir/$name-clang
++.dat
"clang++" "clang++"
264 run_command clang
++ "${cppdir}/${t}_$b.cpp" -O2
-o
"${cppdir}/${t}_$b.clang++.bin"
265 bench_command
"$b" "" "${cppdir}/${t}_$b.clang++.bin" $s
268 prepare_res
$basedir/$name-java.dat
"java" "java"
269 javadir
="${basedir}/java"
271 run_command javac
"${javadir}/${t}_$b.java"
272 bench_command
"$b" "" java
-cp "${javadir}/" "${t}_$b" $s
275 prepare_res
$basedir/$name-gcj.dat
"gcj" "gcj"
277 run_command gcj
--main
=${t}_
$b -O2
"${javadir}/${t}_$b.java" -o
"${javadir}/${t}_$b.gcj.bin"
278 bench_command
"$b" "" "${javadir}/${t}_$b.gcj.bin" $s
281 prepare_res
$basedir/$name-scala.dat
"scala" "scala"
282 scaladir
="${basedir}/scala"
284 run_command scalac
"${scaladir}/${t}_$b.scala" -d
"${scaladir}"
285 bench_command
"$b" "" scala
-cp "${scaladir}/" "${t}_$b" $s
288 prepare_res
$basedir/$name-cs.dat
"c#" "c#"
289 csdir
="${basedir}/cs"
291 run_command gmcs
"$csdir/${t}_$b.cs"
292 bench_command
"$b" "" mono
"$csdir/${t}_$b.exe" $s
295 prepare_res
$basedir/$name-es.dat
"es" "es"
296 esdir
="${basedir}/es"
299 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
301 mv app
${t}_
$b ${t}_
$b.es.bin
303 bench_command
"$b" "" "$esdir/${t}_$b.es.bin" $s
306 prepare_res
$basedir/$name-se.dat
"se" "se"
307 sedir
="${basedir}/se"
310 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
312 bench_command
"$b" "" "$sedir/${t}_$b.se.bin" $s
315 nitdir
="${basedir}/nit"
316 prepare_res
$nitdir/$name-nitg.dat
"nitg" "nitg"
318 run_command .
/nitg
$nitdir/${t}_
$b.nit
-o
"$nitdir/${t}_$b.nitg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
319 bench_command
"$b" "" "$nitdir/${t}_$b.nitg.bin" $s
322 prepare_res
$nitdir/$name-nitg
-s.dat
"nitg-s" "nitg-s"
324 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
-o
"$nitdir/${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
325 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
329 prepare_res
$nitdir/$name-$tg.dat
"$tg" "$tg"
331 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--bm
-typing
-o
"$nitdir/${t}_$b.$tg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
332 bench_command
"$b" "" "$nitdir/${t}_$b.$tg.bin" $s
336 prepare_res
$nitdir/$name-$tg.dat
"$tg" "$tg"
338 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--phand
-typing
-o
"$nitdir/${t}_$b.$tg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
339 bench_command
"$b" "" "$nitdir/${t}_$b.$tg.bin" $s
343 prepare_res
$nitdir/$name-$tg.dat
"$tg" "$tg"
345 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--phmod
-typing
-o
"$nitdir/${t}_$b.$tg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
346 bench_command
"$b" "" "$nitdir/${t}_$b.$tg.bin" $s
349 prepare_res
$nitdir/$name-nitg
-su.dat
"nitg-su" "nitg-su"
351 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"$nitdir/${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
352 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
355 prepare_res
$nitdir/$name-nitg
-e.dat
"nitg-e" "nitg-e"
357 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
-o
"$nitdir/${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
358 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
361 prepare_res
$nitdir/$name-nitg
-eu.dat
"nitg-eu" "nitg-eu"
363 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"$nitdir/${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
364 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
367 plot
$basedir/$name.gnu
371 function bench_typetest_fts_depth
()
374 skip_test
"$name" && return
376 basedir
="./${name}.out"
384 run_command .
/nitg languages
/$name.nit
-o
$basedir/$name.bin
385 run_command
$basedir/$name.bin
$basedir "${t}_$b" "$b"
388 prepare_res
$basedir/$name-cs.dat
"c#" "c#"
389 csdir
="${basedir}/cs"
391 run_command gmcs
"$csdir/${t}_$b.cs"
392 bench_command
"$b" "" mono
"$csdir/${t}_$b.exe" $s
395 prepare_res
$basedir/$name-es.dat
"es" "es"
396 esdir
="${basedir}/es"
399 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
401 mv app
${t}_
$b ${t}_
$b.es.bin
403 bench_command
"$b" "" "$esdir/${t}_$b.es.bin" $s
406 prepare_res
$basedir/$name-se.dat
"se" "se"
407 sedir
="${basedir}/se"
410 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
412 bench_command
"$b" "" "$sedir/${t}_$b.se.bin" $s
415 nitdir
="${basedir}/nit"
416 prepare_res
$nitdir/$name-nitg.dat
"nitg" "nitg"
418 run_command .
/nitg
$nitdir/${t}_
$b.nit
-o
"$nitdir/${t}_$b.nitg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
419 bench_command
"$b" "" "$nitdir/${t}_$b.nitg.bin" $s
422 prepare_res
$nitdir/$name-nitg
-s.dat
"nitg-s" "nitg-s"
424 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
-o
"$nitdir/${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
425 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
428 prepare_res
$nitdir/$name-nitg
-su.dat
"nitg-su" "nitg-su"
430 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"$nitdir/${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
431 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
434 prepare_res
$nitdir/$name-nitg
-e.dat
"nitg-e" "nitg-e"
436 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
-o
"$nitdir/${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
437 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
440 prepare_res
$nitdir/$name-nitg
-eu.dat
"nitg-eu" "nitg-eu"
442 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"$nitdir/${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
443 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
446 plot
$basedir/$name.gnu
448 bench_typetest_fts_depth
450 function bench_typetest_fts_width
()
453 skip_test
"$name" && return
455 basedir
="./${name}.out"
464 run_command .
/nitg languages
/$name.nit
-o
$basedir/$name.bin
465 run_command
$basedir/$name.bin
$basedir "${t}_$b" $depth $b
468 prepare_res
$basedir/$name-cs.dat
"c#" "c#"
469 csdir
="${basedir}/cs"
471 run_command gmcs
"$csdir/${t}_$b.cs"
472 bench_command
"$b" "" mono
"$csdir/${t}_$b.exe" $s
475 prepare_res
$basedir/$name-es.dat
"es" "es"
476 esdir
="${basedir}/es"
479 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
481 mv app
${t}_
$b ${t}_
$b.es.bin
483 bench_command
"$b" "" "$esdir/${t}_$b.es.bin" $s
486 prepare_res
$basedir/$name-se.dat
"se" "se"
487 sedir
="${basedir}/se"
490 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
492 bench_command
"$b" "" "$sedir/${t}_$b.se.bin" $s
495 nitdir
="${basedir}/nit"
497 prepare_res
$nitdir/$name-nitg.dat
"nitg" "nitg"
499 run_command .
/nitg
$nitdir/${t}_
$b.nit
-o
"$nitdir/${t}_$b.nitg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
500 bench_command
"$b" "" "$nitdir/${t}_$b.nitg.bin" $s
503 prepare_res
$nitdir/$name-nitg
-s.dat
"nitg-s" "nitg-s"
505 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
-o
"$nitdir/${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
506 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
509 prepare_res
$nitdir/$name-nitg
-su.dat
"nitg-su" "nitg-su"
511 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"$nitdir/${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
512 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
515 prepare_res
$nitdir/$name-nitg
-e.dat
"nitg-e" "nitg-e"
517 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
-o
"$nitdir/${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
518 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
521 prepare_res
$nitdir/$name-nitg
-eu.dat
"nitg-eu" "nitg-eu"
523 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"$nitdir/${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
524 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
527 plot
$basedir/$name.gnu
529 bench_typetest_fts_width
531 function bench_typetest_fts_nesting
()
534 skip_test
"$name" && return
536 basedir
="./${name}.out"
545 run_command .
/nitg languages
/$name.nit
-o
$basedir/$name.bin
546 run_command
$basedir/$name.bin
$basedir "${t}_$b" $depth $b
549 prepare_res
$basedir/$name-cs.dat
"c#" "c#"
550 csdir
="${basedir}/cs"
552 run_command gmcs
"$csdir/${t}_$b.cs"
553 bench_command
"$b" "" mono
"$csdir/${t}_$b.exe" $s
556 prepare_res
$basedir/$name-es.dat
"es" "es"
557 esdir
="${basedir}/es"
560 run_command ec
-clean
-finalize
${t}_
$b/app
${t}_
$b.e
562 mv app
${t}_
$b ${t}_
$b.es.bin
564 bench_command
"$b" "" "$esdir/${t}_$b.es.bin" $s
567 prepare_res
$basedir/$name-se.dat
"se" "se"
568 sedir
="${basedir}/se"
571 run_command se compile
-no_check app
${t}_
${b}_se.e
-loadpath
${t}_
${b}_se
-o
${t}_
$b.se.bin
573 bench_command
"$b" "" "$sedir/${t}_$b.se.bin" $s
576 nitdir
="${basedir}/nit"
578 prepare_res
$nitdir/$name-nitg.dat
"nitg" "nitg"
580 run_command .
/nitg
$nitdir/${t}_
$b.nit
-o
"$nitdir/${t}_$b.nitg.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
581 bench_command
"$b" "" "$nitdir/${t}_$b.nitg.bin" $s
584 prepare_res
$nitdir/$name-nitg
-s.dat
"nitg-s" "nitg-s"
586 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
-o
"$nitdir/${t}_$b.nitg-s.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
587 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
590 prepare_res
$nitdir/$name-nitg
-su.dat
"nitg-su" "nitg-su"
592 run_command .
/nitg
$nitdir/${t}_
$b.nit
--separate
--no
-check
-covariance
-o
"$nitdir/${t}_$b.nitg-su.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
593 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
596 prepare_res
$nitdir/$name-nitg
-e.dat
"nitg-e" "nitg-e"
598 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
-o
"$nitdir/${t}_$b.nitg-e.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
599 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
602 prepare_res
$nitdir/$name-nitg
-eu.dat
"nitg-eu" "nitg-eu"
604 run_command .
/nitg
$nitdir/${t}_
$b.nit
--erasure
--no
-check
-covariance
--no
-check
-erasure
-cast
-o
"$nitdir/${t}_$b.nitg-eu.bin" --make-flags
"CFLAGS=\"-g -O2 -DNOBOEHM\""
605 bench_command
"$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
608 plot
$basedir/$name.gnu
610 bench_typetest_fts_nesting
612 if test -n
"$died"; then
613 echo "Some commands failed"