X-Git-Url: http://nitlanguage.org diff --git a/benchmarks/bench_engines.sh b/benchmarks/bench_engines.sh index ae8cc5b..4168579 100755 --- a/benchmarks/bench_engines.sh +++ b/benchmarks/bench_engines.sh @@ -96,6 +96,9 @@ function skip_test() else return 0 fi + if test -n "$html"; then + echo >>"$html" "

$1

" + fi echo "*" echo "* $1 *****" echo "*" @@ -113,20 +116,20 @@ function run_compiler() shift if test -n "$fast"; then run_command "$@" ../src/nitg.nit -o "nitg.$title.bin" - bench_command "nitg" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit + bench_command "nitg-g" "nitg --global ../src/test_parser.nit" "./nitg.$title.bin" -v --global --no-cc ../src/test_parser.nit run_command "$@" ../src/nit.nit -o "nit.$title.bin" bench_command "nit" "nit ../src/test_parser.nit ../src/location.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/location.nit - run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin" + run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin" bench_command "shoot" "shoot_logic" "./shoot.$title.bin" run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin" bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16 else run_command "$@" ../src/nitg.nit -o "nitg.$title.bin" - bench_command "nitg" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitmetrics.nit + bench_command "nitg-g" "nitg --global --no-cc ../src/nitmetrics.nit" "./nitg.$title.bin" -v --global --no-cc ../src/nitmetrics.nit bench_command "nitg-s" "nitg --separate ../src/nitg.nit" "./nitg.$title.bin" -v --no-cc --separate ../src/nitg.nit run_command "$@" ../src/nit.nit -o "nit.$title.bin" bench_command "nit" "nit ../src/test_parser.nit ../src/rapid_type_analysis.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/rapid_type_analysis.nit - run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin" + run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin" bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin" bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18 @@ -146,6 +149,7 @@ function usage() echo " -n count: number of execution for each bar (default: $count)" echo " --dry: Do not run the commands, just reuse the data and generate the graph" echo " --fast: Run less and faster tests" + echo " --html: Generate and HTML output" echo " -h: this help" } @@ -157,6 +161,7 @@ while [ "$stop" = false ]; do -n) count="$2"; shift; shift;; --dry) dry_run=true; shift;; --fast) fast=true; shift;; + --html) html="index.html"; echo >"$html" ""; shift;; *) stop=true esac done @@ -171,49 +176,27 @@ fi ## COMPILE ENGINES -test -f ../src/nitc_3 || ../src/ncall.sh -O -test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v +# force to use the last nitg, not the bootstraped one +test -f ./nitg || ../bin/nitg ../src/nitg.nit -v ## EFFECTIVE BENCHS ## -function bench_nitg_bootstrap() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name.dat" "" "Steps of the bootstrap of nitg by nitc" - rm nit?_nit* - cp ../src/nitc_3 ./nitc_nitc.bin - bench_command "c/c c" "nitc_nitc ../src/nitc.nit -> nitc_nitc (stability)" ./nitc_nitc.bin -O ../src/nitc.nit -o nitc_nitc.bin - bench_command "c/c g" "nitc_nitc ../src/nitg.nit -> nitg_nitc" ./nitc_nitc.bin -O ../src/nitg.nit -o nitg_nitc.bin - bench_command "g/c g" "nitg_nitc ../src/nitg.nit -> nitg_nitg" ./nitg_nitc.bin ../src/nitg.nit -o nitg_nitg.bin - bench_command "g/g g" "nitg_nitg ../src/nitg.nit -> nitg_nitg (stability)" ./nitg_nitg.bin ../src/nitg.nit -o nitg_nitg.bin - - plot "$name.gnu" -} -bench_nitg_bootstrap - function bench_steps() { name="$FUNCNAME" skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "Various steps of nitc" - bench_command "parse" "" ../src/nitc_3 --only-parse ../src/nitg.nit - bench_command "metamodel" "" ../src/nitc_3 --only-metamodel ../src/nitg.nit - bench_command "generate c" "" ../src/nitc_3 --no-cc ../src/nitg.nit - bench_command "full" "" ../src/nitc_3 -O ../src/nitg.nit -o "nitg_nitg.bin" - - prepare_res "$name-nitc-g.dat" "nitc-g" "Various steps of nitc --global" - bench_command "parse" "" ../src/nitc_3 --global --only-parse ../src/nitg.nit - bench_command "metamodel" "" ../src/nitc_3 --global --only-metamodel ../src/nitg.nit - bench_command "generate c" "" ../src/nitc_3 --global --no-cc ../src/nitg.nit - bench_command "full" "" ../src/nitc_3 -O --global ../src/nitg.nit -o "nitg_nitc-g.bin" - - prepare_res "$name-nitg.dat" "nitg" "Various steps of nitg --global" + prepare_res "$name-nitg.dat" "nitg-g" "Various steps of nitg --global" bench_command "parse" "" ./nitg --global --only-parse ../src/nitg.nit bench_command "metamodel" "" ./nitg --global --only-metamodel ../src/nitg.nit bench_command "generate c" "" ./nitg --global --no-cc ../src/nitg.nit bench_command "full" "" ./nitg --global ../src/nitg.nit -o "nitg_nitg.bin" + prepare_res "$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate" + bench_command "parse" "" ./nitg --separate --only-parse ../src/nitg.nit + bench_command "metamodel" "" ./nitg --separate --only-metamodel ../src/nitg.nit + bench_command "generate c" "" ./nitg --separate --no-cc ../src/nitg.nit + bench_command "full" "" ./nitg --separate ../src/nitg.nit -o "nitg_nitg-e.bin" + prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure" bench_command "parse" "" ./nitg --erasure --only-parse ../src/nitg.nit bench_command "metamodel" "" ./nitg --erasure --only-metamodel ../src/nitg.nit @@ -225,29 +208,32 @@ function bench_steps() bench_steps # $#: options to compare -function bench_nitg_options() +function bench_nitg-g_options() { tag=$1 shift name="$FUNCNAME-$tag" skip_test "$name" && return - prepare_res "$name.dat" "no options" "nitg without options" - run_compiler "nitg" ./nitg --global - - if test -n "$2"; then - prepare_res "$name-all.dat" "all" "nitg with all options $@" - run_compiler "nitg-$tag" ./nitg --global $@ + prepare_res "$name.dat" "no options" "nitg-g without options" + run_compiler "nitg-g" ./nitg --global + + if test "$1" = NOALL; then + shift + elif test -n "$2"; then + prepare_res "$name-all.dat" "all" "nitg-g with all options $@" + run_compiler "nitg-g-$tag" ./nitg --global $@ fi for opt in "$@"; do - prepare_res "$name$opt.dat" "$opt" "nitg with option $opt" - run_compiler "nitg$opt" ./nitg --global $opt + ot=${opt// /+} + prepare_res "$name$ot.dat" "$opt" "nitg-g with option $opt" + run_compiler "nitg-g$ot" ./nitg --global $opt done plot "$name.gnu" } -bench_nitg_options "hardening" --hardening -bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other +bench_nitg-g_options "slower" --hardening +bench_nitg-g_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other function bench_nitg-s_options() { @@ -258,22 +244,25 @@ function bench_nitg-s_options() prepare_res "$name.dat" "no options" "nitg-s without options" run_compiler "nitg-s" ./nitg --separate - if test -n "$2"; then + if test "$1" = NOALL; then + shift + elif test -n "$2"; then prepare_res "$name-all.dat" "all" "nitg-s with all options $@" run_compiler "nitg-s-$tag" ./nitg --separate $@ fi for opt in "$@"; do - prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt" - run_compiler "nitg-s$opt" ./nitg --separate $opt + ot=${opt// /+} + prepare_res "$name-$ot.dat" "$opt" "nitg-s with option $opt" + run_compiler "nitg-s$ot" ./nitg --separate $opt done plot "$name.gnu" } -bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree --no-union-attribute --no-shortcut-equal --no-shortcut-range -bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other -bench_nitg-s_options "faster" --inline-coloring-numbers -bench_nitg-s_options "typing" --bm-typing --phand-typing +bench_nitg-s_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range "--no-gcc-directive likely" "--no-gcc-directive noreturn" +bench_nitg-s_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other +bench_nitg-s_options "faster" --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" +bench_nitg-s_options "typing" NOALL --bm-typing --phand-typing function bench_nitg-e_options() { @@ -284,60 +273,32 @@ function bench_nitg-e_options() prepare_res "$name.dat" "no options" "nitg-e without options" run_compiler "nitg-e" ./nitg --erasure - if test -n "$2"; then + if test "$1" = NOALL; then + shift + elif test -n "$2"; then prepare_res "$name-all.dat" "all" "nitg-e with all options $@" run_compiler "nitg-e-$tag" ./nitg --erasure $@ fi for opt in "$@"; do - prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt" - run_compiler "nitg-e$opt" ./nitg --erasure $opt + ot=${opt// /+} + prepare_res "$name$ot.dat" "$opt" "nitg-e with option $opt" + run_compiler "nitg-e$ot" ./nitg --erasure $opt done plot "$name.gnu" } bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range -bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast +bench_nitg-e_options "nocheck" --no-check-covariance --no-check-attr-isset --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast bench_nitg-e_options "faster" --inline-coloring-numbers -bench_nitg-e_options "typing" --bm-typing --phand-typing - -function bench_nitc_gc() -{ - name="$FUNCNAME" - skip_test "$name" && return - for gc in nitgc boehm malloc large; do - prepare_res "$name-$gc.dat" "$gc" "nitc with gc=$gc" - export NIT_GC_OPTION="$gc" - run_compiler "nitc" ../src/nitc_3 -O - done - - plot "$name.gnu" -} -bench_nitc_gc - -function bench_nitc_boost() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-slow.dat" "no -O" "nitc without -O" - run_compiler "nitc_slow" ../src/nitc_3 - prepare_res "$name-fast.dat" "-O" "nitc with -O" - run_compiler "nitc" ../src/nitc_3 -O - - plot "$name.gnu" -} -bench_nitc_boost +bench_nitg-e_options "typing" NOALL --bm-typing # --phand-typing function bench_engines() { name="$FUNCNAME" skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "nitc" - run_compiler "nitc" ../src/nitc_3 -O - prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global" - run_compiler "nitc-g" ../src/nitc_3 -O --global - prepare_res "$name-nitg.dat" "nitg" "nitg with --global" - run_compiler "nitg" ./nitg --global + prepare_res "$name-nitg-g.dat" "nitg-g" "nitg with --global" + run_compiler "nitg-g" ./nitg --global prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate" run_compiler "nitg-s" ./nitg --separate prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure" @@ -346,26 +307,6 @@ function bench_engines() } bench_engines -function bench_nitc_vc_nitg-e() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "nitc" - run_compiler "nitc" ../src/nitc_3 -O - prepare_res "$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc" - NIT_GC_OPTION="malloc" run_compiler "nitc" ../src/nitc_3 -O - prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm" - NIT_GC_OPTION="boehm" run_compiler "nitc" ../src/nitc_3 -O - prepare_res "$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc" - NIT_GC_OPTION="malloc" run_compiler "nitg-e-nc-malloc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast - prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast" - run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast - prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure" - run_compiler "nitg-e" ./nitg --erasure - plot "$name.gnu" -} -bench_nitc_vc_nitg-e - function bench_nitg-e_gc() { name="$FUNCNAME" @@ -408,17 +349,33 @@ function bench_policy() } bench_policy +function bench_nullables() +{ + name="$FUNCNAME" + skip_test "$name" && return + prepare_res "$name-nitc.dat" "nitc" "nitc no options" + run_compiler "nitc" ./nitg --separate + prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset" + run_compiler "nitc" ./nitg --separate --no-check-attr-isset + prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute" + run_compiler "nitc" ./nitg --separate --no-union-attribute + prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset" + run_compiler "nitc" ./nitg --separate --no-union-attribute --no-check-attr-isset + plot "$name.gnu" +} +bench_nullables + function bench_compilation_time { name="$FUNCNAME" skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "nitc" + prepare_res "$name-nitg-g.dat" "nitg-g" "nitg --global" for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do - bench_command `basename "$i" .nit` "" ../src/nitc_3 -O "$i" --no-cc + bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc done - prepare_res "$name-nitg.dat" "nitg" "nitg" + prepare_res "$name-nitg-s.dat" "nitg-s" "nitg --separate" for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do - bench_command `basename "$i" .nit` "" ./nitg --global "$i" --no-cc + bench_command `basename "$i" .nit` "" ./nitg --separate "$i" --no-cc done prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure" for i in ../examples/hello_world.nit ../src/test_parser.nit ../src/nitg.nit; do @@ -428,6 +385,10 @@ function bench_compilation_time } bench_compilation_time +if test -n "$html"; then + echo >>"$html" "" +fi + if test -n "$died"; then echo "Some commands failed" exit 1