X-Git-Url: http://nitlanguage.org diff --git a/benchmarks/bench_engines.sh b/benchmarks/bench_engines.sh index 33c0525..8e7f591 100755 --- a/benchmarks/bench_engines.sh +++ b/benchmarks/bench_engines.sh @@ -26,14 +26,6 @@ source ./bench_plot.sh # Can be overrided with 'the option -n' count=2 -### HELPER FUNCTIONS ## - -function die() -{ - echo >&2 "error: $*" - died=1 -} - # HELPER FOR NIT # # Run standards benchs on a compiler command @@ -43,7 +35,10 @@ function run_compiler() { local title=$1 shift - if test -n "$fast"; then + if test "$fast" = truetrue; then + run_command "$@" ../examples/hello_world.nit -o "hello.$title.bin" + bench_command "hello" "hello_world" "./hello.$title.bin" + elif test -n "$fast"; then run_command "$@" ../src/nitc.nit -o "nitc.$title.bin" bench_command "nitc-g" "nitc --global ../src/test_parser.nit" "./nitc.$title.bin" -v --global --no-cc ../src/test_parser.nit run_command "$@" ../src/nit.nit -o "nit.$title.bin" @@ -53,15 +48,19 @@ function run_compiler() run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin" bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16 else + rm -r out 2> /dev/null + mkdir out 2> /dev/null run_command "$@" ../src/nitc.nit -o "nitc.$title.bin" bench_command "nitc-g" "nitc --global --no-cc ../src/nitls.nit" "./nitc.$title.bin" -v --global --no-cc ../src/nitls.nit bench_command "nitc-s" "nitc --separate ../src/nitc.nit" "./nitc.$title.bin" -v --no-cc --separate ../src/nitc.nit run_command "$@" ../src/nit.nit -o "nit.$title.bin" - bench_command "nit" "nit ../src/test_parser.nit ../src/nitls.nit" "./nit.$title.bin" -v ../src/test_parser.nit -- -n ../src/nitls.nit + bench_command "nit-queens" "nit queens.nit 8" "./nit.$title.bin" ../lib/ai/examples/queens.nit -q 8 + bench_command "nit-nitcc" "nit nitcc.nit calc.sablecc" "./nit.$title.bin" ../contrib/nitcc/src/nitcc.nit ../contrib/nitcc/examples/calc.sablecc + rm calc* 2> /dev/null # remove generated cruft run_command "$@" ../src/nitdoc.nit -o "nitdoc.$title.bin" - rm -r out 2> /dev/null - mkdir out 2> /dev/null bench_command "nitdoc" "nitdoc ../src/nitls.nit" "./nitdoc.$title.bin" -v ../src/nitls.nit -d out + run_command "$@" ../src/nitlight.nit -o "nitlight.$title.bin" + bench_command "nitlight" "nitlight ../lib/[a-f]*/" "./nitlight.$title.bin" ../lib/[a-f]*/ -d out run_command "$@" ../examples/shoot/src/shoot_logic.nit -o "shoot.$title.bin" bench_command "shoot" "shoot_logic 15" "./shoot.$title.bin" 15 run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin" @@ -72,9 +71,14 @@ function run_compiler() bench_command "queens" "bench_queens 13" "./queens.$title.bin" 13 run_command "$@" "../lib/ai/examples/puzzle.nit" -o "puzzle.$title.bin" bench_command "puzzle" "puzzle 15-hard" "./puzzle.$title.bin" kleg.mondcafjhbi + run_command "$@" "markdown/engines/nitmd/nitmd.nit" -o "nitmd.$title.bin" + bench_command "nitmd" "markdown" "./nitmd.$title.bin" markdown/benches/out/mixed.md 80 + run_command "$@" ../contrib/jwrapper/src/jwrapper.nit -o "jwrapper.$title.bin" + bench_command "jwrapper" "jwrapper ant.jar" "./jwrapper.$title.bin" /usr/share/java/ant.jar -o out/ant_jar.nit + rm -r tmp 2> /dev/null # remove jwrapper output directory fi - rm -r *.bin .nit_compile out + rm -r *.bin out 2> /dev/null } ## HANDLE OPTIONS ## @@ -86,7 +90,6 @@ function usage() echo " -n count: number of execution for each bar (default: $count)" echo " --dry: Do not run the commands, just reuse the data and generate the graph" echo " --fast: Run less and faster tests" - echo " --html: Generate and HTML output" echo " -h: this help" } @@ -97,12 +100,15 @@ while [ "$stop" = false ]; do -h) usage; exit;; -n) count="$2"; shift; shift;; --dry) dry_run=true; shift;; - --fast) fast=true; shift;; - --html) html="index.html"; echo >"$html" ""; shift;; + --fast) fast=true$fast; shift;; + --html) shift;; # Deprecated *) stop=true esac done +html="index.html" +echo >"$html" "" + xml="bench_engines.xml" echo "" > "$xml" @@ -119,6 +125,12 @@ fi # get the bootstrapped nitc cp ../bin/nitc . +if test -z "$fast"; then + make -C markdown/benches + make -C ../contrib/nitcc + make pre-build -C ../contrib/jwrapper +fi + ## EFFECTIVE BENCHS ## function bench_steps() @@ -147,109 +159,72 @@ function bench_steps() } bench_steps -# $#: options to compare -function bench_nitc-g_options() +# Simple script to compare various options on nitc +# +# usage: *name* *common* [NOALL] *options*... +# +# * *name*, the name of the bench +# * *common*, options to use on each case +# * NOALL, optional flag to avoid a last case including all additional options +# * *options*, sequences of options, one for each case +# +# Example: `bench_nitc_options "foo" "-a -b" -c "-d -e"` generates 4 cases: +# +# * only *common*: `nitc -a -b` +# * *common* and first *options*: `nitc -a -b -c` +# * *common* and second *options*: `nitc -a -b -d -e` +# * all: *common* and all *options*: `nitc -a -b -c -d -e` +function bench_nitc_options() { tag=$1 shift - name="$FUNCNAME-$tag" + common=$1 + shift + name="$FUNCNAME-$tag$common" + name=${name// /} skip_test "$name" && return - prepare_res "$name.dat" "no options" "nitc-g without options" - run_compiler "nitc-g" ./nitc --global + prepare_res "$name.dat" "no options" "nitc $common without more options" + run_compiler "nitc-$name" ./nitc $common if test "$1" = NOALL; then + withall= shift - elif test -n "$2"; then - prepare_res "$name-all.dat" "all" "nitc-g with all options $@" - run_compiler "nitc-g-$tag" ./nitc --global $@ + else + withall=true fi for opt in "$@"; do - ot=${opt// /+} - prepare_res "$name$ot.dat" "$opt" "nitc-g with option $opt" - run_compiler "nitc-g$ot" ./nitc --global $opt + ot=${opt// /} + prepare_res "$name$ot.dat" "$opt" "nitc with option $opt" + run_compiler "nitc-$name" ./nitc $common $opt done - plot "$name.gnu" -} -bench_nitc-g_options "slower" --hardening --no-shortcut-range -bench_nitc-g_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert - -function bench_nitc-s_options() -{ - tag=$1 - shift - name="$FUNCNAME-$tag" - skip_test "$name" && return - prepare_res "$name.dat" "no options" "nitc-s without options" - run_compiler "nitc-s" ./nitc --separate - - if test "$1" = NOALL; then - shift - elif test -n "$2"; then - prepare_res "$name-all.dat" "all" "nitc-s with all options $@" - run_compiler "nitc-s-$tag" ./nitc --separate $@ + if test -n "$2" -a -n "$withall"; then + prepare_res "$name-all.dat" "all" "nitc with all options $@" + run_compiler "nitc-$name" ./nitc $common $@ fi - for opt in "$@"; do - ot=${opt// /+} - prepare_res "$name-$ot.dat" "$opt" "nitc-s with option $opt" - run_compiler "nitc-s$ot" ./nitc --separate $opt - done - plot "$name.gnu" } -bench_nitc-s_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn" -bench_nitc-s_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert -bench_nitc-s_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" "" -function bench_nitc-e_options() -{ - tag=$1 - shift - name="$FUNCNAME-$tag" - skip_test "$name" && return - prepare_res "$name.dat" "no options" "nitc-e without options" - run_compiler "nitc-e" ./nitc --erasure +bench_nitc_options "slower" --global --hardening --no-shortcut-range +bench_nitc_options "nocheck" --global --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert - if test "$1" = NOALL; then - shift - elif test -n "$2"; then - prepare_res "$name-all.dat" "all" "nitc-e with all options $@" - run_compiler "nitc-e-$tag" ./nitc --erasure $@ - fi +bench_nitc_options "slower" --separate --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern "--no-gcc-directive likely --no-gcc-directive noreturn" "--no-tag-primitives" "--colo-dead-methods" --type-poset +bench_nitc_options "nocheck" --separate --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert +bench_nitc_options "faster" --separate --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph "--inline-some-methods --direct-call-monomorph" - for opt in "$@"; do - ot=${opt// /+} - prepare_res "$name$ot.dat" "$opt" "nitc-e with option $opt" - run_compiler "nitc-e$ot" ./nitc --erasure $opt - done +bench_nitc_options "slower" --erasure --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern +bench_nitc_options "nocheck" --erasure --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast --no-check-all +bench_nitc_options "faster" --erasure --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta - plot "$name.gnu" -} -bench_nitc-e_options "slower" --hardening --no-shortcut-equal --no-union-attribute --no-shortcut-range --no-inline-intern -bench_nitc-e_options "nocheck" --no-check-null --no-check-autocast --no-check-attr-isset --no-check-covariance --no-check-assert --no-check-erasure-cast -bench_nitc-e_options "faster" --skip-dead-methods --inline-coloring-numbers --inline-some-methods --direct-call-monomorph --rta +bench_nitc_options "engine" "" NOALL "--separate" "--erasure" "--separate --semi-global" "--erasure --semi-global" "--erasure --semi-global --rta" "--global" +bench_nitc_options "policy" "" NOALL "--separate" "--erasure" "--separate --no-check-covariance" "--erasure --no-check-covariance --no-check-erasure-cast" +bench_nitc_options "nullables" "" "--no-check-attr-isset" "--no-union-attribute" +bench_nitc_options "linkboost" "" NOALL --trampoline-call --colors-are-symbols "--colors-are-symbols --trampoline-call" "--separate --link-boost" "--separate --colors-are-symbols --guard-call" "--separate --colors-are-symbols --direct-call-monomorph0" "--substitute-monomorph" +bench_nitc_options "monomorph" "" --direct-call-monomorph0 --direct-call-monomorph -function bench_engines() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate" - run_compiler "nitc-s" ./nitc --separate - prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure" - run_compiler "nitc-e" ./nitc --erasure - prepare_res "$name-nitc-sg.dat" "nitc-sg" "nitc with --separate --semi-global" - run_compiler "nitc-sg" ./nitc --separate --semi-global - prepare_res "$name-nitc-eg.dat" "nitc-eg" "nitc with --erasure --semi-global" - run_compiler "nitc-eg" ./nitc --erasure --semi-global - prepare_res "$name-nitc-egt.dat" "nitc-egt" "nitc with --erasure --semi-global --rta" - run_compiler "nitc-egt" ./nitc --erasure --semi-global --rta - prepare_res "$name-nitc-g.dat" "nitc-g" "nitc with --global" - run_compiler "nitc-g" ./nitc --global - plot "$name.gnu" -} -bench_engines +bench_nitc_options "misc" "" --log --typing-test-metrics --invocation-metrics --isset-checks-metrics --tables-metrics --no-stacktrace --release --debug #FIXME add --sloppy function bench_nitc-e_gc() { @@ -279,38 +254,6 @@ function bench_cc_nitc-e() } bench_cc_nitc-e -function bench_policy() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate" - run_compiler "nitc-s" ./nitc --separate - prepare_res "$name-nitc-e.dat" "nitc-e" "nitc with --erasure" - run_compiler "nitc-e" ./nitc --erasure - prepare_res "$name-nitc-su.dat" "nitc-su" "nitc with --separate --no-check-covariance" - run_compiler "nitc-su" ./nitc --separate --no-check-covariance - prepare_res "$name-nitc-eu.dat" "nitc-eu" "nitc with --erasure --no-check-covariance --no-check-erasure-cast" - run_compiler "nitc-eu" ./nitc --erasure --no-check-covariance --no-check-erasure-cast - plot "$name.gnu" -} -bench_policy - -function bench_nullables() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "nitc no options" - run_compiler "nitc" ./nitc --separate - prepare_res "$name-nitc-ni.dat" "nitc-ni" "nitc --no-check-attr-isset" - run_compiler "nitc" ./nitc --separate --no-check-attr-isset - prepare_res "$name-nitc-nu.dat" "nitc-nu" "nitc --no-union-attribute" - run_compiler "nitc" ./nitc --separate --no-union-attribute - prepare_res "$name-nitc-nu-ni.dat" "nitc-nu-ni" "nitc --no-union-attribute --no-check-attr-isset" - run_compiler "nitc" ./nitc --separate --no-union-attribute --no-check-attr-isset - plot "$name.gnu" -} -bench_nullables - function bench_compilation_time { name="$FUNCNAME" @@ -331,47 +274,7 @@ function bench_compilation_time } bench_compilation_time -function bench_linkboost() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc-st.dat" "nitc-st" "nitc with --separate --trampoline-call" - run_compiler "nitc-st" ./nitc --separate --trampoline-call - prepare_res "$name-nitc-s.dat" "nitc-s" "nitc with --separate" - run_compiler "nitc-s" ./nitc --separate - prepare_res "$name-nitc-sc.dat" "nitc-sc" "nitc with --separate --colors-are-symbols" - run_compiler "nitc-sc" ./nitc --separate --colors-are-symbols - prepare_res "$name-nitc-sct.dat" "nitc-sct" "nitc with --separate --colors-are-symbols --trampoline-call" - run_compiler "nitc-sct" ./nitc --separate --colors-are-symbols --trampoline-call - prepare_res "$name-nitc-sl.dat" "nitc-sl" "nitc with --separate --link-boost" - run_compiler "nitc-scts" ./nitc --separate --link-boost - prepare_res "$name-nitc-scgc.dat" "nitc-scgc" "nitc with --separate --colors-are-symbols --guard-call" - run_compiler "nitc-scgc" ./nitc --separate --colors-are-symbols --guard-call - prepare_res "$name-nitc-scd.dat" "nitc-scd" "nitc with --separate --colors-are-symbols --direct-call-monomorph0" - run_compiler "nitc-scd" ./nitc --separate --colors-are-symbols --direct-call-monomorph0 - plot "$name.gnu" -} -bench_linkboost - -function bench_call_monomorph() -{ - name="$FUNCNAME" - skip_test "$name" && return - prepare_res "$name-nitc.dat" "nitc" "nitc with --separate" - run_compiler "nitc" ./nitc - prepare_res "$name-nitc-d0.dat" "nitc-d0" "nitc with --separate --direct-call-monomorph0" - run_compiler "nitc-d0" ./nitc --direct-call-monomorph0 - prepare_res "$name-nitc-d1.dat" "nitc-d" "nitc with --separate --direct-call-monomorph" - run_compiler "nitc-d1" ./nitc --direct-call-monomorph - prepare_res "$name-nitc-d2.dat" "nitc-d2" "nitc with --separate --direct-call-monomorph2" - run_compiler "nitc-d2" ./nitc --direct-call-monomorph --direct-call-monomorph0 - plot "$name.gnu" -} -bench_call_monomorph - -if test -n "$html"; then - echo >>"$html" "" -fi +echo >>"$html" "" echo >>"$xml" ""