From afc387e858f4ffd83e3c0fa48b1afb43b44a7e2d Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Tue, 18 Aug 2015 14:42:45 -0400 Subject: [PATCH] benchs/strings: Updated bench_strings, simplified code Signed-off-by: Lucas Bajolet --- benchmarks/strings/bench_strings.sh | 522 +++++++++++++++-------------------- 1 file changed, 222 insertions(+), 300 deletions(-) diff --git a/benchmarks/strings/bench_strings.sh b/benchmarks/strings/bench_strings.sh index a710dc0..7a09065 100755 --- a/benchmarks/strings/bench_strings.sh +++ b/benchmarks/strings/bench_strings.sh @@ -20,375 +20,297 @@ source ../bench_plot.sh # Can be overrided with 'the option -n' count=5 +benches="iteration +concat +substring +index +compiler +basic" + function usage() { echo "run_bench: [options]* bench_name args" + echo "" + echo "Options:" echo " -v: verbose mode" echo " -n count: number of execution for each bar (default: $count)" echo " -h: this help" echo "" echo "Benches : " - echo " iter: bench iterations" - echo " - usage : iter max_nb_cct loops strlen" - echo " cct: concatenation benching" - echo " - usage : cct max_nb_cct loops strlen" - echo " substr: substring benching" - echo " - usage : substr max_nb_cct loops strlen" - echo " array: Benchmark for the to_s in array" - echo " - usage : array nb_cct loops max_arrlen" + echo " index: indexed access benchmark" + echo " - usage : index loops strlen_min strlen_inc strlen_max" + echo " concat: string concatenation benchmark" + echo " - usage : concat loops strlen min_cct cct_inc max_cct" + echo " iteration: iteration benchmark" + echo " - usage : iteration loops strlen_min strlen_inc strlen_max" + echo " substring: substring benchmark" + echo " - usage : substring loops strlen_min strlen_inc strlen_max" + echo " compiler: compiler benchmark" + echo " - usage : compiler" + echo " basic : basic functionnality test of the variants on Hello World" + echo " - usage : basic" } -function bench_array() +function bench_index() { - if [ -d arraytos ]; then - rm arraytos/* - else - mkdir arraytos + if [ $# -lt 4 ]; then + echo "Wrong arguments for benchmark index." + usage + exit fi - cd arraytos + echo "Generating executable index_bench for variant $variant" + ../../bin/nitc --global index_bench.nit - if $verbose; then - echo "*** Benching Array.to_s performance ***" - fi + bench_indexed_variant "string" $1 $2 $3 $4 + bench_indexed_variant "buffer" $1 $2 $3 $4 - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_rope.nit + rm index_bench +} - prepare_res arr_tos_ropes.out arr_tos_ropes ropes - if $verbose; then - echo "Ropes :" - fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i ropes$i ./array_tos --loops $2 --strlen $i --ccts $1 +# $1: string or buffer +# $2: loops +# $3: strlen min +# $4: strlen inc +# $5: strlen max +function bench_indexed_variant() +{ + tmp="${variant}_$1" + prepare_res index_$tmp.out $tmp $tmp + for i in `seq "$3" "$4" "$5"`; do + bench_command $i index_$tmp$i ./index_bench -m $1 --loops $2 --strlen $i done +} - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_flatstr.nit - - prepare_res arr_tos_flat.out arr_tos_flat flatstring - if $verbose; then - echo "FlatStrings :" +function bench_concat() +{ + if [ $# -lt 5 ]; then + echo "Wrong arguments for benchmark concat." + usage + exit fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i flatstring$i ./array_tos --loops $2 --strlen $i --ccts $1 - done - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_buffer.nit + echo "Generating executable chain_concat for variant $variant" + ../../bin/nitc --global chain_concat.nit - prepare_res arr_tos_buf.out arr_tos_buf flatbuffer - if $verbose; then - echo "FlatBuffers :" - fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i flatbuffer$i ./array_tos --loops $2 --strlen $i --ccts $1 - done + bench_concat_variant "string" $1 $2 $3 $4 $5 + bench_concat_variant "buffer" $1 $2 $3 $4 $5 - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_manual.nit + rm chain_concat +} - prepare_res arr_tos_man.out arr_tos_man memmove - if $verbose; then - echo "Memmove :" - fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i memmove$i ./array_tos --loops $2 --strlen $i --ccts $1 +# $1: string or buffer +# $2: loops +# $3: strlen +# $4: concatenations min +# $5: concatenations inc +# $6: concatenations max +function bench_concat_variant() +{ + tmp="${variant}_$1" + prepare_res out/concat/concat_$tmp.out $tmp $tmp + for i in `seq "$4" "$5" "$6"`; do + bench_command $i $tmp$i ./chain_concat -m $1 --loops $2 --strlen $3 --ccts $i done +} - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_man_buf.nit - - prepare_res arr_tos_man_buf.out arr_tos_man_buf flatbuf_with_capacity - if $verbose; then - echo "FlatBuffer.with_capacity :" +function bench_iteration() +{ + if [ $# -lt 4 ]; then + echo "Wrong arguments for benchmark iteration." + usage + exit fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i flatbuf_with_capacity$i ./array_tos --loops $2 --strlen $i --ccts $1 - done + echo "Generating executable iteration_bench for variant $variant" + ../../bin/nitc --global iteration_bench.nit - ../../../bin/nitc --global ../array_tos.nit -m ../array_to_s_vars/array_to_s_rope_buf.nit + bench_iterate_variant "iterator" "string" $1 $2 $3 $4 + bench_iterate_variant "index" "string" $1 $2 $3 $4 + bench_iterate_variant "iterator" "buffer" $1 $2 $3 $4 + bench_iterate_variant "index" "buffer" $1 $2 $3 $4 - prepare_res arr_tos_rope_buf.out arr_tos_rope_buf ropebuf - if $verbose; then - echo "RopeBuffer :" - fi - for i in `seq 1 "$3"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $1, Loops = $2" - fi - bench_command $i ropebuf$i ./array_tos --loops $2 --strlen $i --ccts $1 - done - - plot array_tos.gnu + rm iteration_bench +} - cd .. +# $1: iterator or index +# $2: string or buffer +# $3: loops +# $4: strlen min +# $5: strlen increment +# $6: strlen max +function bench_iterate_variant() +{ + tmp="${variant}_$1_$2" + prepare_res out/iteration/iteration_$tmp.out $tmp $tmp + for i in `seq "$4" "$5" "$6"`; do + bench_command $i $tmp$i ./iteration_bench -m $2 --iter-mode $1 --loops $3 --strlen $i + done } -function bench_concat() +function bench_substring() { - if [ -d string_concat ]; then - rm string_concat/* - else - mkdir string_concat + if [ $# -lt 4 ]; then + echo "Wrong arguments for benchmark substring." + usage + exit fi - cd string_concat - - ../../../bin/nitc --global ../chain_concat.nit + echo "Generating executable substr_bench for variant $variant" + ../../bin/nitc --global substr_bench.nit - if $verbose; then - echo "*** Benching concat performance ***" - fi + bench_substring_variant "string" $1 $2 $3 $4 + bench_substring_variant "buffer" $1 $2 $3 $4 - prepare_res concat_flat.out concat_flat flatstring - if $verbose; then - echo "FlatStrings :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $2, Loops = $3" - fi - bench_command $i flatstring$i ./chain_concat -m flatstr --loops $2 --strlen $3 --ccts $i - done + rm substr_bench +} - prepare_res concat_buf.out concat_buf flatbuffer - if $verbose; then - echo "FlatBuffers :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $2, Loops = $3" - fi - bench_command $i flatbuffer$i ./chain_concat -m flatbuf --loops $2 --strlen $3 --ccts $i +# $1: string or buffer +# $2: loops +# $3: strlen min +# $4: strlen increment +# $5: strlen max +function bench_substring_variant() +{ + tmp="${variant}_$1" + prepare_res out/substring/substring_$tmp.out $tmp $tmp + for i in `seq "$3" "$4" "$5"`; do + bench_command $i $tmp$i ./substr_bench -m $1 --loops $2 --strlen $i done +} - prepare_res concat_ropes.out concat_ropes ropes - if $verbose; then - echo "Ropes :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, Concats/loop = $2, Loops = $3" - fi - bench_command $i ropes$i ./chain_concat -m ropestr --loops $2 --strlen $3 --ccts $i - done +function bench_compiler() +{ + prepare_res out/compiler/compiler_$variant.out compiler_$variant compiler_$variant - prepare_res concat_buf_ropes.out concat_buf_ropes buffered_ropes - if $verbose; then - echo "Rope Buffer:" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "string length = $i, concats/loop = $2, loops = $3" - fi - bench_command $i buf_ropes$i ./chain_concat -m ropebuf --loops $2 --strlen $3 --ccts $i - done + echo "Pre-compiling nitc" + # Do it twice before bench to have stable times when generating C + ../../bin/nitc ../../src/nitc.nit -o ../../bin/nitc + echo "nitc (1/2)" + ../../bin/nitc ../../src/nitc.nit -o ../../bin/nitc + echo "nitc (2/2)" - plot concat.gnu + bench_command nitc nitc_$variant ../../bin/nitc ../../src/nitc.nit - cd .. + rm nitc } -function bench_iteration() +function bench_basic() { - if [ -d string_iter ]; then - rm string_iter/* - else - mkdir string_iter - fi - cd string_iter + ../../bin/nitc ../../examples/hello_world.nit + ./hello_world + rm hello_world +} - if $verbose; then - echo "*** Benching iteration performance ***" +function launch_bench() +{ + echo "---------------------------------------------------------" + echo " Trying variant $variant for benchmark $bench" + echo "---------------------------------------------------------" + git diff-index --quiet HEAD || { + echo "Cannot run benches on a dirty working directory." + echo "Please commit or stash your modifications and relaunch the command." + exit 1 + } + git am $curr_rev || { + echo "Error when applying patch $curr_rev" + git am --abort; + exit 1; + } + if [ "$need_bootstrap" = true ]; then + prepare_compiler fi + bench_$bench "$@"; + git reset --hard $head +} - ../../../bin/nitc --global ../iteration_bench.nit +function prepare_compiler() +{ + cd ../../c_src + rm nitc + make clean + cd ../src + ./ncall.sh + mv nitc.good ../bin/nitc + cd ../benchmarks/strings +} - prepare_res iter_flat_iter.out iter_flat_iter flatstring_iter - if $verbose; then - echo "FlatStrings by iterator :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats = $i, Loops = $3" - fi - bench_command $i flatstr_iter$i ./iteration_bench -m flatstr --iter-mode iterator --loops $2 --strlen $3 --ccts $i +function main() +{ + stop=false + while [ "$stop" = false ]; do + case "$1" in + -v) verbose=true; shift;; + -h) usage; exit;; + -n) count="$2"; shift; shift;; + *) stop=true + esac done - prepare_res iter_flat_index.out iter_flat_index flatstring_index - if $verbose; then - echo "FlatStrings by index :" + if [ $# -lt 1 ]; then + usage; + exit; fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats = $i, Loops = $3" - fi - bench_command $i flatstr_index$i ./iteration_bench -m flatstr --iter-mode index --loops $2 --strlen $3 --ccts $i - done - prepare_res iter_buf_iter.out iter_buf_iter flatbuffer_iter - if $verbose; then - echo "FlatBuffers by iterator :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats = $i, Loops = $3" + isok=false + for i in $benches; do + if [ $1 = $i ]; then + isok=true; fi - bench_command $i flatbuf_iter$i ./iteration_bench -m flatbuf --iter-mode iterator --loops $2 --strlen $3 --ccts $i done - - prepare_res iter_buf_index.out iter_buf_index flatbuffer_index - if $verbose; then - echo "FlatBuffers by index:" + if [ "$isok" = false ]; then + usage; + exit; fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats = $i, Loops = $3" - fi - bench_command $i flatbuf_index$i ./iteration_bench -m flatbuf --iter-mode index --loops $2 --strlen $3 --ccts $i - done - prepare_res iter_ropes_iter.out iter_ropes_iter ropes_iter - if $verbose; then - echo "Ropes by iterator :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" - fi - bench_command $i ropes_iter$i ./iteration_bench -m ropestr --iter-mode iterator --loops $2 --strlen $3 --ccts $i - done + bench=$1 + shift; - prepare_res iter_ropes_index.out iter_ropes_index ropes_index - if $verbose; then - echo "Ropes by index :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" - fi - bench_command $i ropes_index$i ./iteration_bench -m ropestr --iter-mode index --loops $2 --strlen $3 --ccts $i - done + git diff-index --quiet HEAD || { + echo "Cannot run benches on a dirty working directory." + echo "Please commit or stash your modifications and relaunch the command." + exit 1 + } - prepare_res iter_buf_ropes_iter.out iter_buf_ropes_iter buf_ropes_iter - if $verbose; then - echo "RopeBuffer by iterator :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" - fi - bench_command $i buf_ropes_iter$i ./iteration_bench -m ropebuf --iter-mode iterator --loops $2 --strlen $3 --ccts $i - done + head=`git rev-parse HEAD` + variant="HEAD" + need_plot=true + need_bootstrap=false - prepare_res iter_buf_ropes_index.out iter_buf_ropes_index buf_ropes_index - if $verbose; then - echo "RopeBuffer by index :" + if [ "$bench" = "basic" ]; then + need_plot=false fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" - fi - bench_command $i buf_ropes_index$i ./iteration_bench -m ropebuf --iter-mode index --loops $2 --strlen $3 --ccts $i - done - - plot iter.gnu - cd .. -} - -function bench_substr() -{ - if [ -d string_substr ]; then - rm string_substr/* - else - mkdir string_substr + if [ ! -d out ]; then + mkdir out fi - cd string_substr + cd out - if $verbose; then - echo "*** Benching substring performance ***" + if [ -d $bench ]; then + rm $bench/* + else + mkdir $bench fi + cd .. - ../../../bin/nitc --global ../substr_bench.nit - - prepare_res substr_flat.out substr_flat flatstring - if $verbose; then - echo "FlatStrings :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, loops = $2, Loops = $3" - fi - bench_command $i flatstring$i ./substr_bench -m flatstr --loops $2 --strlen $3 --ccts $i - done + echo "---------------------------------------------------------" + echo " Trying variant HEAD for benchmark $bench" + echo "---------------------------------------------------------" + bench_$bench "$@"; - prepare_res substr_buf.out substr_buf flatbuffer - if $verbose; then - echo "FlatBuffers :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, loops = $2, Loops = $3" - fi - bench_command $i flatbuffer$i ./substr_bench -m flatbuf --loops $2 --strlen $3 --ccts $i + for i in lib_variants/regular/*; do + curr_rev=$i + variant=`basename "$i" | cut -f 1 -d '.'` + launch_bench "$@" done - prepare_res substr_ropes.out substr_ropes ropes - if $verbose; then - echo "Ropes :" - fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, loops = $2, Loops = $3" - fi - bench_command $i ropes$i ./substr_bench -m ropestr --loops $2 --strlen $3 --ccts $i + need_bootstrap=true + for i in lib_variants/need_bootstrap/*; do + curr_rev=$i + variant=`basename "$i" | cut -f 1 -d '.'` + launch_bench "$@" done - prepare_res substr_buf_ropes.out substr_buf_ropes buf_ropes - if $verbose; then - echo "RopeBuffers :" + if [ "${need_plot}" = true ]; then + plot out/$bench/$bench.gnu fi - for i in `seq 1 "$1"`; do - if $verbose; then - echo "String length = $i, loops = $2, Loops = $3" - fi - bench_command $i buf_ropes$i ./substr_bench -m ropebuf --loops $2 --strlen $3 --ccts $i - done - - plot substr.gnu - - cd .. } -stop=false -while [ "$stop" = false ]; do - case "$1" in - -v) verbose=true; shift;; - -h) usage; exit;; - -n) count="$2"; shift; shift;; - *) stop=true - esac -done - -if test $# -ne 4; then - usage - exit -fi - -case "$1" in - iter) shift; bench_iteration $@ ;; - cct) shift; bench_concat $@ ;; - substr) shift; bench_substr $@ ;; - array) shift; bench_array $@ ;; - *) usage; exit;; -esac +main "$@"; -- 1.7.9.5