X-Git-Url: http://nitlanguage.org

diff --git a/src/run_bench.sh b/src/run_bench.sh
index 36e0f78..66967ab 100755
--- a/src/run_bench.sh
+++ b/src/run_bench.sh
@@ -27,8 +27,8 @@ count=2
 
 function die()
 {
-	echo >&2 "DIE: $*"
-	exit 1
+	echo >&2 "error: $*"
+	died=1
 }
 
 # Run a single command multiple time and store the execution times
@@ -72,7 +72,7 @@ function run_command()
 {
 	if [ "$dry_run" = "true" ]; then return; fi
 	echo " $ $@"
-	"$@"
+	"$@" || die "$@: failed"
 }
 
 # perl function to compute min/max/avg.
@@ -168,7 +168,7 @@ function skip_test()
 	fi
 	if test "$NOTSKIPED" = "all"; then
 		: # Execute anyway
-	elif echo "$1" | grep "$NOTSKIPED" >/dev/null 2>&1; then
+	elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
 		: # Found one to execute
 	else
 		return 0
@@ -196,14 +196,26 @@ function run_compiler()
 {
 	local title=$1
 	shift
-	run_command "$@" nitg.nit -o "nitg.$title.bin"
-	bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
-	run_command "$@" nit.nit -o "nit.$title.bin"
-	bench_command "nit" "nit test_parser.nit test_parser.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
-	run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
-	bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
-	run_command "$@" ../tests/shootout_binarytrees.nit -o "bintrees.$title.bin"
-	bench_command "bintrees" "shootout_binarytrees" "./bintrees.$title.bin" 17
+	if test -n "$fast"; then
+		run_command "$@" nitg.nit -o "nitg.$title.bin"
+		bench_command "nitg" "nitg test_parser.nit" "./nitg.$title.bin" -v test_parser.nit
+		run_command "$@" nit.nit -o "nit.$title.bin"
+		bench_command "nit" "nit test_parser.nit location.nit" "./nit.$title.bin" -v test_parser.nit -- -n location.nit
+		run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
+		bench_command "shoot" "shoot_logic" "./shoot.$title.bin"
+		run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
+		bench_command "bintrees" "bench_bintree_gen 16" "./bintrees.$title.bin" 16
+	else
+		run_command "$@" nitg.nit -o "nitg.$title.bin"
+		bench_command "nitg" "nitg --no-cc nitstats.nit" "./nitg.$title.bin" -v --no-cc nitstats.nit
+		bench_command "nitg-s" "nitg --separate nitg.nit" "./nitg.$title.bin" -v --separate nitg.nit
+		run_command "$@" nit.nit -o "nit.$title.bin"
+		bench_command "nit" "nit test_parser.nit rapid_type_analysis.nit" "./nit.$title.bin" -v test_parser.nit -- -n rapid_type_analysis.nit
+		run_command "$@" ../examples/shoot/shoot_logic.nit -o "shoot.$title.bin"
+		bench_command "shoot" "shoot_logic 30" "./shoot.$title.bin" 30
+		run_command "$@" ../tests/bench_bintree_gen.nit -o "bintrees.$title.bin"
+		bench_command "bintrees" "bench_bintree_gen 18" "./bintrees.$title.bin" 18
+	fi
 }
 
 ## HANDLE OPTIONS ##
@@ -214,6 +226,7 @@ function usage()
 	echo "  -v: verbose mode"
 	echo "  -n count: number of execution for each bar (default: $count)"
 	echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
+	echo "  --fast: Run less and faster tests"
 	echo "  -h: this help"
 }
 
@@ -224,6 +237,7 @@ while [ "$stop" = false ]; do
 		-h) usage; exit;;
 		-n) count="$2"; shift; shift;;
 		--dry) dry_run=true; shift;;
+		--fast) fast=true; shift;;
 		*) stop=true
 	esac
 done
@@ -236,6 +250,11 @@ if test -z "$NOTSKIPED"; then
 	echo "* all: run all the benches"
 fi
 
+## COMPILE ENGINES
+
+test -f ./nitc_3 || ./ncall.sh -O
+test -f ./nitg || ./nitc_3 nitg.nit -O -v
+
 ## EFFECTIVE BENCHS ##
 
 function bench_nitg_bootstrap()
@@ -276,11 +295,11 @@ function bench_steps()
 	bench_command "generate c" "" ./nitg --no-cc nitg.nit
 	bench_command "full" "" ./nitg nitg.nit -o "nitg_nitg.bin"
 
-	prepare_res "$name-nitg-s.dat" "nitg-s" "Various steps of nitg --separate"
-	bench_command "parse" "" ./nitg --separate --only-parse nitg.nit
-	bench_command "metamodel" "" ./nitg --separate --only-metamodel nitg.nit
-	bench_command "generate c" "" ./nitg --separate --no-cc nitg.nit
-	bench_command "full" "" ./nitg --separate nitg.nit -o "nitg_nitg-s.bin"
+	prepare_res "$name-nitg-e.dat" "nitg-e" "Various steps of nitg --erasure"
+	bench_command "parse" "" ./nitg --erasure --only-parse nitg.nit
+	bench_command "metamodel" "" ./nitg --erasure --only-metamodel nitg.nit
+	bench_command "generate c" "" ./nitg --erasure --no-cc nitg.nit
+	bench_command "full" "" ./nitg --erasure nitg.nit -o "nitg_nitg-e.bin"
 
 	plot "$name.gnu"
 }
@@ -289,11 +308,18 @@ bench_steps
 # $#: options to compare
 function bench_nitg_options()
 {
-	name="$FUNCNAME"
+	tag=$1
+	shift
+	name="$FUNCNAME-$tag"
 	skip_test "$name" && return
 	prepare_res "$name.dat" "no options" "nitg without options"
 	run_compiler "nitg" ./nitg
 
+	if test -n "$2"; then
+		prepare_res "$name-all.dat" "all" "nitg with all options $@"
+		run_compiler "nitg-$tag" ./nitg $@
+	fi
+
 	for opt in "$@"; do
 		prepare_res "$name$opt.dat" "$opt" "nitg with option $opt"
 		run_compiler "nitg$opt" ./nitg $opt
@@ -301,7 +327,60 @@ function bench_nitg_options()
 
 	plot "$name.gnu"
 }
-bench_nitg_options --hardening
+bench_nitg_options "hardening" --hardening
+bench_nitg_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
+
+function bench_nitg-s_options()
+{
+	tag=$1
+	shift
+	name="$FUNCNAME-$tag"
+	skip_test "$name" && return
+	prepare_res "$name.dat" "no options" "nitg-s without options"
+	run_compiler "nitg-s" ./nitg --separate
+
+	if test -n "$2"; then
+		prepare_res "$name-all.dat" "all" "nitg-s with all options $@"
+		run_compiler "nitg-s-$tag" ./nitg --separate $@
+	fi
+
+	for opt in "$@"; do
+		prepare_res "$name$opt.dat" "$opt" "nitg-s with option $opt"
+		run_compiler "nitg-s$opt" ./nitg --separate $opt
+	done
+
+	plot "$name.gnu"
+}
+bench_nitg-s_options "slower" --hardening --no-inline-intern --generic-resolution-tree --no-union-attribute --no-shortcut-equal --no-shortcut-range
+bench_nitg-s_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other
+bench_nitg-s_options "faster" --inline-coloring-numbers
+bench_nitg-s_options "typing" --bm-typing --phmod-typing --phand-typing
+
+function bench_nitg-e_options()
+{
+	tag=$1
+	shift
+	name="$FUNCNAME-$tag"
+	skip_test "$name" && return
+	prepare_res "$name.dat" "no options" "nitg-e without options"
+	run_compiler "nitg-e" ./nitg --erasure
+
+	if test -n "$2"; then
+		prepare_res "$name-all.dat" "all" "nitg-e with all options $@"
+		run_compiler "nitg-e-$tag" ./nitg --erasure $@
+	fi
+
+	for opt in "$@"; do
+		prepare_res "$name$opt.dat" "$opt" "nitg-e with option $opt"
+		run_compiler "nitg-e$opt" ./nitg --erasure $opt
+	done
+
+	plot "$name.gnu"
+}
+bench_nitg-e_options "slower" --hardening --no-inline-intern --no-union-attribute --no-shortcut-equal --no-shortcut-range
+bench_nitg-e_options "nocheck" --no-check-covariance --no-check-initialization --no-check-assert --no-check-autocast --no-check-other --no-check-erasure-cast
+bench_nitg-e_options "faster" --inline-coloring-numbers
+bench_nitg-e_options "typing" --bm-typing --phmod-typing --phand-typing
 
 function bench_nitc_gc()
 {
@@ -340,10 +419,78 @@ function bench_engines()
 	run_compiler "nitc-g" ./nitc_3 -O --global
 	prepare_res "$name-nitg.dat" "nitg" "nitg"
 	run_compiler "nitg" ./nitg
+	prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
+	run_compiler "nitg-s" ./nitg --separate
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
+	run_compiler "nitg-e" ./nitg --erasure
 	plot "$name.gnu"
 }
 bench_engines
 
+function bench_nitc_vc_nitg-e()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	prepare_res "$name-nitc.dat" "nitc" "nitc"
+	run_compiler "nitc" ./nitc_3 -O
+	prepare_res "$name-nitc-malloc.dat" "nitc-malloc" "nitc with malloc"
+	NIT_GC_OPTION="malloc" run_compiler "nitc" ./nitc_3 -O
+	prepare_res "$name-nitc-bohem.dat" "nitc-boehm" "nitc with boehm"
+	NIT_GC_OPTION="boehm" run_compiler "nitc" ./nitc_3 -O
+	prepare_res "$name-nitg-e-nockeck-malloc.dat" "nitg-e-nc-malloc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast and malloc"
+	run_compiler "nitg-e-nc-malloc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast --make-flags "CFLAGS=\"-O2 -DNOBOEHM\""
+	prepare_res "$name-nitg-e-nockeck.dat" "nitg-e-nc" "nitg with --erasure --no-check-autocast --no-check-erasure-cast"
+	run_compiler "nitg-e-nc" ./nitg --erasure --no-check-autocast --no-check-erasure-cast
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
+	run_compiler "nitg-e" ./nitg --erasure
+	plot "$name.gnu"
+}
+bench_nitc_vc_nitg-e
+
+function bench_nitg-e_gc()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	prepare_res "$name-nitg-e-malloc.dat" "nitg-e-malloc" "nitg with --erasure and malloc"
+	run_compiler "nitg-e-malloc" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+	prepare_res "$name-nitg-e-noatomic.dat" "nitg-e-noatomic" "nitg with --erasure and no atomic"
+	run_compiler "nitg-e-noatomic" ./nitg --erasure --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM_ATOMIC\""
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
+	run_compiler "nitg-e" ./nitg --erasure
+	plot "$name.gnu"
+}
+bench_nitg-e_gc
+
+function bench_cc_nitg-e()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	for o in "gcc0:CC=\"ccache gcc\" CFLAGS=-O0" "cl0:CC=\"ccache clang\" CFLAGS=-O0" "gccs:CC=\"ccache gcc\" CFLAGS=-Os" "cls:CC=\"ccache clang\" CFLAGS=-Os" "gcc2:CC=\"ccache gcc\" CFLAGS=-O2" "cl2:CC=\"ccache clang\" CFLAGS=-O2" "gcc3:CC=\"ccache gcc\" CFLAGS=-O3"  "cl3:CC=\"ccache clang\" CFLAGS=-O3"; do
+		f=`echo "$o" | cut -f1 -d:`
+		o=`echo "$o" | cut -f2 -d:`
+		prepare_res "$name-nitg-e-$f.dat" "nitg-e-$f" "nitg with --erasure --make-flags $o"
+		run_compiler "nitg-e-$f" ./nitg --erasure --make-flags "$o"
+	done
+	plot "$name.gnu"
+}
+bench_cc_nitg-e
+
+function bench_policy()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+	prepare_res "$name-nitg-s.dat" "nitg-s" "nitg with --separate"
+	run_compiler "nitg-s" ./nitg --separate
+	prepare_res "$name-nitg-su" "nitg-su" "nitg with --separate --no-check-covariance"
+	run_compiler "nitg-su" ./nitg --separate --no-check-covariance
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg with --erasure"
+	run_compiler "nitg-e" ./nitg --erasure
+	prepare_res "$name-nitg-eu" "nitg-eu" "nitg with --erasure --no-check-covariance --no-check-erasure-cast"
+	run_compiler "nitg-eu" ./nitg --erasure --no-check-covariance --no-check-erasure-cast
+	plot "$name.gnu"
+}
+bench_policy
+
 function bench_compilation_time
 {
 	name="$FUNCNAME"
@@ -356,10 +503,97 @@ function bench_compilation_time
 	for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
 		bench_command `basename "$i" .nit` "" ./nitg "$i" --no-cc
 	done
-	prepare_res "$name-nitg_g.dat" "nitg/g" "nitg/g"
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg --erasure"
 	for i in ../examples/hello_world.nit test_parser.nit nitg.nit; do
-		bench_command `basename "$i" .nit` "" ./nitg.bin "$i" --no-cc
+		bench_command `basename "$i" .nit` "" ./nitg --erasure "$i" --no-cc
 	done
 	plot "$name.gnu"
 }
 bench_compilation_time
+
+function bench_typetest_languages()
+{
+	name="$FUNCNAME"
+	skip_test "$name" && return
+
+	t=t
+	s=20
+	seq="w2_h2 w50_h2 w2_h25 w50_h25"
+	for b in $seq; do
+		run_command ./nitg benchs/gen.nit
+		run_command ./gen.bin "${t}_$b" "$b"
+	done
+
+	prepare_res "$name-g++.dat" "g++" "g++"
+	for b in $seq; do
+		run_command g++ "${t}_$b.cpp" -O2 -o "${t}_$b.g++.bin"
+		bench_command "$b" "" "./${t}_$b.g++.bin" $s
+	done
+
+	prepare_res "$name-clang++.dat" "clang++" "clang++"
+	for b in $seq; do
+		run_command clang++ "${t}_$b.cpp" -O2 -o "${t}_$b.clang++.bin"
+		bench_command "$b" "" "./${t}_$b.clang++.bin" $s
+	done
+
+	prepare_res "$name-java.dat" "java" "java"
+	for b in $seq; do
+		run_command javac ${t}_$b.java
+		bench_command "$b" "" java "${t}_$b" $s
+	done
+
+	prepare_res "$name-es.dat" "es" "es"
+	for b in $seq; do
+		run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
+		chmod +x app${t}_$b
+		mv app${t}_$b  ${t}_$b.es.bin
+		bench_command "$b" "" "./${t}_$b.es.bin" $s
+	done
+
+	prepare_res "$name-se.dat" "se" "se"
+	for b in $seq; do
+		run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
+		bench_command "$b" "" "./${t}_$b.se.bin" $s
+	done
+
+	#too slow
+	#prepare_res "$name-nitg.dat" "nitg" "nitg"
+	#for b in $seq; do
+	#	run_command ./nitg "${t}_$b.nit" -o "${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+	#	bench_command "$b" "" "./${t}_$b.nitg.bin" $s
+	#done
+
+	prepare_res "$name-nitg-s.dat" "nitg-s" "nitg-s"
+	for b in $seq; do
+		run_command ./nitg ${t}_$b.nit --separate -o "${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+		bench_command "$b" "" "./${t}_$b.nitg-s.bin" $s
+	done
+
+	prepare_res "$name-nitg-su.dat" "nitg-su" "nitg-su"
+	for b in $seq; do
+		run_command ./nitg ${t}_$b.nit --separate --no-check-covariance -o "${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+		bench_command "$b" "" "./${t}_$b.nitg-su.bin" $s
+	done
+
+
+	prepare_res "$name-nitg-e.dat" "nitg-e" "nitg-e"
+	for b in $seq; do
+		run_command ./nitg ${t}_$b.nit --erasure -o "${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+		bench_command "$b" "" "./${t}_$b.nitg-e.bin" $s
+	done
+
+	prepare_res "$name-nitg-eu.dat" "nitg-eu" "nitg-eu"
+	for b in $seq; do
+		run_command ./nitg ${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
+		bench_command "$b" "" "./${t}_$b.nitg-eu.bin" $s
+	done
+
+	plot "$name.gnu"
+}
+bench_typetest_languages
+
+if test -n "$died"; then
+	echo "Some commands failed"
+	exit 1
+fi
+exit 0