X-Git-Url: http://nitlanguage.org

diff --git a/tests/tests.sh b/tests/tests.sh
index c6e9158..f470498 100755
--- a/tests/tests.sh
+++ b/tests/tests.sh
@@ -19,156 +19,236 @@
 
 # Set lang do default to avoid failed tests because of locale
 export LANG=C
+export NIT_TESTING=true
 
 usage()
 {
 	e=`basename "$0"`
 	cat<<END
 Usage: $e [options] modulenames
--o option   Pass option to nitc
+-o option   Pass option to the engine
 -v          Verbose (show tests steps)
 -h          This help
+--tap       Produce TAP output
+--engine    Use a specific engine (default=nitc)
+--noskip    Do not skip a test even if the .skip file matches
+--[no]soso  Force enable (or disable) SOSO
 END
 }
 
+# $1 is the pattern of the test
+# $2 is the file to compare to
+# the result is:
+#   0: if the file to compare to do not exists
+#   1: if the file match
+#   2: if the file match with soso
+#   3: if the file do not match
+function compare_to_result()
+{
+	local pattern="$1"
+	local sav="$2"
+	if [ ! -r "$sav" ]; then return 0; fi
+	diff -u "out/$pattern.res" "$sav" > "out/$pattern.diff.sav.log"
+	if [ "$?" == 0 ]; then
+		return 1
+	fi
+	[ -z "$soso" ] && return 3
+	sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
+	sed '/[Ww]arning/d;/[Ee]rror/d' "$sav" > "out/$pattern.sav2"
+	grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
+	grep '[Ee]rror' "$sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
+	diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
+	if [ "$?" == 0 ]; then
+		return 2
+	else
+		return 3
+	fi
+}
+
 # As argument: the pattern used for the file
 function process_result()
 {
+	((tapcount=tapcount+1))
 	# Result
 	pattern=$1
+	description=$2
 	SAV=""
-	FAIL=""
+	NSAV=""
+	FIXME=""
+	NFIXME=""
 	SOSO=""
+	NSOSO=""
 	SOSOF=""
-	if [ -r "sav/$pattern.sav" ]; then
-		diff -u "out/$pattern.res" "sav/$pattern.sav" > "out/$pattern.diff.sav.log"
-		if [ "$?" == 0 ]; then
-			SAV=OK
+	NSOSOF=""
+	for sav in "sav/$engine/$pattern.res" "sav/$pattern.res" "sav/$pattern.sav"; do
+		compare_to_result "$pattern" "$sav"
+		case "$?" in
+			0)
+				;; # no file
+			1)
+				SAV="$sav" ;;
+			2)
+				SOSO="$sav" ;;
+			3)
+				NSAV="$sav";;
+		esac
+	done
+	for sav in "sav/$engine/fixme/$pattern.res" "sav/fixme/$pattern.res" "sav/$pattern.fail"; do
+		compare_to_result "$pattern" "$sav"
+		case "$?" in
+			0)
+				;; # no file
+			1)
+				FIXME="$sav" ;;
+			2)
+				SOSOF="$sav" ;;
+			3)
+				NFIXME="$sav";;
+		esac
+	done
+	grep 'NOT YET IMPLEMENTED' "out/$pattern.res" >/dev/null
+	NYI="$?"
+	if [ -n "$SAV" ]; then
+		if [ -n "$tap" ]; then
+			echo "ok - $description"
+		elif [ -z "$FIXME" ]; then
+			echo "[ok] out/$pattern.res $SAV"
 		else
-			SAV=NOK
+			echo "[ok] out/$pattern.res $SAV - but $FIXME remains!"
 		fi
-		sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
-		sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.sav" > "out/$pattern.sav2"
-		grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
-		grep '[Ee]rror' "sav/$pattern.sav" >/dev/null && echo "Error" >> "out/$pattern.sav2"
-		diff -u "out/$pattern.res2" "out/$pattern.sav2" > "out/$pattern.diff.sav2.log"
-		if [ "$?" == 0 ]; then
-			SOSO=OK
+		ok="$ok $pattern"
+	elif [ -n "$FIXME" ]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - $description # TODO expected failure"
 		else
-			SOSO=NOK
+			echo "[fixme] out/$pattern.res $FIXME"
 		fi
-	fi
-	if [ -r "sav/$pattern.fail" ]; then
-		diff -u "out/$pattern.res" "sav/$pattern.fail" > "out/$pattern.diff.fail.log"
-		if [ "$?" == 0 ]; then
-			FAIL=OK
+		todos="$todos $pattern"
+	elif [ -n "$SOSO" ]; then
+		if [ -n "$tap" ]; then
+			echo "ok - $description # SOSO"
 		else
-			FAIL=NOK
+			echo "[soso] out/$pattern.res $SOSO"
 		fi
-		sed '/[Ww]arning/d;/[Ee]rror/d' "out/$pattern.res" > "out/$pattern.res2"
-		sed '/[Ww]arning/d;/[Ee]rror/d' "sav/$pattern.fail" > "out/$pattern.fail2"
-		grep '[Ee]rror' "out/$pattern.res" >/dev/null && echo "Error" >> "out/$pattern.res2"
-		grep '[Ee]rror' "sav/$pattern.fail" >/dev/null && echo "Error" >> "out/$pattern.fail2"
-		diff -u "out/$pattern.res2" "out/$pattern.fail2" > "out/$pattern.diff.fail2.log"
-		if [ "$?" == 0 ]; then
-			SOSOF=OK
+		ok="$ok $pattern"
+	elif [ "x$NYI" = "x0" ]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - $description # TODO not yet implemented"
 		else
-			SOSOF=NOK
+			echo "[todo] out/$pattern.res -> not yet implemented"
 		fi
-	fi
-	if [ "x$SAV" = "xOK" ]; then
-		if [ "x$FAIL" = "x" ]; then
-			echo "[ok] out/$pattern.res"
+		todos="$todos $pattern"
+	elif [ -n "$SOSOF" ]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - $description # TODO SOSO expected failure"
 		else
-			echo "[ok] out/$pattern.res - but sav/$pattern.fail remains!"
+			echo "[fixme soso] out/$pattern.res $SOSOF"
+		fi
+		todos="$todos $pattern"
+	elif [ -n "$NSAV" ]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - $description"
+		else
+			echo "[======= fail out/$pattern.res $NSAV =======]"
 		fi
-		ok="$ok $pattern"
-	elif [ "x$FAIL" = "xOK" ]; then
-		echo "[fail] out/$pattern.res"
-		ok="$ok $pattern"
-	elif [ "x$SOSO" = "xOK" ]; then
-		echo "[soso] out/$pattern.res sav/$pattern.sav"
-		ok="$ok $pattern"
-	elif [ "x$SOSOF" = "xOK" ]; then
-		echo "[fail soso] out/$pattern.res sav/$pattern.fail"
-		ok="$ok $pattern"
-	elif [ "x$SAV" = "xNOK" ]; then
-		echo "[======= fail out/$pattern.res sav/$pattern.sav =======]"
 		nok="$nok $pattern"
 		echo "$ii" >> "$ERRLIST"
-	elif [ "x$FAIL" = "xNOK" ]; then
-		echo "[======= changed out/$pattern.res sav/$pattern.fail ======]"
+	elif [ -n "$NFIXME" ]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - $description"
+		else
+			echo "[======= changed out/$pattern.res $NFIXME ======]"
+		fi
 		nok="$nok $pattern"
 		echo "$ii" >> "$ERRLIST"
 	else
-		echo "[=== no sav ===] out/$pattern.res"
+		if [ -n "$tap" ]; then
+			echo "ok - $description # skip no sav"
+		else
+			echo "[=== no sav ===] out/$pattern.res"
+		fi
 		nos="$nos $pattern"
 	fi
 }
 
-find_nitc()
+need_skip()
 {
-	recent=`ls -t ../src/nitc ../src/nitc_[0-9] ../bin/nitc ../c_src/nitc 2>/dev/null | head -1`
-	if [[ "x$recent" == "x" ]]; then
-		echo 'Could not find nitc, aborting'
-		exit 1
+	test "$noskip" = true && return 1
+	if echo "$1" | grep -f "$engine.skip" >/dev/null 2>&1; then
+		((tapcount=tapcount+1))
+		if [ -n "$tap" ]; then
+			echo "ok - $2 # skip"
+		else
+			echo "=> $2: [skip]"
+		fi
+		return 0
 	fi
-	echo 'Using nitc from: '$recent
-	NITC=$recent
+	return 1
 }
 
-make_alts0()
+find_nitc()
 {
-	ii="$1"
-	xalt="$2"
-	fs=""
-	for alt in `sed -n "s/.*#!*\($xalt[0-9]*\)#.*/\1/p" "$ii" | sort -u`; do
-		f=`basename "$ii" .nit`
-		d=`dirname "$ii"`
-		ff="$f"
-		i="$ii"
-
-		if [ "x$alt" != "x" ]; then
-			test -d alt || mkdir -p alt
-			i="alt/${f}_$alt.nit"
-			ff="${ff}_$alt"
-			sed "s/^\(\s*\)#$alt#/\\1/g;/\S\s*#$alt#/d;/#!$alt#/d" "$ii" > "$i"
+	((tapcount=tapcount+1))
+	name="$enginebinname"
+	recent=`ls -t ../src/$name ../src/$name_[0-9] ../bin/$name ../c_src/$name 2>/dev/null | head -1`
+	if [[ "x$recent" == "x" ]]; then
+		if [ -n "$tap" ]; then
+			echo "not ok - find engine $name"
+			echo "Bail out! Could not find engine $name, aborting"
+		else
+			echo "Could not find engine $name, aborting"
 		fi
-		ff="$ff$MARK"
-		fs="$fs $i"
-	done
-	echo "$fs"
-}
-make_alts()
-{
-	ii="$1"
-	fs="$1"
-	for xalt in `sed -n 's/.*#!*\([0-9]*alt\)[0-9]*#.*/\1/p' "$ii" | sort -u`; do
-		fs2=""
-		for f in $fs; do
-			fs2="$fs2 `make_alts0 $f $xalt`"
-		done
-		fs="$fs $fs2"
-	done
-	echo "$fs"
+		exit 1
+	fi
+	if [ -n "$tap" ]; then
+		echo "ok - find engine $name: $recent"
+	else
+		echo "Using engine $name from: $recent"
+	fi
+	NITC=$recent
 }
 
-# The default nitc compiler
-[ -z "$NITC" ] && find_nitc
-
-# Set NIT_DIR if needed
-[ -z "$NIT_DIR" ] && export NIT_DIR=..
-
 verbose=false
 stop=false
+tapcount=0
+engine=nitc
+noskip=
 while [ $stop = false ]; do
 	case $1 in
 		-o) OPT="$OPT $2"; shift; shift;;
 		-v) verbose=true; shift;;
 		-h) usage; exit;;
+		--tap) tap=true; shift;;
+		--engine) engine="$2"; shift; shift;;
+		--noskip) noskip=true; shift;;
+		--soso) soso=true; shift;;
+		--nososo) nososo=true; shift;;
 		*) stop=true
 	esac
 done
+enginebinname=$engine
+case $engine in
+	nitc) ;;
+	nitg) OPT="--global $OPT";;
+	nitg-s) enginebinname=nitg; OPT="--separate $OPT";;
+	nitg-e) enginebinname=nitg; OPT="--erasure $OPT";;
+	nit) engine=niti ;;
+	niti) enginebinname=nit ;;
+esac
+
+# The default nitc compiler
+[ -z "$NITC" ] && find_nitc
+
+# Set NIT_DIR if needed
+[ -z "$NIT_DIR" ] && export NIT_DIR=..
+
+if sh -c "timelimit echo" 1>/dev/null 2>&1; then
+	TIMEOUT="timelimit -t 600"
+elif sh -c "timeout 1 echo" 1>/dev/null 2>&1; then
+	TIMEOUT="timeout 600s"
+else
+	echo "No timelimit or timeout command detected. Tests may hang :("
+fi
 
 # Mark to distinguish files among tests
 # MARK=
@@ -193,6 +273,7 @@ fi
 
 ok=""
 nok=""
+todos=""
 
 # CLEAN the out directory
 rm -rf out/ 2>/dev/null
@@ -203,6 +284,10 @@ for ii in "$@"; do
 		echo "File '$ii' does not exist."
 		continue
 	fi
+	f=`basename "$ii" .nit`
+
+	# Sould we skip the file for this engine?
+	need_skip $f $f && continue
 
 	tmp=${ii/../AA}
 	if [ "x$tmp" = "x$ii" ]; then
@@ -211,11 +296,14 @@ for ii in "$@"; do
 		includes="-I alt"
 	fi
 
-	f=`basename "$ii" .nit`
-	for i in `make_alts $ii`; do
+	for i in "$ii" `./alterner.pl --start '#' --altsep '_' $ii`; do
 		bf=`basename $i .nit`
 		ff="out/$bf"
-		echo -n "=> $bf: "
+
+		# Sould we skip the alternative for this engine?
+		need_skip $bf $bf && continue
+
+		test -z "$tap" && echo -n "=> $bf: "
 
 		if [ -f "$f.inputs" ]; then
 			inputs="$f.inputs"
@@ -223,32 +311,40 @@ for ii in "$@"; do
 			inputs=/dev/null
 		fi
 
-		# Compile
-		if [ "x$verbose" = "xtrue" ]; then
-			echo ""
-			echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
-		fi
-		$NITC --no-color $OPT -o "$ff.bin" "$i" $includes <"$inputs" 2> "$ff.cmp.err" > "$ff.compile.log"
-		ERR=$?
-		if [ "x$verbose" = "xtrue" ]; then
-			cat "$ff.compile.log"
-			cat >&2 "$ff.cmp.err"
+		if [ "$engine" = "niti" ]; then
+			cat > "./$ff.bin" <<END
+exec $NITC --no-color $OPT "$i" $includes -- "\$@"
+END
+			chmod +x "./$ff.bin"
+			> "$ff.cmp.err"
+			> "$ff.compile.log"
+			ERR=0
+		else
+			# Compile
+			if [ "x$verbose" = "xtrue" ]; then
+				echo ""
+				echo $NITC --no-color $OPT -o "$ff.bin" "$i" "$includes"
+			fi
+			NIT_NO_STACK=1 $TIMEOUT $NITC --no-color $OPT -o "$ff.bin" "$i" $includes 2> "$ff.cmp.err" > "$ff.compile.log"
+			ERR=$?
+			if [ "x$verbose" = "xtrue" ]; then
+				cat "$ff.compile.log"
+				cat >&2 "$ff.cmp.err"
+			fi
 		fi
-		egrep '^[A-Z0-9_]*$' "$ff.compile.log" > "$ff.res"
 		if [ "$ERR" != 0 ]; then
-			echo -n "! "
-			cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
-			process_result $bf
+			test -z "$tap" && echo -n "! "
+			cat "$ff.compile.log" "$ff.cmp.err" > "$ff.res"
+			process_result $bf $bf
 		elif [ -x "./$ff.bin" ]; then
-			cp "$ff.cmp.err" "$ff.res"
-			echo -n ". "
+			test -z "$tap" && echo -n ". "
 			# Execute
 			args=""
 			if [ "x$verbose" = "xtrue" ]; then
 				echo ""
 				echo "NIT_NO_STACK=1 ./$ff.bin" $args
 			fi
-			NIT_NO_STACK=1 "./$ff.bin" $args < "$inputs" >> "$ff.res" 2>"$ff.err"
+			NIT_NO_STACK=1 $TIMEOUT "./$ff.bin" $args < "$inputs" > "$ff.res" 2>"$ff.err"
 			if [ "x$verbose" = "xtrue" ]; then
 				cat "$ff.res"
 				cat >&2 "$ff.err"
@@ -258,10 +354,9 @@ for ii in "$@"; do
 			elif [ -d "$ff.write" ]; then
 				LANG=C /bin/ls -F $ff.write >> "$ff.res"
 			fi
-			if [ -s "$ff.err" ]; then
-				cat "$ff.err" >> "$ff.res"
-			fi
-			process_result $bf
+			cp "$ff.res"  "$ff.res2"
+			cat "$ff.cmp.err" "$ff.err" "$ff.res2" > "$ff.res"
+			process_result $bf $bf
 
 			if [ -f "$f.args" ]; then
 				fargs=$f.args
@@ -271,13 +366,20 @@ for ii in "$@"; do
 					args="$line"
 					bff=$bf"_args"$cptr
 					fff=$ff"_args"$cptr
+					name="$bf args $cptr"
+
+					# Sould we skip the input for this engine?
+					need_skip $bff "  $name" && continue
+
 					rm -rf "$fff.res" "$fff.err" "$fff.write" 2> /dev/null
 					if [ "x$verbose" = "xtrue" ]; then
 						echo ""
 						echo "NIT_NO_STACK=1 ./$ff.bin" $args
 					fi
-					echo -n "==> args #"$cptr " "
-					sh -c "NIT_NO_STACK=1 ./$ff.bin  ''$args < $inputs > $fff.res 2>$fff.err"
+					test -z "$tap" && echo -n "==> $name "
+					echo "./$ff.bin $args" > "./$fff.bin"
+					chmod +x "./$fff.bin"
+					sh -c "NIT_NO_STACK=1 $TIMEOUT ./$fff.bin < $inputs > $fff.res 2>$fff.err"
 					if [ "x$verbose" = "xtrue" ]; then
 						cat "$fff.res"
 						cat >&2 "$fff.err"
@@ -288,21 +390,31 @@ for ii in "$@"; do
 						LANG=C /bin/ls -F $fff.write >> "$fff.res"
 					fi
 					if [ -s "$fff.err" ]; then
-						cat "$fff.err" >> "$fff.res"
+						cp "$fff.res"  "$fff.res2"
+						cat "$fff.err" "$fff.res2" > "$fff.res"
 					fi
-					process_result $bff
+					process_result $bff "  $name"
 				done < $fargs
 			fi
 		else
-			echo -n "! "
-			cat "$ff.cmp.err" "$ff.compile.log" > "$ff.res"
-			#echo "Compilation error" > "$ff.res"
-			process_result $bf
+			test -z "$tap" && echo -n "! "
+			cat "$ff.cmp.err" > "$ff.res"
+			echo "Compilation error" > "$ff.res"
+			process_result $bf "$bf"
 		fi
 	done
 done
 
-echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos | wc -w`
+if [ -n "$tap" ]; then
+	echo "1..$tapcount"
+	echo "# ok:" `echo $ok | wc -w`
+	echo "# not ok:" `echo $nok | wc -w`
+	echo "# no sav:" `echo $nos | wc -w`
+	echo "# todo/fixme:" `echo $todos | wc -w`
+	exit
+fi
+
+echo "ok: " `echo $ok | wc -w` "/" `echo $ok $nok $nos $todos | wc -w`
 
 if [ -n "$nok" ]; then
 	echo "fail: $nok"
@@ -311,6 +423,9 @@ fi
 if [ -n "$nos" ]; then
 	echo "no sav: $nos"
 fi
+if [ -n "$todos" ]; then
+	echo "todo/fixme: $todos"
+fi
 
 # write $ERRLIST
 if [ "x$ERRLIST" != "x" ]; then