From: Jean Privat <jean@pryen.org>
Date: Mon, 6 Jul 2009 17:29:26 +0000 (-0400)
Subject: tests: tests.sh compares results with expected buggy results
X-Git-Tag: v0.3~185
X-Git-Url: http://nitlanguage.org

tests: tests.sh compares results with expected buggy results

New sav/*.fail files stores buggy results in order to track unfixed bugs
or planned regressions.

Signed-off-by: Jean Privat <jean@pryen.org>
---

diff --git a/tests/tests.sh b/tests/tests.sh
index 6a797b4..607b690 100755
--- a/tests/tests.sh
+++ b/tests/tests.sh
@@ -131,16 +131,42 @@ for ii in "$@"; do
 		fi
 
 		# Result
+		SAV=""
+		FAIL=""
 		if [ -r "sav/$ff.sav" ]; then
-			diff -u "$ff.res" "sav/$ff.sav" > "$ff.diff.log"
+			diff -u "$ff.res" "sav/$ff.sav" > "$ff.diff.sav.log"
 			if [ "$?" == 0 ]; then
+				SAV=OK
+			else
+				SAV=NOK
+			fi
+		fi
+		if [ -r "sav/$ff.fail" ]; then
+			diff -u "$ff.res" "sav/$ff.fail" > "$ff.diff.fail.log"
+			if [ "$?" == 0 ]; then
+				FAIL=OK
+			else
+				FAIL=NOK
+			fi
+		fi
+		if [ "x$SAV" = "xOK" ]; then
+			if [ "x$FAIL" = "x" ]; then
 				echo "[ok] $ff.res"
-				ok="$ok $ff"
 			else
-				echo "[======= fail $ff.res sav/$ff.sav =======]"
-				nok="$nok $ff"
-				echo "$ii" >> "$ERRLIST"
+				echo "[ok] $ff.res - but sav/$ff.fail remains!"
 			fi
+			ok="$ok $ff"
+		elif [ "x$FAIL" = "xOK" ]; then
+			echo "[fail] $ff.res"
+			ok="$ok $ff"
+		elif [ "x$SAV" = "xNOK" ]; then
+			echo "[======= fail $ff.res sav/$ff.sav =======]"
+			nok="$nok $ff"
+			echo "$ii" >> "$ERRLIST"
+		elif [ "x$FAIL" = "xNOK" ]; then
+			echo "[======= changed $ff.res sav/$ff.fail ======]"
+			nok="$nok $ff"
+			echo "$ii" >> "$ERRLIST"
 		else
 			echo "[=== no sav ===] $ff.res"
 			nos="$nos $ff"