var opt_file = new OptionString("Specify test suite location", "-t", "--target-file")
# --pattern
var opt_pattern = new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
+ # --autosav
+ var opt_autosav = new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
end
# Used to test nitunit test files.
# Test to be executed after the whole test suite.
var after_module: nullable TestCase = null
+ fun show_status
+ do
+ toolcontext.show_unit_status("Test-suite of module " + mmodule.full_name, test_cases)
+ end
+
# Execute the test suite
fun run do
+ show_status
if not toolcontext.test_dir.file_exists then
toolcontext.test_dir.mkdir
end
toolcontext.info("Execute test-suite {mmodule.name}", 1)
var before_module = self.before_module
if not before_module == null then before_module.run
- for case in test_cases do case.run
+ for case in test_cases do
+ case.run
+ toolcontext.clear_progress_bar
+ toolcontext.show_unit(case)
+ show_status
+ end
+
var after_module = self.after_module
if not after_module == null then after_module.run
+
+ show_status
+ print ""
end
# Write the test unit for `self` in a nit compilable file.
return
end
var include_dir = module_file.filename.dirname
- var cmd = "{nitc} --no-color '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
+ var cmd = "{nitc} --no-color -q '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
var res = toolcontext.safe_exec(cmd)
var f = new FileReader.open("{file}.out")
var msg = f.read_all
# A test case is a unit test considering only a `MMethodDef`.
class TestCase
+ super UnitTest
# Test suite wich `self` belongs to.
var test_suite: TestSuite
# Test method to be compiled and tested.
var test_method: MMethodDef
+ redef fun full_name do return test_method.full_name
+
+ redef fun location do return test_method.location
+
# `ToolContext` to use to display messages and find `nitc` bin.
var toolcontext: ToolContext
var test_file = test_suite.test_file
var res_name = "{test_file}_{method_name.escape_to_c}"
var res = toolcontext.safe_exec("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
- var f = new FileReader.open("{res_name}.out1")
- var msg = f.read_all
- f.close
+ var raw_output = "{res_name}.out1".to_path.read_all
+ self.raw_output = raw_output
# set test case result
- var loc = test_method.location
if res != 0 then
- error = msg
- toolcontext.warning(loc, "failure",
- "ERROR: {method_name} (in file {test_file}.nit): {msg}")
+ error = "Runtime Error in file {test_file}.nit"
toolcontext.modelbuilder.failed_tests += 1
+ else
+ # no error, check with res file, if any.
+ var mmodule = test_method.mclassdef.mmodule
+ var file = mmodule.filepath
+ if file != null then
+ var tries = [ file.dirname / mmodule.name + ".sav" / test_method.name + ".res",
+ file.dirname / "sav" / test_method.name + ".res" ,
+ file.dirname / test_method.name + ".res" ]
+ var savs = [ for t in tries do if t.file_exists then t ]
+ if savs.length == 1 then
+ var sav = savs.first
+ toolcontext.info("Diff output with {sav}", 1)
+ res = toolcontext.safe_exec("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
+ if res == 0 then
+ # OK
+ else if toolcontext.opt_autosav.value then
+ raw_output.write_to_file(sav)
+ info = "Expected output updated: {sav} (--autoupdate)"
+ else
+ self.raw_output = "Diff\n" + "{res_name}.diff".to_path.read_all
+ error = "Difference with expected output: diff -u {sav} {res_name}.out1"
+ toolcontext.modelbuilder.failed_tests += 1
+ end
+ else if savs.length > 1 then
+ toolcontext.info("Conflicting diffs: {savs.join(", ")}", 1)
+ error = "Conflicting expected output: {savs.join(", ", " and ")} all exist"
+ toolcontext.modelbuilder.failed_tests += 1
+ else if not raw_output.is_empty then
+ toolcontext.info("No diff: {tries.join(", ", " or ")} not found", 1)
+ if toolcontext.opt_autosav.value then
+ var sav = tries.first
+ sav.dirname.mkdir
+ raw_output.write_to_file(sav)
+ info = "Expected output saved: {sav} (--autoupdate)"
+ end
+ end
+ end
end
- toolcontext.check_errors
+ is_done = true
end
- # Error occured during test-case execution.
- var error: nullable String = null
-
- # Was the test case executed at least one?
- var was_exec = false
-
- # Return the `TestCase` in XML format compatible with Jenkins.
- fun to_xml: HTMLTag do
+ redef fun xml_classname do
var mclassdef = test_method.mclassdef
- var tc = new HTMLTag("testcase")
- # NOTE: jenkins expects a '.' in the classname attr
- tc.attr("classname", "nitunit." + mclassdef.mmodule.full_name + "." + mclassdef.mclass.full_name)
- tc.attr("name", test_method.mproperty.full_name)
- if was_exec then
- tc.add new HTMLTag("system-err")
- var n = new HTMLTag("system-out")
- n.append "out"
- tc.add n
- var error = self.error
- if error != null then
- n = new HTMLTag("error")
- n.attr("message", error.to_s)
- tc.add n
- end
- end
- return tc
+ return "nitunit." + mclassdef.mmodule.full_name + "." + mclassdef.mclass.full_name
+ end
+
+ redef fun xml_name do
+ return test_method.mproperty.full_name
end
end
redef class MClassDef
# Is the class a TestClass?
- # i.e. begins with "Test"
+ # i.e. is a subclass of `TestSuite`
private fun is_test: Bool do
var in_hierarchy = self.in_hierarchy
if in_hierarchy == null then return false
nitlanguage / nit.git / blobdiff