Sometimes, it is easier to validate a `TestCase` by comparing its output with a text file containing the expected result.
-For each TestCase `test_bar` of a TestSuite `test_mod.nit`, if the corresponding file `test_mod.sav/test_bar.res` exists, then the output of the test is compared with the file.
+For each TestCase `test_bar` of a TestSuite `test_mod.nit`, a corresponding file with the expected output is looked for:
+
+* "test_mod.sav/test_bar.res". I.e. test-cases grouped by test-suites.
+
+ This is the default and is useful if there is a lot of test-suites and test-cases in a directory
+
+* "sav/test_bar.res". I.e. all test-cases grouped in a common sub-directory.
+
+ Useful if there is a lot of test-suites OR test-cases in a directory.
+
+* "test_bar.res" raw in the directory.
+
+ Useful is there is a few test-suites and test-cases in a directory.
+
+All 3 are exclusive. If more than one exists, the test-case is failed.
+
+If a corresponding file then the output of the test-case is compared with the file.
The `diff(1)` command is used to perform the comparison.
The test is failed if non-zero is returned by `diff`.
If no corresponding `.res` file exists, then the output of the TestCase is ignored.
+To helps the management of the expected results, the option `--autosav` can be used to automatically create and update them.
+
+
## Configuring TestSuites
`TestSuites` also provide methods to configure the test run:
### `-t`, `--target-file`
Specify test suite location.
+### `--autosav`
+Automatically create/update .res files for black box testing.
+
+If a black block test fails because a difference between the expected result and the current result then the expected result file is updated (and the test is passed).
+
+If a test-case of a test-suite passes but that some output is generated, then an expected result file is created.
+
+It is expected that the created/updated files are checked since the tests are considered passed.
+A VCS like `git` is often a good tool to check the creation and modification of those files.
+
## SUITE GENERATION
### `--gen-suite`
var toolcontext = new ToolContext
-toolcontext.option_context.add_option(toolcontext.opt_full, toolcontext.opt_output, toolcontext.opt_dir, toolcontext.opt_noact, toolcontext.opt_pattern, toolcontext.opt_file, toolcontext.opt_gen_unit, toolcontext.opt_gen_force, toolcontext.opt_gen_private, toolcontext.opt_gen_show, toolcontext.opt_nitc)
+toolcontext.option_context.add_option(toolcontext.opt_full, toolcontext.opt_output, toolcontext.opt_dir, toolcontext.opt_noact, toolcontext.opt_pattern, toolcontext.opt_file, toolcontext.opt_autosav, toolcontext.opt_gen_unit, toolcontext.opt_gen_force, toolcontext.opt_gen_private, toolcontext.opt_gen_show, toolcontext.opt_nitc)
toolcontext.tooldescription = "Usage: nitunit [OPTION]... <file.nit>...\nExecutes the unit tests from Nit source files."
toolcontext.process_options(args)
# The location where the error occurred, if it makes sense.
var error_location: nullable Location = null is writable
+ # Additional noteworthy information when a test success.
+ var info: nullable String = null
+
# A colorful `[OK]` or `[KO]`.
fun status_tag(color: nullable Bool): String do
color = color or else true
else
res = "{status_tag(color)} {full_name}"
if more_message != null then res += more_message
+ var info = self.info
+ if info != null then
+ res += "\n {info}"
+ end
end
return res
end
var opt_file = new OptionString("Specify test suite location", "-t", "--target-file")
# --pattern
var opt_pattern = new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
+ # --autosav
+ var opt_autosav = new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
end
# Used to test nitunit test files.
var test_file = test_suite.test_file
var res_name = "{test_file}_{method_name.escape_to_c}"
var res = toolcontext.safe_exec("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
- self.raw_output = "{res_name}.out1".to_path.read_all
+ var raw_output = "{res_name}.out1".to_path.read_all
+ self.raw_output = raw_output
# set test case result
if res != 0 then
error = "Runtime Error in file {test_file}.nit"
var mmodule = test_method.mclassdef.mmodule
var file = mmodule.filepath
if file != null then
- var sav = file.dirname / mmodule.name + ".sav" / test_method.name + ".res"
- if sav.file_exists then
+ var tries = [ file.dirname / mmodule.name + ".sav" / test_method.name + ".res",
+ file.dirname / "sav" / test_method.name + ".res" ,
+ file.dirname / test_method.name + ".res" ]
+ var savs = [ for t in tries do if t.file_exists then t ]
+ if savs.length == 1 then
+ var sav = savs.first
toolcontext.info("Diff output with {sav}", 1)
res = toolcontext.safe_exec("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
- if res != 0 then
+ if res == 0 then
+ # OK
+ else if toolcontext.opt_autosav.value then
+ raw_output.write_to_file(sav)
+ info = "Expected output updated: {sav} (--autoupdate)"
+ else
self.raw_output = "Diff\n" + "{res_name}.diff".to_path.read_all
error = "Difference with expected output: diff -u {sav} {res_name}.out1"
toolcontext.modelbuilder.failed_tests += 1
end
- else
- toolcontext.info("No diff: {sav} not found", 2)
+ else if savs.length > 1 then
+ toolcontext.info("Conflicting diffs: {savs.join(", ")}", 1)
+ error = "Conflicting expected output: {savs.join(", ", " and ")} all exist"
+ toolcontext.modelbuilder.failed_tests += 1
+ else if not raw_output.is_empty then
+ toolcontext.info("No diff: {tries.join(", ", " or ")} not found", 1)
+ if toolcontext.opt_autosav.value then
+ var sav = tries.first
+ sav.dirname.mkdir
+ raw_output.write_to_file(sav)
+ info = "Expected output saved: {sav} (--autoupdate)"
+ end
end
end
end
-==== Test-suite of module test_nitunit4::test_nitunit4 | tests: 3
+==== Test-suite of module test_nitunit4::test_nitunit4 | tests: 4
[KO] test_nitunit4$TestTestSuite$test_foo
test_nitunit4/test_nitunit4.nit:22,2--26,4: Runtime Error in file nitunit.out/gen_test_nitunit4.nit
Output
[OK] test_nitunit4$TestTestSuite$test_bar
[KO] test_nitunit4$TestTestSuite$test_baz
- test_nitunit4/test_nitunit4.nit:32,2--34,4: Difference with expected output: diff -u test_nitunit4/test_nitunit4.sav/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1
+ test_nitunit4/test_nitunit4.nit:32,2--34,4: Difference with expected output: diff -u test_nitunit4/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1
Output
Diff
- --- expected:test_nitunit4/test_nitunit4.sav/test_baz.res
+ --- expected:test_nitunit4/test_baz.res
+++ got:nitunit.out/gen_test_nitunit4_test_baz.out1
@@ -1 +1,3 @@
-Bad result file
+Tested method
+After Test
+[KO] test_nitunit4$TestTestSuite$test_sav_conflict
+ test_nitunit4/test_nitunit4.nit:36,2--38,4: Conflicting expected output: test_nitunit4/test_nitunit4.sav/test_sav_conflict.res, test_nitunit4/sav/test_sav_conflict.res and test_nitunit4/test_sav_conflict.res all exist
+ Output
+ Before Test
+ Tested method
+ After Test
-Docunits: Entities: 12; Documented ones: 0; With nitunits: 0
-Test suites: Classes: 1; Test Cases: 3; Failures: 2
-[FAILURE] 2/3 tests failed.
+
+Docunits: Entities: 13; Documented ones: 0; With nitunits: 0
+Test suites: Classes: 1; Test Cases: 4; Failures: 3
+[FAILURE] 3/4 tests failed.
`nitunit.out` is not removed for investigation.
<testsuites><testsuite package="test_nitunit4>"></testsuite><testsuite package="test_nitunit4::nitunit4"></testsuite><testsuite package="test_nitunit4"><testcase classname="nitunit.test_nitunit4::test_nitunit4.test_nitunit4::TestTestSuite" name="test_nitunit4::TestTestSuite::test_foo"><error>Runtime Error in file nitunit.out/gen_test_nitunit4.nit</error><system-err>Before Test
Tested method
</system-err></testcase><testcase classname="nitunit.test_nitunit4::test_nitunit4.test_nitunit4::TestTestSuite" name="test_nitunit4::TestTestSuite::test_bar"><system-err>Before Test
Tested method
After Test
-</system-err></testcase><testcase classname="nitunit.test_nitunit4::test_nitunit4.test_nitunit4::TestTestSuite" name="test_nitunit4::TestTestSuite::test_baz"><error>Difference with expected output: diff -u test_nitunit4/test_nitunit4.sav/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1</error><system-err>Diff
---- expected:test_nitunit4/test_nitunit4.sav/test_baz.res
+</system-err></testcase><testcase classname="nitunit.test_nitunit4::test_nitunit4.test_nitunit4::TestTestSuite" name="test_nitunit4::TestTestSuite::test_baz"><error>Difference with expected output: diff -u test_nitunit4/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1</error><system-err>Diff
+--- expected:test_nitunit4/test_baz.res
+++ got:nitunit.out/gen_test_nitunit4_test_baz.out1
@@ -1 +1,3 @@
-Bad result file
+Before Test
+Tested method
+After Test
+</system-err></testcase><testcase classname="nitunit.test_nitunit4::test_nitunit4.test_nitunit4::TestTestSuite" name="test_nitunit4::TestTestSuite::test_sav_conflict"><error>Conflicting expected output: test_nitunit4/test_nitunit4.sav/test_sav_conflict.res, test_nitunit4/sav/test_sav_conflict.res and test_nitunit4/test_sav_conflict.res all exist</error><system-err>Before Test
+Tested method
+After Test
</system-err></testcase></testsuite><testsuite package="test_nitunit4::test_nitunit4"></testsuite><testsuite></testsuite><testsuite package="test_nitunit4::test_nitunit4_base"></testsuite><testsuite></testsuite></testsuites>
\ No newline at end of file
fun test_baz do
print "Tested method"
end
+
+ fun test_sav_conflict do
+ print "Tested method"
+ end
end
nitlanguage / nit.git / commitdiff