From: Jean Privat Date: Sun, 5 Jun 2016 16:10:23 +0000 (-0400) Subject: Merge: Honor `source.exclude` in ini files X-Git-Url: http://nitlanguage.org?hp=81c4a3477869e46e0ea919d67874fd6b4f2ab8fe Merge: Honor `source.exclude` in ini files `source.exclude` is used in very specific cases to prevent some .nit file of directory to belong to a package. This means two things: * when the loader is scanning/loading a full package or group, and finds an excluded group or module, then the excluded entity will be not included in the package (and not scanned/loaded) * when the loader is explicitly requested to load an excluded entity, then it will be detached from the parent package. The excluded modules will be considered as standalone modules. Pull-Request: #2160 Reviewed-by: Alexis Laferrière --- diff --git a/share/man/nitunit.md b/share/man/nitunit.md index 6f09402..23b7382 100644 --- a/share/man/nitunit.md +++ b/share/man/nitunit.md @@ -173,7 +173,23 @@ end Sometimes, it is easier to validate a `TestCase` by comparing its output with a text file containing the expected result. -For each TestCase `test_bar` of a TestSuite `test_mod.nit`, if the corresponding file `test_mod.sav/test_bar.res` exists, then the output of the test is compared with the file. +For each TestCase `test_bar` of a TestSuite `test_mod.nit`, a corresponding file with the expected output is looked for: + +* "test_mod.sav/test_bar.res". I.e. test-cases grouped by test-suites. + + This is the default and is useful if there is a lot of test-suites and test-cases in a directory + +* "sav/test_bar.res". I.e. all test-cases grouped in a common sub-directory. + + Useful if there is a lot of test-suites OR test-cases in a directory. + +* "test_bar.res" raw in the directory. + + Useful is there is a few test-suites and test-cases in a directory. + +All 3 are exclusive. If more than one exists, the test-case is failed. + +If a corresponding file then the output of the test-case is compared with the file. The `diff(1)` command is used to perform the comparison. The test is failed if non-zero is returned by `diff`. @@ -195,6 +211,9 @@ Hello! If no corresponding `.res` file exists, then the output of the TestCase is ignored. +To helps the management of the expected results, the option `--autosav` can be used to automatically create and update them. + + ## Configuring TestSuites `TestSuites` also provide methods to configure the test run: @@ -292,6 +311,16 @@ Examples: `TestFoo`, `TestFoo*`, `TestFoo::test_foo`, `TestFoo::test_foo*`, `tes ### `-t`, `--target-file` Specify test suite location. +### `--autosav` +Automatically create/update .res files for black box testing. + +If a black block test fails because a difference between the expected result and the current result then the expected result file is updated (and the test is passed). + +If a test-case of a test-suite passes but that some output is generated, then an expected result file is created. + +It is expected that the created/updated files are checked since the tests are considered passed. +A VCS like `git` is often a good tool to check the creation and modification of those files. + ## SUITE GENERATION ### `--gen-suite` diff --git a/src/nitunit.nit b/src/nitunit.nit index 8d79e80..c9aa8ae 100644 --- a/src/nitunit.nit +++ b/src/nitunit.nit @@ -20,7 +20,7 @@ import testing var toolcontext = new ToolContext -toolcontext.option_context.add_option(toolcontext.opt_full, toolcontext.opt_output, toolcontext.opt_dir, toolcontext.opt_noact, toolcontext.opt_pattern, toolcontext.opt_file, toolcontext.opt_gen_unit, toolcontext.opt_gen_force, toolcontext.opt_gen_private, toolcontext.opt_gen_show, toolcontext.opt_nitc) +toolcontext.option_context.add_option(toolcontext.opt_full, toolcontext.opt_output, toolcontext.opt_dir, toolcontext.opt_noact, toolcontext.opt_pattern, toolcontext.opt_file, toolcontext.opt_autosav, toolcontext.opt_gen_unit, toolcontext.opt_gen_force, toolcontext.opt_gen_private, toolcontext.opt_gen_show, toolcontext.opt_nitc) toolcontext.tooldescription = "Usage: nitunit [OPTION]... ...\nExecutes the unit tests from Nit source files." toolcontext.process_options(args) diff --git a/src/testing/testing_base.nit b/src/testing/testing_base.nit index b01d7dd..36ce1f9 100644 --- a/src/testing/testing_base.nit +++ b/src/testing/testing_base.nit @@ -198,6 +198,9 @@ abstract class UnitTest # The location where the error occurred, if it makes sense. var error_location: nullable Location = null is writable + # Additional noteworthy information when a test success. + var info: nullable String = null + # A colorful `[OK]` or `[KO]`. fun status_tag(color: nullable Bool): String do color = color or else true @@ -236,6 +239,10 @@ abstract class UnitTest else res = "{status_tag(color)} {full_name}" if more_message != null then res += more_message + var info = self.info + if info != null then + res += "\n {info}" + end end return res end diff --git a/src/testing/testing_suite.nit b/src/testing/testing_suite.nit index 309393c..a37d243 100644 --- a/src/testing/testing_suite.nit +++ b/src/testing/testing_suite.nit @@ -24,6 +24,8 @@ redef class ToolContext var opt_file = new OptionString("Specify test suite location", "-t", "--target-file") # --pattern var opt_pattern = new OptionString("Only run test case with name that match pattern", "-p", "--pattern") + # --autosav + var opt_autosav = new OptionBool("Automatically create/update .res files for black box testing", "--autosav") end # Used to test nitunit test files. @@ -269,7 +271,8 @@ class TestCase var test_file = test_suite.test_file var res_name = "{test_file}_{method_name.escape_to_c}" var res = toolcontext.safe_exec("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 '{res_name}.diff' 2>&1 1 then + toolcontext.info("Conflicting diffs: {savs.join(", ")}", 1) + error = "Conflicting expected output: {savs.join(", ", " and ")} all exist" + toolcontext.modelbuilder.failed_tests += 1 + else if not raw_output.is_empty then + toolcontext.info("No diff: {tries.join(", ", " or ")} not found", 1) + if toolcontext.opt_autosav.value then + var sav = tries.first + sav.dirname.mkdir + raw_output.write_to_file(sav) + info = "Expected output saved: {sav} (--autoupdate)" + end end end end diff --git a/tests/sav/nitunit_args9.res b/tests/sav/nitunit_args9.res index 7d1f2bd2..773614a 100644 --- a/tests/sav/nitunit_args9.res +++ b/tests/sav/nitunit_args9.res @@ -1,4 +1,4 @@ -==== Test-suite of module test_nitunit4::test_nitunit4 | tests: 3 +==== Test-suite of module test_nitunit4::test_nitunit4 | tests: 4 [KO] test_nitunit4$TestTestSuite$test_foo test_nitunit4/test_nitunit4.nit:22,2--26,4: Runtime Error in file nitunit.out/gen_test_nitunit4.nit Output @@ -9,10 +9,10 @@ [OK] test_nitunit4$TestTestSuite$test_bar [KO] test_nitunit4$TestTestSuite$test_baz - test_nitunit4/test_nitunit4.nit:32,2--34,4: Difference with expected output: diff -u test_nitunit4/test_nitunit4.sav/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1 + test_nitunit4/test_nitunit4.nit:32,2--34,4: Difference with expected output: diff -u test_nitunit4/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1 Output Diff - --- expected:test_nitunit4/test_nitunit4.sav/test_baz.res + --- expected:test_nitunit4/test_baz.res +++ got:nitunit.out/gen_test_nitunit4_test_baz.out1 @@ -1 +1,3 @@ -Bad result file @@ -20,10 +20,17 @@ +Tested method +After Test +[KO] test_nitunit4$TestTestSuite$test_sav_conflict + test_nitunit4/test_nitunit4.nit:36,2--38,4: Conflicting expected output: test_nitunit4/test_nitunit4.sav/test_sav_conflict.res, test_nitunit4/sav/test_sav_conflict.res and test_nitunit4/test_sav_conflict.res all exist + Output + Before Test + Tested method + After Test -Docunits: Entities: 12; Documented ones: 0; With nitunits: 0 -Test suites: Classes: 1; Test Cases: 3; Failures: 2 -[FAILURE] 2/3 tests failed. + +Docunits: Entities: 13; Documented ones: 0; With nitunits: 0 +Test suites: Classes: 1; Test Cases: 4; Failures: 3 +[FAILURE] 3/4 tests failed. `nitunit.out` is not removed for investigation. Runtime Error in file nitunit.out/gen_test_nitunit4.nitBefore Test Tested method @@ -32,12 +39,15 @@ Runtime error: Assert failed (test_nitunit4/test_nitunit4_base.nit:31) Before Test Tested method After Test -Difference with expected output: diff -u test_nitunit4/test_nitunit4.sav/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1Diff ---- expected:test_nitunit4/test_nitunit4.sav/test_baz.res +Difference with expected output: diff -u test_nitunit4/test_baz.res nitunit.out/gen_test_nitunit4_test_baz.out1Diff +--- expected:test_nitunit4/test_baz.res +++ got:nitunit.out/gen_test_nitunit4_test_baz.out1 @@ -1 +1,3 @@ -Bad result file +Before Test +Tested method +After Test +Conflicting expected output: test_nitunit4/test_nitunit4.sav/test_sav_conflict.res, test_nitunit4/sav/test_sav_conflict.res and test_nitunit4/test_sav_conflict.res all existBefore Test +Tested method +After Test \ No newline at end of file diff --git a/tests/test_nitunit4/sav/test_sav_conflict.res b/tests/test_nitunit4/sav/test_sav_conflict.res new file mode 100644 index 0000000..7b3a785 --- /dev/null +++ b/tests/test_nitunit4/sav/test_sav_conflict.res @@ -0,0 +1 @@ +BAD diff --git a/tests/test_nitunit4/test_nitunit4.sav/test_baz.res b/tests/test_nitunit4/test_baz.res similarity index 100% rename from tests/test_nitunit4/test_nitunit4.sav/test_baz.res rename to tests/test_nitunit4/test_baz.res diff --git a/tests/test_nitunit4/test_nitunit4.nit b/tests/test_nitunit4/test_nitunit4.nit index cd13aca..8864263 100644 --- a/tests/test_nitunit4/test_nitunit4.nit +++ b/tests/test_nitunit4/test_nitunit4.nit @@ -32,4 +32,8 @@ class TestTestSuite fun test_baz do print "Tested method" end + + fun test_sav_conflict do + print "Tested method" + end end diff --git a/tests/test_nitunit4/test_nitunit4.sav/test_sav_conflict.res b/tests/test_nitunit4/test_nitunit4.sav/test_sav_conflict.res new file mode 100644 index 0000000..7b3a785 --- /dev/null +++ b/tests/test_nitunit4/test_nitunit4.sav/test_sav_conflict.res @@ -0,0 +1 @@ +BAD diff --git a/tests/test_nitunit4/test_sav_conflict.res b/tests/test_nitunit4/test_sav_conflict.res new file mode 100644 index 0000000..7b3a785 --- /dev/null +++ b/tests/test_nitunit4/test_sav_conflict.res @@ -0,0 +1 @@ +BAD