37be2f5bca89ebd118c63c11492b1a6913874758
1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from external files.
20 private import annotation
22 redef class ToolContext
24 var opt_file
= new OptionString("Specify test suite location", "-t", "--target-file")
26 var opt_pattern
= new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
28 var opt_autosav
= new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
31 # Used to test nitunit test files.
34 # `ModelBuilder` used to parse test files.
35 var mbuilder
: ModelBuilder
37 # Parse a file and return the contained `MModule`.
38 private fun parse_module_unit
(file
: String): nullable MModule do
39 var mmodule
= mbuilder
.parse
([file
]).first
40 if mbuilder
.get_mmodule_annotation
("test_suite", mmodule
) == null then return null
45 # Compile and execute the test suite for a NitUnit `file`.
46 fun test_module_unit
(file
: String): nullable TestSuite do
47 var toolcontext
= mbuilder
.toolcontext
48 var mmodule
= parse_module_unit
(file
)
49 # is the module a test_suite?
50 if mmodule
== null then return null
51 var suite
= new TestSuite(mmodule
, toolcontext
)
52 # method to execute before all tests in the module
53 var before_module
= mmodule
.before_test
54 if before_module
!= null then
55 toolcontext
.modelbuilder
.total_tests
+= 1
56 suite
.before_module
= new TestCase(suite
, before_module
, toolcontext
)
58 # generate all test cases
59 for mclassdef
in mmodule
.mclassdefs
do
60 if not mclassdef
.is_test
then continue
61 if not suite_match_pattern
(mclassdef
) then continue
62 toolcontext
.modelbuilder
.total_classes
+= 1
63 for mpropdef
in mclassdef
.mpropdefs
do
64 if not mpropdef
isa MMethodDef or not mpropdef
.is_test
then continue
65 if not case_match_pattern
(mpropdef
) then continue
66 toolcontext
.modelbuilder
.total_tests
+= 1
67 var test
= new TestCase(suite
, mpropdef
, toolcontext
)
71 # method to execute after all tests in the module
72 var after_module
= mmodule
.after_test
73 if after_module
!= null then
74 toolcontext
.modelbuilder
.total_tests
+= 1
75 suite
.after_module
= new TestCase(suite
, after_module
, toolcontext
)
81 # Is the test suite name match the pattern option?
82 private fun suite_match_pattern
(suite
: MClassDef): Bool do
83 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
84 if pattern
== null then return true
85 var ps
= pattern
.split_with
("::")
87 if ps
.length
== 1 and p
.first
.is_lower
then return true
88 if ps
.length
== 2 and p
.first
.is_lower
then return false
89 if p
.has_suffix
("*") then
90 p
= p
.substring
(0, p
.length
- 1)
91 if suite
.name
.has_prefix
(p
) then return true
93 if suite
.name
== p
then return true
98 # Is the test case name match the pattern option?
99 private fun case_match_pattern
(case
: MPropDef): Bool do
100 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
101 if pattern
== null then return true
102 var ps
= pattern
.split_with
("::")
104 if ps
.length
== 1 and p
.first
.is_upper
then return true
105 if ps
.length
== 2 and p
.first
.is_upper
then return false
106 if p
.has_suffix
("*") then
107 p
= p
.substring
(0, p
.length
- 1)
108 if case
.name
.has_prefix
(p
) then return true
110 if case
.name
== p
then return true
116 # A test suite contains all the test cases for a `MModule`.
119 # `MModule` under test
122 # `ToolContext` to use to display messages.
123 var toolcontext
: ToolContext
125 # List of `TestCase` to be executed in this suite.
126 var test_cases
= new Array[TestCase]
128 # Add a `TestCase` to the suite.
129 fun add_test
(case
: TestCase) do test_cases
.add case
131 # Test to be executed before the whole test suite.
132 var before_module
: nullable TestCase = null
134 # Test to be executed after the whole test suite.
135 var after_module
: nullable TestCase = null
139 toolcontext
.show_unit_status
("Test-suite of module " + mmodule
.full_name
, test_cases
)
142 # Execute the test suite
145 if not toolcontext
.test_dir
.file_exists
then
146 toolcontext
.test_dir
.mkdir
150 toolcontext
.info
("Execute test-suite {mmodule.name}", 1)
151 var before_module
= self.before_module
152 if not before_module
== null then before_module
.run
153 for case
in test_cases
do
155 toolcontext
.clear_progress_bar
156 toolcontext
.show_unit
(case
)
160 var after_module
= self.after_module
161 if not after_module
== null then after_module
.run
167 # Write the test unit for `self` in a nit compilable file.
169 var file
= new Template
170 file
.addn
"intrude import test_suite"
171 file
.addn
"import {mmodule.name}\n"
172 file
.addn
"var name = args.first"
173 for case
in test_cases
do
174 case
.write_to_nit
(file
)
176 file
.write_to_file
("{test_file}.nit")
179 # Return the test suite in XML format compatible with Jenkins.
180 # Contents depends on the `run` execution.
181 fun to_xml
: HTMLTag do
182 var n
= new HTMLTag("testsuite")
183 n
.attr
("package", mmodule
.name
)
184 var failure
= self.failure
185 if failure
!= null then
186 var f
= new HTMLTag("failure")
187 f
.attr
("message", failure
.to_s
)
190 for test
in test_cases
do n
.add test
.to_xml
195 # Generated test file name.
196 fun test_file
: String do
197 return toolcontext
.test_dir
/ "gen_{mmodule.name.escape_to_c}"
200 # Compile all `test_cases` cases in one file.
203 var nitc
= toolcontext
.find_nitc
206 var module_file
= mmodule
.location
.file
207 if module_file
== null then
208 toolcontext
.error
(null, "Error: cannot find module file for {mmodule.name}.")
209 toolcontext
.check_errors
212 var include_dir
= module_file
.filename
.dirname
213 var cmd
= "{nitc} --no-color '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
214 var res
= toolcontext
.safe_exec
(cmd
)
215 var f
= new FileReader.open
("{file}.out")
218 # set test case result
219 var loc
= mmodule
.location
222 toolcontext
.warning
(loc
, "failure", "FAILURE: {mmodule.name} (in file {file}.nit): {msg}")
223 toolcontext
.modelbuilder
.failed_tests
+= 1
225 toolcontext
.check_errors
228 # Error occured during test-suite compilation.
229 var failure
: nullable String = null
232 # A test case is a unit test considering only a `MMethodDef`.
236 # Test suite wich `self` belongs to.
237 var test_suite
: TestSuite
239 # Test method to be compiled and tested.
240 var test_method
: MMethodDef
242 redef fun full_name
do return test_method
.full_name
244 redef fun location
do return test_method
.location
246 # `ToolContext` to use to display messages and find `nitc` bin.
247 var toolcontext
: ToolContext
249 # Generate the test unit for `self` in `file`.
250 fun write_to_nit
(file
: Template) do
251 var name
= test_method
.name
252 file
.addn
"if name == \"{name}\
" then"
253 if test_method
.mproperty
.is_toplevel
then
256 file
.addn
"\tvar subject = new {test_method.mclassdef.name}.nitunit"
257 file
.addn
"\tsubject.before_test"
258 file
.addn
"\tsubject.{name}"
259 file
.addn
"\tsubject.after_test"
264 # Execute the test case.
266 toolcontext
.info
("Execute test-case {test_method.name}", 1)
268 if toolcontext
.opt_noact
.value
then return
270 var method_name
= test_method
.name
271 var test_file
= test_suite
.test_file
272 var res_name
= "{test_file}_{method_name.escape_to_c}"
273 var res
= toolcontext
.safe_exec
("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
274 var raw_output
= "{res_name}.out1".to_path
.read_all
275 self.raw_output
= raw_output
276 # set test case result
278 error
= "Runtime Error in file {test_file}.nit"
279 toolcontext
.modelbuilder
.failed_tests
+= 1
281 # no error, check with res file, if any.
282 var mmodule
= test_method
.mclassdef
.mmodule
283 var file
= mmodule
.filepath
285 var tries
= [ file
.dirname
/ mmodule
.name
+ ".sav" / test_method
.name
+ ".res",
286 file
.dirname
/ "sav" / test_method
.name
+ ".res" ,
287 file
.dirname
/ test_method
.name
+ ".res" ]
288 var savs
= [ for t
in tries
do if t
.file_exists
then t
]
289 if savs
.length
== 1 then
291 toolcontext
.info
("Diff output with {sav}", 1)
292 res
= toolcontext
.safe_exec
("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
295 else if toolcontext
.opt_autosav
.value
then
296 raw_output
.write_to_file
(sav
)
297 info
= "Expected output updated: {sav} (--autoupdate)"
299 self.raw_output
= "Diff\n" + "{res_name}.diff".to_path
.read_all
300 error
= "Difference with expected output: diff -u {sav} {res_name}.out1"
301 toolcontext
.modelbuilder
.failed_tests
+= 1
303 else if savs
.length
> 1 then
304 toolcontext
.info
("Conflicting diffs: {savs.join(", ")}", 1)
305 error
= "Conflicting expected output: {savs.join(", ", " and ")} all exist"
306 toolcontext
.modelbuilder
.failed_tests
+= 1
307 else if not raw_output
.is_empty
then
308 toolcontext
.info
("No diff: {tries.join(", ", " or ")} not found", 1)
309 if toolcontext
.opt_autosav
.value
then
310 var sav
= tries
.first
312 raw_output
.write_to_file
(sav
)
313 info
= "Expected output saved: {sav} (--autoupdate)"
321 redef fun xml_classname
do
322 var mclassdef
= test_method
.mclassdef
323 return "nitunit." + mclassdef
.mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
326 redef fun xml_name
do
327 return test_method
.mproperty
.full_name
331 redef class MMethodDef
332 # TODO use annotations?
334 # Is the method a test_method?
335 # i.e. begins with "test_"
336 private fun is_test
: Bool do return name
.has_prefix
("test_")
338 # Is the method a "before_module"?
339 private fun is_before_module
: Bool do return mproperty
.is_toplevel
and name
== "before_module"
341 # Is the method a "after_module"?
342 private fun is_after_module
: Bool do return mproperty
.is_toplevel
and name
== "after_module"
345 redef class MClassDef
346 # Is the class a TestClass?
347 # i.e. is a subclass of `TestSuite`
348 private fun is_test
: Bool do
349 var in_hierarchy
= self.in_hierarchy
350 if in_hierarchy
== null then return false
351 for sup
in in_hierarchy
.greaters
do
352 if sup
.name
== "TestSuite" then return true
359 # "before_module" method for this module.
360 private fun before_test
: nullable MMethodDef do
361 for mclassdef
in mclassdefs
do
362 if not mclassdef
.name
== "Object" then continue
363 for mpropdef
in mclassdef
.mpropdefs
do
364 if mpropdef
isa MMethodDef and mpropdef
.is_before_module
then return mpropdef
370 # "after_module" method for this module.
371 private fun after_test
: nullable MMethodDef do
372 for mclassdef
in mclassdefs
do
373 if not mclassdef
.name
== "Object" then continue
374 for mpropdef
in mclassdef
.mpropdefs
do
375 if mpropdef
isa MMethodDef and mpropdef
.is_after_module
then return mpropdef
382 redef class ModelBuilder
383 # Number of test classes generated.
384 var total_classes
= 0
386 # Number of tests generated.
389 # Number of failed tests.
392 # Run NitUnit test file for mmodule (if exists).
393 fun test_unit
(mmodule
: MModule): HTMLTag do
394 var ts
= new HTMLTag("testsuite")
395 toolcontext
.info
("nitunit: test-suite test_{mmodule}", 2)
396 var f
= toolcontext
.opt_file
.value
397 var test_file
= "test_{mmodule.name}.nit"
400 else if not test_file
.file_exists
then
401 var module_file
= mmodule
.location
.file
402 if module_file
== null then
403 toolcontext
.info
("Skip test for {mmodule}, no file found", 2)
406 var include_dir
= module_file
.filename
.dirname
407 test_file
= "{include_dir}/{test_file}"
409 if not test_file
.file_exists
then
410 toolcontext
.info
("Skip test for {mmodule}, no file {test_file} found", 2)
413 var tester
= new NitUnitTester(self)
414 var res
= tester
.test_module_unit
(test_file
)
416 toolcontext
.info
("Skip test for {mmodule}, no test suite found", 2)