1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from external files.
20 private import annotation
21 private import realtime
23 redef class ToolContext
25 var opt_pattern
= new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
27 var opt_autosav
= new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
30 # Used to test nitunit test files.
33 # `ModelBuilder` used to parse test files.
34 var mbuilder
: ModelBuilder
36 # Compile and execute `mmodule` as a test suite.
37 fun test_module_unit
(mmodule
: MModule): TestSuite do
38 var toolcontext
= mbuilder
.toolcontext
39 var suite
= new TestSuite(mmodule
, toolcontext
)
40 # method to execute before all tests in the module
41 var before_module
= mmodule
.before_test
42 if before_module
!= null then
43 toolcontext
.modelbuilder
.total_tests
+= 1
44 suite
.before_module
= new TestCase(suite
, before_module
, toolcontext
)
46 # generate all test cases
47 for mclassdef
in mmodule
.mclassdefs
do
48 if not mclassdef
.is_test
then continue
49 if not suite_match_pattern
(mclassdef
) then continue
50 toolcontext
.modelbuilder
.total_classes
+= 1
51 for mpropdef
in mclassdef
.mpropdefs
do
52 if not mpropdef
isa MMethodDef or not mpropdef
.is_test
then continue
53 if not case_match_pattern
(mpropdef
) then continue
54 toolcontext
.modelbuilder
.total_tests
+= 1
55 var test
= new TestCase(suite
, mpropdef
, toolcontext
)
59 # method to execute after all tests in the module
60 var after_module
= mmodule
.after_test
61 if after_module
!= null then
62 toolcontext
.modelbuilder
.total_tests
+= 1
63 suite
.after_module
= new TestCase(suite
, after_module
, toolcontext
)
69 # Is the test suite name match the pattern option?
70 private fun suite_match_pattern
(suite
: MClassDef): Bool do
71 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
72 if pattern
== null then return true
73 var ps
= pattern
.split_with
("::")
75 if ps
.length
== 1 and p
.first
.is_lower
then return true
76 if ps
.length
== 2 and p
.first
.is_lower
then return false
77 if p
.has_suffix
("*") then
78 p
= p
.substring
(0, p
.length
- 1)
79 if suite
.name
.has_prefix
(p
) then return true
81 if suite
.name
== p
then return true
86 # Is the test case name match the pattern option?
87 private fun case_match_pattern
(case
: MPropDef): Bool do
88 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
89 if pattern
== null then return true
90 var ps
= pattern
.split_with
("::")
92 if ps
.length
== 1 and p
.first
.is_upper
then return true
93 if ps
.length
== 2 and p
.first
.is_upper
then return false
94 if p
.has_suffix
("*") then
95 p
= p
.substring
(0, p
.length
- 1)
96 if case
.name
.has_prefix
(p
) then return true
98 if case
.name
== p
then return true
104 # A test suite contains all the test cases for a `MModule`.
107 # `MModule` under test
110 # `ToolContext` to use to display messages.
111 var toolcontext
: ToolContext
113 # List of `TestCase` to be executed in this suite.
114 var test_cases
= new Array[TestCase]
116 # Add a `TestCase` to the suite.
117 fun add_test
(case
: TestCase) do test_cases
.add case
119 # Test to be executed before the whole test suite.
120 var before_module
: nullable TestCase = null
122 # Test to be executed after the whole test suite.
123 var after_module
: nullable TestCase = null
125 # Display test suite status in std-out.
127 var test_cases
= self.test_cases
.to_a
128 if before_module
!= null then test_cases
.add before_module
.as(not null)
129 if after_module
!= null then test_cases
.add after_module
.as(not null)
130 toolcontext
.show_unit_status
("Test-suite of module " + mmodule
.full_name
, test_cases
)
133 # Execute the test suite
137 if not toolcontext
.test_dir
.file_exists
then
138 toolcontext
.test_dir
.mkdir
142 if failure
!= null then
143 for case
in test_cases
do
144 case
.fail
"Compilation Error"
145 case
.raw_output
= failure
146 toolcontext
.clear_progress_bar
147 toolcontext
.show_unit
(case
)
153 toolcontext
.info
("Execute test-suite {mmodule.name}", 1)
155 var before_module
= self.before_module
156 var after_module
= self.after_module
158 if before_module
!= null then
160 toolcontext
.clear_progress_bar
161 toolcontext
.show_unit
(before_module
)
162 if before_module
.error
!= null then
163 for case
in test_cases
do
164 case
.fail
"Nitunit Error: before_module test failed"
165 toolcontext
.clear_progress_bar
166 toolcontext
.show_unit
(case
)
168 if after_module
!= null then
169 after_module
.fail
"Nitunit Error: before_module test failed"
170 toolcontext
.clear_progress_bar
171 toolcontext
.show_unit
(after_module
)
179 for case
in test_cases
do
181 toolcontext
.clear_progress_bar
182 toolcontext
.show_unit
(case
)
186 if not after_module
== null then
188 toolcontext
.clear_progress_bar
189 toolcontext
.show_unit
(after_module
)
197 # Write the test unit for `self` in a nit compilable file.
199 var file
= new Template
200 file
.addn
"intrude import test_suite"
201 file
.addn
"import {mmodule.name}\n"
202 file
.addn
"var name = args.first"
203 var before_module
= self.before_module
204 if before_module
!= null then
205 before_module
.write_to_nit
(file
)
207 for case
in test_cases
do
208 case
.write_to_nit
(file
)
210 var after_module
= self.after_module
211 if after_module
!= null then
212 after_module
.write_to_nit
(file
)
214 file
.write_to_file
("{test_file}.nit")
217 # Return the test suite in XML format compatible with Jenkins.
218 # Contents depends on the `run` execution.
219 fun to_xml
: HTMLTag do
220 var n
= new HTMLTag("testsuite")
221 n
.attr
("package", mmodule
.name
)
222 for test
in test_cases
do n
.add test
.to_xml
226 # Generated test file name.
227 fun test_file
: String do
228 return toolcontext
.test_dir
/ "gen_{mmodule.name.escape_to_c}"
231 # Compile all `test_cases` cases in one file.
234 var nitc
= toolcontext
.find_nitc
237 var module_file
= mmodule
.location
.file
238 if module_file
== null then
239 toolcontext
.error
(null, "Error: cannot find module file for {mmodule.name}.")
240 toolcontext
.check_errors
243 var include_dir
= module_file
.filename
.dirname
244 var cmd
= "{nitc} --no-color -q '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
245 var res
= toolcontext
.safe_exec
(cmd
)
246 var f
= new FileReader.open
("{file}.out")
254 # Set environment variables for test suite execution
256 var loc
= mmodule
.location
.file
257 if loc
== null then return
258 toolcontext
.set_testing_path
(loc
.filename
)
261 # Error occured during test-suite compilation.
262 var failure
: nullable String = null
265 # A test case is a unit test considering only a `MMethodDef`.
269 # Test suite wich `self` belongs to.
270 var test_suite
: TestSuite
272 # Test method to be compiled and tested.
273 var test_method
: MMethodDef
275 redef fun full_name
do return test_method
.full_name
277 redef fun location
do return test_method
.location
279 # `ToolContext` to use to display messages and find `nitc` bin.
280 var toolcontext
: ToolContext
282 # Generate the test unit for `self` in `file`.
283 fun write_to_nit
(file
: Template) do
284 var name
= test_method
.name
285 file
.addn
"if name == \"{name}\
" then"
286 if test_method
.mproperty
.is_toplevel
then
289 file
.addn
"\tvar subject = new {test_method.mclassdef.name}.nitunit"
290 file
.addn
"\tsubject.before_test"
291 file
.addn
"\tsubject.{name}"
292 file
.addn
"\tsubject.after_test"
297 # Execute the test case.
299 toolcontext
.info
("Execute test-case {test_method.name}", 1)
301 if toolcontext
.opt_noact
.value
then return
303 var method_name
= test_method
.name
304 var test_file
= test_suite
.test_file
305 var res_name
= "{test_file}_{method_name.escape_to_c}"
306 var clock
= new Clock
307 var res
= toolcontext
.safe_exec
("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
308 if not toolcontext
.opt_no_time
.value
then real_time
= clock
.total
310 var raw_output
= "{res_name}.out1".to_path
.read_all
311 self.raw_output
= raw_output
312 # set test case result
314 error
= "Runtime Error in file {test_file}.nit"
315 toolcontext
.modelbuilder
.failed_tests
+= 1
317 # no error, check with res file, if any.
318 var mmodule
= test_method
.mclassdef
.mmodule
319 var file
= mmodule
.filepath
321 var tries
= [ file
.dirname
/ mmodule
.name
+ ".sav" / test_method
.name
+ ".res",
322 file
.dirname
/ "sav" / test_method
.name
+ ".res" ,
323 file
.dirname
/ test_method
.name
+ ".res" ]
324 var savs
= [ for t
in tries
do if t
.file_exists
then t
]
325 if savs
.length
== 1 then
327 toolcontext
.info
("Diff output with {sav}", 1)
328 res
= toolcontext
.safe_exec
("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
331 else if toolcontext
.opt_autosav
.value
then
332 raw_output
.write_to_file
(sav
)
333 info
= "Expected output updated: {sav} (--autoupdate)"
335 self.raw_output
= "Diff\n" + "{res_name}.diff".to_path
.read_all
336 error
= "Difference with expected output: diff -u {sav} {res_name}.out1"
337 toolcontext
.modelbuilder
.failed_tests
+= 1
339 else if savs
.length
> 1 then
340 toolcontext
.info
("Conflicting diffs: {savs.join(", ")}", 1)
341 error
= "Conflicting expected output: {savs.join(", ", " and ")} all exist"
342 toolcontext
.modelbuilder
.failed_tests
+= 1
343 else if not raw_output
.is_empty
then
344 toolcontext
.info
("No diff: {tries.join(", ", " or ")} not found", 1)
345 if toolcontext
.opt_autosav
.value
then
346 var sav
= tries
.first
348 raw_output
.write_to_file
(sav
)
349 info
= "Expected output saved: {sav} (--autoupdate)"
357 # Make the test case fail without testing it
359 # Useful when the compilation or the before_test failed.
360 fun fail
(message
: String) do
363 toolcontext
.modelbuilder
.failed_tests
+= 1
366 redef fun xml_classname
do
367 var a
= test_method
.full_name
.split
("$")
368 return "nitunit.{a[0]}.{a[1]}"
371 redef fun xml_name
do
372 var a
= test_method
.full_name
.split
("$")
377 redef class MMethodDef
378 # TODO use annotations?
380 # Is the method a test_method?
381 # i.e. begins with "test_"
382 private fun is_test
: Bool do return name
.has_prefix
("test_")
384 # Is the method a "before_module"?
385 private fun is_before_module
: Bool do return name
== "before_module"
387 # Is the method a "after_module"?
388 private fun is_after_module
: Bool do return name
== "after_module"
391 redef class MClassDef
392 # Is the class a TestClass?
393 # i.e. is a subclass of `TestSuite`
394 private fun is_test
: Bool do
395 var in_hierarchy
= self.in_hierarchy
396 if in_hierarchy
== null then return false
397 for sup
in in_hierarchy
.greaters
do
398 if sup
.name
== "TestSuite" then return true
405 # "before_module" method for this module.
406 private fun before_test
: nullable MMethodDef do
407 for mclassdef
in mclassdefs
do
408 if not mclassdef
.name
== "Sys" then continue
409 for mpropdef
in mclassdef
.mpropdefs
do
410 if mpropdef
isa MMethodDef and mpropdef
.is_before_module
then return mpropdef
416 # "after_module" method for this module.
417 private fun after_test
: nullable MMethodDef do
418 for mclassdef
in mclassdefs
do
419 if not mclassdef
.name
== "Sys" then continue
420 for mpropdef
in mclassdef
.mpropdefs
do
421 if mpropdef
isa MMethodDef and mpropdef
.is_after_module
then return mpropdef
428 redef class ModelBuilder
429 # Number of test classes generated.
430 var total_classes
= 0
432 # Number of tests generated.
435 # Number of failed tests.
438 # Run NitUnit test suite for `mmodule` (if it is one).
439 fun test_unit
(mmodule
: MModule): nullable HTMLTag do
440 # is the module a test_suite?
441 if get_mmodule_annotation
("test_suite", mmodule
) == null then return null
442 toolcontext
.info
("nitunit: test-suite {mmodule}", 2)
444 var tester
= new NitUnitTester(self)
445 var res
= tester
.test_module_unit
(mmodule
)