1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from external files.
20 private import parse_annotations
21 private import realtime
23 redef class ToolContext
25 var opt_pattern
= new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
27 var opt_autosav
= new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
30 # Used to test nitunit test files.
33 # `ModelBuilder` used to parse test files.
34 var mbuilder
: ModelBuilder
36 # Compile and execute `mmodule` as a test suite.
37 fun test_module_unit
(mmodule
: MModule): TestSuite do
38 var toolcontext
= mbuilder
.toolcontext
39 var suite
= new TestSuite(mmodule
, toolcontext
)
40 # method to execute before all tests in the module
41 for mmethod
in mmodule
.before_all
do
42 toolcontext
.modelbuilder
.total_tests
+= 1
43 suite
.before_all
.add
new TestCase(suite
, mmethod
, toolcontext
)
45 # generate all test cases
46 for mclassdef
in mmodule
.mclassdefs
do
47 if not mclassdef
.is_test
then continue
48 if not suite_match_pattern
(mclassdef
) then continue
49 toolcontext
.modelbuilder
.total_classes
+= 1
51 var test_class
= new TestClass
53 # method to execute before all tests in the class
54 for mmethod
in mclassdef
.before_all
do
55 toolcontext
.modelbuilder
.total_tests
+= 1
56 test_class
.before_all
.add
new TestCase(suite
, mmethod
, toolcontext
)
59 var before
= mclassdef
.before
60 var after
= mclassdef
.after
62 for mpropdef
in mclassdef
.mpropdefs
do
63 if not mpropdef
isa MMethodDef or not mpropdef
.is_test
then continue
64 if not case_match_pattern
(mpropdef
) then continue
65 toolcontext
.modelbuilder
.total_tests
+= 1
66 var test
= new TestCase(suite
, mpropdef
, toolcontext
)
69 test_class
.test_cases
.add test
72 # method to execute after all tests in the class
73 for mmethod
in mclassdef
.after_all
do
74 toolcontext
.modelbuilder
.total_tests
+= 1
75 test_class
.after_all
.add
new TestCase(suite
, mmethod
, toolcontext
)
78 suite
.test_classes
.add test_class
80 # method to execute after all tests in the module
81 for mmethod
in mmodule
.after_all
do
82 toolcontext
.modelbuilder
.total_tests
+= 1
83 suite
.after_all
.add
new TestCase(suite
, mmethod
, toolcontext
)
89 # Is the test suite name match the pattern option?
90 private fun suite_match_pattern
(suite
: MClassDef): Bool do
91 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
92 if pattern
== null then return true
93 var ps
= pattern
.split_with
("::")
95 if ps
.length
== 1 and p
.first
.is_lower
then return true
96 if ps
.length
== 2 and p
.first
.is_lower
then return false
97 if p
.has_suffix
("*") then
98 p
= p
.substring
(0, p
.length
- 1)
99 if suite
.name
.has_prefix
(p
) then return true
101 if suite
.name
== p
then return true
106 # Is the test case name match the pattern option?
107 private fun case_match_pattern
(case
: MPropDef): Bool do
108 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
109 if pattern
== null then return true
110 var ps
= pattern
.split_with
("::")
112 if ps
.length
== 1 and p
.first
.is_upper
then return true
113 if ps
.length
== 2 and p
.first
.is_upper
then return false
114 if p
.has_suffix
("*") then
115 p
= p
.substring
(0, p
.length
- 1)
116 if case
.name
.has_prefix
(p
) then return true
118 if case
.name
== p
then return true
124 # A test suite contains all the test cases for a `MModule`.
127 # `MModule` under test
130 # `ToolContext` to use to display messages.
131 var toolcontext
: ToolContext
133 # List of `TestCase` to be executed in this suite.
134 var test_classes
= new Array[TestClass]
136 # Tests to be executed before the whole test suite.
137 var before_all
= new Array[TestCase]
139 # Tests to be executed after the whole test suite.
140 var after_all
= new Array[TestCase]
142 # Display test suite status in std-out.
144 var test_cases
= new Array[TestCase]
145 for test_class
in test_classes
do
146 test_cases
.add_all test_class
.before_all
147 test_cases
.add_all test_class
.test_cases
148 test_cases
.add_all test_class
.after_all
150 test_cases
.add_all before_all
151 test_cases
.add_all after_all
152 toolcontext
.show_unit_status
("Test-suite of module " + mmodule
.full_name
, test_cases
)
155 # Execute the test suite
159 if not toolcontext
.test_dir
.file_exists
then
160 toolcontext
.test_dir
.mkdir
164 if failure
!= null then
165 for test_class
in test_classes
do
166 for case
in test_class
.test_cases
do
167 case
.fail
"Compilation Error"
168 case
.raw_output
= failure
169 toolcontext
.clear_progress_bar
170 toolcontext
.show_unit
(case
)
177 toolcontext
.info
("Execute test-suite {mmodule.name}", 1)
179 for before_module
in before_all
do
181 toolcontext
.clear_progress_bar
182 toolcontext
.show_unit
(before_module
)
183 if before_module
.error
!= null then
184 for test_class
in test_classes
do
185 for case
in test_class
.before_all
do
186 case
.fail
"Nitunit Error: before module test failed"
187 toolcontext
.clear_progress_bar
188 toolcontext
.show_unit
(case
)
190 for case
in test_class
.test_cases
do
191 case
.fail
"Nitunit Error: before module test failed"
192 toolcontext
.clear_progress_bar
193 toolcontext
.show_unit
(case
)
195 for case
in test_class
.after_all
do
196 case
.fail
"Nitunit Error: before module test failed"
197 toolcontext
.clear_progress_bar
198 toolcontext
.show_unit
(case
)
201 for after_module
in after_all
do
202 after_module
.fail
"Nitunit Error: before module test failed"
203 toolcontext
.clear_progress_bar
204 toolcontext
.show_unit
(after_module
)
212 for test_class
in test_classes
do
213 for case
in test_class
.before_all
do
215 toolcontext
.clear_progress_bar
216 toolcontext
.show_unit
(case
)
217 if case
.error
!= null then
218 for scase
in test_class
.test_cases
do
219 scase
.fail
"Nitunit Error: before class test failed"
220 toolcontext
.clear_progress_bar
221 toolcontext
.show_unit
(scase
)
223 for scase
in test_class
.after_all
do
224 scase
.fail
"Nitunit Error: before class test failed"
225 toolcontext
.clear_progress_bar
226 toolcontext
.show_unit
(scase
)
233 for case
in test_class
.test_cases
do
235 toolcontext
.clear_progress_bar
236 toolcontext
.show_unit
(case
)
239 for after_class
in test_class
.after_all
do
241 toolcontext
.clear_progress_bar
242 toolcontext
.show_unit
(after_class
)
247 for after_module
in after_all
do
249 toolcontext
.clear_progress_bar
250 toolcontext
.show_unit
(after_module
)
258 # Write the test unit for `self` in a nit compilable file.
260 var file
= new Template
261 file
.addn
"intrude import core"
262 file
.addn
"import {mmodule.name}\n"
263 file
.addn
"var name = args.first"
264 for before_module
in before_all
do
265 before_module
.write_to_nit
(file
)
267 for test_class
in test_classes
do
268 for case
in test_class
.before_all
do
269 case
.write_to_nit
(file
)
271 for case
in test_class
.test_cases
do
272 case
.write_to_nit
(file
)
274 for case
in test_class
.after_all
do
275 case
.write_to_nit
(file
)
278 for after_module
in after_all
do
279 after_module
.write_to_nit
(file
)
281 file
.write_to_file
("{test_file}.nit")
284 # Return the test suite in XML format compatible with Jenkins.
285 # Contents depends on the `run` execution.
286 fun to_xml
: HTMLTag do
287 var n
= new HTMLTag("testsuite")
288 n
.attr
("package", mmodule
.name
)
289 for test_class
in test_classes
do
290 for test
in test_class
.test_cases
do n
.add test
.to_xml
295 # Generated test file name.
296 fun test_file
: String do
297 return toolcontext
.test_dir
/ "gen_{mmodule.name.escape_to_c}"
300 # Compile all `test_cases` cases in one file.
303 var nitc
= toolcontext
.find_nitc
306 var module_file
= mmodule
.location
.file
307 if module_file
== null then
308 toolcontext
.error
(null, "Error: cannot find module file for {mmodule.name}.")
309 toolcontext
.check_errors
312 var include_dir
= module_file
.filename
.dirname
313 var cmd
= "{nitc} --no-color -q '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
314 var res
= toolcontext
.safe_exec
(cmd
)
315 var f
= new FileReader.open
("{file}.out")
323 # Set environment variables for test suite execution
325 var loc
= mmodule
.location
.file
326 if loc
== null then return
327 toolcontext
.set_testing_path
(loc
.filename
)
330 # Error occured during test-suite compilation.
331 var failure
: nullable String = null
334 # A test class contains multiple test cases
336 # For each test class, methods can be executed before and after all cases.
338 # List of `TestCase` to be executed in this suite.
339 var test_cases
= new Array[TestCase]
341 # Tests to be executed before the whole test suite.
342 var before_all
= new Array[TestCase]
344 # Tests to be executed after the whole test suite.
345 var after_all
= new Array[TestCase]
348 # A test case is a unit test considering only a `MMethodDef`.
352 # Test suite wich `self` belongs to.
353 var test_suite
: TestSuite
355 # Test method to be compiled and tested.
356 var test_method
: MMethodDef
358 # Cases to execute before this one
359 var before
= new Array[MMethodDef]
361 # Cases to execute after this one
362 var after
= new Array[MMethodDef]
364 redef fun full_name
do return test_method
.full_name
366 redef fun location
do return test_method
.location
368 # `ToolContext` to use to display messages and find `nitc` bin.
369 var toolcontext
: ToolContext
371 # Generate the test unit for `self` in `file`.
372 fun write_to_nit
(file
: Template) do
373 var name
= test_method
.name
374 file
.addn
"if name == \"{name}\
" then"
375 if test_method
.mproperty
.is_toplevel
then
378 file
.addn
"\tvar subject = new {test_method.mclassdef.name}.intern"
379 for mmethod
in before
do
380 file
.addn
"\tsubject.{mmethod.name}"
382 file
.addn
"\tsubject.{name}"
383 for mmethod
in after
do
384 file
.addn
"\tsubject.{mmethod.name}"
390 # Execute the test case.
392 toolcontext
.info
("Execute test-case {test_method.name}", 1)
394 if toolcontext
.opt_noact
.value
then return
396 var method_name
= test_method
.name
397 var test_file
= test_suite
.test_file
398 var res_name
= "{test_file}_{method_name.escape_to_c}"
399 var clock
= new Clock
400 var res
= toolcontext
.safe_exec
("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
401 if not toolcontext
.opt_no_time
.value
then real_time
= clock
.total
403 var raw_output
= "{res_name}.out1".to_path
.read_all
404 self.raw_output
= raw_output
405 # set test case result
407 error
= "Runtime Error in file {test_file}.nit"
408 toolcontext
.modelbuilder
.failed_tests
+= 1
410 # no error, check with res file, if any.
411 var mmodule
= test_method
.mclassdef
.mmodule
412 var file
= mmodule
.filepath
414 var tries
= [ file
.dirname
/ mmodule
.name
+ ".sav" / test_method
.name
+ ".res",
415 file
.dirname
/ "sav" / test_method
.name
+ ".res" ,
416 file
.dirname
/ test_method
.name
+ ".res" ]
417 var savs
= [ for t
in tries
do if t
.file_exists
then t
]
418 if savs
.length
== 1 then
420 toolcontext
.info
("Diff output with {sav}", 1)
421 res
= toolcontext
.safe_exec
("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
424 else if toolcontext
.opt_autosav
.value
then
425 raw_output
.write_to_file
(sav
)
426 info
= "Expected output updated: {sav} (--autoupdate)"
428 self.raw_output
= "Diff\n" + "{res_name}.diff".to_path
.read_all
429 error
= "Difference with expected output: diff -u {sav} {res_name}.out1"
430 toolcontext
.modelbuilder
.failed_tests
+= 1
432 else if savs
.length
> 1 then
433 toolcontext
.info
("Conflicting diffs: {savs.join(", ")}", 1)
434 error
= "Conflicting expected output: {savs.join(", ", " and ")} all exist"
435 toolcontext
.modelbuilder
.failed_tests
+= 1
436 else if not raw_output
.is_empty
then
437 toolcontext
.info
("No diff: {tries.join(", ", " or ")} not found", 1)
438 if toolcontext
.opt_autosav
.value
then
439 var sav
= tries
.first
441 raw_output
.write_to_file
(sav
)
442 info
= "Expected output saved: {sav} (--autoupdate)"
450 # Make the test case fail without testing it
452 # Useful when the compilation or the before_test failed.
453 fun fail
(message
: String) do
456 toolcontext
.modelbuilder
.failed_tests
+= 1
459 redef fun xml_classname
do
460 var a
= test_method
.full_name
.split
("$")
461 return "nitunit.{a[0]}.{a[1]}"
464 redef fun xml_name
do
465 var a
= test_method
.full_name
.split
("$")
470 redef class MClassDef
471 # Methods tagged with `before` in this class definition
472 private fun before
: Array[MMethodDef] do
473 var res
= new ArraySet[MMethodDef]
474 for mpropdef
in mpropdefs
do
475 if mpropdef
isa MMethodDef and mpropdef
.is_before
then
479 var in_hierarchy
= self.in_hierarchy
480 if in_hierarchy
== null then return res
.to_a
481 for mclassdef
in in_hierarchy
.direct_greaters
do
482 res
.add_all mclassdef
.before
485 mmodule
.linearize_mpropdefs
(lin
)
489 # Methods tagged with `before_all` in this class definition
490 private fun before_all
: Array[MMethodDef] do
491 var res
= new ArraySet[MMethodDef]
492 for mpropdef
in mpropdefs
do
493 if mpropdef
isa MMethodDef and mpropdef
.is_before_all
then
497 var in_hierarchy
= self.in_hierarchy
498 if in_hierarchy
== null then return res
.to_a
499 for mclassdef
in in_hierarchy
.direct_greaters
do
500 res
.add_all mclassdef
.before_all
503 mmodule
.linearize_mpropdefs
(lin
)
507 # Methods tagged with `after` in this class definition
508 private fun after
: Array[MMethodDef] do
509 var res
= new ArraySet[MMethodDef]
510 for mpropdef
in mpropdefs
do
511 if mpropdef
isa MMethodDef and mpropdef
.is_after
then
515 var in_hierarchy
= self.in_hierarchy
516 if in_hierarchy
== null then return res
.to_a
517 for mclassdef
in in_hierarchy
.direct_greaters
do
518 res
.add_all mclassdef
.after
521 mmodule
.linearize_mpropdefs
(lin
)
525 # Methods tagged with `after_all` in this class definition
526 private fun after_all
: Array[MMethodDef] do
527 var res
= new ArraySet[MMethodDef]
528 for mpropdef
in mpropdefs
do
529 if mpropdef
isa MMethodDef and mpropdef
.is_after_all
then
533 var in_hierarchy
= self.in_hierarchy
534 if in_hierarchy
== null then return res
.to_a
535 for mclassdef
in in_hierarchy
.direct_greaters
do
536 res
.add_all mclassdef
.after_all
539 mmodule
.linearize_mpropdefs
(lin
)
545 # Methods tagged with `before_all` at the module level (in `Sys`)
546 private fun before_all
: Array[MMethodDef] do
547 var res
= new Array[MMethodDef]
548 for mmodule
in in_importation
.greaters
do
549 for mclassdef
in mmodule
.mclassdefs
do
550 if mclassdef
.name
!= "Sys" then continue
551 for mpropdef
in mclassdef
.mpropdefs
do
552 if not mpropdef
isa MMethodDef or not mpropdef
.is_before_all
then continue
558 linearize_mpropdefs
(lin
)
562 # Methods tagged with `after_all` at the module level (in `Sys`)
563 private fun after_all
: Array[MMethodDef] do
564 var res
= new Array[MMethodDef]
565 for mmodule
in in_importation
.greaters
do
566 for mclassdef
in mmodule
.mclassdefs
do
567 if mclassdef
.name
!= "Sys" then continue
568 for mpropdef
in mclassdef
.mpropdefs
do
569 if not mpropdef
isa MMethodDef or not mpropdef
.is_after_all
then continue
575 linearize_mpropdefs
(lin
)
580 redef class ModelBuilder
581 # Number of test classes generated.
582 var total_classes
= 0
584 # Number of tests generated.
587 # Number of failed tests.
590 # Run NitUnit test suite for `mmodule` (if it is one).
591 fun test_unit
(mmodule
: MModule): nullable HTMLTag do
592 # is the module a test_suite?
593 if not mmodule
.is_test
then return null
594 toolcontext
.info
("nitunit: test-suite {mmodule}", 2)
596 var tester
= new NitUnitTester(self)
597 var res
= tester
.test_module_unit
(mmodule
)