1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from external files.
20 private import annotation
22 redef class ToolContext
24 var opt_file
= new OptionString("Specify test suite location", "-t", "--target-file")
26 var opt_pattern
= new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
29 # Used to test nitunit test files.
32 # `ModelBuilder` used to parse test files.
33 var mbuilder
: ModelBuilder
35 # Parse a file and return the contained `MModule`.
36 private fun parse_module_unit
(file
: String): nullable MModule do
37 var mmodule
= mbuilder
.parse
([file
]).first
38 if mbuilder
.get_mmodule_annotation
("test_suite", mmodule
) == null then return null
43 # Compile and execute the test suite for a NitUnit `file`.
44 fun test_module_unit
(file
: String): nullable TestSuite do
45 var toolcontext
= mbuilder
.toolcontext
46 var mmodule
= parse_module_unit
(file
)
47 # is the module a test_suite?
48 if mmodule
== null then return null
49 var suite
= new TestSuite(mmodule
, toolcontext
)
50 # method to execute before all tests in the module
51 var before_module
= mmodule
.before_test
52 if before_module
!= null then
53 toolcontext
.modelbuilder
.total_tests
+= 1
54 suite
.before_module
= new TestCase(suite
, before_module
, toolcontext
)
56 # generate all test cases
57 for mclassdef
in mmodule
.mclassdefs
do
58 if not mclassdef
.is_test
then continue
59 if not suite_match_pattern
(mclassdef
) then continue
60 toolcontext
.modelbuilder
.total_classes
+= 1
61 for mpropdef
in mclassdef
.mpropdefs
do
62 if not mpropdef
isa MMethodDef or not mpropdef
.is_test
then continue
63 if not case_match_pattern
(mpropdef
) then continue
64 toolcontext
.modelbuilder
.total_tests
+= 1
65 var test
= new TestCase(suite
, mpropdef
, toolcontext
)
69 # method to execute after all tests in the module
70 var after_module
= mmodule
.after_test
71 if after_module
!= null then
72 toolcontext
.modelbuilder
.total_tests
+= 1
73 suite
.after_module
= new TestCase(suite
, after_module
, toolcontext
)
79 # Is the test suite name match the pattern option?
80 private fun suite_match_pattern
(suite
: MClassDef): Bool do
81 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
82 if pattern
== null then return true
83 var ps
= pattern
.split_with
("::")
85 if ps
.length
== 1 and p
.first
.is_lower
then return true
86 if ps
.length
== 2 and p
.first
.is_lower
then return false
87 if p
.has_suffix
("*") then
88 p
= p
.substring
(0, p
.length
- 1)
89 if suite
.name
.has_prefix
(p
) then return true
91 if suite
.name
== p
then return true
96 # Is the test case name match the pattern option?
97 private fun case_match_pattern
(case
: MPropDef): Bool do
98 var pattern
= mbuilder
.toolcontext
.opt_pattern
.value
99 if pattern
== null then return true
100 var ps
= pattern
.split_with
("::")
102 if ps
.length
== 1 and p
.first
.is_upper
then return true
103 if ps
.length
== 2 and p
.first
.is_upper
then return false
104 if p
.has_suffix
("*") then
105 p
= p
.substring
(0, p
.length
- 1)
106 if case
.name
.has_prefix
(p
) then return true
108 if case
.name
== p
then return true
114 # A test suite contains all the test cases for a `MModule`.
117 # `MModule` under test
120 # `ToolContext` to use to display messages.
121 var toolcontext
: ToolContext
123 # List of `TestCase` to be executed in this suite.
124 var test_cases
= new Array[TestCase]
126 # Add a `TestCase` to the suite.
127 fun add_test
(case
: TestCase) do test_cases
.add case
129 # Test to be executed before the whole test suite.
130 var before_module
: nullable TestCase = null
132 # Test to be executed after the whole test suite.
133 var after_module
: nullable TestCase = null
135 # Execute the test suite
137 if not toolcontext
.test_dir
.file_exists
then
138 toolcontext
.test_dir
.mkdir
142 toolcontext
.info
("Execute test-suite {mmodule.name}", 1)
143 var before_module
= self.before_module
144 if not before_module
== null then before_module
.run
145 for case
in test_cases
do case
.run
146 var after_module
= self.after_module
147 if not after_module
== null then after_module
.run
150 # Write the test unit for `self` in a nit compilable file.
152 var file
= new Template
153 file
.addn
"intrude import test_suite"
154 file
.addn
"import {mmodule.name}\n"
155 file
.addn
"var name = args.first"
156 for case
in test_cases
do
157 case
.write_to_nit
(file
)
159 file
.write_to_file
("{test_file}.nit")
162 # Return the test suite in XML format compatible with Jenkins.
163 # Contents depends on the `run` execution.
164 fun to_xml
: HTMLTag do
165 var n
= new HTMLTag("testsuite")
166 n
.attr
("package", mmodule
.name
)
167 var failure
= self.failure
168 if failure
!= null then
169 var f
= new HTMLTag("failure")
170 f
.attr
("message", failure
.to_s
)
173 for test
in test_cases
do n
.add test
.to_xml
178 # Generated test file name.
179 fun test_file
: String do
180 return toolcontext
.test_dir
/ "gen_{mmodule.name.escape_to_c}"
183 # Compile all `test_cases` cases in one file.
186 var nitc
= toolcontext
.find_nitc
189 var module_file
= mmodule
.location
.file
190 if module_file
== null then
191 toolcontext
.error
(null, "Error: cannot find module file for {mmodule.name}.")
192 toolcontext
.check_errors
195 var include_dir
= module_file
.filename
.dirname
196 var cmd
= "{nitc} --no-color '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
197 var res
= toolcontext
.safe_exec
(cmd
)
198 var f
= new FileReader.open
("{file}.out")
201 # set test case result
202 var loc
= mmodule
.location
205 toolcontext
.warning
(loc
, "failure", "FAILURE: {mmodule.name} (in file {file}.nit): {msg}")
206 toolcontext
.modelbuilder
.failed_tests
+= 1
208 toolcontext
.check_errors
211 # Error occured during test-suite compilation.
212 var failure
: nullable String = null
215 # A test case is a unit test considering only a `MMethodDef`.
218 # Test suite wich `self` belongs to.
219 var test_suite
: TestSuite
221 # Test method to be compiled and tested.
222 var test_method
: MMethodDef
224 # `ToolContext` to use to display messages and find `nitc` bin.
225 var toolcontext
: ToolContext
227 # Generate the test unit for `self` in `file`.
228 fun write_to_nit
(file
: Template) do
229 var name
= test_method
.name
230 file
.addn
"if name == \"{name}\
" then"
231 if test_method
.mproperty
.is_toplevel
then
234 file
.addn
"\tvar subject = new {test_method.mclassdef.name}.nitunit"
235 file
.addn
"\tsubject.before_test"
236 file
.addn
"\tsubject.{name}"
237 file
.addn
"\tsubject.after_test"
242 # Execute the test case.
244 toolcontext
.info
("Execute test-case {test_method.name}", 1)
246 if toolcontext
.opt_noact
.value
then return
248 var method_name
= test_method
.name
249 var test_file
= test_suite
.test_file
250 var res_name
= "{test_file}_{method_name.escape_to_c}"
251 var res
= toolcontext
.safe_exec
("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
252 var f
= new FileReader.open
("{res_name}.out1")
255 # set test case result
256 var loc
= test_method
.location
259 toolcontext
.warning
(loc
, "failure",
260 "ERROR: {method_name} (in file {test_file}.nit): {msg}")
261 toolcontext
.modelbuilder
.failed_tests
+= 1
263 var mmodule
= test_method
.mclassdef
.mmodule
264 var file
= mmodule
.filepath
266 var sav
= file
.dirname
/ mmodule
.name
+ ".sav" / test_method
.name
+ ".res"
267 if sav
.file_exists
then
268 toolcontext
.info
("Diff output with {sav}", 1)
269 res
= toolcontext
.safe_exec
("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
271 msg
= "Diff\n" + "{res_name}.diff".to_path
.read_all
273 toolcontext
.warning
(loc
, "failure",
274 "ERROR: {method_name} (in file {test_file}.nit): {msg}")
275 toolcontext
.modelbuilder
.failed_tests
+= 1
278 toolcontext
.info
("No diff: {sav} not found", 2)
282 toolcontext
.check_errors
285 # Error occured during test-case execution.
286 var error
: nullable String = null
288 # Was the test case executed at least one?
291 # Return the `TestCase` in XML format compatible with Jenkins.
292 fun to_xml
: HTMLTag do
293 var mclassdef
= test_method
.mclassdef
294 var tc
= new HTMLTag("testcase")
295 # NOTE: jenkins expects a '.' in the classname attr
296 tc
.attr
("classname", "nitunit." + mclassdef
.mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
)
297 tc
.attr
("name", test_method
.mproperty
.full_name
)
299 tc
.add
new HTMLTag("system-out")
300 var n
= new HTMLTag("system-err")
302 var error
= self.error
303 if error
!= null then
304 n
.append error
.trunc
(8192).filter_nonprintable
305 n
= new HTMLTag("error")
306 n
.attr
("message", "Runtime Error")
314 redef class MMethodDef
315 # TODO use annotations?
317 # Is the method a test_method?
318 # i.e. begins with "test_"
319 private fun is_test
: Bool do return name
.has_prefix
("test_")
321 # Is the method a "before_module"?
322 private fun is_before_module
: Bool do return mproperty
.is_toplevel
and name
== "before_module"
324 # Is the method a "after_module"?
325 private fun is_after_module
: Bool do return mproperty
.is_toplevel
and name
== "after_module"
328 redef class MClassDef
329 # Is the class a TestClass?
330 # i.e. begins with "Test"
331 private fun is_test
: Bool do
332 var in_hierarchy
= self.in_hierarchy
333 if in_hierarchy
== null then return false
334 for sup
in in_hierarchy
.greaters
do
335 if sup
.name
== "TestSuite" then return true
342 # "before_module" method for this module.
343 private fun before_test
: nullable MMethodDef do
344 for mclassdef
in mclassdefs
do
345 if not mclassdef
.name
== "Object" then continue
346 for mpropdef
in mclassdef
.mpropdefs
do
347 if mpropdef
isa MMethodDef and mpropdef
.is_before_module
then return mpropdef
353 # "after_module" method for this module.
354 private fun after_test
: nullable MMethodDef do
355 for mclassdef
in mclassdefs
do
356 if not mclassdef
.name
== "Object" then continue
357 for mpropdef
in mclassdef
.mpropdefs
do
358 if mpropdef
isa MMethodDef and mpropdef
.is_after_module
then return mpropdef
365 redef class ModelBuilder
366 # Number of test classes generated.
367 var total_classes
= 0
369 # Number of tests generated.
372 # Number of failed tests.
375 # Run NitUnit test file for mmodule (if exists).
376 fun test_unit
(mmodule
: MModule): HTMLTag do
377 var ts
= new HTMLTag("testsuite")
378 toolcontext
.info
("nitunit: test-suite test_{mmodule}", 2)
379 var f
= toolcontext
.opt_file
.value
380 var test_file
= "test_{mmodule.name}.nit"
383 else if not test_file
.file_exists
then
384 var module_file
= mmodule
.location
.file
385 if module_file
== null then
386 toolcontext
.info
("Skip test for {mmodule}, no file found", 2)
389 var include_dir
= module_file
.filename
.dirname
390 test_file
= "{include_dir}/{test_file}"
392 if not test_file
.file_exists
then
393 toolcontext
.info
("Skip test for {mmodule}, no file {test_file} found", 2)
396 var tester
= new NitUnitTester(self)
397 var res
= tester
.test_module_unit
(test_file
)
399 toolcontext
.info
("Skip test for {mmodule}, no test suite found", 2)