src/testing/testing_suite.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from external files.
  16 module testing_suite
  17
  18 import testing_base
  19 import html
  20
  21 redef class ToolContext
  22         # -- target-file
  23         var opt_file = new OptionString("Specify test suite location.", "-t", "--target-file")
  24         # --pattern
  25         var opt_pattern = new OptionString("Only run test case with name that match pattern. Examples: 'TestFoo', 'TestFoo*', 'TestFoo::test_foo', 'TestFoo::test_foo*', 'test_foo', 'test_foo*'", "-p", "--pattern")
  26 end
  27
  28 # Used to test nitunit test files.
  29 class NitUnitTester
  30
  31         # `ModelBuilder` used to parse test files.
  32         var mbuilder: ModelBuilder
  33
  34         # Parse a file and return the contained `MModule`.
  35         private fun parse_module_unit(file: String): nullable MModule do
  36                 var mmodule = mbuilder.parse([file]).first
  37                 if mbuilder.get_mmodule_annotation("test_suite", mmodule) == null then return null
  38                 mbuilder.run_phases
  39                 return mmodule
  40         end
  41
  42         # Compile and execute the test suite for a NitUnit `file`.
  43         fun test_module_unit(file: String): nullable TestSuite do
  44                 var toolcontext = mbuilder.toolcontext
  45                 var mmodule = parse_module_unit(file)
  46                 # is the module a test_suite?
  47                 if mmodule == null then return null
  48                 var suite = new TestSuite(mmodule, toolcontext)
  49                 # method to execute before all tests in the module
  50                 var before_module = mmodule.before_test
  51                 if before_module != null then
  52                         toolcontext.modelbuilder.total_tests += 1
  53                         suite.before_module = new TestCase(suite, before_module, toolcontext)
  54                 end
  55                 # generate all test cases
  56                 for mclassdef in mmodule.mclassdefs do
  57                         if not mclassdef.is_test then continue
  58                         if not suite_match_pattern(mclassdef) then continue
  59                         toolcontext.modelbuilder.total_classes += 1
  60                         var before_test = mclassdef.before_test
  61                         var after_test = mclassdef.after_test
  62                         for mpropdef in mclassdef.mpropdefs do
  63                                 if not mpropdef isa MMethodDef or not mpropdef.is_test then continue
  64                                 if not case_match_pattern(mpropdef) then continue
  65                                 toolcontext.modelbuilder.total_tests += 1
  66                                 var test = new TestCase(suite, mpropdef, toolcontext)
  67                                 test.before_test = before_test
  68                                 test.after_test = after_test
  69                                 suite.add_test test
  70                         end
  71                 end
  72                 # method to execute after all tests in the module
  73                 var after_module = mmodule.after_test
  74                 if after_module != null then
  75                         toolcontext.modelbuilder.total_tests += 1
  76                         suite.after_module = new TestCase(suite, after_module, toolcontext)
  77                 end
  78                 suite.run
  79                 return suite
  80         end
  81
  82         # Is the test suite name match the pattern option?
  83         private fun suite_match_pattern(suite: MClassDef): Bool do
  84                 var pattern = mbuilder.toolcontext.opt_pattern.value
  85                 if pattern == null then return true
  86                 var ps = pattern.split_with("::")
  87                 var p = ps.first
  88                 if ps.length == 1 and p.first.is_lower then return true
  89                 if ps.length == 2 and p.first.is_lower then return false
  90                 if p.has_suffix("*") then
  91                         p = p.substring(0, p.length - 1)
  92                         if suite.name.has_prefix(p) then return true
  93                 else
  94                         if suite.name == p then return true
  95                 end
  96                 return false
  97         end
  98
  99         # Is the test case name match the pattern option?
 100         private fun case_match_pattern(case: MPropDef): Bool do
 101                 var pattern = mbuilder.toolcontext.opt_pattern.value
 102                 if pattern == null then return true
 103                 var ps = pattern.split_with("::")
 104                 var p = ps.last
 105                 if ps.length == 1 and p.first.is_upper then return true
 106                 if ps.length == 2 and p.first.is_upper then return false
 107                 if p.has_suffix("*") then
 108                         p = p.substring(0, p.length - 1)
 109                         if case.name.has_prefix(p) then return true
 110                 else
 111                         if case.name == p then return true
 112                 end
 113                 return false
 114         end
 115 end
 116
 117 # A test suite contains all the test cases for a `MModule`.
 118 class TestSuite
 119
 120         # `MModule` under test
 121         var mmodule: MModule
 122
 123         # `ToolContext` to use to display messages.
 124         var toolcontext: ToolContext
 125
 126         # List of `TestCase` to be executed in this suite.
 127         var test_cases = new Array[TestCase]
 128
 129         # Add a `TestCase` to the suite.
 130         fun add_test(case: TestCase) do test_cases.add case
 131
 132         # Test to be executed before the whole test suite.
 133         var before_module: nullable TestCase = null
 134
 135         # Test to be executed after the whole test suite.
 136         var after_module: nullable TestCase = null
 137
 138         # Execute the test suite
 139         fun run do
 140                 if not toolcontext.test_dir.file_exists then
 141                         toolcontext.test_dir.mkdir
 142                 end
 143                 toolcontext.info("Execute test-suite {mmodule.name}", 1)
 144                 var before_module = self.before_module
 145                 if not before_module == null then run_case(before_module)
 146                 for case in test_cases do run_case(case)
 147                 var after_module = self.after_module
 148                 if not after_module == null then run_case(after_module)
 149         end
 150
 151         # Execute a test case
 152         fun run_case(test_case: TestCase) do
 153                 test_case.write_to_nit
 154                 test_case.compile
 155                 test_case.run
 156         end
 157
 158         # Return the test suite in XML format compatible with Jenkins.
 159         # Contents depends on the `run` execution.
 160         fun to_xml: HTMLTag do
 161                 var n = new HTMLTag("testsuite")
 162                 n.attr("package", mmodule.name)
 163                 for test in test_cases do n.add test.to_xml
 164                 return n
 165         end
 166 end
 167
 168 # A test case is a unit test considering only a `MMethodDef`.
 169 class TestCase
 170
 171         # Test suite wich `self` belongs to.
 172         var test_suite: TestSuite
 173
 174         # Test method to be compiled and tested.
 175         var test_method: MMethodDef
 176
 177         # `ToolContext` to use to display messages and find `nitg` bin.
 178         var toolcontext: ToolContext
 179
 180         # `MMethodDef` to call before the test case.
 181         var before_test: nullable MMethodDef = null
 182
 183         # `MMethodDef` to call after the test case.
 184         var after_test: nullable MMethodDef = null
 185
 186         # Generated test file name.
 187         fun test_file: String do
 188                 var dir = toolcontext.test_dir
 189                 var mod = test_method.mclassdef.mmodule.name
 190                 var cls = test_method.mclassdef.name
 191                 var name = test_method.name
 192                 return "{dir}/{mod}_{cls}_{name}"
 193         end
 194
 195         # Generate the test unit in a nit file.
 196         fun write_to_nit do
 197                 var name = test_method.name
 198                 var file = new Template
 199                 file.addn "intrude import test_suite"
 200                 file.addn "import {test_method.mclassdef.mmodule.name}\n"
 201                 if test_method.mproperty.is_toplevel then
 202                         file.addn name
 203                 else
 204                         file.addn "var subject = new {test_method.mclassdef.name}.nitunit"
 205                         if before_test != null then file.addn "subject.{before_test.name}"
 206                         file.addn "subject.{name}"
 207                         if after_test != null then file.addn "subject.{after_test.name}"
 208                 end
 209                 file.write_to_file("{test_file}.nit")
 210         end
 211
 212         # Compile all test cases in once.
 213         fun compile do
 214                 # find nitg
 215                 var nit_dir = toolcontext.nit_dir
 216                 var nitg = "{nit_dir or else ""}/bin/nitg"
 217                 if nit_dir == null or not nitg.file_exists then
 218                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 219                         toolcontext.check_errors
 220                 end
 221                 # compile test suite
 222                 var file = test_file
 223                 var include_dir = test_method.mclassdef.mmodule.location.file.filename.dirname
 224                 var cmd = "{nitg} --no-color '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
 225                 var res = sys.system(cmd)
 226                 var f = new IFStream.open("{file}.out")
 227                 var msg = f.read_all
 228                 f.close
 229                 # set test case result
 230                 var loc = test_method.location
 231                 if res != 0 then
 232                         failure = msg
 233                         toolcontext.warning(loc, "FAILURE: {test_method.name} (in file {file}.nit): {msg}")
 234                         toolcontext.modelbuilder.failed_tests += 1
 235                 end
 236                 toolcontext.check_errors
 237         end
 238
 239         # Execute the test case.
 240         fun run do
 241                 toolcontext.info("Execute test-case {test_method.name}", 1)
 242                 was_exec = true
 243                 if toolcontext.opt_noact.value then return
 244                 # execute
 245                 var file = test_file
 246                 var res = sys.system("./{file}.bin > '{file}.out1' 2>&1 </dev/null")
 247                 var f = new IFStream.open("{file}.out1")
 248                 var msg = f.read_all
 249                 f.close
 250                 # set test case result
 251                 var loc = test_method.location
 252                 if res != 0 then
 253                         error = msg
 254                         toolcontext.warning(loc, "ERROR: {test_method.name} (in file {file}.nit): {msg}")
 255                         toolcontext.modelbuilder.failed_tests += 1
 256                 end
 257                 toolcontext.check_errors
 258         end
 259
 260         # Error occured during execution.
 261         var error: nullable String = null
 262
 263         # Error occured during compilation.
 264         var failure: nullable String = null
 265
 266         # Was the test case executed at least one?
 267         var was_exec = false
 268
 269         # Return the `TestCase` in XML format compatible with Jenkins.
 270         fun to_xml: HTMLTag do
 271                 var mclassdef = test_method.mclassdef
 272                 var tc = new HTMLTag("testcase")
 273                 # NOTE: jenkins expects a '.' in the classname attr
 274                 tc.attr("classname", mclassdef.mmodule.full_name + "." + mclassdef.mclass.full_name)
 275                 tc.attr("name", test_method.mproperty.full_name)
 276                 if was_exec then
 277                         tc.add  new HTMLTag("system-err")
 278                         var n = new HTMLTag("system-out")
 279                         n.append "out"
 280                         tc.add n
 281                         if error != null then
 282                                 n = new HTMLTag("error")
 283                                 n.attr("message", error.to_s)
 284                                 tc.add n
 285                         end
 286                         if failure != null then
 287                                 n = new HTMLTag("failure")
 288                                 n.attr("message", failure.to_s)
 289                                 tc.add n
 290                         end
 291                 end
 292                 return tc
 293         end
 294 end
 295
 296 redef class MMethodDef
 297         # TODO use annotations?
 298
 299         # Is the method a test_method?
 300         # i.e. begins with "test_"
 301         private fun is_test: Bool do return name.has_prefix("test_")
 302
 303         # Is the method a "before_test"?
 304         private fun is_before: Bool do return name == "before_test"
 305
 306         # Is the method a "after_test"?
 307         private fun is_after: Bool do return name == "after_test"
 308
 309         # Is the method a "before_module"?
 310         private fun is_before_module: Bool do return mproperty.is_toplevel and name == "before_module"
 311
 312         # Is the method a "after_module"?
 313         private fun is_after_module: Bool do return mproperty.is_toplevel and name == "after_module"
 314 end
 315
 316 redef class MClassDef
 317         # Is the class a TestClass?
 318         # i.e. begins with "Test"
 319         private fun is_test: Bool do
 320                 for sup in in_hierarchy.greaters do
 321                         if sup.name == "TestSuite" then return true
 322                 end
 323                 return false
 324         end
 325
 326         # "before_test" method for this classdef.
 327         private fun before_test: nullable MMethodDef do
 328                 for mpropdef in mpropdefs do
 329                         if mpropdef isa MMethodDef and mpropdef.is_before then return mpropdef
 330                 end
 331                 return null
 332         end
 333
 334         # "after_test" method for this classdef.
 335         private fun after_test: nullable MMethodDef do
 336                 for mpropdef in mpropdefs do
 337                         if mpropdef isa MMethodDef and mpropdef.is_after then return mpropdef
 338                 end
 339                 return null
 340         end
 341 end
 342
 343 redef class MModule
 344         # "before_module" method for this module.
 345         private fun before_test: nullable MMethodDef do
 346                 for mclassdef in mclassdefs do
 347                         if not mclassdef.name == "Object" then continue
 348                         for mpropdef in mclassdef.mpropdefs do
 349                                 if mpropdef isa MMethodDef and mpropdef.is_before_module then return mpropdef
 350                         end
 351                 end
 352                 return null
 353         end
 354
 355         # "after_module" method for this module.
 356         private fun after_test: nullable MMethodDef do
 357                 for mclassdef in mclassdefs do
 358                         if not mclassdef.name == "Object" then continue
 359                         for mpropdef in mclassdef.mpropdefs do
 360                                 if mpropdef isa MMethodDef and mpropdef.is_after_module then return mpropdef
 361                         end
 362                 end
 363                 return null
 364         end
 365 end
 366
 367 redef class ModelBuilder
 368         var total_classes = 0
 369         var total_tests = 0
 370         var failed_tests = 0
 371
 372         # Run NitUnit test file for mmodule (if exists).
 373         fun test_unit(mmodule: MModule): HTMLTag do
 374                 var ts = new HTMLTag("testsuite")
 375                 toolcontext.info("nitunit: test-suite test_{mmodule}", 2)
 376                 var f = toolcontext.opt_file.value
 377                 var test_file = "test_{mmodule.name}.nit"
 378                 if f != null then
 379                         test_file = f
 380                 else if not test_file.file_exists then
 381                         var include_dir = mmodule.location.file.filename.dirname
 382                         test_file = "{include_dir}/{test_file}"
 383                 end
 384                 if not test_file.file_exists then
 385                         toolcontext.info("Skip test for {mmodule}, no file {test_file} found", 1)
 386                         return ts
 387                 end
 388                 var tester = new NitUnitTester(self)
 389                 var res = tester.test_module_unit(test_file)
 390                 if res == null then
 391                         toolcontext.info("Skip test for {mmodule}, no test suite found", 1)
 392                         return ts
 393                 end
 394                 return res.to_xml
 395         end
 396 end