src/testing/testing_suite.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from external files.
  16 module testing_suite
  17
  18 import testing_base
  19 import html
  20 private import annotation
  21
  22 redef class ToolContext
  23         # -- target-file
  24         var opt_file = new OptionString("Specify test suite location", "-t", "--target-file")
  25         # --pattern
  26         var opt_pattern = new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
  27 end
  28
  29 # Used to test nitunit test files.
  30 class NitUnitTester
  31
  32         # `ModelBuilder` used to parse test files.
  33         var mbuilder: ModelBuilder
  34
  35         # Parse a file and return the contained `MModule`.
  36         private fun parse_module_unit(file: String): nullable MModule do
  37                 var mmodule = mbuilder.parse([file]).first
  38                 if mbuilder.get_mmodule_annotation("test_suite", mmodule) == null then return null
  39                 mbuilder.run_phases
  40                 return mmodule
  41         end
  42
  43         # Compile and execute the test suite for a NitUnit `file`.
  44         fun test_module_unit(file: String): nullable TestSuite do
  45                 var toolcontext = mbuilder.toolcontext
  46                 var mmodule = parse_module_unit(file)
  47                 # is the module a test_suite?
  48                 if mmodule == null then return null
  49                 var suite = new TestSuite(mmodule, toolcontext)
  50                 # method to execute before all tests in the module
  51                 var before_module = mmodule.before_test
  52                 if before_module != null then
  53                         toolcontext.modelbuilder.total_tests += 1
  54                         suite.before_module = new TestCase(suite, before_module, toolcontext)
  55                 end
  56                 # generate all test cases
  57                 for mclassdef in mmodule.mclassdefs do
  58                         if not mclassdef.is_test then continue
  59                         if not suite_match_pattern(mclassdef) then continue
  60                         toolcontext.modelbuilder.total_classes += 1
  61                         var before_test = mclassdef.before_test
  62                         var after_test = mclassdef.after_test
  63                         for mpropdef in mclassdef.mpropdefs do
  64                                 if not mpropdef isa MMethodDef or not mpropdef.is_test then continue
  65                                 if not case_match_pattern(mpropdef) then continue
  66                                 toolcontext.modelbuilder.total_tests += 1
  67                                 var test = new TestCase(suite, mpropdef, toolcontext)
  68                                 test.before_test = before_test
  69                                 test.after_test = after_test
  70                                 suite.add_test test
  71                         end
  72                 end
  73                 # method to execute after all tests in the module
  74                 var after_module = mmodule.after_test
  75                 if after_module != null then
  76                         toolcontext.modelbuilder.total_tests += 1
  77                         suite.after_module = new TestCase(suite, after_module, toolcontext)
  78                 end
  79                 suite.run
  80                 return suite
  81         end
  82
  83         # Is the test suite name match the pattern option?
  84         private fun suite_match_pattern(suite: MClassDef): Bool do
  85                 var pattern = mbuilder.toolcontext.opt_pattern.value
  86                 if pattern == null then return true
  87                 var ps = pattern.split_with("::")
  88                 var p = ps.first
  89                 if ps.length == 1 and p.first.is_lower then return true
  90                 if ps.length == 2 and p.first.is_lower then return false
  91                 if p.has_suffix("*") then
  92                         p = p.substring(0, p.length - 1)
  93                         if suite.name.has_prefix(p) then return true
  94                 else
  95                         if suite.name == p then return true
  96                 end
  97                 return false
  98         end
  99
 100         # Is the test case name match the pattern option?
 101         private fun case_match_pattern(case: MPropDef): Bool do
 102                 var pattern = mbuilder.toolcontext.opt_pattern.value
 103                 if pattern == null then return true
 104                 var ps = pattern.split_with("::")
 105                 var p = ps.last
 106                 if ps.length == 1 and p.first.is_upper then return true
 107                 if ps.length == 2 and p.first.is_upper then return false
 108                 if p.has_suffix("*") then
 109                         p = p.substring(0, p.length - 1)
 110                         if case.name.has_prefix(p) then return true
 111                 else
 112                         if case.name == p then return true
 113                 end
 114                 return false
 115         end
 116 end
 117
 118 # A test suite contains all the test cases for a `MModule`.
 119 class TestSuite
 120
 121         # `MModule` under test
 122         var mmodule: MModule
 123
 124         # `ToolContext` to use to display messages.
 125         var toolcontext: ToolContext
 126
 127         # List of `TestCase` to be executed in this suite.
 128         var test_cases = new Array[TestCase]
 129
 130         # Add a `TestCase` to the suite.
 131         fun add_test(case: TestCase) do test_cases.add case
 132
 133         # Test to be executed before the whole test suite.
 134         var before_module: nullable TestCase = null
 135
 136         # Test to be executed after the whole test suite.
 137         var after_module: nullable TestCase = null
 138
 139         # Execute the test suite
 140         fun run do
 141                 if not toolcontext.test_dir.file_exists then
 142                         toolcontext.test_dir.mkdir
 143                 end
 144                 write_to_nit
 145                 compile
 146                 toolcontext.info("Execute test-suite {mmodule.name}", 1)
 147                 var before_module = self.before_module
 148                 if not before_module == null then before_module.run
 149                 for case in test_cases do case.run
 150                 var after_module = self.after_module
 151                 if not after_module == null then after_module.run
 152         end
 153
 154         # Write the test unit for `self` in a nit compilable file.
 155         fun write_to_nit do
 156                 var file = new Template
 157                 file.addn "intrude import test_suite"
 158                 file.addn "import {mmodule.name}\n"
 159                 file.addn "var name = args.first"
 160                 for case in test_cases do
 161                         case.write_to_nit(file)
 162                 end
 163                 file.write_to_file("{test_file}.nit")
 164         end
 165
 166         # Return the test suite in XML format compatible with Jenkins.
 167         # Contents depends on the `run` execution.
 168         fun to_xml: HTMLTag do
 169                 var n = new HTMLTag("testsuite")
 170                 n.attr("package", mmodule.name)
 171                 var failure = self.failure
 172                 if failure != null then
 173                         var f = new HTMLTag("failure")
 174                         f.attr("message", failure.to_s)
 175                         n.add f
 176                 else
 177                         for test in test_cases do n.add test.to_xml
 178                 end
 179                 return n
 180         end
 181
 182         # Generated test file name.
 183         fun test_file: String do
 184                 return toolcontext.test_dir / "gen_{mmodule.name.escape_to_c}"
 185         end
 186
 187         # Compile all `test_cases` cases in one file.
 188         fun compile do
 189                 # find nitc
 190                 var nit_dir = toolcontext.nit_dir
 191                 var nitc = nit_dir/"bin/nitc"
 192                 if not nitc.file_exists then
 193                         toolcontext.error(null, "Error: cannot find nitc. Set envvar NIT_DIR.")
 194                         toolcontext.check_errors
 195                 end
 196                 # compile test suite
 197                 var file = test_file
 198                 var module_file = mmodule.location.file
 199                 if module_file == null then
 200                         toolcontext.error(null, "Error: cannot find module file for {mmodule.name}.")
 201                         toolcontext.check_errors
 202                         return
 203                 end
 204                 var include_dir = module_file.filename.dirname
 205                 var cmd = "{nitc} --no-color '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
 206                 var res = sys.system(cmd)
 207                 var f = new FileReader.open("{file}.out")
 208                 var msg = f.read_all
 209                 f.close
 210                 # set test case result
 211                 var loc = mmodule.location
 212                 if res != 0 then
 213                         failure = msg
 214                         toolcontext.warning(loc, "failure", "FAILURE: {mmodule.name} (in file {file}.nit): {msg}")
 215                         toolcontext.modelbuilder.failed_tests += 1
 216                 end
 217                 toolcontext.check_errors
 218         end
 219
 220         # Error occured during test-suite compilation.
 221         var failure: nullable String = null
 222 end
 223
 224 # A test case is a unit test considering only a `MMethodDef`.
 225 class TestCase
 226
 227         # Test suite wich `self` belongs to.
 228         var test_suite: TestSuite
 229
 230         # Test method to be compiled and tested.
 231         var test_method: MMethodDef
 232
 233         # `ToolContext` to use to display messages and find `nitc` bin.
 234         var toolcontext: ToolContext
 235
 236         # `MMethodDef` to call before the test case.
 237         var before_test: nullable MMethodDef = null
 238
 239         # `MMethodDef` to call after the test case.
 240         var after_test: nullable MMethodDef = null
 241
 242         # Generate the test unit for `self` in `file`.
 243         fun write_to_nit(file: Template) do
 244                 var name = test_method.name
 245                 file.addn "if name == \"{name}\" then"
 246                 if test_method.mproperty.is_toplevel then
 247                         file.addn "\t{name}"
 248                 else
 249                         file.addn "\tvar subject = new {test_method.mclassdef.name}.nitunit"
 250                         if before_test != null then file.addn "\tsubject.{before_test.name}"
 251                         file.addn "\tsubject.{name}"
 252                         if after_test != null then file.addn "\tsubject.{after_test.name}"
 253                 end
 254                 file.addn "end"
 255         end
 256
 257         # Execute the test case.
 258         fun run do
 259                 toolcontext.info("Execute test-case {test_method.name}", 1)
 260                 was_exec = true
 261                 if toolcontext.opt_noact.value then return
 262                 # execute
 263                 var method_name = test_method.name
 264                 var test_file = test_suite.test_file
 265                 var res_name = "{test_file}_{method_name.escape_to_c}"
 266                 var res = sys.system("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
 267                 var f = new FileReader.open("{res_name}.out1")
 268                 var msg = f.read_all
 269                 f.close
 270                 # set test case result
 271                 var loc = test_method.location
 272                 if res != 0 then
 273                         error = msg
 274                         toolcontext.warning(loc, "failure",
 275                            "ERROR: {method_name} (in file {test_file}.nit): {msg}")
 276                         toolcontext.modelbuilder.failed_tests += 1
 277                 end
 278                 toolcontext.check_errors
 279         end
 280
 281         # Error occured during test-case execution.
 282         var error: nullable String = null
 283
 284         # Was the test case executed at least one?
 285         var was_exec = false
 286
 287         # Return the `TestCase` in XML format compatible with Jenkins.
 288         fun to_xml: HTMLTag do
 289                 var mclassdef = test_method.mclassdef
 290                 var tc = new HTMLTag("testcase")
 291                 # NOTE: jenkins expects a '.' in the classname attr
 292                 tc.attr("classname", "nitunit." + mclassdef.mmodule.full_name + "." + mclassdef.mclass.full_name)
 293                 tc.attr("name", test_method.mproperty.full_name)
 294                 if was_exec then
 295                         tc.add  new HTMLTag("system-err")
 296                         var n = new HTMLTag("system-out")
 297                         n.append "out"
 298                         tc.add n
 299                         var error = self.error
 300                         if error != null then
 301                                 n = new HTMLTag("error")
 302                                 n.attr("message", error.to_s)
 303                                 tc.add n
 304                         end
 305                 end
 306                 return tc
 307         end
 308 end
 309
 310 redef class MMethodDef
 311         # TODO use annotations?
 312
 313         # Is the method a test_method?
 314         # i.e. begins with "test_"
 315         private fun is_test: Bool do return name.has_prefix("test_")
 316
 317         # Is the method a "before_test"?
 318         private fun is_before: Bool do return name == "before_test"
 319
 320         # Is the method a "after_test"?
 321         private fun is_after: Bool do return name == "after_test"
 322
 323         # Is the method a "before_module"?
 324         private fun is_before_module: Bool do return mproperty.is_toplevel and name == "before_module"
 325
 326         # Is the method a "after_module"?
 327         private fun is_after_module: Bool do return mproperty.is_toplevel and name == "after_module"
 328 end
 329
 330 redef class MClassDef
 331         # Is the class a TestClass?
 332         # i.e. begins with "Test"
 333         private fun is_test: Bool do
 334                 var in_hierarchy = self.in_hierarchy
 335                 if in_hierarchy == null then return false
 336                 for sup in in_hierarchy.greaters do
 337                         if sup.name == "TestSuite" then return true
 338                 end
 339                 return false
 340         end
 341
 342         # "before_test" method for this classdef.
 343         private fun before_test: nullable MMethodDef do
 344                 for mpropdef in mpropdefs do
 345                         if mpropdef isa MMethodDef and mpropdef.is_before then return mpropdef
 346                 end
 347                 return null
 348         end
 349
 350         # "after_test" method for this classdef.
 351         private fun after_test: nullable MMethodDef do
 352                 for mpropdef in mpropdefs do
 353                         if mpropdef isa MMethodDef and mpropdef.is_after then return mpropdef
 354                 end
 355                 return null
 356         end
 357 end
 358
 359 redef class MModule
 360         # "before_module" method for this module.
 361         private fun before_test: nullable MMethodDef do
 362                 for mclassdef in mclassdefs do
 363                         if not mclassdef.name == "Object" then continue
 364                         for mpropdef in mclassdef.mpropdefs do
 365                                 if mpropdef isa MMethodDef and mpropdef.is_before_module then return mpropdef
 366                         end
 367                 end
 368                 return null
 369         end
 370
 371         # "after_module" method for this module.
 372         private fun after_test: nullable MMethodDef do
 373                 for mclassdef in mclassdefs do
 374                         if not mclassdef.name == "Object" then continue
 375                         for mpropdef in mclassdef.mpropdefs do
 376                                 if mpropdef isa MMethodDef and mpropdef.is_after_module then return mpropdef
 377                         end
 378                 end
 379                 return null
 380         end
 381 end
 382
 383 redef class ModelBuilder
 384         # Number of test classes generated.
 385         var total_classes = 0
 386
 387         # Number of tests generated.
 388         var total_tests = 0
 389
 390         # Number of failed tests.
 391         var failed_tests = 0
 392
 393         # Run NitUnit test file for mmodule (if exists).
 394         fun test_unit(mmodule: MModule): HTMLTag do
 395                 var ts = new HTMLTag("testsuite")
 396                 toolcontext.info("nitunit: test-suite test_{mmodule}", 2)
 397                 var f = toolcontext.opt_file.value
 398                 var test_file = "test_{mmodule.name}.nit"
 399                 if f != null then
 400                         test_file = f
 401                 else if not test_file.file_exists then
 402                         var module_file = mmodule.location.file
 403                         if module_file == null then
 404                                 toolcontext.info("Skip test for {mmodule}, no file found", 2)
 405                                 return ts
 406                         end
 407                         var include_dir = module_file.filename.dirname
 408                         test_file = "{include_dir}/{test_file}"
 409                 end
 410                 if not test_file.file_exists then
 411                         toolcontext.info("Skip test for {mmodule}, no file {test_file} found", 2)
 412                         return ts
 413                 end
 414                 var tester = new NitUnitTester(self)
 415                 var res = tester.test_module_unit(test_file)
 416                 if res == null then
 417                         toolcontext.info("Skip test for {mmodule}, no test suite found", 2)
 418                         return ts
 419                 end
 420                 return res.to_xml
 421         end
 422 end