src/testing/testing_suite.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from external files.
  16 module testing_suite
  17
  18 import testing_base
  19 import html
  20 private import parse_annotations
  21 private import annotation
  22 private import realtime
  23
  24 redef class ToolContext
  25         # --pattern
  26         var opt_pattern = new OptionString("Only run test case with name that match pattern", "-p", "--pattern")
  27         # --autosav
  28         var opt_autosav = new OptionBool("Automatically create/update .res files for black box testing", "--autosav")
  29 end
  30
  31 # Used to test nitunit test files.
  32 class NitUnitTester
  33
  34         # `ModelBuilder` used to parse test files.
  35         var mbuilder: ModelBuilder
  36
  37         # Compile and execute `mmodule` as a test suite.
  38         fun test_module_unit(mmodule: MModule): TestSuite do
  39                 var toolcontext = mbuilder.toolcontext
  40                 var suite = new TestSuite(mmodule, toolcontext)
  41                 # method to execute before all tests in the module
  42                 var before_module = mmodule.before_test
  43                 if before_module != null then
  44                         toolcontext.modelbuilder.total_tests += 1
  45                         suite.before_module = new TestCase(suite, before_module, toolcontext)
  46                 end
  47                 # generate all test cases
  48                 for mclassdef in mmodule.mclassdefs do
  49                         if not mclassdef.is_test then continue
  50                         if not suite_match_pattern(mclassdef) then continue
  51                         toolcontext.modelbuilder.total_classes += 1
  52                         for mpropdef in mclassdef.mpropdefs do
  53                                 if not mpropdef isa MMethodDef or not mpropdef.is_test then continue
  54                                 if not case_match_pattern(mpropdef) then continue
  55                                 toolcontext.modelbuilder.total_tests += 1
  56                                 var test = new TestCase(suite, mpropdef, toolcontext)
  57                                 suite.add_test test
  58                         end
  59                 end
  60                 # method to execute after all tests in the module
  61                 var after_module = mmodule.after_test
  62                 if after_module != null then
  63                         toolcontext.modelbuilder.total_tests += 1
  64                         suite.after_module = new TestCase(suite, after_module, toolcontext)
  65                 end
  66                 suite.run
  67                 return suite
  68         end
  69
  70         # Is the test suite name match the pattern option?
  71         private fun suite_match_pattern(suite: MClassDef): Bool do
  72                 var pattern = mbuilder.toolcontext.opt_pattern.value
  73                 if pattern == null then return true
  74                 var ps = pattern.split_with("::")
  75                 var p = ps.first
  76                 if ps.length == 1 and p.first.is_lower then return true
  77                 if ps.length == 2 and p.first.is_lower then return false
  78                 if p.has_suffix("*") then
  79                         p = p.substring(0, p.length - 1)
  80                         if suite.name.has_prefix(p) then return true
  81                 else
  82                         if suite.name == p then return true
  83                 end
  84                 return false
  85         end
  86
  87         # Is the test case name match the pattern option?
  88         private fun case_match_pattern(case: MPropDef): Bool do
  89                 var pattern = mbuilder.toolcontext.opt_pattern.value
  90                 if pattern == null then return true
  91                 var ps = pattern.split_with("::")
  92                 var p = ps.last
  93                 if ps.length == 1 and p.first.is_upper then return true
  94                 if ps.length == 2 and p.first.is_upper then return false
  95                 if p.has_suffix("*") then
  96                         p = p.substring(0, p.length - 1)
  97                         if case.name.has_prefix(p) then return true
  98                 else
  99                         if case.name == p then return true
 100                 end
 101                 return false
 102         end
 103 end
 104
 105 # A test suite contains all the test cases for a `MModule`.
 106 class TestSuite
 107
 108         # `MModule` under test
 109         var mmodule: MModule
 110
 111         # `ToolContext` to use to display messages.
 112         var toolcontext: ToolContext
 113
 114         # List of `TestCase` to be executed in this suite.
 115         var test_cases = new Array[TestCase]
 116
 117         # Add a `TestCase` to the suite.
 118         fun add_test(case: TestCase) do test_cases.add case
 119
 120         # Test to be executed before the whole test suite.
 121         var before_module: nullable TestCase = null
 122
 123         # Test to be executed after the whole test suite.
 124         var after_module: nullable TestCase = null
 125
 126         # Display test suite status in std-out.
 127         fun show_status do
 128                 var test_cases = self.test_cases.to_a
 129                 if before_module != null then test_cases.add before_module.as(not null)
 130                 if after_module != null then test_cases.add after_module.as(not null)
 131                 toolcontext.show_unit_status("Test-suite of module " + mmodule.full_name, test_cases)
 132         end
 133
 134         # Execute the test suite
 135         fun run do
 136                 set_env
 137                 show_status
 138                 if not toolcontext.test_dir.file_exists then
 139                         toolcontext.test_dir.mkdir
 140                 end
 141                 write_to_nit
 142                 compile
 143                 if failure != null then
 144                         for case in test_cases do
 145                                 case.fail "Compilation Error"
 146                                 case.raw_output = failure
 147                                 toolcontext.clear_progress_bar
 148                                 toolcontext.show_unit(case)
 149                         end
 150                         show_status
 151                         print ""
 152                         return
 153                 end
 154                 toolcontext.info("Execute test-suite {mmodule.name}", 1)
 155
 156                 var before_module = self.before_module
 157                 var after_module = self.after_module
 158
 159                 if before_module != null then
 160                         before_module.run
 161                         toolcontext.clear_progress_bar
 162                         toolcontext.show_unit(before_module)
 163                         if before_module.error != null then
 164                                 for case in test_cases do
 165                                         case.fail "Nitunit Error: before_module test failed"
 166                                         toolcontext.clear_progress_bar
 167                                         toolcontext.show_unit(case)
 168                                 end
 169                                 if after_module != null then
 170                                         after_module.fail "Nitunit Error: before_module test failed"
 171                                         toolcontext.clear_progress_bar
 172                                         toolcontext.show_unit(after_module)
 173                                 end
 174                                 show_status
 175                                 print ""
 176                                 return
 177                         end
 178                 end
 179
 180                 for case in test_cases do
 181                         case.run
 182                         toolcontext.clear_progress_bar
 183                         toolcontext.show_unit(case)
 184                         show_status
 185                 end
 186
 187                 if not after_module == null then
 188                         after_module.run
 189                         toolcontext.clear_progress_bar
 190                         toolcontext.show_unit(after_module)
 191                         show_status
 192                 end
 193
 194                 show_status
 195                 print ""
 196         end
 197
 198         # Write the test unit for `self` in a nit compilable file.
 199         fun write_to_nit do
 200                 var file = new Template
 201                 file.addn "intrude import core"
 202                 file.addn "import {mmodule.name}\n"
 203                 file.addn "var name = args.first"
 204                 var before_module = self.before_module
 205                 if before_module != null then
 206                         before_module.write_to_nit(file)
 207                 end
 208                 for case in test_cases do
 209                         case.write_to_nit(file)
 210                 end
 211                 var after_module = self.after_module
 212                 if after_module != null then
 213                         after_module.write_to_nit(file)
 214                 end
 215                 file.write_to_file("{test_file}.nit")
 216         end
 217
 218         # Return the test suite in XML format compatible with Jenkins.
 219         # Contents depends on the `run` execution.
 220         fun to_xml: HTMLTag do
 221                 var n = new HTMLTag("testsuite")
 222                 n.attr("package", mmodule.name)
 223                 for test in test_cases do n.add test.to_xml
 224                 return n
 225         end
 226
 227         # Generated test file name.
 228         fun test_file: String do
 229                 return toolcontext.test_dir / "gen_{mmodule.name.escape_to_c}"
 230         end
 231
 232         # Compile all `test_cases` cases in one file.
 233         fun compile do
 234                 # find nitc
 235                 var nitc = toolcontext.find_nitc
 236                 # compile test suite
 237                 var file = test_file
 238                 var module_file = mmodule.location.file
 239                 if module_file == null then
 240                         toolcontext.error(null, "Error: cannot find module file for {mmodule.name}.")
 241                         toolcontext.check_errors
 242                         return
 243                 end
 244                 var include_dir = module_file.filename.dirname
 245                 var cmd = "{nitc} --no-color -q '{file}.nit' -I {include_dir} -o '{file}.bin' > '{file}.out' 2>&1 </dev/null"
 246                 var res = toolcontext.safe_exec(cmd)
 247                 var f = new FileReader.open("{file}.out")
 248                 var msg = f.read_all
 249                 f.close
 250                 if res != 0 then
 251                         failure = msg
 252                 end
 253         end
 254
 255         # Set environment variables for test suite execution
 256         fun set_env do
 257                 var loc = mmodule.location.file
 258                 if loc == null then return
 259                 toolcontext.set_testing_path(loc.filename)
 260         end
 261
 262         # Error occured during test-suite compilation.
 263         var failure: nullable String = null
 264 end
 265
 266 # A test case is a unit test considering only a `MMethodDef`.
 267 class TestCase
 268         super UnitTest
 269
 270         # Test suite wich `self` belongs to.
 271         var test_suite: TestSuite
 272
 273         # Test method to be compiled and tested.
 274         var test_method: MMethodDef
 275
 276         redef fun full_name do return test_method.full_name
 277
 278         redef fun location do return test_method.location
 279
 280         # `ToolContext` to use to display messages and find `nitc` bin.
 281         var toolcontext: ToolContext
 282
 283         # Generate the test unit for `self` in `file`.
 284         fun write_to_nit(file: Template) do
 285                 var name = test_method.name
 286                 file.addn "if name == \"{name}\" then"
 287                 if test_method.mproperty.is_toplevel then
 288                         file.addn "\t{name}"
 289                 else
 290                         file.addn "\tvar subject = new {test_method.mclassdef.name}.nitunit"
 291                         file.addn "\tsubject.before_test"
 292                         file.addn "\tsubject.{name}"
 293                         file.addn "\tsubject.after_test"
 294                 end
 295                 file.addn "end"
 296         end
 297
 298         # Execute the test case.
 299         fun run do
 300                 toolcontext.info("Execute test-case {test_method.name}", 1)
 301                 was_exec = true
 302                 if toolcontext.opt_noact.value then return
 303                 # execute
 304                 var method_name = test_method.name
 305                 var test_file = test_suite.test_file
 306                 var res_name = "{test_file}_{method_name.escape_to_c}"
 307                 var clock = new Clock
 308                 var res = toolcontext.safe_exec("{test_file}.bin {method_name} > '{res_name}.out1' 2>&1 </dev/null")
 309                 if not toolcontext.opt_no_time.value then real_time = clock.total
 310
 311                 var raw_output = "{res_name}.out1".to_path.read_all
 312                 self.raw_output = raw_output
 313                 # set test case result
 314                 if res != 0 then
 315                         error = "Runtime Error in file {test_file}.nit"
 316                         toolcontext.modelbuilder.failed_tests += 1
 317                 else
 318                         # no error, check with res file, if any.
 319                         var mmodule = test_method.mclassdef.mmodule
 320                         var file = mmodule.filepath
 321                         if file != null then
 322                                 var tries = [ file.dirname / mmodule.name + ".sav" / test_method.name + ".res",
 323                                         file.dirname / "sav" / test_method.name + ".res" ,
 324                                         file.dirname / test_method.name + ".res" ]
 325                                 var savs = [ for t in tries do if t.file_exists then t ]
 326                                 if savs.length == 1 then
 327                                         var sav = savs.first
 328                                         toolcontext.info("Diff output with {sav}", 1)
 329                                         res = toolcontext.safe_exec("diff -u --label 'expected:{sav}' --label 'got:{res_name}.out1' '{sav}' '{res_name}.out1' > '{res_name}.diff' 2>&1 </dev/null")
 330                                         if res == 0 then
 331                                                 # OK
 332                                         else if toolcontext.opt_autosav.value then
 333                                                 raw_output.write_to_file(sav)
 334                                                 info = "Expected output updated: {sav} (--autoupdate)"
 335                                         else
 336                                                 self.raw_output = "Diff\n" + "{res_name}.diff".to_path.read_all
 337                                                 error = "Difference with expected output: diff -u {sav} {res_name}.out1"
 338                                                 toolcontext.modelbuilder.failed_tests += 1
 339                                         end
 340                                 else if savs.length > 1 then
 341                                         toolcontext.info("Conflicting diffs: {savs.join(", ")}", 1)
 342                                         error = "Conflicting expected output: {savs.join(", ", " and ")} all exist"
 343                                         toolcontext.modelbuilder.failed_tests += 1
 344                                 else if not raw_output.is_empty then
 345                                         toolcontext.info("No diff: {tries.join(", ", " or ")} not found", 1)
 346                                         if toolcontext.opt_autosav.value then
 347                                                 var sav = tries.first
 348                                                 sav.dirname.mkdir
 349                                                 raw_output.write_to_file(sav)
 350                                                 info = "Expected output saved: {sav} (--autoupdate)"
 351                                         end
 352                                 end
 353                         end
 354                 end
 355                 is_done = true
 356         end
 357
 358         # Make the test case fail without testing it
 359         #
 360         # Useful when the compilation or the before_test failed.
 361         fun fail(message: String) do
 362                 is_done = true
 363                 error = message
 364                 toolcontext.modelbuilder.failed_tests += 1
 365         end
 366
 367         redef fun xml_classname do
 368                 var a = test_method.full_name.split("$")
 369                 return "nitunit.{a[0]}.{a[1]}"
 370         end
 371
 372         redef fun xml_name do
 373                 var a = test_method.full_name.split("$")
 374                 return a[2]
 375         end
 376 end
 377
 378 redef class MMethodDef
 379         # TODO use annotations?
 380
 381         # Is the method a test_method?
 382         # i.e. begins with "test_"
 383         private fun is_test: Bool do return name.has_prefix("test_")
 384
 385         # Is the method a "before_module"?
 386         private fun is_before_module: Bool do return name == "before_module"
 387
 388         # Is the method a "after_module"?
 389         private fun is_after_module: Bool do return name == "after_module"
 390 end
 391
 392 redef class MModule
 393         # "before_module" method for this module.
 394         private fun before_test: nullable MMethodDef do
 395                 for mclassdef in mclassdefs do
 396                         if not mclassdef.name == "Sys" then continue
 397                         for mpropdef in mclassdef.mpropdefs do
 398                                 if mpropdef isa MMethodDef and mpropdef.is_before_module then return mpropdef
 399                         end
 400                 end
 401                 return null
 402         end
 403
 404         # "after_module" method for this module.
 405         private fun after_test: nullable MMethodDef do
 406                 for mclassdef in mclassdefs do
 407                         if not mclassdef.name == "Sys" then continue
 408                         for mpropdef in mclassdef.mpropdefs do
 409                                 if mpropdef isa MMethodDef and mpropdef.is_after_module then return mpropdef
 410                         end
 411                 end
 412                 return null
 413         end
 414 end
 415
 416 redef class ModelBuilder
 417         # Number of test classes generated.
 418         var total_classes = 0
 419
 420         # Number of tests generated.
 421         var total_tests = 0
 422
 423         # Number of failed tests.
 424         var failed_tests = 0
 425
 426         # Run NitUnit test suite for `mmodule` (if it is one).
 427         fun test_unit(mmodule: MModule): nullable HTMLTag do
 428                 # is the module a test_suite?
 429                 if not mmodule.is_test then return null
 430                 toolcontext.info("nitunit: test-suite {mmodule}", 2)
 431
 432                 var tester = new NitUnitTester(self)
 433                 var res = tester.test_module_unit(mmodule)
 434                 return res.to_xml
 435         end
 436 end