src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 private import parser_util
  19 import testing_base
  20 import markdown
  21 import html
  22
  23 # Extractor, Executor and Reporter for the tests in a module
  24 class NitUnitExecutor
  25         super HTMLDecorator
  26
  27         # Toolcontext used to parse Nit code blocks.
  28         var toolcontext: ToolContext
  29
  30         # The prefix of the generated Nit source-file
  31         var prefix: String
  32
  33         # The module to import, if any
  34         var mmodule: nullable MModule
  35
  36         # The XML node associated to the module
  37         var testsuite: HTMLTag
  38
  39         # All blocks of code from a same `ADoc`
  40         var blocks = new Array[Buffer]
  41
  42         # All failures from a same `ADoc`
  43         var failures = new Array[String]
  44
  45         # Markdown processor used to parse markdown comments and extract code.
  46         var mdproc = new MarkdownProcessor
  47
  48         init do
  49                 mdproc.emitter.decorator = new NitunitDecorator(self)
  50         end
  51
  52         # The associated documentation object
  53         var mdoc: nullable MDoc = null
  54
  55         # used to generate distinct names
  56         var cpt = 0
  57
  58         # The entry point for a new `ndoc` node
  59         # Fill `docunits` with new discovered unit of tests.
  60         #
  61         # `tc` (testcase) is the pre-filled XML node
  62         fun extract(mdoc: MDoc, tc: HTMLTag)
  63         do
  64                 blocks.clear
  65                 failures.clear
  66
  67                 self.mdoc = mdoc
  68
  69                 # Populate `blocks` from the markdown decorator
  70                 mdproc.process(mdoc.content.join("\n"))
  71
  72                 toolcontext.check_errors
  73
  74                 if not failures.is_empty then
  75                         for msg in failures do
  76                                 var ne = new HTMLTag("failure")
  77                                 ne.attr("message", msg)
  78                                 tc.add ne
  79                                 toolcontext.modelbuilder.unit_entities += 1
  80                                 toolcontext.modelbuilder.failed_entities += 1
  81                         end
  82                         if blocks.is_empty then testsuite.add(tc)
  83                 end
  84
  85                 if blocks.is_empty then return
  86                 for block in blocks do
  87                         docunits.add new DocUnit(mdoc, tc, block.write_to_string)
  88                 end
  89         end
  90
  91         # All extracted docunits
  92         var docunits = new Array[DocUnit]
  93
  94         # Execute all the docunits
  95         fun run_tests
  96         do
  97                 var simple_du = new Array[DocUnit]
  98                 for du in docunits do
  99                         var ast = toolcontext.parse_something(du.block)
 100                         if ast isa AExpr then
 101                                 simple_du.add du
 102                         else
 103                                 test_single_docunit(du)
 104                         end
 105                 end
 106
 107                 test_simple_docunits(simple_du)
 108         end
 109
 110         # Executes multiples doc-units in a shared program.
 111         # Used for docunits simple block of code (without modules, classes, functions etc.)
 112         #
 113         # In case of compilation error, the method fallbacks to `test_single_docunit` to
 114         # * locate exactly the compilation problem in the problematic docunit.
 115         # * permit the execution of the other docunits that may be correct.
 116         fun test_simple_docunits(dus: Array[DocUnit])
 117         do
 118                 if dus.is_empty then return
 119
 120                 var file = "{prefix}-0.nit"
 121
 122                 var dir = file.dirname
 123                 if dir != "" then dir.mkdir
 124                 var f
 125                 f = create_unitfile(file)
 126                 var i = 0
 127                 for du in dus do
 128
 129                         i += 1
 130                         f.write("fun run_{i} do\n")
 131                         f.write("# {du.testcase.attrs["name"]}\n")
 132                         f.write(du.block)
 133                         f.write("end\n")
 134                 end
 135                 f.write("var a = args.first.to_i\n")
 136                 for j in [1..i] do
 137                         f.write("if a == {j} then run_{j}\n")
 138                 end
 139                 f.close
 140
 141                 if toolcontext.opt_noact.value then return
 142
 143                 var res = compile_unitfile(file)
 144
 145                 if res != 0 then
 146                         # Compilation error.
 147                         # Fall-back to individual modes:
 148                         for du in dus do
 149                                 test_single_docunit(du)
 150                         end
 151                         return
 152                 end
 153
 154                 i = 0
 155                 for du in dus do
 156                         var tc = du.testcase
 157                         toolcontext.modelbuilder.unit_entities += 1
 158                         i += 1
 159                         toolcontext.info("Execute doc-unit {du.testcase.attrs["name"]} in {file} {i}", 1)
 160                         var res2 = toolcontext.safe_exec("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
 161
 162                         f = new FileReader.open("{file}.out1")
 163                         var n2
 164                         n2 = new HTMLTag("system-err")
 165                         tc.add n2
 166                         var content = f.read_all
 167                         var msg = content.trunc(8192).filter_nonprintable
 168                         n2.append(msg)
 169                         f.close
 170
 171                         n2 = new HTMLTag("system-out")
 172                         tc.add n2
 173                         n2.append(du.block)
 174
 175                         if res2 != 0 then
 176                                 var ne = new HTMLTag("error")
 177                                 ne.attr("message", "Runtime error")
 178                                 tc.add ne
 179                                 toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): Runtime error\n{msg}")
 180                                 toolcontext.modelbuilder.failed_entities += 1
 181                         end
 182                         toolcontext.check_errors
 183
 184                         testsuite.add(tc)
 185                 end
 186         end
 187
 188         # Executes a single doc-unit in its own program.
 189         # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
 190         fun test_single_docunit(du: DocUnit)
 191         do
 192                 var tc = du.testcase
 193                 toolcontext.modelbuilder.unit_entities += 1
 194
 195                 cpt += 1
 196                 var file = "{prefix}-{cpt}.nit"
 197
 198                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 199
 200                 var f
 201                 f = create_unitfile(file)
 202                 f.write(du.block)
 203                 f.close
 204
 205                 if toolcontext.opt_noact.value then return
 206
 207                 var res = compile_unitfile(file)
 208                 var res2 = 0
 209                 if res == 0 then
 210                         res2 = toolcontext.safe_exec("{file.to_program_name}.bin >'{file}.out1' 2>&1 </dev/null")
 211                 end
 212
 213                 f = new FileReader.open("{file}.out1")
 214                 var n2
 215                 n2 = new HTMLTag("system-err")
 216                 tc.add n2
 217                 var content = f.read_all
 218                 var msg = content.trunc(8192).filter_nonprintable
 219                 n2.append(msg)
 220                 f.close
 221
 222                 n2 = new HTMLTag("system-out")
 223                 tc.add n2
 224                 n2.append(du.block)
 225
 226
 227                 if res != 0 then
 228                         var ne = new HTMLTag("failure")
 229                         ne.attr("message", "Compilation Error")
 230                         tc.add ne
 231                         toolcontext.warning(du.mdoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
 232                         toolcontext.modelbuilder.failed_entities += 1
 233                 else if res2 != 0 then
 234                         var ne = new HTMLTag("error")
 235                         ne.attr("message", "Runtime Error")
 236                         tc.add ne
 237                         toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
 238                         toolcontext.modelbuilder.failed_entities += 1
 239                 end
 240                 toolcontext.check_errors
 241
 242                 testsuite.add(tc)
 243         end
 244
 245         # Create and fill the header of a unit file `file`.
 246         #
 247         # A unit file is a Nit source file generated from one
 248         # or more docunits that will be compiled and executed.
 249         #
 250         # The handled on the file is returned and must be completed and closed.
 251         #
 252         # `file` should be a valid filepath for a Nit source file.
 253         private fun create_unitfile(file: String): Writer
 254         do
 255                 var dir = file.dirname
 256                 if dir != "" then dir.mkdir
 257                 var f
 258                 f = new FileWriter.open(file)
 259                 f.write("# GENERATED FILE\n")
 260                 f.write("# Docunits extracted from comments\n")
 261                 if mmodule != null then
 262                         f.write("import {mmodule.name}\n")
 263                 end
 264                 f.write("\n")
 265                 return f
 266         end
 267
 268         # Compile an unit file and return the compiler return code
 269         #
 270         # Can terminate the program if the compiler is not found
 271         private fun compile_unitfile(file: String): Int
 272         do
 273                 var nitc = toolcontext.find_nitc
 274                 var opts = new Array[String]
 275                 if mmodule != null then
 276                         opts.add "-I {mmodule.filepath.dirname}"
 277                 end
 278                 var cmd = "{nitc} --ignore-visibility --no-color '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 279                 var res = toolcontext.safe_exec(cmd)
 280                 return res
 281         end
 282 end
 283
 284 private class NitunitDecorator
 285         super HTMLDecorator
 286
 287         var executor: NitUnitExecutor
 288
 289         redef fun add_code(v, block) do
 290                 var code = block.raw_content
 291                 var meta = block.meta or else "nit"
 292                 # Do not try to test non-nit code.
 293                 if meta != "nit" then return
 294                 # Try to parse code blocks
 295                 var ast = executor.toolcontext.parse_something(code)
 296
 297                 var mdoc = executor.mdoc
 298                 assert mdoc != null
 299
 300                 # Skip pure comments
 301                 if ast isa TComment then return
 302
 303                 # The location is computed according to the starts of the mdoc and the block
 304                 # Note, the following assumes that all the comments of the mdoc are correctly aligned.
 305                 var loc = block.block.location
 306                 var line_offset = loc.line_start + mdoc.location.line_start - 2
 307                 var column_offset = loc.column_start + mdoc.location.column_start
 308                 # Hack to handle precise location in blocks
 309                 # TODO remove when markdown is more reliable
 310                 if block isa BlockFence then
 311                         # Skip the starting fence
 312                         line_offset += 1
 313                 else
 314                         # Account a standard 4 space indentation
 315                         column_offset += 4
 316                 end
 317
 318                 # We want executable code
 319                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
 320                         var message
 321                         var l = ast.location
 322                         # Get real location of the node (or error)
 323                         var location = new Location(mdoc.location.file,
 324                                 l.line_start + line_offset,
 325                                 l.line_end + line_offset,
 326                                 l.column_start + column_offset,
 327                                 l.column_end + column_offset)
 328                         if ast isa AError then
 329                                 message = ast.message
 330                         else
 331                                 message = "Error: Invalid Nit code."
 332                         end
 333
 334                         executor.toolcontext.warning(location, "invalid-block", "{message} To suppress this message, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).")
 335                         executor.failures.add("{location}: {message}")
 336                         return
 337                 end
 338
 339                 # Create a first block
 340                 # Or create a new block for modules that are more than a main part
 341                 if executor.blocks.is_empty or ast isa AModule then
 342                         executor.blocks.add(new Buffer)
 343                 end
 344
 345                 # Add it to the file
 346                 executor.blocks.last.append code
 347         end
 348 end
 349
 350 # A unit-test to run
 351 class DocUnit
 352         # The doc that contains self
 353         var mdoc: MDoc
 354
 355         # The XML node that contains the information about the execution
 356         var testcase: HTMLTag
 357
 358         # The text of the code to execute
 359         var block: String
 360 end
 361
 362 redef class ModelBuilder
 363         # Total number analyzed `MEntity`
 364         var total_entities = 0
 365
 366         # The number of `MEntity` that have some documentation
 367         var doc_entities = 0
 368
 369         # The total number of executed docunits
 370         var unit_entities = 0
 371
 372         # The number failed docunits
 373         var failed_entities = 0
 374
 375         # Extracts and executes all the docunits in the `mmodule`
 376         # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
 377         fun test_markdown(mmodule: MModule): HTMLTag
 378         do
 379                 var ts = new HTMLTag("testsuite")
 380                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 381
 382                 var nmodule = mmodule2node(mmodule)
 383                 if nmodule == null then return ts
 384
 385                 # usualy, only the original module must be imported in the unit test.
 386                 var o = mmodule
 387                 var g = o.mgroup
 388                 if g != null and g.mpackage.name == "core" then
 389                         # except for a unit test in a module of `core`
 390                         # in this case, the whole `core` must be imported
 391                         o = get_mmodule_by_name(nmodule, g, g.mpackage.name).as(not null)
 392                 end
 393
 394                 ts.attr("package", mmodule.full_name)
 395
 396                 var prefix = toolcontext.test_dir
 397                 prefix = prefix.join_path(mmodule.to_s)
 398                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 399
 400                 var tc
 401
 402                 do
 403                         total_entities += 1
 404                         var nmoduledecl = nmodule.n_moduledecl
 405                         if nmoduledecl == null then break label x
 406                         var ndoc = nmoduledecl.n_doc
 407                         if ndoc == null then break label x
 408                         doc_entities += 1
 409                         tc = new HTMLTag("testcase")
 410                         # NOTE: jenkins expects a '.' in the classname attr
 411                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 412                         tc.attr("name", "<module>")
 413                         d2m.extract(ndoc.to_mdoc, tc)
 414                 end label x
 415                 for nclassdef in nmodule.n_classdefs do
 416                         var mclassdef = nclassdef.mclassdef
 417                         if mclassdef == null then continue
 418                         if nclassdef isa AStdClassdef then
 419                                 total_entities += 1
 420                                 var ndoc = nclassdef.n_doc
 421                                 if ndoc != null then
 422                                         doc_entities += 1
 423                                         tc = new HTMLTag("testcase")
 424                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 425                                         tc.attr("name", "<class>")
 426                                         d2m.extract(ndoc.to_mdoc, tc)
 427                                 end
 428                         end
 429                         for npropdef in nclassdef.n_propdefs do
 430                                 var mpropdef = npropdef.mpropdef
 431                                 if mpropdef == null then continue
 432                                 total_entities += 1
 433                                 var ndoc = npropdef.n_doc
 434                                 if ndoc != null then
 435                                         doc_entities += 1
 436                                         tc = new HTMLTag("testcase")
 437                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 438                                         tc.attr("name", mpropdef.mproperty.full_name)
 439                                         d2m.extract(ndoc.to_mdoc, tc)
 440                                 end
 441                         end
 442                 end
 443
 444                 d2m.run_tests
 445
 446                 return ts
 447         end
 448
 449         # Extracts and executes all the docunits in the readme of the `mgroup`
 450         # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
 451         fun test_group(mgroup: MGroup): HTMLTag
 452         do
 453                 var ts = new HTMLTag("testsuite")
 454                 toolcontext.info("nitunit: doc-unit group {mgroup}", 2)
 455
 456                 # usually, only the default module must be imported in the unit test.
 457                 var o = mgroup.default_mmodule
 458
 459                 ts.attr("package", mgroup.full_name)
 460
 461                 var prefix = toolcontext.test_dir
 462                 prefix = prefix.join_path(mgroup.to_s)
 463                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 464
 465                 var tc
 466
 467                 total_entities += 1
 468                 var mdoc = mgroup.mdoc
 469                 if mdoc == null then return ts
 470
 471                 doc_entities += 1
 472                 tc = new HTMLTag("testcase")
 473                 # NOTE: jenkins expects a '.' in the classname attr
 474                 tc.attr("classname", "nitunit." + mgroup.full_name)
 475                 tc.attr("name", "<group>")
 476                 d2m.extract(mdoc, tc)
 477
 478                 d2m.run_tests
 479
 480                 return ts
 481         end
 482
 483         # Test a document object unrelated to a Nit entity
 484         fun test_mdoc(mdoc: MDoc): HTMLTag
 485         do
 486                 var ts = new HTMLTag("testsuite")
 487                 var file = mdoc.location.to_s
 488
 489                 toolcontext.info("nitunit: doc-unit file {file}", 2)
 490
 491                 ts.attr("package", file)
 492
 493                 var prefix = toolcontext.test_dir / "file"
 494                 var d2m = new NitUnitExecutor(toolcontext, prefix, null, ts)
 495
 496                 var tc
 497
 498                 total_entities += 1
 499                 doc_entities += 1
 500
 501                 tc = new HTMLTag("testcase")
 502                 # NOTE: jenkins expects a '.' in the classname attr
 503                 tc.attr("classname", "nitunit.<file>")
 504                 tc.attr("name", file)
 505
 506                 d2m.extract(mdoc, tc)
 507                 d2m.run_tests
 508
 509                 return ts
 510         end
 511 end