src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 import testing_base
  19 intrude import docdown
  20
  21 # Extractor, Executor and Reporter for the tests in a module
  22 class NitUnitExecutor
  23         super Doc2Mdwn
  24
  25         # The prefix of the generated Nit source-file
  26         var prefix: String
  27
  28         # The module to import
  29         var mmodule: MModule
  30
  31         # The XML node associated to the module
  32         var testsuite: HTMLTag
  33
  34         # All blocks of code from a same `ADoc`
  35         var blocks = new Array[Array[String]]
  36
  37         # All failures from a same `ADoc`
  38         var failures = new Array[String]
  39
  40         redef fun process_code(n: HTMLTag, text: String, tag: nullable String)
  41         do
  42                 # Skip non-blocks
  43                 if n.tag != "pre" then return
  44
  45                 # Skip strict non-nit
  46                 if tag != null and tag != "nit" and tag != "" then
  47                         return
  48                 end
  49
  50                 # Try to parse it
  51                 var ast = toolcontext.parse_something(text)
  52
  53                 # Skip pure comments
  54                 if ast isa TComment then return
  55
  56                 # We want executable code
  57                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
  58                         var message = ""
  59                         if ast isa AError then message = " At {ast.location}: {ast.message}."
  60                         toolcontext.warning(ndoc.location, "invalid-block", "Error: There is a block of code that is not valid Nit, thus not considered a nitunit. To suppress this warning, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).{message}")
  61                         failures.add("{ndoc.location}: Invalid block of code.{message}")
  62                         return
  63                 end
  64
  65                 # Create a first block
  66                 # Or create a new block for modules that are more than a main part
  67                 if blocks.is_empty or ast isa AModule then
  68                         blocks.add(new Array[String])
  69                 end
  70
  71                 # Add it to the file
  72                 blocks.last.add(text)
  73         end
  74
  75         # The associated node to localize warnings
  76         var ndoc: nullable ADoc = null
  77
  78         # used to generate distinct names
  79         var cpt = 0
  80
  81         # The entry point for a new `ndoc` node
  82         # Fill `docunits` with new discovered unit of tests.
  83         #
  84         # `tc` (testcase) is the pre-filled XML node
  85         fun extract(ndoc: ADoc, tc: HTMLTag)
  86         do
  87                 blocks.clear
  88                 failures.clear
  89
  90                 self.ndoc = ndoc
  91
  92                 work(ndoc.to_mdoc)
  93
  94                 toolcontext.check_errors
  95
  96                 if not failures.is_empty then
  97                         for msg in failures do
  98                                 var ne = new HTMLTag("failure")
  99                                 ne.attr("message", msg)
 100                                 tc.add ne
 101                                 toolcontext.modelbuilder.failed_entities += 1
 102                         end
 103                         if blocks.is_empty then testsuite.add(tc)
 104                 end
 105
 106                 if blocks.is_empty then return
 107
 108                 for block in blocks do
 109                         docunits.add new DocUnit(ndoc, tc, block.join(""))
 110                 end
 111         end
 112
 113         # All extracted docunits
 114         var docunits = new Array[DocUnit]
 115
 116         # Execute all the docunits
 117         fun run_tests
 118         do
 119                 var simple_du = new Array[DocUnit]
 120                 for du in docunits do
 121                         var ast = toolcontext.parse_something(du.block)
 122                         if ast isa AExpr then
 123                                 simple_du.add du
 124                         else
 125                                 test_single_docunit(du)
 126                         end
 127                 end
 128
 129                 test_simple_docunits(simple_du)
 130         end
 131
 132         # Executes multiples doc-units in a shared program.
 133         # Used for docunits simple block of code (without modules, classes, functions etc.)
 134         #
 135         # In case of compilation error, the method fallbacks to `test_single_docunit` to
 136         # * locate exactly the compilation problem in the problematic docunit.
 137         # * permit the execution of the other docunits that may be correct.
 138         fun test_simple_docunits(dus: Array[DocUnit])
 139         do
 140                 if dus.is_empty then return
 141
 142                 var file = "{prefix}-0.nit"
 143
 144                 var dir = file.dirname
 145                 if dir != "" then dir.mkdir
 146                 var f
 147                 f = new FileWriter.open(file)
 148                 f.write("# GENERATED FILE\n")
 149                 f.write("# Docunits extracted from comments\n")
 150                 f.write("import {mmodule.name}\n")
 151                 f.write("\n")
 152                 var i = 0
 153                 for du in dus do
 154
 155                         i += 1
 156                         f.write("fun run_{i} do\n")
 157                         f.write("# {du.testcase.attrs["name"]}\n")
 158                         f.write(du.block)
 159                         f.write("end\n")
 160                 end
 161                 f.write("var a = args.first.to_i\n")
 162                 for j in [1..i] do
 163                         f.write("if a == {j} then run_{j}\n")
 164                 end
 165                 f.close
 166
 167                 if toolcontext.opt_noact.value then return
 168
 169                 var nit_dir = toolcontext.nit_dir
 170                 var nitg = nit_dir/"bin/nitg"
 171                 if not nitg.file_exists then
 172                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 173                         toolcontext.check_errors
 174                 end
 175                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 176                 var res = sys.system(cmd)
 177
 178                 if res != 0 then
 179                         # Compilation error.
 180                         # Fall-back to individual modes:
 181                         for du in dus do
 182                                 test_single_docunit(du)
 183                         end
 184                         return
 185                 end
 186
 187                 i = 0
 188                 for du in dus do
 189                         var tc = du.testcase
 190                         toolcontext.modelbuilder.unit_entities += 1
 191                         i += 1
 192                         toolcontext.info("Execute doc-unit {du.testcase.attrs["name"]} in {file} {i}", 1)
 193                         var res2 = sys.system("{file.to_program_name}.bin {i} >>'{file}.out1' 2>&1 </dev/null")
 194
 195                         var msg
 196                         f = new FileReader.open("{file}.out1")
 197                         var n2
 198                         n2 = new HTMLTag("system-err")
 199                         tc.add n2
 200                         msg = f.read_all
 201                         f.close
 202
 203                         n2 = new HTMLTag("system-out")
 204                         tc.add n2
 205                         n2.append(du.block)
 206
 207                         if res2 != 0 then
 208                                 var ne = new HTMLTag("error")
 209                                 ne.attr("message", msg)
 210                                 tc.add ne
 211                                 toolcontext.warning(du.ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 212                                 toolcontext.modelbuilder.failed_entities += 1
 213                         end
 214                         toolcontext.check_errors
 215
 216                         testsuite.add(tc)
 217                 end
 218         end
 219
 220         # Executes a single doc-unit in its own program.
 221         # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
 222         fun test_single_docunit(du: DocUnit)
 223         do
 224                 var tc = du.testcase
 225                 toolcontext.modelbuilder.unit_entities += 1
 226
 227                 cpt += 1
 228                 var file = "{prefix}-{cpt}.nit"
 229
 230                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 231
 232                 var dir = file.dirname
 233                 if dir != "" then dir.mkdir
 234                 var f
 235                 f = new FileWriter.open(file)
 236                 f.write("# GENERATED FILE\n")
 237                 f.write("# Example extracted from a documentation\n")
 238                 f.write("import {mmodule.name}\n")
 239                 f.write("\n")
 240                 f.write(du.block)
 241                 f.close
 242
 243                 if toolcontext.opt_noact.value then return
 244
 245                 var nit_dir = toolcontext.nit_dir
 246                 var nitg = nit_dir/"bin/nitg"
 247                 if not nitg.file_exists then
 248                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 249                         toolcontext.check_errors
 250                 end
 251                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 252                 var res = sys.system(cmd)
 253                 var res2 = 0
 254                 if res == 0 then
 255                         res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
 256                 end
 257
 258                 var msg
 259                 f = new FileReader.open("{file}.out1")
 260                 var n2
 261                 n2 = new HTMLTag("system-err")
 262                 tc.add n2
 263                 msg = f.read_all
 264                 f.close
 265
 266                 n2 = new HTMLTag("system-out")
 267                 tc.add n2
 268                 n2.append(du.block)
 269
 270
 271                 if res != 0 then
 272                         var ne = new HTMLTag("failure")
 273                         ne.attr("message", msg)
 274                         tc.add ne
 275                         toolcontext.warning(du.ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 276                         toolcontext.modelbuilder.failed_entities += 1
 277                 else if res2 != 0 then
 278                         var ne = new HTMLTag("error")
 279                         ne.attr("message", msg)
 280                         tc.add ne
 281                         toolcontext.warning(du.ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 282                         toolcontext.modelbuilder.failed_entities += 1
 283                 end
 284                 toolcontext.check_errors
 285
 286                 testsuite.add(tc)
 287         end
 288 end
 289
 290 # A unit-test to run
 291 class DocUnit
 292         # The original comment node
 293         var ndoc: ADoc
 294
 295         # The XML node that contains the information about the execution
 296         var testcase: HTMLTag
 297
 298         # The text of the code to execute
 299         var block: String
 300 end
 301
 302 redef class ModelBuilder
 303         # Total number analyzed `MEntity`
 304         var total_entities = 0
 305
 306         # The number of `MEntity` that have some documentation
 307         var doc_entities = 0
 308
 309         # The total number of executed docunits
 310         var unit_entities = 0
 311
 312         # The number failed docunits
 313         var failed_entities = 0
 314
 315         # Extracts and executes all the docunits in the `mmodule`
 316         # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
 317         fun test_markdown(mmodule: MModule): HTMLTag
 318         do
 319                 var ts = new HTMLTag("testsuite")
 320                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 321
 322                 var nmodule = mmodule2node(mmodule)
 323                 if nmodule == null then return ts
 324
 325                 # usualy, only the original module must be imported in the unit test.
 326                 var o = mmodule
 327                 var g = o.mgroup
 328                 if g != null and g.mproject.name == "standard" then
 329                         # except for a unit test in a module of standard
 330                         # in this case, the whole standard must be imported
 331                         o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
 332                 end
 333
 334                 ts.attr("package", mmodule.full_name)
 335
 336                 var prefix = toolcontext.test_dir
 337                 prefix = prefix.join_path(mmodule.to_s)
 338                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 339
 340                 var tc
 341
 342                 do
 343                         total_entities += 1
 344                         var nmoduledecl = nmodule.n_moduledecl
 345                         if nmoduledecl == null then break label x
 346                         var ndoc = nmoduledecl.n_doc
 347                         if ndoc == null then break label x
 348                         doc_entities += 1
 349                         tc = new HTMLTag("testcase")
 350                         # NOTE: jenkins expects a '.' in the classname attr
 351                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 352                         tc.attr("name", "<module>")
 353                         d2m.extract(ndoc, tc)
 354                 end label x
 355                 for nclassdef in nmodule.n_classdefs do
 356                         var mclassdef = nclassdef.mclassdef
 357                         if mclassdef == null then continue
 358                         if nclassdef isa AStdClassdef then
 359                                 total_entities += 1
 360                                 var ndoc = nclassdef.n_doc
 361                                 if ndoc != null then
 362                                         doc_entities += 1
 363                                         tc = new HTMLTag("testcase")
 364                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 365                                         tc.attr("name", "<class>")
 366                                         d2m.extract(ndoc, tc)
 367                                 end
 368                         end
 369                         for npropdef in nclassdef.n_propdefs do
 370                                 var mpropdef = npropdef.mpropdef
 371                                 if mpropdef == null then continue
 372                                 total_entities += 1
 373                                 var ndoc = npropdef.n_doc
 374                                 if ndoc != null then
 375                                         doc_entities += 1
 376                                         tc = new HTMLTag("testcase")
 377                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 378                                         tc.attr("name", mpropdef.mproperty.full_name)
 379                                         d2m.extract(ndoc, tc)
 380                                 end
 381                         end
 382                 end
 383
 384                 d2m.run_tests
 385
 386                 return ts
 387         end
 388 end