src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 import testing_base
  19 intrude import docdown
  20
  21 # Extractor, Executor and Reporter for the tests in a module
  22 class NitUnitExecutor
  23         super Doc2Mdwn
  24
  25         # The prefix of the generated Nit source-file
  26         var prefix: String
  27
  28         # The module to import
  29         var mmodule: MModule
  30
  31         # The XML node associated to the module
  32         var testsuite: HTMLTag
  33
  34         # All blocks of code from a same `ADoc`
  35         var blocks = new Array[Array[String]]
  36
  37         # All failures from a same `ADoc`
  38         var failures = new Array[String]
  39
  40         redef fun process_code(n: HTMLTag, text: String, tag: nullable String)
  41         do
  42                 # Skip non-blocks
  43                 if n.tag != "pre" then return
  44
  45                 # Skip strict non-nit
  46                 if tag != null and tag != "nit" and tag != "" then
  47                         return
  48                 end
  49
  50                 # Try to parse it
  51                 var ast = toolcontext.parse_something(text)
  52
  53                 # Skip pure comments
  54                 if ast isa TComment then return
  55
  56                 # We want executable code
  57                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
  58                         var message = ""
  59                         if ast isa AError then message = " At {ast.location}: {ast.message}."
  60                         toolcontext.warning(ndoc.location, "invalid-block", "Error: There is a block of code that is not valid Nit, thus not considered a nitunit. To suppress this warning, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).{message}")
  61                         failures.add("{ndoc.location}: Invalid block of code.{message}")
  62                         return
  63                 end
  64
  65                 # Create a first block
  66                 # Or create a new block for modules that are more than a main part
  67                 if blocks.is_empty or ast isa AModule then
  68                         blocks.add(new Array[String])
  69                 end
  70
  71                 # Add it to the file
  72                 blocks.last.add(text)
  73         end
  74
  75         # The associated node to localize warnings
  76         var ndoc: nullable ADoc = null
  77
  78         # used to generate distinct names
  79         var cpt = 0
  80
  81         # The entry point for a new `ndoc` node
  82         # Fill `docunits` with new discovered unit of tests.
  83         #
  84         # `tc` (testcase) is the pre-filled XML node
  85         fun extract(ndoc: ADoc, tc: HTMLTag)
  86         do
  87                 blocks.clear
  88                 failures.clear
  89
  90                 self.ndoc = ndoc
  91
  92                 work(ndoc.to_mdoc)
  93
  94                 toolcontext.check_errors
  95
  96                 if not failures.is_empty then
  97                         for msg in failures do
  98                                 var ne = new HTMLTag("failure")
  99                                 ne.attr("message", msg)
 100                                 tc.add ne
 101                                 toolcontext.modelbuilder.failed_entities += 1
 102                         end
 103                         if blocks.is_empty then testsuite.add(tc)
 104                 end
 105
 106                 if blocks.is_empty then return
 107
 108                 for block in blocks do
 109                         docunits.add new DocUnit(ndoc, tc, block.join(""))
 110                 end
 111         end
 112
 113         # All extracted docunits
 114         var docunits = new Array[DocUnit]
 115
 116         # Execute all the docunits
 117         fun run_tests
 118         do
 119                 for du in docunits do
 120                                 test_single_docunit(du)
 121                 end
 122         end
 123
 124         # Executes a single doc-unit in its own program.
 125         fun test_single_docunit(du: DocUnit)
 126         do
 127                 var tc = du.testcase
 128                 toolcontext.modelbuilder.unit_entities += 1
 129
 130                 cpt += 1
 131                 var file = "{prefix}-{cpt}.nit"
 132
 133                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 134
 135                 var dir = file.dirname
 136                 if dir != "" then dir.mkdir
 137                 var f
 138                 f = new OFStream.open(file)
 139                 f.write("# GENERATED FILE\n")
 140                 f.write("# Example extracted from a documentation\n")
 141                 f.write("import {mmodule.name}\n")
 142                 f.write("\n")
 143                 f.write(du.block)
 144                 f.close
 145
 146                 if toolcontext.opt_noact.value then return
 147
 148                 var nit_dir = toolcontext.nit_dir
 149                 var nitg = nit_dir/"bin/nitg"
 150                 if not nitg.file_exists then
 151                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 152                         toolcontext.check_errors
 153                 end
 154                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 155                 var res = sys.system(cmd)
 156                 var res2 = 0
 157                 if res == 0 then
 158                         res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
 159                 end
 160
 161                 var msg
 162                 f = new IFStream.open("{file}.out1")
 163                 var n2
 164                 n2 = new HTMLTag("system-err")
 165                 tc.add n2
 166                 msg = f.read_all
 167                 f.close
 168
 169                 n2 = new HTMLTag("system-out")
 170                 tc.add n2
 171                 n2.append(du.block)
 172
 173
 174                 if res != 0 then
 175                         var ne = new HTMLTag("failure")
 176                         ne.attr("message", msg)
 177                         tc.add ne
 178                         toolcontext.warning(du.ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 179                         toolcontext.modelbuilder.failed_entities += 1
 180                 else if res2 != 0 then
 181                         var ne = new HTMLTag("error")
 182                         ne.attr("message", msg)
 183                         tc.add ne
 184                         toolcontext.warning(du.ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 185                         toolcontext.modelbuilder.failed_entities += 1
 186                 end
 187                 toolcontext.check_errors
 188
 189                 testsuite.add(tc)
 190         end
 191 end
 192
 193 # A unit-test to run
 194 class DocUnit
 195         # The original comment node
 196         var ndoc: ADoc
 197
 198         # The XML node that contains the information about the execution
 199         var testcase: HTMLTag
 200
 201         # The text of the code to execute
 202         var block: String
 203 end
 204
 205 class SearchAssertVisitor
 206         super Visitor
 207         var foundit = false
 208         redef fun visit(node)
 209         do
 210                 if foundit then
 211                         return
 212                 else if node isa AAssertExpr then
 213                         foundit = true
 214                         return
 215                 else
 216                         node.visit_all(self)
 217                 end
 218         end
 219 end
 220
 221 redef class ModelBuilder
 222         var total_entities = 0
 223         var doc_entities = 0
 224         var unit_entities = 0
 225         var failed_entities = 0
 226
 227         fun test_markdown(mmodule: MModule): HTMLTag
 228         do
 229                 var ts = new HTMLTag("testsuite")
 230                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 231                 if not mmodule2nmodule.has_key(mmodule) then return ts
 232
 233                 var nmodule = mmodule2nmodule[mmodule]
 234
 235                 # usualy, only the original module must be imported in the unit test.
 236                 var o = mmodule
 237                 var g = o.mgroup
 238                 if g != null and g.mproject.name == "standard" then
 239                         # except for a unit test in a module of standard
 240                         # in this case, the whole standard must be imported
 241                         o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
 242                 end
 243
 244                 ts.attr("package", mmodule.full_name)
 245
 246                 var prefix = toolcontext.test_dir
 247                 prefix = prefix.join_path(mmodule.to_s)
 248                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 249
 250                 var tc
 251
 252                 do
 253                         total_entities += 1
 254                         var nmoduledecl = nmodule.n_moduledecl
 255                         if nmoduledecl == null then break label x
 256                         var ndoc = nmoduledecl.n_doc
 257                         if ndoc == null then break label x
 258                         doc_entities += 1
 259                         tc = new HTMLTag("testcase")
 260                         # NOTE: jenkins expects a '.' in the classname attr
 261                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 262                         tc.attr("name", "<module>")
 263                         d2m.extract(ndoc, tc)
 264                 end label x
 265                 for nclassdef in nmodule.n_classdefs do
 266                         var mclassdef = nclassdef.mclassdef
 267                         if mclassdef == null then continue
 268                         if nclassdef isa AStdClassdef then
 269                                 total_entities += 1
 270                                 var ndoc = nclassdef.n_doc
 271                                 if ndoc != null then
 272                                         doc_entities += 1
 273                                         tc = new HTMLTag("testcase")
 274                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 275                                         tc.attr("name", "<class>")
 276                                         d2m.extract(ndoc, tc)
 277                                 end
 278                         end
 279                         for npropdef in nclassdef.n_propdefs do
 280                                 var mpropdef = npropdef.mpropdef
 281                                 if mpropdef == null then continue
 282                                 total_entities += 1
 283                                 var ndoc = npropdef.n_doc
 284                                 if ndoc != null then
 285                                         doc_entities += 1
 286                                         tc = new HTMLTag("testcase")
 287                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 288                                         tc.attr("name", mpropdef.mproperty.full_name)
 289                                         d2m.extract(ndoc, tc)
 290                                 end
 291                         end
 292                 end
 293
 294                 d2m.run_tests
 295
 296                 return ts
 297         end
 298 end