src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 import testing_base
  19 intrude import markdown
  20
  21 # Extractor, Executor and Reporter for the tests in a module
  22 class NitUnitExecutor
  23         super Doc2Mdwn
  24
  25         # The module to import
  26         var mmodule: MModule
  27
  28         # The prefix of the generated Nit source-file
  29         var prefix: String
  30
  31         # The XML node associated to the module
  32         var testsuite: HTMLTag
  33
  34         # Initialize a new e
  35         init(toolcontext: ToolContext, prefix: String, mmodule: MModule, testsuite: HTMLTag)
  36         do
  37                 super(toolcontext)
  38                 self.prefix = prefix
  39                 self.mmodule = mmodule
  40                 self.testsuite = testsuite
  41         end
  42
  43         # All blocks of code from a same `ADoc`
  44         var blocks = new Array[Array[String]]
  45
  46         redef fun process_code(n: HTMLTag, text: String)
  47         do
  48                 # Try to parse it
  49                 var ast = toolcontext.parse_something(text)
  50
  51                 # We want executable code
  52                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
  53                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  54                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of code that is not valid Nit, thus not considered a nitunit")
  55                                 if ast isa AError then toolcontext.warning(ast.location, "syntax-error", ast.message)
  56                                 ndoc = null # To avoid multiple warning in the same node
  57                         end
  58                         return
  59                 end
  60
  61                 # Search `assert` in the AST
  62                 var v = new SearchAssertVisitor
  63                 v.enter_visit(ast)
  64                 if not v.foundit then
  65                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  66                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of Nit code without `assert`, thus not considered a nitunit")
  67                                 ndoc = null # To avoid multiple warning in the same node
  68                         end
  69                         return
  70                 end
  71
  72                 # Create a first block
  73                 # Or create a new block for modules that are more than a main part
  74                 if blocks.is_empty or ast isa AModule then
  75                         blocks.add(new Array[String])
  76                 end
  77
  78                 # Add it to the file
  79                 blocks.last.add(text)
  80         end
  81
  82         # The associated node to localize warnings
  83         var ndoc: nullable ADoc
  84
  85         # used to generate distinct names
  86         var cpt = 0
  87
  88         # The entry point for a new `ndoc` node
  89         # Fill the prepated `tc` (testcase) XTM node
  90         fun extract(ndoc: ADoc, tc: HTMLTag)
  91         do
  92                 blocks.clear
  93
  94                 self.ndoc = ndoc
  95
  96                 work(ndoc.to_mdoc)
  97                 toolcontext.check_errors
  98
  99                 if blocks.is_empty then return
 100
 101                 for block in blocks do test_block(ndoc, tc, block)
 102         end
 103
 104         # Execute a block
 105         fun test_block(ndoc: ADoc, tc: HTMLTag, block: Array[String])
 106         do
 107                 toolcontext.modelbuilder.unit_entities += 1
 108
 109                 cpt += 1
 110                 var file = "{prefix}{cpt}.nit"
 111
 112                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 113
 114                 var dir = file.dirname
 115                 if dir != "" then dir.mkdir
 116                 var f
 117                 f = new OFStream.open(file)
 118                 f.write("# GENERATED FILE\n")
 119                 f.write("# Example extracted from a documentation\n")
 120                 f.write("import {mmodule.name}\n")
 121                 f.write("\n")
 122                 for text in block do
 123                         f.write(text)
 124                 end
 125                 f.close
 126
 127                 if toolcontext.opt_noact.value then return
 128
 129                 var nit_dir = toolcontext.nit_dir
 130                 var nitg = "{nit_dir or else ""}/bin/nitg"
 131                 if nit_dir == null or not nitg.file_exists then
 132                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 133                         toolcontext.check_errors
 134                 end
 135                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 136                 var res = sys.system(cmd)
 137                 var res2 = 0
 138                 if res == 0 then
 139                         res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
 140                 end
 141
 142                 var msg
 143                 f = new IFStream.open("{file}.out1")
 144                 var n2
 145                 n2 = new HTMLTag("system-err")
 146                 tc.add n2
 147                 msg = f.read_all
 148                 f.close
 149
 150                 n2 = new HTMLTag("system-out")
 151                 tc.add n2
 152                 for text in block do n2.append(text)
 153
 154
 155                 if res != 0 then
 156                         var ne = new HTMLTag("failure")
 157                         ne.attr("message", msg)
 158                         tc.add ne
 159                         toolcontext.warning(ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 160                         toolcontext.modelbuilder.failed_entities += 1
 161                 else if res2 != 0 then
 162                         var ne = new HTMLTag("error")
 163                         ne.attr("message", msg)
 164                         tc.add ne
 165                         toolcontext.warning(ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 166                         toolcontext.modelbuilder.failed_entities += 1
 167                 end
 168                 toolcontext.check_errors
 169
 170                 testsuite.add(tc)
 171         end
 172 end
 173
 174 class SearchAssertVisitor
 175         super Visitor
 176         var foundit = false
 177         redef fun visit(node)
 178         do
 179                 if foundit then
 180                         return
 181                 else if node isa AAssertExpr then
 182                         foundit = true
 183                         return
 184                 else
 185                         node.visit_all(self)
 186                 end
 187         end
 188 end
 189
 190 redef class ModelBuilder
 191         var total_entities = 0
 192         var doc_entities = 0
 193         var unit_entities = 0
 194         var failed_entities = 0
 195
 196         fun test_markdown(mmodule: MModule): HTMLTag
 197         do
 198                 var ts = new HTMLTag("testsuite")
 199                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 200                 if not mmodule2nmodule.has_key(mmodule) then return ts
 201
 202                 var nmodule = mmodule2nmodule[mmodule]
 203
 204                 # usualy, only the original module must be imported in the unit test.
 205                 var o = mmodule
 206                 var g = o.mgroup
 207                 if g != null and g.mproject.name == "standard" then
 208                         # except for a unit test in a module of standard
 209                         # in this case, the whole standard must be imported
 210                         o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
 211                 end
 212
 213                 ts.attr("package", mmodule.full_name)
 214
 215                 var prefix = toolcontext.test_dir
 216                 prefix = prefix.join_path(mmodule.to_s)
 217                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 218
 219                 var tc
 220
 221                 do
 222                         total_entities += 1
 223                         var nmoduledecl = nmodule.n_moduledecl
 224                         if nmoduledecl == null then break label x
 225                         var ndoc = nmoduledecl.n_doc
 226                         if ndoc == null then break label x
 227                         doc_entities += 1
 228                         tc = new HTMLTag("testcase")
 229                         # NOTE: jenkins expects a '.' in the classname attr
 230                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 231                         tc.attr("name", "<module>")
 232                         d2m.extract(ndoc, tc)
 233                 end label x
 234                 for nclassdef in nmodule.n_classdefs do
 235                         var mclassdef = nclassdef.mclassdef
 236                         if mclassdef == null then continue
 237                         if nclassdef isa AStdClassdef then
 238                                 total_entities += 1
 239                                 var ndoc = nclassdef.n_doc
 240                                 if ndoc != null then
 241                                         doc_entities += 1
 242                                         tc = new HTMLTag("testcase")
 243                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 244                                         tc.attr("name", "<class>")
 245                                         d2m.extract(ndoc, tc)
 246                                 end
 247                         end
 248                         for npropdef in nclassdef.n_propdefs do
 249                                 var mpropdef = npropdef.mpropdef
 250                                 if mpropdef == null then continue
 251                                 total_entities += 1
 252                                 var ndoc = npropdef.n_doc
 253                                 if ndoc != null then
 254                                         doc_entities += 1
 255                                         tc = new HTMLTag("testcase")
 256                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 257                                         tc.attr("name", mpropdef.mproperty.full_name)
 258                                         d2m.extract(ndoc, tc)
 259                                 end
 260                         end
 261                 end
 262
 263                 return ts
 264         end
 265 end