src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 import testing_base
  19 intrude import docdown
  20
  21 # Extractor, Executor and Reporter for the tests in a module
  22 class NitUnitExecutor
  23         super Doc2Mdwn
  24
  25         # The prefix of the generated Nit source-file
  26         var prefix: String
  27
  28         # The module to import
  29         var mmodule: MModule
  30
  31         # The XML node associated to the module
  32         var testsuite: HTMLTag
  33
  34         # All blocks of code from a same `ADoc`
  35         var blocks = new Array[Array[String]]
  36
  37         redef fun process_code(n: HTMLTag, text: String, tag: nullable String)
  38         do
  39                 # Skip non-blocks
  40                 if n.tag != "pre" then return
  41
  42                 # Skip strict non-nit
  43                 if tag != null and tag != "nit" and tag != "" then
  44                         return
  45                 end
  46
  47                 # Try to parse it
  48                 var ast = toolcontext.parse_something(text)
  49
  50                 # We want executable code
  51                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
  52                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  53                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of code that is not valid Nit, thus not considered a nitunit")
  54                                 if ast isa AError then toolcontext.warning(ast.location, "syntax-error", ast.message)
  55                                 ndoc = null # To avoid multiple warning in the same node
  56                         end
  57                         return
  58                 end
  59
  60                 # Search `assert` in the AST
  61                 var v = new SearchAssertVisitor
  62                 v.enter_visit(ast)
  63                 if not v.foundit then
  64                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  65                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of Nit code without `assert`, thus not considered a nitunit")
  66                                 ndoc = null # To avoid multiple warning in the same node
  67                         end
  68                         return
  69                 end
  70
  71                 # Create a first block
  72                 # Or create a new block for modules that are more than a main part
  73                 if blocks.is_empty or ast isa AModule then
  74                         blocks.add(new Array[String])
  75                 end
  76
  77                 # Add it to the file
  78                 blocks.last.add(text)
  79         end
  80
  81         # The associated node to localize warnings
  82         var ndoc: nullable ADoc = null
  83
  84         # used to generate distinct names
  85         var cpt = 0
  86
  87         # The entry point for a new `ndoc` node
  88         # Fill the prepated `tc` (testcase) XTM node
  89         fun extract(ndoc: ADoc, tc: HTMLTag)
  90         do
  91                 blocks.clear
  92
  93                 self.ndoc = ndoc
  94
  95                 work(ndoc.to_mdoc)
  96                 toolcontext.check_errors
  97
  98                 if blocks.is_empty then return
  99
 100                 for block in blocks do test_block(ndoc, tc, block)
 101         end
 102
 103         # Execute a block
 104         fun test_block(ndoc: ADoc, tc: HTMLTag, block: Array[String])
 105         do
 106                 toolcontext.modelbuilder.unit_entities += 1
 107
 108                 cpt += 1
 109                 var file = "{prefix}-{cpt}.nit"
 110
 111                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 112
 113                 var dir = file.dirname
 114                 if dir != "" then dir.mkdir
 115                 var f
 116                 f = new OFStream.open(file)
 117                 f.write("# GENERATED FILE\n")
 118                 f.write("# Example extracted from a documentation\n")
 119                 f.write("import {mmodule.name}\n")
 120                 f.write("\n")
 121                 for text in block do
 122                         f.write(text)
 123                 end
 124                 f.close
 125
 126                 if toolcontext.opt_noact.value then return
 127
 128                 var nit_dir = toolcontext.nit_dir
 129                 var nitg = nit_dir/"bin/nitg"
 130                 if not nitg.file_exists then
 131                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 132                         toolcontext.check_errors
 133                 end
 134                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 135                 var res = sys.system(cmd)
 136                 var res2 = 0
 137                 if res == 0 then
 138                         res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
 139                 end
 140
 141                 var msg
 142                 f = new IFStream.open("{file}.out1")
 143                 var n2
 144                 n2 = new HTMLTag("system-err")
 145                 tc.add n2
 146                 msg = f.read_all
 147                 f.close
 148
 149                 n2 = new HTMLTag("system-out")
 150                 tc.add n2
 151                 for text in block do n2.append(text)
 152
 153
 154                 if res != 0 then
 155                         var ne = new HTMLTag("failure")
 156                         ne.attr("message", msg)
 157                         tc.add ne
 158                         toolcontext.warning(ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 159                         toolcontext.modelbuilder.failed_entities += 1
 160                 else if res2 != 0 then
 161                         var ne = new HTMLTag("error")
 162                         ne.attr("message", msg)
 163                         tc.add ne
 164                         toolcontext.warning(ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 165                         toolcontext.modelbuilder.failed_entities += 1
 166                 end
 167                 toolcontext.check_errors
 168
 169                 testsuite.add(tc)
 170         end
 171 end
 172
 173 class SearchAssertVisitor
 174         super Visitor
 175         var foundit = false
 176         redef fun visit(node)
 177         do
 178                 if foundit then
 179                         return
 180                 else if node isa AAssertExpr then
 181                         foundit = true
 182                         return
 183                 else
 184                         node.visit_all(self)
 185                 end
 186         end
 187 end
 188
 189 redef class ModelBuilder
 190         var total_entities = 0
 191         var doc_entities = 0
 192         var unit_entities = 0
 193         var failed_entities = 0
 194
 195         fun test_markdown(mmodule: MModule): HTMLTag
 196         do
 197                 var ts = new HTMLTag("testsuite")
 198                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 199                 if not mmodule2nmodule.has_key(mmodule) then return ts
 200
 201                 var nmodule = mmodule2nmodule[mmodule]
 202
 203                 # usualy, only the original module must be imported in the unit test.
 204                 var o = mmodule
 205                 var g = o.mgroup
 206                 if g != null and g.mproject.name == "standard" then
 207                         # except for a unit test in a module of standard
 208                         # in this case, the whole standard must be imported
 209                         o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
 210                 end
 211
 212                 ts.attr("package", mmodule.full_name)
 213
 214                 var prefix = toolcontext.test_dir
 215                 prefix = prefix.join_path(mmodule.to_s)
 216                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 217
 218                 var tc
 219
 220                 do
 221                         total_entities += 1
 222                         var nmoduledecl = nmodule.n_moduledecl
 223                         if nmoduledecl == null then break label x
 224                         var ndoc = nmoduledecl.n_doc
 225                         if ndoc == null then break label x
 226                         doc_entities += 1
 227                         tc = new HTMLTag("testcase")
 228                         # NOTE: jenkins expects a '.' in the classname attr
 229                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 230                         tc.attr("name", "<module>")
 231                         d2m.extract(ndoc, tc)
 232                 end label x
 233                 for nclassdef in nmodule.n_classdefs do
 234                         var mclassdef = nclassdef.mclassdef
 235                         if mclassdef == null then continue
 236                         if nclassdef isa AStdClassdef then
 237                                 total_entities += 1
 238                                 var ndoc = nclassdef.n_doc
 239                                 if ndoc != null then
 240                                         doc_entities += 1
 241                                         tc = new HTMLTag("testcase")
 242                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 243                                         tc.attr("name", "<class>")
 244                                         d2m.extract(ndoc, tc)
 245                                 end
 246                         end
 247                         for npropdef in nclassdef.n_propdefs do
 248                                 var mpropdef = npropdef.mpropdef
 249                                 if mpropdef == null then continue
 250                                 total_entities += 1
 251                                 var ndoc = npropdef.n_doc
 252                                 if ndoc != null then
 253                                         doc_entities += 1
 254                                         tc = new HTMLTag("testcase")
 255                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 256                                         tc.attr("name", mpropdef.mproperty.full_name)
 257                                         d2m.extract(ndoc, tc)
 258                                 end
 259                         end
 260                 end
 261
 262                 return ts
 263         end
 264 end