src/testing/testing_doc.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Testing from code comments.
  16 module testing_doc
  17
  18 import testing_base
  19 intrude import markdown
  20
  21 # Extractor, Executor and Reporter for the tests in a module
  22 class NitUnitExecutor
  23         super Doc2Mdwn
  24
  25         # The prefix of the generated Nit source-file
  26         var prefix: String
  27
  28         # The module to import
  29         var mmodule: MModule
  30
  31         # The XML node associated to the module
  32         var testsuite: HTMLTag
  33
  34         # All blocks of code from a same `ADoc`
  35         var blocks = new Array[Array[String]]
  36
  37         redef fun process_code(n: HTMLTag, text: String)
  38         do
  39                 # Try to parse it
  40                 var ast = toolcontext.parse_something(text)
  41
  42                 # We want executable code
  43                 if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
  44                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  45                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of code that is not valid Nit, thus not considered a nitunit")
  46                                 if ast isa AError then toolcontext.warning(ast.location, "syntax-error", ast.message)
  47                                 ndoc = null # To avoid multiple warning in the same node
  48                         end
  49                         return
  50                 end
  51
  52                 # Search `assert` in the AST
  53                 var v = new SearchAssertVisitor
  54                 v.enter_visit(ast)
  55                 if not v.foundit then
  56                         if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
  57                                 toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of Nit code without `assert`, thus not considered a nitunit")
  58                                 ndoc = null # To avoid multiple warning in the same node
  59                         end
  60                         return
  61                 end
  62
  63                 # Create a first block
  64                 # Or create a new block for modules that are more than a main part
  65                 if blocks.is_empty or ast isa AModule then
  66                         blocks.add(new Array[String])
  67                 end
  68
  69                 # Add it to the file
  70                 blocks.last.add(text)
  71         end
  72
  73         # The associated node to localize warnings
  74         var ndoc: nullable ADoc = null
  75
  76         # used to generate distinct names
  77         var cpt = 0
  78
  79         # The entry point for a new `ndoc` node
  80         # Fill the prepated `tc` (testcase) XTM node
  81         fun extract(ndoc: ADoc, tc: HTMLTag)
  82         do
  83                 blocks.clear
  84
  85                 self.ndoc = ndoc
  86
  87                 work(ndoc.to_mdoc)
  88                 toolcontext.check_errors
  89
  90                 if blocks.is_empty then return
  91
  92                 for block in blocks do test_block(ndoc, tc, block)
  93         end
  94
  95         # Execute a block
  96         fun test_block(ndoc: ADoc, tc: HTMLTag, block: Array[String])
  97         do
  98                 toolcontext.modelbuilder.unit_entities += 1
  99
 100                 cpt += 1
 101                 var file = "{prefix}{cpt}.nit"
 102
 103                 toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
 104
 105                 var dir = file.dirname
 106                 if dir != "" then dir.mkdir
 107                 var f
 108                 f = new OFStream.open(file)
 109                 f.write("# GENERATED FILE\n")
 110                 f.write("# Example extracted from a documentation\n")
 111                 f.write("import {mmodule.name}\n")
 112                 f.write("\n")
 113                 for text in block do
 114                         f.write(text)
 115                 end
 116                 f.close
 117
 118                 if toolcontext.opt_noact.value then return
 119
 120                 var nit_dir = toolcontext.nit_dir
 121                 var nitg = "{nit_dir or else ""}/bin/nitg"
 122                 if nit_dir == null or not nitg.file_exists then
 123                         toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
 124                         toolcontext.check_errors
 125                 end
 126                 var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
 127                 var res = sys.system(cmd)
 128                 var res2 = 0
 129                 if res == 0 then
 130                         res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
 131                 end
 132
 133                 var msg
 134                 f = new IFStream.open("{file}.out1")
 135                 var n2
 136                 n2 = new HTMLTag("system-err")
 137                 tc.add n2
 138                 msg = f.read_all
 139                 f.close
 140
 141                 n2 = new HTMLTag("system-out")
 142                 tc.add n2
 143                 for text in block do n2.append(text)
 144
 145
 146                 if res != 0 then
 147                         var ne = new HTMLTag("failure")
 148                         ne.attr("message", msg)
 149                         tc.add ne
 150                         toolcontext.warning(ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 151                         toolcontext.modelbuilder.failed_entities += 1
 152                 else if res2 != 0 then
 153                         var ne = new HTMLTag("error")
 154                         ne.attr("message", msg)
 155                         tc.add ne
 156                         toolcontext.warning(ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
 157                         toolcontext.modelbuilder.failed_entities += 1
 158                 end
 159                 toolcontext.check_errors
 160
 161                 testsuite.add(tc)
 162         end
 163 end
 164
 165 class SearchAssertVisitor
 166         super Visitor
 167         var foundit = false
 168         redef fun visit(node)
 169         do
 170                 if foundit then
 171                         return
 172                 else if node isa AAssertExpr then
 173                         foundit = true
 174                         return
 175                 else
 176                         node.visit_all(self)
 177                 end
 178         end
 179 end
 180
 181 redef class ModelBuilder
 182         var total_entities = 0
 183         var doc_entities = 0
 184         var unit_entities = 0
 185         var failed_entities = 0
 186
 187         fun test_markdown(mmodule: MModule): HTMLTag
 188         do
 189                 var ts = new HTMLTag("testsuite")
 190                 toolcontext.info("nitunit: doc-unit {mmodule}", 2)
 191                 if not mmodule2nmodule.has_key(mmodule) then return ts
 192
 193                 var nmodule = mmodule2nmodule[mmodule]
 194
 195                 # usualy, only the original module must be imported in the unit test.
 196                 var o = mmodule
 197                 var g = o.mgroup
 198                 if g != null and g.mproject.name == "standard" then
 199                         # except for a unit test in a module of standard
 200                         # in this case, the whole standard must be imported
 201                         o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
 202                 end
 203
 204                 ts.attr("package", mmodule.full_name)
 205
 206                 var prefix = toolcontext.test_dir
 207                 prefix = prefix.join_path(mmodule.to_s)
 208                 var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
 209
 210                 var tc
 211
 212                 do
 213                         total_entities += 1
 214                         var nmoduledecl = nmodule.n_moduledecl
 215                         if nmoduledecl == null then break label x
 216                         var ndoc = nmoduledecl.n_doc
 217                         if ndoc == null then break label x
 218                         doc_entities += 1
 219                         tc = new HTMLTag("testcase")
 220                         # NOTE: jenkins expects a '.' in the classname attr
 221                         tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
 222                         tc.attr("name", "<module>")
 223                         d2m.extract(ndoc, tc)
 224                 end label x
 225                 for nclassdef in nmodule.n_classdefs do
 226                         var mclassdef = nclassdef.mclassdef
 227                         if mclassdef == null then continue
 228                         if nclassdef isa AStdClassdef then
 229                                 total_entities += 1
 230                                 var ndoc = nclassdef.n_doc
 231                                 if ndoc != null then
 232                                         doc_entities += 1
 233                                         tc = new HTMLTag("testcase")
 234                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 235                                         tc.attr("name", "<class>")
 236                                         d2m.extract(ndoc, tc)
 237                                 end
 238                         end
 239                         for npropdef in nclassdef.n_propdefs do
 240                                 var mpropdef = npropdef.mpropdef
 241                                 if mpropdef == null then continue
 242                                 total_entities += 1
 243                                 var ndoc = npropdef.n_doc
 244                                 if ndoc != null then
 245                                         doc_entities += 1
 246                                         tc = new HTMLTag("testcase")
 247                                         tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
 248                                         tc.attr("name", mpropdef.mproperty.full_name)
 249                                         d2m.extract(ndoc, tc)
 250                                 end
 251                         end
 252                 end
 253
 254                 return ts
 255         end
 256 end