1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from code comments.
18 private import parser_util
23 # Extractor, Executor and Reporter for the tests in a module
27 # Toolcontext used to parse Nit code blocks.
28 var toolcontext
: ToolContext
30 # The prefix of the generated Nit source-file
33 # The module to import, if any
34 var mmodule
: nullable MModule
36 # The XML node associated to the module
37 var testsuite
: HTMLTag
39 # All blocks of code from a same `ADoc`
40 var blocks
= new Array[Buffer]
42 # All failures from a same `ADoc`
43 var failures
= new Array[String]
45 # Markdown processor used to parse markdown comments and extract code.
46 var mdproc
= new MarkdownProcessor
49 mdproc
.emitter
.decorator
= new NitunitDecorator(self)
52 # The associated documentation object
53 var mdoc
: nullable MDoc = null
55 # used to generate distinct names
58 # The entry point for a new `ndoc` node
59 # Fill `docunits` with new discovered unit of tests.
61 # `tc` (testcase) is the pre-filled XML node
62 fun extract
(mdoc
: MDoc, tc
: HTMLTag)
69 # Populate `blocks` from the markdown decorator
70 mdproc
.process
(mdoc
.content
.join
("\n"))
72 toolcontext
.check_errors
74 if not failures
.is_empty
then
75 for msg
in failures
do
76 var ne
= new HTMLTag("failure")
77 ne
.attr
("message", msg
)
79 toolcontext
.modelbuilder
.unit_entities
+= 1
80 toolcontext
.modelbuilder
.failed_entities
+= 1
82 if blocks
.is_empty
then testsuite
.add
(tc
)
85 if blocks
.is_empty
then return
86 for block
in blocks
do
87 docunits
.add
new DocUnit(mdoc
, tc
, block
.write_to_string
)
91 # All extracted docunits
92 var docunits
= new Array[DocUnit]
94 # Execute all the docunits
97 var simple_du
= new Array[DocUnit]
99 var ast
= toolcontext
.parse_something
(du
.block
)
100 if ast
isa AExpr then
103 test_single_docunit
(du
)
107 test_simple_docunits
(simple_du
)
110 # Executes multiples doc-units in a shared program.
111 # Used for docunits simple block of code (without modules, classes, functions etc.)
113 # In case of compilation error, the method fallbacks to `test_single_docunit` to
114 # * locate exactly the compilation problem in the problematic docunit.
115 # * permit the execution of the other docunits that may be correct.
116 fun test_simple_docunits
(dus
: Array[DocUnit])
118 if dus
.is_empty
then return
120 var file
= "{prefix}-0.nit"
122 var dir
= file
.dirname
123 if dir
!= "" then dir
.mkdir
125 f
= create_unitfile
(file
)
130 f
.write
("fun run_{i} do\n")
131 f
.write
("# {du.testcase.attrs["name"]}\n")
135 f
.write
("var a = args.first.to_i\n")
137 f
.write
("if a == {j} then run_{j}\n")
141 if toolcontext
.opt_noact
.value
then return
143 var res
= compile_unitfile
(file
)
147 # Fall-back to individual modes:
149 test_single_docunit
(du
)
157 toolcontext
.modelbuilder
.unit_entities
+= 1
159 toolcontext
.info
("Execute doc-unit {du.testcase.attrs["name"]} in {file} {i}", 1)
160 var res2
= sys
.system
("{file.to_program_name}.bin {i} >>'{file}.out1' 2>&1 </dev/null")
163 f
= new FileReader.open
("{file}.out1")
165 n2
= new HTMLTag("system-err")
170 n2
= new HTMLTag("system-out")
175 var ne
= new HTMLTag("error")
176 ne
.attr
("message", msg
)
178 toolcontext
.warning
(du
.mdoc
.location
, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
179 toolcontext
.modelbuilder
.failed_entities
+= 1
181 toolcontext
.check_errors
187 # Executes a single doc-unit in its own program.
188 # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
189 fun test_single_docunit
(du
: DocUnit)
192 toolcontext
.modelbuilder
.unit_entities
+= 1
195 var file
= "{prefix}-{cpt}.nit"
197 toolcontext
.info
("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
200 f
= create_unitfile
(file
)
204 if toolcontext
.opt_noact
.value
then return
206 var res
= compile_unitfile
(file
)
209 res2
= sys
.system
("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
213 f
= new FileReader.open
("{file}.out1")
215 n2
= new HTMLTag("system-err")
220 n2
= new HTMLTag("system-out")
226 var ne
= new HTMLTag("failure")
227 ne
.attr
("message", msg
)
229 toolcontext
.warning
(du
.mdoc
.location
, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
230 toolcontext
.modelbuilder
.failed_entities
+= 1
231 else if res2
!= 0 then
232 var ne
= new HTMLTag("error")
233 ne
.attr
("message", msg
)
235 toolcontext
.warning
(du
.mdoc
.location
, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
236 toolcontext
.modelbuilder
.failed_entities
+= 1
238 toolcontext
.check_errors
243 # Create and fill the header of a unit file `file`.
245 # A unit file is a Nit source file generated from one
246 # or more docunits that will be compiled and executed.
248 # The handled on the file is returned and must be completed and closed.
250 # `file` should be a valid filepath for a Nit source file.
251 private fun create_unitfile
(file
: String): Writer
253 var dir
= file
.dirname
254 if dir
!= "" then dir
.mkdir
256 f
= new FileWriter.open
(file
)
257 f
.write
("# GENERATED FILE\n")
258 f
.write
("# Docunits extracted from comments\n")
259 if mmodule
!= null then
260 f
.write
("import {mmodule.name}\n")
266 # Compile an unit file and return the compiler return code
268 # Can terminate the program if the compiler is not found
269 private fun compile_unitfile
(file
: String): Int
271 var nit_dir
= toolcontext
.nit_dir
272 var nitc
= nit_dir
/"bin/nitc"
273 if not nitc
.file_exists
then
274 toolcontext
.error
(null, "Error: cannot find nitc. Set envvar NIT_DIR.")
275 toolcontext
.check_errors
277 var opts
= new Array[String]
278 if mmodule
!= null then
279 opts
.add
"-I {mmodule.location.file.filename.dirname}"
281 var cmd
= "{nitc} --ignore-visibility --no-color '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
282 var res
= sys
.system
(cmd
)
287 private class NitunitDecorator
290 var executor
: NitUnitExecutor
292 redef fun add_code
(v
, block
) do
293 var code
= code_from_block
(block
)
295 if block
isa BlockFence and block
.meta
!= null then
296 meta
= block
.meta
.to_s
298 # Do not try to test non-nit code.
299 if meta
!= "nit" then return
300 # Try to parse code blocks
301 var ast
= executor
.toolcontext
.parse_something
(code
)
304 if ast
isa TComment then return
306 # We want executable code
307 if not (ast
isa AModule or ast
isa ABlockExpr or ast
isa AExpr) then
309 if ast
isa AError then message
= " At {ast.location}: {ast.message}."
310 executor
.toolcontext
.warning
(executor
.mdoc
.location
, "invalid-block", "Error: there is a block of invalid Nit code, thus not considered a nitunit. To suppress this warning, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).{message}")
311 executor
.failures
.add
("{executor.mdoc.location}: Invalid block of code.{message}")
315 # Create a first block
316 # Or create a new block for modules that are more than a main part
317 if executor
.blocks
.is_empty
or ast
isa AModule then
318 executor
.blocks
.add
(new Buffer)
322 executor
.blocks
.last
.append code
325 # Extracts code as String from a `BlockCode`.
326 fun code_from_block
(block
: BlockCode): String do
327 var infence
= block
isa BlockFence
328 var text
= new FlatBuffer
329 var line
= block
.block
.first_line
330 while line
!= null do
331 if not line
.is_empty
then
333 if not infence
and str
.has_prefix
(" ") then
334 text
.append str
.substring
(4, str
.length
- line
.trailing
)
342 return text
.write_to_string
348 # The doc that contains self
351 # The XML node that contains the information about the execution
352 var testcase
: HTMLTag
354 # The text of the code to execute
358 redef class ModelBuilder
359 # Total number analyzed `MEntity`
360 var total_entities
= 0
362 # The number of `MEntity` that have some documentation
365 # The total number of executed docunits
366 var unit_entities
= 0
368 # The number failed docunits
369 var failed_entities
= 0
371 # Extracts and executes all the docunits in the `mmodule`
372 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
373 fun test_markdown
(mmodule
: MModule): HTMLTag
375 var ts
= new HTMLTag("testsuite")
376 toolcontext
.info
("nitunit: doc-unit {mmodule}", 2)
378 var nmodule
= mmodule2node
(mmodule
)
379 if nmodule
== null then return ts
381 # usualy, only the original module must be imported in the unit test.
384 if g
!= null and g
.mproject
.name
== "core" then
385 # except for a unit test in a module of `core`
386 # in this case, the whole `core` must be imported
387 o
= get_mmodule_by_name
(nmodule
, g
, g
.mproject
.name
).as(not null)
390 ts
.attr
("package", mmodule
.full_name
)
392 var prefix
= toolcontext
.test_dir
393 prefix
= prefix
.join_path
(mmodule
.to_s
)
394 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
400 var nmoduledecl
= nmodule
.n_moduledecl
401 if nmoduledecl
== null then break label x
402 var ndoc
= nmoduledecl
.n_doc
403 if ndoc
== null then break label x
405 tc
= new HTMLTag("testcase")
406 # NOTE: jenkins expects a '.' in the classname attr
407 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ ".<module>")
408 tc
.attr
("name", "<module>")
409 d2m
.extract
(ndoc
.to_mdoc
, tc
)
411 for nclassdef
in nmodule
.n_classdefs
do
412 var mclassdef
= nclassdef
.mclassdef
413 if mclassdef
== null then continue
414 if nclassdef
isa AStdClassdef then
416 var ndoc
= nclassdef
.n_doc
419 tc
= new HTMLTag("testcase")
420 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
)
421 tc
.attr
("name", "<class>")
422 d2m
.extract
(ndoc
.to_mdoc
, tc
)
425 for npropdef
in nclassdef
.n_propdefs
do
426 var mpropdef
= npropdef
.mpropdef
427 if mpropdef
== null then continue
429 var ndoc
= npropdef
.n_doc
432 tc
= new HTMLTag("testcase")
433 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
)
434 tc
.attr
("name", mpropdef
.mproperty
.full_name
)
435 d2m
.extract
(ndoc
.to_mdoc
, tc
)
445 # Extracts and executes all the docunits in the readme of the `mgroup`
446 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
447 fun test_group
(mgroup
: MGroup): HTMLTag
449 var ts
= new HTMLTag("testsuite")
450 toolcontext
.info
("nitunit: doc-unit group {mgroup}", 2)
452 # usually, only the default module must be imported in the unit test.
453 var o
= mgroup
.default_mmodule
455 ts
.attr
("package", mgroup
.full_name
)
457 var prefix
= toolcontext
.test_dir
458 prefix
= prefix
.join_path
(mgroup
.to_s
)
459 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
464 var mdoc
= mgroup
.mdoc
465 if mdoc
== null then return ts
468 tc
= new HTMLTag("testcase")
469 # NOTE: jenkins expects a '.' in the classname attr
470 tc
.attr
("classname", "nitunit." + mgroup
.full_name
)
471 tc
.attr
("name", "<group>")
472 d2m
.extract
(mdoc
, tc
)
479 # Test a document object unrelated to a Nit entity
480 fun test_mdoc
(mdoc
: MDoc): HTMLTag
482 var ts
= new HTMLTag("testsuite")
483 var file
= mdoc
.location
.to_s
485 toolcontext
.info
("nitunit: doc-unit file {file}", 2)
487 ts
.attr
("package", file
)
489 var prefix
= toolcontext
.test_dir
/ "file"
490 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, null, ts
)
497 tc
= new HTMLTag("testcase")
498 # NOTE: jenkins expects a '.' in the classname attr
499 tc
.attr
("classname", "nitunit.<file>")
500 tc
.attr
("name", file
)
502 d2m
.extract
(mdoc
, tc
)