1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from code comments.
18 private import parser_util
23 # Extractor, Executor and Reporter for the tests in a module
27 # Toolcontext used to parse Nit code blocks.
28 var toolcontext
: ToolContext
30 # The prefix of the generated Nit source-file
33 # The module to import, if any
34 var mmodule
: nullable MModule
36 # The XML node associated to the module
37 var testsuite
: HTMLTag
39 # The current test-case xml element
40 var tc
: HTMLTag is noautoinit
42 # All failures from a same `ADoc`
43 var failures
= new Array[String]
45 # Markdown processor used to parse markdown comments and extract code.
46 var mdproc
= new MarkdownProcessor
49 mdproc
.emitter
.decorator
= new NitunitDecorator(self)
52 # The associated documentation object
53 var mdoc
: nullable MDoc = null
55 # used to generate distinct names
58 # The last docunit extracted from a mdoc.
60 # Is used because a new code-block might just be added to it.
61 var last_docunit
: nullable DocUnit = null
63 # The entry point for a new `ndoc` node
64 # Fill `docunits` with new discovered unit of tests.
66 # `tc` (testcase) is the pre-filled XML node
67 fun extract
(mdoc
: MDoc, tc
: HTMLTag)
75 # Populate `blocks` from the markdown decorator
76 mdproc
.process
(mdoc
.content
.join
("\n"))
78 toolcontext
.check_errors
80 if not failures
.is_empty
then
81 for msg
in failures
do
82 var ne
= new HTMLTag("failure")
83 ne
.attr
("message", msg
)
85 toolcontext
.modelbuilder
.unit_entities
+= 1
86 toolcontext
.modelbuilder
.failed_entities
+= 1
88 if last_docunit
== null then testsuite
.add
(tc
)
92 # All extracted docunits
93 var docunits
= new Array[DocUnit]
95 # Execute all the docunits
98 var simple_du
= new Array[DocUnit]
100 var ast
= toolcontext
.parse_something
(du
.block
)
101 if ast
isa AExpr then
104 test_single_docunit
(du
)
108 test_simple_docunits
(simple_du
)
111 # Executes multiples doc-units in a shared program.
112 # Used for docunits simple block of code (without modules, classes, functions etc.)
114 # In case of compilation error, the method fallbacks to `test_single_docunit` to
115 # * locate exactly the compilation problem in the problematic docunit.
116 # * permit the execution of the other docunits that may be correct.
117 fun test_simple_docunits
(dus
: Array[DocUnit])
119 if dus
.is_empty
then return
121 var file
= "{prefix}-0.nit"
123 var dir
= file
.dirname
124 if dir
!= "" then dir
.mkdir
126 f
= create_unitfile
(file
)
131 f
.write
("fun run_{i} do\n")
132 f
.write
("# {du.testcase.attrs["name"]}\n")
136 f
.write
("var a = args.first.to_i\n")
138 f
.write
("if a == {j} then run_{j}\n")
142 if toolcontext
.opt_noact
.value
then return
144 var res
= compile_unitfile
(file
)
148 # Fall-back to individual modes:
150 test_single_docunit
(du
)
158 toolcontext
.modelbuilder
.unit_entities
+= 1
160 toolcontext
.info
("Execute doc-unit {du.testcase.attrs["name"]} in {file} {i}", 1)
161 var res2
= toolcontext
.safe_exec
("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
163 f
= new FileReader.open
("{file}.out1")
165 n2
= new HTMLTag("system-err")
167 var content
= f
.read_all
168 var msg
= content
.trunc
(8192).filter_nonprintable
172 n2
= new HTMLTag("system-out")
177 var ne
= new HTMLTag("error")
178 ne
.attr
("message", "Runtime error")
180 toolcontext
.warning
(du
.mdoc
.location
, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): Runtime error\n{msg}")
181 toolcontext
.modelbuilder
.failed_entities
+= 1
183 toolcontext
.check_errors
189 # Executes a single doc-unit in its own program.
190 # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
191 fun test_single_docunit
(du
: DocUnit)
194 toolcontext
.modelbuilder
.unit_entities
+= 1
197 var file
= "{prefix}-{cpt}.nit"
199 toolcontext
.info
("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
202 f
= create_unitfile
(file
)
206 if toolcontext
.opt_noact
.value
then return
208 var res
= compile_unitfile
(file
)
211 res2
= toolcontext
.safe_exec
("{file.to_program_name}.bin >'{file}.out1' 2>&1 </dev/null")
214 f
= new FileReader.open
("{file}.out1")
216 n2
= new HTMLTag("system-err")
218 var content
= f
.read_all
219 var msg
= content
.trunc
(8192).filter_nonprintable
223 n2
= new HTMLTag("system-out")
229 var ne
= new HTMLTag("failure")
230 ne
.attr
("message", "Compilation Error")
232 toolcontext
.warning
(du
.mdoc
.location
, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
233 toolcontext
.modelbuilder
.failed_entities
+= 1
234 else if res2
!= 0 then
235 var ne
= new HTMLTag("error")
236 ne
.attr
("message", "Runtime Error")
238 toolcontext
.warning
(du
.mdoc
.location
, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
239 toolcontext
.modelbuilder
.failed_entities
+= 1
241 toolcontext
.check_errors
246 # Create and fill the header of a unit file `file`.
248 # A unit file is a Nit source file generated from one
249 # or more docunits that will be compiled and executed.
251 # The handled on the file is returned and must be completed and closed.
253 # `file` should be a valid filepath for a Nit source file.
254 private fun create_unitfile
(file
: String): Writer
256 var dir
= file
.dirname
257 if dir
!= "" then dir
.mkdir
259 f
= new FileWriter.open
(file
)
260 f
.write
("# GENERATED FILE\n")
261 f
.write
("# Docunits extracted from comments\n")
262 if mmodule
!= null then
263 f
.write
("import {mmodule.name}\n")
269 # Compile an unit file and return the compiler return code
271 # Can terminate the program if the compiler is not found
272 private fun compile_unitfile
(file
: String): Int
274 var nitc
= toolcontext
.find_nitc
275 var opts
= new Array[String]
276 if mmodule
!= null then
277 opts
.add
"-I {mmodule.filepath.dirname}"
279 var cmd
= "{nitc} --ignore-visibility --no-color '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
280 var res
= toolcontext
.safe_exec
(cmd
)
285 private class NitunitDecorator
288 var executor
: NitUnitExecutor
290 redef fun add_code
(v
, block
) do
291 var code
= block
.raw_content
292 var meta
= block
.meta
or else "nit"
293 # Do not try to test non-nit code.
294 if meta
!= "nit" then return
295 # Try to parse code blocks
296 var ast
= executor
.toolcontext
.parse_something
(code
)
298 var mdoc
= executor
.mdoc
302 if ast
isa TComment then return
304 # The location is computed according to the starts of the mdoc and the block
305 # Note, the following assumes that all the comments of the mdoc are correctly aligned.
306 var loc
= block
.block
.location
307 var line_offset
= loc
.line_start
+ mdoc
.location
.line_start
- 2
308 var column_offset
= loc
.column_start
+ mdoc
.location
.column_start
309 # Hack to handle precise location in blocks
310 # TODO remove when markdown is more reliable
311 if block
isa BlockFence then
312 # Skip the starting fence
315 # Account a standard 4 space indentation
319 # We want executable code
320 if not (ast
isa AModule or ast
isa ABlockExpr or ast
isa AExpr) then
323 # Get real location of the node (or error)
324 var location
= new Location(mdoc
.location
.file
,
325 l
.line_start
+ line_offset
,
326 l
.line_end
+ line_offset
,
327 l
.column_start
+ column_offset
,
328 l
.column_end
+ column_offset
)
329 if ast
isa AError then
330 message
= ast
.message
332 message
= "Error: Invalid Nit code."
335 executor
.toolcontext
.warning
(location
, "invalid-block", "{message} To suppress this message, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).")
336 executor
.failures
.add
("{location}: {message}")
340 # Create a first block
341 # Or create a new block for modules that are more than a main part
342 var last_docunit
= executor
.last_docunit
343 if last_docunit
== null or ast
isa AModule then
344 last_docunit
= new DocUnit(executor
.mdoc
.as(not null), executor
.tc
, "")
345 executor
.docunits
.add last_docunit
349 last_docunit
.block
+= code
353 # A unit-test extracted from some documentation.
355 # A docunit is extracted from the code-blocks of mdocs.
356 # Each mdoc can contains more than one docunit, and a single docunit can be made of more that a single code-block.
358 # The doc that contains self
361 # The XML node that contains the information about the execution
362 var testcase
: HTMLTag
364 # The text of the code to execute.
366 # This is the verbatim content on one, or more, code-blocks from `mdoc`
370 redef class ModelBuilder
371 # Total number analyzed `MEntity`
372 var total_entities
= 0
374 # The number of `MEntity` that have some documentation
377 # The total number of executed docunits
378 var unit_entities
= 0
380 # The number failed docunits
381 var failed_entities
= 0
383 # Extracts and executes all the docunits in the `mmodule`
384 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
385 fun test_markdown
(mmodule
: MModule): HTMLTag
387 var ts
= new HTMLTag("testsuite")
388 toolcontext
.info
("nitunit: doc-unit {mmodule}", 2)
390 var nmodule
= mmodule2node
(mmodule
)
391 if nmodule
== null then return ts
393 # usualy, only the original module must be imported in the unit test.
396 if g
!= null and g
.mpackage
.name
== "core" then
397 # except for a unit test in a module of `core`
398 # in this case, the whole `core` must be imported
399 o
= get_mmodule_by_name
(nmodule
, g
, g
.mpackage
.name
).as(not null)
402 ts
.attr
("package", mmodule
.full_name
)
404 var prefix
= toolcontext
.test_dir
405 prefix
= prefix
.join_path
(mmodule
.to_s
)
406 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
412 var nmoduledecl
= nmodule
.n_moduledecl
413 if nmoduledecl
== null then break label x
414 var ndoc
= nmoduledecl
.n_doc
415 if ndoc
== null then break label x
417 tc
= new HTMLTag("testcase")
418 # NOTE: jenkins expects a '.' in the classname attr
419 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ ".<module>")
420 tc
.attr
("name", "<module>")
421 d2m
.extract
(ndoc
.to_mdoc
, tc
)
423 for nclassdef
in nmodule
.n_classdefs
do
424 var mclassdef
= nclassdef
.mclassdef
425 if mclassdef
== null then continue
426 if nclassdef
isa AStdClassdef then
428 var ndoc
= nclassdef
.n_doc
431 tc
= new HTMLTag("testcase")
432 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
)
433 tc
.attr
("name", "<class>")
434 d2m
.extract
(ndoc
.to_mdoc
, tc
)
437 for npropdef
in nclassdef
.n_propdefs
do
438 var mpropdef
= npropdef
.mpropdef
439 if mpropdef
== null then continue
441 var ndoc
= npropdef
.n_doc
444 tc
= new HTMLTag("testcase")
445 tc
.attr
("classname", "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
)
446 tc
.attr
("name", mpropdef
.mproperty
.full_name
)
447 d2m
.extract
(ndoc
.to_mdoc
, tc
)
457 # Extracts and executes all the docunits in the readme of the `mgroup`
458 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
459 fun test_group
(mgroup
: MGroup): HTMLTag
461 var ts
= new HTMLTag("testsuite")
462 toolcontext
.info
("nitunit: doc-unit group {mgroup}", 2)
464 # usually, only the default module must be imported in the unit test.
465 var o
= mgroup
.default_mmodule
467 ts
.attr
("package", mgroup
.full_name
)
469 var prefix
= toolcontext
.test_dir
470 prefix
= prefix
.join_path
(mgroup
.to_s
)
471 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
476 var mdoc
= mgroup
.mdoc
477 if mdoc
== null then return ts
480 tc
= new HTMLTag("testcase")
481 # NOTE: jenkins expects a '.' in the classname attr
482 tc
.attr
("classname", "nitunit." + mgroup
.full_name
)
483 tc
.attr
("name", "<group>")
484 d2m
.extract
(mdoc
, tc
)
491 # Test a document object unrelated to a Nit entity
492 fun test_mdoc
(mdoc
: MDoc): HTMLTag
494 var ts
= new HTMLTag("testsuite")
495 var file
= mdoc
.location
.to_s
497 toolcontext
.info
("nitunit: doc-unit file {file}", 2)
499 ts
.attr
("package", file
)
501 var prefix
= toolcontext
.test_dir
/ "file"
502 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, null, ts
)
509 tc
= new HTMLTag("testcase")
510 # NOTE: jenkins expects a '.' in the classname attr
511 tc
.attr
("classname", "nitunit.<file>")
512 tc
.attr
("name", file
)
514 d2m
.extract
(mdoc
, tc
)