1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from code comments.
18 private import parser_util
23 # Extractor, Executor and Reporter for the tests in a module
27 # Toolcontext used to parse Nit code blocks.
28 var toolcontext
: ToolContext
30 # The prefix of the generated Nit source-file
33 # The module to import, if any
34 var mmodule
: nullable MModule
36 # The XML node associated to the module
37 var testsuite
: HTMLTag
39 # Markdown processor used to parse markdown comments and extract code.
40 var mdproc
= new MarkdownProcessor
43 mdproc
.emitter
.decorator
= new NitunitDecorator(self)
46 # The associated documentation object
47 var mdoc
: nullable MDoc = null
49 # used to generate distinct names
52 # The last docunit extracted from a mdoc.
54 # Is used because a new code-block might just be added to it.
55 var last_docunit
: nullable DocUnit = null
57 var xml_classname
: String is noautoinit
59 var xml_name
: String is noautoinit
61 # The entry point for a new `ndoc` node
62 # Fill `docunits` with new discovered unit of tests.
63 fun extract
(mdoc
: MDoc, xml_classname
, xml_name
: String)
66 self.xml_classname
= xml_classname
67 self.xml_name
= xml_name
71 # Populate `blocks` from the markdown decorator
72 mdproc
.process
(mdoc
.content
.join
("\n"))
74 toolcontext
.check_errors
77 # All extracted docunits
78 var docunits
= new Array[DocUnit]
80 # Execute all the docunits
83 var simple_du
= new Array[DocUnit]
85 # Skip existing errors
86 if du
.error
!= null then continue
88 var ast
= toolcontext
.parse_something
(du
.block
)
92 test_single_docunit
(du
)
96 test_simple_docunits
(simple_du
)
99 testsuite
.add du
.to_xml
103 # Executes multiples doc-units in a shared program.
104 # Used for docunits simple block of code (without modules, classes, functions etc.)
106 # In case of compilation error, the method fallbacks to `test_single_docunit` to
107 # * locate exactly the compilation problem in the problematic docunit.
108 # * permit the execution of the other docunits that may be correct.
109 fun test_simple_docunits
(dus
: Array[DocUnit])
111 if dus
.is_empty
then return
113 var file
= "{prefix}-0.nit"
115 var dir
= file
.dirname
116 if dir
!= "" then dir
.mkdir
118 f
= create_unitfile
(file
)
123 f
.write
("fun run_{i} do\n")
124 f
.write
("# {du.full_name}\n")
128 f
.write
("var a = args.first.to_i\n")
130 f
.write
("if a == {j} then run_{j}\n")
134 if toolcontext
.opt_noact
.value
then return
136 var res
= compile_unitfile
(file
)
140 # Fall-back to individual modes:
142 test_single_docunit
(du
)
150 toolcontext
.info
("Execute doc-unit {du.full_name} in {file} {i}", 1)
151 var res2
= toolcontext
.safe_exec
("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
153 var content
= "{file}.out1".to_path
.read_all
154 var msg
= content
.trunc
(8192).filter_nonprintable
158 toolcontext
.warning
(du
.location
, "error", "ERROR: {du.full_name} (in {file}): Runtime error\n{msg}")
159 toolcontext
.modelbuilder
.failed_entities
+= 1
161 toolcontext
.check_errors
165 # Executes a single doc-unit in its own program.
166 # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
167 fun test_single_docunit
(du
: DocUnit)
170 var file
= "{prefix}-{cpt}.nit"
172 toolcontext
.info
("Execute doc-unit {du.full_name} in {file}", 1)
175 f
= create_unitfile
(file
)
179 if toolcontext
.opt_noact
.value
then return
181 var res
= compile_unitfile
(file
)
184 res2
= toolcontext
.safe_exec
("{file.to_program_name}.bin >'{file}.out1' 2>&1 </dev/null")
187 var content
= "{file}.out1".to_path
.read_all
188 var msg
= content
.trunc
(8192).filter_nonprintable
192 toolcontext
.warning
(du
.location
, "failure", "FAILURE: {du.full_name} (in {file}):\n{msg}")
193 toolcontext
.modelbuilder
.failed_entities
+= 1
194 else if res2
!= 0 then
195 toolcontext
.warning
(du
.location
, "error", "ERROR: {du.full_name} (in {file}):\n{msg}")
196 toolcontext
.modelbuilder
.failed_entities
+= 1
198 toolcontext
.check_errors
201 # Create and fill the header of a unit file `file`.
203 # A unit file is a Nit source file generated from one
204 # or more docunits that will be compiled and executed.
206 # The handled on the file is returned and must be completed and closed.
208 # `file` should be a valid filepath for a Nit source file.
209 private fun create_unitfile
(file
: String): Writer
211 var dir
= file
.dirname
212 if dir
!= "" then dir
.mkdir
214 f
= new FileWriter.open
(file
)
215 f
.write
("# GENERATED FILE\n")
216 f
.write
("# Docunits extracted from comments\n")
217 if mmodule
!= null then
218 f
.write
("import {mmodule.name}\n")
224 # Compile an unit file and return the compiler return code
226 # Can terminate the program if the compiler is not found
227 private fun compile_unitfile
(file
: String): Int
229 var nitc
= toolcontext
.find_nitc
230 var opts
= new Array[String]
231 if mmodule
!= null then
232 opts
.add
"-I {mmodule.filepath.dirname}"
234 var cmd
= "{nitc} --ignore-visibility --no-color '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
235 var res
= toolcontext
.safe_exec
(cmd
)
240 private class NitunitDecorator
243 var executor
: NitUnitExecutor
245 redef fun add_code
(v
, block
) do
246 var code
= block
.raw_content
247 var meta
= block
.meta
or else "nit"
248 # Do not try to test non-nit code.
249 if meta
!= "nit" then return
250 # Try to parse code blocks
251 var ast
= executor
.toolcontext
.parse_something
(code
)
253 var mdoc
= executor
.mdoc
257 if ast
isa TComment then return
259 # The location is computed according to the starts of the mdoc and the block
260 # Note, the following assumes that all the comments of the mdoc are correctly aligned.
261 var loc
= block
.block
.location
262 var line_offset
= loc
.line_start
+ mdoc
.location
.line_start
- 2
263 var column_offset
= loc
.column_start
+ mdoc
.location
.column_start
264 # Hack to handle precise location in blocks
265 # TODO remove when markdown is more reliable
266 if block
isa BlockFence then
267 # Skip the starting fence
270 # Account a standard 4 space indentation
274 # We want executable code
275 if not (ast
isa AModule or ast
isa ABlockExpr or ast
isa AExpr) then
278 # Get real location of the node (or error)
279 var location
= new Location(mdoc
.location
.file
,
280 l
.line_start
+ line_offset
,
281 l
.line_end
+ line_offset
,
282 l
.column_start
+ column_offset
,
283 l
.column_end
+ column_offset
)
284 if ast
isa AError then
285 message
= ast
.message
287 message
= "Error: Invalid Nit code."
290 executor
.toolcontext
.warning
(location
, "invalid-block", "{message} To suppress this message, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).")
291 executor
.toolcontext
.modelbuilder
.failed_entities
+= 1
295 du
.error
= "{location}: {message}"
299 # Create a first block
300 # Or create a new block for modules that are more than a main part
301 var last_docunit
= executor
.last_docunit
302 if last_docunit
== null or ast
isa AModule then
303 last_docunit
= new_docunit
304 executor
.last_docunit
= last_docunit
308 last_docunit
.block
+= code
310 # In order to retrieve precise positions,
311 # the real position of each line of the raw_content is stored.
312 # See `DocUnit::real_location`
313 line_offset
-= loc
.line_start
- 1
314 for i
in [loc
.line_start
..loc
.line_end
] do
315 last_docunit
.lines
.add i
+ line_offset
316 last_docunit
.columns
.add column_offset
320 # Return and register a new empty docunit
321 fun new_docunit
: DocUnit
323 var mdoc
= executor
.mdoc
327 var name
= executor
.xml_name
328 if executor
.docunits
.not_empty
and executor
.docunits
.last
.mdoc
== mdoc
then
329 next_number
= executor
.docunits
.last
.number
+ 1
330 name
+= "+" + next_number
.to_s
333 var res
= new DocUnit(mdoc
, next_number
, "", executor
.xml_classname
, name
)
334 executor
.docunits
.add res
335 executor
.toolcontext
.modelbuilder
.unit_entities
+= 1
340 # A unit-test extracted from some documentation.
342 # A docunit is extracted from the code-blocks of mdocs.
343 # Each mdoc can contains more than one docunit, and a single docunit can be made of more that a single code-block.
347 # The doc that contains self
350 # The numbering of self in mdoc (starting with 0)
353 # The name of the unit to show in messages
354 fun full_name
: String do
355 var mentity
= mdoc
.original_mentity
356 if mentity
!= null then return mentity
.full_name
357 return xml_classname
+ "." + xml_name
360 # The text of the code to execute.
362 # This is the verbatim content on one, or more, code-blocks from `mdoc`
365 # For each line in `block`, the associated line in the mdoc
367 # Is used to give precise locations
368 var lines
= new Array[Int]
370 # For each line in `block`, the associated column in the mdoc
372 # Is used to give precise locations
373 var columns
= new Array[Int]
375 # The location of the whole docunit.
377 # If `self` is made of multiple code-blocks, then the location
378 # starts at the first code-books and finish at the last one, thus includes anything between.
379 var location
: Location is lazy
do
380 return new Location(mdoc
.location
.file
, lines
.first
, lines
.last
+1, columns
.first
+1, 0)
383 # Compute the real location of a node on the `ast` based on `mdoc.location`
385 # The result is basically: ast_location + markdown location of the piece + mdoc.location
387 # The fun is that a single docunit can be made of various pieces of code blocks.
388 fun real_location
(ast_location
: Location): Location
391 var res
= new Location(mdoc
.location
.file
, lines
[ast_location
.line_start-1
],
392 lines
[ast_location
.line_end-1
],
393 columns
[ast_location
.line_start-1
] + ast_location
.column_start
,
394 columns
[ast_location
.line_end-1
] + ast_location
.column_end
)
401 res
.open
("system-out").append
(block
)
405 redef var xml_classname
409 redef class ModelBuilder
410 # Total number analyzed `MEntity`
411 var total_entities
= 0
413 # The number of `MEntity` that have some documentation
416 # The total number of executed docunits
417 var unit_entities
= 0
419 # The number failed docunits
420 var failed_entities
= 0
422 # Extracts and executes all the docunits in the `mmodule`
423 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
424 fun test_markdown
(mmodule
: MModule): HTMLTag
426 var ts
= new HTMLTag("testsuite")
427 toolcontext
.info
("nitunit: doc-unit {mmodule}", 2)
429 var nmodule
= mmodule2node
(mmodule
)
430 if nmodule
== null then return ts
432 # usualy, only the original module must be imported in the unit test.
435 if g
!= null and g
.mpackage
.name
== "core" then
436 # except for a unit test in a module of `core`
437 # in this case, the whole `core` must be imported
438 o
= get_mmodule_by_name
(nmodule
, g
, g
.mpackage
.name
).as(not null)
441 ts
.attr
("package", mmodule
.full_name
)
443 var prefix
= toolcontext
.test_dir
444 prefix
= prefix
.join_path
(mmodule
.to_s
)
445 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
449 var nmoduledecl
= nmodule
.n_moduledecl
450 if nmoduledecl
== null then break label x
451 var ndoc
= nmoduledecl
.n_doc
452 if ndoc
== null then break label x
454 # NOTE: jenkins expects a '.' in the classname attr
455 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + mmodule
.full_name
+ ".<module>", "<module>")
457 for nclassdef
in nmodule
.n_classdefs
do
458 var mclassdef
= nclassdef
.mclassdef
459 if mclassdef
== null then continue
460 if nclassdef
isa AStdClassdef then
462 var ndoc
= nclassdef
.n_doc
465 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
, "<class>")
468 for npropdef
in nclassdef
.n_propdefs
do
469 var mpropdef
= npropdef
.mpropdef
470 if mpropdef
== null then continue
472 var ndoc
= npropdef
.n_doc
475 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + mmodule
.full_name
+ "." + mclassdef
.mclass
.full_name
, mpropdef
.mproperty
.full_name
)
485 # Extracts and executes all the docunits in the readme of the `mgroup`
486 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
487 fun test_group
(mgroup
: MGroup): HTMLTag
489 var ts
= new HTMLTag("testsuite")
490 toolcontext
.info
("nitunit: doc-unit group {mgroup}", 2)
492 # usually, only the default module must be imported in the unit test.
493 var o
= mgroup
.default_mmodule
495 ts
.attr
("package", mgroup
.full_name
)
497 var prefix
= toolcontext
.test_dir
498 prefix
= prefix
.join_path
(mgroup
.to_s
)
499 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
)
502 var mdoc
= mgroup
.mdoc
503 if mdoc
== null then return ts
506 # NOTE: jenkins expects a '.' in the classname attr
507 d2m
.extract
(mdoc
, "nitunit." + mgroup
.full_name
, "<group>")
514 # Test a document object unrelated to a Nit entity
515 fun test_mdoc
(mdoc
: MDoc): HTMLTag
517 var ts
= new HTMLTag("testsuite")
518 var file
= mdoc
.location
.to_s
520 toolcontext
.info
("nitunit: doc-unit file {file}", 2)
522 ts
.attr
("package", file
)
524 var prefix
= toolcontext
.test_dir
/ "file"
525 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, null, ts
)
530 # NOTE: jenkins expects a '.' in the classname attr
531 d2m
.extract
(mdoc
, "nitunit.<file>", file
)