1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Testing from code comments.
18 private import parser_util
24 # Extractor, Executor and Reporter for the tests in a module
27 # Toolcontext used to parse Nit code blocks.
28 var toolcontext
: ToolContext
30 # The prefix of the generated Nit source-file
33 # The module to import, if any
34 var mmodule
: nullable MModule
36 # The XML node associated to the module
37 var testsuite
: HTMLTag
39 # The name of the suite
42 # Markdown parse used to parse markdown comments and extract code
43 private var md_parser
= new MdParser
45 # Markdown visitor used to extract markdown code blocks
46 private var md_visitor
= new NitunitMdVisitor(self) is lazy
48 # The associated documentation object
49 var mdoc
: nullable MDoc = null
51 # used to generate distinct names
54 # The last docunit extracted from a mdoc.
56 # Is used because a new code-block might just be added to it.
57 var last_docunit
: nullable DocUnit = null
59 # Unit class name in XML output
60 var xml_classname
: String is noautoinit
62 # Unit name in xml output
63 var xml_name
: String is noautoinit
65 # The entry point for a new `ndoc` node
66 # Fill `docunits` with new discovered unit of tests.
67 fun extract
(mdoc
: MDoc, xml_classname
, xml_name
: String)
70 self.xml_classname
= xml_classname
71 self.xml_name
= xml_name
75 # Populate `blocks` from the markdown decorator
76 var md_node
= md_parser
.parse
(mdoc
.content
.join
("\n"))
77 md_visitor
.enter_visit
(md_node
)
80 # All extracted docunits
81 var docunits
= new Array[DocUnit]
83 # Display current testing status
84 fun show_status
do toolcontext
.show_unit_status
(name
, docunits
)
86 # Update display when a test case is done
87 fun mark_done
(du
: DocUnit)
90 toolcontext
.clear_progress_bar
91 toolcontext
.show_unit
(du
)
95 # Execute all the docunits
98 if docunits
.is_empty
then
102 # Try to group each nitunit into a single source file to fasten the compilation
103 var simple_du
= new Array[DocUnit] # du that are simple statements
104 var single_du
= new Array[DocUnit] # du that are modules or include classes
106 for du
in docunits
do
107 # Skip existing errors
108 if du
.error
!= null then
112 var ast
= toolcontext
.parse_something
(du
.block
)
113 if ast
isa AExpr then
120 # Try to mass compile all the simple du as a single nit module
121 compile_simple_docunits
(simple_du
)
122 # Try to mass compile all the single du in a single nitc invocation with many modules
123 compile_single_docunits
(single_du
)
124 # If the mass compilation fail, then each one will be compiled individually
126 # Now test them in order
127 for du
in docunits
do
128 if du
.error
!= null then
129 # Nothing to execute. Conclude
130 else if du
.is_compiled
then
131 # Already compiled. Execute it.
132 execute_simple_docunit
(du
)
134 # A mass compilation failed
135 # Need to try to recompile it, then execute it
136 test_single_docunit
(du
)
145 for du
in docunits
do
146 testsuite
.add du
.to_xml
150 # Compiles multiples doc-units in a shared program.
151 # Used for docunits simple block of code (without modules, classes, functions etc.)
153 # In case of success, the docunits are compiled and the caller can call `execute_simple_docunit`.
155 # In case of compilation error, the docunits are let uncompiled.
156 # The caller should fallbacks to `test_single_docunit` to
157 # * locate exactly the compilation problem in the problematic docunit.
158 # * permit the execution of the other docunits that may be correct.
159 fun compile_simple_docunits
(dus
: Array[DocUnit])
161 if dus
.is_empty
then return
163 var file
= "{prefix}-0.nit"
165 toolcontext
.info
("Compile {dus.length} simple(s) doc-unit(s) in {file}", 1)
167 var dir
= file
.dirname
168 if dir
!= "" then dir
.mkdir
170 f
= create_unitfile
(file
)
174 f
.write
("fun run_{i} do\n")
175 f
.write
("# {du.full_name}\n")
179 f
.write
("var a = args.first.to_i\n")
181 f
.write
("if a == {j} then run_{j}\n")
185 if toolcontext
.opt_noact
.value
then return
187 var res
= compile_unitfile
(file
)
191 # They should be generated and compiled independently
195 # Compilation was a success.
196 # Store what need to be executed for each one.
202 du
.is_compiled
= true
206 # Execute a docunit compiled by `test_single_docunit`
207 fun execute_simple_docunit
(du
: DocUnit)
209 var file
= du
.test_file
.as(not null)
210 var i
= du
.test_arg
or else 0
211 toolcontext
.info
("Execute doc-unit {du.full_name} in {file} {i}", 1)
212 var clock
= new Clock
213 var res2
= toolcontext
.safe_exec
("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
214 if not toolcontext
.opt_no_time
.value
then du
.real_time
= clock
.total
217 var content
= "{file}.out1".to_path
.read_all
218 du
.raw_output
= content
221 du
.error
= "Runtime error in {file} with argument {i}"
222 toolcontext
.modelbuilder
.failed_entities
+= 1
226 # Produce a single unit file for the docunit `du`.
227 fun generate_single_docunit
(du
: DocUnit): String
230 var file
= "{prefix}-{cpt}.nit"
233 f
= create_unitfile
(file
)
241 # Executes a single doc-unit in its own program.
242 # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
243 fun test_single_docunit
(du
: DocUnit)
245 var file
= generate_single_docunit
(du
)
247 toolcontext
.info
("Compile doc-unit {du.full_name} in {file}", 1)
249 if toolcontext
.opt_noact
.value
then return
251 var res
= compile_unitfile
(file
)
252 var content
= "{file}.out1".to_path
.read_all
253 du
.raw_output
= content
258 du
.error
= "Compilation error in {file}"
259 toolcontext
.modelbuilder
.failed_entities
+= 1
263 du
.is_compiled
= true
264 execute_simple_docunit
(du
)
267 # Create and fill the header of a unit file `file`.
269 # A unit file is a Nit source file generated from one
270 # or more docunits that will be compiled and executed.
272 # The handled on the file is returned and must be completed and closed.
274 # `file` should be a valid filepath for a Nit source file.
275 private fun create_unitfile
(file
: String): Writer
277 var mmodule
= self.mmodule
278 var dir
= file
.dirname
279 if dir
!= "" then dir
.mkdir
281 f
= new FileWriter.open
(file
)
282 f
.write
("# GENERATED FILE\n")
283 f
.write
("# Docunits extracted from comments\n")
284 if mmodule
!= null then
285 f
.write
("intrude import {mmodule.name}\n")
291 # Compile a unit file and return the compiler return code
293 # Can terminate the program if the compiler is not found
294 private fun compile_unitfile
(file
: String): Int
296 var mmodule
= self.mmodule
297 var nitc
= toolcontext
.find_nitc
298 var opts
= new Array[String]
299 if mmodule
!= null then
300 # FIXME playing this way with the include dir is not safe nor robust
301 opts
.add
"-I {mmodule.filepath.as(not null).dirname}"
303 var cmd
= "{nitc} --ignore-visibility --no-color -q '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
304 var res
= toolcontext
.safe_exec
(cmd
)
308 # Compile a unit file and return the compiler return code
310 # Can terminate the program if the compiler is not found
311 private fun compile_single_docunits
(dus
: Array[DocUnit]): Int
313 # Generate all unitfiles
314 var files
= new Array[String]
316 files
.add generate_single_docunit
(du
)
319 if files
.is_empty
then return 0
321 toolcontext
.info
("Compile {dus.length} single(s) doc-unit(s) at once", 1)
324 var nitc
= toolcontext
.find_nitc
325 var opts
= new Array[String]
326 if mmodule
!= null then
327 # FIXME playing this way with the include dir is not safe nor robust
328 opts
.add
"-I {mmodule.filepath.dirname}"
330 var cmd
= "{nitc} --ignore-visibility --no-color -q '{files.join("' '")}' {opts.join(" ")} > '{prefix}.out1' 2>&1 </dev/null --dir {prefix.dirname}"
331 var res
= toolcontext
.safe_exec
(cmd
)
333 # Mass compilation failure
337 # Rename each file into it expected binary name
339 var f
= du
.test_file
.as(not null)
340 toolcontext
.safe_exec
("mv '{f.strip_extension(".nit")}' '{f}.bin'")
341 du
.is_compiled
= true
348 private class NitunitMdVisitor
351 var executor
: NitUnitExecutor
353 redef fun visit
(node
) do node
.accept_nitunit
(self)
355 fun parse_code
(block
: MdCodeBlock) do
356 var code
= block
.literal
357 if code
== null then return
359 var meta
= block
.info
or else "nit"
360 # Do not try to test non-nit code.
361 if meta
!= "nit" then return
363 # Try to parse code blocks
364 var executor
= self.executor
365 var ast
= executor
.toolcontext
.parse_something
(code
)
367 var mdoc
= executor
.mdoc
371 if ast
isa TComment then return
373 # The location is computed according to the starts of the mdoc and the block
374 # Note, the following assumes that all the comments of the mdoc are correctly aligned.
375 var loc
= block
.location
376 var line_offset
= loc
.line_start
+ mdoc
.location
.line_start
- 2
377 var column_offset
= loc
.column_start
+ mdoc
.location
.column_start
378 # Hack to handle precise location in blocks
379 # TODO remove when markdown is more reliable
380 if block
isa MdFencedCodeBlock then
381 # Skip the starting fence
384 # Account a standard 4 space indentation
388 # We want executable code
389 if not (ast
isa AModule or ast
isa ABlockExpr or ast
isa AExpr) then
392 # Get real location of the node (or error)
393 var location
= new Location(mdoc
.location
.file
,
394 l
.line_start
+ line_offset
,
395 l
.line_end
+ line_offset
,
396 l
.column_start
+ column_offset
,
397 l
.column_end
+ column_offset
)
398 if ast
isa AError then
399 message
= ast
.message
401 message
= "Error: Invalid Nit code."
406 du
.error_location
= location
408 executor
.toolcontext
.modelbuilder
.failed_entities
+= 1
412 # Create a first block
413 # Or create a new block for modules that are more than a main part
414 var last_docunit
= executor
.last_docunit
415 if last_docunit
== null or ast
isa AModule then
416 last_docunit
= new_docunit
417 executor
.last_docunit
= last_docunit
421 last_docunit
.block
+= code
423 # In order to retrieve precise positions,
424 # the real position of each line of the raw_content is stored.
425 # See `DocUnit::real_location`
426 line_offset
-= loc
.line_start
- 1
427 for i
in [loc
.line_start
..loc
.line_end
] do
428 last_docunit
.lines
.add i
+ line_offset
429 last_docunit
.columns
.add column_offset
433 # Return and register a new empty docunit
434 fun new_docunit
: DocUnit do
435 var mdoc
= executor
.mdoc
439 var name
= executor
.xml_name
440 if executor
.docunits
.not_empty
and executor
.docunits
.last
.mdoc
== mdoc
then
441 next_number
= executor
.docunits
.last
.number
+ 1
442 name
+= "#" + next_number
.to_s
445 var res
= new DocUnit(mdoc
, next_number
, "", executor
.xml_classname
, name
)
446 executor
.docunits
.add res
447 executor
.toolcontext
.modelbuilder
.unit_entities
+= 1
453 private fun accept_nitunit
(v
: NitunitMdVisitor) do visit_all
(v
)
456 redef class MdCodeBlock
457 redef fun accept_nitunit
(v
) do v
.parse_code
(self)
460 # A unit-test extracted from some documentation.
462 # A docunit is extracted from the code-blocks of mdocs.
463 # Each mdoc can contains more than one docunit, and a single docunit can be made of more that a single code-block.
467 # The doc that contains self
470 # The numbering of self in mdoc (starting with 0)
473 # The generated Nit source file that contains the unit-test
475 # Note that a same generated file can be used for multiple tests.
476 # See `test_arg` that is used to distinguish them
477 var test_file
: nullable String = null
479 # Was `test_file` successfully compiled?
480 var is_compiled
= false
482 # The command-line argument to use when executing the test, if any.
483 var test_arg
: nullable Int = null
485 redef fun full_name
do
486 var mentity
= mdoc
.original_mentity
487 if mentity
!= null then
488 var res
= mentity
.full_name
494 return xml_classname
+ "." + xml_name
498 # The text of the code to execute.
500 # This is the verbatim content on one, or more, code-blocks from `mdoc`
503 # For each line in `block`, the associated line in the mdoc
505 # Is used to give precise locations
506 var lines
= new Array[Int]
508 # For each line in `block`, the associated column in the mdoc
510 # Is used to give precise locations
511 var columns
= new Array[Int]
513 # The location of the whole docunit.
515 # If `self` is made of multiple code-blocks, then the location
516 # starts at the first code-books and finish at the last one, thus includes anything between.
517 redef var location
is lazy
do
518 return new Location(mdoc
.location
.file
, lines
.first
, lines
.last
+1, columns
.first
+1, 0)
521 # Compute the real location of a node on the `ast` based on `mdoc.location`
523 # The result is basically: ast_location + markdown location of the piece + mdoc.location
525 # The fun is that a single docunit can be made of various pieces of code blocks.
526 fun real_location
(ast_location
: Location): Location
530 var res
= new Location(mdoc
.location
.file
,
531 lines
[ast_location
.line_start-1
],
532 lines
[ast_location
.line_end-1
],
533 columns
[ast_location
.line_start-1
] + ast_location
.column_start
,
534 columns
[ast_location
.line_end-1
] + ast_location
.column_end
)
542 res
.open
("system-out").append
(block
)
546 redef var xml_classname
550 redef class ModelBuilder
551 # Total number analyzed `MEntity`
552 var total_entities
= 0
554 # The number of `MEntity` that have some documentation
557 # The total number of executed docunits
558 var unit_entities
= 0
560 # The number failed docunits
561 var failed_entities
= 0
563 # Extracts and executes all the docunits in the `mmodule`
564 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
565 fun test_markdown
(mmodule
: MModule): HTMLTag
567 var ts
= new HTMLTag("testsuite")
568 toolcontext
.info
("nitunit: doc-unit {mmodule}", 2)
570 var nmodule
= mmodule2node
(mmodule
)
571 if nmodule
== null then return ts
573 # usualy, only the original module must be imported in the unit test.
576 if g
!= null and g
.mpackage
.name
== "core" then
577 # except for a unit test in a module of `core`
578 # in this case, the whole `core` must be imported
579 o
= get_mmodule_by_name
(nmodule
, g
, g
.mpackage
.name
).as(not null)
582 ts
.attr
("package", mmodule
.full_name
)
584 var prefix
= toolcontext
.test_dir
585 prefix
= prefix
.join_path
(mmodule
.to_s
)
586 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
, "Docunits of module {mmodule.full_name}")
590 var nmoduledecl
= nmodule
.n_moduledecl
591 if nmoduledecl
== null then break label x
592 var ndoc
= nmoduledecl
.n_doc
593 if ndoc
== null then break label x
595 # NOTE: jenkins expects a '.' in the classname attr
596 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + mmodule
.full_name
+ ".<module>", "<module>")
598 for nclassdef
in nmodule
.n_classdefs
do
599 var mclassdef
= nclassdef
.mclassdef
600 if mclassdef
== null then continue
601 if nclassdef
isa AStdClassdef then
603 var ndoc
= nclassdef
.n_doc
606 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + mclassdef
.full_name
.replace
("$", "."), "<class>")
609 for npropdef
in nclassdef
.n_propdefs
do
610 var mpropdef
= npropdef
.mpropdef
611 if mpropdef
== null then continue
613 var ndoc
= npropdef
.n_doc
616 var a
= mpropdef
.full_name
.split
("$")
617 d2m
.extract
(ndoc
.to_mdoc
, "nitunit." + a
[0] + "." + a
[1], a
[2])
627 # Extracts and executes all the docunits in the readme of the `mgroup`
628 # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
629 fun test_group
(mgroup
: MGroup): HTMLTag
631 var ts
= new HTMLTag("testsuite")
632 toolcontext
.info
("nitunit: doc-unit group {mgroup}", 2)
634 # usually, only the default module must be imported in the unit test.
635 var o
= mgroup
.default_mmodule
637 ts
.attr
("package", mgroup
.full_name
)
639 var prefix
= toolcontext
.test_dir
640 prefix
= prefix
.join_path
(mgroup
.to_s
)
641 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, o
, ts
, "Docunits of group {mgroup.full_name}")
644 var mdoc
= mgroup
.mdoc
645 if mdoc
== null then return ts
648 # NOTE: jenkins expects a '.' in the classname attr
649 d2m
.extract
(mdoc
, "nitunit." + mgroup
.mpackage
.name
+ "." + mgroup
.name
+ ".<group>", "<group>")
656 # Test a document object unrelated to a Nit entity
657 fun test_mdoc
(mdoc
: MDoc): HTMLTag
659 var ts
= new HTMLTag("testsuite")
660 var file
= mdoc
.location
.file
.as(not null).filename
662 toolcontext
.info
("nitunit: doc-unit file {file}", 2)
664 ts
.attr
("package", file
)
666 var prefix
= toolcontext
.test_dir
/ "file"
667 var d2m
= new NitUnitExecutor(toolcontext
, prefix
, null, ts
, "Docunits of file {file}")
672 # NOTE: jenkins expects a '.' in the classname attr
673 d2m
.extract
(mdoc
, "nitunit.<file>", file
)