# Testing from code comments.
module testing_doc
+private import parser_util
import testing_base
-intrude import markdown
+import markdown
+import html
# Extractor, Executor and Reporter for the tests in a module
class NitUnitExecutor
- super Doc2Mdwn
+ super HTMLDecorator
+
+ # Toolcontext used to parse Nit code blocks.
+ var toolcontext: ToolContext
# The prefix of the generated Nit source-file
var prefix: String
- # The module to import
- var mmodule: MModule
+ # The module to import, if any
+ var mmodule: nullable MModule
# The XML node associated to the module
var testsuite: HTMLTag
- # All blocks of code from a same `ADoc`
- var blocks = new Array[Array[String]]
-
- redef fun process_code(n: HTMLTag, text: String)
- do
- # Try to parse it
- var ast = toolcontext.parse_something(text)
-
- # We want executable code
- if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
- if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
- toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of code that is not valid Nit, thus not considered a nitunit")
- if ast isa AError then toolcontext.warning(ast.location, "syntax-error", ast.message)
- ndoc = null # To avoid multiple warning in the same node
- end
- return
- end
-
- # Search `assert` in the AST
- var v = new SearchAssertVisitor
- v.enter_visit(ast)
- if not v.foundit then
- if ndoc != null and n.tag == "pre" and toolcontext.opt_warn.value > 1 then
- toolcontext.warning(ndoc.location, "invalid-block", "Warning: There is a block of Nit code without `assert`, thus not considered a nitunit")
- ndoc = null # To avoid multiple warning in the same node
- end
- return
- end
+ # The name of the suite
+ var name: String
- # Create a first block
- # Or create a new block for modules that are more than a main part
- if blocks.is_empty or ast isa AModule then
- blocks.add(new Array[String])
- end
+ # Markdown processor used to parse markdown comments and extract code.
+ var mdproc = new MarkdownProcessor
- # Add it to the file
- blocks.last.add(text)
+ init do
+ mdproc.emitter.decorator = new NitunitDecorator(self)
end
- # The associated node to localize warnings
- var ndoc: nullable ADoc = null
+ # The associated documentation object
+ var mdoc: nullable MDoc = null
# used to generate distinct names
var cpt = 0
+ # The last docunit extracted from a mdoc.
+ #
+ # Is used because a new code-block might just be added to it.
+ var last_docunit: nullable DocUnit = null
+
+ var xml_classname: String is noautoinit
+
+ var xml_name: String is noautoinit
+
# The entry point for a new `ndoc` node
- # Fill the prepated `tc` (testcase) XTM node
- fun extract(ndoc: ADoc, tc: HTMLTag)
+ # Fill `docunits` with new discovered unit of tests.
+ fun extract(mdoc: MDoc, xml_classname, xml_name: String)
do
- blocks.clear
+ last_docunit = null
+ self.xml_classname = xml_classname
+ self.xml_name = xml_name
- self.ndoc = ndoc
+ self.mdoc = mdoc
- work(ndoc.to_mdoc)
- toolcontext.check_errors
+ # Populate `blocks` from the markdown decorator
+ mdproc.process(mdoc.content.join("\n"))
+ end
- if blocks.is_empty then return
+ # All extracted docunits
+ var docunits = new Array[DocUnit]
- for block in blocks do test_block(ndoc, tc, block)
+ fun show_status
+ do
+ toolcontext.show_unit_status(name, docunits)
end
- # Execute a block
- fun test_block(ndoc: ADoc, tc: HTMLTag, block: Array[String])
+ fun mark_done(du: DocUnit)
do
- toolcontext.modelbuilder.unit_entities += 1
+ du.is_done = true
+ toolcontext.clear_progress_bar
+ toolcontext.show_unit(du)
+ show_status
+ end
- cpt += 1
- var file = "{prefix}{cpt}.nit"
+ # Execute all the docunits
+ fun run_tests
+ do
+ if docunits.is_empty then
+ return
+ end
+
+ # Try to group each nitunit into a single source file to fasten the compilation
+ var simple_du = new Array[DocUnit]
+ show_status
+ for du in docunits do
+ # Skip existing errors
+ if du.error != null then
+ continue
+ end
+
+ var ast = toolcontext.parse_something(du.block)
+ if ast isa AExpr then
+ simple_du.add du
+ end
+ end
+ test_simple_docunits(simple_du)
+
+ # Now test them in order
+ for du in docunits do
+ if du.error != null then
+ # Nothing to execute. Conclude
+ else if du.test_file != null then
+ # Already compiled. Execute it.
+ execute_simple_docunit(du)
+ else
+ # Need to try to compile it, then execute it
+ test_single_docunit(du)
+ end
+ mark_done(du)
+ end
+
+ # Final status
+ show_status
+ print ""
+
+ for du in docunits do
+ testsuite.add du.to_xml
+ end
+ end
- toolcontext.info("Execute doc-unit {tc.attrs["name"]} in {file}", 1)
+ # Executes multiples doc-units in a shared program.
+ # Used for docunits simple block of code (without modules, classes, functions etc.)
+ #
+ # In case of compilation error, the method fallbacks to `test_single_docunit` to
+ # * locate exactly the compilation problem in the problematic docunit.
+ # * permit the execution of the other docunits that may be correct.
+ fun test_simple_docunits(dus: Array[DocUnit])
+ do
+ if dus.is_empty then return
+
+ var file = "{prefix}-0.nit"
var dir = file.dirname
if dir != "" then dir.mkdir
var f
- f = new OFStream.open(file)
- f.write("# GENERATED FILE\n")
- f.write("# Example extracted from a documentation\n")
- f.write("import {mmodule.name}\n")
- f.write("\n")
- for text in block do
- f.write(text)
+ f = create_unitfile(file)
+ var i = 0
+ for du in dus do
+
+ i += 1
+ f.write("fun run_{i} do\n")
+ f.write("# {du.full_name}\n")
+ f.write(du.block)
+ f.write("end\n")
+ end
+ f.write("var a = args.first.to_i\n")
+ for j in [1..i] do
+ f.write("if a == {j} then run_{j}\n")
end
f.close
if toolcontext.opt_noact.value then return
- var nit_dir = toolcontext.nit_dir
- var nitg = "{nit_dir or else ""}/bin/nitg"
- if nit_dir == null or not nitg.file_exists then
- toolcontext.error(null, "Cannot find nitg. Set envvar NIT_DIR.")
- toolcontext.check_errors
+ var res = compile_unitfile(file)
+
+ if res != 0 then
+ # Compilation error.
+ # They will be executed independently
+ return
+ end
+
+ # Compilation was a success.
+ # Store what need to be executed for each one.
+ i = 0
+ for du in dus do
+ i += 1
+ du.test_file = file
+ du.test_arg = i
end
- var cmd = "{nitg} --ignore-visibility --no-color '{file}' -I {mmodule.location.file.filename.dirname} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
- var res = sys.system(cmd)
- var res2 = 0
- if res == 0 then
- res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
+ end
+
+ # Execute a docunit compiled by `test_single_docunit`
+ fun execute_simple_docunit(du: DocUnit)
+ do
+ var file = du.test_file.as(not null)
+ var i = du.test_arg.as(not null)
+ toolcontext.info("Execute doc-unit {du.full_name} in {file} {i}", 1)
+ var res2 = toolcontext.safe_exec("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
+ du.was_exec = true
+
+ var content = "{file}.out1".to_path.read_all
+ du.raw_output = content
+
+ if res2 != 0 then
+ du.error = "Runtime error in {file} with argument {i}"
+ toolcontext.modelbuilder.failed_entities += 1
end
+ end
- var msg
- f = new IFStream.open("{file}.out1")
- var n2
- n2 = new HTMLTag("system-err")
- tc.add n2
- msg = f.read_all
+ # Executes a single doc-unit in its own program.
+ # Used for docunits larger than a single block of code (with modules, classes, functions etc.)
+ fun test_single_docunit(du: DocUnit)
+ do
+ cpt += 1
+ var file = "{prefix}-{cpt}.nit"
+
+ toolcontext.info("Execute doc-unit {du.full_name} in {file}", 1)
+
+ var f
+ f = create_unitfile(file)
+ f.write(du.block)
f.close
- n2 = new HTMLTag("system-out")
- tc.add n2
- for text in block do n2.append(text)
+ if toolcontext.opt_noact.value then return
+
+ var res = compile_unitfile(file)
+ var res2 = 0
+ if res == 0 then
+ res2 = toolcontext.safe_exec("{file.to_program_name}.bin >'{file}.out1' 2>&1 </dev/null")
+ du.was_exec = true
+ end
+ var content = "{file}.out1".to_path.read_all
+ du.raw_output = content
if res != 0 then
- var ne = new HTMLTag("failure")
- ne.attr("message", msg)
- tc.add ne
- toolcontext.warning(ndoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
+ du.error = "Compilation error in {file}"
toolcontext.modelbuilder.failed_entities += 1
else if res2 != 0 then
- var ne = new HTMLTag("error")
- ne.attr("message", msg)
- tc.add ne
- toolcontext.warning(ndoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
+ du.error = "Runtime error in {file}"
toolcontext.modelbuilder.failed_entities += 1
end
- toolcontext.check_errors
+ end
+
+ # Create and fill the header of a unit file `file`.
+ #
+ # A unit file is a Nit source file generated from one
+ # or more docunits that will be compiled and executed.
+ #
+ # The handled on the file is returned and must be completed and closed.
+ #
+ # `file` should be a valid filepath for a Nit source file.
+ private fun create_unitfile(file: String): Writer
+ do
+ var dir = file.dirname
+ if dir != "" then dir.mkdir
+ var f
+ f = new FileWriter.open(file)
+ f.write("# GENERATED FILE\n")
+ f.write("# Docunits extracted from comments\n")
+ if mmodule != null then
+ f.write("import {mmodule.name}\n")
+ end
+ f.write("\n")
+ return f
+ end
- testsuite.add(tc)
+ # Compile an unit file and return the compiler return code
+ #
+ # Can terminate the program if the compiler is not found
+ private fun compile_unitfile(file: String): Int
+ do
+ var nitc = toolcontext.find_nitc
+ var opts = new Array[String]
+ if mmodule != null then
+ opts.add "-I {mmodule.filepath.dirname}"
+ end
+ var cmd = "{nitc} --ignore-visibility --no-color -q '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
+ var res = toolcontext.safe_exec(cmd)
+ return res
end
end
-class SearchAssertVisitor
- super Visitor
- var foundit = false
- redef fun visit(node)
- do
- if foundit then
- return
- else if node isa AAssertExpr then
- foundit = true
+private class NitunitDecorator
+ super HTMLDecorator
+
+ var executor: NitUnitExecutor
+
+ redef fun add_code(v, block) do
+ var code = block.raw_content
+ var meta = block.meta or else "nit"
+ # Do not try to test non-nit code.
+ if meta != "nit" then return
+ # Try to parse code blocks
+ var ast = executor.toolcontext.parse_something(code)
+
+ var mdoc = executor.mdoc
+ assert mdoc != null
+
+ # Skip pure comments
+ if ast isa TComment then return
+
+ # The location is computed according to the starts of the mdoc and the block
+ # Note, the following assumes that all the comments of the mdoc are correctly aligned.
+ var loc = block.block.location
+ var line_offset = loc.line_start + mdoc.location.line_start - 2
+ var column_offset = loc.column_start + mdoc.location.column_start
+ # Hack to handle precise location in blocks
+ # TODO remove when markdown is more reliable
+ if block isa BlockFence then
+ # Skip the starting fence
+ line_offset += 1
+ else
+ # Account a standard 4 space indentation
+ column_offset += 4
+ end
+
+ # We want executable code
+ if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
+ var message
+ var l = ast.location
+ # Get real location of the node (or error)
+ var location = new Location(mdoc.location.file,
+ l.line_start + line_offset,
+ l.line_end + line_offset,
+ l.column_start + column_offset,
+ l.column_end + column_offset)
+ if ast isa AError then
+ message = ast.message
+ else
+ message = "Error: Invalid Nit code."
+ end
+
+ var du = new_docunit
+ du.block += code
+ du.error_location = location
+ du.error = message
+ executor.toolcontext.modelbuilder.failed_entities += 1
return
+ end
+
+ # Create a first block
+ # Or create a new block for modules that are more than a main part
+ var last_docunit = executor.last_docunit
+ if last_docunit == null or ast isa AModule then
+ last_docunit = new_docunit
+ executor.last_docunit = last_docunit
+ end
+
+ # Add it to the file
+ last_docunit.block += code
+
+ # In order to retrieve precise positions,
+ # the real position of each line of the raw_content is stored.
+ # See `DocUnit::real_location`
+ line_offset -= loc.line_start - 1
+ for i in [loc.line_start..loc.line_end] do
+ last_docunit.lines.add i + line_offset
+ last_docunit.columns.add column_offset
+ end
+ end
+
+ # Return and register a new empty docunit
+ fun new_docunit: DocUnit
+ do
+ var mdoc = executor.mdoc
+ assert mdoc != null
+
+ var next_number = 1
+ var name = executor.xml_name
+ if executor.docunits.not_empty and executor.docunits.last.mdoc == mdoc then
+ next_number = executor.docunits.last.number + 1
+ name += "#" + next_number.to_s
+ end
+
+ var res = new DocUnit(mdoc, next_number, "", executor.xml_classname, name)
+ executor.docunits.add res
+ executor.toolcontext.modelbuilder.unit_entities += 1
+ return res
+ end
+end
+
+# A unit-test extracted from some documentation.
+#
+# A docunit is extracted from the code-blocks of mdocs.
+# Each mdoc can contains more than one docunit, and a single docunit can be made of more that a single code-block.
+class DocUnit
+ super UnitTest
+
+ # The doc that contains self
+ var mdoc: MDoc
+
+ # The numbering of self in mdoc (starting with 0)
+ var number: Int
+
+ # The generated Nit source file that contains the unit-test
+ #
+ # Note that a same generated file can be used for multiple tests.
+ # See `test_arg` that is used to distinguish them
+ var test_file: nullable String = null
+
+ # The command-line argument to use when executing the test, if any.
+ var test_arg: nullable Int = null
+
+ redef fun full_name do
+ var mentity = mdoc.original_mentity
+ if mentity != null then
+ var res = mentity.full_name
+ if number > 1 then
+ res += "#{number}"
+ end
+ return res
else
- node.visit_all(self)
+ return xml_classname + "." + xml_name
end
end
+
+ # The text of the code to execute.
+ #
+ # This is the verbatim content on one, or more, code-blocks from `mdoc`
+ var block: String
+
+ # For each line in `block`, the associated line in the mdoc
+ #
+ # Is used to give precise locations
+ var lines = new Array[Int]
+
+ # For each line in `block`, the associated column in the mdoc
+ #
+ # Is used to give precise locations
+ var columns = new Array[Int]
+
+ # The location of the whole docunit.
+ #
+ # If `self` is made of multiple code-blocks, then the location
+ # starts at the first code-books and finish at the last one, thus includes anything between.
+ redef var location is lazy do
+ return new Location(mdoc.location.file, lines.first, lines.last+1, columns.first+1, 0)
+ end
+
+ # Compute the real location of a node on the `ast` based on `mdoc.location`
+ #
+ # The result is basically: ast_location + markdown location of the piece + mdoc.location
+ #
+ # The fun is that a single docunit can be made of various pieces of code blocks.
+ fun real_location(ast_location: Location): Location
+ do
+ var mdoc = self.mdoc
+ var res = new Location(mdoc.location.file, lines[ast_location.line_start-1],
+ lines[ast_location.line_end-1],
+ columns[ast_location.line_start-1] + ast_location.column_start,
+ columns[ast_location.line_end-1] + ast_location.column_end)
+ return res
+ end
+
+ redef fun to_xml
+ do
+ var res = super
+ res.open("system-out").append(block)
+ return res
+ end
+
+ redef var xml_classname
+ redef var xml_name
end
redef class ModelBuilder
+ # Total number analyzed `MEntity`
var total_entities = 0
+
+ # The number of `MEntity` that have some documentation
var doc_entities = 0
+
+ # The total number of executed docunits
var unit_entities = 0
+
+ # The number failed docunits
var failed_entities = 0
+ # Extracts and executes all the docunits in the `mmodule`
+ # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
fun test_markdown(mmodule: MModule): HTMLTag
do
var ts = new HTMLTag("testsuite")
toolcontext.info("nitunit: doc-unit {mmodule}", 2)
- if not mmodule2nmodule.has_key(mmodule) then return ts
- var nmodule = mmodule2nmodule[mmodule]
+ var nmodule = mmodule2node(mmodule)
+ if nmodule == null then return ts
# usualy, only the original module must be imported in the unit test.
var o = mmodule
var g = o.mgroup
- if g != null and g.mproject.name == "standard" then
- # except for a unit test in a module of standard
- # in this case, the whole standard must be imported
- o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
+ if g != null and g.mpackage.name == "core" then
+ # except for a unit test in a module of `core`
+ # in this case, the whole `core` must be imported
+ o = get_mmodule_by_name(nmodule, g, g.mpackage.name).as(not null)
end
ts.attr("package", mmodule.full_name)
var prefix = toolcontext.test_dir
prefix = prefix.join_path(mmodule.to_s)
- var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts)
-
- var tc
+ var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts, "Docunits of module {mmodule.full_name}")
do
total_entities += 1
var ndoc = nmoduledecl.n_doc
if ndoc == null then break label x
doc_entities += 1
- tc = new HTMLTag("testcase")
# NOTE: jenkins expects a '.' in the classname attr
- tc.attr("classname", "nitunit." + mmodule.full_name + ".<module>")
- tc.attr("name", "<module>")
- d2m.extract(ndoc, tc)
+ d2m.extract(ndoc.to_mdoc, "nitunit." + mmodule.full_name + ".<module>", "<module>")
end label x
for nclassdef in nmodule.n_classdefs do
var mclassdef = nclassdef.mclassdef
var ndoc = nclassdef.n_doc
if ndoc != null then
doc_entities += 1
- tc = new HTMLTag("testcase")
- tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
- tc.attr("name", "<class>")
- d2m.extract(ndoc, tc)
+ d2m.extract(ndoc.to_mdoc, "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name, "<class>")
end
end
for npropdef in nclassdef.n_propdefs do
var ndoc = npropdef.n_doc
if ndoc != null then
doc_entities += 1
- tc = new HTMLTag("testcase")
- tc.attr("classname", "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name)
- tc.attr("name", mpropdef.mproperty.full_name)
- d2m.extract(ndoc, tc)
+ d2m.extract(ndoc.to_mdoc, "nitunit." + mmodule.full_name + "." + mclassdef.mclass.full_name, mpropdef.mproperty.full_name)
end
end
end
+ d2m.run_tests
+
+ return ts
+ end
+
+ # Extracts and executes all the docunits in the readme of the `mgroup`
+ # Returns a JUnit-compatible `<testsuite>` XML element that contains the results of the executions.
+ fun test_group(mgroup: MGroup): HTMLTag
+ do
+ var ts = new HTMLTag("testsuite")
+ toolcontext.info("nitunit: doc-unit group {mgroup}", 2)
+
+ # usually, only the default module must be imported in the unit test.
+ var o = mgroup.default_mmodule
+
+ ts.attr("package", mgroup.full_name)
+
+ var prefix = toolcontext.test_dir
+ prefix = prefix.join_path(mgroup.to_s)
+ var d2m = new NitUnitExecutor(toolcontext, prefix, o, ts, "Docunits of group {mgroup.full_name}")
+
+ total_entities += 1
+ var mdoc = mgroup.mdoc
+ if mdoc == null then return ts
+
+ doc_entities += 1
+ # NOTE: jenkins expects a '.' in the classname attr
+ d2m.extract(mdoc, "nitunit." + mgroup.full_name, "<group>")
+
+ d2m.run_tests
+
+ return ts
+ end
+
+ # Test a document object unrelated to a Nit entity
+ fun test_mdoc(mdoc: MDoc): HTMLTag
+ do
+ var ts = new HTMLTag("testsuite")
+ var file = mdoc.location.file.filename
+
+ toolcontext.info("nitunit: doc-unit file {file}", 2)
+
+ ts.attr("package", file)
+
+ var prefix = toolcontext.test_dir / "file"
+ var d2m = new NitUnitExecutor(toolcontext, prefix, null, ts, "Docunits of file {file}")
+
+ total_entities += 1
+ doc_entities += 1
+
+ # NOTE: jenkins expects a '.' in the classname attr
+ d2m.extract(mdoc, "nitunit.<file>", file)
+ d2m.run_tests
+
return ts
end
end
nitlanguage / nit.git / blobdiff