nitunit: teach test_suite to compare outputs with a saved result file
[nit.git] / src / testing / testing_doc.nit
index 1d6096e..05c1f67 100644 (file)
@@ -76,6 +76,7 @@ class NitUnitExecutor
                                var ne = new HTMLTag("failure")
                                ne.attr("message", msg)
                                tc.add ne
+                               toolcontext.modelbuilder.unit_entities += 1
                                toolcontext.modelbuilder.failed_entities += 1
                        end
                        if blocks.is_empty then testsuite.add(tc)
@@ -156,14 +157,15 @@ class NitUnitExecutor
                        toolcontext.modelbuilder.unit_entities += 1
                        i += 1
                        toolcontext.info("Execute doc-unit {du.testcase.attrs["name"]} in {file} {i}", 1)
-                       var res2 = sys.system("{file.to_program_name}.bin {i} >>'{file}.out1' 2>&1 </dev/null")
+                       var res2 = toolcontext.safe_exec("{file.to_program_name}.bin {i} >'{file}.out1' 2>&1 </dev/null")
 
-                       var msg
                        f = new FileReader.open("{file}.out1")
                        var n2
                        n2 = new HTMLTag("system-err")
                        tc.add n2
-                       msg = f.read_all
+                       var content = f.read_all
+                       var msg = content.trunc(8192).filter_nonprintable
+                       n2.append(msg)
                        f.close
 
                        n2 = new HTMLTag("system-out")
@@ -172,9 +174,9 @@ class NitUnitExecutor
 
                        if res2 != 0 then
                                var ne = new HTMLTag("error")
-                               ne.attr("message", msg)
+                               ne.attr("message", "Runtime error")
                                tc.add ne
-                               toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
+                               toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): Runtime error\n{msg}")
                                toolcontext.modelbuilder.failed_entities += 1
                        end
                        toolcontext.check_errors
@@ -205,15 +207,16 @@ class NitUnitExecutor
                var res = compile_unitfile(file)
                var res2 = 0
                if res == 0 then
-                       res2 = sys.system("{file.to_program_name}.bin >>'{file}.out1' 2>&1 </dev/null")
+                       res2 = toolcontext.safe_exec("{file.to_program_name}.bin >'{file}.out1' 2>&1 </dev/null")
                end
 
-               var msg
                f = new FileReader.open("{file}.out1")
                var n2
                n2 = new HTMLTag("system-err")
                tc.add n2
-               msg = f.read_all
+               var content = f.read_all
+               var msg = content.trunc(8192).filter_nonprintable
+               n2.append(msg)
                f.close
 
                n2 = new HTMLTag("system-out")
@@ -223,15 +226,15 @@ class NitUnitExecutor
 
                if res != 0 then
                        var ne = new HTMLTag("failure")
-                       ne.attr("message", msg)
+                       ne.attr("message", "Compilation Error")
                        tc.add ne
-                       toolcontext.warning(du.mdoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
+                       toolcontext.warning(du.mdoc.location, "failure", "FAILURE: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
                        toolcontext.modelbuilder.failed_entities += 1
                else if res2 != 0 then
                        var ne = new HTMLTag("error")
-                       ne.attr("message", msg)
+                       ne.attr("message", "Runtime Error")
                        tc.add ne
-                       toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}): {msg}")
+                       toolcontext.warning(du.mdoc.location, "error", "ERROR: {tc.attrs["classname"]}.{tc.attrs["name"]} (in {file}):\n{msg}")
                        toolcontext.modelbuilder.failed_entities += 1
                end
                toolcontext.check_errors
@@ -267,18 +270,13 @@ class NitUnitExecutor
        # Can terminate the program if the compiler is not found
        private fun compile_unitfile(file: String): Int
        do
-               var nit_dir = toolcontext.nit_dir
-               var nitc = nit_dir/"bin/nitc"
-               if not nitc.file_exists then
-                       toolcontext.error(null, "Error: cannot find nitc. Set envvar NIT_DIR.")
-                       toolcontext.check_errors
-               end
+               var nitc = toolcontext.find_nitc
                var opts = new Array[String]
                if mmodule != null then
-                       opts.add "-I {mmodule.location.file.filename.dirname}"
+                       opts.add "-I {mmodule.filepath.dirname}"
                end
                var cmd = "{nitc} --ignore-visibility --no-color '{file}' {opts.join(" ")} >'{file}.out1' 2>&1 </dev/null -o '{file}.bin'"
-               var res = sys.system(cmd)
+               var res = toolcontext.safe_exec(cmd)
                return res
        end
 end
@@ -289,25 +287,52 @@ private class NitunitDecorator
        var executor: NitUnitExecutor
 
        redef fun add_code(v, block) do
-               var code = code_from_block(block)
-               var meta = "nit"
-               if block isa BlockFence and block.meta != null then
-                       meta = block.meta.to_s
-               end
+               var code = block.raw_content
+               var meta = block.meta or else "nit"
                # Do not try to test non-nit code.
                if meta != "nit" then return
                # Try to parse code blocks
                var ast = executor.toolcontext.parse_something(code)
 
+               var mdoc = executor.mdoc
+               assert mdoc != null
+
                # Skip pure comments
                if ast isa TComment then return
 
+               # The location is computed according to the starts of the mdoc and the block
+               # Note, the following assumes that all the comments of the mdoc are correctly aligned.
+               var loc = block.block.location
+               var line_offset = loc.line_start + mdoc.location.line_start - 2
+               var column_offset = loc.column_start + mdoc.location.column_start
+               # Hack to handle precise location in blocks
+               # TODO remove when markdown is more reliable
+               if block isa BlockFence then
+                       # Skip the starting fence
+                       line_offset += 1
+               else
+                       # Account a standard 4 space indentation
+                       column_offset += 4
+               end
+
                # We want executable code
                if not (ast isa AModule or ast isa ABlockExpr or ast isa AExpr) then
-                       var message = ""
-                       if ast isa AError then message = " At {ast.location}: {ast.message}."
-                       executor.toolcontext.warning(executor.mdoc.location, "invalid-block", "Error: there is a block of invalid Nit code, thus not considered a nitunit. To suppress this warning, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).{message}")
-                       executor.failures.add("{executor.mdoc.location}: Invalid block of code.{message}")
+                       var message
+                       var l = ast.location
+                       # Get real location of the node (or error)
+                       var location = new Location(mdoc.location.file,
+                               l.line_start + line_offset,
+                               l.line_end + line_offset,
+                               l.column_start + column_offset,
+                               l.column_end + column_offset)
+                       if ast isa AError then
+                               message = ast.message
+                       else
+                               message = "Error: Invalid Nit code."
+                       end
+
+                       executor.toolcontext.warning(location, "invalid-block", "{message} To suppress this message, enclose the block with a fence tagged `nitish` or `raw` (see `man nitdoc`).")
+                       executor.failures.add("{location}: {message}")
                        return
                end
 
@@ -320,26 +345,6 @@ private class NitunitDecorator
                # Add it to the file
                executor.blocks.last.append code
        end
-
-       # Extracts code as String from a `BlockCode`.
-       fun code_from_block(block: BlockCode): String do
-               var infence = block isa BlockFence
-               var text = new FlatBuffer
-               var line = block.block.first_line
-               while line != null do
-                       if not line.is_empty then
-                               var str = line.value
-                               if not infence and str.has_prefix("    ") then
-                                       text.append str.substring(4, str.length - line.trailing)
-                               else
-                                       text.append str
-                               end
-                       end
-                       text.append "\n"
-                       line = line.next
-               end
-               return text.write_to_string
-       end
 end
 
 # A unit-test to run
@@ -380,10 +385,10 @@ redef class ModelBuilder
                # usualy, only the original module must be imported in the unit test.
                var o = mmodule
                var g = o.mgroup
-               if g != null and g.mproject.name == "standard" then
-                       # except for a unit test in a module of standard
-                       # in this case, the whole standard must be imported
-                       o = get_mmodule_by_name(nmodule, g, g.mproject.name).as(not null)
+               if g != null and g.mpackage.name == "core" then
+                       # except for a unit test in a module of `core`
+                       # in this case, the whole `core` must be imported
+                       o = get_mmodule_by_name(nmodule, g, g.mpackage.name).as(not null)
                end
 
                ts.attr("package", mmodule.full_name)