src/testing/testing_base.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Base options for testing tools.
  16 module testing_base
  17
  18 import modelize
  19 private import parser_util
  20 import html
  21 import console
  22
  23 redef class ToolContext
  24         # opt --full
  25         var opt_full = new OptionBool("Process also imported modules", "--full")
  26         # opt --output
  27         var opt_output = new OptionString("Output name (default is 'nitunit.xml')", "-o", "--output")
  28         # opt --dirr
  29         var opt_dir = new OptionString("Working directory (default is '.nitunit')", "--dir")
  30         # opt --no-act
  31         var opt_noact = new OptionBool("Does not compile and run tests", "--no-act")
  32         # opt --nitc
  33         var opt_nitc = new OptionString("nitc compiler to use", "--nitc")
  34         # opt --no-time
  35         var opt_no_time = new OptionBool("Disable time information in XML", "--no-time")
  36
  37         # Working directory for testing.
  38         fun test_dir: String do
  39                 var dir = opt_dir.value
  40                 if dir == null then return "nitunit.out"
  41                 return dir
  42         end
  43
  44         # Search the `nitc` compiler to use
  45         #
  46         # If not `nitc` is suitable, then prints an error and quit.
  47         fun find_nitc: String
  48         do
  49                 var nitc = opt_nitc.value
  50                 if nitc != null then
  51                         if not nitc.file_exists then
  52                                 fatal_error(null, "error: cannot find `{nitc}` given by --nitc.")
  53                                 abort
  54                         end
  55                         return nitc
  56                 end
  57
  58                 nitc = "NITC".environ
  59                 if nitc != "" then
  60                         if not nitc.file_exists then
  61                                 fatal_error(null, "error: cannot find `{nitc}` given by NITC.")
  62                                 abort
  63                         end
  64                         return nitc
  65                 end
  66
  67                 var nit_dir = nit_dir or else "."
  68                 nitc = nit_dir / "bin/nitc"
  69                 if not nitc.file_exists then
  70                         fatal_error(null, "Error: cannot find nitc. Set envvar NIT_DIR or NITC or use the --nitc option.")
  71                         abort
  72                 end
  73                 return nitc
  74         end
  75
  76         # Execute a system command in a more safe context than `Sys::system`.
  77         fun safe_exec(command: String): Int
  78         do
  79                 info(command, 2)
  80                 var real_command = """
  81 bash -c "
  82 ulimit -f {{{ulimit_file}}} 2> /dev/null
  83 ulimit -t {{{ulimit_usertime}}} 2> /dev/null
  84 {{{command}}}
  85 "
  86 """
  87                 return system(real_command)
  88         end
  89
  90         # The maximum size (in KB) of files written by a command executed trough `safe_exec`
  91         #
  92         # Default: 64MB
  93         var ulimit_file = 65536 is writable
  94
  95         # The maximum amount of cpu time (in seconds) for a command executed trough `safe_exec`
  96         #
  97         # Default: 10 CPU minute
  98         var ulimit_usertime = 600 is writable
  99
 100         # Show a single-line status to use as a progression.
 101         #
 102         # If `has_progress_bar` is true, then the output is a progress bar.
 103         # The printed the line starts with `'\r'` and is not ended by a `'\n'`.
 104         # So it is expected that:
 105         # * no other output is printed between two calls
 106         # * the last `show_unit_status` is followed by a new-line
 107         #
 108         # If `has_progress_bar` is false, then only the first and last state is shown
 109         fun show_unit_status(name: String, tests: SequenceRead[UnitTest])
 110         do
 111                 var line = "\r\x1B[K==== {name} ["
 112                 var done = tests.length
 113                 var fails = 0
 114                 for t in tests do
 115                         if not t.is_done then
 116                                 line += " "
 117                                 done -= 1
 118                         else if t.error == null then
 119                                 line += ".".green.bold
 120                         else
 121                                 line += "X".red.bold
 122                                 fails += 1
 123                         end
 124                 end
 125
 126                 if not has_progress_bar then
 127                         if done == 0 then
 128                                 print "==== {name} | tests: {tests.length}"
 129                         end
 130                         return
 131                 end
 132
 133                 if done < tests.length then
 134                         line += "] {done}/{tests.length}"
 135                 else
 136                         line += "] tests: {tests.length} "
 137                         if fails == 0 then
 138                                 line += "OK".green.bold
 139                         else
 140                                 line += "KO: {fails}".red.bold
 141                         end
 142                 end
 143                 printn "{line}"
 144         end
 145
 146         # Is a progress bar printed?
 147         #
 148         # true if color (because and non-verbose mode
 149         # (because verbose mode messes up with the progress bar).
 150         fun has_progress_bar: Bool
 151         do
 152                 return not opt_no_color.value and opt_verbose.value <= 0
 153         end
 154
 155         # Clear the line if `has_progress_bar` (no-op else)
 156         fun clear_progress_bar
 157         do
 158                 if has_progress_bar then printn "\r\x1B[K"
 159         end
 160
 161         # Show the full description of the test-case.
 162         #
 163         # The output honors `--no-color`.
 164         #
 165         # `more message`, if any, is added after the error message.
 166         fun show_unit(test: UnitTest, more_message: nullable String) do
 167                 print test.to_screen(more_message, not opt_no_color.value)
 168         end
 169
 170         # Set the `NIT_TESTING_PATH` environment variable with `path`.
 171         #
 172         # If `path == null` then `NIT_TESTING_PATH` is set with the empty string.
 173         fun set_testing_path(path: nullable String) do
 174                 "NIT_TESTING_PATH".setenv(path or else "")
 175         end
 176 end
 177
 178 # A unit test is an elementary test discovered, run and reported by nitunit.
 179 #
 180 # This class factorizes `DocUnit` and `TestCase`.
 181 abstract class UnitTest
 182         # The name of the unit to show in messages
 183         fun full_name: String is abstract
 184
 185         # The location of the unit test to show in messages.
 186         fun location: Location is abstract
 187
 188         # Flag that indicates if the unit test was compiled/run.
 189         var is_done: Bool = false is writable
 190
 191         # Error message occurred during test-case execution (or compilation).
 192         #
 193         # e.g.: `Runtime Error`
 194         var error: nullable String = null is writable
 195
 196         # Was the test case executed at least once?
 197         #
 198         # This will indicate the status of the test (failture or error)
 199         var was_exec = false is writable
 200
 201         # The raw output of the execution (or compilation)
 202         #
 203         # It merges the standard output and error output
 204         var raw_output: nullable String = null is writable
 205
 206         # The location where the error occurred, if it makes sense.
 207         var error_location: nullable Location = null is writable
 208
 209         # Additional noteworthy information when a test success.
 210         var info: nullable String = null
 211
 212         # Time for the execution, in seconds
 213         var real_time: Float = 0.0 is writable
 214
 215         # A colorful `[OK]` or `[KO]`.
 216         fun status_tag(color: nullable Bool): String do
 217                 color = color or else true
 218                 if not is_done then
 219                         return "[  ]"
 220                 else if error != null then
 221                         var res = "[KO]"
 222                         if color then res = res.red.bold
 223                         return res
 224                 else
 225                         var res = "[OK]"
 226                         if color then res = res.green.bold
 227                         return res
 228                 end
 229         end
 230
 231         # The full (color) description of the test-case.
 232         #
 233         # `more message`, if any, is added after the error message.
 234         fun to_screen(more_message: nullable String, color: nullable Bool): String do
 235                 color = color or else true
 236                 var res
 237                 var error = self.error
 238                 if error != null then
 239                         if more_message != null then error += " " + more_message
 240                         var loc = error_location or else location
 241                         if color then
 242                                 res = "{status_tag(color)} {full_name}\n     {loc.to_s.yellow}: {error}\n{loc.colored_line("1;31")}"
 243                         else
 244                                 res = "{status_tag(color)} {full_name}\n     {loc}: {error}"
 245                         end
 246                         var output = self.raw_output
 247                         if output != null then
 248                                 res += "\n     Output\n\t{output.chomp.replace("\n", "\n\t")}\n"
 249                         end
 250                 else
 251                         res = "{status_tag(color)} {full_name}"
 252                         if more_message != null then res += more_message
 253                         var info = self.info
 254                         if info != null then
 255                                 res += "\n     {info}"
 256                         end
 257                 end
 258                 return res
 259         end
 260
 261         # Return a `<testcase>` XML node in format compatible with Jenkins unit tests.
 262         fun to_xml: HTMLTag do
 263                 var tc = new HTMLTag("testcase")
 264                 tc.attr("classname", xml_classname)
 265                 tc.attr("name", xml_name)
 266                 tc.attr("time", real_time.to_s)
 267
 268                 var output = self.raw_output
 269                 if output != null then output = output.trunc(8192).filter_nonprintable
 270                 var error = self.error
 271                 if error != null then
 272                         var node
 273                         if was_exec then
 274                                 node = tc.open("error").attr("message", error)
 275                         else
 276                                 node = tc.open("failure").attr("message", error)
 277                         end
 278                         if output != null then
 279                                 node.append(output)
 280                         end
 281                 else if output != null then
 282                         tc.open("system-err").append(output)
 283                 end
 284                 return tc
 285         end
 286
 287         # The `classname` attribute of the XML format.
 288         #
 289         # NOTE: jenkins expects a '.' in the classname attr
 290         #
 291         # See to_xml
 292         fun xml_classname: String is abstract
 293
 294         # The `name` attribute of the XML format.
 295         #
 296         # See to_xml
 297         fun xml_name: String is abstract
 298 end
 299
 300 redef class String
 301         # If needed, truncate `self` at `max_length` characters and append an informative `message`.
 302         #
 303         # ~~~
 304         # assert "hello".trunc(10) == "hello"
 305         # assert "hello".trunc(2) == "he[truncated. Full size is 5]"
 306         # assert "hello".trunc(2, "...") == "he..."
 307         # ~~~
 308         fun trunc(max_length: Int, message: nullable String): String
 309         do
 310                 if length <= max_length then return self
 311                 if message == null then message = "[truncated. Full size is {length}]"
 312                 return substring(0, max_length) + message
 313         end
 314
 315         # Use a special notation for whitespace characters that are not `'\n'` (LFD) or `' '` (space).
 316         #
 317         # ~~~
 318         # assert "hello".filter_nonprintable == "hello"
 319         # assert "\r\n\t".filter_nonprintable == "^13\n^9"
 320         # ~~~
 321         fun filter_nonprintable: String
 322         do
 323                 var buf = new Buffer
 324                 for c in self do
 325                         var cp = c.code_point
 326                         if cp < 32 and c != '\n' then
 327                                 buf.append "^{cp}"
 328                         else
 329                                 buf.add c
 330                         end
 331                 end
 332                 return buf.to_s
 333         end
 334 end