src/testing/testing_base.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Base options for testing tools.
  16 module testing_base
  17
  18 import modelize
  19 private import parser_util
  20 import html
  21 import console
  22
  23 redef class ToolContext
  24         # opt --full
  25         var opt_full = new OptionBool("Process also imported modules", "--full")
  26         # opt --output
  27         var opt_output = new OptionString("Output name (default is 'nitunit.xml')", "-o", "--output")
  28         # opt --dirr
  29         var opt_dir = new OptionString("Working directory (default is '.nitunit')", "--dir")
  30         # opt --no-act
  31         var opt_noact = new OptionBool("Does not compile and run tests", "--no-act")
  32         # opt --nitc
  33         var opt_nitc = new OptionString("nitc compiler to use", "--nitc")
  34
  35         # Working directory for testing.
  36         fun test_dir: String do
  37                 var dir = opt_dir.value
  38                 if dir == null then return "nitunit.out"
  39                 return dir
  40         end
  41
  42         # Search the `nitc` compiler to use
  43         #
  44         # If not `nitc` is suitable, then prints an error and quit.
  45         fun find_nitc: String
  46         do
  47                 var nitc = opt_nitc.value
  48                 if nitc != null then
  49                         if not nitc.file_exists then
  50                                 fatal_error(null, "error: cannot find `{nitc}` given by --nitc.")
  51                                 abort
  52                         end
  53                         return nitc
  54                 end
  55
  56                 nitc = "NITC".environ
  57                 if nitc != "" then
  58                         if not nitc.file_exists then
  59                                 fatal_error(null, "error: cannot find `{nitc}` given by NITC.")
  60                                 abort
  61                         end
  62                         return nitc
  63                 end
  64
  65                 var nit_dir = nit_dir
  66                 nitc = nit_dir/"bin/nitc"
  67                 if not nitc.file_exists then
  68                         fatal_error(null, "Error: cannot find nitc. Set envvar NIT_DIR or NITC or use the --nitc option.")
  69                         abort
  70                 end
  71                 return nitc
  72         end
  73
  74         # Execute a system command in a more safe context than `Sys::system`.
  75         fun safe_exec(command: String): Int
  76         do
  77                 info(command, 2)
  78                 var real_command = """
  79 bash -c "
  80 ulimit -f {{{ulimit_file}}} 2> /dev/null
  81 ulimit -t {{{ulimit_usertime}}} 2> /dev/null
  82 {{{command}}}
  83 "
  84 """
  85                 return system(real_command)
  86         end
  87
  88         # The maximum size (in KB) of files written by a command executed trough `safe_exec`
  89         #
  90         # Default: 64MB
  91         var ulimit_file = 65536 is writable
  92
  93         # The maximum amount of cpu time (in seconds) for a command executed trough `safe_exec`
  94         #
  95         # Default: 10 CPU minute
  96         var ulimit_usertime = 600 is writable
  97
  98         # Show a single-line status to use as a progression.
  99         #
 100         # Note that the line starts with `'\r'` and is not ended by a `'\n'`.
 101         # So it is expected that:
 102         # * no other output is printed between two calls
 103         # * the last `show_unit_status` is followed by a new-line
 104         fun show_unit_status(name: String, tests: SequenceRead[UnitTest])
 105         do
 106                 var esc = 27.code_point.to_s
 107                 var line = "\r{esc}[K* {name} ["
 108                 var done = tests.length
 109                 for t in tests do
 110                         if not t.is_done then
 111                                 line += " "
 112                                 done -= 1
 113                         else if t.error == null then
 114                                 line += ".".green.bold
 115                         else
 116                                 line += "X".red.bold
 117                         end
 118                 end
 119
 120                 if not has_status then
 121                         if done == 0 then
 122                                 print "* {name} ({tests.length} tests)"
 123                         end
 124                         return
 125                 end
 126
 127                 line += "] {done}/{tests.length}"
 128                 printn "{line}"
 129         end
 130
 131         # Is a status bar printed?
 132         #
 133         # true if color and non-verbose mode
 134         fun has_status: Bool
 135         do
 136                 return not opt_no_color.value and opt_verbose.value <= 0
 137         end
 138
 139         # Clear the line if `has_status` (no-op else)
 140         fun clear_status
 141         do
 142                 if has_status then printn "\r\x1B[K"
 143         end
 144
 145         # Show the full description of the test-case.
 146         #
 147         # The output honors `--no-color`.
 148         #
 149         # `more message`, if any, is added after the error message.
 150         fun show_unit(test: UnitTest, more_message: nullable String) do
 151                 print test.to_screen(more_message, not opt_no_color.value)
 152         end
 153 end
 154
 155 # A unit test is an elementary test discovered, run and reported by nitunit.
 156 #
 157 # This class factorizes `DocUnit` and `TestCase`.
 158 abstract class UnitTest
 159         # The name of the unit to show in messages
 160         fun full_name: String is abstract
 161
 162         # The location of the unit test to show in messages.
 163         fun location: Location is abstract
 164
 165         # Flag that indicates if the unit test was compiled/run.
 166         var is_done: Bool = false is writable
 167
 168         # Error message occurred during test-case execution (or compilation).
 169         #
 170         # e.g.: `Runtime Error`
 171         var error: nullable String = null is writable
 172
 173         # Was the test case executed at least once?
 174         #
 175         # This will indicate the status of the test (failture or error)
 176         var was_exec = false is writable
 177
 178         # The raw output of the execution (or compilation)
 179         #
 180         # It merges the standard output and error output
 181         var raw_output: nullable String = null is writable
 182
 183         # The location where the error occurred, if it makes sense.
 184         var error_location: nullable Location = null is writable
 185
 186         # A colorful `[OK]` or `[KO]`.
 187         fun status_tag(color: nullable Bool): String do
 188                 color = color or else true
 189                 if not is_done then
 190                         return "[  ]"
 191                 else if error != null then
 192                         var res = "[KO]"
 193                         if color then res = res.red.bold
 194                         return res
 195                 else
 196                         var res = "[OK]"
 197                         if color then res = res.green.bold
 198                         return res
 199                 end
 200         end
 201
 202         # The full (color) description of the test-case.
 203         #
 204         # `more message`, if any, is added after the error message.
 205         fun to_screen(more_message: nullable String, color: nullable Bool): String do
 206                 color = color or else true
 207                 var res
 208                 var error = self.error
 209                 if error != null then
 210                         if more_message != null then error += " " + more_message
 211                         var loc = error_location or else location
 212                         if color then
 213                                 res = "{status_tag(color)} {full_name}\n     {loc.to_s.yellow}: {error}\n{loc.colored_line("1;31")}"
 214                         else
 215                                 res = "{status_tag(color)} {full_name}\n     {loc}: {error}"
 216                         end
 217                         var output = self.raw_output
 218                         if output != null then
 219                                 res += "\n     Output\n\t{output.chomp.replace("\n", "\n\t")}\n"
 220                         end
 221                 else
 222                         res = "{status_tag(color)} {full_name}"
 223                         if more_message != null then res += more_message
 224                 end
 225                 return res
 226         end
 227
 228         # Return a `<testcase>` XML node in format compatible with Jenkins unit tests.
 229         fun to_xml: HTMLTag do
 230                 var tc = new HTMLTag("testcase")
 231                 tc.attr("classname", xml_classname)
 232                 tc.attr("name", xml_name)
 233                 var error = self.error
 234                 if error != null then
 235                         if was_exec then
 236                                 tc.open("error").append(error)
 237                         else
 238                                 tc.open("failure").append(error)
 239                         end
 240                 end
 241                 var output = self.raw_output
 242                 if output != null then
 243                         tc.open("system-err").append(output.trunc(8192).filter_nonprintable)
 244                 end
 245                 return tc
 246         end
 247
 248         # The `classname` attribute of the XML format.
 249         #
 250         # NOTE: jenkins expects a '.' in the classname attr
 251         #
 252         # See to_xml
 253         fun xml_classname: String is abstract
 254
 255         # The `name` attribute of the XML format.
 256         #
 257         # See to_xml
 258         fun xml_name: String is abstract
 259 end
 260
 261 redef class String
 262         # If needed, truncate `self` at `max_length` characters and append an informative `message`.
 263         #
 264         # ~~~
 265         # assert "hello".trunc(10) == "hello"
 266         # assert "hello".trunc(2) == "he[truncated. Full size is 5]"
 267         # assert "hello".trunc(2, "...") == "he..."
 268         # ~~~
 269         fun trunc(max_length: Int, message: nullable String): String
 270         do
 271                 if length <= max_length then return self
 272                 if message == null then message = "[truncated. Full size is {length}]"
 273                 return substring(0, max_length) + message
 274         end
 275
 276         # Use a special notation for whitespace characters that are not `'\n'` (LFD) or `' '` (space).
 277         #
 278         # ~~~
 279         # assert "hello".filter_nonprintable == "hello"
 280         # assert "\r\n\t".filter_nonprintable == "^13\n^9"
 281         # ~~~
 282         fun filter_nonprintable: String
 283         do
 284                 var buf = new Buffer
 285                 for c in self do
 286                         var cp = c.code_point
 287                         if cp < 32 and c != '\n' then
 288                                 buf.append "^{cp}"
 289                         else
 290                                 buf.add c
 291                         end
 292                 end
 293                 return buf.to_s
 294         end
 295 end