nitunit: do not show the status bar if no-color or verbose
[nit.git] / src / testing / testing_base.nit
1 # This file is part of NIT ( http://www.nitlanguage.org ).
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 # Base options for testing tools.
16 module testing_base
17
18 import modelize
19 private import parser_util
20 import html
21 import console
22
23 redef class ToolContext
24 # opt --full
25 var opt_full = new OptionBool("Process also imported modules", "--full")
26 # opt --output
27 var opt_output = new OptionString("Output name (default is 'nitunit.xml')", "-o", "--output")
28 # opt --dirr
29 var opt_dir = new OptionString("Working directory (default is '.nitunit')", "--dir")
30 # opt --no-act
31 var opt_noact = new OptionBool("Does not compile and run tests", "--no-act")
32 # opt --nitc
33 var opt_nitc = new OptionString("nitc compiler to use", "--nitc")
34
35 # Working directory for testing.
36 fun test_dir: String do
37 var dir = opt_dir.value
38 if dir == null then return "nitunit.out"
39 return dir
40 end
41
42 # Search the `nitc` compiler to use
43 #
44 # If not `nitc` is suitable, then prints an error and quit.
45 fun find_nitc: String
46 do
47 var nitc = opt_nitc.value
48 if nitc != null then
49 if not nitc.file_exists then
50 fatal_error(null, "error: cannot find `{nitc}` given by --nitc.")
51 abort
52 end
53 return nitc
54 end
55
56 nitc = "NITC".environ
57 if nitc != "" then
58 if not nitc.file_exists then
59 fatal_error(null, "error: cannot find `{nitc}` given by NITC.")
60 abort
61 end
62 return nitc
63 end
64
65 var nit_dir = nit_dir
66 nitc = nit_dir/"bin/nitc"
67 if not nitc.file_exists then
68 fatal_error(null, "Error: cannot find nitc. Set envvar NIT_DIR or NITC or use the --nitc option.")
69 abort
70 end
71 return nitc
72 end
73
74 # Execute a system command in a more safe context than `Sys::system`.
75 fun safe_exec(command: String): Int
76 do
77 info(command, 2)
78 var real_command = """
79 bash -c "
80 ulimit -f {{{ulimit_file}}} 2> /dev/null
81 ulimit -t {{{ulimit_usertime}}} 2> /dev/null
82 {{{command}}}
83 "
84 """
85 return system(real_command)
86 end
87
88 # The maximum size (in KB) of files written by a command executed trough `safe_exec`
89 #
90 # Default: 64MB
91 var ulimit_file = 65536 is writable
92
93 # The maximum amount of cpu time (in seconds) for a command executed trough `safe_exec`
94 #
95 # Default: 10 CPU minute
96 var ulimit_usertime = 600 is writable
97
98 # Show a single-line status to use as a progression.
99 #
100 # Note that the line starts with `'\r'` and is not ended by a `'\n'`.
101 # So it is expected that:
102 # * no other output is printed between two calls
103 # * the last `show_unit_status` is followed by a new-line
104 fun show_unit_status(name: String, tests: SequenceRead[UnitTest], more_message: nullable String)
105 do
106 var esc = 27.code_point.to_s
107 var line = "\r{esc}[K* {name} ["
108 var done = tests.length
109 for t in tests do
110 if not t.is_done then
111 line += " "
112 done -= 1
113 else if t.error == null then
114 line += ".".green.bold
115 else
116 line += "X".red.bold
117 end
118 end
119
120 if not has_status then
121 if done == 0 then
122 print "* {name} ({tests.length} tests)"
123 end
124 return
125 end
126
127 line += "] {done}/{tests.length}"
128 if more_message != null then
129 line += " " + more_message
130 end
131 printn "{line}"
132 end
133
134 # Is a status bar printed?
135 #
136 # true if color and non-verbose mode
137 fun has_status: Bool
138 do
139 return not opt_no_color.value and opt_verbose.value <= 0
140 end
141
142 # Shoe the full description of the test-case.
143 #
144 # The output honors `--no-color`.
145 #
146 # `more message`, if any, is added after the error message.
147 fun show_unit(test: UnitTest, more_message: nullable String) do
148 print test.to_screen(more_message, not opt_no_color.value)
149 end
150 end
151
152 # A unit test is an elementary test discovered, run and reported by nitunit.
153 #
154 # This class factorizes `DocUnit` and `TestCase`.
155 abstract class UnitTest
156 # The name of the unit to show in messages
157 fun full_name: String is abstract
158
159 # The location of the unit test to show in messages.
160 fun location: Location is abstract
161
162 # Flag that indicates if the unit test was compiled/run.
163 var is_done: Bool = false is writable
164
165 # Error message occurred during test-case execution (or compilation).
166 #
167 # e.g.: `Runtime Error`
168 var error: nullable String = null is writable
169
170 # Was the test case executed at least once?
171 #
172 # This will indicate the status of the test (failture or error)
173 var was_exec = false is writable
174
175 # The raw output of the execution (or compilation)
176 #
177 # It merges the standard output and error output
178 var raw_output: nullable String = null is writable
179
180 # The location where the error occurred, if it makes sense.
181 var error_location: nullable Location = null is writable
182
183 # A colorful `[OK]` or `[KO]`.
184 fun status_tag(color: nullable Bool): String do
185 color = color or else true
186 if not is_done then
187 return "[ ]"
188 else if error != null then
189 var res = "[KO]"
190 if color then res = res.red.bold
191 return res
192 else
193 var res = "[OK]"
194 if color then res = res.green.bold
195 return res
196 end
197 end
198
199 # The full (color) description of the test-case.
200 #
201 # `more message`, if any, is added after the error message.
202 fun to_screen(more_message: nullable String, color: nullable Bool): String do
203 color = color or else true
204 var res
205 var error = self.error
206 if error != null then
207 if more_message != null then error += " " + more_message
208 var loc = error_location or else location
209 if color then
210 res = "{status_tag(color)} {full_name}\n {loc.to_s.yellow}: {error}\n{loc.colored_line("1;31")}"
211 else
212 res = "{status_tag(color)} {full_name}\n {loc}: {error}"
213 end
214 var output = self.raw_output
215 if output != null then
216 res += "\n Output\n\t{output.chomp.replace("\n", "\n\t")}\n"
217 end
218 else
219 res = "{status_tag(color)} {full_name}"
220 if more_message != null then res += more_message
221 end
222 return res
223 end
224
225 # Return a `<testcase>` XML node in format compatible with Jenkins unit tests.
226 fun to_xml: HTMLTag do
227 var tc = new HTMLTag("testcase")
228 tc.attr("classname", xml_classname)
229 tc.attr("name", xml_name)
230 var error = self.error
231 if error != null then
232 if was_exec then
233 tc.open("error").append(error)
234 else
235 tc.open("failure").append(error)
236 end
237 end
238 var output = self.raw_output
239 if output != null then
240 tc.open("system-err").append(output.trunc(8192).filter_nonprintable)
241 end
242 return tc
243 end
244
245 # The `classname` attribute of the XML format.
246 #
247 # NOTE: jenkins expects a '.' in the classname attr
248 #
249 # See to_xml
250 fun xml_classname: String is abstract
251
252 # The `name` attribute of the XML format.
253 #
254 # See to_xml
255 fun xml_name: String is abstract
256 end
257
258 redef class String
259 # If needed, truncate `self` at `max_length` characters and append an informative `message`.
260 #
261 # ~~~
262 # assert "hello".trunc(10) == "hello"
263 # assert "hello".trunc(2) == "he[truncated. Full size is 5]"
264 # assert "hello".trunc(2, "...") == "he..."
265 # ~~~
266 fun trunc(max_length: Int, message: nullable String): String
267 do
268 if length <= max_length then return self
269 if message == null then message = "[truncated. Full size is {length}]"
270 return substring(0, max_length) + message
271 end
272
273 # Use a special notation for whitespace characters that are not `'\n'` (LFD) or `' '` (space).
274 #
275 # ~~~
276 # assert "hello".filter_nonprintable == "hello"
277 # assert "\r\n\t".filter_nonprintable == "^13\n^9"
278 # ~~~
279 fun filter_nonprintable: String
280 do
281 var buf = new Buffer
282 for c in self do
283 var cp = c.code_point
284 if cp < 32 and c != '\n' then
285 buf.append "^{cp}"
286 else
287 buf.add c
288 end
289 end
290 return buf.to_s
291 end
292 end