contrib/nitester: use turing skip list
[nit.git] / contrib / nitester / src / nitester.nit
index b853863..9374c66 100644 (file)
@@ -31,6 +31,9 @@ abstract class Processor
        # Controller rank is always 0
        var controller_rank: Rank = 0.rank
 
+       # Rank on this processor
+       fun rank: Rank is abstract
+
        # Where to store data for transfer between nodes
        #
        # Require: `buffer.length % 4 == 0`
@@ -49,7 +52,7 @@ abstract class Processor
        # Tag of a new task packet of size `tasks_per_packet`
        var task_tag: Tag = 0.tag
 
-       # Tag to return a set of `Result` throught `buffer`
+       # Tag to return a set of `Result` thought `buffer`
        var result_tag: Tag = 1.tag
 
        # Tag to notify `Worker` when to quit
@@ -62,12 +65,12 @@ abstract class Processor
        var done_tag: Tag = 5.tag
 
        # Number of tasks within each task assignation with `task_tag`
-       var tasks_per_packet = 4
+       var tasks_per_packet = 1
 
        # Run the main logic of this node
        fun run is abstract
 
-       # Engines targetted by this execution
+       # Engines targeted by this execution
        var engines: Array[String] is noinit
 
        # All known engines, used to detect errors in `engines`
@@ -160,14 +163,30 @@ abstract class Processor
        # All tasks to be performed
        var tasks = new Array[Task]
 
-       # Gather and registar all tasks
+       # Gather and register all tasks
        fun create_tasks
        do
-               var c = 0
-               for engine in engines do for prog in test_programs do
-                       tasks.add new Task(engine, prog)
-                       c += 1
+               # At this point we are in our local nit
+               var skip_path = "tests/turing.skip"
+               var skip
+               if skip_path.file_exists then
+                       var skip_file = new IFStream.open(skip_path)
+                       skip = skip_file.read_lines
+                       skip_file.close
+               else
+                       skip = new Array[String]
                end
+
+               for prog in test_programs do for engine in engines do
+
+                       # Is is blacklisted?
+                       for s in skip do if not s.is_empty and prog.has(s) then
+                               if verbose > 0 and rank == 0 then print "Skipping test '{prog}' because of '{s}' in turing.skip"
+                               continue label
+                       end
+
+                       tasks.add new Task(engine, prog)
+               end label
        end
 end
 
@@ -175,6 +194,8 @@ end
 class Controller
        super Processor
 
+       redef fun rank do return controller_rank
+
        # Id as `Int` of the next task to distribute
        var next_task_id = 0
 
@@ -250,6 +271,8 @@ class Controller
                                        if res == 5 then result.fail = true
                                        if res == 6 then result.soso = true
                                        if res == 7 then result.skip = true
+                                       if res == 8 then result.todo = true
+                                       if res == 9 then result.skip_exec = true
                                        if res == 0 then result.unknown = true
 
                                        results.add result
@@ -305,6 +328,8 @@ class Controller
                print "* {results.fixmes.length} fixmes"
                print "* {results.sosos.length} sosos"
                print "* {results.skips.length} skips"
+               print "* {results.todos.length} todos"
+               print "* {results.skip_execs.length} skip execs"
                print "* {results.unknowns.length} unknowns (bug in tests.sh or nitester)"
        end
 
@@ -324,7 +349,7 @@ class Worker
        super Processor
 
        # The `Rank` of `self`
-       var rank: Rank
+       redef var rank: Rank
 
        # Compilation directory
        var comp_dir = "/dev/shm/nit_compile{rank}" is lazy
@@ -338,9 +363,6 @@ class Worker
        # Output file of the `tests.sh` script
        var tests_sh_out = "/dev/shm/nit_local_out{rank}" is lazy
 
-       # Path to the local copy of the Nit repository
-       var nit_copy_dir = "/dev/shm/nit{rank}/" is lazy
-
        # Source Nit repository, must be already updated and `make` before execution
        var nit_source_dir = "~/nit"
 
@@ -365,7 +387,6 @@ class Worker
        fun setup
        do
                if verbose > 0 then sys.system "hostname"
-               sys.system "git clone {nit_source_dir} {nit_copy_dir}"
        end
 
        # Clean up the testing environment
@@ -375,7 +396,6 @@ class Worker
        do
                if comp_dir.file_exists then comp_dir.rmdir
                if out_dir.file_exists then out_dir.rmdir
-               if nit_copy_dir.file_exists then nit_copy_dir.rmdir
                if tests_sh_out.file_exists then tests_sh_out.file_delete
        end
 
@@ -398,7 +418,7 @@ class Worker
                                # Receive tasks to execute
                                mpi.recv_into(task_buffer, 0, 1, status.source, status.tag, comm_world)
                                var first_id = task_buffer[0]
-                               for task_id in [first_id .. first_id + tasks_per_packet] do
+                               for task_id in [first_id .. first_id + tasks_per_packet[ do
 
                                        # If id is over all known tasks, stop right here
                                        if task_id >= tasks.length then break
@@ -408,7 +428,7 @@ class Worker
                                        var cmd = "XMLDIR={xml_dir} ERRLIST={out_dir}/errlist TMPDIR={out_dir} " +
                                                "CCACHE_DIR={ccache_dir} CCACHE_TEMPDIR={ccache_dir} CCACHE_BASEDIR={comp_dir} " +
                                                "./tests.sh --compdir {comp_dir} --outdir {out_dir} " +
-                                               " --node --engine {task.engine} {nit_copy_dir / "tests" / task.test_program} > {tests_sh_out}"
+                                               " --node --engine {task.engine} {task.test_program} > {tests_sh_out}"
 
                                        # Execute test
                                        sys.system cmd
@@ -449,6 +469,8 @@ class Worker
                                                if line.has("[======= fail") then res = 5
                                                if line.has("[======= soso") then res = 6
                                                if line.has("[skip]") then res = 7
+                                               if line.has("[todo]") then res = 8
+                                               if line.has("[skip exec]") then res = 9
 
                                                if res == null then
                                                        res = 0
@@ -555,9 +577,15 @@ class Result
        # Is `self` result a _soso_?
        var soso = false
 
-       # Is `self` skipped test?
+       # Has `self` been skipped?
        var skip = false
 
+       # Is `self` TODO?
+       var todo = false
+
+       # Has the execution of `self` been skipped?
+       var skip_exec = false
+
        # Is `self` an unknown result, probably an error
        var unknown = false
 
@@ -569,6 +597,10 @@ class Result
                if ok_empty then err = "0k"
                if fixme then err = "fixme"
                if fail then err = "fail"
+               if soso then err = "soso"
+               if skip then err = "skip"
+               if todo then err = "todo"
+               if skip_exec then err = "skip_exec"
 
                return "{task} arg{arg} alt{alt} => {err}"
        end
@@ -585,6 +617,8 @@ class ResultSet
        var fails = new HashSet[Result]
        var sosos = new HashSet[Result]
        var skips = new HashSet[Result]
+       var todos = new HashSet[Result]
+       var skip_execs = new HashSet[Result]
        var unknowns = new HashSet[Result]
 
        # TODO remove
@@ -599,6 +633,8 @@ class ResultSet
                if result.fail then fails.add result
                if result.soso then sosos.add result
                if result.skip then skips.add result
+               if result.todo then todos.add result
+               if result.skip_exec then skip_execs.add result
                if result.unknown then unknowns.add result
 
                super