Merge: Sys is top

[nit.git] / contrib / nitester / src / nitester.nit

diff --git a/contrib/nitester/src/nitester.nit b/contrib/nitester/src/nitester.nit
index 922a8a0..fb51b4d 100644 (file)
--- a/contrib/nitester/src/nitester.nit
+++ b/contrib/nitester/src/nitester.nit
@@ -31,6 +31,9 @@ abstract class Processor
        # Controller rank is always 0
        var controller_rank: Rank = 0.rank
 
        # Controller rank is always 0
        var controller_rank: Rank = 0.rank
 

+       # Rank on this processor
+       fun rank: Rank is abstract
+

        # Where to store data for transfer between nodes
        #
        # Require: `buffer.length % 4 == 0`
        # Where to store data for transfer between nodes
        #
        # Require: `buffer.length % 4 == 0`

@@ -49,7 +52,7 @@ abstract class Processor
        # Tag of a new task packet of size `tasks_per_packet`
        var task_tag: Tag = 0.tag
 
        # Tag of a new task packet of size `tasks_per_packet`
        var task_tag: Tag = 0.tag
 

-       # Tag to return a set of `Result` throught `buffer`
+       # Tag to return a set of `Result` thought `buffer`

        var result_tag: Tag = 1.tag
 
        # Tag to notify `Worker` when to quit
        var result_tag: Tag = 1.tag
 
        # Tag to notify `Worker` when to quit

@@ -67,7 +70,10 @@ abstract class Processor
        # Run the main logic of this node
        fun run is abstract
 
        # Run the main logic of this node
        fun run is abstract
 

-       # Engines targetted by this execution
+       # Hash or name of the branch to test
+       var branch_hash: String is noinit
+
+       # Engines targeted by this execution

        var engines: Array[String] is noinit
 
        # All known engines, used to detect errors in `engines`
        var engines: Array[String] is noinit
 
        # All known engines, used to detect errors in `engines`

@@ -86,6 +92,10 @@ abstract class Processor
        fun read_cli_options
        do
                var opt_ctx = new OptionContext
        fun read_cli_options
        do
                var opt_ctx = new OptionContext

+               var opt_hash = new OptionString(
+                       "Branch to test",
+                       "--hash", "-h")
+               opt_hash.mandatory = true

                var opt_engines = new OptionString(
                        "Engines to test, separated with commas ({all_engines.join(", ")} or all)",
                        "--engine", "-e")
                var opt_engines = new OptionString(
                        "Engines to test, separated with commas ({all_engines.join(", ")} or all)",
                        "--engine", "-e")

@@ -97,7 +107,7 @@ abstract class Processor
                        "Clean up all nitester files (and do not run tests)",
                        "--cleanup", "-C")
 
                        "Clean up all nitester files (and do not run tests)",
                        "--cleanup", "-C")
 

-               opt_ctx.add_option(opt_engines, opt_help, opt_verbose, opt_cleanup)
+               opt_ctx.add_option(opt_hash, opt_engines, opt_help, opt_verbose, opt_cleanup)

                opt_ctx.parse args
 
                # --help?
                opt_ctx.parse args
 
                # --help?

@@ -121,7 +131,6 @@ abstract class Processor
                                else
                                        full_path.file_delete
                                end
                                else
                                        full_path.file_delete
                                end

-                               stat.free

                        end
                        mpi.finalize
                        exit 0
                        end
                        mpi.finalize
                        exit 0

@@ -132,6 +141,9 @@ abstract class Processor
                if rest.is_empty then opt_ctx.usage_error "This tool needs at least one test_program.nit"
                test_programs = rest
 
                if rest.is_empty then opt_ctx.usage_error "This tool needs at least one test_program.nit"
                test_programs = rest
 

+               # hash
+               branch_hash = opt_hash.value.as(not null)
+

                # gather and check engines
                var engines_str = opt_engines.value
                var engines
                # gather and check engines
                var engines_str = opt_engines.value
                var engines

@@ -160,12 +172,29 @@ abstract class Processor
        # All tasks to be performed
        var tasks = new Array[Task]
 
        # All tasks to be performed
        var tasks = new Array[Task]
 

-       # Gather and registar all tasks
+       # Gather and register all tasks

        fun create_tasks
        do
        fun create_tasks
        do

-               for prog in test_programs do for engine in engines do
-                       tasks.add new Task(engine, prog)
+               # At this point we are in our local nit
+               var skip_path = "tests/turing.skip"
+               var skip
+               if skip_path.file_exists then
+                       var skip_file = new FileReader.open(skip_path)
+                       skip = skip_file.read_lines
+                       skip_file.close
+               else
+                       skip = new Array[String]

                end
                end

+
+               for engine in engines do for prog in test_programs do
+                       # Is is blacklisted?
+                       for s in skip do if not s.is_empty and prog.has(s) then
+                               if verbose > 0 and rank == 0 then print "Skipping test '{prog}' because of '{s}' in turing.skip"
+                               continue label
+                       end
+
+                       tasks.add new Task(engine, prog)
+               end label

        end
 end
 
        end
 end
 

@@ -173,6 +202,8 @@ end
 class Controller
        super Processor
 
 class Controller
        super Processor
 

+       redef fun rank do return controller_rank
+

        # Id as `Int` of the next task to distribute
        var next_task_id = 0
 
        # Id as `Int` of the next task to distribute
        var next_task_id = 0
 

@@ -248,6 +279,8 @@ class Controller
                                        if res == 5 then result.fail = true
                                        if res == 6 then result.soso = true
                                        if res == 7 then result.skip = true
                                        if res == 5 then result.fail = true
                                        if res == 6 then result.soso = true
                                        if res == 7 then result.skip = true

+                                       if res == 8 then result.todo = true
+                                       if res == 9 then result.skip_exec = true

                                        if res == 0 then result.unknown = true
 
                                        results.add result
                                        if res == 0 then result.unknown = true
 
                                        results.add result

@@ -268,7 +301,7 @@ class Controller
                                mpi.recv_empty(status.source, status.tag, comm_world)
                                at_work.remove(status.source)
 
                                mpi.recv_empty(status.source, status.tag, comm_world)
                                at_work.remove(status.source)
 

-                               if verbose > 1 then print "worker {status.source} is done ({at_work.length} still at work)"
+                               if verbose > 0 then print "Worker {status.source} is done ({at_work.length} still at work)"

                        else
                                print "Unexpected tag {status.tag}"
                                shutdown
                        else
                                print "Unexpected tag {status.tag}"
                                shutdown

@@ -303,6 +336,8 @@ class Controller
                print "* {results.fixmes.length} fixmes"
                print "* {results.sosos.length} sosos"
                print "* {results.skips.length} skips"
                print "* {results.fixmes.length} fixmes"
                print "* {results.sosos.length} sosos"
                print "* {results.skips.length} skips"

+               print "* {results.todos.length} todos"
+               print "* {results.skip_execs.length} skip execs"

                print "* {results.unknowns.length} unknowns (bug in tests.sh or nitester)"
        end
 
                print "* {results.unknowns.length} unknowns (bug in tests.sh or nitester)"
        end
 

@@ -322,14 +357,11 @@ class Worker
        super Processor
 
        # The `Rank` of `self`
        super Processor
 
        # The `Rank` of `self`

-       var rank: Rank
+       redef var rank: Rank

 
        # Compilation directory
        var comp_dir = "/dev/shm/nit_compile{rank}" is lazy
 
 
        # Compilation directory
        var comp_dir = "/dev/shm/nit_compile{rank}" is lazy
 

-       # Output file directory
-       var out_dir = "/dev/shm/nit_out{rank}" is lazy
-

        # Directory to store the xml files produced for Jenkins
        var xml_dir = "~/jenkins_xml/"
 
        # Directory to store the xml files produced for Jenkins
        var xml_dir = "~/jenkins_xml/"
 

@@ -337,7 +369,10 @@ class Worker
        var tests_sh_out = "/dev/shm/nit_local_out{rank}" is lazy
 
        # Source Nit repository, must be already updated and `make` before execution
        var tests_sh_out = "/dev/shm/nit_local_out{rank}" is lazy
 
        # Source Nit repository, must be already updated and `make` before execution

-       var nit_source_dir = "~/nit"
+       var local_nit = "/dev/shm/nit{rank}" is lazy
+
+       # Remote Nit repository (actually the local source)
+       var remote_nit = "~/nit/"

 
        # Compiled `Regex` to detect the argument of an execution
        var re_arg: Regex = "arg [0-9]+".to_re
 
        # Compiled `Regex` to detect the argument of an execution
        var re_arg: Regex = "arg [0-9]+".to_re

@@ -360,6 +395,25 @@ class Worker
        fun setup
        do
                if verbose > 0 then sys.system "hostname"
        fun setup
        do
                if verbose > 0 then sys.system "hostname"

+
+               if local_nit.file_exists then local_nit.rmdir
+
+               exec_and_check "git clone {remote_nit} {local_nit}"
+               local_nit.chdir
+               exec_and_check "git config remote.origin.fetch +refs/remotes/origin/pr/*:refs/remotes/origin/pr/*"
+               exec_and_check "git fetch origin --quiet"
+               exec_and_check "git checkout {branch_hash}"
+               exec_and_check "cp {remote_nit}/bin/*  bin/"
+               exec_and_check "src/git-gen-version.sh"
+       end
+
+       private fun exec_and_check(cmd: String)
+       do
+               if verbose > 0 then
+                       print "+ {cmd}"
+                       var res = sys.system(cmd)
+                       assert res == 0 else print "Command '{cmd}' failed."
+               end

        end
 
        # Clean up the testing environment
        end
 
        # Clean up the testing environment

@@ -368,8 +422,8 @@ class Worker
        fun cleanup
        do
                if comp_dir.file_exists then comp_dir.rmdir
        fun cleanup
        do
                if comp_dir.file_exists then comp_dir.rmdir

-               if out_dir.file_exists then out_dir.rmdir

                if tests_sh_out.file_exists then tests_sh_out.file_delete
                if tests_sh_out.file_exists then tests_sh_out.file_delete

+               if local_nit.file_exists then local_nit.file_delete

        end
 
        # Single C `int` to hold the next task id received from the `Controller`
        end
 
        # Single C `int` to hold the next task id received from the `Controller`

@@ -391,23 +445,24 @@ class Worker
                                # Receive tasks to execute
                                mpi.recv_into(task_buffer, 0, 1, status.source, status.tag, comm_world)
                                var first_id = task_buffer[0]
                                # Receive tasks to execute
                                mpi.recv_into(task_buffer, 0, 1, status.source, status.tag, comm_world)
                                var first_id = task_buffer[0]

-                               for task_id in [first_id .. first_id + tasks_per_packet] do
+                               for task_id in [first_id .. first_id + tasks_per_packet[ do

 
                                        # If id is over all known tasks, stop right here
                                        if task_id >= tasks.length then break
                                        var task = tasks[task_id]
 
 
                                        # If id is over all known tasks, stop right here
                                        if task_id >= tasks.length then break
                                        var task = tasks[task_id]
 

+                                       "tests".chdir
+

                                        # Command line to execute test
                                        # Command line to execute test

-                                       var cmd = "XMLDIR={xml_dir} ERRLIST={out_dir}/errlist TMPDIR={out_dir} " +
+                                       var cmd = "XMLDIR={xml_dir} " +

                                                "CCACHE_DIR={ccache_dir} CCACHE_TEMPDIR={ccache_dir} CCACHE_BASEDIR={comp_dir} " +
                                                "CCACHE_DIR={ccache_dir} CCACHE_TEMPDIR={ccache_dir} CCACHE_BASEDIR={comp_dir} " +

-                                               "./tests.sh --compdir {comp_dir} --outdir {out_dir} " +
-                                               " --node --engine {task.engine} {task.test_program} > {tests_sh_out}"
+                                               "./tests.sh --node --engine {task.engine} {task.test_program} > {tests_sh_out}"

 
                                        # Execute test
                                        sys.system cmd
 
                                        # Test results were written to file, read them
 
                                        # Execute test
                                        sys.system cmd
 
                                        # Test results were written to file, read them

-                                       var fstream = new IFStream.open(tests_sh_out)
+                                       var fstream = new FileReader.open(tests_sh_out)

                                        var content = fstream.read_all
                                        fstream.close
 
                                        var content = fstream.read_all
                                        fstream.close
 

@@ -442,6 +497,8 @@ class Worker
                                                if line.has("[======= fail") then res = 5
                                                if line.has("[======= soso") then res = 6
                                                if line.has("[skip]") then res = 7
                                                if line.has("[======= fail") then res = 5
                                                if line.has("[======= soso") then res = 6
                                                if line.has("[skip]") then res = 7

+                                               if line.has("[todo]") then res = 8
+                                               if line.has("[skip exec]") then res = 9

 
                                                if res == null then
                                                        res = 0
 
                                                if res == null then
                                                        res = 0

@@ -460,6 +517,8 @@ class Worker
                                                end
                                        end
 
                                                end
                                        end
 

+                                       if verbose > 1 then print "Done testing: {task}"
+

                                        self.results_count = c
                                end
 
                                        self.results_count = c
                                end
 

@@ -489,7 +548,7 @@ class Worker
        fun send_results
        do
                if results_count > 0 then
        fun send_results
        do
                if results_count > 0 then

-                       if verbose > 1 then print "sending {results_count} results"
+                       if verbose > 2 then print "Sending {results_count} results"

                        mpi.send_from(buffer, 0, results_count*4, controller_rank, result_tag, comm_world)
                        results_count = 0
                end
                        mpi.send_from(buffer, 0, results_count*4, controller_rank, result_tag, comm_world)
                        results_count = 0
                end

@@ -548,9 +607,15 @@ class Result
        # Is `self` result a _soso_?
        var soso = false
 
        # Is `self` result a _soso_?
        var soso = false
 

-       # Is `self` skipped test?
+       # Has `self` been skipped?

        var skip = false
 
        var skip = false
 

+       # Is `self` TODO?
+       var todo = false
+
+       # Has the execution of `self` been skipped?
+       var skip_exec = false
+

        # Is `self` an unknown result, probably an error
        var unknown = false
 
        # Is `self` an unknown result, probably an error
        var unknown = false
 

@@ -562,6 +627,10 @@ class Result
                if ok_empty then err = "0k"
                if fixme then err = "fixme"
                if fail then err = "fail"
                if ok_empty then err = "0k"
                if fixme then err = "fixme"
                if fail then err = "fail"

+               if soso then err = "soso"
+               if skip then err = "skip"
+               if todo then err = "todo"
+               if skip_exec then err = "skip_exec"

 
                return "{task} arg{arg} alt{alt} => {err}"
        end
 
                return "{task} arg{arg} alt{alt} => {err}"
        end

@@ -578,6 +647,8 @@ class ResultSet
        var fails = new HashSet[Result]
        var sosos = new HashSet[Result]
        var skips = new HashSet[Result]
        var fails = new HashSet[Result]
        var sosos = new HashSet[Result]
        var skips = new HashSet[Result]

+       var todos = new HashSet[Result]
+       var skip_execs = new HashSet[Result]

        var unknowns = new HashSet[Result]
 
        # TODO remove
        var unknowns = new HashSet[Result]
 
        # TODO remove

@@ -592,6 +663,8 @@ class ResultSet
                if result.fail then fails.add result
                if result.soso then sosos.add result
                if result.skip then skips.add result
                if result.fail then fails.add result
                if result.soso then sosos.add result
                if result.skip then skips.add result

+               if result.todo then todos.add result
+               if result.skip_exec then skip_execs.add result

                if result.unknown then unknowns.add result
 
                super
                if result.unknown then unknowns.add result
 
                super