Merge: nitester over MPI and jenkins
authorJean Privat <jean@pryen.org>
Mon, 10 Nov 2014 19:28:49 +0000 (14:28 -0500)
committerJean Privat <jean@pryen.org>
Mon, 10 Nov 2014 19:28:49 +0000 (14:28 -0500)
Add compatibility with Jenkins and some improvements to the tester.

This PR should be testable over MPI on Jenkins.

Pull-Request: #886
Reviewed-by: Jean Privat <jean@pryen.org>

contrib/nitester/src/nitester.nit
lib/ai/examples/puzzle.nit
misc/jenkins/nitester-wrapper.sh [new file with mode: 0755]
misc/jenkins/unitrun.sh
src/metrics/generate_hierarchies.nit
src/nitunit.nit
tests/listfull.sh
tests/sav/nitmetrics_args1.res
tests/sav/nitunit_args1.res
tests/sav/puzzle_args1.res
tests/tests.sh

index 5edfe22..37b5594 100644 (file)
@@ -62,7 +62,7 @@ abstract class Processor
        var done_tag: Tag = 5.tag
 
        # Number of tasks within each task assignation with `task_tag`
-       var tasks_per_packet = 4
+       var tasks_per_packet = 1
 
        # Run the main logic of this node
        fun run is abstract
@@ -163,10 +163,8 @@ abstract class Processor
        # Gather and registar all tasks
        fun create_tasks
        do
-               var c = 0
-               for engine in engines do for prog in test_programs do
+               for prog in test_programs do for engine in engines do
                        tasks.add new Task(engine, prog)
-                       c += 1
                end
        end
 end
@@ -250,6 +248,8 @@ class Controller
                                        if res == 5 then result.fail = true
                                        if res == 6 then result.soso = true
                                        if res == 7 then result.skip = true
+                                       if res == 8 then result.todo = true
+                                       if res == 9 then result.skip_exec = true
                                        if res == 0 then result.unknown = true
 
                                        results.add result
@@ -305,6 +305,8 @@ class Controller
                print "* {results.fixmes.length} fixmes"
                print "* {results.sosos.length} sosos"
                print "* {results.skips.length} skips"
+               print "* {results.todos.length} todos"
+               print "* {results.skip_execs.length} skip execs"
                print "* {results.unknowns.length} unknowns (bug in tests.sh or nitester)"
        end
 
@@ -332,12 +334,12 @@ class Worker
        # Output file directory
        var out_dir = "/dev/shm/nit_out{rank}" is lazy
 
+       # Directory to store the xml files produced for Jenkins
+       var xml_dir = "~/jenkins_xml/"
+
        # Output file of the `tests.sh` script
        var tests_sh_out = "/dev/shm/nit_local_out{rank}" is lazy
 
-       # Path to the local copy of the Nit repository
-       var nit_copy_dir = "/dev/shm/nit{rank}/" is lazy
-
        # Source Nit repository, must be already updated and `make` before execution
        var nit_source_dir = "~/nit"
 
@@ -362,7 +364,6 @@ class Worker
        fun setup
        do
                if verbose > 0 then sys.system "hostname"
-               sys.system "git clone {nit_source_dir} {nit_copy_dir}"
        end
 
        # Clean up the testing environment
@@ -372,7 +373,6 @@ class Worker
        do
                if comp_dir.file_exists then comp_dir.rmdir
                if out_dir.file_exists then out_dir.rmdir
-               if nit_copy_dir.file_exists then nit_copy_dir.rmdir
                if tests_sh_out.file_exists then tests_sh_out.file_delete
        end
 
@@ -395,17 +395,17 @@ class Worker
                                # Receive tasks to execute
                                mpi.recv_into(task_buffer, 0, 1, status.source, status.tag, comm_world)
                                var first_id = task_buffer[0]
-                               for task_id in [first_id .. first_id + tasks_per_packet] do
+                               for task_id in [first_id .. first_id + tasks_per_packet[ do
 
                                        # If id is over all known tasks, stop right here
                                        if task_id >= tasks.length then break
                                        var task = tasks[task_id]
 
                                        # Command line to execute test
-                                       var cmd = "XMLDIR={out_dir} ERRLIST={out_dir}/errlist TMPDIR={out_dir} " +
+                                       var cmd = "XMLDIR={xml_dir} ERRLIST={out_dir}/errlist TMPDIR={out_dir} " +
                                                "CCACHE_DIR={ccache_dir} CCACHE_TEMPDIR={ccache_dir} CCACHE_BASEDIR={comp_dir} " +
-                                               "./tests.sh --compdir {comp_dir} --outdir {out_dir} -o \"--make-flags '-j1'\"" +
-                                               " --node --engine {task.engine} {nit_copy_dir / "tests" / task.test_program} > {tests_sh_out}"
+                                               "./tests.sh --compdir {comp_dir} --outdir {out_dir} " +
+                                               " --node --engine {task.engine} {task.test_program} > {tests_sh_out}"
 
                                        # Execute test
                                        sys.system cmd
@@ -446,6 +446,8 @@ class Worker
                                                if line.has("[======= fail") then res = 5
                                                if line.has("[======= soso") then res = 6
                                                if line.has("[skip]") then res = 7
+                                               if line.has("[todo]") then res = 8
+                                               if line.has("[skip exec]") then res = 9
 
                                                if res == null then
                                                        res = 0
@@ -552,9 +554,15 @@ class Result
        # Is `self` result a _soso_?
        var soso = false
 
-       # Is `self` skipped test?
+       # Has `self` been skipped?
        var skip = false
 
+       # Is `self` TODO?
+       var todo = false
+
+       # Has the execution of `self` been skipped?
+       var skip_exec = false
+
        # Is `self` an unknown result, probably an error
        var unknown = false
 
@@ -566,6 +574,10 @@ class Result
                if ok_empty then err = "0k"
                if fixme then err = "fixme"
                if fail then err = "fail"
+               if soso then err = "soso"
+               if skip then err = "skip"
+               if todo then err = "todo"
+               if skip_exec then err = "skip_exec"
 
                return "{task} arg{arg} alt{alt} => {err}"
        end
@@ -582,6 +594,8 @@ class ResultSet
        var fails = new HashSet[Result]
        var sosos = new HashSet[Result]
        var skips = new HashSet[Result]
+       var todos = new HashSet[Result]
+       var skip_execs = new HashSet[Result]
        var unknowns = new HashSet[Result]
 
        # TODO remove
@@ -596,6 +610,8 @@ class ResultSet
                if result.fail then fails.add result
                if result.soso then sosos.add result
                if result.skip then skips.add result
+               if result.todo then todos.add result
+               if result.skip_exec then skip_execs.add result
                if result.unknown then unknowns.add result
 
                super
index 782fee6..d07c1be 100644 (file)
@@ -285,6 +285,6 @@ for arg in args do
                break
        end
 
-       print "Solved, after looking at {r.steps} positions during {c.lapse}"
+       print "Solved, after looking at {r.steps} positions"
        pb.print_plan(r.plan)
 end
diff --git a/misc/jenkins/nitester-wrapper.sh b/misc/jenkins/nitester-wrapper.sh
new file mode 100755 (executable)
index 0000000..2f9e20f
--- /dev/null
@@ -0,0 +1,47 @@
+#!/bin/bash
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is a wrapper for `nitester` which also manages a local repo
+#
+# The first argument _must_ be the hash of the commit at the head of the
+# branch to test. The other arguments are passed on to `nitester`.
+
+hash=$1
+shift
+
+set +x
+
+local_repo=nit/
+remote_repo=privat
+
+tools_dir=misc/jenkins/
+
+cd $local_repo
+git clean -fdxq .
+
+git fetch $remote_repo
+git checkout $hash
+
+# Make nitg and tools
+$tools_dir/unitrun.sh "run-make-0initial_make" make
+
+# Make nitester
+$tools_dir/unitrun.sh "run-make-nitester" make -C contrib/nitester/
+
+# Run tests
+cd tests
+mkdir -p out
+rm ~/jenkins_xml/*.xml
+mpirun -np 30 ../contrib/nitester/bin/nitester $@
index 54d875c..10dfd38 100755 (executable)
@@ -24,9 +24,18 @@ fi
 name=$1
 shift
 
+# Detect a working time command
+if env time --quiet -f%U true 2>/dev/null; then
+       TIME="env time --quiet -f%U -o '${name}.t.out'"
+elif env time -f%U true 2>/dev/null; then
+       TIME="env time -f%U -o '${name}.t.out'"
+else
+       TIME=
+fi
+
 # Magic here! This tee and save both stdout and stderr in distinct files without messing with them
 # Time  just get the user time
-/usr/bin/time -f%U --quiet -o "${name}.t.out" "$@" > >(tee "${name}.out") 2> >(tee "${name}.2.out" >&2)
+$TIME "$@" > >(tee "${name}.out") 2> >(tee "${name}.2.out" >&2)
 res=$?
 
 c=`echo "${name%-*}" | tr "-" "."`
index e902b05..3522245 100644 (file)
@@ -53,12 +53,12 @@ do
                dot.mprojects.add(g.mproject)
        end
        var projectpath = toolcontext.output_dir.join_path("project_hierarchy.dot")
-       print "generating {projectpath}"
+       print "generating project_hierarchy.dot"
        dot.write_to_file(projectpath)
 
        var modulepath = toolcontext.output_dir.join_path("module_hierarchy.dot")
        dot.mprojects.add_all(model.mprojects)
-       print "generating {modulepath}"
+       print "generating module_hierarchy.dot"
        dot.write_to_file(modulepath)
 end
 
index 0a8973d..c057beb 100644 (file)
@@ -73,9 +73,8 @@ end
 var file = toolcontext.opt_output.value
 if file == null then file = "nitunit.xml"
 page.write_to_file(file)
-print "Results saved in {file}"
 # print docunits results
-print "\nDocUnits:"
+print "DocUnits:"
 if modelbuilder.unit_entities == 0 then
        print "No doc units found"
 else if modelbuilder.failed_entities == 0 and not toolcontext.opt_noact.value then
index 087e573..778d0d3 100755 (executable)
@@ -1,5 +1,7 @@
 #!/bin/sh
-printf "%s\n" "$@" *.nit \
+printf "%s\n" "$@" \
+       ../src/nit*.nit \
+       ../src/test_*.nit \
        ../examples/*.nit \
        ../examples/*/*.nit \
        ../examples/shoot/src/shoot_logic.nit \
@@ -12,5 +14,4 @@ printf "%s\n" "$@" *.nit \
        ../lib/*/examples/*.nit \
        ../contrib/friendz/src/solver_cmd.nit \
        ../contrib/pep8analysis/src/pep8analysis.nit \
-       ../src/nit*.nit \
-       ../src/test_*.nit
+       *.nit
index 481e468..e5ac8bd 100644 (file)
@@ -470,8 +470,8 @@ Average number of composing class definition by runtime class: 2.00
 Total size of tables (classes and instances): 38 (not including stuff like info for subtyping or call-next-method)
 Average size of table by runtime class: 6.33
 Values never redefined: 32 (84.21%)
-generating out/nitmetrics_args1.write/project_hierarchy.dot
-generating out/nitmetrics_args1.write/module_hierarchy.dot
+generating project_hierarchy.dot
+generating module_hierarchy.dot
 
 # Inheritance metrics
 
index 3b5b590..3a07d20 100644 (file)
@@ -4,8 +4,6 @@ test_nitunit.nit:23,2--25,0: FAILURE: nitunit.test_nitunit.test_nitunit::X.test_
 
 test_test_nitunit.nit:36,2--40,4: ERROR: test_foo1 (in file .nitunit/test_test_nitunit_TestX_test_foo1.nit): Runtime error: Assert failed (test_test_nitunit.nit:39)
 
-Results saved in out/nitunit_args1.write
-
 DocUnits:
 Entities: 27; Documented ones: 3; With nitunits: 3; Failures: 2
 
index 17ccd5a..6c361fe 100644 (file)
@@ -2,5 +2,5 @@ Initial: eabf.cdgh
 eab
 f.c
 dgh
-Solved, after looking at 14 positions during 0.0s
+Solved, after looking at 14 positions
 Solution in 10 moves: right(>) down(v) left(<) left(<) up(^) right(>) right(>) up(^) left(<) left(<)
index 8179c9c..76ecf2c 100755 (executable)
@@ -457,7 +457,9 @@ todos=""
 if [ "x$XMLDIR" = "x" ]; then
        xml="tests-$engine.xml"
 else
-       xml="$XMLDIR/tests-$engine.xml"
+       sum=`echo $@ | md5sum | cut -f1 -d " "`
+       xml="$XMLDIR/tests-$engine-$sum.xml"
+       mkdir -p "$XMLDIR"
 fi
 
 echo >$xml "<testsuites><testsuite>"