Merge: ci: basic bench with old branch

This adds a job where a basic execution is compared against origin/master is order to catch important time or space regressions. Because the test machine is shared with many jobs, the results are not that reliable and some additional checks by the integrator might be required.

However, the script can help developers to do a basic check to document their PR.

Pull-Request: #2739
Reviewed-by: Alexis Laferrière <alexis.laf@xymus.net>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70b91e1..eef488d 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -379,8 +379,6 @@ build_catalog:
     - ./oot.sh pre-build
     - cd ..
     - nitcatalog -d catalog.out lib/ examples/ contrib/ contrib/oot/
-  dependencies:
-    - build_more_tools
   artifacts:
     paths:
       - catalog.out
@@ -403,3 +401,14 @@ test_full_nitcs_macos:
     - macos
   dependencies:
     - build_tools_macos
+
+bench_old:
+  stage: more_test
+  tags:
+    - perf
+  dependencies:
+    - build_tools
+  script:
+    - benchmarks/bench_old.sh
+  allow_failure: true # time is unreliable. manual check required
+  services: []

diff --git a/benchmarks/bench_old.sh b/benchmarks/bench_old.sh
new file mode 100755 (executable)
index 0000000..0111aa7
--- /dev/null
+++ b/benchmarks/bench_old.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script run and compare nitc with the one in origin/master
+
+# execute a command multiple time and store its executin metrics
+# $1 = output file (.tsv)
+# $2. = command to execute
+bench_one() {
+       out=$1
+       shift
+       > "$out"
+       echo "> $*"
+       echo -e "user\treal\t%cpu\tmem"
+       for i in `seq 6`; do
+               /usr/bin/time -o "$out" -a -f "%U\t%e\t%P\t%M" "$@" >/dev/null
+               tail -n 1 "$out"
+       done
+}
+
+# Get column $2, $3-th lowest value of tsv file $1
+get() {
+       cut -f "$2" < "$1" | sort -n | head -n "$3" | tail -n 1
+}
+
+# $1 + result% = $2
+percent() {
+       perl -e "printf '%+.1f', ($2/$1-1)*100;"
+}
+
+# compare two tsv generated by `bench_one`.
+# displays numbers and returns 1 if some execution metrics is above a threshold
+compare() {
+       # Chose the median value
+       u1=`get "$1" 1 3`
+       e1=`get "$1" 2 3`
+       m1=`get "$1" 4 3`
+       u2=`get "$2" 1 3`
+       e2=`get "$2" 2 3`
+       m2=`get "$2" 4 3`
+       ud=`percent "$u1" "$u2"`
+       ed=`percent "$e1" "$e2"`
+       md=`percent "$m1" "$m2"`
+       echo "Before: user=$u1 real=$e1 mem=$m1"
+       echo "After: user=$u2 ($ud%) real=$e2 ($ed%) mem=$m2 ($md%)"
+       # Fail if above threshold
+       perl -e "exit ($ud>=3 || $ed>=5 || $md>=0.3)"
+}
+
+set -e
+#set -x
+
+old=`git show --pretty="%h %D" --no-patch origin/master`
+new=`git show --pretty="%h %D" --no-patch`
+if [ ! -f bin/nitc_old ]; then
+       git checkout origin/master
+       nitc --semi-global src/nitc.nit -o bin/nitc_old -v
+       nitc --semi-global src/nit.nit -o bin/nit_old -v
+       git checkout -
+fi
+
+bench_one "nitc_old.tsv" nitc_old --no-cc src/nitc.nit
+bench_one "nitc_new.tsv" nitc --no-cc src/nitc.nit
+bench_one "nit_old.tsv" nit_old -- src/nitls.nit -p lib/core
+bench_one "nit_new.tsv" nit -- src/nitls.nit -p lib/core
+
+echo
+echo "Before is $old"
+echo "After is $new"
+echo "For nitc src/nitls.nit:"
+ret=0
+compare "nitc_old.tsv" "nitc_new.tsv" || ret=1
+echo "For nit src/nitls.nit -p lib/core:"
+compare "nit_old.tsv" "nit_new.tsv" || ret=1
+exit "$ret"