ci: basic bench with old branch

Signed-off-by: Jean Privat <jean@pryen.org>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70b91e1..eef488d 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -379,8 +379,6 @@ build_catalog:
     - ./oot.sh pre-build
     - cd ..
     - nitcatalog -d catalog.out lib/ examples/ contrib/ contrib/oot/
-  dependencies:
-    - build_more_tools
   artifacts:
     paths:
       - catalog.out
@@ -403,3 +401,14 @@ test_full_nitcs_macos:
     - macos
   dependencies:
     - build_tools_macos
+
+bench_old:
+  stage: more_test
+  tags:
+    - perf
+  dependencies:
+    - build_tools
+  script:
+    - benchmarks/bench_old.sh
+  allow_failure: true # time is unreliable. manual check required
+  services: []

diff --git a/benchmarks/bench_old.sh b/benchmarks/bench_old.sh
new file mode 100755 (executable)
index 0000000..0111aa7
--- /dev/null
+++ b/benchmarks/bench_old.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script run and compare nitc with the one in origin/master
+
+# execute a command multiple time and store its executin metrics
+# $1 = output file (.tsv)
+# $2. = command to execute
+bench_one() {
+       out=$1
+       shift
+       > "$out"
+       echo "> $*"
+       echo -e "user\treal\t%cpu\tmem"
+       for i in `seq 6`; do
+               /usr/bin/time -o "$out" -a -f "%U\t%e\t%P\t%M" "$@" >/dev/null
+               tail -n 1 "$out"
+       done
+}
+
+# Get column $2, $3-th lowest value of tsv file $1
+get() {
+       cut -f "$2" < "$1" | sort -n | head -n "$3" | tail -n 1
+}
+
+# $1 + result% = $2
+percent() {
+       perl -e "printf '%+.1f', ($2/$1-1)*100;"
+}
+
+# compare two tsv generated by `bench_one`.
+# displays numbers and returns 1 if some execution metrics is above a threshold
+compare() {
+       # Chose the median value
+       u1=`get "$1" 1 3`
+       e1=`get "$1" 2 3`
+       m1=`get "$1" 4 3`
+       u2=`get "$2" 1 3`
+       e2=`get "$2" 2 3`
+       m2=`get "$2" 4 3`
+       ud=`percent "$u1" "$u2"`
+       ed=`percent "$e1" "$e2"`
+       md=`percent "$m1" "$m2"`
+       echo "Before: user=$u1 real=$e1 mem=$m1"
+       echo "After: user=$u2 ($ud%) real=$e2 ($ed%) mem=$m2 ($md%)"
+       # Fail if above threshold
+       perl -e "exit ($ud>=3 || $ed>=5 || $md>=0.3)"
+}
+
+set -e
+#set -x
+
+old=`git show --pretty="%h %D" --no-patch origin/master`
+new=`git show --pretty="%h %D" --no-patch`
+if [ ! -f bin/nitc_old ]; then
+       git checkout origin/master
+       nitc --semi-global src/nitc.nit -o bin/nitc_old -v
+       nitc --semi-global src/nit.nit -o bin/nit_old -v
+       git checkout -
+fi
+
+bench_one "nitc_old.tsv" nitc_old --no-cc src/nitc.nit
+bench_one "nitc_new.tsv" nitc --no-cc src/nitc.nit
+bench_one "nit_old.tsv" nit_old -- src/nitls.nit -p lib/core
+bench_one "nit_new.tsv" nit -- src/nitls.nit -p lib/core
+
+echo
+echo "Before is $old"
+echo "After is $new"
+echo "For nitc src/nitls.nit:"
+ret=0
+compare "nitc_old.tsv" "nitc_new.tsv" || ret=1
+echo "For nit src/nitls.nit -p lib/core:"
+compare "nit_old.tsv" "nit_new.tsv" || ret=1
+exit "$ret"