From e22eff6c202e52a7a41b748dee68408ae6b77fe0 Mon Sep 17 00:00:00 2001
From: Jean Privat <jean@pryen.org>
Date: Fri, 22 Feb 2019 22:13:05 -0500
Subject: [PATCH] ci: basic bench with old branch

Signed-off-by: Jean Privat <jean@pryen.org>
---
 .gitlab-ci.yml          |   13 +++++--
 benchmarks/bench_old.sh |   87 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+), 2 deletions(-)
 create mode 100755 benchmarks/bench_old.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70b91e1..eef488d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -379,8 +379,6 @@ build_catalog:
     - ./oot.sh pre-build
     - cd ..
     - nitcatalog -d catalog.out lib/ examples/ contrib/ contrib/oot/
-  dependencies:
-    - build_more_tools
   artifacts:
     paths:
       - catalog.out
@@ -403,3 +401,14 @@ test_full_nitcs_macos:
     - macos
   dependencies:
     - build_tools_macos
+
+bench_old:
+  stage: more_test
+  tags:
+    - perf
+  dependencies:
+    - build_tools
+  script:
+    - benchmarks/bench_old.sh
+  allow_failure: true # time is unreliable. manual check required
+  services: []
diff --git a/benchmarks/bench_old.sh b/benchmarks/bench_old.sh
new file mode 100755
index 0000000..0111aa7
--- /dev/null
+++ b/benchmarks/bench_old.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script run and compare nitc with the one in origin/master
+
+# execute a command multiple time and store its executin metrics
+# $1 = output file (.tsv)
+# $2. = command to execute
+bench_one() {
+	out=$1
+	shift
+	> "$out"
+	echo "> $*"
+	echo -e "user\treal\t%cpu\tmem"
+	for i in `seq 6`; do
+		/usr/bin/time -o "$out" -a -f "%U\t%e\t%P\t%M" "$@" >/dev/null
+		tail -n 1 "$out"
+	done
+}
+
+# Get column $2, $3-th lowest value of tsv file $1
+get() {
+	cut -f "$2" < "$1" | sort -n | head -n "$3" | tail -n 1
+}
+
+# $1 + result% = $2
+percent() {
+	perl -e "printf '%+.1f', ($2/$1-1)*100;"
+}
+
+# compare two tsv generated by `bench_one`.
+# displays numbers and returns 1 if some execution metrics is above a threshold
+compare() {
+	# Chose the median value
+	u1=`get "$1" 1 3`
+	e1=`get "$1" 2 3`
+	m1=`get "$1" 4 3`
+	u2=`get "$2" 1 3`
+	e2=`get "$2" 2 3`
+	m2=`get "$2" 4 3`
+	ud=`percent "$u1" "$u2"`
+	ed=`percent "$e1" "$e2"`
+	md=`percent "$m1" "$m2"`
+	echo "Before: user=$u1 real=$e1 mem=$m1"
+	echo "After: user=$u2 ($ud%) real=$e2 ($ed%) mem=$m2 ($md%)"
+	# Fail if above threshold
+	perl -e "exit ($ud>=3 || $ed>=5 || $md>=0.3)"
+}
+
+set -e
+#set -x
+
+old=`git show --pretty="%h %D" --no-patch origin/master`
+new=`git show --pretty="%h %D" --no-patch`
+if [ ! -f bin/nitc_old ]; then
+	git checkout origin/master
+	nitc --semi-global src/nitc.nit -o bin/nitc_old -v
+	nitc --semi-global src/nit.nit -o bin/nit_old -v
+	git checkout -
+fi
+
+bench_one "nitc_old.tsv" nitc_old --no-cc src/nitc.nit
+bench_one "nitc_new.tsv" nitc --no-cc src/nitc.nit
+bench_one "nit_old.tsv" nit_old -- src/nitls.nit -p lib/core
+bench_one "nit_new.tsv" nit -- src/nitls.nit -p lib/core
+
+echo
+echo "Before is $old"
+echo "After is $new"
+echo "For nitc src/nitls.nit:"
+ret=0
+compare "nitc_old.tsv" "nitc_new.tsv" || ret=1
+echo "For nit src/nitls.nit -p lib/core:"
+compare "nit_old.tsv" "nit_new.tsv" || ret=1
+exit "$ret"
-- 
1.7.9.5