benchmarks/bench_old.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script run and compare nitc with the one in origin/master
  17
  18 # execute a command multiple time and store its executin metrics
  19 # $1 = output file (.tsv)
  20 # $2. = command to execute
  21 bench_one() {
  22         out=$1
  23         shift
  24         > "$out"
  25         echo "> $*"
  26         echo -e "user\treal\t%cpu\tmem"
  27         for i in `seq 6`; do
  28                 /usr/bin/time -o "$out" -a -f "%U\t%e\t%P\t%M" "$@" >/dev/null
  29                 tail -n 1 "$out"
  30         done
  31 }
  32
  33 # Get column $2, $3-th lowest value of tsv file $1
  34 get() {
  35         cut -f "$2" < "$1" | sort -n | head -n "$3" | tail -n 1
  36 }
  37
  38 # $1 + result% = $2
  39 percent() {
  40         perl -e "printf '%+.1f', ($2/$1-1)*100;"
  41 }
  42
  43 # compare two tsv generated by `bench_one`.
  44 # displays numbers and returns 1 if some execution metrics is above a threshold
  45 compare() {
  46         # Chose the median value
  47         u1=`get "$1" 1 3`
  48         e1=`get "$1" 2 3`
  49         m1=`get "$1" 4 3`
  50         u2=`get "$2" 1 3`
  51         e2=`get "$2" 2 3`
  52         m2=`get "$2" 4 3`
  53         ud=`percent "$u1" "$u2"`
  54         ed=`percent "$e1" "$e2"`
  55         md=`percent "$m1" "$m2"`
  56         echo "Before: user=$u1 real=$e1 mem=$m1"
  57         echo "After: user=$u2 ($ud%) real=$e2 ($ed%) mem=$m2 ($md%)"
  58         # Fail if above threshold
  59         perl -e "exit ($ud>=3 || $ed>=5 || $md>=0.3)"
  60 }
  61
  62 set -e
  63 #set -x
  64
  65 old=`git show --pretty="%h %D" --no-patch origin/master`
  66 new=`git show --pretty="%h %D" --no-patch`
  67 if [ ! -f bin/nitc_old ]; then
  68         git checkout origin/master
  69         nitc --semi-global src/nitc.nit -o bin/nitc_old -v
  70         nitc --semi-global src/nit.nit -o bin/nit_old -v
  71         git checkout -
  72 fi
  73
  74 bench_one "nitc_old.tsv" nitc_old --no-cc src/nitc.nit
  75 bench_one "nitc_new.tsv" nitc --no-cc src/nitc.nit
  76 bench_one "nit_old.tsv" nit_old -- src/nitls.nit -p lib/core
  77 bench_one "nit_new.tsv" nit -- src/nitls.nit -p lib/core
  78
  79 echo
  80 echo "Before is $old"
  81 echo "After is $new"
  82 echo "For nitc src/nitls.nit:"
  83 ret=0
  84 compare "nitc_old.tsv" "nitc_new.tsv" || ret=1
  85 echo "For nit src/nitls.nit -p lib/core:"
  86 compare "nit_old.tsv" "nit_new.tsv" || ret=1
  87 exit "$ret"