benchmarks/bench_languages.sh

   1 #!/bin/bash
   2 # This file is part of NIT ( http://www.nitlanguage.org ).
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This shell script helps running benchmarks
  17
  18 # TODO: cleanup and libify the helper-parts
  19
  20 source ./bench_plot.sh
  21
  22 ## CONFIGURATION OPTIONS ##
  23
  24 # Default number of times a command must be run with bench_command
  25 # Can be overrided with 'the option -n'
  26 count=2
  27
  28 ### HELPER FUNCTIONS ##
  29
  30 function die()
  31 {
  32         echo >&2 "error: $*"
  33         died=1
  34 }
  35
  36 # Run a single command multiple time and store the execution times
  37 # in the current $res file.
  38 #
  39 #  $1: title of the command
  40 #  $2: long desription of the command
  41 #  rest: the command to execute
  42 function bench_command()
  43 {
  44         if [ "$dry_run" = "true" ]; then return; fi
  45         local title="$1"
  46         local desc="$2"
  47         shift
  48         shift
  49         if test "$verbose" = true; then outputopts="/dev/stdout"; else outputopts="/dev/null"; fi
  50         timeout="time.out"
  51         echo "$title" > "$timeout"
  52         echo "# $desc" >> "$timeout"
  53         echo "\$ $@" >> "$timeout"
  54         echo
  55         echo "** [$title] $desc **"
  56         echo " $ $@"
  57
  58         # Execute the commands $count times
  59         for i in `seq 1 "$count"`; do
  60                 (ulimit -t 300; /usr/bin/time -f "%U" -o "$timeout" -a "$@") > $outputopts 2>&1 || die "$1: failed"
  61                 echo -n "$i. "
  62                 tail -n 1 "$timeout"
  63         done
  64
  65         line=`compute_stats "$timeout"`
  66         echo "$line ($res)"
  67         echo $line >> "$res"
  68         rm $timeout
  69 }
  70
  71 # Run a simple command witout storing the execution time
  72 # Used to display command on verbose and skip long executions when dry_run is given
  73 # $@ command to execute
  74 function run_command()
  75 {
  76         if [ "$dry_run" = "true" ]; then return; fi
  77         echo " $ $@"
  78         (ulimit -t 180; "$@") || die "$@: failed"
  79 }
  80
  81 # Check if the test should be skiped according to its name
  82 # $1: name of the test
  83 # $2: description of the test
  84 # $NOTSKIPED: arguments
  85 function skip_test()
  86 {
  87         if test -z "$NOTSKIPED"; then
  88                 echo "* $1"
  89                 return 0
  90         fi
  91         if test "$NOTSKIPED" = "all"; then
  92                 : # Execute anyway
  93         elif echo "$1" | egrep "$NOTSKIPED" >/dev/null 2>&1; then
  94                 : # Found one to execute
  95         else
  96                 return 0
  97         fi
  98         echo "*"
  99         echo "* $1 *****"
 100         echo "*"
 101         return 1
 102 }
 103
 104 ## HANDLE OPTIONS ##
 105
 106 function usage()
 107 {
 108         echo "run_bench: [options]* benchname"
 109         echo "  -v: verbose mode"
 110         echo "  -n count: number of execution for each bar (default: $count)"
 111         echo "  --dry: Do not run the commands, just reuse the data and generate the graph"
 112         echo "  --fast: Run less and faster tests"
 113         echo "  -h: this help"
 114 }
 115
 116 stop=false
 117 while [ "$stop" = false ]; do
 118         case "$1" in
 119                 -v) verbose=true; shift;;
 120                 -h) usage; exit;;
 121                 -n) count="$2"; shift; shift;;
 122                 --dry) dry_run=true; shift;;
 123                 --fast) fast=true; shift;;
 124                 *) stop=true
 125         esac
 126 done
 127
 128 NOTSKIPED="$*"
 129
 130 if test -z "$NOTSKIPED"; then
 131         usage
 132         echo "List of available benches:"
 133         echo "* all: run all the benches"
 134 fi
 135
 136 ## COMPILE ENGINES
 137 cd ../src
 138 test -f ./nitc_3 || ./ncall.sh -O
 139 cd ../benchmarks
 140 test -f ./nitg || ../src/nitc_3 ../src/nitg.nit -O -v
 141
 142 ## EFFECTIVE BENCHS ##
 143
 144 function bench_language()
 145 {
 146         name="$1"
 147         skip_test "$name" && return
 148         rootdir=`pwd`
 149         basedir="./${name}.out"
 150
 151         mkdir $basedir
 152
 153         t=t
 154         s=20
 155         seq="2 4 8"
 156         for b in $seq; do
 157                 run_command ./nitg languages/$name.nit -o $basedir/$name.bin
 158                 run_command $basedir/$name.bin $basedir "${t}_$b" "$b"
 159         done
 160
 161         prepare_res $basedir/$name-g++.dat "g++" "g++"
 162         cppdir="${basedir}/cpp"
 163         for b in $seq; do
 164                 run_command g++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.g++.bin"
 165                 bench_command "$b" "" "${cppdir}/${t}_$b.g++.bin" $s
 166         done
 167
 168         prepare_res $basedir/$name-clang++.dat "clang++" "clang++"
 169         for b in $seq; do
 170                 run_command clang++ "${cppdir}/${t}_$b.cpp" -O2 -o "${cppdir}/${t}_$b.clang++.bin"
 171                 bench_command "$b" "" "${cppdir}/${t}_$b.clang++.bin" $s
 172         done
 173
 174         prepare_res $basedir/$name-java.dat "java" "java"
 175         javadir="${basedir}/java"
 176         for b in $seq; do
 177                 run_command javac "${javadir}/${t}_$b.java"
 178                 bench_command "$b" "" java -cp "${javadir}/" "${t}_$b" $s
 179         done
 180
 181         prepare_res $basedir/$name-gcj.dat "gcj" "gcj"
 182         for b in $seq; do
 183                 run_command gcj --main=${t}_$b -O2 "${javadir}/${t}_$b.java" -o "${javadir}/${t}_$b.gcj.bin"
 184                 bench_command "$b" "" "${javadir}/${t}_$b.gcj.bin" $s
 185         done
 186
 187         prepare_res $basedir/$name-scala.dat "scala" "scala"
 188         scaladir="${basedir}/scala"
 189         for b in $seq; do
 190                 run_command scalac "${scaladir}/${t}_$b.scala" -d "${scaladir}"
 191                 bench_command "$b" "" scala -cp "${scaladir}/" "${t}_$b" $s
 192         done
 193
 194         prepare_res $basedir/$name-cs.dat "c#" "c#"
 195         csdir="${basedir}/cs"
 196         for b in $seq; do
 197                 run_command gmcs "$csdir/${t}_$b.cs"
 198                 bench_command "$b" "" mono "$csdir/${t}_$b.exe" $s
 199         done
 200
 201         prepare_res $basedir/$name-es.dat "es" "es"
 202         esdir="${basedir}/es"
 203         for b in $seq; do
 204                 cd $esdir
 205                 run_command ec -clean -finalize ${t}_$b/app${t}_$b.e
 206                 chmod +x app${t}_$b
 207                 mv app${t}_$b ${t}_$b.es.bin
 208                 cd $rootdir
 209                 bench_command "$b" "" "$esdir/${t}_$b.es.bin" $s
 210         done
 211
 212         prepare_res $basedir/$name-se.dat "se" "se"
 213         sedir="${basedir}/se"
 214         for b in $seq; do
 215                 cd $sedir
 216                 run_command se compile -no_check app${t}_${b}_se.e -loadpath ${t}_${b}_se -o ${t}_$b.se.bin
 217                 cd $rootdir
 218                 bench_command "$b" "" "$sedir/${t}_$b.se.bin" $s
 219         done
 220
 221         nitdir="${basedir}/nit"
 222         prepare_res $nitdir/$name-nitg.dat "nitg" "nitg"
 223         for b in $seq; do
 224                 run_command ./nitg $nitdir/${t}_$b.nit --global -o "$nitdir/${t}_$b.nitg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 225                 bench_command "$b" "" "$nitdir/${t}_$b.nitg.bin" $s
 226         done
 227
 228         prepare_res $nitdir/$name-nitg-s.dat "nitg-s" "nitg-s"
 229         for b in $seq; do
 230                 run_command ./nitg $nitdir/${t}_$b.nit --separate -o "$nitdir/${t}_$b.nitg-s.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 231                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-s.bin" $s
 232         done
 233
 234 <<XXX
 235         tg="nitg-s-bm"
 236         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 237         for b in $seq; do
 238                 run_command ./nitg $nitdir/${t}_$b.nit --separate --bm-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 239                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 240         done
 241
 242         tg="nitg-s-pha"
 243         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 244         for b in $seq; do
 245                 run_command ./nitg $nitdir/${t}_$b.nit --separate --phand-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 246                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 247         done
 248
 249         tg="nitg-s-phm"
 250         prepare_res $nitdir/$name-$tg.dat "$tg" "$tg"
 251         for b in $seq; do
 252                 run_command ./nitg $nitdir/${t}_$b.nit --separate --phmod-typing -o "$nitdir/${t}_$b.$tg.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 253                 bench_command "$b" "" "$nitdir/${t}_$b.$tg.bin" $s
 254         done
 255
 256         prepare_res $nitdir/$name-nitg-su.dat "nitg-su" "nitg-su"
 257         for b in $seq; do
 258                 run_command ./nitg $nitdir/${t}_$b.nit --separate --no-check-covariance -o "$nitdir/${t}_$b.nitg-su.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 259                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-su.bin" $s
 260         done
 261
 262         prepare_res $nitdir/$name-nitg-e.dat "nitg-e" "nitg-e"
 263         for b in $seq; do
 264                 run_command ./nitg $nitdir/${t}_$b.nit --erasure -o "$nitdir/${t}_$b.nitg-e.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 265                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-e.bin" $s
 266         done
 267
 268         prepare_res $nitdir/$name-nitg-eu.dat "nitg-eu" "nitg-eu"
 269         for b in $seq; do
 270                 run_command ./nitg $nitdir/${t}_$b.nit --erasure --no-check-covariance --no-check-erasure-cast -o "$nitdir/${t}_$b.nitg-eu.bin" --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\""
 271                 bench_command "$b" "" "$nitdir/${t}_$b.nitg-eu.bin" $s
 272         done
 273 XXX
 274
 275         plot $basedir/$name.gnu
 276 }
 277
 278 for name in languages/*.nit; do
 279         n=`basename $name .nit`
 280         bench_language $n
 281 done
 282
 283 if test -n "$died"; then
 284         echo "Some commands failed"
 285         exit 1
 286 fi
 287 exit 0