From: Jean Privat Date: Wed, 30 Jul 2014 18:08:58 +0000 (-0400) Subject: Merge: Toplevel for sys and exit X-Git-Tag: v0.6.7~6 X-Git-Url: http://nitlanguage.org?hp=70a712bd6d5bb8236f27bf414dc968366469e1df Merge: Toplevel for sys and exit Because the new c_src can deal with it Pull-Request: #628 Reviewed-by: Alexandre Terrasa Reviewed-by: Lucas Bajolet --- diff --git a/.gitignore b/.gitignore index eed3179..9b98202 100644 --- a/.gitignore +++ b/.gitignore @@ -10,13 +10,11 @@ EIFGENs bin/nit* doc/stdlib doc/nitc -doc/newmodel *.aux *.log *.out *.toc -doc/nit_version.sty src/parser/.nit.sablecc3 src/parser/.nit.sablecc3.dump @@ -28,6 +26,7 @@ src/*.dot src/*.dat src/*.gnu src/*.bin +src/nitg_0 c_src/*.o c_src/*.cksum @@ -47,6 +46,7 @@ tests/alt tests/errlist tests/out tests/*.xml +tests/nitunit *.stub.nit.[ch] diff --git a/benchmarks/bench_strings.sh b/benchmarks/bench_strings.sh new file mode 100755 index 0000000..811eddd --- /dev/null +++ b/benchmarks/bench_strings.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./bench_common.sh +source ./bench_plot.sh + +# Default number of times a command must be run with bench_command +# Can be overrided with 'the option -n' +count=2 + +function usage() +{ + echo "run_bench: [options]* bench_name args" + echo " -v: verbose mode" + echo " -n count: number of execution for each bar (default: $count)" + echo " -h: this help" + echo "" + echo "Benches : " + echo " all : all benches" + echo " - usage : * max_nb_cct loops strlen" + echo " iter: bench iterations" + echo " - usage : iter max_nb_cct loops strlen" + echo " cct: concatenation benching" + echo " - usage : cct max_nb_cct loops strlen" + echo " substr: substring benching" + echo " - usage : substr max_nb_cct loops strlen" +} + +function benches() +{ + bench_concat $@; + bench_iteration $@; + bench_substr $@; +} + +function bench_concat() +{ + if $verbose; then + echo "*** Benching concat performance ***" + fi + + prepare_res concat_ropes.out concat_ropes ropes + if $verbose; then + echo "Ropes :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, Concats/loop = $2, Loops = $3" + fi + bench_command $i ropes$i ./chain_concat -m rope --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res concat_flat.out concat_flat flatstring + if $verbose; then + echo "FlatStrings :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, Concats/loop = $2, Loops = $3" + fi + bench_command $i flatstring$i ./chain_concat -m flatstr --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res concat_buf.out concat_buf flatbuffer + if $verbose; then + echo "FlatBuffers :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, Concats/loop = $2, Loops = $3" + fi + bench_command $i flatbuffer$i ./chain_concat -m flatbuf --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + plot concat.gnu +} + +function bench_iteration() +{ + if $verbose; then + echo "*** Benching iteration performance ***" + fi + + prepare_res iter_ropes_iter.out iter_ropes_iter ropes_iter + if $verbose; then + echo "Ropes by iterator :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" + fi + bench_command $i ropes_iter$i ./iteration_bench -m rope --iter-mode iterator --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res iter_ropes_index.out iter_ropes_index ropes_index + if $verbose; then + echo "Ropes by index :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats (depth of the rope) = $i, Loops = $3" + fi + bench_command $i ropes_index$i ./iteration_bench -m rope --iter-mode index --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res iter_flat_iter.out iter_flat_iter flatstring_iter + if $verbose; then + echo "FlatStrings by iterator :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats = $i, Loops = $3" + fi + bench_command $i flatstr_iter$i ./iteration_bench -m flatstr --iter-mode iterator --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res iter_flat_index.out iter_flat_index flatstring_index + if $verbose; then + echo "FlatStrings by index :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats = $i, Loops = $3" + fi + bench_command $i flatstr_index$i ./iteration_bench -m flatstr --iter-mode index --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res iter_buf_iter.out iter_buf_iter flatbuffer_iter + if $verbose; then + echo "FlatBuffers by iterator :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats = $i, Loops = $3" + fi + bench_command $i flatbuf_iter$i ./iteration_bench -m flatbuf --iter-mode iterator --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res iter_buf_index.out iter_buf_index flatbuffer_index + if $verbose; then + echo "FlatBuffers by index:" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String base length = $1, Concats = $i, Loops = $3" + fi + bench_command $i flatbuf_index$i ./iteration_bench -m flatbuf --iter-mode index --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + plot iter.gnu +} + +function bench_substr() +{ + if $verbose; then + echo "*** Benching substring performance ***" + fi + + prepare_res substr_ropes.out substr_ropes ropes + if $verbose; then + echo "Ropes :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, loops = $2, Loops = $3" + fi + bench_command $i ropes$i ./substr_bench -m rope --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res substr_flat.out substr_flat flatstring + if $verbose; then + echo "FlatStrings :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, loops = $2, Loops = $3" + fi + bench_command $i flatstring$i ./substr_bench -m flatstr --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + prepare_res substr_buf.out substr_buf flatbuffer + if $verbose; then + echo "FlatBuffers :" + fi + for i in `seq 1 "$1"`; do + if $verbose; then + echo "String length = $i, loops = $2, Loops = $3" + fi + bench_command $i flatbuffer$i ./substr_bench -m flatbuf --loops $2 --strlen $3 --ccts $i "NIT_GC_CHOOSER=large" + done + + plot substr.gnu +} + +stop=false +while [ "$stop" = false ]; do + case "$1" in + -v) verbose=true; shift;; + -h) usage; exit;; + -n) count="$2"; shift; shift;; + *) stop=true + esac +done + +if test $# -ne 4; then + usage + exit +fi + +if $verbose; then + echo "Compiling" +fi + +../bin/nitg --global ./strings/chain_concat.nit --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\"" +../bin/nitg --global ./strings/iteration_bench.nit --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\"" +../bin/nitg --global ./strings/substr_bench.nit --make-flags "CFLAGS=\"-g -O2 -DNOBOEHM\"" + +case "$1" in + iter) shift; bench_iteration $@ ;; + cct) shift; bench_concat $@ ;; + substr) shift; bench_substr $@ ;; + all) shift; benches $@ ;; + *) usage; exit;; +esac diff --git a/benchmarks/strings/chain_concat.nit b/benchmarks/strings/chain_concat.nit new file mode 100644 index 0000000..35466f9 --- /dev/null +++ b/benchmarks/strings/chain_concat.nit @@ -0,0 +1,78 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# This file is free software, which comes along with NIT. This software is +# distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. You can modify it is you want, provided this header +# is kept unaltered, and a notification of the changes is added. +# You are allowed to redistribute it and sell it, alone or is a part of +# another product. + +# Benches measuring the performance of several concatenations on Text variants +module chain_concat + +import opts + +fun bench_flatstr(str_size: Int, nb_ccts: Int, loops: Int) +do + var lft = "a" * str_size + + for i in [0..loops] do + var str: String = lft + for j in [0..nb_ccts] do + str += lft + end + end +end + +fun bench_rope(str_size: Int, nb_ccts: Int, loops: Int) +do + var lft = "a" * str_size + + for i in [0..loops] do + var str: String = new RopeString.from(lft) + for j in [0..nb_ccts] do + str += lft + end + end +end + +fun bench_flatbuf(str_size: Int, nb_ccts: Int, loops: Int) +do + var lft = "a" * str_size + + for i in [0..loops] do + var buf = new FlatBuffer.from(lft) + for j in [0..nb_ccts] do + buf.append(lft) + end + buf.to_s + end +end + +var opts = new OptionContext +var mode = new OptionEnum(["rope", "flatstr", "flatbuf"], "Mode", -1, "-m") +var nb_ccts = new OptionInt("Number of concatenations per loop", -1, "--ccts") +var loops = new OptionInt("Number of loops to be done", -1, "--loops") +var strlen = new OptionInt("Length of the base string", -1, "--strlen") +opts.add_option(mode, nb_ccts, loops, strlen) + +opts.parse(args) + +if nb_ccts.value == -1 or loops.value == -1 or strlen.value == -1 then + opts.usage + exit -1 +end + +var modval = mode.value + +if modval == 0 then + bench_rope(strlen.value, nb_ccts.value, loops.value) +else if modval == 1 then + bench_flatstr(strlen.value, nb_ccts.value, loops.value) +else if modval == 2 then + bench_flatbuf(strlen.value, nb_ccts.value, loops.value) +else + opts.usage + exit -1 +end diff --git a/benchmarks/strings/iteration_bench.nit b/benchmarks/strings/iteration_bench.nit new file mode 100644 index 0000000..a8047d2 --- /dev/null +++ b/benchmarks/strings/iteration_bench.nit @@ -0,0 +1,163 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# This file is free software, which comes along with NIT. This software is +# distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. You can modify it is you want, provided this header +# is kept unaltered, and a notification of the changes is added. +# You are allowed to redistribute it and sell it, alone or is a part of +# another product. + +# Benches for iteration on variants of Text +module iteration_bench + +import opts + +fun bench_rope_iter(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x:String = new RopeString.from(a) + for i in [0 .. nb_cct] do x += a + var cnt = 0 + var c: Char + while cnt != loops do + for i in x do + c = i + end + cnt += 1 + end +end + +fun bench_rope_index(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x:String = new RopeString.from(a) + for i in [0 .. nb_cct] do x += a + var cnt = 0 + var c: Char + var pos = 0 + while cnt != loops do + pos = 0 + while pos < x.length do + c = x[pos] + pos += 1 + end + cnt += 1 + end +end + +fun bench_flatstr_iter(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = a + for i in [0 .. nb_cct] do x += a + var cnt = 0 + var c: Char + while cnt != loops do + for i in x do + c = i + end + cnt += 1 + end +end + +fun bench_flatstr_index(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = a + for i in [0 .. nb_cct] do x += a + var cnt = 0 + var c: Char + var pos = 0 + while cnt != loops do + pos = 0 + while pos < x.length do + c = x[pos] + pos += 1 + end + cnt += 1 + end +end + +fun bench_flatbuf_iter(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = new FlatBuffer.from(a) + for i in [0 .. nb_cct] do x.append a + var cnt = 0 + var c: Char + while cnt != loops do + for i in x do + c = i + end + cnt += 1 + end +end + +fun bench_flatbuf_index(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = new FlatBuffer.from(a) + for i in [0 .. nb_cct] do x.append a + var cnt = 0 + var c: Char + var pos = 0 + while cnt != loops do + pos = 0 + while pos < x.length do + c = x[pos] + pos += 1 + end + cnt += 1 + end +end + +var opts = new OptionContext +var mode = new OptionEnum(["rope", "flatstr", "flatbuf"], "Mode", -1, "-m") +var access_mode = new OptionEnum(["iterator", "index"], "Iteration mode", -1, "--iter-mode") +var nb_ccts = new OptionInt("Number of concatenations done to the string (in the case of the rope, this will increase its depth)", -1, "--ccts") +var loops = new OptionInt("Number of loops to be done", -1, "--loops") +var strlen = new OptionInt("Length of the base string", -1, "--strlen") +opts.add_option(mode, nb_ccts, loops, strlen, access_mode) + +opts.parse(args) + +if nb_ccts.value == -1 or loops.value == -1 or strlen.value == -1 then + opts.usage + exit(-1) +end + +var modval = mode.value +var iterval = access_mode.value + +if modval == 0 then + if iterval == 0 then + bench_rope_iter(nb_ccts.value, loops.value, strlen.value) + else if iterval == 1 then + bench_rope_index(nb_ccts.value, loops.value, strlen.value) + else + opts.usage + exit(-1) + end +else if modval == 1 then + if iterval == 0 then + bench_flatstr_iter(nb_ccts.value, loops.value, strlen.value) + else if iterval == 1 then + bench_flatstr_index(nb_ccts.value, loops.value, strlen.value) + else + opts.usage + exit(-1) + end +else if modval == 2 then + if iterval == 0 then + bench_flatbuf_iter(nb_ccts.value, loops.value, strlen.value) + else if iterval == 1 then + bench_flatbuf_index(nb_ccts.value, loops.value, strlen.value) + else + opts.usage + exit(-1) + end +else + opts.usage + exit(-1) +end diff --git a/benchmarks/strings/substr_bench.nit b/benchmarks/strings/substr_bench.nit new file mode 100644 index 0000000..5021e66 --- /dev/null +++ b/benchmarks/strings/substr_bench.nit @@ -0,0 +1,77 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# This file is free software, which comes along with NIT. This software is +# distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. You can modify it is you want, provided this header +# is kept unaltered, and a notification of the changes is added. +# You are allowed to redistribute it and sell it, alone or is a part of +# another product. + +# Benches on the substring operation on variants of Text +module substr_bench + +import opts + +fun bench_rope(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x:String = new RopeString.from(a) + for i in [0 .. nb_cct] do x += a + var cnt = 0 + while cnt != loops do + x.substring(0,5) + cnt += 1 + end +end + +fun bench_flatstr(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = a + for i in [0 .. nb_cct] do x += a + var cnt = 0 + while cnt != loops do + x.substring(0,5) + cnt += 1 + end +end + +fun bench_flatbuf(nb_cct: Int, loops: Int, strlen: Int) +do + var a = "a" * strlen + var x = new FlatBuffer.from(a) + for i in [0 .. nb_cct] do x.append a + var cnt = 0 + while cnt != loops do + x.substring(0,5) + cnt += 1 + end +end + +var opts = new OptionContext +var mode = new OptionEnum(["rope", "flatstr", "flatbuf"], "Mode", -1, "-m") +var nb_ccts = new OptionInt("Number of concatenations done to the string (in the case of the rope, this will increase its depth)", -1, "--ccts") +var loops = new OptionInt("Number of loops to be done", -1, "--loops") +var strlen = new OptionInt("Length of the base string", -1, "--strlen") +opts.add_option(mode, nb_ccts, loops, strlen) + +opts.parse(args) + +if nb_ccts.value == -1 or loops.value == -1 or strlen.value == -1 then + opts.usage + exit(-1) +end + +var modval = mode.value + +if modval == 0 then + bench_rope(nb_ccts.value, loops.value, strlen.value) +else if modval == 1 then + bench_flatstr(nb_ccts.value, loops.value, strlen.value) +else if modval == 2 then + bench_flatbuf(nb_ccts.value, loops.value, strlen.value) +else + opts.usage + exit(-1) +end diff --git a/examples/mnit_simple/org.nitlanguage.simple.txt b/examples/mnit_simple/org.nitlanguage.test_all.txt similarity index 100% rename from examples/mnit_simple/org.nitlanguage.simple.txt rename to examples/mnit_simple/org.nitlanguage.test_all.txt diff --git a/examples/mnit_simple/src/complete_simple_android.nit b/examples/mnit_simple/src/complete_simple_android.nit index 57eead6..9ee8b06 100644 --- a/examples/mnit_simple/src/complete_simple_android.nit +++ b/examples/mnit_simple/src/complete_simple_android.nit @@ -1,8 +1,21 @@ -#FIXME: Improper way of resolving unjustified metadata conflict +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Frédéric Vachon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + module complete_simple_android is - app_name("test all") java_package("org.nitlanguage.test_all") - app_version(1, 0) end import test_bundle diff --git a/examples/mnit_simple/src/simple.nit b/examples/mnit_simple/src/simple.nit index dd6f65d..6bfc9b1 100644 --- a/examples/mnit_simple/src/simple.nit +++ b/examples/mnit_simple/src/simple.nit @@ -16,7 +16,7 @@ # Very simple application module simple is - app_name("mnit Simple example") + app_name("mnit Simple example") # On Android, this name is hidden by the value in `res/values/strings.xml` app_version(0, 2, git_revision) end diff --git a/examples/mnit_simple/src/simple_android.nit b/examples/mnit_simple/src/simple_android.nit index d92ecc6..833d832 100644 --- a/examples/mnit_simple/src/simple_android.nit +++ b/examples/mnit_simple/src/simple_android.nit @@ -15,7 +15,6 @@ # limitations under the License. module simple_android is - java_package("org.nitlanguage.simple") android_manifest("""""") end diff --git a/examples/nitcorn/Makefile b/examples/nitcorn/Makefile new file mode 100644 index 0000000..96a6ee3 --- /dev/null +++ b/examples/nitcorn/Makefile @@ -0,0 +1,3 @@ +all: + mkdir -p bin/ + ../../bin/nitg --dir bin src/nitcorn_hello_world.nit src/file_server_on_port_80.nit diff --git a/examples/nitcorn/src/file_server_on_port_80.nit b/examples/nitcorn/src/file_server_on_port_80.nit new file mode 100644 index 0000000..cf91d9f --- /dev/null +++ b/examples/nitcorn/src/file_server_on_port_80.nit @@ -0,0 +1,52 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Basic file server on port 80, usually requires `root` to execute +# +# To be safe, it is recommended to run this program with its own username: +# `sudo file_server -u nitcorn:www` +module file_server_on_port_80 + +import nitcorn +import privileges + +# Prepare options +var opts = new OptionContext +var opt_drop = new OptionUserAndGroup.for_dropping_privileges +opt_drop.mandatory = true +var opt_help = new OptionBool("Print this message", "--help", "-h") +opts.add_option(opt_drop, opt_help) +opts.parse(args) + +# Check options errors and help +if not opts.errors.is_empty or opt_help.value then + print opts.errors + print "Usage: file_server [Options]" + opts.usage + exit 1 +end + +# Serve everything with a standard FilesHandler +var vh = new VirtualHost("localhost:80") +vh.routes.add new Route(null, new FileServer("www/hello_world/")) +var factory = new HttpFactory.and_libevent +factory.config.virtual_hosts.add vh + +# Drop to a low-privileged user +var user_group = opt_drop.value +if user_group != null then user_group.drop_privileges + +factory.run diff --git a/examples/nitcorn/src/nitcorn_hello_world.nit b/examples/nitcorn/src/nitcorn_hello_world.nit new file mode 100644 index 0000000..fd7fe73 --- /dev/null +++ b/examples/nitcorn/src/nitcorn_hello_world.nit @@ -0,0 +1,65 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hello World Web server example +# +# The main page, `index.html`, is served dynamicly with `MyAction`. +# The rest of the Web site fetches files from the local directory +# `www/hello_world/`. +module nitcorn_hello_world + +import nitcorn + +class MyAction + super Action + + redef fun answer(http_request, turi) + do + var response = new HttpResponse(200) + var title = "Hello World from Nitcorn!" + response.body = """ + + + + + + {{{title}}} + + +
+

{{{title}}}

+

See also a directory.

+
+ +""" + return response + end +end + +var vh = new VirtualHost("localhost:8080") + +# Serve index.html with our custom handler +vh.routes.add new Route("/index.html", new MyAction) + +# Serve everything else with a standard `FileServer` with a root at "www/hello_world/" +vh.routes.add new Route(null, new FileServer("www/hello_world/")) + +# Avoid executing when running tests +if "NIT_TESTING".environ == "true" then exit 0 + +var factory = new HttpFactory.and_libevent +factory.config.virtual_hosts.add vh +factory.run diff --git a/examples/nitcorn/www/hello_world/dir/a.txt b/examples/nitcorn/www/hello_world/dir/a.txt new file mode 100644 index 0000000..8e512cf --- /dev/null +++ b/examples/nitcorn/www/hello_world/dir/a.txt @@ -0,0 +1 @@ +aaaAAAAAaaaa diff --git a/examples/nitcorn/www/hello_world/dir/b.txt b/examples/nitcorn/www/hello_world/dir/b.txt new file mode 100644 index 0000000..55fe151 --- /dev/null +++ b/examples/nitcorn/www/hello_world/dir/b.txt @@ -0,0 +1 @@ +BBBBBBbbbbbbbbbbbbbbb diff --git a/examples/nitcorn/www/hello_world/favicon.ico b/examples/nitcorn/www/hello_world/favicon.ico new file mode 100644 index 0000000..0f66f25 Binary files /dev/null and b/examples/nitcorn/www/hello_world/favicon.ico differ diff --git a/lib/bufferized_ropes.nit b/lib/bufferized_ropes.nit index a47bc0a..18cfa4c 100644 --- a/lib/bufferized_ropes.nit +++ b/lib/bufferized_ropes.nit @@ -105,6 +105,8 @@ redef class RopeString end end + redef fun +(o) do return insert_at(o.to_s, length) + # Inserts a String `str` at position `pos` redef fun insert_at(str, pos) do diff --git a/lib/libevent.nit b/lib/libevent.nit new file mode 100644 index 0000000..525915c --- /dev/null +++ b/lib/libevent.nit @@ -0,0 +1,290 @@ +# This file is part of NIT (http://www.nitlanguage.org). +# +# Copyright 2013 Jean-Philippe Caissy +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Low-level wrapper around the libevent library to manage events on file descriptors +# +# For mor information, refer to the libevent documentation at +# http://monkey.org/~provos/libevent/doxygen-2.0.1/ +module libevent is pkgconfig("libevent") + +in "C header" `{ + #include + #include + #include + #include + #include + + #include + #include + #include +`} + +in "C" `{ + // Callback forwarded to 'Connection.write_callback' + static void c_write_cb(struct bufferevent *bev, Connection ctx) { + Connection_write_callback(ctx); + } + + // Callback forwarded to 'Connection.read_callback_native' + static void c_read_cb(struct bufferevent *bev, Connection ctx) + { + // TODO move to Nit code + struct evbuffer *input = bufferevent_get_input(bev); + size_t len = evbuffer_get_length(input); + char* cstr = malloc(len); + evbuffer_remove(input, cstr, len); + Connection_read_callback_native(ctx, cstr, len); + } + + // Callback forwarded to 'Connection.event_callback' + static void c_event_cb(struct bufferevent *bev, short events, Connection ctx) + { + Connection_event_callback(ctx, events); + + // TODO move to Nit code + if (events & BEV_EVENT_ERROR) + perror("Error from bufferevent"); + if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) { + bufferevent_free(bev); + Connection_decr_ref(ctx); + } + } + + // Callback fowarded to 'ConnectionFactory.spawn_connection' + static void accept_conn_cb(struct evconnlistener *listener, evutil_socket_t fd, + struct sockaddr *address, int socklen, ConnectionFactory ctx) + { + // TODO move to Nit code + struct event_base *base = evconnlistener_get_base(listener); + struct bufferevent *bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE); + + Connection nit_con = ConnectionFactory_spawn_connection(ctx, bev); + Connection_incr_ref(nit_con); + + bufferevent_setcb(bev, + (bufferevent_data_cb)c_read_cb, + (bufferevent_data_cb)c_write_cb, + (bufferevent_event_cb)c_event_cb, nit_con); + bufferevent_enable(bev, EV_READ|EV_WRITE); + } +`} + +# Structure to hold information and state for a Libevent dispatch loop. +# +# The event_base lies at the center of Libevent; every application will +# have one. It keeps track of all pending and active events, and +# notifies your application of the active ones. +extern class NativeEventBase `{ struct event_base * `} + + # Create a new event_base to use with the rest of Libevent + new `{ return event_base_new(); `} + fun is_valid: Bool do return not address_is_null + #fun creation_ok + + # Event dispatching loop + # + # This loop will run the event base until either there are no more added + # events, or until something calls `exit_loop`. + fun dispatch `{ event_base_dispatch(recv); `} + + # Exit the event loop + # + # TODO support timer + fun exit_loop `{ event_base_loopexit(recv, NULL); `} + + # Destroy this instance + fun destroy `{ event_base_free(recv); `} +end + +# Spawned to manage a specific connection +# +# TODO, use polls +class Connection + # Closing this connection has been requested, but may not yet be `closed` + var close_requested = false + + # This connection is closed + var closed = false + + # The native libevent linked to `self` + var native_buffer_event: NativeBufferEvent + + # Close this connection if possible, otherwise mark it to be closed later + fun close + do + var success = native_buffer_event.destroy + close_requested = true + closed = success + end + + # Callback method on a write event + fun write_callback + do + if close_requested and not closed then close + end + + private fun read_callback_native(cstr: NativeString, len: Int) + do + read_callback(cstr.to_s_with_length(len)) + end + + # Callback method when data is available to read + fun read_callback(content: String) + do + if close_requested and not closed then close + end + + # Callback method on events + fun event_callback(events: Int) do end + + # Write a string to the connection + fun write(str: String) + do + var res = native_buffer_event.write(str.to_cstring, str.length) + end + + # Write a file to the connection + # + # require: `path.file_exists` + fun write_file(path: String) + do + assert path.file_exists + + var file = new IFStream.open(path) + var output = native_buffer_event.output_buffer + var fd = file.fd + var length = file.file_stat.size + + output.add_file(fd, 0, length) + end +end + +# A buffer event structure, strongly associated to a connection, an input buffer and an output_buffer +extern class NativeBufferEvent `{ struct bufferevent * `} + fun write(line: NativeString, length: Int): Int `{ + return bufferevent_write(recv, line, length); + `} + + # Check if we have anything left in our buffers. If so, we set our connection to be closed + # on a callback. Otherwise we close it and free it right away. + fun destroy: Bool `{ + struct evbuffer* out = bufferevent_get_output(recv); + struct evbuffer* in = bufferevent_get_input(recv); + if(evbuffer_get_length(in) > 0 || evbuffer_get_length(out) > 0) { + return 0; + } else { + bufferevent_free(recv); + return 1; + } + `} + + # The output buffer associated to `self` + fun output_buffer: OutputNativeEvBuffer `{ return bufferevent_get_output(recv); `} + + # The input buffer associated to `self` + fun input_buffer: InputNativeEvBuffer `{ return bufferevent_get_input(recv); `} +end + +# A single buffer +extern class NativeEvBuffer `{ struct evbuffer * `} + # Length of data in this buffer + fun length: Int `{ return evbuffer_get_length(recv); `} +end + +extern class InputNativeEvBuffer + super NativeEvBuffer + + # Empty/clear `length` data from buffer + fun drain(length: Int) `{ evbuffer_drain(recv, length); `} +end + +extern class OutputNativeEvBuffer + super NativeEvBuffer + + # Add file to buffer + fun add_file(fd, offset, length: Int): Bool `{ + return evbuffer_add_file(recv, fd, offset, length); + `} +end + +# A listener acting on an interface and port, spawns `Connection` on new connections +extern class ConnectionListener `{ struct evconnlistener * `} + + private new bind_to(base: NativeEventBase, address: NativeString, port: Int, factory: ConnectionFactory) + import ConnectionFactory.spawn_connection, error_callback, Connection.read_callback_native, + Connection.write_callback, Connection.event_callback `{ + + struct sockaddr_in sin; + struct evconnlistener *listener; + ConnectionFactory_incr_ref(factory); + + struct hostent *hostent = gethostbyname(address); + + memset(&sin, 0, sizeof(sin)); + sin.sin_family = hostent->h_addrtype; + sin.sin_port = htons(port); + memcpy( &(sin.sin_addr.s_addr), (const void*)hostent->h_addr, hostent->h_length ); + + listener = evconnlistener_new_bind(base, + (evconnlistener_cb)accept_conn_cb, factory, + LEV_OPT_CLOSE_ON_FREE | LEV_OPT_REUSEABLE, -1, + (struct sockaddr*)&sin, sizeof(sin)); + + if (listener != NULL) { + evconnlistener_set_error_cb(listener, (evconnlistener_errorcb)ConnectionListener_error_callback); + } + + return listener; + `} + + # Get the `NativeEventBase` associated to `self` + fun base: NativeEventBase `{ return evconnlistener_get_base(recv); `} + + # Callback method on listening error + fun error_callback do + var cstr = socket_error + sys.stderr.write "libevent error: '{cstr}'" + end + + # Error with sockets + fun socket_error: NativeString `{ + // TODO move to Nit and maybe NativeEventBase + int err = EVUTIL_SOCKET_ERROR(); + return evutil_socket_error_to_string(err); + `} +end + +# Factory to listen on sockets and create new `Connection` +class ConnectionFactory + var event_base: NativeEventBase + + # On new connection, create the handler `Connection` object + fun spawn_connection(nat_buf_ev: NativeBufferEvent): Connection + do + return new Connection(nat_buf_ev) + end + + # Listen on `address`:`port` for new connection, which will callback `spawn_connection` + fun bind_to(address: String, port: Int): nullable ConnectionListener + do + var listener = new ConnectionListener.bind_to(event_base, address.to_cstring, port, self) + if listener.address_is_null then + sys.stderr.write "libevent warning: Opening {address}:{port} failed\n" + end + return listener + end +end diff --git a/lib/nitcorn/README.md b/lib/nitcorn/README.md new file mode 100644 index 0000000..f9e952d --- /dev/null +++ b/lib/nitcorn/README.md @@ -0,0 +1,30 @@ +The nitcorn Web server framework creates server-side Web apps in Nit + +# Examples + +Want to see `nitcorn` in action? Examples are available at ../../examples/nitcorn/src/. + +# Features and TODO list + + - [x] Virtual hosts and routes + - [x] Configuration change on the fly + - [x] Sessions + - [x] Reading cookies + - [ ] Full cookie support + - [ ] Close interfaces on the fly + - [ ] Better logging + - [ ] Info/status page + - [ ] `ProxyAction` to redirect a request to an external server + - [ ] `ModuleAction` which forwards the request to an independant Nit program + +## Bugs / Limitations + +* The size of requests is limited, so no big uploads + +# Credits + +This nitcorn library is a fork from an independant project originally created in 2013 by +Jean-Philippe Caissy, Guillaume Auger, Frederic Sevillano, Justin Michaud-Ouellette, +Stephan Michaud and Maxime Bélanger. + +It has been adapted to a library, and is currently maintained, by Alexis Laferrière. diff --git a/lib/nitcorn/file_server.nit b/lib/nitcorn/file_server.nit new file mode 100644 index 0000000..348383e --- /dev/null +++ b/lib/nitcorn/file_server.nit @@ -0,0 +1,128 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2013 Jean-Philippe Caissy +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Provides the `FileServer` action, which is a standard and minimal file server +module file_server + +import reactor +import sessions +import media_types + +redef class String + # Returns a `String` copy of `self` without any of the prefixed '/'s + # + # Examples: + # + # assert "/home/".strip_start_slashes == "home/" + # assert "////home/".strip_start_slashes == "home/" + # assert "../home/".strip_start_slashes == "../home/" + fun strip_start_slashes: String + do + for i in chars.length.times do if chars[i] != '/' then return substring_from(i) + return "" + end +end + +# A simple file server +class FileServer + super Action + + # Root of `self` file system + var root: String + + redef fun answer(request, turi) + do + var response + + var local_file = root.join_path(turi.strip_start_slashes) + local_file = local_file.simplify_path + + # HACK + if turi == "/" then local_file = root + + # Is it reachable? + if local_file.has_prefix(root) then + # Does it exists? + if local_file.file_exists then + response = new HttpResponse(200) + + if local_file.file_stat.is_dir then + # Show index.html instead of the directory listing + var index_file = local_file.join_path("index.html") + if index_file.file_exists then + local_file = index_file + else + index_file = local_file.join_path("index.htm") + if index_file.file_exists then local_file = index_file + end + end + + if local_file.file_stat.is_dir then + # Show the directory listing + var title = turi + var files = local_file.files + + var links = new Array[String] + if local_file.length > 1 then + # The extra / is a hack + var path = "/" + (turi + "/..").simplify_path + links.add ".." + end + for file in files do + var path = (turi + "/" + file).simplify_path + links.add "{file}" + end + + response.body = """ + + + + + + {{{title}}} + + +
+

{{{title}}}

+
    +
  • {{{links.join("
    • \n\t\t\t
  • ")}}}
    • +
+
+ +""" + + response.header["Content-Type"] = media_types["html"].as(not null) + else + # It's a single file + var file = new IFStream.open(local_file) + response.body = file.read_all + + var ext = local_file.file_extension + if ext != null then + var media_type = media_types[ext] + if media_type != null then response.header["Content-Type"] = media_type + end + + file.close + end + + else response = new HttpResponse(404) + else response = new HttpResponse(403) + + return response + end +end diff --git a/lib/nitcorn/http_request.nit b/lib/nitcorn/http_request.nit new file mode 100644 index 0000000..6431ace --- /dev/null +++ b/lib/nitcorn/http_request.nit @@ -0,0 +1,204 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2013 Frederic Sevillano +# Copyright 2013 Jean-Philippe Caissy +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Provides the `HttpRequest` class and services to create it +module http_request + +import standard + +# A request received over HTTP, is build by `HttpRequestParser` +class HttpRequest + private init do end + + # HTTP protocol version + var http_version: String + + # Method of this request (GET or POST) + var method: String + + # The host targetter by this request (usually the server) + var host: String + + # The full URL requested by the client (including the `query_string`) + var url: String + + # The resource requested by the client (only the page, not the `query_string`) + var uri: String + + # The string following `?` in the requested URL + var query_string = "" + + # The header of this request + var header = new HashMap[String, String] + + # The content of the cookie of this request + var cookie = new HashMap[String, String] + + # The arguments passed with the GET method, + var get_args = new HashMap[String, String] + + # The arguments passed with the POST method + var post_args = new HashMap[String, String] +end + +# Utility class to parse a request string and build a `HttpRequest` +# +# The main method is `parse_http_request`. +class HttpRequestParser + # The current `HttpRequest` under construction + private var http_request: HttpRequest + + # Untreated body + private var body = "" + + # Lines of the header + private var header_fields = new Array[String] + + # Words of the first line + private var first_line = new Array[String] + + init do end + + fun parse_http_request(full_request: String): nullable HttpRequest + do + clear_data + + var http_request = new HttpRequest + self.http_request = http_request + + segment_http_request(full_request) + + # Parse first line, looks like "GET dir/index.html?user=xymus HTTP/1.0" + http_request.method = first_line[0] + http_request.url = first_line[1] + http_request.http_version = first_line[2] + + # GET args + if http_request.url.has('?') then + http_request.uri = first_line[1].substring(0, first_line[1].index_of('?')) + http_request.query_string = first_line[1].substring_from(first_line[1].index_of('?')+1) + http_request.get_args = parse_url + else + http_request.uri = first_line[1] + end + + # POST args + if http_request.method == "POST" then + var lines = body.split_with('&') + for line in lines do + var parts = line.split_once_on('=') + if parts.length > 1 then + var decoded = parts[1].replace('+', " ").from_percent_encoding + if decoded == null then + print "decode error" + continue + end + http_request.post_args[parts[0]] = decoded + else + print "POST Error: {line} format error on {line}" + end + end + end + + # Headers + for i in header_fields do + var temp_field = i.split_with(": ") + + if temp_field.length == 2 then + http_request.header[temp_field[0]] = temp_field[1] + end + end + + # Cookies + if http_request.header.keys.has("Cookie") then + var cookie = http_request.header["Cookie"] + for couple in cookie.split_with(';') do + var words = couple.trim.split_with('=') + if words.length != 2 then continue + http_request.cookie[words[0]] = words[1] + end + end + + return http_request + end + + private fun clear_data + do + first_line.clear + header_fields.clear + end + + private fun segment_http_request(http_request: String): Bool + do + var header_end = http_request.search("\r\n\r\n") + + if header_end == null then + header_fields = http_request.split_with("\r\n") + else + header_fields = http_request.substring(0, header_end.from).split_with("\r\n") + body = http_request.substring(header_end.after, http_request.length-1) + end + + # If a line of the http_request is long it may change line, it has " " at the + # end to indicate this. This section turns them into 1 line. + if header_fields.length > 1 and header_fields[0].has_suffix(" ") then + var temp_req = header_fields[0].substring(0, header_fields[0].length-1) + header_fields[1] + + first_line = temp_req.split_with(' ') + header_fields.shift + header_fields.shift + + if first_line.length != 3 then return false + else + first_line = header_fields[0].split_with(' ') + header_fields.shift + + if first_line.length != 3 then return false + end + + # Cut off the header in lines + var pos = 0 + while pos < header_fields.length do + if pos < header_fields.length-1 and header_fields[pos].has_suffix(" ") then + header_fields[pos] = header_fields[pos].substring(0, header_fields[pos].length-1) + header_fields[pos+1] + header_fields.remove_at(pos+1) + pos = pos-1 + end + pos = pos+1 + end + + return true + end + + # Extract args from the URL + private fun parse_url: HashMap[String, String] + do + var query_strings = new HashMap[String, String] + + if http_request.url.has('?') then + var get_args = http_request.query_string.split_with("&") + for param in get_args do + var key_value = param.split_with("=") + if key_value.length < 2 then continue + query_strings[key_value[0]] = key_value[1] + end + end + + return query_strings + end +end diff --git a/lib/nitcorn/http_response.nit b/lib/nitcorn/http_response.nit new file mode 100644 index 0000000..666974b --- /dev/null +++ b/lib/nitcorn/http_response.nit @@ -0,0 +1,132 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2013 Frederic Sevillano +# Copyright 2013 Jean-Philippe Caissy +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Provides the `HttpResponse` class and `http_status_codes` +module http_response + +# A response to send over HTTP +class HttpResponse + + # HTTP protocol version + var http_version = "HTTP/1.0" is writable + + # Status code of this response (200, 404, etc.) + var status_code: Int is writable + + # Return the message associated to `status_code` + fun status_message: nullable String do return http_status_codes[status_code] + + # Headers of this response as a `Map` + var header = new HashMap[String, String] + + # Body of this response + var body = "" is writable + + # Finalize this response before sending it over HTTP + fun finalize + do + # Set the content length if not already set + if not header.keys.has("Content-Length") then + header["Content-Length"] = body.length.to_s + end + + # Set server ID + if not header.keys.has("Server") then header["Server"] = "unitcorn" + end + + # Get this reponse as a string according to HTTP protocol + redef fun to_s: String + do + finalize + + var buf = new FlatBuffer + buf.append("{http_version} {status_code} {status_message or else ""}\r\n") + for key, value in header do + buf.append("{key}: {value}\r\n") + end + buf.append("\r\n{body}") + return buf.to_s + end +end + +# Helper class to associate HTTP status code to their message +# +# You probably want the default instance available as the top-level method +# `http_status_codes`. +class HttpStatusCodes + + # All know code and their message + var codes = new HashMap[Int, String] + + protected init do insert_status_codes + + # Get the message associated to the status `code`, return `null` in unknown + fun [](code: Int): nullable String + do + if codes.keys.has(code) then + return codes[code] + else return null + end + + private fun insert_status_codes + do + codes[100] = "Continue" + codes[101] = "Switching Protocols" + codes[200] = "OK" + codes[201] = "Created" + codes[202] = "Accepted" + codes[203] = "Non-Authoritative Information" + codes[204] = "No Content" + codes[205] = "Reset Content" + codes[206] = "Partial Content" + codes[300] = "Multiple Choices" + codes[301] = "Moved Permanently" + codes[302] = "Found" + codes[303] = "See Other" + codes[304] = "Not Modified" + codes[305] = "Use Proxy" + codes[307] = "Temporary Redirect" + codes[400] = "Bad Request" + codes[401] = "Unauthorized" + codes[402] = "Payment Requred" + codes[403] = "Forbidden" + codes[404] = "Not Found" + codes[405] = "Method Not Allowed" + codes[406] = "Not Acceptable" + codes[407] = "Proxy Authentication Required" + codes[408] = "Request Timeout" + codes[409] = "Conflict" + codes[410] = "Gone" + codes[411] = "Length Required" + codes[412] = "Precondition Failed" + codes[413] = "Request Entity Too Large" + codes[414] = "Request-URI Too Long" + codes[415] = "Unsupported Media Type" + codes[416] = "Requested Range Not Satisfiable" + codes[417] = "Expectation Failed" + codes[500] = "Internal Server Error" + codes[501] = "Not Implemented" + codes[502] = "Bad Gateway" + codes[503] = "Service Unavailable" + codes[504] = "Gateway Timeout" + codes[505] = "HTTP Version Not Supported" + end +end + +# Get the default instance of `HttpStatusCodes` +fun http_status_codes: HttpStatusCodes do return once new HttpStatusCodes diff --git a/lib/nitcorn/media_types.nit b/lib/nitcorn/media_types.nit new file mode 100644 index 0000000..bf5d74a --- /dev/null +++ b/lib/nitcorn/media_types.nit @@ -0,0 +1,101 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2013 Justin Michaud-Ouellette +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Services to identify Internet media types (or MIME types, Content-types) +module media_types + +# Map of known MIME types +class MediaTypes + protected var types = new HashMap[String, String] + + # Get the type/subtype associated to a file extension `ext` + fun [](ext: String): nullable String + do + if types.keys.has(ext) then return types[ext] + return null + end + + init + do + types["html"] = "text/html" + types["htm"] = "text/html" + types["shtml"] = "text/html" + types["css"] = "text/css" + types["xml"] = "text/xml" + types["rss"] = "text/xml" + types["gif"] = "image/gif" + types["jpg"] = "image/jpeg" + types["jpeg"] = "image/jpeg" + types["js"] = "application/x-javascript" + types["txt"] = "text/plain" + types["htc"] = "text/x-component" + types["mml"] = "text/mathml" + types["png"] = "image/png" + types["ico"] = "image/x-icon" + types["jng"] = "image/x-jng" + types["wbmp"] = "image/vnd.wap.wbmp" + types["jar"] = "application/java-archive" + types["war"] = "application/java-archive" + types["ear"] = "application/java-archive" + types["hqx"] = "application/mac-binhex40" + types["pdf"] = "application/pdf" + types["cco"] = "application/x-cocoa" + types["jardiff"] = "application/x-java-archive-diff" + types["jnlp"] = "application/x-java-jnlp-file" + types["run"] = "application/x-makeself" + types["pl"] = "application/x-perl" + types["pm"] = "application/x-perl" + types["pdb"] = "application/x-pilot" + types["prc"] = "application/x-pilot" + types["rar"] = "application/x-rar-compressed" + types["rpm"] = "application/x-redhat-package-manager" + types["sea"] = "application/x-sea" + types["swf"] = "application/x-shockwave-flash" + types["sit"] = "application/x-stuffit" + types["tcl"] = "application/x-tcl" + types["tk"] = "application/x-tcl" + types["der"] = "application/x-x509-ca-cert" + types["pem"] = "application/x-x509-ca-cert" + types["crt"] = "application/x-x509-ca-cert" + types["xpi"] = "application/x-xpinstall" + types["zip"] = "application/zip" + types["deb"] = "application/octet-stream" + types["bin"] = "application/octet-stream" + types["exe"] = "application/octet-stream" + types["dll"] = "application/octet-stream" + types["dmg"] = "application/octet-stream" + types["eot"] = "application/octet-stream" + types["iso"] = "application/octet-stream" + types["img"] = "application/octet-stream" + types["msi"] = "application/octet-stream" + types["msp"] = "application/octet-stream" + types["msm"] = "application/octet-stream" + types["mp3"] = "audio/mpeg" + types["ra"] = "audio/x-realaudio" + types["mpeg"] = "video/mpeg" + types["mpg"] = "video/mpeg" + types["mov"] = "video/quicktime" + types["flv"] = "video/x-flv" + types["avi"] = "video/x-msvideo" + types["wmv"] = "video/x-ms-wmv" + types["asx"] = "video/x-ms-asf" + types["asf"] = "video/x-ms-asf" + types["mng"] = "video/x-mng" + end +end + +fun media_types: MediaTypes do return once new MediaTypes diff --git a/lib/nitcorn/nitcorn.nit b/lib/nitcorn/nitcorn.nit new file mode 100644 index 0000000..2a5b43a --- /dev/null +++ b/lib/nitcorn/nitcorn.nit @@ -0,0 +1,63 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The nitcorn Web server framework creates server-side Web apps in Nit +# +# The main classes are: +# * `Action` to answer to requests. +# * `Route` to represent a path to an action. +# * `VirtualHost` to listen on a specific interface and behave accordingly +# * `HttpFactory` which is the base dispatcher class. +# +# Basic usage example: +# ~~~~ +# class MyAction +# super Action +# +# redef fun answer(http_request, turi) +# do +# var response = new HttpResponse(200) +# response.body = """ +# +# +# +# Hello World +# +# +#

Hello World

+# +# """ +# return response +# end +# end +# +# var vh = new VirtualHost("localhost:80") +# +# # Serve index.html with our custom handler +# vh.routes.add new Route("/index.html", new MyAction) +# +# # Serve everything else with a standard `FileServer` +# vh.routes.add new Route(null, new FileServer("/var/www/")) +# +# var factory = new HttpFactory.and_libevent +# factory.config.virtual_hosts.add vh +# factory.run +# ~~~~ +module nitcorn + +import reactor +import file_server +import sessions diff --git a/lib/nitcorn/reactor.nit b/lib/nitcorn/reactor.nit new file mode 100644 index 0000000..a8f9236 --- /dev/null +++ b/lib/nitcorn/reactor.nit @@ -0,0 +1,182 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2013 Jean-Philippe Caissy +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core of the `nitcorn` project, provides `HttpFactory` and `Action` +module reactor + +import more_collections +import libevent + +import server_config +import http_request +import http_response + +# A server handling a single connection +class HttpServer + super Connection + + # The associated `HttpFactory` + var factory: HttpFactory + + init(buf_ev: NativeBufferEvent, factory: HttpFactory) do self.factory = factory + + private var parser = new HttpRequestParser is lazy + + redef fun read_callback(str) + do + # TODO support bigger inputs (such as big forms and file upload) + + var request_object = parser.parse_http_request(str.to_s) + + if request_object != null then delegate_answer request_object + end + + # Answer to a request + fun delegate_answer(request: HttpRequest) + do + # Find target virtual host + var virtual_host = null + if request.header.keys.has("Host") then + var host = request.header["Host"] + if host.index_of(':') == -1 then host += ":80" + for vh in factory.config.virtual_hosts do + for i in vh.interfaces do if i.to_s == host then + virtual_host = vh + break label + end + end label + end + + # Get a response from the virtual host + var response + if virtual_host != null then + var route = virtual_host.routes[request.uri] + if route != null then + var handler = route.handler + var root = route.path + var turi + if root != null then + turi = ("/" + request.uri.substring_from(root.length)).simplify_path + else turi = request.uri + response = handler.answer(request, turi) + else response = new HttpResponse(405) + else response = new HttpResponse(405) + + # Send back a response + write response.to_s + close + end +end + +redef abstract class Action + # Handle a request with the relative URI `truncated_uri` + # + # `request` is fully formed request object and has a reference to the session + # if one preexists. + # + # `truncated_uri` is the ending of the fulle request URI, truncated from the route + # leading to this `Action`. + fun answer(request: HttpRequest, truncated_uri: String): HttpResponse is abstract +end + +# Factory to create `HttpServer` instances, and hold the libevent base handler +class HttpFactory + super ConnectionFactory + + # Configuration of this server + # + # It should be populated after this object has instanciated + var config = new ServerConfig.with_factory(self) + + # Instanciate a server and libvent + # + # You can use this to create the first `HttpFactory`, which is the most common. + init and_libevent do init(new NativeEventBase) + + redef fun spawn_connection(buf_ev) do return new HttpServer(buf_ev, self) + + # Launch the main loop of this server + fun run + do + event_base.dispatch + event_base.destroy + end +end + +redef class ServerConfig + # Handle to retreive the `HttpFactory` on config change + private var factory: HttpFactory + + private init with_factory(factory: HttpFactory) do self.factory = factory +end + +redef class Sys + # Active listeners + private var listeners = new HashMap2[String, Int, ConnectionListener] + + # Hosts needong each listener + private var listeners_count = new HashMap2[String, Int, Int] + + # Activate a listener on `interfac` if there's not already one + private fun listen_on(interfac: Interface, factory: HttpFactory) + do + if interfac.registered then return + + var name = interfac.name + var port = interfac.port + + var listener = listeners[name, port] + if listener == null then + listener = factory.bind_to(name, port) + if listener != null then + sys.listeners[name, port] = listener + listeners_count[name, port] = 1 + end + else + listeners_count[name, port] += 1 + end + + interfac.registered = true + end + + # TODO close listener +end + +redef class Interface + # Has `self` been registered by `listen_on`? + private var registered = false +end + +redef class Interfaces + redef fun add(e) + do + super + if vh.server_config != null then sys.listen_on(e, vh.server_config.factory) + end + + # TODO remove +end + +redef class VirtualHosts + redef fun add(e) + do + super + for i in e.interfaces do sys.listen_on(i, config.factory) + end + + # TODO remove +end diff --git a/lib/nitcorn/server_config.nit b/lib/nitcorn/server_config.nit new file mode 100644 index 0000000..2decb16 --- /dev/null +++ b/lib/nitcorn/server_config.nit @@ -0,0 +1,134 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Classes and services to configure the server +# +# The classes of interest are `VirtualHost`, `Interface`, `Route` and `Action` +module server_config + +# Server instance configuration +class ServerConfig + # Virtual hosts list + var virtual_hosts = new VirtualHosts(self) + + # TODO implement serialization or something like that +end + +# A `VirtualHost` configuration +class VirtualHost + # Back reference to the associated server configuration + var server_config: nullable ServerConfig = null + + # Interfaces on which `self` is active + var interfaces = new Interfaces(self) + + # Routes and thus `Action`s active on `self` + var routes = new Routes(self) + + # Create a virtual host from interfaces as strings + init(interfaces: String ...) + do + for i in interfaces do self.interfaces.add_from_string i + end +end + +# An interface composed of a `name`:`port` +class Interface + # Name of this interface (such as "localhost", "example.org", etc.) + var name: String + + # The port to open + var port: Int + + redef fun to_s do return "{name}:{port}" +end + +# A route to an `Action` according to a `path` +class Route + # Path to this action present in the URI + var path: nullable String + + # `Action` to activate when this route is traveled + var handler: Action +end + +# Action executed to answer a request +abstract class Action +end + +### Intelligent lists ### + +# A list of interfaces with dynamic port listeners +class Interfaces + super Array[Interface] + + # Back reference to the associtated `VirtualHost` + var vh: VirtualHost + + # Add an `Interface` described by `text` formatted as `interface.name.com:port` + fun add_from_string(text: String) + do + assert text.chars.count(':') <= 1 + + var words = text.split(':') + var name = words[0] + var port + if words.length > 1 then + port = words[1].to_i + else port = 80 + + add new Interface(name, port) + end +end + +# A list of virtual hosts with dynamic port listeners +class VirtualHosts + super Array[VirtualHost] + + # Back reference to the server config + var config: ServerConfig + + redef fun add(e) + do + super + + e.server_config = config + end +end + +# A list of routes with the search method `[]` +class Routes + # Back reference to the config of the virtual host + var config: VirtualHost + + private var array = new Array[Route] + + # Add `e` to `self` + fun add(e: Route) do array.add e + + # Remove `e` from `self` + fun remove(e: Route) do array.remove e + + # Get the first `Route` than has `key` as prefix to its path + fun [](key: String): nullable Route + do + for route in array do + var path = route.path + if path == null or key.has_prefix(path) then return route + end + return null + end +end diff --git a/lib/nitcorn/sessions.nit b/lib/nitcorn/sessions.nit new file mode 100644 index 0000000..31bae5d --- /dev/null +++ b/lib/nitcorn/sessions.nit @@ -0,0 +1,109 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Copyright 2014 Alexis Laferrière +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Automated session management +# +# When parsing a request, this module associate a pre-existing session +# to the request if there is one. It will also send the required cookie +# with the response if a session has been associated to the response object. +module sessions + +import md5 + +import server_config +import http_request +import http_response + +# A server side session +class Session + + # Hashed id used both client and server side to identify this `Session` + var id_hash: String + + init + do + self.id_hash = sys.next_session_hash + sys.sessions[self.id_hash] = self + end +end + +redef class Sys + # Active sessions + var sessions = new HashMap[String, Session] + + # Get the next session hash available, and increment the session id cache + fun next_session_hash: String + do + var id = next_session_id_cache + # On firt evocation, seed the pseudo random number generator + if id == null then + srand + id = 1000000.rand + end + + next_session_id_cache = id + 1 + + return id.to_id_hash + end + + private var next_session_id_cache: nullable Int = null + + # Salt used to hash the session id + protected var session_salt = "Default unitcorn session salt" +end + +redef class Int + # Salt and hash and id to use as `Session.id_hash` + private fun to_id_hash: String do return (self.to_s+sys.session_salt).md5 +end + +redef class HttpResponse + # A `Session` to associate with a response + var session: nullable Session = null is writable + + redef fun finalize + do + super + + var session = self.session + if session != null then + header["Set-Cookie"] = "session={session.id_hash}; HttpOnly" + end + end +end + +redef class HttpRequest + # The `Session` associated to this request + var session: nullable Session = null +end + +redef class HttpRequestParser + redef fun parse_http_request(text) + do + var request = super + if request != null then + if request.cookie.keys.has("session") then + var id_hash = request.cookie["session"] + + if sys.sessions.keys.has(id_hash) then + # Restore the session + request.session = sys.sessions[id_hash] + end + end + end + return request + end +end diff --git a/lib/opts.nit b/lib/opts.nit index d6efd3e..19b8ff3 100644 --- a/lib/opts.nit +++ b/lib/opts.nit @@ -245,23 +245,15 @@ end # Context where the options process class OptionContext # Options present in the context - var options: Array[Option] + var options = new Array[Option] # Rest of the options after `parse` is called - var rest: Array[String] + var rest = new Array[String] # Errors found in the context after parsing - var errors: Array[String] + var errors = new Array[String] - private var optmap: Map[String, Option] - - init - do - options = new Array[Option] - optmap = new HashMap[String, Option] - rest = new Array[String] - errors = new Array[String] - end + private var optmap = new HashMap[String, Option] # Add one or more options to the context fun add_option(opts: Option...) do diff --git a/lib/sqlite3/sqlite3.nit b/lib/sqlite3/sqlite3.nit index d194362..d016a4c 100644 --- a/lib/sqlite3/sqlite3.nit +++ b/lib/sqlite3/sqlite3.nit @@ -127,7 +127,6 @@ class Statement fun iterator: StatementIterator do native_statement.reset - native_statement.step return new StatementIterator(self) end end @@ -228,7 +227,7 @@ class StatementEntry var native_string = statement.native_statement.column_text(index) if native_string.address_is_null then return "" - return native_string.to_s + return native_string.to_s_with_copy end # Get this entry as `Blob` @@ -260,11 +259,13 @@ class StatementIterator do self.statement = s self.item = new StatementRow(s) + + self.is_ok = statement.native_statement.step.is_row end redef var item: StatementRow - redef var is_ok = true + redef var is_ok: Bool # require: `self.statement.is_open` redef fun next @@ -290,7 +291,17 @@ interface Sqlite3Data end redef universal Int super Sqlite3Data end redef universal Float super Sqlite3Data end -redef class String super Sqlite3Data end +redef class String + super Sqlite3Data + + # Return `self` between `'`s and escaping any extra `'` + # + # assert "'; DROP TABLE students".to_sql_string == "'''; DROP TABLE students'" + fun to_sql_string: String + do + return "'{self.replace('\'', "''")}'" + end +end # A Sqlite3 blob class Blob diff --git a/lib/standard/file.nit b/lib/standard/file.nit index 862f145..f7883f6 100644 --- a/lib/standard/file.nit +++ b/lib/standard/file.nit @@ -26,6 +26,7 @@ in "C Header" `{ #include #include #include + #include `} # File Abstract Stream @@ -37,8 +38,10 @@ abstract class FStream # The FILE *. var _file: nullable NativeFile = null - fun file_stat: FileStat - do return _file.file_stat end + fun file_stat: FileStat do return _file.file_stat + + # File descriptor of this file + fun fd: Int do return _file.fileno end # File input stream @@ -514,6 +517,7 @@ private extern class NativeFile `{ FILE* `} fun io_write(buf: NativeString, len: Int): Int is extern "file_NativeFile_NativeFile_io_write_2" fun io_close: Int is extern "file_NativeFile_NativeFile_io_close_0" fun file_stat: FileStat is extern "file_NativeFile_NativeFile_file_stat_0" + fun fileno: Int `{ return fileno(recv); `} new io_open_read(path: NativeString) is extern "file_NativeFileCapable_NativeFileCapable_io_open_read_1" new io_open_write(path: NativeString) is extern "file_NativeFileCapable_NativeFileCapable_io_open_write_1" diff --git a/lib/standard/ropes.nit b/lib/standard/ropes.nit index c17e9bd..ebc3377 100644 --- a/lib/standard/ropes.nit +++ b/lib/standard/ropes.nit @@ -99,6 +99,14 @@ private class StringLeaf redef fun to_leaf do return self end +# Used as a cache when using indexed access to a substring in the Rope +private class LeafCache + # Cached leaf + var leaf: Leaf + # Position in Rope + var pos: Int +end + # Basic structure, binary tree with a root node. # # Also shared services by subsequent implementations. @@ -111,6 +119,8 @@ abstract class Rope # Cached version of self as a flat String private var str_representation: nullable NativeString = null + private var leaf_cache: nullable LeafCache = null + # Empty Rope init do from("") @@ -225,7 +235,10 @@ abstract class Rope private fun get_node_from(node: RopeNode, curr_pos: Int, seek_pos: Int, stack: List[PathElement]): Path do assert curr_pos >= 0 - if node isa Leaf then return new Path(node, seek_pos - curr_pos, stack) + if node isa Leaf then + self.leaf_cache = new LeafCache(node, curr_pos) + return new Path(node, seek_pos - curr_pos, stack) + end node = node.as(Concat) if node.left != null then @@ -296,7 +309,18 @@ class RopeString return ret end - redef fun +(o) do return insert_at(o.to_s, length) + redef fun +(o) do + if self.length == 0 then return o.to_s + if o.length == 0 then return self + var str = o.to_s + if str isa FlatString then + return new RopeString.from_root(new Concat(root, new StringLeaf(str))) + else if str isa RopeString then + return new RopeString.from_root(new Concat(root, str.root)) + else + abort + end + end redef fun *(n) do @@ -439,6 +463,7 @@ private class RopeStringChars redef fun [](pos) do assert pos < tgt.length + if tgt.leaf_cache != null and pos >= tgt.leaf_cache.pos and (tgt.leaf_cache.pos + tgt.leaf_cache.leaf.length) > pos then return tgt.leaf_cache.leaf.str.chars[pos - tgt.leaf_cache.pos] var path = tgt.node_at(pos) return path.leaf.str.chars[path.offset] end diff --git a/lib/standard/string.nit b/lib/standard/string.nit index 5d158b5..45e5712 100644 --- a/lib/standard/string.nit +++ b/lib/standard/string.nit @@ -293,6 +293,21 @@ abstract class Text return true end + # Returns `true` if the string contains only Hex chars + # + # assert "048bf".is_hex == true + # assert "ABCDEF".is_hex == true + # assert "0G".is_hex == false + fun is_hex: Bool + do + for c in self.chars do + if not (c >= 'a' and c <= 'f') and + not (c >= 'A' and c <= 'F') and + not (c >= '0' and c <= '9') then return false + end + return true + end + # Are all letters in `self` upper-case ? # # assert "HELLO WORLD".is_upper == true @@ -474,6 +489,71 @@ abstract class Text return res.to_s end + # Encode `self` to percent (or URL) encoding + # + # assert "aBc09-._~".to_percent_encoding == "aBc09-._~" + # assert "%()< >".to_percent_encoding == "%25%28%29%3c%20%3e" + # assert ".com/post?e=asdf&f=123".to_percent_encoding == ".com%2fpost%3fe%3dasdf%26f%3d123" + fun to_percent_encoding: String + do + var buf = new FlatBuffer + + for c in self.chars do + if (c >= '0' and c <= '9') or + (c >= 'a' and c <= 'z') or + (c >= 'A' and c <= 'Z') or + c == '-' or c == '.' or + c == '_' or c == '~' + then + buf.add c + else buf.append "%{c.ascii.to_hex}" + end + + return buf.to_s + end + + # Decode `self` from percent (or URL) encoding to a clear string + # + # Replace invalid use of '%' with '?'. + # + # assert "aBc09-._~".from_percent_encoding == "aBc09-._~" + # assert "%25%28%29%3c%20%3e".from_percent_encoding == "%()< >" + # assert ".com%2fpost%3fe%3dasdf%26f%3d123".from_percent_encoding == ".com/post?e=asdf&f=123" + # assert "%25%28%29%3C%20%3E".from_percent_encoding == "%()< >" + # assert "incomplete %".from_percent_encoding == "incomplete ?" + # assert "invalid % usage".from_percent_encoding == "invalid ? usage" + fun from_percent_encoding: String + do + var buf = new FlatBuffer + + var i = 0 + while i < length do + var c = chars[i] + if c == '%' then + if i + 2 >= length then + # What follows % has been cut off + buf.add '?' + else + i += 1 + var hex_s = substring(i, 2) + if hex_s.is_hex then + var hex_i = hex_s.to_hex + buf.add hex_i.ascii + i += 1 + else + # What follows a % is not Hex + buf.add '?' + i -= 1 + end + end + else buf.add c + + i += 1 + end + + return buf.to_s + end + # Equality of text # Two pieces of text are equals if thez have the same characters in the same order. # diff --git a/lib/standard/string_search.nit b/lib/standard/string_search.nit index 8dcf90a..0786ec9 100644 --- a/lib/standard/string_search.nit +++ b/lib/standard/string_search.nit @@ -355,6 +355,17 @@ redef class Text # @deprecated alias for `split` fun split_with(p: Pattern): Array[SELFTYPE] do return self.split(p) + # Split `self` on the first `=` + # + # assert "hello".split_once_on('l') == ["he", "lo"] + # assert "a, b, c, d, e".split_once_on(", ") == ["a", "b, c, d, e"] + fun split_once_on(p: Pattern): Array[SELFTYPE] + do + var m = p.search_in(self, 0) + if m == null then return [self] + return new Array[SELFTYPE].with_items(substring(0, m.from), substring_from(m.after)) + end + # Replace all occurences of a pattern with a string # # assert "hlelo".replace("le", "el") == "hello" diff --git a/lib/string_experimentations/README b/lib/string_experimentations/README new file mode 100644 index 0000000..fa09366 --- /dev/null +++ b/lib/string_experimentations/README @@ -0,0 +1,17 @@ +This project is a collection of modules used to experiment on different variations of Text and its subclasses. +This is only temporary as these modules will eventually be merged into standard library or discarded for those bringing no real improvements to the language. + +The modules contained here are : + + * utf8: A draft of implementation of UTF-8 as internal encoding for Strings with automatic indexing. + +TODO : + + * utf8: + * Support for the whole API of Text + * Any kind of normalization form for equality (NFC probably) + * Compatibility versions of equality test + * Locale support + * Comparisons + * to_upper/lower fully-compatible with Unicode + * Lazy indexing version diff --git a/lib/string_experimentations/string_experimentations.nit b/lib/string_experimentations/string_experimentations.nit new file mode 100644 index 0000000..1eeb10d --- /dev/null +++ b/lib/string_experimentations/string_experimentations.nit @@ -0,0 +1,18 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# General module for all kinds of string experimentations +module string_experimentations + +import utf8 diff --git a/lib/string_experimentations/utf8.nit b/lib/string_experimentations/utf8.nit new file mode 100644 index 0000000..7261820 --- /dev/null +++ b/lib/string_experimentations/utf8.nit @@ -0,0 +1,413 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Introduces UTF-8 as internal encoding for Strings in Nit. +module utf8 + +intrude import standard::string +intrude import standard::file + +in "C Header" `{ + +#include +#include +#include + +typedef struct { + long pos; + char* ns; +} UTF8Char; + +`} + +# UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes +# +# A UTF-8 char has its bytes stored in a NativeString (char*) +extern class UnicodeChar `{ UTF8Char* `} + + new(pos: Int, ns: NativeString) `{ + UTF8Char* u = malloc(sizeof(UTF8Char)); + u->pos = pos; + u->ns = ns; + return u; + `} + + # Real length of the char in UTF8 + # + # As per the specification : + # + # Length | UTF-8 octet sequence + # | (binary) + # ---------+------------------------------------------------- + # 1 | 0xxxxxxx + # 2 | 110xxxxx 10xxxxxx + # 3 | 1110xxxx 10xxxxxx 10xxxxxx + # 4 | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + private fun len: Int `{ + char* ns = recv->ns; + int pos = recv->pos; + char nspos = ns[pos]; + if((nspos & 0x80) == 0x00){ return 1;} + if((nspos & 0xE0) == 0xC0){ return 2;} + if((nspos & 0xF0) == 0xE0){ return 3;} + if((nspos & 0xF7) == 0xF0){ return 4;} + // Invalid character + return 1; + `} + + # Position in containing NativeString + private fun pos: Int `{ + return recv->pos; + `} + + private fun pos=(p: Int) `{recv->pos = p;`} + + # C char* wrapping the char + fun ns: NativeString `{ + return recv->ns; + `} + + # Returns the Unicode code point representing the character + # + # Note : A unicode character might not be a visible glyph, but it will be used to determine canonical equivalence + fun code_point: Int import UnicodeChar.len `{ + switch(UnicodeChar_len(recv)){ + case 1: + return (long)(0x7F & (unsigned char)recv->ns[recv->pos]); + case 2: + return 0 | ((0x1F & (unsigned char)recv->ns[recv->pos]) << 6) | (0x3F & (unsigned char)recv->ns[recv->pos+1]); + case 3: + return 0 | ((0x0F & (unsigned char)recv->ns[recv->pos]) << 12) | + ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 6) | + (0x3F & (unsigned char)recv->ns[recv->pos+2]); + case 4: + return 0 | ((0x07 & (unsigned char)recv->ns[recv->pos]) << 18) | + ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 12) | + ((0x3F & (unsigned char)recv->ns[recv->pos+2]) << 6) | + (0x3F & (unsigned char)recv->ns[recv->pos+3]); + } + `} + + # Returns an upper-case version of self + # + # NOTE : Works only on ASCII chars + # TODO : Support unicode for to_upper + fun to_upper: UnicodeChar import UnicodeChar.code_point `{ + int cp = UnicodeChar_code_point(recv); + if(cp < 97 || cp > 122){ return recv; } + char* ns = malloc(2); + ns[1] = '\0'; + char c = recv->ns[recv->pos]; + ns[0] = c - 32; + UTF8Char* ret = malloc(sizeof(UTF8Char)); + ret->ns = ns; + ret->pos = 0; + return ret; + `} + + # Returns an lower-case version of self + # + # NOTE : Works only on ASCII chars + # TODO : Support unicode for to_upper + fun to_lower: UnicodeChar import UnicodeChar.code_point `{ + int cp = UnicodeChar_code_point(recv); + if(cp < 65 || cp > 90){ return recv; } + char* ns = malloc(2); + ns[1] = '\0'; + char c = recv->ns[recv->pos]; + ns[0] = c + 32; + UTF8Char* ret = malloc(sizeof(UTF8Char)); + ret->ns = ns; + ret->pos = 0; + return ret; + `} + + redef fun ==(o) + do + if o isa Char then + if len != 1 then return false + if code_point == o.ascii then return true + else if o isa UnicodeChar then + if len != o.len then return false + if code_point == o.code_point then return true + end + return false + end + + redef fun output import UnicodeChar.code_point `{ + switch(UnicodeChar_len(recv)){ + case 1: + printf("%c", recv->ns[recv->pos]); + break; + case 2: + printf("%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1]); + break; + case 3: + printf("%c%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1], recv->ns[recv->pos + 2]); + break; + case 4: + printf("%c%c%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1], recv->ns[recv->pos + 2], recv->ns[recv->pos + 3]); + break; + } + `} + + redef fun to_s import NativeString.to_s_with_length `{ + int len = utf8___UnicodeChar_len___impl(recv); + char* r = malloc(len + 1); + r[len] = '\0'; + char* src = (recv->ns + recv->pos); + memcpy(r, src, len); + return NativeString_to_s_with_length(r, len); + `} +end + +# A `StringIndex` is used to keep track of the position of characters in a `FlatString` object +# +# It becomes mandatory for UTF-8 strings since characters do not have a fixed size. +private extern class StringIndex `{ UTF8Char* `} + + new(size: Int) `{ return malloc(size*sizeof(UTF8Char)); `} + + # Sets the character at `index` as `item` + fun []=(index: Int, item: UnicodeChar) `{ recv[index] = *item; `} + + # Gets the character at position `id` + fun [](id: Int): UnicodeChar `{ return &recv[id]; `} + + # Copies a part of self starting at index `my_from` of length `length` into `other`, starting at `its_from` + fun copy_to(other: StringIndex, my_from: Int, its_from: Int, length: Int)`{ + UTF8Char* myfrom = recv + my_from*(sizeof(UTF8Char)); + UTF8Char* itsfrom = other + its_from*(sizeof(UTF8Char)); + memcpy(itsfrom, myfrom, length); + `} +end + +redef class FlatString + + # Index of the characters of the FlatString + private var index: StringIndex + + # Length in bytes of the string (e.g. the length of the C string) + var bytelen: Int + + private init with_infos_index(items: NativeString, len: Int, index_from: Int, index_to: Int, index: StringIndex, bytelen: Int) + do + self.items = items + length = len + self.index_from = index_from + self.index_to = index_to + self.index = index + self.bytelen = bytelen + end + + redef fun to_cstring + do + if real_items != null then return real_items.as(not null) + var new_items = calloc_string(bytelen + 1) + self.items.copy_to(new_items, bytelen, index[index_from].pos, 0) + new_items[bytelen] = '\0' + self.real_items = new_items + return new_items + end + + redef fun substring(from, count) + do + assert count >= 0 + + if from < 0 then + count += from + if count < 0 then count = 0 + from = 0 + end + + if count == 0 then return empty + + var real_from = index_from + from + var real_to = real_from + count - 1 + + if real_to > index_to then real_to = index_to + + var sub_bytelen = (index[real_to].pos - index[from].pos) + index[from].len + + return new FlatString.with_infos_index(items, count, real_from, real_to, index, sub_bytelen) + end + + redef fun reversed + do + var native = calloc_string(self.bytelen + 1) + var length = self.length + var index = self.index + var pos = 0 + var i = 0 + var ipos = bytelen + var new_index = new StringIndex(length) + var pos_index = length + while i < length do + var uchar = index[i] + var uchar_len = uchar.len + ipos -= uchar_len + new_index[pos_index] = new UnicodeChar(ipos, native) + pos_index -= 1 + items.copy_to(native, uchar_len, pos, ipos) + pos += uchar_len + i += 1 + end + return new FlatString.with_infos_index(native, length, 0, length-1, new_index, bytelen) + end + + redef fun *(i) + do + assert i >= 0 + + var mylen = self.bytelen + var finlen = mylen * i + + var my_items = self.items + + var my_real_len = length + var my_real_fin_len = my_real_len * i + + var target_string = calloc_string((finlen) + 1) + + var my_index = index + var new_index = new StringIndex(my_real_fin_len) + + target_string[finlen] = '\0' + + var current_last = 0 + var curr_index = 0 + + for iteration in [1 .. i] do + my_items.copy_to(target_string, mylen, index_from, current_last) + my_index.copy_to(new_index, length, 0, curr_index) + current_last += mylen + end + + return new FlatString.with_infos_index(target_string, my_real_fin_len, 0, my_real_fin_len -1, new_index, finlen) + + end + + redef fun to_upper + do + var outstr = calloc_string(self.bytelen + 1) + + var out_index = 0 + var index = self.index + var ipos = 0 + var max = length + var items = self.items + + while ipos < max do + var u = index[ipos].to_upper + u.ns.copy_to(outstr, u.len, u.pos, out_index) + out_index += u.len + ipos += 1 + end + + outstr[self.bytelen] = '\0' + + return outstr.to_s_with_length(self.bytelen) + end + + redef fun to_lower + do + var outstr = calloc_string(self.bytelen + 1) + + var out_index = 0 + var index = self.index + var ipos = 0 + var max = length + var items = self.items + + while ipos < max do + var u = index[ipos].to_lower + u.ns.copy_to(outstr, u.len, u.pos, out_index) + out_index += u.len + ipos += 1 + end + + outstr[self.bytelen] = '\0' + + return outstr.to_s_with_length(self.bytelen) + end + + redef fun output + do + var i = self.index_from + var imax = self.index_to + while i <= imax do + index[i].output + i += 1 + end + end + +end + +redef class NativeString + + # Creates the index for said NativeString + # `length` is the size of the CString (in bytes, up to the first \0) + # real_len is just a way to store the length (UTF-8 characters) + private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UnicodeChar.len `{ + int pos = 0; + int index_pos = 0; + UTF8Char* index = malloc(length*sizeof(UTF8Char)); + while(pos < length){ + UTF8Char* curr = &index[index_pos]; + curr->pos = pos; + curr->ns = recv; + pos += UnicodeChar_len(curr); + index_pos ++; + } + Container_of_Int_item__assign(real_len, index_pos); + return index; + `} + + redef fun to_s: FlatString + do + var len = cstring_length + return to_s_with_length(len) + end + + redef fun to_s_with_length(len: Int): FlatString + do + var real_len = new Container[Int](0) + var x = make_index(len, real_len) + return new FlatString.with_infos_index(self, real_len.item, 0, real_len.item - 1, x, len) + end + + redef fun to_s_with_copy + do + var real_len = new Container[Int](0) + var length = cstring_length + var x = make_index(length, real_len) + var new_self = calloc_string(length + 1) + copy_to(new_self, length, 0, 0) + return new FlatString.with_infos_index(new_self, real_len.item, 0, real_len.item - 1, x, length) + end +end + +redef class OFStream + redef fun write(s) + do + assert _writable + if s isa FlatText then + if s isa FlatString then + write_native(s.to_cstring, s.bytelen) + else + write_native(s.to_cstring, s.length) + end + else for i in s.substrings do write_native(i.to_cstring, i.length) + end +end diff --git a/src/android_annotations.nit b/src/android_annotations.nit index cad98bd..7e9f43a 100644 --- a/src/android_annotations.nit +++ b/src/android_annotations.nit @@ -133,21 +133,21 @@ redef class ModelBuilder end end - var sources = new Array[MModule] - var annotations = null + var annotations = new HashSet[AAnnotation] for mmod in mmodule.in_importation.direct_greaters do var res = priority_annotation_on_modules(name, mmod) - if res != null then - sources.add mmod - annotations = res - end + if res != null then annotations.add res end - if sources.length > 1 then + if annotations.length > 1 then + var locs = new Array[Location] + for annot in annotations do locs.add(annot.location) + toolcontext.error(mmodule.location, - "Priority conflict on annotation {name}, it has been defined in: {sources.join(", ")}") + "Priority conflict on annotation {name}, it has been defined in: {locs.join(", ")}") return null - end - return annotations + else if annotations.length == 1 then + return annotations.first + else return null end end diff --git a/src/common_ffi/java.nit b/src/common_ffi/java.nit index eefb5c6..a43cab6 100644 --- a/src/common_ffi/java.nit +++ b/src/common_ffi/java.nit @@ -119,7 +119,11 @@ class JavaLanguage end end - for p in signature.mparameters do params.add(p.name) + for p in signature.mparameters do + var param_mtype = p.mtype + param_mtype = param_mtype.resolve_for(mclass_type, mclass_type, mmodule, true) + params.add(to_java_call_context.cast_to(param_mtype, p.name)) + end var cname = "(*nit_ffi_jni_env)->CallStatic{jni_signature_alt}Method" var ccall diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..68eac53 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,14 @@ +example.org +fulltest +listtest.sh +modifs +nit.html +nitcc0.lr.dot +nitcc_lexer.nit +nitcc_parser.nit +sometests +test.db +test_jvm/Queue.class +test_jvm/Test2.class +test_jvm/TestJvm.class +test_nity.db diff --git a/tests/README b/tests/README index c41d71d..7042a5c 100644 --- a/tests/README +++ b/tests/README @@ -5,3 +5,8 @@ This directory contains small NIT programs divided in some categories: * 'example' are examples * 'test' are others tests * 'shootout' are benchmarks from http://shootout.alioth.debian.org/ + +The .gitignore contains specific artefacts produced by some tests. +However, it is better to produce these artifacts in the `out` directory instead, +because `out` is cleaned before runing tests (so that old artefacts do not +interfere with new executions of tests) diff --git a/tests/sav/file_server_on_port_80.res b/tests/sav/file_server_on_port_80.res new file mode 100644 index 0000000..1d69303 --- /dev/null +++ b/tests/sav/file_server_on_port_80.res @@ -0,0 +1,4 @@ +Mandatory option -u, --usergroup not found. +Usage: file_server [Options] + -u, --usergroup Drop privileges to user:group or simply user + --help, -h Print this message diff --git a/tests/sav/test_ffi_java_minimal.res b/tests/sav/test_ffi_java_minimal.res index 802992c..a97f5be 100644 --- a/tests/sav/test_ffi_java_minimal.res +++ b/tests/sav/test_ffi_java_minimal.res @@ -1 +1,2 @@ Hello world +777 diff --git a/tests/sav/utf_test.res b/tests/sav/utf_test.res new file mode 100644 index 0000000..4055d93 --- /dev/null +++ b/tests/sav/utf_test.res @@ -0,0 +1,11 @@ +28 +すでa語A本日a 𐍆,A ᓂ . ᓀ 界世a𐍃ーЖロaハ +ハaロЖー𐍃a世界 ᓀ . ᓂ A,𐍆 a日本A語aです +ー𐍃a世 +30fc +10343 +61 +4e16 +ハAロЖー𐍃A世界 ᓀ . ᓂ A,𐍆 A日本A語Aです +ハaロЖー𐍃a世界 ᓀ . ᓂ a,𐍆 a日本a語aです +ハaロЖー𐍃a世界 ᓀ . ᓂ A,𐍆 a日本A語aですハaロЖー𐍃a世界 ᓀ . ᓂ A,𐍆 a日本A語aです diff --git a/tests/test_ffi_java_minimal.nit b/tests/test_ffi_java_minimal.nit index 3705de2..3c80b7c 100644 --- a/tests/test_ffi_java_minimal.nit +++ b/tests/test_ffi_java_minimal.nit @@ -21,6 +21,12 @@ class MyClass fun foo in "Java" `{ System.out.println("Hello world"); `} + + fun bar(i: Int) in "Java" `{ + System.out.println(i); + `} end -(new MyClass).foo +var a = new MyClass +a.foo +a.bar(777) diff --git a/tests/utf_test.nit b/tests/utf_test.nit new file mode 100644 index 0000000..88474c4 --- /dev/null +++ b/tests/utf_test.nit @@ -0,0 +1,42 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# This file is free software, which comes along with NIT. This software is +# distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. You can modify it is you want, provided this header +# is kept unaltered, and a notification of the changes is added. +# You are allowed to redistribute it and sell it, alone or is a part of +# another product. + +import standard +intrude import string_experimentations::utf8 + +var s = "aàハ𐍆".as(FlatString) +assert s.index[0].code_point == 97 +assert s.index[1].code_point == 224 +assert s.index[2].code_point == 12495 +assert s.index[3].code_point == 66374 + +var str = "ハaロЖー𐍃a世界 ᓀ . ᓂ A,𐍆 a日本A語aです".as(FlatString) + +print str.length + +print str.reversed + +str.output + +print "" + +var x = str.substring(4,4).as(FlatString) + +print x + +for i in [0..x.length[ do + print x.index[i + x.index_from].code_point.to_hex +end + +print str.to_upper + +print str.to_lower + +print str * 2