-/* This file is part of NIT ( http://www.nitlanguage.org ).
- *
- * Copyright 2008 Jean Privat <jean@pryen.org>
- * Based on algorithms developped for ( http://www.sablecc.org/ ).
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+$ // This file is part of NIT ( http://www.nitlanguage.org ).
+$ //
+$ // Copyright 2008 Jean Privat <jean@pryen.org>
+$ // Based on algorithms developped for ( http://www.sablecc.org/ ).
+$ //
+$ // Licensed under the Apache License, Version 2.0 (the "License");
+$ // you may not use this file except in compliance with the License.
+$ // You may obtain a copy of the License at
+$ //
+$ // http://www.apache.org/licenses/LICENSE-2.0
+$ //
+$ // Unless required by applicable law or agreed to in writing, software
+$ // distributed under the License is distributed on an "AS IS" BASIS,
+$ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+$ // See the License for the specific language governing permissions and
+$ // limitations under the License.
$ template make_lexer()
var _state: Int = 0
# Name of the stream (as given to tokens)
- readable var _filename: String
+ readable var _filename: String
# Input stream where character are read
var _stream: IStream
var goto_table = _goto_table[_state]
var accept = _accept_table[_state]
- _text.clear
+ var text = _text
+ text.clear
while true do
var c = get_char
if c != -1 then
+ var cr = _cr
+ var line = _line
+ var pos = _pos
if c == 10 then
- if _cr then
- _cr = false
+ if cr then
+ cr = false
else
- _line = _line + 1
- _pos = 0
+ line = line + 1
+ pos = 0
end
else if c == 13 then
- _line = _line + 1
- _pos = 0
- _cr = true
+ line = line + 1
+ pos = 0
+ cr = true
else
- _pos = _pos + 1
- _cr = false
+ pos = pos + 1
+ cr = false
end
- _text.add(c.ascii)
+ text.add(c.ascii)
var first_loop = true # aka until
while dfa_state < -1 or first_loop do
dfa_state = -1
- var tmp1 = goto_table[old_state]
+ var tmp0 = goto_table[old_state]
var low = 0
- var high = tmp1.length - 1
-
- while low <= high do
- var middle = (low + high) / 2
- var tmp2 = tmp1[middle]
-
- if c < tmp2[0] then
- high = middle - 1
- else if c > tmp2[1] then
- low = middle + 1
- else
- dfa_state = tmp2[2]
- low = high + 1 # aka break
+ var high = tmp0.length - 1
+
+ if high >= 0 then
+ var tmp1 = tmp0.intern_items
+ while low <= high do
+ var middle = (low + high) / 2
+ var tmp2 = tmp1[middle].intern_items
+
+ if c < tmp2[0] then
+ high = middle - 1
+ else if c > tmp2[1] then
+ low = middle + 1
+ else
+ dfa_state = tmp2[2]
+ low = high + 1 # aka break
+ end
end
end
first_loop = false # aka until
end
+
+ _cr = cr
+ _line = line
+ _pos = pos
else
dfa_state = -1
end
if accept[dfa_state] != -1 then
accept_state = dfa_state
accept_token = accept[dfa_state]
- accept_length = _text.length
+ accept_length = text.length
accept_pos = _pos
accept_line = _line
end
if accept_state != -1 then
$ foreach {//token}
if accept_token == ${position()-1} then
+ var location = new Location(_filename, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
$ if {not(@text)}
$ if {@parser_index}
- var token_text = _text.substring(0, accept_length)
- var token = new @ename.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
+ var token_text = text.substring(0, accept_length)
+ var token = new @ename.init_tk(token_text, location)
$ end
$ else
- var token = new @ename.init_tk(_filename, start_line + 1, start_pos + 1)
+ var token = new @ename.init_tk(location)
$ end
push_back(accept_length)
_pos = accept_pos
end
$ end foreach
else
- if _text.length > 0 then
- var token = new PError.init_error(_filename, start_line + 1, start_pos + 1, "Unknown token: {_text}")
+ var location = new Location(_filename, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
+ if text.length > 0 then
+ var token = new PError.init_error("Syntax error: unknown token {text}.", location)
return token
else
- var token = new EOF(_filename, start_line + 1, start_pos + 1)
+ var token = new EOF(location)
return token
end
end
$ if {count(goto)!=0}
[
$ foreach {goto}
- [@low, @high, @state] [-sep ','-]
+ [@low, @high, @state][-sep ','-]
$ end foreach
- ] [-sep ','-]
+ ][-sep ','-]
$ else
- nil_array [-sep ','-]
+ nil_array[-sep ','-]
$ end
$ end foreach
- ] [-sep ','-]
+ ][-sep ','-]
$ end foreach
]
end
-
+
private fun nil_array: Array[Array[Int]]
do
return once new Array[Array[Int]]
private fun build_accept_table do
_accept_table = once [
$ foreach {lexer_data/accept_table/state}
- [
- [-foreach {i}-]${.} [-sep ','-] [-end foreach-]
+ [
+ [-foreach {i}-]${.}[-sep ','-][-end foreach-]
- ] [-sep ','-]
+ ][-sep ','-]
$ end foreach
]
end
nitlanguage / nit.git / blobdiff