1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Ad-hoc hand-written lexer for nitcc
16 # This avoid to commit (and rely on) a generated lexer
19 # Required for the tokens definitions
22 # Hand-written lexer of nitcc.
23 # Used only for the bootstrap of the tool.
25 # The text to tokenize
28 # The iterator on text
29 private var iter
: Iterator[Char] is noinit
31 # The current position
34 # The tokens currently produced
35 private var tokens
= new Array[NToken]
37 # Tokenize and returns the tokens
38 fun lex
: Array[NToken]
40 iter
= text
.chars
.iterator
43 if not iter
.is_ok
then break
54 if iter
.item
== '>' then
80 else if c
== '\'' then
82 else if c >= 'a
' and c <= 'z
' then
84 else if c >= 'A
' and c <= 'Z
' then
86 else if c == '/' and iter.is_ok and iter.item == '/' then
87 while iter.is_ok and iter.item != '\n
' do iter.next
96 private fun error(c: Char)
98 print "pos {pos}: Lexer error on '{c}'."
104 var b = new FlatBuffer
110 if not iter
.is_ok
then
116 else if c
== '\'' then
129 private fun id(c: Char)
131 var b = new FlatBuffer
135 if c != '_
' and (c<'a
' or c >'z
') and (c<'0' or c>'9') then
146 private fun kw(c: Char)
148 var b = new FlatBuffer
152 if c != '_
' and (c<'a
' or c >'z
') and (c<'0' or c>'9') then
165 while iter.is_ok and iter.item <= ' ' do