As usual, the new features and changes aim to follow as closely as possible the C API.
With this PR, we have pretty much all the OpenGL ES 2.0 features needed for mineit and for a first version of gamnit.
Pull-Request: #1878
Reviewed-by: Jean Privat <jean@pryen.org>
token = null
end
add("\tredef fun is_accept do return true\n")
- add("\tredef fun make_token(position, text) do\n")
+ var is_ignored = false
if token != null and token.name == "Ignored" then
+ is_ignored = true
+ add("\tredef fun is_ignored do return true\n")
+ end
+ add("\tredef fun make_token(position, source) do\n")
+ if is_ignored then
add("\t\treturn null\n")
else
if token == null then
add("\t\tvar t = new MyNToken\n")
+ add("\t\tt.text = position.extract(source)\n")
else
add("\t\tvar t = new {token.cname}\n")
+ var ttext = token.text
+ if ttext == null then
+ add("\t\tt.text = position.extract(source)\n")
+ else
+ add("\t\tt.text = \"{ttext.escape_to_nit}\"\n")
+ end
end
add("\t\tt.position = position\n")
- add("\t\tt.text = text\n")
add("\t\treturn t\n")
end
add("\tend\n")
add("\tredef fun trans(char) do\n")
add("\t\tvar c = char.code_point\n")
- var haslast = false
+
+ # Collect the sequence of tests in the dispatch sequence
+ # The point here is that for each transition, there is a first and a last
+ # So holes hare to be identified
+ var dispatch = new HashMap[Int, nullable State]
+ var haslast: nullable State = null
+
var last = -1
for sym, next in trans do
- assert not haslast
+ assert haslast == null
assert sym.first > last
- if sym.first > last + 1 then add("\t\tif c <= {sym.first-1} then return null\n")
+ if sym.first > last + 1 then
+ dispatch[sym.first-1] = null
+ end
var l = sym.last
if l == null then
- add("\t\treturn dfastate_{names[next]}\n")
- haslast= true
+ haslast = next
else
- add("\t\tif c <= {l} then return dfastate_{names[next]}\n")
+ dispatch[l] = next
last = l
end
end
- if not haslast then add("\t\treturn null\n")
+
+ # Generate a sequence of `if` for the dispatch
+ if haslast != null then
+ # Special case: handle up-bound first if not an error
+ add("\t\tif c > {last} then return dfastate_{names[haslast]}\n")
+ # previous become the new last case
+ haslast = dispatch[last]
+ dispatch.keys.remove(last)
+ end
+ for c, next in dispatch do
+ if next == null then
+ add("\t\tif c <= {c} then return null\n")
+ else
+ add("\t\tif c <= {c} then return dfastate_{names[next]}\n")
+ end
+ end
+ if haslast == null then
+ add("\t\treturn null\n")
+ else
+ add("\t\treturn dfastate_{names[haslast]}\n")
+ end
+
add("\tend\n")
end
add("end\n")
end
end
+redef class Token
+ # The associated text (if any, ie defined in the parser part)
+ var text: nullable String is noautoinit, writable
+end
+
# A state in a finite automaton
class State
# Outgoing transitions
redef class Token
# The associated expression (if any, ie defined in the lexer part)
var nexpr: nullable Nexpr
- # The associated text (if any, ie defined in the parser part)
- var text: nullable String
# Build a NFA according to nexpr or text
# Does not tag it!
private class DFAState1
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun is_ignored do return true
+ redef fun make_token(position, source) do
return null
end
redef fun trans(char) do
super DFAState
redef fun trans(char) do
var c = char.code_point
+ if c > 92 then return dfastate_2
if c <= 33 then return dfastate_2
if c <= 34 then return dfastate_29
if c <= 91 then return dfastate_2
- if c <= 92 then return dfastate_30
- return dfastate_2
+ return dfastate_30
end
end
private class DFAState3
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_44d_39d
+ t.text = ","
t.position = position
- t.text = text
return t
end
end
private class DFAState5
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new Nnumber
+ t.text = position.extract(source)
t.position = position
- t.text = text
return t
end
redef fun trans(char) do
private class DFAState6
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_58d_39d
+ t.text = ":"
t.position = position
- t.text = text
return t
end
end
private class DFAState7
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_91d_39d
+ t.text = "["
t.position = position
- t.text = text
return t
end
end
private class DFAState8
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_93d_39d
+ t.text = "]"
t.position = position
- t.text = text
return t
end
end
private class DFAState12
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_123d_39d
+ t.text = "\{"
t.position = position
- t.text = text
return t
end
end
private class DFAState13
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39d_125d_39d
+ t.text = "\}"
t.position = position
- t.text = text
return t
end
end
private class DFAState16
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39dtrue_39d
+ t.text = "true"
t.position = position
- t.text = text
return t
end
end
private class DFAState19
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39dnull_39d
+ t.text = "null"
t.position = position
- t.text = text
return t
end
end
private class DFAState23
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new N_39dfalse_39d
+ t.text = "false"
t.position = position
- t.text = text
return t
end
end
private class DFAState27
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new Nnumber
+ t.text = position.extract(source)
t.position = position
- t.text = text
return t
end
redef fun trans(char) do
private class DFAState28
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new Nnumber
+ t.text = position.extract(source)
t.position = position
- t.text = text
return t
end
redef fun trans(char) do
private class DFAState29
super DFAState
redef fun is_accept do return true
- redef fun make_token(position, text) do
+ redef fun make_token(position, source) do
var t = new Nstring
+ t.text = position.extract(source)
t.position = position
- t.text = text
return t
end
end
abstract class Parser
# The list of tokens
# FIXME: provide something better, like a lexer?
- var tokens = new List[NToken]
+ var tokens = new CircularArray[NToken]
# Look at the next token
# Used by generated parsers
#print " expected: {state.error_msg}"
#print " node_stack={node_stack.join(", ")}"
#print " state_stack={state_stack.join(", ")}"
- node_stack.add(token)
+ node_stack.push(token)
var error: NError
if token isa NLexerError then
error = token
protected fun start_state: DFAState is abstract
# Lexize a stream of characters and return a sequence of tokens
- fun lex: List[NToken]
+ fun lex: CircularArray[NToken]
do
- var res = new List[NToken]
+ var res = new CircularArray[NToken]
var state = start_state
var pos = 0
var pos_start = 0
var position = new Position(pos_start, pos, line_start, line, col_start, col)
token.position = position
token.text = text.substring(pos_start, pos-pos_start+1)
- res.add token
+ res.push token
break
end
- var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
- var token = last_state.make_token(position, text.substring(pos_start, pos_end-pos_start+1))
- if token != null then res.add(token)
+ if not last_state.is_ignored then
+ var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
+ var token = last_state.make_token(position, text)
+ if token != null then res.push(token)
+ end
end
if pos >= length then
var token = new NEof
var position = new Position(pos, pos, line, line, col, col)
token.position = position
token.text = ""
- res.add token
+ res.push token
break
end
state = start_state
interface DFAState
fun is_accept: Bool do return false
fun trans(c: Char): nullable DFAState do return null
- fun make_token(position: Position, text: String): nullable NToken is abstract
+ fun make_token(position: Position, source: String): nullable NToken is abstract
+ fun is_ignored: Bool do return false
end
###
redef fun to_s do return "{line_start}:{col_start}-{line_end}:{col_end}"
+ # Extract the content from the given source
+ fun extract(source: String): String
+ do
+ return source.substring(pos_start, pos_end-pos_start+1)
+ end
+
# Get the lines covered by `self` and underline the target columns.
#
# This is useful for pretty printing errors or debug the output
fun children: SequenceRead[nullable Node] is abstract
# A point of view of a depth-first visit of all non-null children
- var depth: Collection[Node] = new DephCollection(self)
+ var depth: Collection[Node] = new DephCollection(self) is lazy
# Visit all the children of the node with the visitor `v`
protected fun visit_children(v: Visitor)
private class DephIterator
super Iterator[Node]
- var stack = new List[Iterator[nullable Node]]
+ var stack = new Array[Iterator[nullable Node]]
init(i: Iterator[nullable Node]) is old_style_init do
- stack.add i
+ stack.push i
end
redef fun is_ok do return not stack.is_empty
nitlanguage / nit.git / commitdiff