X-Git-Url: http://nitlanguage.org

diff --git a/lib/nitcc_runtime.nit b/lib/nitcc_runtime.nit
index d74e246..cfddec3 100644
--- a/lib/nitcc_runtime.nit
+++ b/lib/nitcc_runtime.nit
@@ -15,6 +15,8 @@
 # Runtime library required by parsers and lexers generated by nitcc
 module nitcc_runtime
 
+import serialization
+
 # A abstract parser engine generated by nitcc
 abstract class Parser
 	# The list of tokens
@@ -76,7 +78,7 @@ abstract class Parser
 		#print "  expected: {state.error_msg}"
 		#print "  node_stack={node_stack.join(", ")}"
 		#print "  state_stack={state_stack.join(", ")}"
-		node_stack.add(token)
+		node_stack.push(token)
 		var error: NError
 		if token isa NLexerError then
 			error = token
@@ -165,16 +167,36 @@ abstract class Lexer
 	fun lex: CircularArray[NToken]
 	do
 		var res = new CircularArray[NToken]
+		loop
+			var t = next_token
+			if t != null then res.add t
+			if t isa NEof or t isa NError then break
+		end
+		return res
+	end
+
+	# Cursor current position (in chars, starting from 0)
+	var pos_start = 0
+
+	# Cursor current line (starting from 1)
+	var line_start = 1
+
+	# Cursor current column (in chars, starting from 1)
+	var col_start = 1
+
+	# Move the cursor and return the next token.
+	#
+	# Returns a `NEof` and the end.
+	# Returns `null` if the token is ignored.
+	fun next_token: nullable NToken
+	do
 		var state = start_state
-		var pos = 0
-		var pos_start = 0
-		var pos_end = 0
-		var line = 1
-		var line_start = 1
-		var line_end = 0
-		var col = 1
-		var col_start = 1
-		var col_end = 0
+		var pos = pos_start
+		var pos_end = pos_start - 1
+		var line = line_start
+		var line_end = line_start - 1
+		var col = col_start
+		var col_end = col_start - 1
 		var last_state: nullable DFAState = null
 		var text = stream
 		var length = text.length
@@ -195,37 +217,30 @@ abstract class Lexer
 				next = state.trans(c)
 			end
 			if next == null then
+				var token
 				if pos_start < length then
 					if last_state == null then
-						var token = new NLexerError
+						token = new NLexerError
 						var position = new Position(pos_start, pos, line_start, line, col_start, col)
 						token.position = position
 						token.text = text.substring(pos_start, pos-pos_start+1)
-						res.add token
-						break
+					else if not last_state.is_ignored then
+						var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
+						token = last_state.make_token(position, text)
+					else
+						token = null
 					end
-					var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
-					var token = last_state.make_token(position, text.substring(pos_start, pos_end-pos_start+1))
-					if token != null then res.add(token)
-				end
-				if pos >= length then
-					var token = new NEof
+				else
+					token = new NEof
 					var position = new Position(pos, pos, line, line, col, col)
 					token.position = position
 					token.text = ""
-					res.add token
-					break
 				end
-				state = start_state
 				pos_start = pos_end + 1
-				pos = pos_start
 				line_start = line_end
-				line = line_start
 				col_start = col_end
-				col = col_start
 
-				last_state = null
-				continue
+				return token
 			end
 			state = next
 			pos += 1
@@ -235,7 +250,6 @@ abstract class Lexer
 				col = 1
 			end
 		end
-		return res
 	end
 end
 
@@ -244,7 +258,8 @@ end
 interface DFAState
 	fun is_accept: Bool do return false
 	fun trans(c: Char): nullable DFAState do return null
-	fun make_token(position: Position, text: String): nullable NToken is abstract
+	fun make_token(position: Position, source: String): nullable NToken is abstract
+	fun is_ignored: Bool do return false
 end
 
 ###
@@ -290,6 +305,8 @@ end
 # A position into a input stream
 # Used to give position to tokens
 class Position
+	serialize
+
 	var pos_start: Int
 	var pos_end: Int
 	var line_start: Int
@@ -299,6 +316,12 @@ class Position
 
 	redef fun to_s do return "{line_start}:{col_start}-{line_end}:{col_end}"
 
+	# Extract the content from the given source
+	fun extract(source: String): String
+	do
+		return source.substring(pos_start, pos_end-pos_start+1)
+	end
+
 	# Get the lines covered by `self` and underline the target columns.
 	#
 	# This is useful for pretty printing errors or debug the output
@@ -343,7 +366,7 @@ abstract class Node
 	fun children: SequenceRead[nullable Node] is abstract
 
 	# A point of view of a depth-first visit of all non-null children
-	var depth: Collection[Node] = new DephCollection(self)
+	var depth: Collection[Node] = new DephCollection(self) is lazy
 
 	# Visit all the children of the node with the visitor `v`
 	protected fun visit_children(v: Visitor)
@@ -410,10 +433,10 @@ end
 private class DephIterator
 	super Iterator[Node]
 
-	var stack = new List[Iterator[nullable Node]]
+	var stack = new Array[Iterator[nullable Node]]
 
 	init(i: Iterator[nullable Node]) is old_style_init do
-		stack.add i
+		stack.push i
 	end
 
 	redef fun is_ok do return not stack.is_empty