From e72eedf293da3adccbc662542663fa6bcb6c19f9 Mon Sep 17 00:00:00 2001 From: Jean Privat Date: Fri, 4 Dec 2015 08:58:07 -0500 Subject: [PATCH] nitcc_rt: do not create ignored tokens Signed-off-by: Jean Privat --- contrib/nitcc/src/autom.nit | 7 ++++++- lib/nitcc_runtime.nit | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/contrib/nitcc/src/autom.nit b/contrib/nitcc/src/autom.nit index 946b420..d2ab41d 100644 --- a/contrib/nitcc/src/autom.nit +++ b/contrib/nitcc/src/autom.nit @@ -688,8 +688,13 @@ private class DFAGenerator token = null end add("\tredef fun is_accept do return true\n") - add("\tredef fun make_token(position, text) do\n") + var is_ignored = false if token != null and token.name == "Ignored" then + is_ignored = true + add("\tredef fun is_ignored do return true\n") + end + add("\tredef fun make_token(position, text) do\n") + if is_ignored then add("\t\treturn null\n") else if token == null then diff --git a/lib/nitcc_runtime.nit b/lib/nitcc_runtime.nit index a85b22b..f50f8ee 100644 --- a/lib/nitcc_runtime.nit +++ b/lib/nitcc_runtime.nit @@ -204,9 +204,11 @@ abstract class Lexer res.add token break end - var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end) - var token = last_state.make_token(position, text.substring(pos_start, pos_end-pos_start+1)) - if token != null then res.add(token) + if not last_state.is_ignored then + var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end) + var token = last_state.make_token(position, text.substring(pos_start, pos_end-pos_start+1)) + if token != null then res.add(token) + end end if pos >= length then var token = new NEof @@ -245,6 +247,7 @@ interface DFAState fun is_accept: Bool do return false fun trans(c: Char): nullable DFAState do return null fun make_token(position: Position, text: String): nullable NToken is abstract + fun is_ignored: Bool do return false end ### -- 1.7.9.5