nitcc: move the responsibility of making the substring to the automatom

Signed-off-by: Jean Privat <jean@pryen.org>

diff --git a/contrib/nitcc/src/autom.nit b/contrib/nitcc/src/autom.nit
index 21d23ee..c419caa 100644 (file)
--- a/contrib/nitcc/src/autom.nit
+++ b/contrib/nitcc/src/autom.nit
@@ -693,17 +693,23 @@ private class DFAGenerator
                                        is_ignored = true
                                        add("\tredef fun is_ignored do return true\n")
                                end
-                               add("\tredef fun make_token(position, text) do\n")
+                               add("\tredef fun make_token(position, source) do\n")
                                if is_ignored then
                                        add("\t\treturn null\n")
                                else
                                        if token == null then
                                                add("\t\tvar t = new MyNToken\n")
+                                               add("\t\tt.text = position.extract(source)\n")
                                        else
                                                add("\t\tvar t = new {token.cname}\n")
+                                               var ttext = token.text
+                                               if ttext == null then
+                                                       add("\t\tt.text = position.extract(source)\n")
+                                               else
+                                                       add("\t\tt.text = \"{ttext.escape_to_nit}\"\n")
+                                               end
                                        end
                                        add("\t\tt.position = position\n")
-                                       add("\t\tt.text = text\n")
                                        add("\t\treturn t\n")
                                end
                                add("\tend\n")

diff --git a/lib/nitcc_runtime.nit b/lib/nitcc_runtime.nit
index f50f8ee..737d3e8 100644 (file)
--- a/lib/nitcc_runtime.nit
+++ b/lib/nitcc_runtime.nit
@@ -206,7 +206,7 @@ abstract class Lexer
                                        end
                                        if not last_state.is_ignored then
                                                var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
-                                               var token = last_state.make_token(position, text.substring(pos_start, pos_end-pos_start+1))
+                                               var token = last_state.make_token(position, text)
                                                if token != null then res.add(token)
                                        end
                                end
@@ -246,7 +246,7 @@ end
 interface DFAState
        fun is_accept: Bool do return false
        fun trans(c: Char): nullable DFAState do return null
-       fun make_token(position: Position, text: String): nullable NToken is abstract
+       fun make_token(position: Position, source: String): nullable NToken is abstract
        fun is_ignored: Bool do return false
 end
 
@@ -302,6 +302,12 @@ class Position
 
        redef fun to_s do return "{line_start}:{col_start}-{line_end}:{col_end}"
 
+       # Extract the content from the given source
+       fun extract(source: String): String
+       do
+               return source.substring(pos_start, pos_end-pos_start+1)
+       end
+
        # Get the lines covered by `self` and underline the target columns.
        #
        # This is useful for pretty printing errors or debug the output