nitcc: move up `text` of token earlier to improve generated code

[nit.git] / contrib / nitcc / src / autom.nit

diff --git a/contrib/nitcc/src/autom.nit b/contrib/nitcc/src/autom.nit
index 5ece999..21d23ee 100644 (file)
--- a/contrib/nitcc/src/autom.nit
+++ b/contrib/nitcc/src/autom.nit
@@ -21,7 +21,7 @@ import grammar
 # A finite automaton
 class Automaton
        # The start state
-       var start: State
+       var start: State is noinit
 
        # State that are accept states
        var accept = new Array[State]
@@ -443,7 +443,7 @@ class Automaton
                        ni += 1
                end
 
-               var f = new OFStream.open(filepath) 
+               var f = new FileWriter.open(filepath)
                 f.write("digraph g \{\n")
 
                for s in states do
@@ -638,10 +638,10 @@ private class DFAGenerator
        var automaton: Automaton
        var parser: nullable String
 
-       var out: OStream is noinit
+       var out: Writer is noinit
 
        init do
-               self.out = new OFStream.open(filepath)
+               self.out = new FileWriter.open(filepath)
        end
 
        fun add(s: String) do out.write(s)
@@ -655,7 +655,8 @@ private class DFAGenerator
                        i += 1
                end
 
-               add "# Lexer generated by nitcc for the grammar {name}"
+               add "# Lexer generated by nitcc for the grammar {name}\n"
+               add "module {name}_lexer is no_warning \"missing-doc\"\n"
                add("import nitcc_runtime\n")
 
                var p = parser
@@ -666,12 +667,10 @@ private class DFAGenerator
                add("\tredef fun start_state do return dfastate_{names[automaton.start]}\n")
                add("end\n")
 
-               add("redef class Object\n")
                for s in automaton.states do
                        var n = names[s]
-                       add("\tprivate fun dfastate_{n}: DFAState{n} do return once new DFAState{n}\n")
+                       add("private fun dfastate_{n}: DFAState{n} do return once new DFAState{n}\n")
                end
-               add("end\n")
 
                add("class MyNToken\n")
                add("\tsuper NToken\n")
@@ -689,8 +688,13 @@ private class DFAGenerator
                                        token = null
                                end
                                add("\tredef fun is_accept do return true\n")
-                               add("\tredef fun make_token(position, text) do\n")
+                               var is_ignored = false
                                if token != null and token.name == "Ignored" then
+                                       is_ignored = true
+                                       add("\tredef fun is_ignored do return true\n")
+                               end
+                               add("\tredef fun make_token(position, text) do\n")
+                               if is_ignored then
                                        add("\t\treturn null\n")
                                else
                                        if token == null then
@@ -715,7 +719,7 @@ private class DFAGenerator
                        else
                                add("\tredef fun trans(char) do\n")
 
-                               add("\t\tvar c = char.ascii\n")
+                               add("\t\tvar c = char.code_point\n")
                                var haslast = false
                                var last = -1
                                for sym, next in trans do
@@ -741,6 +745,11 @@ private class DFAGenerator
        end
 end
 
+redef class Token
+       # The associated text (if any, ie defined in the parser part)
+       var text: nullable String is noautoinit, writable
+end
+
 # A state in a finite automaton
 class State
        # Outgoing transitions
@@ -792,14 +801,14 @@ class TSymbol
                if f <= 32 then
                        res = "#{f}"
                else
-                       res = f.ascii.to_s
+                       res = f.code_point.to_s
                end
                var l = last
                if f == l then return res
                res += " .. "
                if l == null then return res
                if l <= 32 or l >= 127 then return res + "#{l}"
-               return res + l.ascii.to_s
+               return res + l.code_point.to_s
        end
 end