Merge: Loose Tokens

[nit.git] / src / parser / parser_nodes.nit

diff --git a/src/parser/parser_nodes.nit b/src/parser/parser_nodes.nit
index fe79ef1..9131c3e 100644 (file)
--- a/src/parser/parser_nodes.nit
+++ b/src/parser/parser_nodes.nit
@@ -311,6 +311,35 @@ abstract class Token
        # May have disappeared in the AST
        var next_token: nullable Token = null
 
+       # Is `self` a token discarded from the AST?
+       #
+       # Loose tokens are not present in the AST.
+       # It means they were identified by the lexer but were discarded by the parser.
+       # It also means that they are not visited or manipulated by AST-related functions.
+       #
+       # Each loose token is attached to the non-loose token that precedes or follows it.
+       # The rules are the following:
+       #
+       # * tokens that follow a non-loose token on a same line are attached to it.
+       #   See `next_looses`.
+       # * other tokens, thus that precede a non-loose token on the same line or the next one,
+       # are attached to this one. See `prev_looses`.
+       #
+       # Loose tokens are mostly end of lines (`TEol`) and comments (`TComment`).
+       # Whitespace are ignored by the lexer, so they are not even considered as loose tokens.
+       # See `blank_before` to get the whitespace that separate tokens.
+       var is_loose = false
+
+       # Loose tokens that precede `self`.
+       #
+       # These tokens start the line or belong to a line with only loose tokens.
+       var prev_looses = new Array[Token] is lazy
+
+       # Loose tokens that follow `self`
+       #
+       # These tokens are on the same line than `self`.
+       var next_looses = new Array[Token] is lazy
+
        # The verbatim blank text between `prev_token` and `self`
        fun blank_before: String
        do
@@ -946,6 +975,36 @@ class THexNumber
        super TokenLiteral
 end
 
+# A literal binary integer
+class TBinNumber
+       super TokenLiteral
+end
+
+# A literal octal integer
+class TOctNumber
+       super TokenLiteral
+end
+
+# A literal decimal byte
+class TBytenum
+       super TokenLiteral
+end
+
+# A literal hexadecimal byte
+class THexBytenum
+       super TokenLiteral
+end
+
+# A literal binary byte
+class TBinBytenum
+       super TokenLiteral
+end
+
+# A literal octal byte
+class TOctBytenum
+       super TokenLiteral
+end
+
 # A literal floating point number
 class TFloat
        super TokenLiteral
@@ -2462,6 +2521,59 @@ class AHexIntExpr
        var n_hex_number: THexNumber is writable, noinit
 end
 
+# An integer literal in binary format
+class ABinIntExpr
+       super AIntExpr
+
+       # The binary token
+       var n_bin_number: TBinNumber is writable, noinit
+end
+
+# An integer literal in octal format
+class AOctIntExpr
+       super AIntExpr
+
+       # The octal token
+       var n_oct_number: TOctNumber is writable, noinit
+end
+
+# An byte literal
+class AByteExpr
+       super AExpr
+end
+
+# An byte literal in decimal format
+class ADecByteExpr
+       super AByteExpr
+
+       # The decimal token
+       var n_bytenum: TBytenum is writable, noinit
+end
+
+# An byte literal in hexadecimal format
+class AHexByteExpr
+       super AByteExpr
+
+       # The hexadecimal token
+       var n_hex_bytenum: THexBytenum is writable, noinit
+end
+
+# An byte literal in binary format
+class ABinByteExpr
+       super AByteExpr
+
+       # The binary token
+       var n_bin_bytenum: TBinBytenum is writable, noinit
+end
+
+# An byte literal in octal format
+class AOctByteExpr
+       super AByteExpr
+
+       # The octal token
+       var n_oct_bytenum: TOctBytenum is writable, noinit
+end
+
 # A float literal
 class AFloatExpr
        super AExpr