contrib/pep8analysis/src/parser/lexer.nit

   1 # Lexer and its tokens.
   2 # This file was generated by SableCC (http://www.sablecc.org/).
   3 module lexer
   4
   5 intrude import parser_nodes
   6 private import tables
   7
   8 redef class Token
   9     var _text: nullable String
  10
  11     redef fun text
  12     do
  13         var res = _text
  14         if res != null then return res
  15         res = location.text
  16         _text = res
  17         return res
  18     end
  19
  20     fun parser_index: Int is abstract
  21 end
  22
  23 redef class TEol
  24     redef fun parser_index: Int
  25     do
  26         return 0
  27     end
  28
  29     init init_tk(loc: Location)
  30     do
  31                 _location = loc
  32     end
  33 end
  34
  35 redef class TNumber
  36     redef fun parser_index: Int
  37     do
  38         return 1
  39     end
  40
  41     init init_tk(loc: Location)
  42     do
  43                 _location = loc
  44     end
  45 end
  46
  47 redef class TFloat
  48     redef fun parser_index: Int
  49     do
  50         return 2
  51     end
  52
  53     init init_tk(loc: Location)
  54     do
  55                 _location = loc
  56     end
  57 end
  58
  59 redef class TChar
  60     redef fun parser_index: Int
  61     do
  62         return 3
  63     end
  64
  65     init init_tk(loc: Location)
  66     do
  67                 _location = loc
  68     end
  69 end
  70
  71 redef class TString
  72     redef fun parser_index: Int
  73     do
  74         return 4
  75     end
  76
  77     init init_tk(loc: Location)
  78     do
  79                 _location = loc
  80     end
  81 end
  82
  83 redef class THex
  84     redef fun parser_index: Int
  85     do
  86         return 5
  87     end
  88
  89     init init_tk(loc: Location)
  90     do
  91                 _location = loc
  92     end
  93 end
  94
  95 redef class TColon
  96     redef fun parser_index: Int
  97     do
  98         return 6
  99     end
 100
 101     init init_tk(loc: Location)
 102     do
 103                 _location = loc
 104     end
 105 end
 106
 107 redef class TComma
 108     redef fun parser_index: Int
 109     do
 110         return 7
 111     end
 112
 113     init init_tk(loc: Location)
 114     do
 115                 _location = loc
 116     end
 117 end
 118
 119 redef class TComment
 120     redef fun parser_index: Int
 121     do
 122         return 8
 123     end
 124
 125     init init_tk(loc: Location)
 126     do
 127                 _location = loc
 128     end
 129 end
 130
 131 redef class TTkByte
 132     redef fun parser_index: Int
 133     do
 134         return 9
 135     end
 136
 137     init init_tk(loc: Location)
 138     do
 139                 _location = loc
 140     end
 141 end
 142
 143 redef class TTkWord
 144     redef fun parser_index: Int
 145     do
 146         return 10
 147     end
 148
 149     init init_tk(loc: Location)
 150     do
 151                 _location = loc
 152     end
 153 end
 154
 155 redef class TTkBlock
 156     redef fun parser_index: Int
 157     do
 158         return 11
 159     end
 160
 161     init init_tk(loc: Location)
 162     do
 163                 _location = loc
 164     end
 165 end
 166
 167 redef class TTkAscii
 168     redef fun parser_index: Int
 169     do
 170         return 12
 171     end
 172
 173     init init_tk(loc: Location)
 174     do
 175                 _location = loc
 176     end
 177 end
 178
 179 redef class TTkAddrss
 180     redef fun parser_index: Int
 181     do
 182         return 13
 183     end
 184
 185     init init_tk(loc: Location)
 186     do
 187                 _location = loc
 188     end
 189 end
 190
 191 redef class TTkEquate
 192     redef fun parser_index: Int
 193     do
 194         return 14
 195     end
 196
 197     init init_tk(loc: Location)
 198     do
 199                 _location = loc
 200     end
 201 end
 202
 203 redef class TTkBurn
 204     redef fun parser_index: Int
 205     do
 206         return 15
 207     end
 208
 209     init init_tk(loc: Location)
 210     do
 211                 _location = loc
 212     end
 213 end
 214
 215 redef class TEndBlock
 216     redef fun parser_index: Int
 217     do
 218         return 16
 219     end
 220
 221     init init_tk(loc: Location)
 222     do
 223                 _location = loc
 224     end
 225 end
 226
 227 redef class TId
 228     redef fun parser_index: Int
 229     do
 230         return 17
 231     end
 232
 233     init init_tk(loc: Location)
 234     do
 235                 _location = loc
 236     end
 237 end
 238
 239
 240 redef class EOF
 241     redef fun parser_index: Int
 242     do
 243         return 18
 244     end
 245
 246     init(loc: Location)
 247     do
 248         _text = ""
 249                 _location = loc
 250     end
 251 end
 252
 253 redef class AError
 254     var message: String
 255
 256     init init_error(message: String, loc: Location)
 257     do
 258                 init(loc)
 259                 self.message = message
 260     end
 261 end
 262
 263
 264 # The lexer extract NIT tokens from an input stream.
 265 # It is better user with the Parser
 266 class Lexer
 267         super TablesCapable
 268         # Last peeked token
 269         var _token: nullable Token
 270
 271         # Lexer current state
 272         var _state: Int = 0
 273
 274         # The source file
 275         var _file: SourceFile
 276
 277         # Current character in the stream
 278         var _stream_pos: Int = 0
 279
 280         # Current line number in the input stream
 281         var _line: Int = 0
 282
 283         # Current column in the input stream
 284         var _pos: Int = 0
 285
 286         # Was the last character a cariage-return?
 287         var _cr: Bool = false
 288
 289         # Constante state values
 290         private fun state_initial: Int do return 0 end
 291
 292         # Create a new lexer for a stream (and a name)
 293         init(file: SourceFile)
 294         do
 295                 _file = file
 296         end
 297
 298         # Give the next token (but do not consume it)
 299         fun peek: Token
 300         do
 301                 while _token == null do
 302                         _token = get_token
 303                 end
 304                 return _token.as(not null)
 305         end
 306
 307         # Give and consume the next token
 308         fun next: Token
 309         do
 310                 var result = _token
 311                 while result == null do
 312                         result = get_token
 313                 end
 314                 _token = null
 315                 return result
 316         end
 317
 318         # Get a token, or null if it is discarded
 319         private fun get_token: nullable Token
 320         do
 321                 var dfa_state = 0
 322
 323                 var sp = _stream_pos
 324                 var start_stream_pos = sp
 325                 var start_pos = _pos
 326                 var start_line = _line
 327                 var string = _file.string
 328                 var string_len = string.length
 329
 330                 var accept_state = -1
 331                 var accept_token = -1
 332                 var accept_length = -1
 333                 var accept_pos = -1
 334                 var accept_line = -1
 335
 336                 loop
 337                         if sp >= string_len then
 338                                 dfa_state = -1
 339                         else
 340                                 var c = string[sp].ascii
 341                                 sp += 1
 342
 343                                 var cr = _cr
 344                                 var line = _line
 345                                 var pos = _pos
 346                                 if c == 10 then
 347                                         if cr then
 348                                                 cr = false
 349                                                 _file.line_starts[line] = sp
 350                                         else
 351                                                 line = line + 1
 352                                                 pos = 0
 353                                                 _file.line_starts[line] = sp
 354                                         end
 355                                 else if c == 13 then
 356                                         line = line + 1
 357                                         pos = 0
 358                                         cr = true
 359                                         _file.line_starts[line] = sp
 360                                 else
 361                                         pos = pos + 1
 362                                         cr = false
 363                                 end
 364
 365                                 loop
 366                                         var old_state = dfa_state
 367                                         if dfa_state < -1 then
 368                                                 old_state = -2 - dfa_state
 369                                         end
 370
 371                                         dfa_state = -1
 372
 373                                         var low = 0
 374                                         var high = lexer_goto(old_state, 0) - 1
 375
 376                                         if high >= 0 then
 377                                                 while low <= high do
 378                                                         var middle = (low + high) / 2
 379                                                         var offset = middle * 3 + 1 # +1 because length is at 0
 380
 381                                                         if c < lexer_goto(old_state, offset) then
 382                                                                 high = middle - 1
 383                                                         else if c > lexer_goto(old_state, offset+1) then
 384                                                                 low = middle + 1
 385                                                         else
 386                                                                 dfa_state = lexer_goto(old_state, offset+2)
 387                                                                 break
 388                                                         end
 389                                                 end
 390                                         end
 391                                         if dfa_state > -2 then break
 392                                 end
 393
 394                                 _cr = cr
 395                                 _line = line
 396                                 _pos = pos
 397                         end
 398
 399                         if dfa_state >= 0 then
 400                                 var tok = lexer_accept(dfa_state)
 401                                 if tok != -1 then
 402                                         accept_state = dfa_state
 403                                         accept_token = tok
 404                                         accept_length = sp - start_stream_pos
 405                                         accept_pos = _pos
 406                                         accept_line = _line
 407                                 end
 408                         else
 409                                 if accept_state != -1 then
 410                                         var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
 411                                         _pos = accept_pos
 412                                         _line = accept_line
 413                                         _stream_pos = start_stream_pos + accept_length
 414                                         if accept_token == 0 then
 415                                                 return null
 416                                         end
 417                                         if accept_token == 1 then
 418                                                 return new TEol.init_tk(location)
 419                                         end
 420                                         if accept_token == 2 then
 421                                                 return new TNumber.init_tk(location)
 422                                         end
 423                                         if accept_token == 3 then
 424                                                 return new TFloat.init_tk(location)
 425                                         end
 426                                         if accept_token == 4 then
 427                                                 return new TChar.init_tk(location)
 428                                         end
 429                                         if accept_token == 5 then
 430                                                 return new TString.init_tk(location)
 431                                         end
 432                                         if accept_token == 6 then
 433                                                 return new THex.init_tk(location)
 434                                         end
 435                                         if accept_token == 7 then
 436                                                 return new TColon.init_tk(location)
 437                                         end
 438                                         if accept_token == 8 then
 439                                                 return new TComma.init_tk(location)
 440                                         end
 441                                         if accept_token == 9 then
 442                                                 return new TComment.init_tk(location)
 443                                         end
 444                                         if accept_token == 10 then
 445                                                 return new TTkByte.init_tk(location)
 446                                         end
 447                                         if accept_token == 11 then
 448                                                 return new TTkWord.init_tk(location)
 449                                         end
 450                                         if accept_token == 12 then
 451                                                 return new TTkBlock.init_tk(location)
 452                                         end
 453                                         if accept_token == 13 then
 454                                                 return new TTkAscii.init_tk(location)
 455                                         end
 456                                         if accept_token == 14 then
 457                                                 return new TTkAddrss.init_tk(location)
 458                                         end
 459                                         if accept_token == 15 then
 460                                                 return new TTkEquate.init_tk(location)
 461                                         end
 462                                         if accept_token == 16 then
 463                                                 return new TTkBurn.init_tk(location)
 464                                         end
 465                                         if accept_token == 17 then
 466                                                 return new TEndBlock.init_tk(location)
 467                                         end
 468                                         if accept_token == 18 then
 469                                                 return new TId.init_tk(location)
 470                                         end
 471                                 else
 472                                         _stream_pos = sp
 473                                         var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
 474                                         if sp > start_stream_pos then
 475                                                 var text = string.substring(start_stream_pos, sp-start_stream_pos)
 476                                                 var token = new AError.init_error("Syntax error: unknown token {text}.", location)
 477                                                 return token
 478                                         else
 479                                                 var token = new EOF(location)
 480                                                 return token
 481                                         end
 482                                 end
 483                         end
 484                 end
 485         end
 486 end
 487