contrib/pep8analysis/src/parser/lexer.nit

   1 # Lexer and its tokens.
   2 # This file was generated by SableCC (http://www.sablecc.org/).
   3 module lexer is generated, no_warning("missing-doc", "old-init")
   4
   5 intrude import parser_nodes
   6 private import tables
   7
   8 redef class Token
   9     var text_cache: nullable String
  10
  11     redef fun text
  12     do
  13         var res = _text_cache
  14         if res != null then return res
  15         res = location.text
  16         _text_cache = res
  17         return res
  18     end
  19
  20     fun parser_index: Int is abstract
  21 end
  22
  23 redef class TEol
  24     redef fun parser_index: Int
  25     do
  26         return 0
  27     end
  28
  29     init init_tk(loc: Location)
  30     do
  31                 _location = loc
  32     end
  33 end
  34
  35 redef class TNumber
  36     redef fun parser_index: Int
  37     do
  38         return 1
  39     end
  40
  41     init init_tk(loc: Location)
  42     do
  43                 _location = loc
  44     end
  45 end
  46
  47 redef class TFloat
  48     redef fun parser_index: Int
  49     do
  50         return 2
  51     end
  52
  53     init init_tk(loc: Location)
  54     do
  55                 _location = loc
  56     end
  57 end
  58
  59 redef class TChar
  60     redef fun parser_index: Int
  61     do
  62         return 3
  63     end
  64
  65     init init_tk(loc: Location)
  66     do
  67                 _location = loc
  68     end
  69 end
  70
  71 redef class TString
  72     redef fun parser_index: Int
  73     do
  74         return 4
  75     end
  76
  77     init init_tk(loc: Location)
  78     do
  79                 _location = loc
  80     end
  81 end
  82
  83 redef class THex
  84     redef fun parser_index: Int
  85     do
  86         return 5
  87     end
  88
  89     init init_tk(loc: Location)
  90     do
  91                 _location = loc
  92     end
  93 end
  94
  95 redef class TColon
  96     redef fun parser_index: Int
  97     do
  98         return 6
  99     end
 100
 101     init init_tk(loc: Location)
 102     do
 103                 _location = loc
 104     end
 105 end
 106
 107 redef class TComma
 108     redef fun parser_index: Int
 109     do
 110         return 7
 111     end
 112
 113     init init_tk(loc: Location)
 114     do
 115                 _location = loc
 116     end
 117 end
 118
 119 redef class TComment
 120     redef fun parser_index: Int
 121     do
 122         return 8
 123     end
 124
 125     init init_tk(loc: Location)
 126     do
 127                 _location = loc
 128     end
 129 end
 130
 131 redef class TTkByte
 132     redef fun parser_index: Int
 133     do
 134         return 9
 135     end
 136
 137     init init_tk(loc: Location)
 138     do
 139                 _location = loc
 140     end
 141 end
 142
 143 redef class TTkWord
 144     redef fun parser_index: Int
 145     do
 146         return 10
 147     end
 148
 149     init init_tk(loc: Location)
 150     do
 151                 _location = loc
 152     end
 153 end
 154
 155 redef class TTkBlock
 156     redef fun parser_index: Int
 157     do
 158         return 11
 159     end
 160
 161     init init_tk(loc: Location)
 162     do
 163                 _location = loc
 164     end
 165 end
 166
 167 redef class TTkAscii
 168     redef fun parser_index: Int
 169     do
 170         return 12
 171     end
 172
 173     init init_tk(loc: Location)
 174     do
 175                 _location = loc
 176     end
 177 end
 178
 179 redef class TTkAddrss
 180     redef fun parser_index: Int
 181     do
 182         return 13
 183     end
 184
 185     init init_tk(loc: Location)
 186     do
 187                 _location = loc
 188     end
 189 end
 190
 191 redef class TTkEquate
 192     redef fun parser_index: Int
 193     do
 194         return 14
 195     end
 196
 197     init init_tk(loc: Location)
 198     do
 199                 _location = loc
 200     end
 201 end
 202
 203 redef class TTkBurn
 204     redef fun parser_index: Int
 205     do
 206         return 15
 207     end
 208
 209     init init_tk(loc: Location)
 210     do
 211                 _location = loc
 212     end
 213 end
 214
 215 redef class TEndBlock
 216     redef fun parser_index: Int
 217     do
 218         return 16
 219     end
 220
 221     init init_tk(loc: Location)
 222     do
 223                 _location = loc
 224     end
 225 end
 226
 227 redef class TId
 228     redef fun parser_index: Int
 229     do
 230         return 17
 231     end
 232
 233     init init_tk(loc: Location)
 234     do
 235                 _location = loc
 236     end
 237 end
 238
 239
 240 redef class EOF
 241     redef fun parser_index: Int
 242     do
 243         return 18
 244     end
 245
 246     init(loc: Location)
 247     do
 248         _text_cache = ""
 249                 _location = loc
 250     end
 251 end
 252
 253 redef class AError
 254     var message: String
 255
 256     init init_error(message: String, loc: Location)
 257     do
 258                 init(loc)
 259                 self.message = message
 260     end
 261 end
 262
 263
 264 # The lexer extract NIT tokens from an input stream.
 265 # It is better user with the Parser
 266 class Lexer
 267         super TablesCapable
 268         # Last peeked token
 269         var token: nullable Token
 270
 271         # Lexer current state
 272         var state: Int = 0
 273
 274         # The source file
 275         var file: SourceFile
 276
 277         # Current character in the stream
 278         var stream_pos: Int = 0
 279
 280         # Current line number in the input stream
 281         var line: Int = 0
 282
 283         # Current column in the input stream
 284         var pos: Int = 0
 285
 286         # Was the last character a cariage-return?
 287         var cr: Bool = false
 288
 289         # Constante state values
 290         private fun state_initial: Int do return 0 end
 291
 292         # Create a new lexer for a stream (and a name)
 293         init(file: SourceFile)
 294         do
 295                 _file = file
 296         end
 297
 298         # Give the next token (but do not consume it)
 299         fun peek: Token
 300         do
 301                 while _token == null do
 302                         _token = get_token
 303                 end
 304                 return _token.as(not null)
 305         end
 306
 307         # Give and consume the next token
 308         fun next: Token
 309         do
 310                 var result = _token
 311                 while result == null do
 312                         result = get_token
 313                 end
 314                 _token = null
 315                 return result
 316         end
 317
 318         # Get a token, or null if it is discarded
 319         private fun get_token: nullable Token
 320         do
 321                 var dfa_state = 0
 322
 323                 var sp = _stream_pos
 324                 var start_stream_pos = sp
 325                 var start_pos = _pos
 326                 var start_line = _line
 327                 var string = _file.string
 328                 var string_len = string.length
 329
 330                 var accept_state = -1
 331                 var accept_token = -1
 332                 var accept_length = -1
 333                 var accept_pos = -1
 334                 var accept_line = -1
 335
 336                 loop
 337                         if sp >= string_len then
 338                                 dfa_state = -1
 339                         else
 340                                 var c = string[sp].code_point
 341                                 if c >= 255 then c = 255
 342                                 sp += 1
 343
 344                                 var cr = _cr
 345                                 var line = _line
 346                                 var pos = _pos
 347                                 if c == 10 then
 348                                         if cr then
 349                                                 cr = false
 350                                                 _file.line_starts[line] = sp
 351                                         else
 352                                                 line = line + 1
 353                                                 pos = 0
 354                                                 _file.line_starts[line] = sp
 355                                         end
 356                                 else if c == 13 then
 357                                         line = line + 1
 358                                         pos = 0
 359                                         cr = true
 360                                         _file.line_starts[line] = sp
 361                                 else
 362                                         pos = pos + 1
 363                                         cr = false
 364                                 end
 365
 366                                 loop
 367                                         var old_state = dfa_state
 368                                         if dfa_state < -1 then
 369                                                 old_state = -2 - dfa_state
 370                                         end
 371
 372                                         dfa_state = -1
 373
 374                                         var low = 0
 375                                         var high = lexer_goto(old_state, 0) - 1
 376
 377                                         if high >= 0 then
 378                                                 while low <= high do
 379                                                         var middle = (low + high) / 2
 380                                                         var offset = middle * 3 + 1 # +1 because length is at 0
 381
 382                                                         if c < lexer_goto(old_state, offset) then
 383                                                                 high = middle - 1
 384                                                         else if c > lexer_goto(old_state, offset+1) then
 385                                                                 low = middle + 1
 386                                                         else
 387                                                                 dfa_state = lexer_goto(old_state, offset+2)
 388                                                                 break
 389                                                         end
 390                                                 end
 391                                         end
 392                                         if dfa_state > -2 then break
 393                                 end
 394
 395                                 _cr = cr
 396                                 _line = line
 397                                 _pos = pos
 398                         end
 399
 400                         if dfa_state >= 0 then
 401                                 var tok = lexer_accept(dfa_state)
 402                                 if tok != -1 then
 403                                         accept_state = dfa_state
 404                                         accept_token = tok
 405                                         accept_length = sp - start_stream_pos
 406                                         accept_pos = _pos
 407                                         accept_line = _line
 408                                 end
 409                         else
 410                                 if accept_state != -1 then
 411                                         var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
 412                                         _pos = accept_pos
 413                                         _line = accept_line
 414                                         _stream_pos = start_stream_pos + accept_length
 415                                         if accept_token == 0 then
 416                                                 return null
 417                                         end
 418                                         if accept_token == 1 then
 419                                                 return new TEol.init_tk(location)
 420                                         end
 421                                         if accept_token == 2 then
 422                                                 return new TNumber.init_tk(location)
 423                                         end
 424                                         if accept_token == 3 then
 425                                                 return new TFloat.init_tk(location)
 426                                         end
 427                                         if accept_token == 4 then
 428                                                 return new TChar.init_tk(location)
 429                                         end
 430                                         if accept_token == 5 then
 431                                                 return new TString.init_tk(location)
 432                                         end
 433                                         if accept_token == 6 then
 434                                                 return new THex.init_tk(location)
 435                                         end
 436                                         if accept_token == 7 then
 437                                                 return new TColon.init_tk(location)
 438                                         end
 439                                         if accept_token == 8 then
 440                                                 return new TComma.init_tk(location)
 441                                         end
 442                                         if accept_token == 9 then
 443                                                 return new TComment.init_tk(location)
 444                                         end
 445                                         if accept_token == 10 then
 446                                                 return new TTkByte.init_tk(location)
 447                                         end
 448                                         if accept_token == 11 then
 449                                                 return new TTkWord.init_tk(location)
 450                                         end
 451                                         if accept_token == 12 then
 452                                                 return new TTkBlock.init_tk(location)
 453                                         end
 454                                         if accept_token == 13 then
 455                                                 return new TTkAscii.init_tk(location)
 456                                         end
 457                                         if accept_token == 14 then
 458                                                 return new TTkAddrss.init_tk(location)
 459                                         end
 460                                         if accept_token == 15 then
 461                                                 return new TTkEquate.init_tk(location)
 462                                         end
 463                                         if accept_token == 16 then
 464                                                 return new TTkBurn.init_tk(location)
 465                                         end
 466                                         if accept_token == 17 then
 467                                                 return new TEndBlock.init_tk(location)
 468                                         end
 469                                         if accept_token == 18 then
 470                                                 return new TId.init_tk(location)
 471                                         end
 472                                 else
 473                                         _stream_pos = sp
 474                                         var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
 475                                         if sp > start_stream_pos then
 476                                                 var text = string.substring(start_stream_pos, sp-start_stream_pos)
 477                                                 var token = new AError.init_error("Syntax error: unknown token {text}.", location)
 478                                                 return token
 479                                         else
 480                                                 var token = new EOF(location)
 481                                                 return token
 482                                         end
 483                                 end
 484                         end
 485                 end
 486         end
 487 end
 488