Utils and tools related to parsers and AST

Introduced classes

class InjectedLexer

nitc :: InjectedLexer

A modified lexer that feed tokens before and after the real tokens.

Redefined classes

redef class ToolContext

nitc :: parser_util $ ToolContext

Global context for tools

All class definitions

class InjectedLexer

nitc $ InjectedLexer

A modified lexer that feed tokens before and after the real tokens.
redef class ToolContext

nitc :: parser_util $ ToolContext

Global context for tools
package_diagram nitc::parser_util parser_util nitc\>parser\> parser nitc::parser_util->nitc\>parser\> nitc::toolcontext toolcontext nitc::parser_util->nitc::toolcontext nitc nitc nitc\>parser\>->nitc ordered_tree ordered_tree nitc\>parser\>->ordered_tree console console nitc\>parser\>->console core core nitc\>parser\>->core opts opts nitc::toolcontext->opts nitc::location location nitc::toolcontext->nitc::location nitc::version version nitc::toolcontext->nitc::version template template nitc::toolcontext->template more_collections more_collections nitc::toolcontext->more_collections ...nitc ... ...nitc->nitc ...ordered_tree ... ...ordered_tree->ordered_tree ...console ... ...console->console ...core ... ...core->core ...opts ... ...opts->opts ...nitc::location ... ...nitc::location->nitc::location ...nitc::version ... ...nitc::version->nitc::version ...template ... ...template->template ...more_collections ... ...more_collections->more_collections nitc::platform platform nitc::platform->nitc::parser_util nitc::serialization_model_phase serialization_model_phase nitc::serialization_model_phase->nitc::parser_util nitc::deriving deriving nitc::deriving->nitc::parser_util nitc::parallelization_phase parallelization_phase nitc::parallelization_phase->nitc::parser_util nitc::i18n_phase i18n_phase nitc::i18n_phase->nitc::parser_util nitc::regex_phase regex_phase nitc::regex_phase->nitc::parser_util nitc::actors_injection_phase actors_injection_phase nitc::actors_injection_phase->nitc::parser_util nitc::app_annotations app_annotations nitc::app_annotations->nitc::parser_util nitc::testing_base testing_base nitc::testing_base->nitc::parser_util nitc::test_parser test_parser nitc::test_parser->nitc::parser_util nitc::c_compiler_options c_compiler_options nitc::c_compiler_options->nitc::platform nitc::abstract_compiler abstract_compiler nitc::abstract_compiler->nitc::platform nitc::c_compiler_options... ... nitc::c_compiler_options...->nitc::c_compiler_options nitc::abstract_compiler... ... nitc::abstract_compiler...->nitc::abstract_compiler nitc::frontend frontend nitc::frontend->nitc::serialization_model_phase nitc::frontend->nitc::deriving nitc::frontend->nitc::parallelization_phase nitc::frontend->nitc::i18n_phase nitc::frontend->nitc::regex_phase nitc::frontend->nitc::actors_injection_phase nitc::serialization_code_gen_phase serialization_code_gen_phase nitc::serialization_code_gen_phase->nitc::serialization_model_phase nitc::frontend... ... nitc::frontend...->nitc::frontend nitc::serialization_code_gen_phase... ... nitc::serialization_code_gen_phase...->nitc::serialization_code_gen_phase nitc::actors_generation_phase actors_generation_phase nitc::actors_generation_phase->nitc::actors_injection_phase nitc::actors_generation_phase... ... nitc::actors_generation_phase...->nitc::actors_generation_phase nitc::android_annotations android_annotations nitc::android_annotations->nitc::app_annotations nitc::ios ios nitc::ios->nitc::app_annotations nitc::android_annotations... ... nitc::android_annotations...->nitc::android_annotations nitc::ios... ... nitc::ios...->nitc::ios nitc::testing_doc testing_doc nitc::testing_doc->nitc::testing_base nitc::testing_suite testing_suite nitc::testing_suite->nitc::testing_base nitc::testing_gen testing_gen nitc::testing_gen->nitc::testing_base nitc::testing_doc... ... nitc::testing_doc...->nitc::testing_doc nitc::testing_suite... ... nitc::testing_suite...->nitc::testing_suite nitc::testing_gen... ... nitc::testing_gen...->nitc::testing_gen a_star-m a_star-m a_star-m->nitc::test_parser a_star-m... ... a_star-m...->a_star-m

Ancestors

module abstract_collection

core :: abstract_collection

Abstract collection classes and services.
module abstract_text

core :: abstract_text

Abstract class for manipulation of sequences of characters
module array

core :: array

This module introduces the standard array structure.
module bitset

core :: bitset

Services to handle BitSet
module bytes

core :: bytes

Services for byte streams and arrays
module caching

serialization :: caching

Services for caching serialization engines
module circular_array

core :: circular_array

Efficient data structure to access both end of the sequence.
module codec_base

core :: codec_base

Base for codecs to use with streams
module codecs

core :: codecs

Group module for all codec-related manipulations
module collection

core :: collection

This module define several collection classes.
module console

console :: console

Defines some ANSI Terminal Control Escape Sequences.
module core

core :: core

Standard classes and methods used by default by Nit programs and libraries.
module engine_tools

serialization :: engine_tools

Advanced services for serialization engines
module environ

core :: environ

Access to the environment variables of the process
module error

core :: error

Standard error-management infrastructure.
module exec

core :: exec

Invocation and management of operating system sub-processes.
module file

core :: file

File manipulations (create, read, write, etc.)
module fixed_ints

core :: fixed_ints

Basic integers of fixed-precision
module fixed_ints_text

core :: fixed_ints_text

Text services to complement fixed_ints
module flat

core :: flat

All the array-based text representations
module gc

core :: gc

Access to the Nit internal garbage collection mechanism
module hash_collection

core :: hash_collection

Introduce HashMap and HashSet.
module inspect

serialization :: inspect

Refine Serializable::inspect to show more useful information
module iso8859_1

core :: iso8859_1

Codec for ISO8859-1 I/O
module kernel

core :: kernel

Most basic classes and methods.
module lexer

nitc :: lexer

Lexer and its tokens.
module lexer_work

nitc :: lexer_work

Internal algorithm and data structures for the Nit lexer
module list

core :: list

This module handle double linked lists
module location

nitc :: location

Nit source-file and locations in source-file
module math

core :: math

Mathematical operations
module meta

meta :: meta

Simple user-defined meta-level to manipulate types of instances as object.
module more_collections

more_collections :: more_collections

Highly specific, but useful, collections-related classes.
module native

core :: native

Native structures for text and bytes
module numeric

core :: numeric

Advanced services for Numeric types
module opts

opts :: opts

Management of options on the command line
module ordered_tree

ordered_tree :: ordered_tree

Manipulation and presentation of ordered trees.
module parser_nodes

nitc :: parser_nodes

AST nodes of the Nit language
module parser_prod

nitc :: parser_prod

Production AST nodes full definition.
module parser_work

nitc :: parser_work

Internal algorithm and data structures for the Nit parser
module poset

poset :: poset

Pre order sets and partial order set (ie hierarchies)
module protocol

core :: protocol

module queue

core :: queue

Queuing data structures and wrappers
module range

core :: range

Module for range of discrete objects.
module re

core :: re

Regular expression support for all services based on Pattern
module ropes

core :: ropes

Tree-based representation of a String.
module serialization

serialization :: serialization

General serialization services
module serialization_core

serialization :: serialization_core

Abstract services to serialize Nit objects to different formats
module sorter

core :: sorter

This module contains classes used to compare things and sorts arrays.
module stream

core :: stream

Input and output streams of characters
module tables

nitc :: tables

Module that interfaces the parsing tables.
module template

template :: template

Basic template system
module text

core :: text

All the classes and methods related to the manipulation of text entities
module time

core :: time

Management of time and dates
module union_find

core :: union_find

union–find algorithm using an efficient disjoint-set data structure
module utf8

core :: utf8

Codec for UTF-8 I/O
module version

nitc :: version

This file was generated by git-gen-version.sh

Parents

module parser

nitc :: parser

Parser.
module toolcontext

nitc :: toolcontext

Common command-line tool infrastructure than handle options and error messages

Children

module actors_injection_phase

nitc :: actors_injection_phase

Injects model for the classes annotated with "is actor" so
module app_annotations

nitc :: app_annotations

Annotations to gather metadata on app.nit projects
module deriving

nitc :: deriving

Injection of automatic method definitions for standard methods, based on the attributes of the classes
module i18n_phase

nitc :: i18n_phase

Basic support of internationalization through the generation of id-to-string tables
module parallelization_phase

nitc :: parallelization_phase

Phase generating threads for functions annotated with threaded annotation
module platform

nitc :: platform

Platform system, used to customize the behavior of the compiler.
module regex_phase

nitc :: regex_phase

Check for error in regular expressions from string literals
module serialization_model_phase

nitc :: serialization_model_phase

Phase generating methods (model-only) to serialize Nit objects
module test_parser

nitc :: test_parser

Program used to test the NIT parser
module testing_base

nitc :: testing_base

Base options for testing tools.

Descendants

module a_star-m

a_star-m

module abstract_compiler

nitc :: abstract_compiler

Abstract compiler
module actors_generation_phase

nitc :: actors_generation_phase

Generate a support module for each module that contain a class annotated with is actor
module android

nitc :: android

Compile program for the Android platform
module android_annotations

nitc :: android_annotations

Additionnal annotations to gather metadata on Android projects
module api

nitc :: api

Components required to build a web server about the nit model.
module api_auth

nitc :: api_auth

module api_base

nitc :: api_base

Base classes used by nitweb.
module api_docdown

nitc :: api_docdown

Nitdoc specific Markdown format handling for Nitweb
module api_feedback

nitc :: api_feedback

Feedback related features
module api_light

nitc :: api_light

Highlight and collect messages from a piece of code
module api_model

nitc :: api_model

module c_compiler_options

nitc :: c_compiler_options

Offers the annotations cflags and ldflags to specify
module code_gen

nitc :: code_gen

Main frontend phases plus code generation phases
module commands_docdown

nitc :: commands_docdown

Doc down related queries
module compiler

nitc :: compiler

Compilation to C
module compiler_ffi

nitc :: compiler_ffi

Full FFI support for the compiler
module compiler_serialization

nitc :: compiler_serialization

Serialization support for the compiler
module dynamic_loading_ffi

nitc :: dynamic_loading_ffi

Execute FFI code by creating and loading shared libraries
module emscripten

nitc :: emscripten

Compile to JavaScript using the Emscripten SDK
module extra_java_files

nitc :: extra_java_files

Intro the annotation extra_java_files to compile extra java files
module ffi

nitc :: ffi

Full FFI support, independent of the compiler
module frontend

nitc :: frontend

Collect and orchestration of main frontend phases
module global_compiler

nitc :: global_compiler

Global compilation of a Nit program
module highlight

nitc :: highlight

Highlighting of Nit AST
module html_commands

nitc :: html_commands

Render commands results as HTML
module html_model

nitc :: html_model

Translate mentities to html blocks.
module htmlight

nitc :: htmlight

Highlighting of Nit AST with HTML
module interpreter

nitc :: interpreter

Interpretation of Nit programs
module ios

nitc :: ios

Compile programs for the iOS platform
module java

nitc :: java

FFI support for the Java language
module java_compiler

nitc :: java_compiler

Compile Nit code to Java code
module json_commands

nitc :: json_commands

Translate command results to json
module json_model

nitc :: json_model

Make model entities Serializable.
module light

nitc :: light

Light FFI support for the compiler
module light_only

nitc :: light_only

Compiler support for the light FFI only, detects unsupported usage of callbacks
module md_commands

nitc :: md_commands

Render commands results as Markdown
module memory_logger

nitc :: memory_logger

Extension to inject memory-tracing instrumentation in code generated by nitc.
module nit

nitc :: nit

A naive Nit interpreter
module nitc

nitc :: nitc

A Nit compiler
module nitcatalog

nitc :: nitcatalog

Basic catalog generator for Nit packages
module nitdoc

nitc :: nitdoc

Generator of static API documentation for the Nit language
module nith

nitc :: nith

A ligHt Nit compiler
module nitj

nitc :: nitj

Compile Nit into Java code runnable on the Java Virtual Machine.
module nitlight

nitc :: nitlight

Tool that produces highlighting for Nit programs
module nitmetrics

nitc :: nitmetrics

A program that collects various metrics on nit programs and libraries
module nitpackage

nitc :: nitpackage

Helpful features about packages
module nitpick

nitc :: nitpick

A program that collect potential style and code issues
module nitrestful

nitc :: nitrestful

Tool generating boilerplate code linking RESTful actions to Nit methods
module nitsaf

nitc :: nitsaf

Nit Static Analysis Framework client example.
module nitserial

nitc :: nitserial

Serialization support compiler, a tool to support deserialization of live generic types
module nitsmells

nitc :: nitsmells

module nituml

nitc :: nituml

UML generator in dot format.
module nitunit

nitc :: nitunit

Testing tool.
module nitvm

nitc :: nitvm

The Nit virtual machine launcher
module nitweb

nitc :: nitweb

Runs a webserver based on nitcorn that render things from model.
module nitx

nitc :: nitx

nitx, a command tool that displays useful data about Nit code
module on_demand_compiler

nitc :: on_demand_compiler

Compiles extern code within a module to a static library, as needed
module separate_compiler

nitc :: separate_compiler

Separate compilation of a Nit program
module separate_erasure_compiler

nitc :: separate_erasure_compiler

Separate compilation of a Nit program with generic type erasure
module serialization_code_gen_phase

nitc :: serialization_code_gen_phase

Phase generating methods (code) to serialize Nit objects
module static

nitc :: static

Nitdoc generation framework
module static_base

nitc :: static_base

Base entities shared by all the nitdoc code
module static_cards

nitc :: static_cards

Cards templates for the static documentation
module static_html

nitc :: static_html

Render documentation pages as HTML
module static_index

nitc :: static_index

Manage indexing of Nit model for Nitdoc QuickSearch.
module static_structure

nitc :: static_structure

Composes the pages of the static documentation
module term

nitc :: term

module test_astbuilder

nitc :: test_astbuilder

Program used to test the clone method of the astbuilder tool
module test_highlight

nitc :: test_highlight

Program used to test the Nit highlighter
module test_model_visitor

nitc :: test_model_visitor

Example of model_visitor
module test_neo

nitc :: test_neo

Test for neo model saving and loading.
module test_test_phase

nitc :: test_test_phase

Example of simple module that aims to do some specific work on nit programs.
module testing

nitc :: testing

Test unit generation and execution for Nit.
module testing_doc

nitc :: testing_doc

Testing from code comments.
module testing_gen

nitc :: testing_gen

Test Suites generation.
module testing_suite

nitc :: testing_suite

Testing from external files.
module xcode_templates

nitc :: xcode_templates

Templates and other services to create XCode projects
# Utils and tools related to parsers and AST
module parser_util

intrude import parser
import toolcontext

redef class ToolContext
	# Parse a full module given as a string
	# Fatal error if the `string` is not a syntactically correct module
	fun parse_module(string: String): AModule
	do
		var source = new SourceFile.from_string("", string)
		var lexer = new Lexer(source)
		var parser = new Parser(lexer)
		var tree = parser.parse

		var eof = tree.n_eof
		if eof isa AError then
			self.fatal_error(null, "Fatal Error: {eof.message}.")
			abort
		end
		return tree.n_base.as(not null)
	end

	# Parse a full classdef given as a string
	# Fatal error if the `string` is not a syntactically correct class definition
	fun parse_classdef(string: String): AClassdef
	do
		var nmodule = parse_module(string)
		var nclassdefs = nmodule.n_classdefs
		if nclassdefs.length != 1 then
			self.fatal_error(null, "Fatal Error: not a classdef.")
			abort
		end
		return nclassdefs.first
	end

	# Parse a full propdef given as a string
	# Fatal error if the `string` is not a syntactically correct property definition
	fun parse_propdef(string: String): APropdef
	do
		var mod_string = "class Dummy\n{string}\nend"
		var nclassdef = parse_classdef(mod_string)
		var npropdefs = nclassdef.n_propdefs
		if npropdefs.length != 1 then
			self.fatal_error(null, "Fatal Error: not a propdef.")
			abort
		end
		return npropdefs.first
	end

	# Parse a full statement block given as a string
	# Fatal error if the `string` is not a syntactically correct statement block
	fun parse_stmts(string: String): AExpr
	do
		var mod_string = "do\n{string}\nend"
		var nmodule = parse_module(mod_string)
		var nblock = nmodule.n_classdefs.first.n_propdefs.first.as(AMethPropdef).n_block.as(ABlockExpr).n_expr.first.as(ADoExpr).n_block.as(not null)
		return nblock
	end

	# Parse a full expression given as a string
	# Fatal error if the `string` is not a syntactically correct expression
	fun parse_expr(string: String): AExpr
	do
		var mod_string = "var dummy = \n{string}"
		var nmodule = parse_module(mod_string)
		var nexpr = nmodule.n_classdefs.first.n_propdefs.first.as(AMethPropdef).n_block.as(ABlockExpr).n_expr.first.as(AVardeclExpr).n_expr.as(not null)
		return nexpr
	end

	# Parse a super class declaration
	# Fatal error if the `string` is not a syntactically correct super class declaration
	fun parse_superclass(string: String): APropdef
	do
		var mod_string = "class Dummy\nsuper {string}\nend"
		var nclassdef = parse_classdef(mod_string).as(AStdClassdef)
		var nsuperclasses = nclassdef.n_propdefs
		if nsuperclasses.length != 1 then
			self.fatal_error(null, "Fatal Error: not a super class declaration.")
			abort
		end
		return nsuperclasses.first
	end

	# Try to parse the `string` as something
	#
	# Returns the first possible syntacticaly correct type among:
	#
	# - a type `AType`
	# - a single `Token`
	# - an expression `AExpr`
	# - a block of statements `ABlockExpr`
	# - a full module `AModule`
	# - a `AError` if nothing else matches
	#
	#     var tc = new ToolContext
	#     assert tc.parse_something("foo") isa TId
	#     assert tc.parse_something("foo[bar]") isa AExpr
	#     assert tc.parse_something("Foo[Bar]") isa AType
	#     assert tc.parse_something("foo\nbar") isa ABlockExpr
	#     assert tc.parse_something("fun foo do bar\nfoo") isa AModule
	#     assert tc.parse_something("fun fun") isa AParserError
	#     assert tc.parse_something("?%^&") isa ALexerError
	fun parse_something(string: String): ANode
	do
		var source = new SourceFile.from_string("", string)
		var error
		var tree
		var eof
		var lexer

		lexer = new InjectedLexer(source)
		lexer.injected_before.add new TKwvar
		lexer.injected_before.add new TId
		lexer.injected_before.add new TColumn
		lexer.injected_before.add new TClassid
		lexer.injected_before.add new TObra
		lexer.injected_after.add new TCbra
		tree = (new Parser(lexer)).parse
		eof = tree.n_eof
		if not eof isa AError then
			var ntype = tree.n_base.n_classdefs.first.n_propdefs.first.as(AMethPropdef).n_block.as(ABlockExpr).n_expr.first.as(AVardeclExpr).n_type.n_types.first
			ntype.parent = null
			return ntype
		end
		error = eof

		lexer = new Lexer(source)
		var first = lexer.next
		if first isa EOF then return first
		var second = lexer.next
		if second isa EOF and not second isa AError then
			first.parent = null
			return first
		end

		lexer = new InjectedLexer(source)
		lexer.injected_before.add new TKwvar
		lexer.injected_before.add new TId
		lexer.injected_before.add new TAssign
		lexer.injected_before.add new TOpar
		lexer.injected_after.add new TCpar
		tree = (new Parser(lexer)).parse
		eof = tree.n_eof
		if not eof isa AError then
			var nexpr = tree.n_base.n_classdefs.first.n_propdefs.first.as(AMethPropdef).n_block.as(ABlockExpr).n_expr.first.as(AVardeclExpr).n_expr.as(AParExpr).n_expr
			nexpr.parent = null
			return nexpr
		end
		if eof.location > error.location then error = eof

		lexer = new InjectedLexer(source)
		lexer.injected_before.add new TKwdo
		lexer.injected_before.add new TEol
		lexer.injected_after.add new TEol
		lexer.injected_after.add new TKwend
		tree = (new Parser(lexer)).parse
		eof = tree.n_eof
		if not eof isa AError then
			var nblock = tree.n_base.n_classdefs.first.n_propdefs.first.as(AMethPropdef).n_block.as(ABlockExpr).n_expr.first.as(ADoExpr).n_block.as(ABlockExpr)
			nblock.n_kwend = null # drop injected token
			nblock.parent = null
			return nblock
		end
		if eof.location > error.location then error = eof

		lexer = new Lexer(source)
		tree = (new Parser(lexer)).parse
		eof = tree.n_eof
		if not eof isa AError then
			return tree.n_base.as(not null)
		end
		if eof.location > error.location then error = eof

		return error
	end

	# Parse the input of the user as something
	fun interactive_parse(prompt: String): ANode
	do
		var oldtext = ""

		loop
			printn prompt
			printn " "
			var s = sys.stdin.read_line
			if s == "" then continue
			if s.chars.first == ':' then
				var res = new TString
				res.text = s
				return res
			end

			var text = oldtext + s + "\n"
			oldtext = ""
			var n = parse_something(text)

			if n isa AParserError and n.token isa EOF then
				# Unexpected end of file, thus continuing
				if oldtext == "" then prompt = "." * prompt.length
				oldtext = text
				continue
			end

			return n
		end
	end
end

# A modified lexer that feed tokens before and after the real tokens.
class InjectedLexer
	super Lexer

	# The tokens to use before the real tokens (in order).
	var injected_before = new List[Token]

	# The tokens to use after the real tokens (in order).
	# The real EOF token is produced after these tokens.
	var injected_after = new List[Token]
	private var is_finished = false

	redef fun get_token
	do
		if not injected_before.is_empty then
			var tok = injected_before.shift
			return tok
		end
		if not is_finished then
			var next = super
			if not next isa EOF then return next
			injected_after.push(next)
			is_finished = true
		end

		var tok = injected_after.shift
		return tok
	end
end
src/parser_util.nit:15,1--253,3