Basic support of internationalization through the generation of id-to-string tables

Introduced classes

private class I18NPhase

nitc :: I18NPhase

class POFile

nitc :: POFile

A GNU gettext .po/.pot file
class PObject

nitc :: PObject

.po file entry
private class StringFinder

nitc :: StringFinder

Redefined classes

redef abstract class ANode

nitc :: i18n_phase $ ANode

Root of the AST class-hierarchy
redef class AStringExpr

nitc :: i18n_phase $ AStringExpr

A simple string. eg. "abc"
redef class ASuperstringExpr

nitc :: i18n_phase $ ASuperstringExpr

A superstring literal. eg "a{x}b{y}c"
redef abstract class Text

nitc :: i18n_phase $ Text

High-level abstraction for all text representations
redef class ToolContext

nitc :: i18n_phase $ ToolContext

Global context for tools

All class definitions

redef abstract class ANode

nitc :: i18n_phase $ ANode

Root of the AST class-hierarchy
redef class AStringExpr

nitc :: i18n_phase $ AStringExpr

A simple string. eg. "abc"
redef class ASuperstringExpr

nitc :: i18n_phase $ ASuperstringExpr

A superstring literal. eg "a{x}b{y}c"
private class I18NPhase

nitc $ I18NPhase

class POFile

nitc $ POFile

A GNU gettext .po/.pot file
class PObject

nitc $ PObject

.po file entry
private class StringFinder

nitc $ StringFinder

redef abstract class Text

nitc :: i18n_phase $ Text

High-level abstraction for all text representations
redef class ToolContext

nitc :: i18n_phase $ ToolContext

Global context for tools
package_diagram nitc::i18n_phase i18n_phase nitc::parser_util parser_util nitc::i18n_phase->nitc::parser_util nitc::astbuilder astbuilder nitc::i18n_phase->nitc::astbuilder nitc\>parser\> parser nitc::parser_util->nitc\>parser\> nitc::toolcontext toolcontext nitc::parser_util->nitc::toolcontext nitc::typing typing nitc::astbuilder->nitc::typing ...nitc\>parser\> ... ...nitc\>parser\>->nitc\>parser\> ...nitc::toolcontext ... ...nitc::toolcontext->nitc::toolcontext ...nitc::typing ... ...nitc::typing->nitc::typing nitc::frontend frontend nitc::frontend->nitc::i18n_phase nitc::frontend... ... nitc::frontend...->nitc::frontend

Ancestors

module abstract_collection

core :: abstract_collection

Abstract collection classes and services.
module abstract_text

core :: abstract_text

Abstract class for manipulation of sequences of characters
module annotation

nitc :: annotation

Management and utilities on annotations
module array

core :: array

This module introduces the standard array structure.
module bitset

core :: bitset

Services to handle BitSet
module bytes

core :: bytes

Services for byte streams and arrays
module caching

serialization :: caching

Services for caching serialization engines
module circular_array

core :: circular_array

Efficient data structure to access both end of the sequence.
module codec_base

core :: codec_base

Base for codecs to use with streams
module codecs

core :: codecs

Group module for all codec-related manipulations
module collection

core :: collection

This module define several collection classes.
module console

console :: console

Defines some ANSI Terminal Control Escape Sequences.
module core

core :: core

Standard classes and methods used by default by Nit programs and libraries.
module digraph

graph :: digraph

Implementation of directed graphs, also called digraphs.
module engine_tools

serialization :: engine_tools

Advanced services for serialization engines
module environ

core :: environ

Access to the environment variables of the process
module error

core :: error

Standard error-management infrastructure.
module exec

core :: exec

Invocation and management of operating system sub-processes.
module file

core :: file

File manipulations (create, read, write, etc.)
module fixed_ints

core :: fixed_ints

Basic integers of fixed-precision
module fixed_ints_text

core :: fixed_ints_text

Text services to complement fixed_ints
module flat

core :: flat

All the array-based text representations
module flow

nitc :: flow

Intraprocedural static flow.
module gc

core :: gc

Access to the Nit internal garbage collection mechanism
module hash_collection

core :: hash_collection

Introduce HashMap and HashSet.
module ini

ini :: ini

Read and write INI configuration files
module inspect

serialization :: inspect

Refine Serializable::inspect to show more useful information
module iso8859_1

core :: iso8859_1

Codec for ISO8859-1 I/O
module kernel

core :: kernel

Most basic classes and methods.
module lexer

nitc :: lexer

Lexer and its tokens.
module lexer_work

nitc :: lexer_work

Internal algorithm and data structures for the Nit lexer
module list

core :: list

This module handle double linked lists
module literal

nitc :: literal

Parsing of literal values in the abstract syntax tree.
module loader

nitc :: loader

Loading of Nit source files
module local_var_init

nitc :: local_var_init

Verify that local variables are initialized before their usage
module location

nitc :: location

Nit source-file and locations in source-file
module math

core :: math

Mathematical operations
module mdoc

nitc :: mdoc

Documentation of model entities
module meta

meta :: meta

Simple user-defined meta-level to manipulate types of instances as object.
module mmodule

nitc :: mmodule

modules and module hierarchies in the metamodel
module mmodule_data

nitc :: mmodule_data

Define and retrieve data in modules
module model

nitc :: model

Classes, types and properties
module model_base

nitc :: model_base

The abstract concept of model and related common things
module modelbuilder_base

nitc :: modelbuilder_base

Load nit source files and build the associated model
module modelize

nitc :: modelize

Create a model from nit source files
module modelize_class

nitc :: modelize_class

Analysis and verification of class definitions to instantiate model element
module modelize_property

nitc :: modelize_property

Analysis and verification of property definitions to instantiate model element
module more_collections

more_collections :: more_collections

Highly specific, but useful, collections-related classes.
module mpackage

nitc :: mpackage

Modelisation of a Nit package
module native

core :: native

Native structures for text and bytes
module nitpm_shared

nitc :: nitpm_shared

Services related to the Nit package manager
module numeric

core :: numeric

Advanced services for Numeric types
module opts

opts :: opts

Management of options on the command line
module ordered_tree

ordered_tree :: ordered_tree

Manipulation and presentation of ordered trees.
module parser

nitc :: parser

Parser.
module parser_nodes

nitc :: parser_nodes

AST nodes of the Nit language
module parser_prod

nitc :: parser_prod

Production AST nodes full definition.
module parser_work

nitc :: parser_work

Internal algorithm and data structures for the Nit parser
module phase

nitc :: phase

Phases of the processing of nit programs
module poset

poset :: poset

Pre order sets and partial order set (ie hierarchies)
module protocol

core :: protocol

module queue

core :: queue

Queuing data structures and wrappers
module range

core :: range

Module for range of discrete objects.
module re

core :: re

Regular expression support for all services based on Pattern
module ropes

core :: ropes

Tree-based representation of a String.
module scope

nitc :: scope

Identification and scoping of local variables and labels.
module serialization

serialization :: serialization

General serialization services
module serialization_core

serialization :: serialization_core

Abstract services to serialize Nit objects to different formats
module sorter

core :: sorter

This module contains classes used to compare things and sorts arrays.
module stream

core :: stream

Input and output streams of characters
module tables

nitc :: tables

Module that interfaces the parsing tables.
module template

template :: template

Basic template system
module text

core :: text

All the classes and methods related to the manipulation of text entities
module time

core :: time

Management of time and dates
module toolcontext

nitc :: toolcontext

Common command-line tool infrastructure than handle options and error messages
module typing

nitc :: typing

Intraprocedural resolution of static types and OO-services
module union_find

core :: union_find

union–find algorithm using an efficient disjoint-set data structure
module utf8

core :: utf8

Codec for UTF-8 I/O
module version

nitc :: version

This file was generated by git-gen-version.sh

Parents

module astbuilder

nitc :: astbuilder

Instantiation and transformation of semantic nodes in the AST of expressions and statements
module parser_util

nitc :: parser_util

Utils and tools related to parsers and AST

Children

module frontend

nitc :: frontend

Collect and orchestration of main frontend phases

Descendants

module a_star-m

a_star-m

module api

nitc :: api

Components required to build a web server about the nit model.
module api_auth

nitc :: api_auth

module api_base

nitc :: api_base

Base classes used by nitweb.
module api_docdown

nitc :: api_docdown

Nitdoc specific Markdown format handling for Nitweb
module api_feedback

nitc :: api_feedback

Feedback related features
module api_light

nitc :: api_light

Highlight and collect messages from a piece of code
module api_model

nitc :: api_model

module code_gen

nitc :: code_gen

Main frontend phases plus code generation phases
module commands_docdown

nitc :: commands_docdown

Doc down related queries
module highlight

nitc :: highlight

Highlighting of Nit AST
module html_commands

nitc :: html_commands

Render commands results as HTML
module html_model

nitc :: html_model

Translate mentities to html blocks.
module htmlight

nitc :: htmlight

Highlighting of Nit AST with HTML
module java_compiler

nitc :: java_compiler

Compile Nit code to Java code
module json_commands

nitc :: json_commands

Translate command results to json
module json_model

nitc :: json_model

Make model entities Serializable.
module md_commands

nitc :: md_commands

Render commands results as Markdown
module nit

nitc :: nit

A naive Nit interpreter
module nitc

nitc :: nitc

A Nit compiler
module nitcatalog

nitc :: nitcatalog

Basic catalog generator for Nit packages
module nitdoc

nitc :: nitdoc

Generator of static API documentation for the Nit language
module nitj

nitc :: nitj

Compile Nit into Java code runnable on the Java Virtual Machine.
module nitlight

nitc :: nitlight

Tool that produces highlighting for Nit programs
module nitmetrics

nitc :: nitmetrics

A program that collects various metrics on nit programs and libraries
module nitpackage

nitc :: nitpackage

Helpful features about packages
module nitpick

nitc :: nitpick

A program that collect potential style and code issues
module nitrestful

nitc :: nitrestful

Tool generating boilerplate code linking RESTful actions to Nit methods
module nitsaf

nitc :: nitsaf

Nit Static Analysis Framework client example.
module nitserial

nitc :: nitserial

Serialization support compiler, a tool to support deserialization of live generic types
module nitsmells

nitc :: nitsmells

module nituml

nitc :: nituml

UML generator in dot format.
module nitunit

nitc :: nitunit

Testing tool.
module nitvm

nitc :: nitvm

The Nit virtual machine launcher
module nitweb

nitc :: nitweb

Runs a webserver based on nitcorn that render things from model.
module nitx

nitc :: nitx

nitx, a command tool that displays useful data about Nit code
module static

nitc :: static

Nitdoc generation framework
module static_base

nitc :: static_base

Base entities shared by all the nitdoc code
module static_cards

nitc :: static_cards

Cards templates for the static documentation
module static_html

nitc :: static_html

Render documentation pages as HTML
module static_index

nitc :: static_index

Manage indexing of Nit model for Nitdoc QuickSearch.
module static_structure

nitc :: static_structure

Composes the pages of the static documentation
module term

nitc :: term

module test_astbuilder

nitc :: test_astbuilder

Program used to test the clone method of the astbuilder tool
module test_highlight

nitc :: test_highlight

Program used to test the Nit highlighter
module test_model_visitor

nitc :: test_model_visitor

Example of model_visitor
module test_neo

nitc :: test_neo

Test for neo model saving and loading.
module test_test_phase

nitc :: test_test_phase

Example of simple module that aims to do some specific work on nit programs.
# Basic support of internationalization through the generation of id-to-string tables
module i18n_phase

intrude import literal
private import annotation
private import parser_util
import astbuilder

redef class ToolContext
	# Main phase of `language`
	var localize_phase: Phase = new I18NPhase(self, [literal_phase])
end

private class I18NPhase
	super Phase

	redef fun process_annotated_node(nmodule, nat) do
		if not nat.name == "i18n" then return

		if not nmodule isa AModuledecl then
			toolcontext.error(nmodule.location, "Error: The localized language can only be used on module declarations.")
			return
		end

		var domain = nmodule.n_name.n_id.text

		var lang: nullable String = null
		if nat.n_args.length > 0 then
			lang = nat.arg_as_string(toolcontext.modelbuilder)
			if lang == null then return
		end

		var module_dir = nmodule.location.file.filename.dirname.realpath
		var locale_dir = module_dir / "languages"

		if not locale_dir.file_exists then locale_dir.mkdir

		var amodule = nmodule.parent.as(AModule)

		var vi = new StringFinder(domain, locale_dir, toolcontext, amodule)
		vi.enter_visit(amodule)

		var module_name = nmodule.location.file.filename.basename(".nit")

		var pot_path = locale_dir / module_name
		var arr = vi.strings.values.to_a
		var po = new POFile(arr)
		po.write_template(pot_path)

		if lang != null then
			for i in po.strings do
				i.msgstr = i.msgid
			end

			var lang_dir = locale_dir / lang
			if not lang_dir.file_exists then lang_dir.mkdir

			var messages_dir = lang_dir / "LC_MESSAGES"
			if not messages_dir.file_exists then messages_dir.mkdir

			po.write_to_file(messages_dir / module_name)
		end

		var lit = new LiteralVisitor(toolcontext)
		lit.enter_visit(amodule)
	end
end

private class StringFinder
	super Visitor

	# Strings in the file, used to generate .pot and .po files
	var strings = new HashMap[String, PObject]

	# Domain of the strings to internationalize
	var domain: String

	# Location of the languages file
	var languages_location: String

	# Context for the visitor, used only for the parse_expr
	var toolcontext: ToolContext

	# The module we are working on
	var amodule: AModule

	redef fun visit(n)
	do
		n.accept_string_finder(self)
		n.visit_all(self)
	end

	redef fun enter_visit(e) do
		if e isa AAnnotation then return
		super
	end

	# Adds a String to the list of strings of the module
	#
	# The string needs to be pre-formatted to C standards (escape_to_c)
	fun add_string(s: String, loc: Location) do
		var locstr = "{amodule.mmodule.mgroup.name}::{amodule.mmodule.name} {loc.line_start}--{loc.column_start}:{loc.column_end}"
		if not strings.has_key(s) then
			var po = new PObject([locstr], s, "")
			strings[s] = po
		else
			strings[s].locations.push locstr
		end
	end
end

redef class ANode
	private fun accept_string_finder(v: StringFinder) do end
end

redef class AStringExpr

	redef fun accept_string_finder(v) do
		var str = value.escape_to_gettext
		var code = "\"{str}\".get_translation(\"{v.domain}\", \"{v.languages_location}\")"
		var parse = v.toolcontext.parse_expr(code)
		replace_with(parse)
		v.add_string(str, location)
	end
end

redef class ASuperstringExpr

	redef fun accept_string_finder(v) do
		var fmt = ""
		var exprs = new Array[AExpr]
		for i in n_exprs do
			if i isa AStartStringExpr or i isa AEndStringExpr or i isa AMidStringExpr then
				assert i isa AStringFormExpr
				var str = i.value
				fmt += str.replace("%", "%%")
			else
				fmt += "%"
				exprs.push i
				fmt += (exprs.length-1).to_s
			end
		end
		fmt = fmt.escape_to_gettext
		v.add_string(fmt, location)
		var code = "\"{fmt}\".get_translation(\"{v.domain}\", \"{v.languages_location}\").format()"
		var parse = v.toolcontext.parse_expr(code)
		if not parse isa ACallExpr then
			v.toolcontext.error(location, "Fatal error in i18n annotation, the parsed superstring could not be generated properly")
			return
		end
		var parse_exprs = parse.n_args.n_exprs
		parse_exprs.add_all exprs
		replace_with parse
	end
end

# .po file entry
#
# Locations are optional, they just serve for translation purposes
# to help the translator with the context of the message if necessary
#
# msgid and msgstr are the map of translate to translated strings in the po file.
class PObject
	# Array since the same string can be encountered at several places
	var locations: Array[String]
	# Identifier of the string to translate (i.e. the string itself)
	var msgid: String is writable
	# Translation of the string
	var msgstr: String is writable
end

# A GNU gettext .po/.pot file
class POFile
	super Writable

	# Map of the strings's `msgid` and `msgstr`
	#
	# Read from a PO file
	var strings: Array[PObject]

	redef fun write_to_file(path) do
		if not path.has_suffix(".po") then path += ".po"
		super path
	end

	redef fun write_to(ofs) do
		for i in strings do
			ofs.write("#: {i.locations.join(", ")}\n")
			ofs.write("msgid \"{i.msgid}\"\n")
			ofs.write("msgstr \"{i.msgstr}\"\n\n")
		end
		ofs.write("# Generated file, do not modify\n")
	end

	# Writes the information of the POFile to a .pot template file
	fun write_template(path: String) do
		if not path.has_suffix(".pot") then path += ".pot"
		var f = new FileWriter.open(path)
		write_to(f)
		f.close
	end
end

redef class Text
	private fun escape_to_gettext: String
	do
		return escape_to_c.replace("\{", "\\\{").replace("\}", "\\\}")
	end
end
src/frontend/i18n_phase.nit:15,1--223,3