Compiles extern code within a module to a static library, as needed

Redefined classes

redef class AMethPropdef

nitc :: on_demand_compiler $ AMethPropdef

A definition of all kind of method (including constructors)
redef class AModule

nitc :: on_demand_compiler $ AModule

The main node of a Nit source-file
redef class CCompilationUnit

nitc :: on_demand_compiler $ CCompilationUnit

Accumulates all C code for a compilation unit
redef class ExternCFile

nitc :: on_demand_compiler $ ExternCFile

An extern C file to compile
redef class ExternFile

nitc :: on_demand_compiler $ ExternFile

An extern file to compile
redef class MClassType

nitc :: on_demand_compiler $ MClassType

A type based on a class.
redef class MMethodDef

nitc :: on_demand_compiler $ MMethodDef

A local definition of a method
redef abstract class MType

nitc :: on_demand_compiler $ MType

A global static type
redef class NaiveInterpreter

nitc :: on_demand_compiler $ NaiveInterpreter

The visitor that interprets the Nit Program by walking on the AST
redef class ToolContext

nitc :: on_demand_compiler $ ToolContext

Global context for tools

All class definitions

redef class AMethPropdef

nitc :: on_demand_compiler $ AMethPropdef

A definition of all kind of method (including constructors)
redef class AModule

nitc :: on_demand_compiler $ AModule

The main node of a Nit source-file
redef class CCompilationUnit

nitc :: on_demand_compiler $ CCompilationUnit

Accumulates all C code for a compilation unit
redef class ExternCFile

nitc :: on_demand_compiler $ ExternCFile

An extern C file to compile
redef class ExternFile

nitc :: on_demand_compiler $ ExternFile

An extern file to compile
redef class MClassType

nitc :: on_demand_compiler $ MClassType

A type based on a class.
redef class MMethodDef

nitc :: on_demand_compiler $ MMethodDef

A local definition of a method
redef abstract class MType

nitc :: on_demand_compiler $ MType

A global static type
redef class NaiveInterpreter

nitc :: on_demand_compiler $ NaiveInterpreter

The visitor that interprets the Nit Program by walking on the AST
redef class ToolContext

nitc :: on_demand_compiler $ ToolContext

Global context for tools
package_diagram nitc::on_demand_compiler on_demand_compiler nitc\>ffi\> ffi nitc::on_demand_compiler->nitc\>ffi\> nitc::naive_interpreter naive_interpreter nitc::on_demand_compiler->nitc::naive_interpreter nitc\>platform\> platform nitc\>ffi\>->nitc\>platform\> nitc\>modelize\> modelize nitc\>ffi\>->nitc\>modelize\> nitc\>nitni\> nitni nitc\>ffi\>->nitc\>nitni\> nitc nitc nitc\>ffi\>->nitc nitc\>semantize\> semantize nitc::naive_interpreter->nitc\>semantize\> nitc::mixin mixin nitc::naive_interpreter->nitc::mixin nitc::serialize_model serialize_model nitc::naive_interpreter->nitc::serialize_model nitc::explain_assert_api explain_assert_api nitc::naive_interpreter->nitc::explain_assert_api ...nitc\>platform\> ... ...nitc\>platform\>->nitc\>platform\> ...nitc\>modelize\> ... ...nitc\>modelize\>->nitc\>modelize\> ...nitc\>nitni\> ... ...nitc\>nitni\>->nitc\>nitni\> ...nitc ... ...nitc->nitc ...nitc\>semantize\> ... ...nitc\>semantize\>->nitc\>semantize\> ...nitc::mixin ... ...nitc::mixin->nitc::mixin ...nitc::serialize_model ... ...nitc::serialize_model->nitc::serialize_model ...nitc::explain_assert_api ... ...nitc::explain_assert_api->nitc::explain_assert_api nitc::dynamic_loading_ffi dynamic_loading_ffi nitc::dynamic_loading_ffi->nitc::on_demand_compiler nitc::interpreter interpreter nitc::interpreter->nitc::dynamic_loading_ffi nitc::interpreter... ... nitc::interpreter...->nitc::interpreter

Ancestors

module abstract_collection

core :: abstract_collection

Abstract collection classes and services.
module abstract_text

core :: abstract_text

Abstract class for manipulation of sequences of characters
module annotation

nitc :: annotation

Management and utilities on annotations
module array

core :: array

This module introduces the standard array structure.
module auto_super_init

nitc :: auto_super_init

Computing of super-constructors that must be implicitly called at the begin of constructors.
module bitset

core :: bitset

Services to handle BitSet
module bytes

core :: bytes

Services for byte streams and arrays
module c

nitc :: c

Support for nesting C code within a Nit program using its FFI
module c_compiler_options

nitc :: c_compiler_options

Offers the annotations cflags and ldflags to specify
module c_tools

nitc :: c_tools

provides tools to write C .c and .h files
module caching

serialization :: caching

Services for caching serialization engines
module circular_array

core :: circular_array

Efficient data structure to access both end of the sequence.
module codec_base

core :: codec_base

Base for codecs to use with streams
module codecs

core :: codecs

Group module for all codec-related manipulations
module collection

core :: collection

This module define several collection classes.
module console

console :: console

Defines some ANSI Terminal Control Escape Sequences.
module core

core :: core

Standard classes and methods used by default by Nit programs and libraries.
module cpp

nitc :: cpp

Supports the use of the C++ language through the FFI
module csv

csv :: csv

CSV document handling.
module digraph

graph :: digraph

Implementation of directed graphs, also called digraphs.
module engine_tools

serialization :: engine_tools

Advanced services for serialization engines
module environ

core :: environ

Access to the environment variables of the process
module error

core :: error

Standard error-management infrastructure.
module exec

core :: exec

Invocation and management of operating system sub-processes.
module explain_assert_api

nitc :: explain_assert_api

Explain failed assert to the console (service declaration only)
module extern_classes

nitc :: extern_classes

Manages all extern classes and their associated foreign type.
module extra_java_files

nitc :: extra_java_files

Intro the annotation extra_java_files to compile extra java files
module ffi_base

nitc :: ffi_base

Tools and utilities for implement FFI with different languages
module file

core :: file

File manipulations (create, read, write, etc.)
module fixed_ints

core :: fixed_ints

Basic integers of fixed-precision
module fixed_ints_text

core :: fixed_ints_text

Text services to complement fixed_ints
module flat

core :: flat

All the array-based text representations
module flow

nitc :: flow

Intraprocedural static flow.
module gc

core :: gc

Access to the Nit internal garbage collection mechanism
module hash_collection

core :: hash_collection

Introduce HashMap and HashSet.
module header_dependency

nitc :: header_dependency

Tracks which modules has public header code that must be imported
module ini

ini :: ini

Read and write INI configuration files
module inspect

serialization :: inspect

Refine Serializable::inspect to show more useful information
module iso8859_1

core :: iso8859_1

Codec for ISO8859-1 I/O
module java

nitc :: java

FFI support for the Java language
module kernel

core :: kernel

Most basic classes and methods.
module lexer

nitc :: lexer

Lexer and its tokens.
module lexer_work

nitc :: lexer_work

Internal algorithm and data structures for the Nit lexer
module light_c

nitc :: light_c

Support for nesting C code within a Nit program using its FFI
module light_ffi

nitc :: light_ffi

Light FFI support, independent of the compiler
module light_ffi_base

nitc :: light_ffi_base

Tools and utilities for implement FFI with different languages
module list

core :: list

This module handle double linked lists
module literal

nitc :: literal

Parsing of literal values in the abstract syntax tree.
module loader

nitc :: loader

Loading of Nit source files
module local_var_init

nitc :: local_var_init

Verify that local variables are initialized before their usage
module location

nitc :: location

Nit source-file and locations in source-file
module math

core :: math

Mathematical operations
module mdoc

nitc :: mdoc

Documentation of model entities
module meta

meta :: meta

Simple user-defined meta-level to manipulate types of instances as object.
module mixin

nitc :: mixin

Loading and additional module refinements at link-time.
module mmodule

nitc :: mmodule

modules and module hierarchies in the metamodel
module mmodule_data

nitc :: mmodule_data

Define and retrieve data in modules
module model

nitc :: model

Classes, types and properties
module model_base

nitc :: model_base

The abstract concept of model and related common things
module modelbuilder_base

nitc :: modelbuilder_base

Load nit source files and build the associated model
module modelize

nitc :: modelize

Create a model from nit source files
module modelize_class

nitc :: modelize_class

Analysis and verification of class definitions to instantiate model element
module modelize_property

nitc :: modelize_property

Analysis and verification of property definitions to instantiate model element
module more_collections

more_collections :: more_collections

Highly specific, but useful, collections-related classes.
module mpackage

nitc :: mpackage

Modelisation of a Nit package
module native

core :: native

Native structures for text and bytes
module nitni

nitc :: nitni

Native interface related services (used underneath the FFI)
module nitni_base

nitc :: nitni_base

Native interface related services (used underneath the FFI)
module nitni_callbacks

nitc :: nitni_callbacks

nitni services related to callbacks (used underneath the FFI)
module nitpm_shared

nitc :: nitpm_shared

Services related to the Nit package manager
module numeric

core :: numeric

Advanced services for Numeric types
module objc

nitc :: objc

FFI support for Objective-C
module opts

opts :: opts

Management of options on the command line
module ordered_tree

ordered_tree :: ordered_tree

Manipulation and presentation of ordered trees.
module parser

nitc :: parser

Parser.
module parser_nodes

nitc :: parser_nodes

AST nodes of the Nit language
module parser_prod

nitc :: parser_prod

Production AST nodes full definition.
module parser_util

nitc :: parser_util

Utils and tools related to parsers and AST
module parser_work

nitc :: parser_work

Internal algorithm and data structures for the Nit parser
module phase

nitc :: phase

Phases of the processing of nit programs
module pkgconfig

nitc :: pkgconfig

Offers the PkgconfigPhase to use the external program "pkg-config" in order
module platform

nitc :: platform

Platform system, used to customize the behavior of the compiler.
module poset

poset :: poset

Pre order sets and partial order set (ie hierarchies)
module protocol

core :: protocol

module queue

core :: queue

Queuing data structures and wrappers
module range

core :: range

Module for range of discrete objects.
module rapid_type_analysis

nitc :: rapid_type_analysis

Rapid type analysis on the AST
module re

core :: re

Regular expression support for all services based on Pattern
module ropes

core :: ropes

Tree-based representation of a String.
module scope

nitc :: scope

Identification and scoping of local variables and labels.
module semantize

nitc :: semantize

Process bodies of methods in regard with the model.
module serialization

serialization :: serialization

General serialization services
module serialization_core

serialization :: serialization_core

Abstract services to serialize Nit objects to different formats
module serialization_write

json :: serialization_write

Services to write Nit objects to JSON strings: serialize_to_json and JsonSerializer
module serialize_model

nitc :: serialize_model

Service to serialize POSet to JSON
module sorter

core :: sorter

This module contains classes used to compare things and sorts arrays.
module stream

core :: stream

Input and output streams of characters
module tables

nitc :: tables

Module that interfaces the parsing tables.
module template

template :: template

Basic template system
module text

core :: text

All the classes and methods related to the manipulation of text entities
module time

core :: time

Management of time and dates
module toolcontext

nitc :: toolcontext

Common command-line tool infrastructure than handle options and error messages
module typing

nitc :: typing

Intraprocedural resolution of static types and OO-services
module union_find

core :: union_find

union–find algorithm using an efficient disjoint-set data structure
module utf8

core :: utf8

Codec for UTF-8 I/O
module version

nitc :: version

This file was generated by git-gen-version.sh

Parents

module ffi

nitc :: ffi

Full FFI support, independent of the compiler
module naive_interpreter

nitc :: naive_interpreter

Interpretation of a Nit program directly on the AST

Children

module dynamic_loading_ffi

nitc :: dynamic_loading_ffi

Execute FFI code by creating and loading shared libraries

Descendants

module a_star-m

a_star-m

module interpreter

nitc :: interpreter

Interpretation of Nit programs
module nit

nitc :: nit

A naive Nit interpreter
module test_astbuilder

nitc :: test_astbuilder

Program used to test the clone method of the astbuilder tool
# Compiles extern code within a module to a static library, as needed
module on_demand_compiler

import modelbuilder
import c_tools
import nitni
import ffi
import naive_interpreter
import pkgconfig

redef class ToolContext

	# --compile-dir
	var opt_compile_dir = new OptionString("Directory used to generate temporary files", "--compile-dir")

	init do option_context.add_option opt_compile_dir
end

redef class AMethPropdef
	# Does this method definition use the FFI and is it supported by the interpreter?
	#
	# * Must use the nested foreign code block of the FFI.
	# * Must not have callbacks.
	# * Must be implemented in C.
	# * Must not have a parameter or return typed with a Nit standard class.
	fun supported_by_dynamic_ffi: Bool
	do
		# If the user specfied `is light_ffi`, it must be supported
		var nats = get_annotations("light_ffi")
		if nats.not_empty then return true

		var n_extern_code_block = n_extern_code_block
		if not (n_extern_calls == null and n_extern_code_block != null and
		        n_extern_code_block.is_c) then return false

		for mparam in mpropdef.msignature.mparameters do
			if not mparam.mtype.is_cprimitive then
				return false
			end
		end

		var return_mtype = mpropdef.msignature.return_mtype
		if return_mtype != null and not return_mtype.is_cprimitive then
			return false
		end

		return true
	end
end

redef class NaiveInterpreter
	redef fun start(mainmodule)
	do
		super

		# Delete temporary files
		var compile_dir = compile_dir
		if compile_dir.file_exists then compile_dir.rmdir
	end

	# Where to store generated C and extracted code
	private var compile_dir: String is lazy do
		# Prioritize the user supplied directory
		var opt = modelbuilder.toolcontext.opt_compile_dir.value
		if opt != null then return opt
		return "/tmp/niti_ffi_{process_id}"
	end

	# Identifier for this process, unique between running interpreters
	private fun process_id: Int `{ return getpid(); `}

	# Path of the compiled foreign code library
	#
	# TODO change the ".so" extension per platform.
	fun foreign_code_lib_path(mmodule: MModule): String
	do
		return compile_dir / mmodule.c_name + ".so"
	end

	# External compiler used to generate the foreign code library
	private var c_compiler = "cc"
end

redef class AModule

	# Compile user FFI code and a standardized API into a `.so` file
	#
	# Returns `true` on success.
	fun compile_foreign_lib(v: NaiveInterpreter): Bool
	do
		var mmodule = mmodule
		assert mmodule != null

		var compile_dir = v.compile_dir
		var foreign_code_lib_path = v.foreign_code_lib_path(mmodule)

		if not compile_dir.file_exists then compile_dir.mkdir(0o700)

		# Compile the common FFI part
		ensure_compile_ffi_wrapper
		for mclassdef in mmodule.mclassdefs do for mpropdef in mclassdef.mpropdefs do
			var anode = v.modelbuilder.mpropdef2node(mpropdef)
			if mpropdef isa MMethodDef and anode isa AMethPropdef and anode.supported_by_dynamic_ffi then
				anode.compile_ffi_method(mmodule)
			end
		end
		mmodule.finalize_ffi_wrapper(compile_dir, mmodule)

		# Compile the standard API and its implementation for the .so file
		var ccu = compile_foreign_lib_api(compile_dir)

		var srcs = [for file in ccu.files do new ExternCFile(file, ""): ExternFile]
		srcs.add_all mmodule.ffi_files

		# Compiler options specific to this module
		var ldflags_array = mmodule.ldflags[""]
		if ldflags_array.has("-lrt") and system("sh -c 'uname -s 2>/dev/null || echo not' | grep Darwin >/dev/null") == 0 then
			# Remove -lrt on OS X
			ldflags_array.remove "-lrt"
		end
		var ldflags = ldflags_array.join(" ")

		# Protect pkg-config
		var pkgconfigs = mmodule.pkgconfigs
		var pkg_cflags = ""
		if not pkgconfigs.is_empty then

			# Check if the pkgconfig packages are available
			v.modelbuilder.toolcontext.check_pkgconfig_packages pkgconfigs
			if not v.modelbuilder.toolcontext.check_errors then return false

			pkg_cflags = "`pkg-config --cflags {pkgconfigs.join(" ")}`"
			ldflags += " `pkg-config --libs {pkgconfigs.join(" ")}`"
		end

		# Compile each source file to an object file (or equivalent)
		var object_files = new Array[String]
		for f in srcs do
			f.compile(v, mmodule, object_files, pkg_cflags)
		end

		# Link everything in a shared library
		var cmd = "{v.c_compiler} -Wall -shared -o {foreign_code_lib_path} {object_files.join(" ")} {ldflags}"
		if system(cmd) != 0 then
			v.fatal "FFI Error: Failed to link native code using `{cmd}`"
			return false
		end

		return true
	end

	# Compile the standard API of the `.so` file
	#
	# * The shared structure `nit_call_arg`.
	# * Standardized implementation functions entrypoints that relay calls
	#   to the FFI implementation functions.
	private fun compile_foreign_lib_api(compdir: String): CCompilationUnit
	do
		var mmodule = mmodule
		assert mmodule != null

		# ready extern code compiler
		var ecc = new CCompilationUnit

		ecc.body_decl.add """

#include <string.h>
#include <stdio.h>
#include <inttypes.h>

// C structure behind `CallArg` from the interpreter
typedef union nit_call_arg {
	long value_Int;
	int value_Bool;
	uint32_t value_Char;
	uint8_t value_Byte;
	int8_t value_Int8;
	int16_t value_Int16;
	uint16_t value_UInt16;
	int32_t value_Int32;
	uint32_t value_UInt32;
	double value_Float;
	void* value_Pointer;
} nit_call_arg;

"""

		# types
		var used_types = collect_mtypes
		for t in used_types do
			if not t.is_cprimitive then
				ecc.header_c_types.add "typedef void* {t.cname};\n"
			end
		end

		# TODO callbacks & casts

		for nclassdef in n_classdefs do for npropdef in nclassdef.n_propdefs do
			if npropdef isa AMethPropdef and npropdef.supported_by_dynamic_ffi then
				npropdef.mpropdef.compile_foreign_code_entry ecc
			end
		end

		ecc.write_as_foreign_lib_api(mmodule, compdir)

		return ecc
	end

	# Collect all `MType` use in extern methods within this module
	private fun collect_mtypes: Set[MType]
	do
		var used_types = new HashSet[MType]

		# collect callbacks
		for nclassdef in n_classdefs do for npropdef in nclassdef.n_propdefs do
			if npropdef isa AMethPropdef and npropdef.supported_by_dynamic_ffi then
				var fcs = npropdef.foreign_callbacks
				used_types.add_all fcs.types
			end
		end

		return used_types
	end
end

redef class CCompilationUnit
	# Write this compilation unit as the API of a foreign code library
	private fun write_as_foreign_lib_api(mmodule: MModule, compdir: String)
	do
		# The FFI expects the support header to end with `._nitni.h`
		var base_name = mmodule.c_name + "._nitni"
		var guard = mmodule.c_name.to_s.to_upper + "_API_H"
		var header_comment = """
/*
	Public API to foreign code of the Nit module {{{mmodule.name}}}
*/
"""

		# Header file
		var h_file = base_name+".h"
		var stream = new FileWriter.open(compdir/h_file)
		stream.write header_comment
		stream.write """
#ifndef {{{guard}}}
#define {{{guard}}}
"""
		compile_header_core stream
		stream.write """

#endif
"""
		stream.close

		# Body file
		var c_file = base_name+".c"
		stream = new FileWriter.open(compdir/c_file)
		stream.write header_comment
		stream.write """
#include "{{{h_file}}}"
"""
		compile_body_core stream
		stream.close

		# Only the C files needs compiling
		files.add compdir / c_file
	end
end

redef class MMethodDef
	# Name of the entry point to the implementation function in the foreign lib
	fun foreign_lib_entry_cname: String do return "entry__{cname}"

	# Compile the standardized entry point as part of the foreign lib API
	private fun compile_foreign_code_entry(ecc: CCompilationUnit)
	do
		var msignature = msignature
		if msignature == null then return

		# Return type
		var return_mtype = msignature.return_mtype
		if mproperty.is_init then return_mtype = mclassdef.mclass.mclass_type

		var c_return_type
		if return_mtype != null then
			c_return_type = return_mtype.cname_blind
		else c_return_type = "void"

		var is_init = mproperty.is_init

		# Params
		var params = new Array[String]
		if not is_init then params.add mclassdef.mclass.mclass_type.cname_blind
		for param in msignature.mparameters do params.add param.mtype.cname_blind

		# Declare the implementation function as extern
		var impl_cname = mproperty.build_cname(mclassdef.bound_mtype,
			mclassdef.mmodule, "___impl", long_signature)
		ecc.body_decl.add "extern {c_return_type} {impl_cname}({params.join(", ")});\n"

		# Declare the entry function
		var foreign_lib_entry_cname = "int {foreign_lib_entry_cname}(int argc, nit_call_arg *argv, nit_call_arg *result)"
		var fc = new CFunction(foreign_lib_entry_cname)

		# Check argument count on the library side
		#
		# This may detect inconsistencies between the interpreter and the generated code.
		var expected_argc = msignature.arity
		if not is_init then expected_argc += 1

		fc.exprs.add """
	if (argc != {{{expected_argc}}}) {
		printf("Invalid argument count in `{{{mproperty.full_name}}}`, expected %d, got %d.\\n",
			argc, {{{expected_argc}}});
		return 1;
	}
"""

		# Unpack and prepare args for the user code
		var k = 0
		var args_for_call = new Array[String]
		if not is_init then
			var mtype = mclassdef.mclass.mclass_type
			var arg_name = "arg___self"

			fc.decls.add "	{mtype.cname_blind} {arg_name};\n"
			fc.exprs.add "	{arg_name} = argv[{k}].{mtype.call_arg_field};\n"
			args_for_call.add arg_name

			k += 1
		end
		for param in msignature.mparameters do
			var mtype = param.mtype
			var arg_name = "arg__"+param.name

			fc.decls.add "	{mtype.cname_blind} {arg_name};\n"
			fc.exprs.add "	{arg_name} = argv[{k}].{mtype.call_arg_field};\n"
			args_for_call.add arg_name

			k += 1
		end

		# Call implementation function
		var args_compressed = args_for_call.join(", ")
		var method_call = "{impl_cname}({args_compressed})"
		if return_mtype != null then
			fc.decls.add """
	{{{return_mtype.cname_blind}}} return_value;
"""
			fc.exprs.add """
	return_value = {{{method_call}}};
	result->{{{return_mtype.call_arg_field}}} = return_value;
"""
		else
			fc.exprs.add "	{method_call};\n"
		end

		fc.exprs.add "	return 0;\n"

		ecc.add_exported_function fc
	end
end

redef class MType
	# The interpreter FFI use `void*` to represent intern data types
	redef fun cname_blind do return "void*"

	# Field to store this type in the C structure `nit_call_arg`
	private fun call_arg_field: String do return "value_Pointer"
end

redef class MClassType
	redef fun call_arg_field
	do
		if is_cprimitive and mclass.kind != extern_kind then
			return "value_{name}"
		else return super
	end
end

redef class ExternFile
	# Compile this source file
	private fun compile(v: NaiveInterpreter, mmodule: MModule,
		object_files: Array[String], pkg_cflags: String): Bool is abstract
end

redef class ExternCFile
	redef fun compile(v, mmodule, object_files, pkg_cflags)
	do
		var compile_dir = v.compile_dir
		var cflags = mmodule.cflags[""].join(" ") + " " + pkg_cflags
		var obj = compile_dir / filename.basename(".c") + ".o"

		var cmd = "{v.c_compiler} -Wall -c -fPIC -I {compile_dir} -g -o {obj} {compile_dir / filename} {cflags}"
		if sys.system(cmd) != 0 then
			 v.fatal "FFI Error: Failed to compile C code using `{cmd}`"
			 return false
		end

		object_files.add obj
		return true
	end
end
src/interpreter/dynamic_loading_ffi/on_demand_compiler.nit:15,1--416,3