Services to read JSON: deserialize_json and JsonDeserializer

Introduced classes

class JsonDeserializer

json :: JsonDeserializer

Deserializer from a Json string.

Redefined classes

redef interface Map[K: nullable Object, V: nullable Object]

json :: serialization_read $ Map

Maps are associative collections: key -> item.
redef interface SimpleCollection[E: nullable Object]

json :: serialization_read $ SimpleCollection

Items can be added to these collections.
redef class Sys

json :: serialization_read $ Sys

The main class of the program.
redef abstract class Text

json :: serialization_read $ Text

High-level abstraction for all text representations

All class definitions

class JsonDeserializer

json $ JsonDeserializer

Deserializer from a Json string.
redef interface Map[K: nullable Object, V: nullable Object]

json :: serialization_read $ Map

Maps are associative collections: key -> item.
redef interface SimpleCollection[E: nullable Object]

json :: serialization_read $ SimpleCollection

Items can be added to these collections.
redef class Sys

json :: serialization_read $ Sys

The main class of the program.
redef abstract class Text

json :: serialization_read $ Text

High-level abstraction for all text representations
package_diagram json::serialization_read serialization_read serialization::safe safe json::serialization_read->serialization::safe json::static static json::serialization_read->json::static poset poset serialization::safe->poset serialization serialization serialization::safe->serialization json::error error json::static->json::error ...poset ... ...poset->poset ...serialization ... ...serialization->serialization ...json::error ... ...json::error->json::error json::json json json::json->json::serialization_read json::json... ... json::json...->json::json

Ancestors

module abstract_collection

core :: abstract_collection

Abstract collection classes and services.
module abstract_text

core :: abstract_text

Abstract class for manipulation of sequences of characters
module array

core :: array

This module introduces the standard array structure.
module bitset

core :: bitset

Services to handle BitSet
module bytes

core :: bytes

Services for byte streams and arrays
module caching

serialization :: caching

Services for caching serialization engines
module circular_array

core :: circular_array

Efficient data structure to access both end of the sequence.
module codec_base

core :: codec_base

Base for codecs to use with streams
module codecs

core :: codecs

Group module for all codec-related manipulations
module collection

core :: collection

This module define several collection classes.
module core

core :: core

Standard classes and methods used by default by Nit programs and libraries.
module engine_tools

serialization :: engine_tools

Advanced services for serialization engines
module environ

core :: environ

Access to the environment variables of the process
module error

json :: error

Intro JsonParseError which is exposed by all JSON reading APIs
module error

core :: error

Standard error-management infrastructure.
module exec

core :: exec

Invocation and management of operating system sub-processes.
module file

core :: file

File manipulations (create, read, write, etc.)
module fixed_ints

core :: fixed_ints

Basic integers of fixed-precision
module fixed_ints_text

core :: fixed_ints_text

Text services to complement fixed_ints
module flat

core :: flat

All the array-based text representations
module gc

core :: gc

Access to the Nit internal garbage collection mechanism
module hash_collection

core :: hash_collection

Introduce HashMap and HashSet.
module inspect

serialization :: inspect

Refine Serializable::inspect to show more useful information
module iso8859_1

core :: iso8859_1

Codec for ISO8859-1 I/O
module kernel

core :: kernel

Most basic classes and methods.
module list

core :: list

This module handle double linked lists
module math

core :: math

Mathematical operations
module meta

meta :: meta

Simple user-defined meta-level to manipulate types of instances as object.
module native

core :: native

Native structures for text and bytes
module numeric

core :: numeric

Advanced services for Numeric types
module parser_base

parser_base :: parser_base

Simple base for hand-made parsers of all kinds
module poset

poset :: poset

Pre order sets and partial order set (ie hierarchies)
module protocol

core :: protocol

module queue

core :: queue

Queuing data structures and wrappers
module range

core :: range

Module for range of discrete objects.
module re

core :: re

Regular expression support for all services based on Pattern
module ropes

core :: ropes

Tree-based representation of a String.
module serialization

serialization :: serialization

General serialization services
module serialization_core

serialization :: serialization_core

Abstract services to serialize Nit objects to different formats
module sorter

core :: sorter

This module contains classes used to compare things and sorts arrays.
module stream

core :: stream

Input and output streams of characters
module text

core :: text

All the classes and methods related to the manipulation of text entities
module time

core :: time

Management of time and dates
module union_find

core :: union_find

union–find algorithm using an efficient disjoint-set data structure
module utf8

core :: utf8

Codec for UTF-8 I/O

Parents

module safe

serialization :: safe

Services for safer deserialization engines
module static

json :: static

Static interface to read Nit objects from JSON strings

Children

module json

json :: json

Read and write JSON formatted text using the standard serialization services

Descendants

module a_star-m

a_star-m

module api

github :: api

Nit object oriented interface to Github api.
module at_boot

android :: at_boot

Import this module to launch Service at device boot
module bundle

android :: bundle

A mapping class of String to various value types used by the
module cache

github :: cache

Enable caching on Github API accesses.
module client

gamnit :: client

Client-side network services for games and such
module common

gamnit :: common

Services common to the client and server modules
module commonmark_gen

markdown2 :: commonmark_gen

Generate Nitunit tests from commonmark specification.
module curl_json

neo4j :: curl_json

cURL requests compatible with the JSON REST APIs.
module curl_rest

curl :: curl_rest

module data_store

android :: data_store

Implements app::data_store using shared_preferences
module data_store

linux :: data_store

app::data_store implementation on GNU/Linux
module data_store

ios :: data_store

Implements app::data_store using NSUserDefaults
module error

neo4j :: error

Errors thrown by the neo4j library.
module events

github :: events

Events are emitted by Github Hooks.
module example_angular

popcorn :: example_angular

This is an example of how to use angular.js with popcorn
module github

github :: github

Nit wrapper for Github API
module graph

neo4j :: graph

Provides an interface for services on a Neo4j graphs.
module hooks

github :: hooks

Github hook event listening with nitcorn.
module http_request

android :: http_request

Android implementation of app:http_request
module http_request

linux :: http_request

Implementation of app::http_request using GDK and Curl
module http_request

ios :: http_request

Implementation of app::http_request for iOS
module http_request

app :: http_request

HTTP request services: AsyncHttpRequest and Text::http_get
module http_request_example

app :: http_request_example

Example for the app::http_request main service AsyncHttpRequest
module intent

android :: intent

Services allowing to launch activities and start/stop services using
module intent_api10

android :: intent_api10

Services allowing to launch activities and start/stop services using
module intent_api11

android :: intent_api11

Refines intent module to add API 11 services
module intent_api12

android :: intent_api12

Refines intent module to add API 12 services
module intent_api14

android :: intent_api14

Refines intent module to add API 14 services
module intent_api15

android :: intent_api15

Refines intent module to add API 15 services
module intent_api16

android :: intent_api16

Refines intent module to add API 16 services
module intent_api17

android :: intent_api17

Refines intent module to add API 17 services
module intent_api18

android :: intent_api18

Refines intent module to add API 18 services
module intent_api19

android :: intent_api19

Refines intent module to add API 19 services
module json_graph_store

neo4j :: json_graph_store

Provides JSON as a mean to store graphs.
module loader

github :: loader

module mongodb

mongodb :: mongodb

MongoDB Nit Driver.
module mpi

mpi :: mpi

Implementation of the Message Passing Interface protocol by wrapping OpenMPI
module msgpack

msgpack :: msgpack

MessagePack, an efficient binary serialization format
module msgpack_to_json

msgpack :: msgpack_to_json

Convert MessagePack format to JSON
module native_ui

android :: native_ui

Native services from the android.view and android.widget namespaces
module neo4j

neo4j :: neo4j

Neo4j connector through its JSON REST API using curl.
module network

gamnit :: network

Easy client/server logic for games and simple distributed applications
module nit_activity

android :: nit_activity

Core implementation of app.nit on Android using a custom Java entry point
module pop_auth

popcorn :: pop_auth

Authentification handlers.
module pop_json

popcorn :: pop_json

Introduce useful services for JSON REST API handlers.
module pop_repos

popcorn :: pop_repos

Repositories for data management.
module pop_templates

popcorn :: pop_templates

Template rendering for popcorn
module queries

mongodb :: queries

Mongo queries framework
module restful

nitcorn :: restful

Support module for the nitrestful tool and the restful annotation
module restful_annot

nitcorn :: restful_annot

Example for the restful annotation documented at lib/nitcorn/restful.nit
module sequential_id

neo4j :: sequential_id

Provides a sequential identification scheme for Neo4j nodes.
module serialization_read

msgpack :: serialization_read

Deserialize full Nit objects from MessagePack format
module server

gamnit :: server

Server-side network services for games and such
module service

android :: service

Android service support for app.nit centered around the class Service
module shared_preferences

android :: shared_preferences

Services allowing to save and load datas to internal android device
module shared_preferences_api10

android :: shared_preferences_api10

Services to save/load data using android.content.SharedPreferences for the android platform
module shared_preferences_api11

android :: shared_preferences_api11

Refines shared_preferences module to add API 11 services
module store

json :: store

Store and load json data.
module ui

linux :: ui

Implementation of the app.nit UI module for GNU/Linux
module ui

android :: ui

Views and services to use the Android native user interface
module ui_test

android :: ui_test

Test for app.nit's UI services
module wallet

github :: wallet

Github OAuth tokens management
module wifi

android :: wifi

Simple wrapper of the Android WiFi services
# Services to read JSON: `deserialize_json` and `JsonDeserializer`
module serialization_read

import serialization::caching
private import serialization::engine_tools
import serialization::safe

private import static
import poset

# Deserializer from a Json string.
class JsonDeserializer
	super CachingDeserializer
	super SafeDeserializer

	# Json text to deserialize from.
	private var text: Text

	# Root json object parsed from input text.
	private var root: nullable Object is noinit

	# Depth-first path in the serialized object tree.
	private var path = new Array[Map[String, nullable Object]]

	# Names of the attributes from the root to the object currently being deserialized
	var attributes_path = new Array[String]

	# Last encountered object reference id.
	#
	# See `id_to_object`.
	var just_opened_id: nullable Int = null

	init do
		var root = text.parse_json
		if root isa Map[String, nullable Object] then path.add(root)
		self.root = root
	end

	redef fun deserialize_attribute(name, static_type)
	do
		if path.is_empty then
			# The was a parsing error or the root is not an object
			if not root isa Error then
				errors.add new Error("Deserialization Error: parsed JSON value is not an object.")
			end
			deserialize_attribute_missing = false
			return null
		end

		var current = path.last

		if not current.keys.has(name) then
			# Let the generated code / caller of `deserialize_attribute` raise the missing attribute error
			deserialize_attribute_missing = true
			return null
		end

		var value = current[name]

		attributes_path.add name
		var res = convert_object(value, static_type)
		attributes_path.pop

		deserialize_attribute_missing = false
		return res
	end

	# This may be called multiple times by the same object from constructors
	# in different nclassdef
	redef fun notify_of_creation(new_object)
	do
		var id = just_opened_id
		if id == null then return # Register `new_object` only once
		cache[id] = new_object
	end

	# Convert the simple JSON `object` to a Nit object
	private fun convert_object(object: nullable Object, static_type: nullable String): nullable Object
	do
		if object isa JsonParseError then
			errors.add object
			return null
		end

		if object isa Map[String, nullable Object] then
			var kind = null
			if object.keys.has("__kind") then
				kind = object["__kind"]
			end

			# ref?
			if kind == "ref" then
				if not object.keys.has("__id") then
					errors.add new Error("Deserialization Error: JSON object reference does not declare a `__id`.")
					return object
				end

				var id = object["__id"]
				if not id isa Int then
					errors.add new Error("Deserialization Error: JSON object reference declares a non-integer `__id`.")
					return object
				end

				if not cache.has_id(id) then
					errors.add new Error("Deserialization Error: JSON object reference has an unknown `__id`.")
					return object
				end

				return cache.object_for(id)
			end

			# obj?
			if kind == "obj" or kind == null then
				var id = null
				if object.keys.has("__id") then
					id = object["__id"]

					if not id isa Int then
						errors.add new Error("Deserialization Error: JSON object declaration declares a non-integer `__id`.")
						return object
					end

					if cache.has_id(id) then
						errors.add new Error("Deserialization Error: JSON object with `__id` {id} is deserialized twice.")
						# Keep going
					end
				end

				var class_name = object.get_or_null("__class")
				if class_name == null then
					# Fallback to custom heuristic
					class_name = class_name_heuristic(object)

					if class_name == null and static_type != null then
						# Fallack to the static type, strip the `nullable` prefix
						class_name = static_type.strip_nullable
					end
				end

				if class_name == null then
					errors.add new Error("Deserialization Error: JSON object declaration does not declare a `__class`.")
					return object
				end

				if not class_name isa String then
					errors.add new Error("Deserialization Error: JSON object declaration declares a non-string `__class`.")
					return object
				end

				if not accept(class_name, static_type) then return null

				# advance on path
				path.push object

				just_opened_id = id
				var value = deserialize_class(class_name)
				just_opened_id = null

				# revert on path
				path.pop

				return value
			end

			# char?
			if kind == "char" then
				if not object.keys.has("__val") then
					errors.add new Error("Deserialization Error: JSON `char` object does not declare a `__val`.")
					return object
				end

				var val = object["__val"]

				if not val isa String or val.is_empty then
					errors.add new Error("Deserialization Error: JSON `char` object does not declare a single char in `__val`.")
					return object
				end

				return val.chars.first
			end

			# byte?
			if kind == "byte" then
				var val = object.get_or_null("__val")
				if not val isa Int then
					errors.add new Error("Serialization Error: JSON `byte` object does not declare an integer `__val`.")
					return object
				end

				return val.to_b
			end

			errors.add new Error("Deserialization Error: JSON object has an unknown `__kind`.")
			return object
		end

		# Simple JSON array without serialization metadata
		if object isa Array[nullable Object] then
			# Can we use the static type?
			if static_type != null then
				opened_array = object
				var value = deserialize_class(static_type.strip_nullable)
				opened_array = null
				return value
			end

			# This branch should rarely be used:
			# when an array is the root object which is accepted but illegal in standard JSON,
			# or in strange custom deserialization hacks.

			var array = new Array[nullable Object]
			var types = new HashSet[String]
			var has_nullable = false
			for e in object do
				var res = convert_object(e)
				array.add res

				if res != null then
					types.add res.class_name
				else has_nullable = true
			end

			if types.length == 1 then
				var array_type = types.first

				var typed_array
				if array_type == "ASCIIFlatString" or array_type == "UnicodeFlatString" then
					if has_nullable then
						typed_array = new Array[nullable FlatString]
					else typed_array = new Array[FlatString]
				else if array_type == "Int" then
					if has_nullable then
						typed_array = new Array[nullable Int]
					else typed_array = new Array[Int]
				else if array_type == "Float" then
					if has_nullable then
						typed_array = new Array[nullable Float]
					else typed_array = new Array[Float]
				else
					# TODO support all array types when we separate the constructor
					# `from_deserializer` from the filling of the items.

					if not has_nullable then
						typed_array = new Array[Object]
					else
						# Unsupported array type, return as `Array[nullable Object]`
						return array
					end
				end

				assert typed_array isa Array[nullable Object]

				# Copy item to the new array
				for e in array do typed_array.add e
				return typed_array
			end

			# Uninferrable type, return as `Array[nullable Object]`
			return array
		end

		if object isa String and object.length == 1 and static_type == "Char" then
			# Char serialized as a JSON string
			return object.chars.first
		end

		if object isa Int and static_type == "Byte" then
			# Byte serialized as an integer
			return object.to_b
		end

		return object
	end

	# Current array open for deserialization, used by `SimpleCollection::from_deserializer`
	private var opened_array: nullable Array[nullable Object] = null

	redef fun deserialize(static_type)
	do
		errors.clear
		return convert_object(root, static_type)
	end

	# User customizable heuristic to infer the name of the Nit class to deserialize `json_object`
	#
	# This method is called only when deserializing an object without the metadata `__class`.
	# Use the content of `json_object` to identify what Nit class it should be deserialized into.
	# Or use `self.attributes_path` indicating where the deserialized object will be stored,
	# is is less reliable as some objects don't have an associated attribute:
	# the root/first deserialized object and collection elements.
	#
	# Return the class name as a `String` when it can be inferred,
	# or `null` when the class name cannot be found.
	#
	# If a valid class name is returned, `json_object` will then be deserialized normally.
	# So it must contain the attributes of the corresponding class, as usual.
	#
	# ~~~
	# class MyData
	#     serialize
	#
	#     var data: String
	# end
	#
	# class MyError
	#     serialize
	#
	#     var error: String
	#     var related_data: MyData
	# end
	#
	# class MyJsonDeserializer
	#     super JsonDeserializer
	#
	#     redef fun class_name_heuristic(json_object)
	#     do
	#         # Infer the Nit class from the content of the JSON object.
	#         if json_object.keys.has("error") then return "MyError"
	#         if json_object.keys.has("data") then return "MyData"
	#
	#         # Infer the Nit class from the attribute where it will be stored.
	#         # This line duplicates a previous line, and would only apply when
	#         # `MyData` is within a `MyError`.
	#         if attributes_path.not_empty and attributes_path.last == "related_data" then return "MyData"
	#
	#         return null
	#     end
	# end
	#
	# var json = """{"data": "some data"}"""
	# var deserializer = new MyJsonDeserializer(json)
	# var deserialized = deserializer.deserialize
	# assert deserializer.errors.is_empty
	# assert deserialized isa MyData
	#
	# json = """{"error": "some error message",
	#            "related_data": {"data": "some other data"}}"""
	# deserializer = new MyJsonDeserializer(json)
	# deserialized = deserializer.deserialize
	# assert deserializer.errors.is_empty
	# assert deserialized isa MyError
	# ~~~
	protected fun class_name_heuristic(json_object: Map[String, nullable Object]): nullable String
	do
		return null
	end
end

redef class Text

	# Deserialize a `nullable Object` from this JSON formatted string
	#
	# If a `static_type` is given, only subtypes of the `static_type` are accepted.
	#
	# Warning: Deserialization errors are reported with `print_error` and
	# may be returned as a partial object or as `null`.
	#
	# This method is not appropriate when errors need to be handled programmatically,
	# manually use a `JsonDeserializer` in such cases.
	fun deserialize_json(static_type: nullable String): nullable Object
	do
		var deserializer = new JsonDeserializer(self)
		var res = deserializer.deserialize(static_type)
		if deserializer.errors.not_empty then
			print_error "Deserialization Errors: {deserializer.errors.join(", ")}"
		end
		return res
	end
end

redef class SimpleCollection[E]
	redef init from_deserializer(v)
	do
		super
		if v isa JsonDeserializer then
			v.notify_of_creation self
			init

			var open_array: nullable SequenceRead[nullable Object] = v.opened_array
			if open_array == null then
				# With metadata
				var arr = v.path.last.get_or_null("__items")
				if not arr isa SequenceRead[nullable Object] then
					# If there is nothing, we consider that it is an empty collection.
					if arr != null then v.errors.add new Error("Deserialization Error: invalid format in {self.class_name}")
					return
				end
				open_array = arr
			end

			# Name of the dynamic name of E
			var items_type_name = (new GetName[E]).to_s

			# Fill array
			for o in open_array do
				var obj = v.convert_object(o, items_type_name)
				if obj isa E then
					add obj
				else v.errors.add new AttributeTypeError(self, "items", obj, items_type_name)
			end
		end
	end
end

redef class Map[K, V]
	redef init from_deserializer(v)
	do
		super

		if v isa JsonDeserializer then
			v.notify_of_creation self
			init

			var keys_type_name = (new GetName[K]).to_s
			var values_type_name = (new GetName[V]).to_s

			var length = v.deserialize_attribute("__length")
			var keys = v.path.last.get_or_null("__keys")
			var values = v.path.last.get_or_null("__values")

			if keys == null and values == null then
				# Fallback to a plain object
				for key, value_src in v.path.last do

					var value = v.convert_object(value_src, values_type_name)

					if not key isa K then
						v.errors.add new AttributeTypeError(self, "keys", key, keys_type_name)
						continue
					end

					if not value isa V then
						v.errors.add new AttributeTypeError(self, "values", value, values_type_name)
						continue
					end

					self[key] = value
				end
				return
			end

			# Length is optional
			if length == null and keys isa SequenceRead[nullable Object] then length = keys.length

			# Consistency check
			if not length isa Int or length < 0 or
			   not keys isa SequenceRead[nullable Object] or
			   not values isa SequenceRead[nullable Object] or
			   keys.length != values.length or length != keys.length then

				# If there is nothing or length == 0, we consider that it is an empty Map.
				if (length != null and length != 0) or keys != null or values != null then
					v.errors.add new Error("Deserialization Error: invalid format in {self.class_name}")
				end
				return
			end

			# First, convert all keys to follow the order of the serialization
			var converted_keys = new Array[K]
			for i in length.times do
				var key = v.convert_object(keys[i], keys_type_name)

				if not key isa K then
					v.errors.add new AttributeTypeError(self, "keys", key, keys_type_name)
					continue
				end

				converted_keys.add key
			end

			# Then convert the values and build the map
			for i in length.times do
				var key = converted_keys[i]
				var value = v.convert_object(values[i], values_type_name)

				if not value isa V then
					v.errors.add new AttributeTypeError(self, "values", value, values_type_name)
					continue
				end

				if has_key(key) then
					v.errors.add new Error("Deserialization Error: duplicated key '{key or else "null"}' in {self.class_name}, previous value overwritten")
				end

				self[key] = value
			end
		end
	end
end

# ---
# Metamodel

# Class inheritance graph as a `POSet[String]` serialized to JSON
private fun class_inheritance_metamodel_json: CString is intern

redef class Sys
	redef var class_inheritance_metamodel is lazy do
		var engine = new JsonDeserializer(class_inheritance_metamodel_json.to_s)
		engine.check_subtypes = false
		engine.whitelist.add_all(
			["String", "POSet[String]", "POSetElement[String]",
			 "HashSet[String]", "HashMap[String, POSetElement[String]]"])

		var poset = engine.deserialize
		if engine.errors.not_empty then
			print_error "Deserialization errors in class_inheritance_metamodel:"
			print_error engine.errors.join("\n* ")
			return new POSet[String]
		end

		if poset isa POSet[String] then return poset
		return new POSet[String]
	end
end
lib/json/serialization_read.nit:15,1--529,3