1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Serialize and deserialize Nit objects to binary streams
17 # The serialized data format uses a dictionary structure similar to BSON:
20 # object = 0x01 # null
21 # | 0x02 id attributes # New object
22 # | 0x03 id # Ref to object
24 # | 0x05 int8 # Bool (int8 != 0)
25 # | 0x06 utf8 byte sequence # Char
26 # | 0x07 double(64 bits) # Float
27 # | 0x08 block # String
28 # | 0x09 block # NativeString
29 # | 0x0A flat_array; # Array[nullable Object]
31 # block = int64 int8*;
32 # cstring = int8* 0x00;
35 # attributes = attribute* 0x00;
36 # attribute = cstring object;
40 import ::serialization
::caching
41 private import ::serialization
::engine_tools
43 import more_collections
46 # Special bytes, marking the kind of objects in the stream and the end on an object
48 private fun kind_null
: Byte do return 0x01u
8
49 private fun kind_object_new
: Byte do return 0x02u
8
50 private fun kind_object_ref
: Byte do return 0x03u
8
51 private fun kind_int
: Byte do return 0x04u
8
52 private fun kind_bool
: Byte do return 0x05u
8
53 private fun kind_char
: Byte do return 0x06u
8
54 private fun kind_float
: Byte do return 0x07u
8
55 private fun kind_string
: Byte do return 0x08u
8
56 private fun kind_native_string
: Byte do return 0x09u
8
57 private fun kind_flat_array
: Byte do return 0x0Au
8
59 private fun new_object_end
: Byte do return 0x00u
8
64 # Writes Nit objects to the binary `stream`
66 # The output can be deserialized with `BinaryDeserializer`.
67 class BinarySerializer
68 super CachingSerializer
70 # Target writing stream
71 var stream
: Writer is writable
73 redef fun serialize
(object
)
75 if object
== null then
76 stream
.write_byte kind_null
77 else serialize_reference
(object
)
80 redef fun serialize_attribute
(name
, value
)
82 stream
.write_string name
86 redef fun serialize_reference
(object
)
88 if cache
.has_object
(object
) then
89 # if already serialized, add local reference
90 var id
= cache
.id_for
(object
)
91 stream
.write_byte kind_object_ref
95 object
.serialize_to_binary
self
99 # Write `collection` as a simple list of objects
100 private fun serialize_flat_array
(collection
: Collection[nullable Object])
102 stream
.write_byte kind_flat_array
103 stream
.write_int64 collection
.length
104 for e
in collection
do
105 if not try_to_serialize
(e
) then
107 warn
"Element of {collection} is not serializable, it is a {e}"
114 # Deserialize Nit objects from a binary `stream`
116 # Used with `BinarySerializer`.
117 class BinaryDeserializer
118 super CachingDeserializer
120 # Source `Reader` stream
123 # Last encountered object reference id.
125 # See `cache.received`.
126 private var just_opened_id
: nullable Int = null
128 # Tree of attributes, deserialized but not yet claimed
129 private var unclaimed_attributes
= new UnrolledList[HashMap[String, nullable Object]]
131 # Buffer for one char
132 private var char_buf
: NativeString is lazy
do return new NativeString(4)
134 # Read and deserialize the next attribute name and value
136 # A `peeked_char` can suffix the next attribute name.
138 # Returns `null` on error.
139 private fun deserialize_next_attribute
(peeked_char
: nullable Byte):
140 nullable Couple[String, nullable Object]
142 # Try the next attribute
143 var next_attribute_name
= stream
.read_string
144 var next_object
= deserialize_next_object
146 if stream
.last_error
!= null then return null
148 if peeked_char
!= null then
149 # Replace a char peeked to find an object end
150 next_attribute_name
= "{peeked_char}{next_attribute_name}"
153 return new Couple[String, nullable Object](next_attribute_name
, next_object
)
156 redef fun deserialize_attribute
(name
)
158 if unclaimed_attributes
.last
.keys
.has
(name
) then
159 # Pick in already deserialized attributes
160 var value
= unclaimed_attributes
.last
[name
]
161 unclaimed_attributes
.last
.keys
.remove
(name
)
165 # Read attributes until we find the wanted one named `name`
167 var next
= deserialize_next_attribute
169 # Error was already logged
173 var next_attribute_name
= next
.first
174 var next_object
= next
.second
176 # Got the wanted object
177 if next_attribute_name
== name
then return next_object
179 # An invalid attribute name is an heuristic for invalid data.
180 # Hitting an object end marker will result in an empty string.
181 assert next_attribute_name
.is_valid_id
else
184 if next_attribute_name
.is_empty
then
185 # Reached the end of the object
186 error
= new Error("Deserialization Error: Attributes '{name}' not in stream.")
188 error
= new Error("Deserialization Error: Got an invalid attribute name '{next_attribute_name}', expected '{name}'")
189 # TODO this is invalid data, break even on keep_going
195 # It's not the next attribute, put it aside
196 unclaimed_attributes
.last
[next_attribute_name
] = next_object
200 redef fun notify_of_creation
(new_object
)
202 var id
= just_opened_id
203 if id
== null then return
204 cache
[id
] = new_object
207 # Convert from simple Json object to Nit object
208 private fun deserialize_next_object
: nullable Object
210 var kind
= stream
.read_byte
211 assert kind
isa Byte else
212 # TODO break even on keep_going
216 # After this point, all stream reading errors are caught later
218 if kind
== kind_null
then return null
219 if kind
== kind_int
then return stream
.read_int64
220 if kind
== kind_bool
then return stream
.read_bool
221 if kind
== kind_float
then return stream
.read_double
222 if kind
== kind_char
then
224 var b
= stream
.read_byte
225 if b
== null then return '�'
228 for i
in [1 .. ln
[ do
230 if b
== null then return '�'
233 return bf
.to_s_with_length
(ln
)[0]
235 if kind
== kind_string
then return stream
.read_block
236 if kind
== kind_native_string
then return stream
.read_block
.to_cstring
238 if kind
== kind_flat_array
then
240 var length
= stream
.read_int64
241 var array
= new Array[nullable Object]
242 for i
in length
.times
do
243 array
.add deserialize_next_object
248 if kind
== kind_object_ref
then
250 var id
= stream
.read_int64
251 if stream
.last_error
!= null then return null
253 if not cache
.has_id
(id
) then
254 errors
.add
new Error("Deserialization Error: Unknown reference to id #{id}")
257 return cache
.object_for
(id
)
260 if kind
== kind_object_new
then
262 var id
= stream
.read_int64
263 if stream
.last_error
!= null then return null
265 if cache
.has_id
(id
) then
266 errors
.add
new Error("Deserialization Error: Duplicated use of reference #{id}")
270 var class_name
= stream
.read_string
272 if stream
.last_error
!= null then return null
274 # Use the validity of the `class_name` as heuristic to detect invalid data
275 if not class_name
.is_valid_id
then
276 errors
.add
new Error("Deserialization Error: got an invalid class name '{class_name}'")
280 # Prepare opening a new object
282 unclaimed_attributes
.push
new HashMap[String, nullable Object]
284 var value
= deserialize_class
(class_name
)
286 # Check for the attributes end marker
288 var next_byte
= stream
.read_byte
289 if next_byte
== new_object_end
then break
291 # Fetch an additional attribute, even if it isn't expected
292 deserialize_next_attribute
(next_byte
)
296 unclaimed_attributes
.pop
297 just_opened_id
= null
302 errors
.add
new Error("Deserialization Error: Unknown binary object kind `{kind}`")
303 # TODO fatal error and break even on keep_going
307 redef fun deserialize
311 var value
= deserialize_next_object
313 var error
= stream
.last_error
314 if error
!= null then
327 # Is `self` a valid identifier for a Nit class or property?
328 private fun is_valid_id
: Bool
330 if trim
.is_empty
then return false
333 if not (c
.is_letter
or c
.is_numeric
or c
== '[' or c
== ']' or
334 c
== ' ' or c
== ',' or c
== '_') then return false
342 # Per class serialization behavior
344 redef class Serializable
345 # Write the binary serialization header
347 # The header for a normal object is:
348 # 1. The kind of object on 8 bits, `0x01` for a new object.
349 # 2. The id of this object so it is not serialized more than once.
350 # 3. The name of the object type as a null terminated string.
351 private fun serialize_header_to_binary
(v
: BinarySerializer)
353 var id
= v
.cache
.new_id_for
(self)
354 v
.stream
.write_byte kind_object_new
# is object intro
355 v
.stream
.write_int64 id
356 v
.stream
.write_string class_name
359 # Write a normal object to binary
360 private fun serialize_to_binary
(v
: BinarySerializer)
362 serialize_header_to_binary v
364 v
.stream
.write_byte new_object_end
369 redef fun serialize_to_binary
(v
)
371 v
.stream
.write_byte kind_int
372 v
.stream
.write_int64
self
377 redef fun serialize_to_binary
(v
)
379 v
.stream
.write_byte kind_float
380 v
.stream
.write_double
self
385 redef fun serialize_to_binary
(v
)
387 v
.stream
.write_byte kind_bool
388 v
.stream
.write_bool
self
393 redef fun serialize_to_binary
(v
)
395 v
.stream
.write_byte kind_char
396 for i
in bytes
do v
.stream
.write_byte i
401 redef fun serialize_to_binary
(v
)
403 v
.stream
.write_byte kind_string
404 v
.stream
.write_block
self
408 redef class NativeString
409 redef fun serialize_to_binary
(v
)
411 v
.stream
.write_byte kind_native_string
412 v
.stream
.write_block to_s
416 redef class SimpleCollection[E
]
418 redef fun serialize_to_binary
(v
)
420 serialize_header_to_binary v
422 v
.stream
.write_string
"items"
423 v
.serialize_flat_array
self
425 v
.stream
.write_byte new_object_end
428 redef init from_deserializer
(v
)
430 # Give a chance to other engines, and defs
433 if v
isa BinaryDeserializer then
434 v
.notify_of_creation
self
437 var items
= v
.deserialize_attribute
("items")
438 assert items
isa Array[nullable Object]
440 assert item
isa E
else
441 var item_type
= "null"
442 if item
!= null then item_type
= item
.class_name
444 v
.errors
.add
new Error("Deserialization Error: invalid type '{item_type}' for the collection '{class_name}'")
454 redef class Map[K
, V
]
455 redef fun serialize_to_binary
(v
)
457 serialize_header_to_binary v
461 v
.stream
.write_string
"keys"
462 v
.serialize_flat_array keys
464 v
.stream
.write_string
"values"
465 v
.serialize_flat_array values
467 v
.stream
.write_byte new_object_end
470 # Instantiate a new `Array` from its serialized representation.
471 redef init from_deserializer
(v
)
473 # Give a chance to other engines, and defs
476 if v
isa BinaryDeserializer then
477 v
.notify_of_creation
self
481 var keys
= v
.deserialize_attribute
("keys")
482 var values
= v
.deserialize_attribute
("values")
483 assert keys
isa Array[nullable Object]
484 assert values
isa Array[nullable Object]
486 for i
in keys
.length
.times
do
488 var value
= values
[i
]
490 if not key
isa K
then
491 var item_type
= "null"
492 if key
!= null then item_type
= key
.class_name
494 v
.errors
.add
new Error("Deserialization Error: Invalid key type '{item_type}' for '{class_name}'")
498 if not value
isa V
then
499 var item_type
= "null"
500 if value
!= null then item_type
= value
.class_name
502 v
.errors
.add
new Error("Deserialization Error: Invalid value type '{item_type}' for '{class_name}'")