1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Serialize and deserialize Nit objects to binary streams
17 # The serialized data format uses a dictionary structure similar to BSON:
20 # object = 0x01 # null
21 # | 0x02 id attributes # New object
22 # | 0x03 id # Ref to object
24 # | 0x05 int8 # Bool (int8 != 0)
25 # | 0x06 utf8 byte sequence # Char
26 # | 0x07 double(64 bits) # Float
27 # | 0x08 block # String
28 # | 0x09 block # CString
29 # | 0x0A flat_array; # Array[nullable Object]
31 # block = int64 int8*;
32 # cstring = int8* 0x00;
35 # attributes = attribute* 0x00;
36 # attribute = cstring object;
40 import ::serialization
::caching
41 private import ::serialization
::engine_tools
43 import more_collections
46 # Special bytes, marking the kind of objects in the stream and the end on an object
48 private fun kind_null
: Byte do return 0x01u
8
49 private fun kind_object_new
: Byte do return 0x02u
8
50 private fun kind_object_ref
: Byte do return 0x03u
8
51 private fun kind_int
: Byte do return 0x04u
8
52 private fun kind_bool
: Byte do return 0x05u
8
53 private fun kind_char
: Byte do return 0x06u
8
54 private fun kind_float
: Byte do return 0x07u
8
55 private fun kind_string
: Byte do return 0x08u
8
56 private fun kind_c_string
: Byte do return 0x09u
8
57 private fun kind_flat_array
: Byte do return 0x0Au
8
59 private fun new_object_end
: Byte do return 0x00u
8
64 # Writes Nit objects to the binary `stream`
66 # The output can be deserialized with `BinaryDeserializer`.
67 class BinarySerializer
68 super CachingSerializer
70 # Target writing stream
71 var stream
: Writer is writable
73 redef fun serialize
(object
)
75 if object
== null then
76 stream
.write_byte kind_null
77 else serialize_reference
(object
)
80 redef fun serialize_attribute
(name
, value
)
82 stream
.write_string name
86 redef fun serialize_reference
(object
)
88 if cache
.has_object
(object
) then
89 # if already serialized, add local reference
90 var id
= cache
.id_for
(object
)
91 stream
.write_byte kind_object_ref
95 object
.serialize_to_binary
self
99 # Write `collection` as a simple list of objects
100 private fun serialize_flat_array
(collection
: Collection[nullable Object])
102 stream
.write_byte kind_flat_array
103 stream
.write_int64 collection
.length
104 for e
in collection
do
105 if not try_to_serialize
(e
) then
107 warn
"Element of {collection} is not serializable, it is a {e}"
114 # Deserialize Nit objects from a binary `stream`
116 # Used with `BinarySerializer`.
117 class BinaryDeserializer
118 super CachingDeserializer
120 # Source `Reader` stream
123 # Last encountered object reference id.
125 # See `cache.received`.
126 private var just_opened_id
: nullable Int = null
128 # Tree of attributes, deserialized but not yet claimed
129 private var unclaimed_attributes
= new UnrolledList[HashMap[String, nullable Object]]
131 # Buffer for one char
132 private var char_buf
: CString is lazy
do return new CString(4)
134 # Read and deserialize the next attribute name and value
136 # A `peeked_char` can suffix the next attribute name.
138 # Returns `null` on error.
139 private fun deserialize_next_attribute
(peeked_char
: nullable Byte):
140 nullable Couple[String, nullable Object]
142 # Try the next attribute
143 var next_attribute_name
= stream
.read_string
144 var next_object
= deserialize_next_object
146 if stream
.last_error
!= null then return null
148 if peeked_char
!= null then
149 # Replace a char peeked to find an object end
150 next_attribute_name
= "{peeked_char}{next_attribute_name}"
153 return new Couple[String, nullable Object](next_attribute_name
, next_object
)
156 redef fun deserialize_attribute
(name
, static_type
)
158 if unclaimed_attributes
.last
.keys
.has
(name
) then
159 # Pick in already deserialized attributes
160 var value
= unclaimed_attributes
.last
[name
]
161 unclaimed_attributes
.last
.keys
.remove
(name
)
165 # Read attributes until we find the wanted one named `name`
167 var next
= deserialize_next_attribute
169 # Error was already logged
173 var next_attribute_name
= next
.first
174 var next_object
= next
.second
176 # Got the wanted object
177 if next_attribute_name
== name
then return next_object
179 # An invalid attribute name is an heuristic for invalid data.
180 # Hitting an object end marker will result in an empty string.
181 if not next_attribute_name
.is_valid_id
then
184 if next_attribute_name
.is_empty
then
185 # Reached the end of the object
186 error
= new Error("Deserialization Error: Attributes '{name}' not in stream.")
188 error
= new Error("Deserialization Error: Got an invalid attribute name '{next_attribute_name}', expected '{name}'")
189 # TODO this is invalid data, break even on keep_going
195 # It's not the next attribute, put it aside
196 unclaimed_attributes
.last
[next_attribute_name
] = next_object
200 redef fun notify_of_creation
(new_object
)
202 var id
= just_opened_id
203 if id
== null then return
204 cache
[id
] = new_object
207 # Convert from simple Json object to Nit object
208 private fun deserialize_next_object
: nullable Object
210 var kind
= stream
.read_byte
211 assert kind
isa Byte else
212 # TODO break even on keep_going
216 # After this point, all stream reading errors are caught later
218 if kind
== kind_null
then return null
219 if kind
== kind_int
then return stream
.read_int64
220 if kind
== kind_bool
then return stream
.read_bool
221 if kind
== kind_float
then return stream
.read_double
222 if kind
== kind_char
then
224 var b
= stream
.read_byte
225 if b
== null then return '�'
228 for i
in [1 .. ln
[ do
230 if b
== null then return '�'
233 return bf
.to_s_unsafe
(ln
, copy
=false)[0]
235 if kind
== kind_string
then return stream
.read_block
236 if kind
== kind_c_string
then return stream
.read_block
.to_cstring
238 if kind
== kind_flat_array
then
240 var length
= stream
.read_int64
241 var array
= new Array[nullable Object]
242 for i
in length
.times
do
243 array
.add deserialize_next_object
248 if kind
== kind_object_ref
then
250 var id
= stream
.read_int64
251 if stream
.last_error
!= null then return null
253 if not cache
.has_id
(id
) then
254 errors
.add
new Error("Deserialization Error: Unknown reference to id #{id}")
257 return cache
.object_for
(id
)
260 if kind
== kind_object_new
then
262 var id
= stream
.read_int64
263 if stream
.last_error
!= null then return null
265 if cache
.has_id
(id
) then
266 errors
.add
new Error("Deserialization Error: Duplicated use of reference #{id}")
270 var class_name
= stream
.read_string
272 if stream
.last_error
!= null then return null
274 # Use the validity of the `class_name` as heuristic to detect invalid data
275 if not class_name
.is_valid_id
then
276 errors
.add
new Error("Deserialization Error: got an invalid class name '{class_name}'")
280 # Prepare opening a new object
282 unclaimed_attributes
.push
new HashMap[String, nullable Object]
284 var value
= deserialize_class
(class_name
)
286 # Check for the attributes end marker
288 var next_byte
= stream
.read_byte
289 if next_byte
== new_object_end
then break
291 # Fetch an additional attribute, even if it isn't expected
292 deserialize_next_attribute
(next_byte
)
296 unclaimed_attributes
.pop
297 just_opened_id
= null
302 errors
.add
new Error("Deserialization Error: Unknown binary object kind `{kind}`")
303 # TODO fatal error and break even on keep_going
307 redef fun deserialize
(static_type
)
311 var value
= deserialize_next_object
313 var error
= stream
.last_error
314 if error
!= null then
327 # Is `self` a valid identifier for a Nit class or property?
328 private fun is_valid_id
: Bool
330 if trim
.is_empty
then return false
333 if not (c
.is_letter
or c
.is_numeric
or c
== '[' or c
== ']' or
334 c
== ' ' or c
== ',' or c
== '_') then return false
340 redef fun serialize_to_binary
(v
)
342 v
.stream
.write_byte kind_string
343 v
.stream
.write_block to_s
348 # Per class serialization behavior
350 redef class Serializable
351 # Write the binary serialization header
353 # The header for a normal object is:
354 # 1. The kind of object on 8 bits, `0x01` for a new object.
355 # 2. The id of this object so it is not serialized more than once.
356 # 3. The name of the object type as a null terminated string.
357 private fun serialize_header_to_binary
(v
: BinarySerializer)
359 var id
= v
.cache
.new_id_for
(self)
360 v
.stream
.write_byte kind_object_new
# is object intro
361 v
.stream
.write_int64 id
362 v
.stream
.write_string class_name
365 # Write a normal object to binary
366 private fun serialize_to_binary
(v
: BinarySerializer)
368 serialize_header_to_binary v
370 v
.stream
.write_byte new_object_end
375 redef fun serialize_to_binary
(v
)
377 v
.stream
.write_byte kind_int
378 v
.stream
.write_int64
self
383 redef fun serialize_to_binary
(v
)
385 v
.stream
.write_byte kind_float
386 v
.stream
.write_double
self
391 redef fun serialize_to_binary
(v
)
393 v
.stream
.write_byte kind_bool
394 v
.stream
.write_bool
self
399 redef fun serialize_to_binary
(v
)
401 v
.stream
.write_byte kind_char
402 for i
in bytes
do v
.stream
.write_byte i
407 redef fun serialize_to_binary
(v
)
409 v
.stream
.write_byte kind_c_string
410 v
.stream
.write_block to_s
414 redef class SimpleCollection[E
]
416 redef fun serialize_to_binary
(v
)
418 serialize_header_to_binary v
420 v
.stream
.write_string
"items"
421 v
.serialize_flat_array
self
423 v
.stream
.write_byte new_object_end
426 redef init from_deserializer
(v
)
428 # Give a chance to other engines, and defs
431 if v
isa BinaryDeserializer then
432 v
.notify_of_creation
self
435 var items
= v
.deserialize_attribute
("items")
436 assert items
isa Array[nullable Object]
438 assert item
isa E
else
439 var item_type
= "null"
440 if item
!= null then item_type
= item
.class_name
442 v
.errors
.add
new Error("Deserialization Error: invalid type '{item_type}' for the collection '{class_name}'")
452 redef class Map[K
, V
]
453 redef fun serialize_to_binary
(v
)
455 serialize_header_to_binary v
459 v
.stream
.write_string
"keys"
460 v
.serialize_flat_array keys
462 v
.stream
.write_string
"values"
463 v
.serialize_flat_array values
465 v
.stream
.write_byte new_object_end
468 # Instantiate a new `Array` from its serialized representation.
469 redef init from_deserializer
(v
)
471 # Give a chance to other engines, and defs
474 if v
isa BinaryDeserializer then
475 v
.notify_of_creation
self
479 var keys
= v
.deserialize_attribute
("keys")
480 var values
= v
.deserialize_attribute
("values")
481 assert keys
isa Array[nullable Object]
482 assert values
isa Array[nullable Object]
484 for i
in keys
.length
.times
do
486 var value
= values
[i
]
488 if not key
isa K
then
489 var item_type
= "null"
490 if key
!= null then item_type
= key
.class_name
492 v
.errors
.add
new Error("Deserialization Error: Invalid key type '{item_type}' for '{class_name}'")
496 if not value
isa V
then
497 var item_type
= "null"
498 if value
!= null then item_type
= value
.class_name
500 v
.errors
.add
new Error("Deserialization Error: Invalid value type '{item_type}' for '{class_name}'")