1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Serialize and deserialize Nit objects to binary streams
17 # The serialized data format uses a dictionary structure similar to BSON:
20 # object = 0x01 # null
21 # | 0x02 id attributes # New object
22 # | 0x03 id # Ref to object
24 # | 0x05 int8 # Bool (int8 != 0)
26 # | 0x07 double(64 bits) # Float
27 # | 0x08 block # String
28 # | 0x09 block # NativeString
29 # | 0x0A flat_array; # Array[nullable Object]
31 # block = int64 int8*;
32 # cstring = int8* 0x00;
35 # attributes = attribute* 0x00;
36 # attribute = cstring object;
40 import ::serialization
::caching
41 private import ::serialization
::engine_tools
43 import more_collections
46 # Special bytes, marking the kind of objects in the stream and the end on an object
48 private fun kind_null
: Byte do return 0x01u
8
49 private fun kind_object_new
: Byte do return 0x02u
8
50 private fun kind_object_ref
: Byte do return 0x03u
8
51 private fun kind_int
: Byte do return 0x04u
8
52 private fun kind_bool
: Byte do return 0x05u
8
53 private fun kind_char
: Byte do return 0x06u
8
54 private fun kind_float
: Byte do return 0x07u
8
55 private fun kind_string
: Byte do return 0x08u
8
56 private fun kind_native_string
: Byte do return 0x09u
8
57 private fun kind_flat_array
: Byte do return 0x0Au
8
59 private fun new_object_end
: Byte do return 0x00u
8
64 # Writes Nit objects to the binary `stream`
66 # The output can be deserialized with `BinaryDeserializer`.
67 class BinarySerializer
68 super CachingSerializer
70 # Target writing stream
71 var stream
: Writer is writable
73 redef fun serialize
(object
)
75 if object
== null then
76 stream
.write_byte kind_null
77 else serialize_reference
(object
)
80 redef fun serialize_attribute
(name
, value
)
82 stream
.write_string name
86 redef fun serialize_reference
(object
)
88 if cache
.has_object
(object
) then
89 # if already serialized, add local reference
90 var id
= cache
.id_for
(object
)
91 stream
.write_byte kind_object_ref
95 object
.serialize_to_binary
self
99 # Write `collection` as a simple list of objects
100 private fun serialize_flat_array
(collection
: Collection[nullable Object])
102 stream
.write_byte kind_flat_array
103 stream
.write_int64 collection
.length
104 for e
in collection
do
105 if not try_to_serialize
(e
) then
107 warn
"Element of {collection} is not serializable, it is a {e}"
114 # Deserialize Nit objects from a binary `stream`
116 # Used with `BinarySerializer`.
117 class BinaryDeserializer
118 super CachingDeserializer
120 # Source `Reader` stream
123 # Last encountered object reference id.
125 # See `cache.received`.
126 private var just_opened_id
: nullable Int = null
128 # Tree of attributes, deserialized but not yet claimed
129 private var unclaimed_attributes
= new UnrolledList[HashMap[String, nullable Object]]
131 # Read and deserialize the next attribute name and value
133 # A `peeked_char` can suffix the next attribute name.
135 # Returns `null` on error.
136 private fun deserialize_next_attribute
(peeked_char
: nullable Byte):
137 nullable Couple[String, nullable Object]
139 # Try the next attribute
140 var next_attribute_name
= stream
.read_string
141 var next_object
= deserialize_next_object
143 if stream
.last_error
!= null then return null
145 if peeked_char
!= null then
146 # Replace a char peeked to find an object end
147 next_attribute_name
= "{peeked_char}{next_attribute_name}"
150 return new Couple[String, nullable Object](next_attribute_name
, next_object
)
153 redef fun deserialize_attribute
(name
)
155 if unclaimed_attributes
.last
.keys
.has
(name
) then
156 # Pick in already deserialized attributes
157 var value
= unclaimed_attributes
.last
[name
]
158 unclaimed_attributes
.last
.keys
.remove
(name
)
162 # Read attributes until we find the wanted one named `name`
164 var next
= deserialize_next_attribute
166 # Error was already logged
170 var next_attribute_name
= next
.first
171 var next_object
= next
.second
173 # Got the wanted object
174 if next_attribute_name
== name
then return next_object
176 # An invalid attribute name is an heuristic for invalid data.
177 # Hitting an object end marker will result in an empty string.
178 assert next_attribute_name
.is_valid_id
else
181 if next_attribute_name
.is_empty
then
182 # Reached the end of the object
183 error
= new Error("Deserialization Error: Attributes '{name}' not in stream.")
185 error
= new Error("Deserialization Error: Got an invalid attribute name '{next_attribute_name}', expected '{name}'")
186 # TODO this is invalid data, break even on keep_going
192 # It's not the next attribute, put it aside
193 unclaimed_attributes
.last
[next_attribute_name
] = next_object
197 redef fun notify_of_creation
(new_object
)
199 var id
= just_opened_id
200 if id
== null then return
201 cache
[id
] = new_object
204 # Convert from simple Json object to Nit object
205 private fun deserialize_next_object
: nullable Object
207 var kind
= stream
.read_byte
208 assert kind
isa Byte else
209 # TODO break even on keep_going
213 # After this point, all stream reading errors are caught later
215 if kind
== kind_null
then return null
216 if kind
== kind_int
then return stream
.read_int64
217 if kind
== kind_bool
then return stream
.read_bool
218 if kind
== kind_float
then return stream
.read_double
219 if kind
== kind_char
then
220 var b
= stream
.read_byte
221 if b
== null then return 0
224 if kind
== kind_string
then return stream
.read_block
225 if kind
== kind_native_string
then return stream
.read_block
.to_cstring
227 if kind
== kind_flat_array
then
229 var length
= stream
.read_int64
230 var array
= new Array[nullable Object]
231 for i
in length
.times
do
232 array
.add deserialize_next_object
237 if kind
== kind_object_ref
then
239 var id
= stream
.read_int64
240 if stream
.last_error
!= null then return null
242 if not cache
.has_id
(id
) then
243 errors
.add
new Error("Deserialization Error: Unknown reference to id #{id}")
246 return cache
.object_for
(id
)
249 if kind
== kind_object_new
then
251 var id
= stream
.read_int64
252 if stream
.last_error
!= null then return null
254 if cache
.has_id
(id
) then
255 errors
.add
new Error("Deserialization Error: Duplicated use of reference #{id}")
259 var class_name
= stream
.read_string
261 if stream
.last_error
!= null then return null
263 # Use the validity of the `class_name` as heuristic to detect invalid data
264 if not class_name
.is_valid_id
then
265 errors
.add
new Error("Deserialization Error: got an invalid class name '{class_name}'")
269 # Prepare opening a new object
271 unclaimed_attributes
.push
new HashMap[String, nullable Object]
273 var value
= deserialize_class
(class_name
)
275 # Check for the attributes end marker
277 var next_byte
= stream
.read_byte
278 if next_byte
== new_object_end
then break
280 # Fetch an additional attribute, even if it isn't expected
281 deserialize_next_attribute
(next_byte
)
285 unclaimed_attributes
.pop
286 just_opened_id
= null
291 errors
.add
new Error("Deserialization Error: Unknown binary object kind `{kind}`")
292 # TODO fatal error and break even on keep_going
296 redef fun deserialize
300 var value
= deserialize_next_object
302 var error
= stream
.last_error
303 if error
!= null then
316 # Is `self` a valid identifier for a Nit class or property?
317 private fun is_valid_id
: Bool
319 if trim
.is_empty
then return false
322 if not (c
.is_letter
or c
.is_numeric
or c
== '[' or c
== ']' or
323 c
== ' ' or c
== ',' or c
== '_') then return false
331 # Per class serialization behavior
333 redef class Serializable
334 # Write the binary serialization header
336 # The header for a normal object is:
337 # 1. The kind of object on 8 bits, `0x01` for a new object.
338 # 2. The id of this object so it is not serialized more than once.
339 # 3. The name of the object type as a null terminated string.
340 private fun serialize_header_to_binary
(v
: BinarySerializer)
342 var id
= v
.cache
.new_id_for
(self)
343 v
.stream
.write_byte kind_object_new
# is object intro
344 v
.stream
.write_int64 id
345 v
.stream
.write_string class_name
348 # Write a normal object to binary
349 private fun serialize_to_binary
(v
: BinarySerializer)
351 serialize_header_to_binary v
353 v
.stream
.write_byte new_object_end
358 redef fun serialize_to_binary
(v
)
360 v
.stream
.write_byte kind_int
361 v
.stream
.write_int64
self
366 redef fun serialize_to_binary
(v
)
368 v
.stream
.write_byte kind_float
369 v
.stream
.write_double
self
374 redef fun serialize_to_binary
(v
)
376 v
.stream
.write_byte kind_bool
377 v
.stream
.write_bool
self
382 redef fun serialize_to_binary
(v
)
384 v
.stream
.write_byte kind_char
386 v
.stream
.write_byte
self.ascii
.to_b
391 redef fun serialize_to_binary
(v
)
393 v
.stream
.write_byte kind_string
394 v
.stream
.write_block
self
398 redef class NativeString
399 redef fun serialize_to_binary
(v
)
401 v
.stream
.write_byte kind_native_string
402 v
.stream
.write_block to_s
406 redef class SimpleCollection[E
]
408 redef fun serialize_to_binary
(v
)
410 serialize_header_to_binary v
412 v
.stream
.write_string
"items"
413 v
.serialize_flat_array
self
415 v
.stream
.write_byte new_object_end
418 redef init from_deserializer
(v
)
420 # Give a chance to other engines, and defs
423 if v
isa BinaryDeserializer then
424 v
.notify_of_creation
self
427 var items
= v
.deserialize_attribute
("items")
428 assert items
isa Array[nullable Object]
430 assert item
isa E
else
431 var item_type
= "null"
432 if item
!= null then item_type
= item
.class_name
434 v
.errors
.add
new Error("Deserialization Error: invalid type '{item_type}' for the collection '{class_name}'")
444 redef class Map[K
, V
]
445 redef fun serialize_to_binary
(v
)
447 serialize_header_to_binary v
451 v
.stream
.write_string
"keys"
452 v
.serialize_flat_array keys
454 v
.stream
.write_string
"values"
455 v
.serialize_flat_array values
457 v
.stream
.write_byte new_object_end
460 # Instantiate a new `Array` from its serialized representation.
461 redef init from_deserializer
(v
)
463 # Give a chance to other engines, and defs
466 if v
isa BinaryDeserializer then
467 v
.notify_of_creation
self
471 var keys
= v
.deserialize_attribute
("keys")
472 var values
= v
.deserialize_attribute
("values")
473 assert keys
isa Array[nullable Object]
474 assert values
isa Array[nullable Object]
476 for i
in keys
.length
.times
do
478 var value
= values
[i
]
480 if not key
isa K
then
481 var item_type
= "null"
482 if key
!= null then item_type
= key
.class_name
484 v
.errors
.add
new Error("Deserialization Error: Invalid key type '{item_type}' for '{class_name}'")
488 if not value
isa V
then
489 var item_type
= "null"
490 if value
!= null then item_type
= value
.class_name
492 v
.errors
.add
new Error("Deserialization Error: Invalid value type '{item_type}' for '{class_name}'")