1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Serialize and deserialize Nit objects to binary streams
17 # The serialized data format uses a dictionary structure similar to BSON:
20 # object = 0x01 # null
21 # | 0x02 id attributes # New object
22 # | 0x03 id # Ref to object
24 # | 0x05 int8 # Bool (int8 != 0)
25 # | 0x06 utf8 byte sequence # Char
26 # | 0x07 double(64 bits) # Float
27 # | 0x08 block # String
28 # | 0x09 block # CString
29 # | 0x0A flat_array; # Array[nullable Object]
31 # block = int64 int8*;
32 # cstring = int8* 0x00;
35 # attributes = attribute* 0x00;
36 # attribute = cstring object;
40 import ::serialization
::caching
41 private import ::serialization
::engine_tools
43 import more_collections
46 # Special bytes, marking the kind of objects in the stream and the end on an object
48 private fun kind_null
: Byte do return 0x01u
8
49 private fun kind_object_new
: Byte do return 0x02u
8
50 private fun kind_object_ref
: Byte do return 0x03u
8
51 private fun kind_int
: Byte do return 0x04u
8
52 private fun kind_bool
: Byte do return 0x05u
8
53 private fun kind_char
: Byte do return 0x06u
8
54 private fun kind_float
: Byte do return 0x07u
8
55 private fun kind_string
: Byte do return 0x08u
8
56 private fun kind_c_string
: Byte do return 0x09u
8
57 private fun kind_flat_array
: Byte do return 0x0Au
8
59 private fun new_object_end
: Byte do return 0x00u
8
64 # Writes Nit objects to the binary `stream`
66 # The output can be deserialized with `BinaryDeserializer`.
67 class BinarySerializer
68 super CachingSerializer
70 # Target writing stream
71 var stream
: Writer is writable
73 redef var current_object
= null
75 redef fun serialize
(object
)
77 if object
== null then
78 stream
.write_byte kind_null
79 else serialize_reference
(object
)
82 redef fun serialize_attribute
(name
, value
)
84 stream
.write_string name
88 redef fun serialize_reference
(object
)
90 if cache
.has_object
(object
) then
91 # if already serialized, add local reference
92 var id
= cache
.id_for
(object
)
93 stream
.write_byte kind_object_ref
97 var last_object
= current_object
98 current_object
= object
99 object
.serialize_to_binary
self
100 current_object
= last_object
104 # Write `collection` as a simple list of objects
105 private fun serialize_flat_array
(collection
: Collection[nullable Object])
107 stream
.write_byte kind_flat_array
108 stream
.write_int64 collection
.length
109 for e
in collection
do
110 if not try_to_serialize
(e
) then
112 warn
"Element of {collection} is not serializable, it is a {e}"
119 # Deserialize Nit objects from a binary `stream`
121 # Used with `BinarySerializer`.
122 class BinaryDeserializer
123 super CachingDeserializer
125 # Source `Reader` stream
128 # Last encountered object reference id.
130 # See `cache.received`.
131 private var just_opened_id
: nullable Int = null
133 # Tree of attributes, deserialized but not yet claimed
134 private var unclaimed_attributes
= new UnrolledList[HashMap[String, nullable Object]]
136 # Buffer for one char
137 private var char_buf
: CString is lazy
do return new CString(4)
139 # Read and deserialize the next attribute name and value
141 # A `peeked_char` can suffix the next attribute name.
143 # Returns `null` on error.
144 private fun deserialize_next_attribute
(peeked_char
: nullable Byte):
145 nullable Couple[String, nullable Object]
147 # Try the next attribute
148 var next_attribute_name
= stream
.read_string
149 var next_object
= deserialize_next_object
151 if stream
.last_error
!= null then return null
153 if peeked_char
!= null then
154 # Replace a char peeked to find an object end
155 next_attribute_name
= "{peeked_char}{next_attribute_name}"
158 return new Couple[String, nullable Object](next_attribute_name
, next_object
)
161 redef fun deserialize_attribute
(name
, static_type
)
163 if unclaimed_attributes
.last
.keys
.has
(name
) then
164 # Pick in already deserialized attributes
165 var value
= unclaimed_attributes
.last
[name
]
166 unclaimed_attributes
.last
.keys
.remove
(name
)
170 # Read attributes until we find the wanted one named `name`
172 var next
= deserialize_next_attribute
174 # Error was already logged
178 var next_attribute_name
= next
.first
179 var next_object
= next
.second
181 # Got the wanted object
182 if next_attribute_name
== name
then return next_object
184 # An invalid attribute name is an heuristic for invalid data.
185 # Hitting an object end marker will result in an empty string.
186 if not next_attribute_name
.is_valid_id
then
189 if next_attribute_name
.is_empty
then
190 # Reached the end of the object
191 error
= new Error("Deserialization Error: Attributes '{name}' not in stream.")
193 error
= new Error("Deserialization Error: Got an invalid attribute name '{next_attribute_name}', expected '{name}'")
194 # TODO this is invalid data, break even on keep_going
200 # It's not the next attribute, put it aside
201 unclaimed_attributes
.last
[next_attribute_name
] = next_object
205 redef fun notify_of_creation
(new_object
)
207 var id
= just_opened_id
208 if id
== null then return
209 cache
[id
] = new_object
212 # Convert from simple Json object to Nit object
213 private fun deserialize_next_object
: nullable Object
215 var kind
= stream
.read_byte
216 assert kind
isa Byte else
217 # TODO break even on keep_going
221 # After this point, all stream reading errors are caught later
223 if kind
== kind_null
then return null
224 if kind
== kind_int
then return stream
.read_int64
225 if kind
== kind_bool
then return stream
.read_bool
226 if kind
== kind_float
then return stream
.read_double
227 if kind
== kind_char
then
229 var b
= stream
.read_byte
230 if b
== null then return '�'
233 for i
in [1 .. ln
[ do
235 if b
== null then return '�'
238 return bf
.to_s_unsafe
(ln
, copy
=false)[0]
240 if kind
== kind_string
then return stream
.read_block
241 if kind
== kind_c_string
then return stream
.read_block
.to_cstring
243 if kind
== kind_flat_array
then
245 var length
= stream
.read_int64
246 var array
= new Array[nullable Object]
247 for i
in length
.times
do
248 array
.add deserialize_next_object
253 if kind
== kind_object_ref
then
255 var id
= stream
.read_int64
256 if stream
.last_error
!= null then return null
258 if not cache
.has_id
(id
) then
259 errors
.add
new Error("Deserialization Error: Unknown reference to id #{id}")
262 return cache
.object_for
(id
)
265 if kind
== kind_object_new
then
267 var id
= stream
.read_int64
268 if stream
.last_error
!= null then return null
270 if cache
.has_id
(id
) then
271 errors
.add
new Error("Deserialization Error: Duplicated use of reference #{id}")
275 var class_name
= stream
.read_string
277 if stream
.last_error
!= null then return null
279 # Use the validity of the `class_name` as heuristic to detect invalid data
280 if not class_name
.is_valid_id
then
281 errors
.add
new Error("Deserialization Error: got an invalid class name '{class_name}'")
285 # Prepare opening a new object
287 unclaimed_attributes
.push
new HashMap[String, nullable Object]
289 var value
= deserialize_class
(class_name
)
291 # Check for the attributes end marker
293 var next_byte
= stream
.read_byte
294 if next_byte
== new_object_end
then break
296 # Fetch an additional attribute, even if it isn't expected
297 deserialize_next_attribute
(next_byte
)
301 unclaimed_attributes
.pop
302 just_opened_id
= null
307 errors
.add
new Error("Deserialization Error: Unknown binary object kind `{kind}`")
308 # TODO fatal error and break even on keep_going
312 redef fun deserialize
(static_type
)
316 var value
= deserialize_next_object
318 var error
= stream
.last_error
319 if error
!= null then
332 # Is `self` a valid identifier for a Nit class or property?
333 private fun is_valid_id
: Bool
335 if trim
.is_empty
then return false
338 if not (c
.is_letter
or c
.is_numeric
or c
== '[' or c
== ']' or
339 c
== ' ' or c
== ',' or c
== '_') then return false
345 redef fun serialize_to_binary
(v
)
347 v
.stream
.write_byte kind_string
348 v
.stream
.write_block to_s
353 # Per class serialization behavior
355 redef class Serializable
356 # Write the binary serialization header
358 # The header for a normal object is:
359 # 1. The kind of object on 8 bits, `0x01` for a new object.
360 # 2. The id of this object so it is not serialized more than once.
361 # 3. The name of the object type as a null terminated string.
362 private fun serialize_header_to_binary
(v
: BinarySerializer)
364 var id
= v
.cache
.new_id_for
(self)
365 v
.stream
.write_byte kind_object_new
# is object intro
366 v
.stream
.write_int64 id
367 v
.stream
.write_string class_name
370 # Write a normal object to binary
371 private fun serialize_to_binary
(v
: BinarySerializer)
373 serialize_header_to_binary v
374 v
.serialize_core
self
375 v
.stream
.write_byte new_object_end
380 redef fun serialize_to_binary
(v
)
382 v
.stream
.write_byte kind_int
383 v
.stream
.write_int64
self
388 redef fun serialize_to_binary
(v
)
390 v
.stream
.write_byte kind_float
391 v
.stream
.write_double
self
396 redef fun serialize_to_binary
(v
)
398 v
.stream
.write_byte kind_bool
399 v
.stream
.write_bool
self
404 redef fun serialize_to_binary
(v
)
406 v
.stream
.write_byte kind_char
407 for i
in bytes
do v
.stream
.write_byte i
412 redef fun serialize_to_binary
(v
)
414 v
.stream
.write_byte kind_c_string
415 v
.stream
.write_block to_s
419 redef class SimpleCollection[E
]
421 redef fun serialize_to_binary
(v
)
423 serialize_header_to_binary v
425 v
.stream
.write_string
"items"
426 v
.serialize_flat_array
self
428 v
.stream
.write_byte new_object_end
431 redef init from_deserializer
(v
)
433 # Give a chance to other engines, and defs
436 if v
isa BinaryDeserializer then
437 v
.notify_of_creation
self
440 var items
= v
.deserialize_attribute
("items")
441 assert items
isa Array[nullable Object]
443 assert item
isa E
else
444 var item_type
= "null"
445 if item
!= null then item_type
= item
.class_name
447 v
.errors
.add
new Error("Deserialization Error: invalid type '{item_type}' for the collection '{class_name}'")
457 redef class Map[K
, V
]
458 redef fun serialize_to_binary
(v
)
460 serialize_header_to_binary v
462 v
.serialize_core
self
464 v
.stream
.write_string
"keys"
465 v
.serialize_flat_array keys
467 v
.stream
.write_string
"values"
468 v
.serialize_flat_array values
470 v
.stream
.write_byte new_object_end
473 # Instantiate a new `Array` from its serialized representation.
474 redef init from_deserializer
(v
)
476 # Give a chance to other engines, and defs
479 if v
isa BinaryDeserializer then
480 v
.notify_of_creation
self
484 var keys
= v
.deserialize_attribute
("keys")
485 var values
= v
.deserialize_attribute
("values")
486 assert keys
isa Array[nullable Object]
487 assert values
isa Array[nullable Object]
489 for i
in keys
.length
.times
do
491 var value
= values
[i
]
493 if not key
isa K
then
494 var item_type
= "null"
495 if key
!= null then item_type
= key
.class_name
497 v
.errors
.add
new Error("Deserialization Error: Invalid key type '{item_type}' for '{class_name}'")
501 if not value
isa V
then
502 var item_type
= "null"
503 if value
!= null then item_type
= value
.class_name
505 v
.errors
.add
new Error("Deserialization Error: Invalid value type '{item_type}' for '{class_name}'")