From a5e98961205c653adc0a5fa18ea65af57397d9fb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Alexis=20Laferri=C3=A8re?= Date: Fri, 4 May 2018 10:54:20 -0400 Subject: [PATCH] binary: remove the serialization service, clients should use msgpack MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Alexis Laferrière --- lib/binary/serialization.nit | 514 ------------------------ tests/sav/test_binary_deserialization.res | 22 - tests/sav/test_binary_deserialization_alt1.res | 42 -- tests/sav/test_binary_deserialization_alt2.res | 1 - tests/test_binary_deserialization.nit | 61 --- 5 files changed, 640 deletions(-) delete mode 100644 lib/binary/serialization.nit delete mode 100644 tests/sav/test_binary_deserialization.res delete mode 100644 tests/sav/test_binary_deserialization_alt1.res delete mode 100644 tests/sav/test_binary_deserialization_alt2.res delete mode 100644 tests/test_binary_deserialization.nit diff --git a/lib/binary/serialization.nit b/lib/binary/serialization.nit deleted file mode 100644 index af1b1b4..0000000 --- a/lib/binary/serialization.nit +++ /dev/null @@ -1,514 +0,0 @@ -# This file is part of NIT ( http://www.nitlanguage.org ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Serialize and deserialize Nit objects to binary streams -# -# The serialized data format uses a dictionary structure similar to BSON: -# -# ~~~raw -# object = 0x01 # null -# | 0x02 id attributes # New object -# | 0x03 id # Ref to object -# | 0x04 int64 # Int -# | 0x05 int8 # Bool (int8 != 0) -# | 0x06 utf8 byte sequence # Char -# | 0x07 double(64 bits) # Float -# | 0x08 block # String -# | 0x09 block # CString -# | 0x0A flat_array; # Array[nullable Object] -# -# block = int64 int8*; -# cstring = int8* 0x00; -# id = int64; -# -# attributes = attribute* 0x00; -# attribute = cstring object; -# ~~~ -module serialization - -import ::serialization::caching -private import ::serialization::engine_tools -import binary -import more_collections - -# --- -# Special bytes, marking the kind of objects in the stream and the end on an object - -private fun kind_null: Byte do return 0x01u8 -private fun kind_object_new: Byte do return 0x02u8 -private fun kind_object_ref: Byte do return 0x03u8 -private fun kind_int: Byte do return 0x04u8 -private fun kind_bool: Byte do return 0x05u8 -private fun kind_char: Byte do return 0x06u8 -private fun kind_float: Byte do return 0x07u8 -private fun kind_string: Byte do return 0x08u8 -private fun kind_c_string: Byte do return 0x09u8 -private fun kind_flat_array: Byte do return 0x0Au8 - -private fun new_object_end: Byte do return 0x00u8 - -#--- -# Engines - -# Writes Nit objects to the binary `stream` -# -# The output can be deserialized with `BinaryDeserializer`. -class BinarySerializer - super CachingSerializer - - # Target writing stream - var stream: Writer is writable - - redef var current_object = null - - redef fun serialize(object) - do - if object == null then - stream.write_byte kind_null - else serialize_reference(object) - end - - redef fun serialize_attribute(name, value) - do - stream.write_string name - super - end - - redef fun serialize_reference(object) - do - if cache.has_object(object) then - # if already serialized, add local reference - var id = cache.id_for(object) - stream.write_byte kind_object_ref - stream.write_int64 id - else - # serialize here - var last_object = current_object - current_object = object - object.serialize_to_binary self - current_object = last_object - end - end - - # Write `collection` as a simple list of objects - private fun serialize_flat_array(collection: Collection[nullable Object]) - do - stream.write_byte kind_flat_array - stream.write_int64 collection.length - for e in collection do - if not try_to_serialize(e) then - assert e != null - warn "Element of {collection} is not serializable, it is a {e}" - serialize null - end - end - end -end - -# Deserialize Nit objects from a binary `stream` -# -# Used with `BinarySerializer`. -class BinaryDeserializer - super CachingDeserializer - - # Source `Reader` stream - var stream: Reader - - # Last encountered object reference id. - # - # See `cache.received`. - private var just_opened_id: nullable Int = null - - # Tree of attributes, deserialized but not yet claimed - private var unclaimed_attributes = new UnrolledList[HashMap[String, nullable Object]] - - # Buffer for one char - private var char_buf: CString is lazy do return new CString(4) - - # Read and deserialize the next attribute name and value - # - # A `peeked_char` can suffix the next attribute name. - # - # Returns `null` on error. - private fun deserialize_next_attribute(peeked_char: nullable Byte): - nullable Couple[String, nullable Object] - do - # Try the next attribute - var next_attribute_name = stream.read_string - var next_object = deserialize_next_object - - if stream.last_error != null then return null - - if peeked_char != null then - # Replace a char peeked to find an object end - next_attribute_name = "{peeked_char}{next_attribute_name}" - end - - return new Couple[String, nullable Object](next_attribute_name, next_object) - end - - redef fun deserialize_attribute(name, static_type) - do - if unclaimed_attributes.last.keys.has(name) then - # Pick in already deserialized attributes - var value = unclaimed_attributes.last[name] - unclaimed_attributes.last.keys.remove(name) - return value - end - - # Read attributes until we find the wanted one named `name` - loop - var next = deserialize_next_attribute - if next == null then - # Error was already logged - return null - end - - var next_attribute_name = next.first - var next_object = next.second - - # Got the wanted object - if next_attribute_name == name then return next_object - - # An invalid attribute name is an heuristic for invalid data. - # Hitting an object end marker will result in an empty string. - if not next_attribute_name.is_valid_id then - - var error - if next_attribute_name.is_empty then - # Reached the end of the object - error = new Error("Deserialization Error: Attributes '{name}' not in stream.") - else - error = new Error("Deserialization Error: Got an invalid attribute name '{next_attribute_name}', expected '{name}'") - # TODO this is invalid data, break even on keep_going - end - errors.add error - return null - end - - # It's not the next attribute, put it aside - unclaimed_attributes.last[next_attribute_name] = next_object - end - end - - redef fun notify_of_creation(new_object) - do - var id = just_opened_id - if id == null then return - cache[id] = new_object - end - - # Convert from simple Json object to Nit object - private fun deserialize_next_object: nullable Object - do - var kindi = stream.read_byte - assert kindi >= 0 else - # TODO break even on keep_going - return null - end - var kind = kindi.to_b - - # After this point, all stream reading errors are caught later - - if kind == kind_null then return null - if kind == kind_int then return stream.read_int64 - if kind == kind_bool then return stream.read_bool - if kind == kind_float then return stream.read_double - if kind == kind_char then - var bf = char_buf - var b = stream.read_byte - if b < 0 then return '�' - var ln = b.to_b.u8len - bf[0] = b.to_b - for i in [1 .. ln[ do - b = stream.read_byte - if b < 0 then return '�' - bf[i] = b.to_b - end - return bf.to_s_unsafe(ln, copy=false)[0] - end - if kind == kind_string then return stream.read_block - if kind == kind_c_string then return stream.read_block.to_cstring - - if kind == kind_flat_array then - # An array - var length = stream.read_int64 - var array = new Array[nullable Object] - for i in length.times do - array.add deserialize_next_object - end - return array - end - - if kind == kind_object_ref then - # A reference - var id = stream.read_int64 - if stream.last_error != null then return null - - if not cache.has_id(id) then - errors.add new Error("Deserialization Error: Unknown reference to id #{id}") - return null - end - return cache.object_for(id) - end - - if kind == kind_object_new then - # A new object - var id = stream.read_int64 - if stream.last_error != null then return null - - if cache.has_id(id) then - errors.add new Error("Deserialization Error: Duplicated use of reference #{id}") - return null - end - - var class_name = stream.read_string - - if stream.last_error != null then return null - - # Use the validity of the `class_name` as heuristic to detect invalid data - if not class_name.is_valid_id then - errors.add new Error("Deserialization Error: got an invalid class name '{class_name}'") - return null - end - - # Prepare opening a new object - just_opened_id = id - unclaimed_attributes.push new HashMap[String, nullable Object] - - var value = deserialize_class(class_name) - - # Check for the attributes end marker - loop - var next_byte = stream.read_byte.to_b - if next_byte == new_object_end then break - - # Fetch an additional attribute, even if it isn't expected - deserialize_next_attribute(next_byte) - end - - # Close object - unclaimed_attributes.pop - just_opened_id = null - - return value - end - - errors.add new Error("Deserialization Error: Unknown binary object kind `{kind}`") - # TODO fatal error and break even on keep_going - return null - end - - redef fun deserialize(static_type) - do - errors.clear - - var value = deserialize_next_object - - var error = stream.last_error - if error != null then - errors.add error - return true - end - - return value - end -end - -# --- -# Services - -redef class Text - # Is `self` a valid identifier for a Nit class or property? - private fun is_valid_id: Bool - do - if trim.is_empty then return false - - for c in chars do - if not (c.is_letter or c.is_numeric or c == '[' or c == ']' or - c == ' ' or c == ',' or c == '_') then return false - end - - return true - end - - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_string - v.stream.write_block to_s - end -end - -# --- -# Per class serialization behavior - -redef class Serializable - # Write the binary serialization header - # - # The header for a normal object is: - # 1. The kind of object on 8 bits, `0x01` for a new object. - # 2. The id of this object so it is not serialized more than once. - # 3. The name of the object type as a null terminated string. - private fun serialize_header_to_binary(v: BinarySerializer) - do - var id = v.cache.new_id_for(self) - v.stream.write_byte kind_object_new # is object intro - v.stream.write_int64 id - v.stream.write_string class_name - end - - # Write a normal object to binary - private fun serialize_to_binary(v: BinarySerializer) - do - serialize_header_to_binary v - v.serialize_core self - v.stream.write_byte new_object_end - end -end - -redef class Int - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_int - v.stream.write_int64 self - end -end - -redef class Float - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_float - v.stream.write_double self - end -end - -redef class Bool - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_bool - v.stream.write_bool self - end -end - -redef class Char - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_char - for i in bytes do v.stream.write_byte i - end -end - -redef class CString - redef fun serialize_to_binary(v) - do - v.stream.write_byte kind_c_string - v.stream.write_block to_s - end -end - -redef class SimpleCollection[E] - - redef fun serialize_to_binary(v) - do - serialize_header_to_binary v - - v.stream.write_string "items" - v.serialize_flat_array self - - v.stream.write_byte new_object_end - end - - redef init from_deserializer(v) - do - # Give a chance to other engines, and defs - super - - if v isa BinaryDeserializer then - v.notify_of_creation self - init - - var items = v.deserialize_attribute("items") - assert items isa Array[nullable Object] - for item in items do - assert item isa E else - var item_type = "null" - if item != null then item_type = item.class_name - - v.errors.add new Error("Deserialization Error: invalid type '{item_type}' for the collection '{class_name}'") - continue - end - - add item - end - end - end -end - -redef class Map[K, V] - redef fun serialize_to_binary(v) - do - serialize_header_to_binary v - - v.serialize_core self - - v.stream.write_string "keys" - v.serialize_flat_array keys - - v.stream.write_string "values" - v.serialize_flat_array values - - v.stream.write_byte new_object_end - end - - # Instantiate a new `Array` from its serialized representation. - redef init from_deserializer(v) - do - # Give a chance to other engines, and defs - super - - if v isa BinaryDeserializer then - v.notify_of_creation self - - init - - var keys = v.deserialize_attribute("keys") - var values = v.deserialize_attribute("values") - assert keys isa Array[nullable Object] - assert values isa Array[nullable Object] - - for i in keys.length.times do - var key = keys[i] - var value = values[i] - - if not key isa K then - var item_type = "null" - if key != null then item_type = key.class_name - - v.errors.add new Error("Deserialization Error: Invalid key type '{item_type}' for '{class_name}'") - continue - end - - if not value isa V then - var item_type = "null" - if value != null then item_type = value.class_name - - v.errors.add new Error("Deserialization Error: Invalid value type '{item_type}' for '{class_name}'") - continue - end - - self[key] = value - end - end - end -end diff --git a/tests/sav/test_binary_deserialization.res b/tests/sav/test_binary_deserialization.res deleted file mode 100644 index 577ab47..0000000 --- a/tests/sav/test_binary_deserialization.res +++ /dev/null @@ -1,22 +0,0 @@ -# Src: - -# Dst: - - -# Src: - 1111 qwer> -# Dst: - 1111 qwer> - -# Src: - 1111 qwer>> -# Dst: - 1111 qwer>> - -# Src: - -<- false p4ssw0rd> 1111 f" \/> true> -# Dst: - -<- false p4ssw0rd> 1111 f" \/> true> - diff --git a/tests/sav/test_binary_deserialization_alt1.res b/tests/sav/test_binary_deserialization_alt1.res deleted file mode 100644 index 46a95ec..0000000 --- a/tests/sav/test_binary_deserialization_alt1.res +++ /dev/null @@ -1,42 +0,0 @@ -# Src: - -# Dst: - - -# Src: - 1111 qwer> -# Dst: - 1111 qwer> - -# Src: - 1111 qwer>> -# Dst: - 1111 qwer>> - -# Src: - -<- false p4ssw0rd> 1111 f" \/> true> -# Dst: - -<- false p4ssw0rd> 1111 f" \/> true> - -# Src: - -# Dst: - - -# Src: - -# Dst: - - -# Src: - -# Dst: - - -# Src: - -# Dst: - - diff --git a/tests/sav/test_binary_deserialization_alt2.res b/tests/sav/test_binary_deserialization_alt2.res deleted file mode 100644 index 474fef8..0000000 --- a/tests/sav/test_binary_deserialization_alt2.res +++ /dev/null @@ -1 +0,0 @@ -Deserialization Error: Doesn't know how to deserialize class "NoSerializeClass" diff --git a/tests/test_binary_deserialization.nit b/tests/test_binary_deserialization.nit deleted file mode 100644 index c6bcd0a..0000000 --- a/tests/test_binary_deserialization.nit +++ /dev/null @@ -1,61 +0,0 @@ -# This file is part of NIT ( http://www.nitlanguage.org ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import test_deserialization -import binary::serialization -#alt1# import test_deserialization_serial - -class NoSerializeClass - super Serializable - - var some_attribute: String -end - -var entities = new TestEntities - -var tests = entities.without_generics#alt1##alt2# -#alt1#var tests = entities.with_generics -#alt2#var tests = [new NoSerializeClass("will not deserialize")] - -var dir = "out/test_binary_deserialization" -if not dir.file_exists then dir.mkdir - -var path = dir / "alt0"#alt1##alt2# -#alt1#var path = dir / "alt1" -#alt2#var path = dir / "alt2" - -var writer = new FileWriter.open(path) -var serializer = new BinarySerializer(writer) -for o in tests do - serializer.serialize o -end -writer.close - -var reader = new FileReader.open(path) -var deserializer = new BinaryDeserializer(reader) -for o in tests do - var dst = deserializer.deserialize - - if deserializer.errors.not_empty then - print deserializer.errors.join(", ") - end - - if dst != null then - assert o.is_same_type(dst) - - print "# Src:\n{o}" - print "# Dst:\n{dst}\n" - end -end -reader.close -- 1.7.9.5