Implement Serialize for flexbuffer::Reader (#6635)

* Implement Serialize for flexbuffer::Reader

* bump version

* Remove use of experimantal or-patterns

* Remove more use of experimantal or-patterns

Co-authored-by: Casper Neo <cneo@google.com>

rust/flexbuffers/src/buffer.rs

@@ -1,6 +1,6 @@
 use std::ops::{Deref, Range};
 
-/// The underlying buffer that is used by a flexbuffer Reader. 
+/// The underlying buffer that is used by a flexbuffer Reader.
 ///
 /// This allows for custom buffer implementations as long as they can be viewed as a &[u8].
 pub trait Buffer: Deref<Target = [u8]> + Sized {
@@ -14,12 +14,12 @@ pub trait Buffer: Deref<Target = [u8]> + Sized {
     /// A BufferString which will live at least as long as the Buffer itself.
     ///
     /// Deref's to UTF-8 `str`, and only generated from the `buffer_str` function Result.
-    type BufferString: Deref<Target = str> + Sized;
+    type BufferString: Deref<Target = str> + Sized + serde::ser::Serialize;
 
     /// This method returns an instance of type Self. This allows for lifetimes to be tracked
     /// in cases of deserialization.
     ///
-    /// It also lets custom buffers manage reference counts. 
+    /// It also lets custom buffers manage reference counts.
     ///
     /// Returns None if:
     /// - range start is greater than end
@@ -48,7 +48,7 @@ pub trait Buffer: Deref<Target = [u8]> + Sized {
         Self::empty().buffer_str().unwrap()
     }
 
-    /// Attempts to convert the given buffer to a custom string type. 
+    /// Attempts to convert the given buffer to a custom string type.
     ///
     /// This should fail if the type does not have valid UTF-8 bytes, and must be zero copy.
     fn buffer_str(&self) -> Result<Self::BufferString, std::str::Utf8Error>;
@@ -78,4 +78,3 @@ impl<'de> Buffer for &'de [u8] {
         std::str::from_utf8(self)
     }
 }
-

rust/flexbuffers/src/builder/map.rs

@@ -97,8 +97,7 @@ pub(super) fn sort_map_by_keys(values: &mut [Value], buffer: &[u8]) {
     // preferred over custom sorting or adding another dependency. By construction, this part
     // of the values stack must be alternating (key, value) pairs. The public API must not be
     // able to trigger the above debug_assets that protect this unsafe usage.
-    let pairs: &mut [[Value; 2]] =
-        unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) };
+    let pairs: &mut [[Value; 2]] = unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) };
     #[rustfmt::skip]
     pairs.sort_unstable_by(|[key1, _], [key2, _]| {
         if let Value::Key(a1) = *key1 {

rust/flexbuffers/src/builder/mod.rs

@@ -32,10 +32,11 @@ macro_rules! push_slice {
         fn $push_name<T, S>(&mut self, xs: S)
         where
             T: Into<$scalar> + Copy,
-            S: AsRef<[T]>
+            S: AsRef<[T]>,
         {
             let mut value = Value::$new_vec(xs.as_ref().len());
-            let mut width = xs.as_ref()
+            let mut width = xs
+                .as_ref()
                 .iter()
                 .map(|x| BitWidth::from((*x).into()))
                 .max()
@@ -56,7 +57,7 @@ macro_rules! push_slice {
             value.set_child_width_or_panic(width);
             self.values.push(value);
         }
-    }
+    };
 }
 macro_rules! push_indirect {
     ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => {
@@ -65,15 +66,13 @@ macro_rules! push_indirect {
             let child_width = x.width_or_child_width();
             let address = self.buffer.len();
             store_value(&mut self.buffer, x, child_width);
-            self.values.push(
-                Value::Reference {
-                    address,
-                    child_width,
-                    fxb_type: FlexBufferType::$Indirect,
-                }
-            );
+            self.values.push(Value::Reference {
+                address,
+                child_width,
+                fxb_type: FlexBufferType::$Indirect,
+            });
         }
-    }
+    };
 }
 
 bitflags! {

rust/flexbuffers/src/builder/ser.rs

@@ -36,6 +36,10 @@ impl FlexbufferSerializer {
     pub fn take_buffer(&mut self) -> Vec<u8> {
         self.builder.take_buffer()
     }
+    pub fn reset(&mut self) {
+        self.builder.reset();
+        self.nesting.clear();
+    }
     fn finish_if_not_nested(&mut self) -> Result<(), Error> {
         if self.nesting.is_empty() {
             assert_eq!(self.builder.values.len(), 1);

rust/flexbuffers/src/lib.rs

@@ -41,18 +41,18 @@ extern crate num_enum;
 extern crate serde;
 
 mod bitwidth;
+mod buffer;
 mod builder;
 mod flexbuffer_type;
 mod reader;
-mod buffer;
 
 pub use bitwidth::BitWidth;
+pub use buffer::Buffer;
 pub use builder::Error as SerializationError;
 pub use builder::{
     singleton, Builder, BuilderOptions, FlexbufferSerializer, MapBuilder, Pushable, VectorBuilder,
 };
 pub use flexbuffer_type::FlexBufferType;
-pub use buffer::Buffer;
 pub use reader::Error as ReaderError;
 pub use reader::{DeserializationError, MapReader, Reader, ReaderIterator, VectorReader};
 use serde::{Deserialize, Serialize};
@@ -76,7 +76,7 @@ pub fn from_slice<'de, T: Deserialize<'de>>(buf: &'de [u8]) -> Result<T, Deseria
 
 /// Deserialize a type from a flexbuffer.
 pub fn from_buffer<'de, T: Deserialize<'de>, B: Buffer>(
-    buf: &'de B
+    buf: &'de B,
 ) -> Result<T, DeserializationError> {
     let r = Reader::get_root(buf as &'de [u8])?;
     T::deserialize(r)

rust/flexbuffers/src/reader/mod.rs

@@ -14,7 +14,7 @@
 
 use crate::bitwidth::BitWidth;
 use crate::flexbuffer_type::FlexBufferType;
-use crate::{Buffer, Blob};
+use crate::{Blob, Buffer};
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::ops::Rem;
@@ -22,6 +22,7 @@ use std::str::FromStr;
 mod de;
 mod iter;
 mod map;
+mod serialize;
 mod vector;
 pub use de::DeserializationError;
 pub use iter::ReaderIterator;
@@ -184,13 +185,12 @@ impl<B> std::fmt::Debug for Reader<B> {
     }
 }
 
-
 macro_rules! try_cast_fn {
     ($name: ident, $full_width: ident, $Ty: ident) => {
         pub fn $name(&self) -> $Ty {
             self.$full_width().try_into().unwrap_or_default()
         }
-    }
+    };
 }
 
 fn safe_sub(a: usize, b: usize) -> Result<usize, Error> {
@@ -242,7 +242,7 @@ impl<B: Buffer> Reader<B> {
     }
 
     /// Convenience function to get the underlying buffer. By using `shallow_copy`, this preserves
-    /// the lifetime that the underlying buffer has. 
+    /// the lifetime that the underlying buffer has.
     pub fn buffer(&self) -> B {
         self.buffer.shallow_copy()
     }
@@ -263,7 +263,11 @@ impl<B: Buffer> Reader<B> {
         if let Some(len) = self.fxb_type.fixed_length_vector_length() {
             len
         } else if self.fxb_type.has_length_slot() && self.address >= self.width.n_bytes() {
-            read_usize(&self.buffer, self.address - self.width.n_bytes(), self.width)
+            read_usize(
+                &self.buffer,
+                self.address - self.width.n_bytes(),
+                self.width,
+            )
         } else {
             0
         }
@@ -359,7 +363,8 @@ impl<B: Buffer> Reader<B> {
 
     /// Retrieves the string value up until the first `\0` character.
     pub fn get_key(&self) -> Result<B::BufferString, Error> {
-        let bytes = self.buffer
+        let bytes = self
+            .buffer
             .slice(self.address..self.address + self.get_key_len()?)
             .ok_or(Error::IndexOutOfBounds)?;
         Ok(bytes.buffer_str()?)
@@ -368,9 +373,9 @@ impl<B: Buffer> Reader<B> {
     pub fn get_blob(&self) -> Result<Blob<B>, Error> {
         self.expect_type(FlexBufferType::Blob)?;
         Ok(Blob(
-                self.buffer
-                    .slice(self.address..self.address + self.length())
-                    .ok_or(Error::IndexOutOfBounds)?
+            self.buffer
+                .slice(self.address..self.address + self.length())
+                .ok_or(Error::IndexOutOfBounds)?,
         ))
     }
 
@@ -382,7 +387,9 @@ impl<B: Buffer> Reader<B> {
     /// is out of bounds.
     pub fn get_str(&self) -> Result<B::BufferString, Error> {
         self.expect_type(FlexBufferType::String)?;
-        let bytes = self.buffer.slice(self.address..self.address + self.length());
+        let bytes = self
+            .buffer
+            .slice(self.address..self.address + self.length());
         Ok(bytes.ok_or(Error::ReadUsizeOverflowed)?.buffer_str()?)
     }
 
@@ -666,4 +673,4 @@ fn unpack_type(ty: u8) -> Result<(FlexBufferType, BitWidth), Error> {
     let w = BitWidth::try_from(ty & 3u8).map_err(|_| Error::InvalidPackedType)?;
     let t = FlexBufferType::try_from(ty >> 2).map_err(|_| Error::InvalidPackedType)?;
     Ok((t, w))
-}
+}

rust/flexbuffers/src/reader/serialize.rs

@@ -0,0 +1,76 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::reader::Reader;
+use crate::Buffer;
+use crate::{BitWidth::*, FlexBufferType::*};
+use serde::ser;
+use serde::ser::{SerializeMap, SerializeSeq};
+
+impl<B: Buffer> ser::Serialize for &Reader<B> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: ser::Serializer,
+    {
+        #[allow(deprecated)]
+        match (self.flexbuffer_type(), self.bitwidth()) {
+            (Null, _) => serializer.serialize_unit(),
+            (Int, W8) | (IndirectInt, W8) => serializer.serialize_i8(self.as_i8()),
+            (Int, W16) | (IndirectInt, W16) => serializer.serialize_i16(self.as_i16()),
+            (Int, W32) | (IndirectInt, W32) => serializer.serialize_i32(self.as_i32()),
+            (Int, W64) | (IndirectInt, W64) => serializer.serialize_i64(self.as_i64()),
+            (UInt, W8) | (IndirectUInt, W8) => serializer.serialize_u8(self.as_u8()),
+            (UInt, W16) | (IndirectUInt, W16) => serializer.serialize_u16(self.as_u16()),
+            (UInt, W32) | (IndirectUInt, W32) => serializer.serialize_u32(self.as_u32()),
+            (UInt, W64) | (IndirectUInt, W64) => serializer.serialize_u64(self.as_u64()),
+            (Float, W32) | (IndirectFloat, W32) => serializer.serialize_f32(self.as_f32()),
+            (Float, _) | (IndirectFloat, _) => serializer.serialize_f64(self.as_f64()),
+            (Bool, _) => serializer.serialize_bool(self.as_bool()),
+            (Key, _) | (String, _) => serializer.serialize_str(&self.as_str()),
+            (Map, _) => {
+                let m = self.as_map();
+                let mut map_serializer = serializer.serialize_map(Some(m.len()))?;
+                for (k, v) in m.iter_keys().zip(m.iter_values()) {
+                    map_serializer.serialize_key(&&k)?;
+                    map_serializer.serialize_value(&&v)?;
+                }
+                map_serializer.end()
+            }
+            (Vector, _)
+            | (VectorInt, _)
+            | (VectorUInt, _)
+            | (VectorFloat, _)
+            | (VectorKey, _)
+            | (VectorString, _)
+            | (VectorBool, _)
+            | (VectorInt2, _)
+            | (VectorUInt2, _)
+            | (VectorFloat2, _)
+            | (VectorInt3, _)
+            | (VectorUInt3, _)
+            | (VectorFloat3, _)
+            | (VectorInt4, _)
+            | (VectorUInt4, _)
+            | (VectorFloat4, _) => {
+                let v = self.as_vector();
+                let mut seq_serializer = serializer.serialize_seq(Some(v.len()))?;
+                for x in v.iter() {
+                    seq_serializer.serialize_element(&&x)?;
+                }
+                seq_serializer.end()
+            }
+            (Blob, _) => serializer.serialize_bytes(&self.as_blob().0),
+        }
+    }
+}

rust/flexbuffers/src/reader/vector.rs

@@ -40,7 +40,7 @@ impl<B: Buffer> Default for VectorReader<B> {
     fn default() -> Self {
         VectorReader {
             reader: Reader::default(),
-            length: usize::default()
+            length: usize::default(),
         }
     }
 }

tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs

@@ -134,12 +134,13 @@ impl<A: Arbitrary> Arbitrary for Array4<A> {
 }
 
 quickcheck! {
-    fn qc_serious(x: Struct) -> bool {
-        let mut s = FlexbufferSerializer::new();
-        x.serialize(&mut s).unwrap();
-        let r = Reader::get_root(s.view()).unwrap();
-        println!("{}", r);
-        let x2 = Struct::deserialize(r).unwrap();
-        x == x2
+    fn qc_serious(original: Struct) -> bool {
+        let struct_buf = flexbuffers::to_vec(&original).unwrap();
+        let root = Reader::get_root(&*struct_buf).unwrap();
+        let reader_buf = flexbuffers::to_vec(&root).unwrap();
+        let deserialized: Struct = flexbuffers::from_slice(&struct_buf).unwrap();
+        let reserialized: Struct = flexbuffers::from_slice(&reader_buf).unwrap();
+
+        original == deserialized && original == reserialized
     }
 }