feat(serde_v8): ZeroCopyBuf (#10432)

Bidirectional zero-copy serialization of buffers between v8 & rust
that can be nested in structs/tuples/etc.

core/lib.rs

@@ -15,13 +15,13 @@ mod ops_json;
 pub mod plugin_api;
 mod resources;
 mod runtime;
-mod zero_copy_buf;
 
 // Re-exports
 pub use futures;
 pub use rusty_v8 as v8;
 pub use serde;
 pub use serde_json;
+pub use serde_v8::Buffer as ZeroCopyBuf;
 pub use url;
 
 pub use crate::async_cancel::CancelFuture;
@@ -79,7 +79,6 @@ pub use crate::runtime::Snapshot;
 // pub use crate::runtime_modules::include_js_files!;
 pub use crate::extensions::Extension;
 pub use crate::extensions::OpMiddlewareFn;
-pub use crate::zero_copy_buf::ZeroCopyBuf;
 
 pub fn v8_version() -> &'static str {
   v8::V8::get_version()

serde_v8/src/de.rs

@@ -309,6 +309,16 @@ impl<'de, 'a, 'b, 's, 'x> de::Deserializer<'de>
       return visitor.visit_u64(hack);
     }
 
+    // Magic Buffer
+    if name == magic::buffer::BUF_NAME {
+      let zero_copy_buf =
+        v8::Local::<v8::ArrayBufferView>::try_from(self.input)
+          .map(|view| magic::zero_copy_buf::ZeroCopyBuf::new(self.scope, view))
+          .map_err(|_| Error::ExpectedArray)?;
+      let data: [u8; 32] = unsafe { std::mem::transmute(zero_copy_buf) };
+      return visitor.visit_bytes(&data);
+    }
+
     // Regular struct
     let obj = v8::Local::<v8::Object>::try_from(self.input).unwrap();
     let map = ObjectAccess {

serde_v8/src/lib.rs

@@ -11,6 +11,7 @@ pub mod utils;
 pub use de::{from_v8, from_v8_cached, Deserializer};
 pub use error::{Error, Result};
 pub use keys::KeyCache;
+pub use magic::buffer::MagicBuffer as Buffer;
 pub use magic::Value;
 pub use ser::{to_v8, Serializer};
 pub use serializable::{Serializable, SerializablePkg};

serde_v8/src/magic/buffer.rs

@@ -0,0 +1,132 @@
+use rusty_v8 as v8;
+
+use std::cell::Cell;
+use std::fmt;
+use std::ops::Deref;
+use std::ops::DerefMut;
+
+use super::zero_copy_buf::ZeroCopyBuf;
+
+// An asymmetric wrapper around ZeroCopyBuf,
+// allowing us to use a single type for familiarity
+pub enum MagicBuffer {
+  FromV8(ZeroCopyBuf),
+  ToV8(Cell<Option<Box<[u8]>>>),
+}
+
+impl MagicBuffer {
+  pub fn new<'s>(
+    scope: &mut v8::HandleScope<'s>,
+    view: v8::Local<v8::ArrayBufferView>,
+  ) -> Self {
+    Self::FromV8(ZeroCopyBuf::new(scope, view))
+  }
+}
+
+impl Clone for MagicBuffer {
+  fn clone(&self) -> Self {
+    match self {
+      Self::FromV8(zbuf) => Self::FromV8(zbuf.clone()),
+      Self::ToV8(_) => panic!("Don't Clone a MagicBuffer sent to v8"),
+    }
+  }
+}
+
+impl AsRef<[u8]> for MagicBuffer {
+  fn as_ref(&self) -> &[u8] {
+    &*self
+  }
+}
+
+impl AsMut<[u8]> for MagicBuffer {
+  fn as_mut(&mut self) -> &mut [u8] {
+    &mut *self
+  }
+}
+
+impl Deref for MagicBuffer {
+  type Target = [u8];
+  fn deref(&self) -> &[u8] {
+    match self {
+      Self::FromV8(buf) => &*buf,
+      Self::ToV8(_) => panic!("Don't Deref a MagicBuffer sent to v8"),
+    }
+  }
+}
+
+impl DerefMut for MagicBuffer {
+  fn deref_mut(&mut self) -> &mut [u8] {
+    match self {
+      Self::FromV8(buf) => &mut *buf,
+      Self::ToV8(_) => panic!("Don't Deref a MagicBuffer sent to v8"),
+    }
+  }
+}
+
+impl From<Box<[u8]>> for MagicBuffer {
+  fn from(buf: Box<[u8]>) -> Self {
+    MagicBuffer::ToV8(Cell::new(Some(buf)))
+  }
+}
+
+impl From<Vec<u8>> for MagicBuffer {
+  fn from(vec: Vec<u8>) -> Self {
+    vec.into_boxed_slice().into()
+  }
+}
+
+pub const BUF_NAME: &str = "$__v8_magic_Buffer";
+pub const BUF_FIELD_1: &str = "$__v8_magic_buffer_1";
+pub const BUF_FIELD_2: &str = "$__v8_magic_buffer_2";
+
+impl serde::Serialize for MagicBuffer {
+  fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+  where
+    S: serde::Serializer,
+  {
+    use serde::ser::SerializeStruct;
+
+    let mut s = serializer.serialize_struct(BUF_NAME, 1)?;
+    let boxed: Box<[u8]> = match self {
+      Self::FromV8(_) => unreachable!(),
+      Self::ToV8(x) => x.take().expect("MagicBuffer was empty"),
+    };
+    let hack: [usize; 2] = unsafe { std::mem::transmute(boxed) };
+    let f1: u64 = hack[0] as u64;
+    let f2: u64 = hack[1] as u64;
+    s.serialize_field(BUF_FIELD_1, &f1)?;
+    s.serialize_field(BUF_FIELD_2, &f2)?;
+    s.end()
+  }
+}
+
+impl<'de, 's> serde::Deserialize<'de> for MagicBuffer {
+  fn deserialize<D>(deserializer: D) -> Result<MagicBuffer, D::Error>
+  where
+    D: serde::Deserializer<'de>,
+  {
+    struct ValueVisitor {}
+
+    impl<'de> serde::de::Visitor<'de> for ValueVisitor {
+      type Value = MagicBuffer;
+
+      fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a serde_v8::MagicBuffer")
+      }
+
+      fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+      where
+        E: serde::de::Error,
+      {
+        let p1: &[usize] = unsafe { &*(v as *const [u8] as *const [usize]) };
+        let p2: [usize; 4] = [p1[0], p1[1], p1[2], p1[3]];
+        let zero_copy: ZeroCopyBuf = unsafe { std::mem::transmute(p2) };
+        Ok(MagicBuffer::FromV8(zero_copy))
+      }
+    }
+
+    static FIELDS: [&str; 0] = [];
+    let visitor = ValueVisitor {};
+    deserializer.deserialize_struct(BUF_NAME, &FIELDS, visitor)
+  }
+}

serde_v8/src/magic/mod.rs

@@ -1,6 +1,8 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+pub mod buffer;
 mod field;
 mod value;
+pub mod zero_copy_buf;
 
 pub use field::FieldSerializer;
 pub use value::{Value, FIELD, NAME};

serde_v8/src/ser.rs

@@ -224,9 +224,55 @@ impl<'a> ser::SerializeStruct for MagicSerializer<'a> {
   }
 }
 
+// TODO(@AaronO): refactor this and streamline how we transmute values
+pub struct MagicBufferSerializer<'a, 'b, 'c> {
+  scope: ScopePtr<'a, 'b, 'c>,
+  f1: u64,
+  f2: u64,
+}
+
+impl<'a, 'b, 'c> MagicBufferSerializer<'a, 'b, 'c> {
+  pub fn new(scope: ScopePtr<'a, 'b, 'c>) -> Self {
+    Self {
+      scope,
+      f1: 0,
+      f2: 0,
+    }
+  }
+}
+
+impl<'a, 'b, 'c> ser::SerializeStruct for MagicBufferSerializer<'a, 'b, 'c> {
+  type Ok = JsValue<'a>;
+  type Error = Error;
+
+  fn serialize_field<T: ?Sized + Serialize>(
+    &mut self,
+    key: &'static str,
+    value: &T,
+  ) -> Result<()> {
+    // Get u64 chunk
+    let transmuted: u64 = value.serialize(magic::FieldSerializer {})?;
+    match key {
+      magic::buffer::BUF_FIELD_1 => self.f1 = transmuted,
+      magic::buffer::BUF_FIELD_2 => self.f2 = transmuted,
+      _ => unreachable!(),
+    }
+    Ok(())
+  }
+
+  fn end(self) -> JsResult<'a> {
+    let x: [usize; 2] = [self.f1 as usize, self.f2 as usize];
+    let buf: Box<[u8]> = unsafe { std::mem::transmute(x) };
+    let scope = &mut *self.scope.borrow_mut();
+    let v8_value = boxed_slice_to_uint8array(scope, buf);
+    Ok(v8_value.into())
+  }
+}
+
 // Dispatches between magic and regular struct serializers
 pub enum StructSerializers<'a, 'b, 'c> {
   Magic(MagicSerializer<'a>),
+  MagicBuffer(MagicBufferSerializer<'a, 'b, 'c>),
   Regular(ObjectSerializer<'a, 'b, 'c>),
 }
 
@@ -241,6 +287,7 @@ impl<'a, 'b, 'c> ser::SerializeStruct for StructSerializers<'a, 'b, 'c> {
   ) -> Result<()> {
     match self {
       StructSerializers::Magic(s) => s.serialize_field(key, value),
+      StructSerializers::MagicBuffer(s) => s.serialize_field(key, value),
       StructSerializers::Regular(s) => s.serialize_field(key, value),
     }
   }
@@ -248,6 +295,7 @@ impl<'a, 'b, 'c> ser::SerializeStruct for StructSerializers<'a, 'b, 'c> {
   fn end(self) -> JsResult<'a> {
     match self {
       StructSerializers::Magic(s) => s.end(),
+      StructSerializers::MagicBuffer(s) => s.end(),
       StructSerializers::Regular(s) => s.end(),
     }
   }
@@ -463,12 +511,20 @@ impl<'a, 'b, 'c> ser::Serializer for Serializer<'a, 'b, 'c> {
     name: &'static str,
     _len: usize,
   ) -> Result<Self::SerializeStruct> {
-    if name == magic::NAME {
+    match name {
-      let m: MagicSerializer<'a> = MagicSerializer { v8_value: None };
+      magic::NAME => {
-      return Ok(StructSerializers::Magic(m));
+        let m: MagicSerializer<'a> = MagicSerializer { v8_value: None };
+        Ok(StructSerializers::Magic(m))
+      }
+      magic::buffer::BUF_NAME => {
+        let m = MagicBufferSerializer::new(self.scope);
+        Ok(StructSerializers::MagicBuffer(m))
+      }
+      _ => {
+        let o = ObjectSerializer::new(self.scope);
+        Ok(StructSerializers::Regular(o))
+      }
     }
-    let o = ObjectSerializer::new(self.scope);
-    Ok(StructSerializers::Regular(o))
   }
 
   fn serialize_struct_variant(
@@ -483,3 +539,21 @@ impl<'a, 'b, 'c> ser::Serializer for Serializer<'a, 'b, 'c> {
     Ok(VariantSerializer::new(scope, variant, x))
   }
 }
+
+// Used to map MagicBuffers to v8
+pub fn boxed_slice_to_uint8array<'a>(
+  scope: &mut v8::HandleScope<'a>,
+  buf: Box<[u8]>,
+) -> v8::Local<'a, v8::Uint8Array> {
+  if buf.is_empty() {
+    let ab = v8::ArrayBuffer::new(scope, 0);
+    return v8::Uint8Array::new(scope, ab, 0, 0)
+      .expect("Failed to create UintArray8");
+  }
+  let buf_len = buf.len();
+  let backing_store = v8::ArrayBuffer::new_backing_store_from_boxed_slice(buf);
+  let backing_store_shared = backing_store.make_shared();
+  let ab = v8::ArrayBuffer::with_backing_store(scope, &backing_store_shared);
+  v8::Uint8Array::new(scope, ab, 0, buf_len)
+    .expect("Failed to create UintArray8")
+}

serde_v8/tests/magic.rs

@@ -3,7 +3,7 @@ use rusty_v8 as v8;
 
 use serde::{Deserialize, Serialize};
 
-use serde_v8::utils::{js_exec, v8_init, v8_shutdown};
+use serde_v8::utils::{js_exec, v8_do};
 use std::convert::TryFrom;
 
 #[derive(Deserialize)]
@@ -22,9 +22,7 @@ struct MagicContainer<'s> {
 
 #[test]
 fn magic_basic() {
-  v8_init();
+  v8_do(|| {
-
-  {
     let isolate = &mut v8::Isolate::new(v8::CreateParams::default());
     let handle_scope = &mut v8::HandleScope::new(isolate);
     let context = v8::Context::new(handle_scope);
@@ -52,7 +50,75 @@ fn magic_basic() {
       s2,
       r#"{"magic":true,"contains":{"a":1,"b":3,"c":"abracadabra"}}"#
     );
-  }
+  })
-
+}
-  v8_shutdown();
+
+#[test]
+fn magic_buffer() {
+  v8_do(|| {
+    // Init isolate
+    let isolate = &mut v8::Isolate::new(v8::CreateParams::default());
+    let handle_scope = &mut v8::HandleScope::new(isolate);
+    let context = v8::Context::new(handle_scope);
+    let scope = &mut v8::ContextScope::new(handle_scope, context);
+    let global = context.global(scope);
+
+    // Simple buffer
+    let v8_array = js_exec(scope, "new Uint8Array([1,2,3,4,5])");
+    let zbuf: serde_v8::Buffer = serde_v8::from_v8(scope, v8_array).unwrap();
+    assert_eq!(&*zbuf, &[1, 2, 3, 4, 5]);
+
+    // Multi buffers
+    let v8_arrays =
+      js_exec(scope, "[new Uint8Array([1,2]), new Uint8Array([3,4,5])]");
+    let (z1, z2): (serde_v8::Buffer, serde_v8::Buffer) =
+      serde_v8::from_v8(scope, v8_arrays).unwrap();
+    assert_eq!(&*z1, &[1, 2]);
+    assert_eq!(&*z2, &[3, 4, 5]);
+
+    // Wrapped in option, like our current op-ABI
+    let v8_array = js_exec(scope, "new Uint8Array([1,2,3,4,5])");
+    let zbuf: Option<serde_v8::Buffer> =
+      serde_v8::from_v8(scope, v8_array).unwrap();
+    assert_eq!(&*zbuf.unwrap(), &[1, 2, 3, 4, 5]);
+
+    // Observe mutation in JS
+    let v8_array = js_exec(scope, "new Uint8Array([1,2,3,4,5])");
+    let mut zbuf: serde_v8::Buffer =
+      serde_v8::from_v8(scope, v8_array).unwrap();
+    let key = serde_v8::to_v8(scope, "t1").unwrap();
+    global.set(scope, key, v8_array);
+    (&mut *zbuf)[2] = 42;
+    let eq = js_exec(scope, "t1[2] === 42");
+    assert!(eq.is_true());
+
+    // Serialization
+    let buf: Vec<u8> = vec![1, 2, 3, 99, 5];
+    let zbuf: serde_v8::Buffer = buf.into();
+    let v8_value = serde_v8::to_v8(scope, zbuf).unwrap();
+    let key = serde_v8::to_v8(scope, "t2").unwrap();
+    global.set(scope, key, v8_value);
+    let eq = js_exec(scope, "t2[3] === 99");
+    assert!(eq.is_true());
+
+    // Composite Serialization
+    #[derive(serde::Serialize)]
+    struct Wrapper {
+      a: serde_v8::Buffer,
+      b: serde_v8::Buffer,
+    }
+    let buf1: Vec<u8> = vec![1, 2, 33, 4, 5];
+    let buf2: Vec<u8> = vec![5, 4, 3, 2, 11];
+    let wrapped = Wrapper {
+      a: buf1.into(),
+      b: buf2.into(),
+    };
+    let v8_value = serde_v8::to_v8(scope, wrapped).unwrap();
+    let key = serde_v8::to_v8(scope, "t3").unwrap();
+    global.set(scope, key, v8_value);
+    let eq = js_exec(scope, "t3.a[2] === 33");
+    assert!(eq.is_true());
+    let eq = js_exec(scope, "t3.b[4] === 11");
+    assert!(eq.is_true());
+  })
 }