Support zero-copy data in libdeno.send(). (#838)

This is a large API refactor of deno.h which replaces
deno_send() and deno_set_response() with deno_respond().
It also adds a req_id parameter to the deno_recv_cb.

Make writeFile/writeFileSync use it.

js/dispatch.ts

@@ -28,10 +28,11 @@ export function handleAsyncMsgFromRust(ui8: Uint8Array) {
 export function sendAsync(
   builder: flatbuffers.Builder,
   msgType: fbs.Any,
-  msg: flatbuffers.Offset
+  msg: flatbuffers.Offset,
+  data?: ArrayBufferView
 ): Promise<fbs.Base> {
   maybePushTrace(msgType, false); // add to trace if tracing
-  const [cmdId, resBuf] = sendInternal(builder, msgType, msg, false);
+  const [cmdId, resBuf] = sendInternal(builder, msgType, msg, data, false);
   util.assert(resBuf == null);
   const promise = util.createResolvable<fbs.Base>();
   promiseTable.set(cmdId, promise);
@@ -42,16 +43,16 @@ export function sendAsync(
 export function sendSync(
   builder: flatbuffers.Builder,
   msgType: fbs.Any,
-  msg: flatbuffers.Offset
+  msg: flatbuffers.Offset,
+  data?: ArrayBufferView
 ): null | fbs.Base {
   maybePushTrace(msgType, true); // add to trace if tracing
-  const [cmdId, resBuf] = sendInternal(builder, msgType, msg, true);
+  const [cmdId, resBuf] = sendInternal(builder, msgType, msg, data, true);
   util.assert(cmdId >= 0);
   if (resBuf == null) {
     return null;
   } else {
     const u8 = new Uint8Array(resBuf!);
-    // console.log("recv sync message", util.hexdump(u8));
     const bb = new flatbuffers.ByteBuffer(u8);
     const baseRes = fbs.Base.getRootAsBase(bb);
     errors.maybeThrowError(baseRes);
@@ -63,6 +64,7 @@ function sendInternal(
   builder: flatbuffers.Builder,
   msgType: fbs.Any,
   msg: flatbuffers.Offset,
+  data: undefined | ArrayBufferView,
   sync = true
 ): [number, null | Uint8Array] {
   const cmdId = nextCmdId++;
@@ -73,5 +75,5 @@ function sendInternal(
   fbs.Base.addCmdId(builder, cmdId);
   builder.finish(fbs.Base.endBase(builder));
 
-  return [cmdId, libdeno.send(builder.asUint8Array())];
+  return [cmdId, libdeno.send(builder.asUint8Array(), data)];
 }

js/libdeno.ts

@@ -6,7 +6,7 @@ type MessageCallback = (msg: Uint8Array) => void;
 interface Libdeno {
   recv(cb: MessageCallback): void;
 
-  send(msg: ArrayBufferView): null | Uint8Array;
+  send(control: ArrayBufferView, data?: ArrayBufferView): null | Uint8Array;
 
   print(x: string, isErr?: boolean): void;
 

js/write_file.ts

@@ -41,14 +41,12 @@ function req(
   filename: string,
   data: Uint8Array,
   perm: number
-): [flatbuffers.Builder, fbs.Any, flatbuffers.Offset] {
+): [flatbuffers.Builder, fbs.Any, flatbuffers.Offset, Uint8Array] {
   const builder = new flatbuffers.Builder();
   const filename_ = builder.createString(filename);
-  const dataOffset = fbs.WriteFile.createDataVector(builder, data);
   fbs.WriteFile.startWriteFile(builder);
   fbs.WriteFile.addFilename(builder, filename_);
-  fbs.WriteFile.addData(builder, dataOffset);
   fbs.WriteFile.addPerm(builder, perm);
   const msg = fbs.WriteFile.endWriteFile(builder);
-  return [builder, fbs.Any.WriteFile, msg];
+  return [builder, fbs.Any.WriteFile, msg, data];
 }

libdeno/binding.cc

@@ -25,6 +25,36 @@ Deno* FromIsolate(v8::Isolate* isolate) {
   return static_cast<Deno*>(isolate->GetData(0));
 }
 
+void LazilyCreateDataMap(Deno* d) {
+  if (d->async_data_map.IsEmpty()) {
+    v8::HandleScope handle_scope(d->isolate);
+    // It's important for security reasons that async_data_map is not exposed to
+    // the VM.
+    auto async_data_map = v8::Map::New(d->isolate);
+    d->async_data_map.Reset(d->isolate, async_data_map);
+  }
+  DCHECK(!d->async_data_map.IsEmpty());
+}
+
+void AddDataRef(Deno* d, int32_t req_id, v8::Local<v8::Value> data_v) {
+  LazilyCreateDataMap(d);
+  auto async_data_map = d->async_data_map.Get(d->isolate);
+  auto context = d->context.Get(d->isolate);
+  auto req_id_v = v8::Integer::New(d->isolate, req_id);
+  auto r = async_data_map->Set(context, req_id_v, data_v);
+  CHECK(!r.IsEmpty());
+}
+
+void DeleteDataRef(Deno* d, int32_t req_id) {
+  LazilyCreateDataMap(d);
+  auto context = d->context.Get(d->isolate);
+  // Delete persistent reference to data ArrayBuffer.
+  auto async_data_map = d->async_data_map.Get(d->isolate);
+  auto req_id_v = v8::Integer::New(d->isolate, req_id);
+  auto maybe_deleted = async_data_map->Delete(context, req_id_v);
+  DCHECK(maybe_deleted.IsJust());
+}
+
 // Extracts a C string from a v8::V8 Utf8Value.
 const char* ToCString(const v8::String::Utf8Value& value) {
   return *value ? *value : "<string conversion failed>";
@@ -214,17 +244,40 @@ void Send(const v8::FunctionCallbackInfo<v8::Value>& args) {
   v8::Locker locker(d->isolate);
   v8::EscapableHandleScope handle_scope(isolate);
 
-  CHECK_EQ(args.Length(), 1);
-  v8::Local<v8::Value> ab_v = args[0];
-  CHECK(ab_v->IsArrayBufferView());
-  auto buf = GetContents(isolate, v8::Local<v8::ArrayBufferView>::Cast(ab_v));
+  CHECK_EQ(d->currentArgs, nullptr); // libdeno.send re-entry forbidden.
+  int32_t req_id = d->next_req_id++;
+
+  v8::Local<v8::Value> control_v = args[0];
+  CHECK(control_v->IsArrayBufferView());
+  deno_buf control =
+      GetContents(isolate, v8::Local<v8::ArrayBufferView>::Cast(control_v));
+  deno_buf data = {nullptr, 0u, nullptr, 0u};
+  v8::Local<v8::Value> data_v;
+  if (args.Length() == 2) {
+    if (args[1]->IsArrayBufferView()) {
+      data_v = args[1];
+      data = GetContents(isolate, v8::Local<v8::ArrayBufferView>::Cast(data_v));
+    }
+  } else {
+    CHECK_EQ(args.Length(), 1);
+  }
 
   DCHECK_EQ(d->currentArgs, nullptr);
   d->currentArgs = &args;
 
-  d->cb(d, buf);
+  d->cb(d, req_id, control, data);
 
-  d->currentArgs = nullptr;
+  if (d->currentArgs == nullptr) {
+    // This indicates that deno_repond() was called already.
+  } else {
+    // Asynchronous.
+    d->currentArgs = nullptr;
+    // If the data ArrayBuffer was given, we must maintain a strong reference
+    // to it until deno_respond is called.
+    if (!data_v.IsEmpty()) {
+      AddDataRef(d, req_id, data_v);
+    }
+  }
 }
 
 // Sets the global error handler.
@@ -358,6 +411,7 @@ void InitializeContext(v8::Isolate* isolate, v8::Local<v8::Context> context,
 }
 
 void AddIsolate(Deno* d, v8::Isolate* isolate) {
+  d->next_req_id = 0;
   d->isolate = isolate;
   // Leaving this code here because it will probably be useful later on, but
   // disabling it now as I haven't got tests for the desired behavior.
@@ -400,7 +454,17 @@ int deno_execute(Deno* d, const char* js_filename, const char* js_source) {
   return deno::Execute(context, js_filename, js_source) ? 1 : 0;
 }
 
-int deno_send(Deno* d, deno_buf buf) {
+int deno_respond(Deno* d, int32_t req_id, deno_buf buf) {
+  if (d->currentArgs != nullptr) {
+    // Synchronous response.
+    auto ab = deno::ImportBuf(d->isolate, buf);
+    d->currentArgs->GetReturnValue().Set(ab);
+    d->currentArgs = nullptr;
+    return 0;
+  }
+
+  // Asynchronous response.
+
   v8::Locker locker(d->isolate);
   v8::Isolate::Scope isolate_scope(d->isolate);
   v8::HandleScope handle_scope(d->isolate);
@@ -410,10 +474,12 @@ int deno_send(Deno* d, deno_buf buf) {
 
   v8::TryCatch try_catch(d->isolate);
 
+  deno::DeleteDataRef(d, req_id);
+
   auto recv = d->recv.Get(d->isolate);
   if (recv.IsEmpty()) {
     d->last_exception = "libdeno.recv has not been called.";
-    return 0;
+    return 1;
   }
 
   v8::Local<v8::Value> args[1];
@@ -422,17 +488,10 @@ int deno_send(Deno* d, deno_buf buf) {
 
   if (try_catch.HasCaught()) {
     deno::HandleException(context, try_catch.Exception());
-    return 0;
+    return 1;
   }
 
-  return 1;
-}
-
-void deno_set_response(Deno* d, deno_buf buf) {
-  // printf("deno_set_response: ");
-  // hexdump(buf.data_ptr, buf.data_len);
-  auto ab = deno::ImportBuf(d->isolate, buf);
-  d->currentArgs->GetReturnValue().Set(ab);
+  return 0;
 }
 
 void deno_delete(Deno* d) {

libdeno/deno.h

@@ -20,9 +20,11 @@ typedef struct {
 struct deno_s;
 typedef struct deno_s Deno;
 
-// A callback to receive a message from deno.send javascript call.
-// buf is valid only for the lifetime of the call.
-typedef void (*deno_recv_cb)(Deno* d, deno_buf buf);
+// A callback to receive a message from a libdeno.send() javascript call.
+// control_buf is valid for only for the lifetime of this callback.
+// data_buf is valid until deno_respond() is called.
+typedef void (*deno_recv_cb)(Deno* d, int32_t req_id, deno_buf control_buf,
+                             deno_buf data_buf);
 
 void deno_init();
 const char* deno_v8_version();
@@ -39,21 +41,25 @@ void* deno_get_data(Deno*);
 // 0 = fail, 1 = success
 int deno_execute(Deno* d, const char* js_filename, const char* js_source);
 
-// Routes message to the javascript callback set with deno.recv(). A false
-// return value indicates error. Check deno_last_exception() for exception text.
-// 0 = fail, 1 = success
-// After calling deno_send(), the caller no longer owns `buf` and must not use
-// it; deno_send() is responsible for releasing it's memory.
-// TODO(piscisaureus) In C++ and/or Rust, use a smart pointer or similar to
-// enforce this rule.
-int deno_send(Deno* d, deno_buf buf);
-
-// Call this inside a deno_recv_cb to respond synchronously to messages.
-// If this is not called during the life time of a deno_recv_cb callback
-// the deno.send() call in javascript will return null.
-// After calling deno_set_response(), the caller no longer owns `buf` and must
-// not access it; deno_set_response() is responsible for releasing it's memory.
-void deno_set_response(Deno* d, deno_buf buf);
+// deno_respond sends up to one message back for every deno_recv_cb made.
+//
+// If this is called during deno_recv_cb, the issuing libdeno.send() in
+// javascript will synchronously return the specified buf as an ArrayBuffer (or
+// null if buf is empty).
+//
+// If this is called after deno_recv_cb has returned, the deno_respond
+// will call into the JS callback specified by libdeno.recv().
+//
+// (Ideally, but not currently: After calling deno_respond(), the caller no
+// longer owns `buf` and must not use it; deno_respond() is responsible for
+// releasing its memory.)
+//
+// Calling this function more than once with the same req_id will result in
+// an error.
+//
+// A non-zero return value, means a JS exception was encountered during the
+// libdeno.recv() callback. Check deno_last_exception() for exception text.
+int deno_respond(Deno* d, int32_t req_id, deno_buf buf);
 
 const char* deno_last_exception(Deno* d);
 

libdeno/internal.h

@@ -15,7 +15,9 @@ struct deno_s {
   v8::Persistent<v8::Function> recv;
   v8::Persistent<v8::Function> global_error_handler;
   v8::Persistent<v8::Context> context;
+  v8::Persistent<v8::Map> async_data_map;
   deno_recv_cb cb;
+  int32_t next_req_id;
   void* data;
 };
 }

libdeno/libdeno_test.cc

@@ -45,34 +45,17 @@ deno_buf StrBufNullAllocPtr(const char* str) {
   return buf;
 }
 
-TEST(LibDenoTest, SendSuccess) {
-  Deno* d = deno_new(nullptr, nullptr);
-  EXPECT_TRUE(deno_execute(d, "a.js", "SendSuccess()"));
-  EXPECT_TRUE(deno_send(d, strbuf("abc")));
-  deno_delete(d);
-}
-
-TEST(LibDenoTest, SendWrongByteLength) {
-  Deno* d = deno_new(nullptr, nullptr);
-  EXPECT_TRUE(deno_execute(d, "a.js", "SendWrongByteLength()"));
-  // deno_send the wrong sized message, it should throw.
-  EXPECT_FALSE(deno_send(d, strbuf("abcd")));
-  std::string exception = deno_last_exception(d);
-  EXPECT_GT(exception.length(), 1u);
-  EXPECT_NE(exception.find("assert"), std::string::npos);
-  deno_delete(d);
-}
-
-TEST(LibDenoTest, SendNoCallback) {
-  Deno* d = deno_new(nullptr, nullptr);
-  // We didn't call deno.recv() in JS, should fail.
-  EXPECT_FALSE(deno_send(d, strbuf("abc")));
-  deno_delete(d);
+void assert_null(deno_buf b) {
+  EXPECT_EQ(b.alloc_ptr, nullptr);
+  EXPECT_EQ(b.alloc_len, 0u);
+  EXPECT_EQ(b.data_ptr, nullptr);
+  EXPECT_EQ(b.data_len, 0u);
 }
 
 TEST(LibDenoTest, RecvReturnEmpty) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto _, auto buf) {
+  Deno* d = deno_new(nullptr, [](auto _, int req_id, auto buf, auto data_buf) {
+    assert_null(data_buf);
     count++;
     EXPECT_EQ(static_cast<size_t>(3), buf.data_len);
     EXPECT_EQ(buf.data_ptr[0], 'a');
@@ -86,14 +69,16 @@ TEST(LibDenoTest, RecvReturnEmpty) {
 
 TEST(LibDenoTest, RecvReturnBar) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto deno, auto buf) {
-    count++;
-    EXPECT_EQ(static_cast<size_t>(3), buf.data_len);
-    EXPECT_EQ(buf.data_ptr[0], 'a');
-    EXPECT_EQ(buf.data_ptr[1], 'b');
-    EXPECT_EQ(buf.data_ptr[2], 'c');
-    deno_set_response(deno, strbuf("bar"));
-  });
+  Deno* d =
+      deno_new(nullptr, [](auto deno, int req_id, auto buf, auto data_buf) {
+        assert_null(data_buf);
+        count++;
+        EXPECT_EQ(static_cast<size_t>(3), buf.data_len);
+        EXPECT_EQ(buf.data_ptr[0], 'a');
+        EXPECT_EQ(buf.data_ptr[1], 'b');
+        EXPECT_EQ(buf.data_ptr[2], 'c');
+        deno_respond(deno, req_id, strbuf("bar"));
+      });
   EXPECT_TRUE(deno_execute(d, "a.js", "RecvReturnBar()"));
   EXPECT_EQ(count, 1);
   deno_delete(d);
@@ -107,31 +92,33 @@ TEST(LibDenoTest, DoubleRecvFails) {
 
 TEST(LibDenoTest, SendRecvSlice) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto deno, auto buf) {
-    static const size_t alloc_len = 1024;
-    size_t i = count++;
-    // Check the size and offset of the slice.
-    size_t data_offset = buf.data_ptr - buf.alloc_ptr;
-    EXPECT_EQ(data_offset, i * 11);
-    EXPECT_EQ(buf.data_len, alloc_len - i * 30);
-    EXPECT_EQ(buf.alloc_len, alloc_len);
-    // Check values written by the JS side.
-    EXPECT_EQ(buf.data_ptr[0], 100 + i);
-    EXPECT_EQ(buf.data_ptr[buf.data_len - 1], 100 - i);
-    // Make copy of the backing buffer -- this is currently necessary because
-    // deno_set_response() takes ownership over the buffer, but we are not given
-    // ownership of `buf` by our caller.
-    uint8_t* alloc_ptr = reinterpret_cast<uint8_t*>(malloc(alloc_len));
-    memcpy(alloc_ptr, buf.alloc_ptr, alloc_len);
-    // Make a slice that is a bit shorter than the original.
-    deno_buf buf2{alloc_ptr, alloc_len, alloc_ptr + data_offset,
-                  buf.data_len - 19};
-    // Place some values into the buffer for the JS side to verify.
-    buf2.data_ptr[0] = 200 + i;
-    buf2.data_ptr[buf2.data_len - 1] = 200 - i;
-    // Send back.
-    deno_set_response(deno, buf2);
-  });
+  Deno* d =
+      deno_new(nullptr, [](auto deno, int req_id, auto buf, auto data_buf) {
+        assert_null(data_buf);
+        static const size_t alloc_len = 1024;
+        size_t i = count++;
+        // Check the size and offset of the slice.
+        size_t data_offset = buf.data_ptr - buf.alloc_ptr;
+        EXPECT_EQ(data_offset, i * 11);
+        EXPECT_EQ(buf.data_len, alloc_len - i * 30);
+        EXPECT_EQ(buf.alloc_len, alloc_len);
+        // Check values written by the JS side.
+        EXPECT_EQ(buf.data_ptr[0], 100 + i);
+        EXPECT_EQ(buf.data_ptr[buf.data_len - 1], 100 - i);
+        // Make copy of the backing buffer -- this is currently necessary
+        // because deno_respond() takes ownership over the buffer, but we are
+        // not given ownership of `buf` by our caller.
+        uint8_t* alloc_ptr = reinterpret_cast<uint8_t*>(malloc(alloc_len));
+        memcpy(alloc_ptr, buf.alloc_ptr, alloc_len);
+        // Make a slice that is a bit shorter than the original.
+        deno_buf buf2{alloc_ptr, alloc_len, alloc_ptr + data_offset,
+                      buf.data_len - 19};
+        // Place some values into the buffer for the JS side to verify.
+        buf2.data_ptr[0] = 200 + i;
+        buf2.data_ptr[buf2.data_len - 1] = 200 - i;
+        // Send back.
+        deno_respond(deno, req_id, buf2);
+      });
   EXPECT_TRUE(deno_execute(d, "a.js", "SendRecvSlice()"));
   EXPECT_EQ(count, 5);
   deno_delete(d);
@@ -139,7 +126,8 @@ TEST(LibDenoTest, SendRecvSlice) {
 
 TEST(LibDenoTest, JSSendArrayBufferViewTypes) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto _, auto buf) {
+  Deno* d = deno_new(nullptr, [](auto _, int req_id, auto buf, auto data_buf) {
+    assert_null(data_buf);
     count++;
     size_t data_offset = buf.data_ptr - buf.alloc_ptr;
     EXPECT_EQ(data_offset, 2468u);
@@ -166,7 +154,8 @@ TEST(LibDenoTest, SnapshotBug) {
 
 TEST(LibDenoTest, GlobalErrorHandling) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto _, auto buf) {
+  Deno* d = deno_new(nullptr, [](auto _, int req_id, auto buf, auto data_buf) {
+    assert_null(data_buf);
     count++;
     EXPECT_EQ(static_cast<size_t>(1), buf.data_len);
     EXPECT_EQ(buf.data_ptr[0], 42);
@@ -182,14 +171,25 @@ TEST(LibDenoTest, DoubleGlobalErrorHandlingFails) {
   deno_delete(d);
 }
 
-TEST(LibDenoTest, SendNullAllocPtr) {
+TEST(LibDenoTest, DataBuf) {
   static int count = 0;
-  Deno* d = deno_new(nullptr, [](auto _, auto buf) { count++; });
-  EXPECT_TRUE(deno_execute(d, "a.js", "SendNullAllocPtr()"));
-  deno_buf buf = StrBufNullAllocPtr("abcd");
-  EXPECT_EQ(buf.alloc_ptr, nullptr);
-  EXPECT_EQ(buf.data_len, 4u);
-  EXPECT_TRUE(deno_send(d, buf));
-  EXPECT_EQ(count, 0);
+  static deno_buf data_buf_copy;
+  Deno* d = deno_new(nullptr,
+                     [](auto _, int req_id, deno_buf buf, deno_buf data_buf) {
+                       count++;
+                       data_buf.data_ptr[0] = 4;
+                       data_buf.data_ptr[1] = 2;
+                       data_buf_copy = data_buf;
+                       EXPECT_EQ(2u, buf.data_len);
+                       EXPECT_EQ(2u, data_buf.data_len);
+                       EXPECT_EQ(buf.data_ptr[0], 1);
+                       EXPECT_EQ(buf.data_ptr[1], 2);
+                     });
+  EXPECT_TRUE(deno_execute(d, "a.js", "DataBuf()"));
+  EXPECT_EQ(count, 1);
+  // data_buf was subsequently changed in JS, let's check that our copy reflects
+  // that.
+  EXPECT_EQ(data_buf_copy.data_ptr[0], 9);
+  EXPECT_EQ(data_buf_copy.data_ptr[1], 8);
   deno_delete(d);
 }

libdeno/libdeno_test.js

@@ -25,18 +25,6 @@ global.TypedArraySnapshots = () => {
   assert(snapshotted[3] === 7);
 };
 
-global.SendSuccess = () => {
-  libdeno.recv(msg => {
-    libdeno.print("SendSuccess: ok");
-  });
-};
-
-global.SendWrongByteLength = () => {
-  libdeno.recv(msg => {
-    assert(msg.byteLength === 3);
-  });
-};
-
 global.RecvReturnEmpty = () => {
   const m1 = new Uint8Array("abc".split("").map(c => c.charCodeAt(0)));
   const m2 = m1.slice();
@@ -128,13 +116,20 @@ global.DoubleGlobalErrorHandlingFails = () => {
   libdeno.setGlobalErrorHandler((message, source, line, col, error) => {});
 };
 
-global.SendNullAllocPtr = () => {
-  libdeno.recv(msg => {
-    assert(msg instanceof Uint8Array);
-    assert(msg.byteLength === 4);
-    assert(msg[0] === "a".charCodeAt(0));
-    assert(msg[1] === "b".charCodeAt(0));
-    assert(msg[2] === "c".charCodeAt(0));
-    assert(msg[3] === "d".charCodeAt(0));
-  });
+// Allocate this buf at the top level to avoid GC.
+const dataBuf = new Uint8Array([3, 4]);
+
+global.DataBuf = () => {
+  const a = new Uint8Array([1, 2]);
+  const b = dataBuf;
+  // The second parameter of send should modified by the
+  // privileged side.
+  const r = libdeno.send(a, b);
+  assert(r == null);
+  // b is different.
+  assert(b[0] === 4);
+  assert(b[1] === 2);
+  // Now we modify it again.
+  b[0] = 9;
+  b[1] = 8;
 };

src/handlers.rs

@@ -33,15 +33,21 @@ type OpResult = DenoResult<Buf>;
 
 // TODO Ideally we wouldn't have to box the Op being returned.
 // The box is just to make it easier to get a prototype refactor working.
-type Handler = fn(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op>;
+type Handler =
+  fn(state: Arc<IsolateState>, base: &msg::Base, data: &'static mut [u8])
+    -> Box<Op>;
 
 // Hopefully Rust optimizes this away.
 fn empty_buf() -> Buf {
   Box::new([])
 }
 
-pub fn msg_from_js(state: Arc<IsolateState>, bytes: &[u8]) -> (bool, Box<Op>) {
-  let base = msg::get_root_as_base(bytes);
+pub fn msg_from_js(
+  state: Arc<IsolateState>,
+  control: &[u8],
+  data: &'static mut [u8],
+) -> (bool, Box<Op>) {
+  let base = msg::get_root_as_base(control);
   let is_sync = base.sync();
   let msg_type = base.msg_type();
   let cmd_id = base.cmd_id();
@@ -71,7 +77,7 @@ pub fn msg_from_js(state: Arc<IsolateState>, bytes: &[u8]) -> (bool, Box<Op>) {
     )),
   };
 
-  let op: Box<Op> = handler(state.clone(), &base);
+  let op: Box<Op> = handler(state.clone(), &base, data);
   let boxed_op = Box::new(
     op.or_else(move |err: DenoError| -> DenoResult<Buf> {
       debug!("op err {}", err);
@@ -130,12 +136,21 @@ fn not_implemented() -> DenoError {
   ))
 }
 
-fn handle_exit(_config: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_exit(
+  _config: Arc<IsolateState>,
+  base: &msg::Base,
+  _data: &'static mut [u8],
+) -> Box<Op> {
   let msg = base.msg_as_exit().unwrap();
   std::process::exit(msg.code())
 }
 
-fn handle_start(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_start(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let mut builder = FlatBufferBuilder::new();
 
   let argv = state.argv.iter().map(|s| s.as_str()).collect::<Vec<_>>();
@@ -191,7 +206,12 @@ fn odd_future(err: DenoError) -> Box<Op> {
 }
 
 // https://github.com/denoland/isolate/blob/golang/os.go#L100-L154
-fn handle_code_fetch(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_code_fetch(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_code_fetch().unwrap();
   let cmd_id = base.cmd_id();
   let module_specifier = msg.module_specifier().unwrap();
@@ -228,7 +248,12 @@ fn handle_code_fetch(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
 }
 
 // https://github.com/denoland/isolate/blob/golang/os.go#L156-L169
-fn handle_code_cache(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_code_cache(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_code_cache().unwrap();
   let filename = msg.filename().unwrap();
   let source_code = msg.source_code().unwrap();
@@ -239,7 +264,12 @@ fn handle_code_cache(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   }()))
 }
 
-fn handle_set_timeout(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_set_timeout(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_set_timeout().unwrap();
   let val = msg.timeout() as isize;
   state
@@ -248,7 +278,12 @@ fn handle_set_timeout(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   ok_future(empty_buf())
 }
 
-fn handle_set_env(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_set_env(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_set_env().unwrap();
   let key = msg.key().unwrap();
   let value = msg.value().unwrap();
@@ -261,7 +296,12 @@ fn handle_set_env(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   ok_future(empty_buf())
 }
 
-fn handle_env(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_env(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let cmd_id = base.cmd_id();
 
   if !state.flags.allow_env {
@@ -302,7 +342,12 @@ fn handle_env(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   ))
 }
 
-fn handle_fetch_req(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_fetch_req(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_fetch_req().unwrap();
   let cmd_id = base.cmd_id();
   let id = msg.id();
@@ -436,7 +481,12 @@ macro_rules! blocking {
   };
 }
 
-fn handle_make_temp_dir(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_make_temp_dir(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let base = Box::new(*base);
   let msg = base.msg_as_make_temp_dir().unwrap();
   let cmd_id = base.cmd_id();
@@ -480,7 +530,12 @@ fn handle_make_temp_dir(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   })
 }
 
-fn handle_mkdir(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_mkdir(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_mkdir().unwrap();
   let mode = msg.mode();
   let path = String::from(msg.path().unwrap());
@@ -496,7 +551,12 @@ fn handle_mkdir(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   })
 }
 
-fn handle_remove(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_remove(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_remove().unwrap();
   let path = PathBuf::from(msg.path().unwrap());
   let recursive = msg.recursive();
@@ -520,7 +580,12 @@ fn handle_remove(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
 }
 
 // Prototype https://github.com/denoland/isolate/blob/golang/os.go#L171-L184
-fn handle_read_file(_config: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_read_file(
+  _config: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_read_file().unwrap();
   let cmd_id = base.cmd_id();
   let filename = PathBuf::from(msg.filename().unwrap());
@@ -570,7 +635,12 @@ fn get_mode(_perm: fs::Permissions) -> u32 {
   0
 }
 
-fn handle_stat(_config: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_stat(
+  _config: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_stat().unwrap();
   let cmd_id = base.cmd_id();
   let filename = PathBuf::from(msg.filename().unwrap());
@@ -612,7 +682,11 @@ fn handle_stat(_config: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   })
 }
 
-fn handle_write_file(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_write_file(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
   let msg = base.msg_as_write_file().unwrap();
 
   if !state.flags.allow_write {
@@ -620,12 +694,11 @@ fn handle_write_file(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   }
 
   let filename = String::from(msg.filename().unwrap());
-  let data = Vec::from(msg.data().unwrap());
   let perm = msg.perm();
 
   blocking!(base.sync(), || -> OpResult {
-    debug!("handle_write_file {}", filename);
-    deno_fs::write_file(Path::new(&filename), data.as_slice(), perm)?;
+    debug!("handle_write_file {} {}", filename, data.len());
+    deno_fs::write_file(Path::new(&filename), data, perm)?;
     Ok(empty_buf())
   })
 }
@@ -636,7 +709,12 @@ fn remove_timer(state: Arc<IsolateState>, timer_id: u32) {
 }
 
 // Prototype: https://github.com/ry/isolate/blob/golang/timers.go#L25-L39
-fn handle_timer_start(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_timer_start(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   debug!("handle_timer_start");
   let msg = base.msg_as_timer_start().unwrap();
   let cmd_id = base.cmd_id();
@@ -679,14 +757,24 @@ fn handle_timer_start(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
 }
 
 // Prototype: https://github.com/ry/isolate/blob/golang/timers.go#L40-L43
-fn handle_timer_clear(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_timer_clear(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_timer_clear().unwrap();
   debug!("handle_timer_clear");
   remove_timer(state, msg.id());
   ok_future(empty_buf())
 }
 
-fn handle_rename(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_rename(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   if !state.flags.allow_write {
     return odd_future(permission_denied());
   }
@@ -700,7 +788,12 @@ fn handle_rename(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   })
 }
 
-fn handle_symlink(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_symlink(
+  state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   if !state.flags.allow_write {
     return odd_future(permission_denied());
   }
@@ -720,7 +813,12 @@ fn handle_symlink(state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
   })
 }
 
-fn handle_read_link(_state: Arc<IsolateState>, base: &msg::Base) -> Box<Op> {
+fn handle_read_link(
+  _state: Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
   let msg = base.msg_as_readlink().unwrap();
   let cmd_id = base.cmd_id();
   let name = PathBuf::from(msg.name().unwrap());

src/isolate.rs

@@ -36,12 +36,14 @@ pub type Buf = Box<[u8]>;
 pub type Op = Future<Item = Buf, Error = DenoError> + Send;
 
 // Returns (is_sync, op)
-pub type Dispatch = fn(state: Arc<IsolateState>, buf: &[u8]) -> (bool, Box<Op>);
+pub type Dispatch =
+  fn(state: Arc<IsolateState>, buf: &[u8], data_buf: &'static mut [u8])
+    -> (bool, Box<Op>);
 
 pub struct Isolate {
   ptr: *const libdeno::isolate,
   dispatch: Dispatch,
-  rx: mpsc::Receiver<Buf>,
+  rx: mpsc::Receiver<(i32, Buf)>,
   ntasks: i32,
   pub state: Arc<IsolateState>,
 }
@@ -54,17 +56,17 @@ pub struct IsolateState {
   pub timers: Mutex<HashMap<u32, futures::sync::oneshot::Sender<()>>>,
   pub argv: Vec<String>,
   pub flags: flags::DenoFlags,
-  tx: Mutex<Option<mpsc::Sender<Buf>>>,
+  tx: Mutex<Option<mpsc::Sender<(i32, Buf)>>>,
 }
 
 impl IsolateState {
   // Thread safe.
-  fn send_to_js(&self, buf: Buf) {
+  fn send_to_js(&self, req_id: i32, buf: Buf) {
     let mut g = self.tx.lock().unwrap();
     let maybe_tx = g.as_mut();
     assert!(maybe_tx.is_some(), "Expected tx to not be deleted.");
     let tx = maybe_tx.unwrap();
-    tx.send(buf).expect("tx.send error");
+    tx.send((req_id, buf)).expect("tx.send error");
   }
 }
 
@@ -79,7 +81,7 @@ impl Isolate {
     let (flags, argv_rest) = flags::set_flags(argv);
 
     // This channel handles sending async messages back to the runtime.
-    let (tx, rx) = mpsc::channel::<Buf>();
+    let (tx, rx) = mpsc::channel::<(i32, Buf)>();
 
     let mut isolate = Box::new(Isolate {
       ptr: 0 as *const libdeno::isolate,
@@ -131,12 +133,10 @@ impl Isolate {
     Ok(())
   }
 
-  pub fn set_response(&self, buf: Buf) {
-    unsafe { libdeno::deno_set_response(self.ptr, buf.into()) }
-  }
-
-  pub fn send(&self, buf: Buf) {
-    unsafe { libdeno::deno_send(self.ptr, buf.into()) };
+  pub fn respond(&self, req_id: i32, buf: Buf) {
+    // TODO(zero-copy) Use Buf::leak(buf) to leak the heap allocated buf. And
+    // don't do the memcpy in ImportBuf() (in libdeno/binding.cc)
+    unsafe { libdeno::deno_respond(self.ptr, req_id, buf.into()) }
   }
 
   // TODO Use Park abstraction? Note at time of writing Tokio default runtime
@@ -144,12 +144,12 @@ impl Isolate {
   pub fn event_loop(&mut self) {
     // Main thread event loop.
     while !self.is_idle() {
-      let buf = self.rx.recv().unwrap();
+      let (req_id, buf) = self.rx.recv().unwrap();
       // Receiving a message on rx exactly corresponds to an async task
       // completing.
       self.ntasks_decrement();
       // Call into JS with the buf.
-      self.send(buf);
+      self.respond(req_id, buf);
     }
   }
 
@@ -189,18 +189,38 @@ impl From<Buf> for libdeno::deno_buf {
 }
 
 // Dereferences the C pointer into the Rust Isolate object.
-extern "C" fn pre_dispatch(d: *const libdeno::isolate, buf: libdeno::deno_buf) {
-  let bytes = unsafe { std::slice::from_raw_parts(buf.data_ptr, buf.data_len) };
+extern "C" fn pre_dispatch(
+  d: *const libdeno::isolate,
+  req_id: i32,
+  control_buf: libdeno::deno_buf,
+  data_buf: libdeno::deno_buf,
+) {
+  // control_buf is only valid for the lifetime of this call, thus is
+  // interpretted as a slice.
+  let control_slice = unsafe {
+    std::slice::from_raw_parts(control_buf.data_ptr, control_buf.data_len)
+  };
+
+  // data_buf is valid for the lifetime of the promise, thus a mutable buf with
+  // static lifetime.
+  let data_slice = unsafe {
+    std::slice::from_raw_parts_mut::<'static>(
+      data_buf.data_ptr,
+      data_buf.data_len,
+    )
+  };
+
   let isolate = Isolate::from_c(d);
   let dispatch = isolate.dispatch;
-  let (is_sync, op) = dispatch(isolate.state.clone(), bytes);
+  let (is_sync, op) =
+    dispatch(isolate.state.clone(), control_slice, data_slice);
 
   if is_sync {
     // Execute op synchronously.
     let buf = tokio_util::block_on(op).unwrap();
     if buf.len() != 0 {
       // Set the synchronous response, the value returned from isolate.send().
-      isolate.set_response(buf);
+      isolate.respond(req_id, buf);
     }
   } else {
     // Execute op asynchronously.
@@ -213,7 +233,7 @@ extern "C" fn pre_dispatch(d: *const libdeno::isolate, buf: libdeno::deno_buf) {
 
     let task = op
       .and_then(move |buf| {
-        state.send_to_js(buf);
+        state.send_to_js(req_id, buf);
         Ok(())
       }).map_err(|_| ());
     tokio::spawn(task);
@@ -239,7 +259,8 @@ mod tests {
 
   fn unreachable_dispatch(
     _state: Arc<IsolateState>,
-    _buf: &[u8],
+    _control: &[u8],
+    _data: &'static mut [u8],
   ) -> (bool, Box<Op>) {
     unreachable!();
   }
@@ -267,14 +288,19 @@ mod tests {
     });
   }
 
-  fn dispatch_sync(_state: Arc<IsolateState>, buf: &[u8]) -> (bool, Box<Op>) {
-    assert_eq!(buf[0], 4);
-    assert_eq!(buf[1], 5);
-    assert_eq!(buf[2], 6);
+  fn dispatch_sync(
+    _state: Arc<IsolateState>,
+    control: &[u8],
+    data: &'static mut [u8],
+  ) -> (bool, Box<Op>) {
+    assert_eq!(control[0], 4);
+    assert_eq!(control[1], 5);
+    assert_eq!(control[2], 6);
+    assert_eq!(data.len(), 0);
     // Send back some sync response.
     let vec: Vec<u8> = vec![1, 2, 3];
-    let buf = vec.into_boxed_slice();
-    let op = Box::new(futures::future::ok(buf));
+    let control = vec.into_boxed_slice();
+    let op = Box::new(futures::future::ok(control));
     (true, op)
   }
 }

src/libdeno.rs

@@ -19,7 +19,12 @@ pub struct deno_buf {
   pub data_len: usize,
 }
 
-type DenoRecvCb = unsafe extern "C" fn(d: *const isolate, buf: deno_buf);
+type DenoRecvCb = unsafe extern "C" fn(
+  d: *const isolate,
+  req_id: i32,
+  buf: deno_buf,
+  data_buf: deno_buf,
+);
 
 extern "C" {
   pub fn deno_init();
@@ -29,8 +34,7 @@ extern "C" {
   pub fn deno_delete(i: *const isolate);
   pub fn deno_last_exception(i: *const isolate) -> *const c_char;
   pub fn deno_get_data(i: *const isolate) -> *const c_void;
-  pub fn deno_set_response(i: *const isolate, buf: deno_buf);
-  pub fn deno_send(i: *const isolate, buf: deno_buf);
+  pub fn deno_respond(i: *const isolate, req_id: i32, buf: deno_buf);
   pub fn deno_execute(
     i: *const isolate,
     js_filename: *const c_char,

src/msg.fbs

@@ -199,7 +199,6 @@ table ReadFileRes {
 
 table WriteFile {
   filename: string;
-  data: [ubyte];
   perm: uint;
   // perm specified by https://godoc.org/os#FileMode
 }