Support request method and headers in fetch() (#1188)

Adds a general HttpHeader flatbuffer message for serializing requests
and responses.

BUILD.gn

@@ -55,6 +55,7 @@ main_extern = [
   "$rust_build:dirs",
   "$rust_build:futures",
   "$rust_build:getopts",
+  "$rust_build:http",
   "$rust_build:hyper",
   "$rust_build:hyper_rustls",
   "$rust_build:lazy_static",

Cargo.toml

@@ -14,6 +14,7 @@ dirs = "1.0.4"
 flatbuffers = { path = "third_party/flatbuffers/rust/flatbuffers/" }
 futures = "0.1.25"
 getopts = "0.2.18"
+http = "0.1.13"
 hyper = "0.12.13"
 # The current version of hyper-rustls, 0.14.0, depends on tokio-core, which is
 # deprecated.

js/fetch.ts

@@ -1,5 +1,5 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
-import { assert, log, createResolvable, notImplemented } from "./util";
+import { assert, createResolvable, notImplemented } from "./util";
 import * as flatbuffers from "./flatbuffers";
 import { sendAsync } from "./dispatch";
 import * as msg from "gen/msg_generated";
@@ -162,19 +162,73 @@ class Response implements domTypes.Response {
   }
 }
 
+function msgHttpRequest(
+  builder: flatbuffers.Builder,
+  url: string,
+  method: null | string,
+  headers: null | domTypes.Headers
+): flatbuffers.Offset {
+  const methodOffset = !method ? -1 : builder.createString(method);
+  let fieldsOffset: flatbuffers.Offset = -1;
+  const urlOffset = builder.createString(url);
+  if (headers) {
+    const kvOffsets: flatbuffers.Offset[] = [];
+    for (const [key, val] of headers.entries()) {
+      const keyOffset = builder.createString(key);
+      const valOffset = builder.createString(val);
+      msg.KeyValue.startKeyValue(builder);
+      msg.KeyValue.addKey(builder, keyOffset);
+      msg.KeyValue.addValue(builder, valOffset);
+      kvOffsets.push(msg.KeyValue.endKeyValue(builder));
+    }
+    fieldsOffset = msg.HttpHeader.createFieldsVector(builder, kvOffsets);
+  } else {
+  }
+  msg.HttpHeader.startHttpHeader(builder);
+  msg.HttpHeader.addIsRequest(builder, true);
+  msg.HttpHeader.addUrl(builder, urlOffset);
+  if (methodOffset >= 0) {
+    msg.HttpHeader.addMethod(builder, methodOffset);
+  }
+  if (fieldsOffset >= 0) {
+    msg.HttpHeader.addFields(builder, fieldsOffset);
+  }
+  return msg.HttpHeader.endHttpHeader(builder);
+}
+
 /** Fetch a resource from the network. */
 export async function fetch(
-  input?: domTypes.Request | string,
+  input: domTypes.Request | string,
   init?: domTypes.RequestInit
 ): Promise<Response> {
-  const url = input as string;
-  log("dispatch FETCH_REQ", url);
+  let url: string;
+  let method: string | null = null;
+  let headers: domTypes.Headers | null = null;
+
+  if (typeof input === "string") {
+    url = input;
+    if (init != null) {
+      method = init.method || null;
+      if (init.headers) {
+        headers =
+          init.headers instanceof Headers
+            ? init.headers
+            : new Headers(init.headers);
+      } else {
+        headers = null;
+      }
+    }
+  } else {
+    url = input.url;
+    method = input.method;
+    headers = input.headers;
+  }
 
   // Send Fetch message
   const builder = flatbuffers.createBuilder();
-  const url_ = builder.createString(url);
+  const headerOff = msgHttpRequest(builder, url, method, headers);
   msg.Fetch.startFetch(builder);
-  msg.Fetch.addUrl(builder, url_);
+  msg.Fetch.addHeader(builder, headerOff);
   const resBase = await sendAsync(
     builder,
     msg.Any.Fetch,
@@ -186,17 +240,22 @@ export async function fetch(
   const inner = new msg.FetchRes();
   assert(resBase.inner(inner) != null);
 
-  const status = inner.status();
+  const header = inner.header()!;
   const bodyRid = inner.bodyRid();
+  assert(!header.isRequest());
+  const status = header.status();
 
-  const headersList: Array<[string, string]> = [];
-  const len = inner.headerKeyLength();
-  for (let i = 0; i < len; ++i) {
-    const key = inner.headerKey(i);
-    const value = inner.headerValue(i);
-    headersList.push([key, value]);
-  }
+  const headersList = deserializeHeaderFields(header);
 
   const response = new Response(status, headersList, bodyRid);
   return response;
 }
+
+function deserializeHeaderFields(m: msg.HttpHeader): Array<[string, string]> {
+  const out: Array<[string, string]> = [];
+  for (let i = 0; i < m.fieldsLength(); i++) {
+    const item = m.fields(i)!;
+    out.push([item.key()!, item.value()!]);
+  }
+  return out;
+}

js/fetch_test.ts

@@ -46,3 +46,34 @@ testPerm({ net: true }, async function responseClone() {
     assertEqual(ab[i], ab1[i]);
   }
 });
+
+testPerm({ net: true }, async function fetchRequest() {
+  const addr = "127.0.0.1:4501";
+  const listener = deno.listen("tcp", addr);
+  const buf = new deno.Buffer();
+  listener.accept().then(async conn => {
+    buf.readFrom(conn);
+    await conn.write(
+      new TextEncoder().encode(
+        "HTTP/1.0 404 Not Found\r\nContent-Length: 2\r\n\r\nNF"
+      )
+    );
+    conn.close();
+  });
+  const response = await fetch(`http://${addr}/blah`, {
+    method: "POST",
+    headers: [["Hello", "World"], ["Foo", "Bar"]]
+  });
+  listener.close();
+  assertEqual(response.status, 404);
+  assertEqual(response.headers.get("Content-Length"), "2");
+
+  const actual = new TextDecoder().decode(buf.bytes());
+  const expected = [
+    "POST /blah HTTP/1.1\r\n",
+    "hello: World\r\n",
+    "foo: Bar\r\n",
+    `host: ${addr}\r\n\r\n`
+  ].join("");
+  assertEqual(actual, expected);
+});

js/headers.ts

@@ -79,7 +79,8 @@ class HeadersBase {
 
 // @internal
 // tslint:disable-next-line:variable-name
-export const Headers = DomIterableMixin<string, string, typeof HeadersBase>(
-  HeadersBase,
-  headerMap
-);
+export class Headers extends DomIterableMixin<
+  string,
+  string,
+  typeof HeadersBase
+>(HeadersBase, headerMap) {}

src/main.rs

@@ -2,6 +2,7 @@
 extern crate dirs;
 extern crate flatbuffers;
 extern crate getopts;
+extern crate http;
 extern crate hyper;
 extern crate hyper_rustls;
 extern crate libc;

src/msg.fbs

@@ -194,18 +194,26 @@ table KeyValue {
   value: string;
 }
 
-table Fetch {
-  id: uint;
+// Note this represents The WHOLE header of an http message, not just the key
+// value pairs. That means it includes method and url for Requests and status
+// for responses. This is why it is singular "Header" instead of "Headers".
+table HttpHeader {
+  is_request: bool;
+  // Request only:
+  method: string;
   url: string;
-  // TODO Supply request headers:
-  // header_line: [string];
+  // Response only:
+  status: uint16;
+  // Both:
+  fields: [KeyValue];
+}
+
+table Fetch {
+  header: HttpHeader;
 }
 
 table FetchRes {
-  id: uint;
-  status: int;
-  header_key: [string];
-  header_value: [string];
+  header: HttpHeader;
   body_rid: uint32;
 }
 

src/msg_util.rs

@@ -1,7 +1,18 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
 // Helpers for serialization.
 use flatbuffers;
+use http::header::HeaderName;
+use http::uri::Uri;
+use http::Method;
+use hyper::header::HeaderMap;
+use hyper::header::HeaderValue;
+use hyper::Body;
+use hyper::Request;
+use hyper::Response;
 use msg;
+use std::str::FromStr;
+
+type Headers = HeaderMap<HeaderValue>;
 
 pub fn serialize_key_value<'bldr>(
   builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
@@ -15,6 +26,99 @@ pub fn serialize_key_value<'bldr>(
     &msg::KeyValueArgs {
       key: Some(key),
       value: Some(value),
+      ..Default::default()
     },
   )
 }
+
+pub fn serialize_request_header<'bldr>(
+  builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
+  r: &Request<Body>,
+) -> flatbuffers::WIPOffset<msg::HttpHeader<'bldr>> {
+  let method = builder.create_string(r.method().as_str());
+  let url = builder.create_string(r.uri().to_string().as_ref());
+
+  let mut fields = Vec::new();
+  for (key, val) in r.headers().iter() {
+    let kv = serialize_key_value(builder, key.as_ref(), val.to_str().unwrap());
+    fields.push(kv);
+  }
+  let fields = builder.create_vector(fields.as_ref());
+
+  msg::HttpHeader::create(
+    builder,
+    &msg::HttpHeaderArgs {
+      is_request: true,
+      method: Some(method),
+      url: Some(url),
+      fields: Some(fields),
+      ..Default::default()
+    },
+  )
+}
+
+pub fn serialize_fields<'bldr>(
+  builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
+  headers: &Headers,
+) -> flatbuffers::WIPOffset<
+  flatbuffers::Vector<
+    'bldr,
+    flatbuffers::ForwardsUOffset<msg::KeyValue<'bldr>>,
+  >,
+> {
+  let mut fields = Vec::new();
+  for (key, val) in headers.iter() {
+    let kv = serialize_key_value(builder, key.as_ref(), val.to_str().unwrap());
+    fields.push(kv);
+  }
+  return builder.create_vector(fields.as_ref());
+}
+
+// Not to be confused with serialize_response which has nothing to do with HTTP.
+pub fn serialize_http_response<'bldr>(
+  builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
+  r: &Response<Body>,
+) -> flatbuffers::WIPOffset<msg::HttpHeader<'bldr>> {
+  let status = r.status().as_u16();
+  let fields = serialize_fields(builder, r.headers());
+  msg::HttpHeader::create(
+    builder,
+    &msg::HttpHeaderArgs {
+      is_request: false,
+      status,
+      fields: Some(fields),
+      ..Default::default()
+    },
+  )
+}
+
+pub fn deserialize_request(
+  header_msg: msg::HttpHeader,
+  body: Body,
+) -> Request<Body> {
+  let mut r = Request::new(body);
+
+  assert!(header_msg.is_request() == true);
+
+  let url = header_msg.url().unwrap();
+  let uri = Uri::from_str(url).unwrap();
+  *r.uri_mut() = uri;
+
+  if let Some(method) = header_msg.method() {
+    let method = Method::from_str(method).unwrap();
+    *r.method_mut() = method;
+  }
+
+  if let Some(fields) = header_msg.fields() {
+    let headers = r.headers_mut();
+    for i in 0..fields.len() {
+      let kv = fields.get(i);
+      let key = kv.key().unwrap();
+      let name = HeaderName::from_bytes(key.as_bytes()).unwrap();
+      let value = kv.value().unwrap();
+      let v = HeaderValue::from_str(value).unwrap();
+      headers.insert(name, v);
+    }
+  }
+  r
+}

src/ops.rs

@@ -389,74 +389,49 @@ fn op_fetch(
   assert_eq!(data.len(), 0);
   let inner = base.inner_as_fetch().unwrap();
   let cmd_id = base.cmd_id();
-  let id = inner.id();
-  let url = inner.url().unwrap();
+
+  let header = inner.header().unwrap();
+  assert!(header.is_request());
+  let url = header.url().unwrap();
+
+  let body = hyper::Body::empty();
+  let req = msg_util::deserialize_request(header, body);
 
   if let Err(e) = state.check_net(url) {
     return odd_future(e);
   }
 
-  let url = url.parse::<hyper::Uri>().unwrap();
   let client = http_util::get_client();
 
   debug!("Before fetch {}", url);
-  let future = client.get(url).and_then(move |res| {
-    let status = i32::from(res.status().as_u16());
-    debug!("fetch {}", status);
+  let future =
+    client
+      .request(req)
+      .map_err(DenoError::from)
+      .and_then(move |res| {
+        let builder = &mut FlatBufferBuilder::new();
+        let header_off = msg_util::serialize_http_response(builder, &res);
+        let body = res.into_body();
+        let body_resource = resources::add_hyper_body(body);
+        let inner = msg::FetchRes::create(
+          builder,
+          &msg::FetchResArgs {
+            header: Some(header_off),
+            body_rid: body_resource.rid,
+            ..Default::default()
+          },
+        );
 
-    let headers = {
-      let map = res.headers();
-      let keys = map
-        .keys()
-        .map(|s| s.as_str().to_string())
-        .collect::<Vec<_>>();
-      let values = map
-        .values()
-        .map(|s| s.to_str().unwrap().to_string())
-        .collect::<Vec<_>>();
-      (keys, values)
-    };
-
-    let body = res.into_body();
-    let body_resource = resources::add_hyper_body(body);
-    Ok((status, headers, body_resource))
-  });
-
-  let future = future.map_err(|err| -> DenoError { err.into() }).and_then(
-    move |(status, headers, body_resource)| {
-      debug!("fetch body ");
-      let builder = &mut FlatBufferBuilder::new();
-      // Send the first message without a body. This is just to indicate
-      // what status code.
-      let header_keys: Vec<&str> = headers.0.iter().map(|s| &**s).collect();
-      let header_keys_off =
-        builder.create_vector_of_strings(header_keys.as_slice());
-      let header_values: Vec<&str> = headers.1.iter().map(|s| &**s).collect();
-      let header_values_off =
-        builder.create_vector_of_strings(header_values.as_slice());
-
-      let inner = msg::FetchRes::create(
-        builder,
-        &msg::FetchResArgs {
-          id,
-          status,
-          body_rid: body_resource.rid,
-          header_key: Some(header_keys_off),
-          header_value: Some(header_values_off),
-        },
-      );
-
-      Ok(serialize_response(
-        cmd_id,
-        builder,
-        msg::BaseArgs {
-          inner: Some(inner.as_union_value()),
-          inner_type: msg::Any::FetchRes,
-          ..Default::default()
-        },
-      ))
-    },
-  );
+        Ok(serialize_response(
+          cmd_id,
+          builder,
+          msg::BaseArgs {
+            inner: Some(inner.as_union_value()),
+            inner_type: msg::Any::FetchRes,
+            ..Default::default()
+          },
+        ))
+      });
   Box::new(future)
 }