1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-21 21:50:00 -05:00

Use C++ to do flatbuffer parsing.

- Port protobuf messages to flatbuffers.
- Demo linking to rust from C++ executable.
- Start using the prototype TS libraries.
This commit is contained in:
Ryan Dahl 2018-07-06 11:27:36 -04:00
parent d9cb093989
commit 9778eceaf5
14 changed files with 1881 additions and 186 deletions

View file

@ -8,6 +8,9 @@ import("//build_extra/rust/rust.gni")
config("deno_config") {
include_dirs = [ "third_party/v8" ] # This allows us to v8/src/base/ libraries.
configs = [ "third_party/v8:external_config" ]
if (is_debug) {
defines = [ "DEBUG" ]
}
}
rust_executable("deno") {
@ -26,14 +29,21 @@ rust_component("libc") {
]
}
rust_component("handlers") {
source_root = "src/handlers.rs"
extern = [ ":libc" ]
}
executable("deno_cc") {
sources = [
"src/main.cc",
]
deps = [
":flatbufferjs",
":handlers",
":libdeno",
":msg_cpp",
"//build_extra/rust:stdlib",
]
configs += [ ":deno_config" ]
}
@ -99,8 +109,16 @@ flatbuffer("msg_cpp") {
run_node("bundle") {
out_dir = "$target_gen_dir/bundle/"
sources = [
"js/console.ts",
"js/deno.d.ts",
"js/dispatch.ts",
"js/globals.ts",
"js/main.ts",
"js/msg_generated.ts",
"js/os.ts",
"js/runtime.ts",
"js/types.ts",
"js/util.ts",
"package.json", # The `browserslist` field controls Babel behavior.
]
outputs = [

View file

@ -1,5 +1,3 @@
const print = V8Worker2.print;
// tslint:disable-next-line:no-any
type ConsoleContext = Set<any>;
@ -102,7 +100,7 @@ function stringifyArgs(args: any[]): string {
export class Console {
// tslint:disable-next-line:no-any
log(...args: any[]): void {
print(stringifyArgs(args));
deno.print(stringifyArgs(args));
}
debug = this.log;
@ -110,7 +108,7 @@ export class Console {
// tslint:disable-next-line:no-any
warn(...args: any[]): void {
print(`ERROR: ${stringifyArgs(args)}`);
deno.print(`ERROR: ${stringifyArgs(args)}`);
}
error = this.warn;

4
js/deno.d.ts vendored
View file

@ -1,9 +1,9 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
type MessageCallback = (msg: ArrayBuffer) => void;
type MessageCallback = (channel: string, msg: ArrayBuffer) => void;
interface Deno {
recv(channel: string, cb: MessageCallback): void;
recv(cb: MessageCallback): void;
send(channel: string, msg: ArrayBuffer): null | ArrayBuffer;
print(x: string): void;
}

View file

@ -1,13 +1,11 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
import { typedArrayToArrayBuffer } from "./util";
import { _global } from "./globals";
import { deno as pb } from "./msg.pb";
import { deno as fbs } from "./msg_generated";
export type MessageCallback = (msg: Uint8Array) => void;
//type MessageStructCallback = (msg: pb.IMsg) => void;
const send = V8Worker2.send;
const channels = new Map<string, MessageCallback[]>();
export function sub(channel: string, cb: MessageCallback): void {
@ -19,55 +17,14 @@ export function sub(channel: string, cb: MessageCallback): void {
subscribers.push(cb);
}
/*
export function subMsg(channel: string, cb: MessageStructCallback): void {
sub(channel, (payload: Uint8Array) => {
const msg = pb.Msg.decode(payload);
if (msg.error != null) {
f.onError(new Error(msg.error));
} else {
cb(msg);
}
});
}
*/
export function pub(channel: string, payload: Uint8Array): null | ArrayBuffer {
const msg = pb.BaseMsg.fromObject({ channel, payload });
const ui8 = pb.BaseMsg.encode(msg).finish();
const ab = typedArrayToArrayBuffer(ui8);
return send(ab);
}
// Internal version of "pub".
// TODO add internal version of "sub"
export function pubInternal(channel: string, obj: pb.IMsg): null | pb.Msg {
const msg = pb.Msg.fromObject(obj);
const ui8 = pb.Msg.encode(msg).finish();
const resBuf = pub(channel, ui8);
if (resBuf != null && resBuf.byteLength > 0) {
const res = pb.Msg.decode(new Uint8Array(resBuf));
if (res != null && res.error != null && res.error.length > 0) {
throw Error(res.error);
}
return res;
} else {
return null;
}
}
V8Worker2.recv((ab: ArrayBuffer) => {
const msg = pb.BaseMsg.decode(new Uint8Array(ab));
const subscribers = channels.get(msg.channel);
deno.recv((channel: string, ab: ArrayBuffer) => {
const subscribers = channels.get(channel);
if (subscribers == null) {
throw Error(`No subscribers for channel "${msg.channel}".`);
throw Error(`No subscribers for channel "${channel}".`);
}
const ui8 = new Uint8Array(ab);
for (const subscriber of subscribers) {
subscriber(msg.payload);
subscriber(ui8);
}
});
// Delete the V8Worker2 from the global object, so that no one else can receive
// messages.
_global["V8Worker2"] = null;

View file

@ -1,6 +1,5 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
import * as timer from "./timers";
// If you use the eval function indirectly, by invoking it via a reference
// other than eval, as of ECMAScript 5 it works in the global scope rather than
@ -11,22 +10,23 @@ export const globalEval = eval;
// A reference to the global object.
// TODO The underscore is because it's conflicting with @types/node.
export const _global = globalEval("this");
export const window = globalEval("this");
_global["window"] = _global; // Create a window object.
import "./url";
window["window"] = window; // Create a window object.
// import "./url";
_global["setTimeout"] = timer.setTimeout;
_global["setInterval"] = timer.setInterval;
_global["clearTimeout"] = timer.clearTimer;
_global["clearInterval"] = timer.clearTimer;
// import * as timer from "./timers";
// window["setTimeout"] = timer.setTimeout;
// window["setInterval"] = timer.setInterval;
// window["clearTimeout"] = timer.clearTimer;
// window["clearInterval"] = timer.clearTimer;
import { Console } from "./console";
_global["console"] = new Console();
window["console"] = new Console();
import { fetch } from "./fetch";
_global["fetch"] = fetch;
// import { fetch } from "./fetch";
// window["fetch"] = fetch;
import { TextEncoder, TextDecoder } from "text-encoding";
_global["TextEncoder"] = TextEncoder;
_global["TextDecoder"] = TextDecoder;
// import { TextEncoder, TextDecoder } from "text-encoding";
// window["TextEncoder"] = TextEncoder;
// window["TextDecoder"] = TextDecoder;

View file

@ -4,30 +4,59 @@ import * as ts from "typescript";
import { flatbuffers } from "flatbuffers";
import { deno as fbs } from "./msg_generated";
import { assert } from "./util";
// import * as runtime from "./runtime";
const globalEval = eval;
const window = globalEval("this");
function startMsg(): ArrayBuffer {
const builder = new flatbuffers.Builder();
const msg = fbs.Start.createStart(builder, 0);
fbs.Base.startBase(builder);
fbs.Base.addMsg(builder, msg);
fbs.Base.addMsgType(builder, fbs.Any.Start);
builder.finish(fbs.Base.endBase(builder));
return typedArrayToArrayBuffer(builder.asUint8Array());
}
window["denoMain"] = () => {
deno.print(`ts.version: ${ts.version}`);
const res = deno.send("startDeno2", emptyArrayBuffer());
// deno.print(`after`);
const resUi8 = new Uint8Array(res);
const bb = new flatbuffers.ByteBuffer(resUi8);
const msg = fbs.Msg.getRootAsMsg(bb);
// First we send an empty "Start" message to let the privlaged side know we
// are ready. The response should be a "StartRes" message containing the CLI
// argv and other info.
const res = deno.send("start", startMsg());
// startDebugFlag: debugFlag,
// startMainJs: mainJs,
// startMainMap: mainMap
const cwd = msg.startCwd();
// TODO(ry) Remove this conditional once main.rs gets up to speed.
if (res == null) {
console.log(`The 'Start' message got a null response. Normally this would
be an error but main.rs currently does this."); Exiting without error.`);
return;
}
// Deserialize res into startResMsg.
const bb = new flatbuffers.ByteBuffer(new Uint8Array(res));
const base = fbs.Base.getRootAsBase(bb);
assert(fbs.Any.StartRes === base.msgType());
const startResMsg = new fbs.StartRes();
assert(base.msg(startResMsg) != null);
const cwd = startResMsg.cwd();
deno.print(`cwd: ${cwd}`);
const argv: string[] = [];
for (let i = 0; i < msg.startArgvLength(); i++) {
argv.push(msg.startArgv(i));
for (let i = 0; i < startResMsg.argvLength(); i++) {
argv.push(startResMsg.argv(i));
}
deno.print(`argv ${argv}`);
/* TODO(ry) Uncomment to test further message passing.
const inputFn = argv[0];
const mod = runtime.resolveModule(inputFn, `${cwd}/`);
mod.compileAndRun();
*/
};
function typedArrayToArrayBuffer(ta: Uint8Array): ArrayBuffer {
@ -36,7 +65,3 @@ function typedArrayToArrayBuffer(ta: Uint8Array): ArrayBuffer {
ta.byteOffset + ta.byteLength
) as ArrayBuffer;
}
function emptyArrayBuffer(): ArrayBuffer {
return typedArrayToArrayBuffer(new Uint8Array([]));
}

File diff suppressed because it is too large Load diff

View file

@ -1,32 +1,54 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
import { ModuleInfo } from "./types";
import { pubInternal } from "./dispatch";
import { deno as pb } from "./msg.pb";
import { assert } from "./util";
import { deno as fbs } from "./msg_generated";
import { assert, typedArrayToArrayBuffer } from "./util";
import { flatbuffers } from "flatbuffers";
export function exit(exitCode = 0): void {
assert(false, "Not Implemented");
/*
pubInternal("os", {
command: pb.Msg.Command.EXIT,
command: fbs.Command.EXIT,
exitCode
});
*/
}
export function codeFetch(
moduleSpecifier: string,
containingFile: string
): ModuleInfo {
const res = pubInternal("os", {
command: pb.Msg.Command.CODE_FETCH,
codeFetchModuleSpecifier: moduleSpecifier,
codeFetchContainingFile: containingFile
});
assert(res.command === pb.Msg.Command.CODE_FETCH_RES);
console.log("Hello from codeFetch");
// Send CodeFetch message
const builder = new flatbuffers.Builder();
const moduleSpecifier_ = builder.createString(moduleSpecifier);
const containingFile_ = builder.createString(containingFile);
fbs.CodeFetch.startCodeFetch(builder);
fbs.CodeFetch.addModuleSpecifier(builder, moduleSpecifier_);
fbs.CodeFetch.addContainingFile(builder, containingFile_);
const msg = fbs.CodeFetch.endCodeFetch(builder);
fbs.Base.startBase(builder);
fbs.Base.addMsg(builder, msg);
fbs.Base.addMsgType(builder, fbs.Any.CodeFetch);
builder.finish(fbs.Base.endBase(builder));
const payload = typedArrayToArrayBuffer(builder.asUint8Array());
const resBuf = deno.send("x", payload);
console.log("CodeFetch sent");
// Process CodeFetchRes
const bb = new flatbuffers.ByteBuffer(new Uint8Array(resBuf));
const baseRes = fbs.Base.getRootAsBase(bb);
assert(fbs.Any.CodeFetchRes === baseRes.msgType());
const codeFetchRes = new fbs.CodeFetchRes();
assert(baseRes.msg(codeFetchRes) != null);
return {
moduleName: res.codeFetchResModuleName,
filename: res.codeFetchResFilename,
sourceCode: res.codeFetchResSourceCode,
outputCode: res.codeFetchResOutputCode
moduleName: codeFetchRes.moduleName(),
filename: codeFetchRes.filename(),
sourceCode: codeFetchRes.sourceCode(),
outputCode: codeFetchRes.outputCode(),
};
}
@ -35,20 +57,40 @@ export function codeCache(
sourceCode: string,
outputCode: string
): void {
pubInternal("os", {
command: pb.Msg.Command.CODE_CACHE,
codeCacheFilename: filename,
codeCacheSourceCode: sourceCode,
codeCacheOutputCode: outputCode
});
const builder = new flatbuffers.Builder();
const filename_ = builder.createString(filename);
const sourceCode_ = builder.createString(sourceCode);
const outputCode_ = builder.createString(outputCode);
fbs.CodeCache.startCodeCache(builder);
fbs.CodeCache.addFilename(builder, filename_);
fbs.CodeCache.addSourceCode(builder, sourceCode_);
fbs.CodeCache.addOutputCode(builder, outputCode_);
const msg = fbs.CodeCache.endCodeCache(builder);
fbs.Base.startBase(builder);
fbs.Base.addMsg(builder, msg);
builder.finish(fbs.Base.endBase(builder));
// Maybe need to do another step?
// Base.finishBaseBuffer(builder, base);
const payload = typedArrayToArrayBuffer(builder.asUint8Array());
const resBuf = deno.send("x", payload);
assert(resBuf === null);
}
export function readFileSync(filename: string): Uint8Array {
assert(false, "Not Implemented");
return null;
/*
const res = pubInternal("os", {
command: pb.Msg.Command.READ_FILE_SYNC,
command: fbs.Command.READ_FILE_SYNC,
readFileSyncFilename: filename
});
return res.readFileSyncData;
*/
}
export function writeFileSync(
@ -56,10 +98,13 @@ export function writeFileSync(
data: Uint8Array,
perm: number
): void {
assert(false, "Not Implemented");
/*
pubInternal("os", {
command: pb.Msg.Command.WRITE_FILE_SYNC,
command: fbs.Command.WRITE_FILE_SYNC,
writeFileSyncFilename: filename,
writeFileSyncData: data,
writeFileSyncPerm: perm
});
*/
}

View file

@ -12,9 +12,9 @@ import * as ts from "typescript";
import * as util from "./util";
import { log } from "./util";
import * as os from "./os";
import * as sourceMaps from "./v8_source_maps";
import { _global, globalEval } from "./globals";
import * as deno from "./deno";
//import * as sourceMaps from "./v8_source_maps";
import { window, globalEval } from "./globals";
//import * as deno from "./deno";
const EOL = "\n";
@ -22,18 +22,20 @@ const EOL = "\n";
export type AmdFactory = (...args: any[]) => undefined | object;
export type AmdDefine = (deps: string[], factory: AmdFactory) => void;
// Uncaught exceptions are sent to window.onerror by v8worker2.
// https://git.io/vhOsf
/*
// Uncaught exceptions are sent to window.onerror by the privlaged binding.
window.onerror = (message, source, lineno, colno, error) => {
// TODO Currently there is a bug in v8_source_maps.ts that causes a segfault
// if it is used within window.onerror. To workaround we uninstall the
// Error.prepareStackTrace handler. Users will get unmapped stack traces on
// uncaught exceptions until this issue is fixed.
Error.prepareStackTrace = null;
//Error.prepareStackTrace = null;
console.log(error.message, error.stack);
os.exit(1);
};
*/
/*
export function setup(mainJs: string, mainMap: string): void {
sourceMaps.install({
installPrepareStackTrace: true,
@ -52,6 +54,7 @@ export function setup(mainJs: string, mainMap: string): void {
}
});
}
*/
// This class represents a module. We call it FileModule to make it explicit
// that each module represents a single file.
@ -79,6 +82,7 @@ export class FileModule {
}
compileAndRun(): void {
util.log("compileAndRun", this.sourceCode);
if (!this.outputCode) {
// If there is no cached outputCode, then compile the code.
util.assert(
@ -89,7 +93,6 @@ export class FileModule {
this.outputCode = compiler.compile(this.fileName);
os.codeCache(this.fileName, this.sourceCode, this.outputCode);
}
util.log("compileAndRun", this.sourceCode);
execute(this.fileName, this.outputCode);
}
@ -142,17 +145,11 @@ export function resolveModule(
moduleSpecifier: string,
containingFile: string
): null | FileModule {
//util.log("resolveModule", { moduleSpecifier, containingFile });
util.log("resolveModule", { moduleSpecifier, containingFile });
util.assert(moduleSpecifier != null && moduleSpecifier.length > 0);
// We ask golang to sourceCodeFetch. It will load the sourceCode and if
// there is any outputCode cached, it will return that as well.
let fetchResponse;
try {
fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
} catch (e) {
// TODO Only catch "no such file or directory" errors. Need error codes.
return null;
}
const fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
const { filename, sourceCode, outputCode } = fetchResponse;
if (sourceCode.length === 0) {
return null;
@ -180,10 +177,10 @@ function resolveModuleName(
function execute(fileName: string, outputCode: string): void {
util.assert(outputCode && outputCode.length > 0);
_global["define"] = makeDefine(fileName);
window["define"] = makeDefine(fileName);
outputCode += `\n//# sourceURL=${fileName}`;
globalEval(outputCode);
_global["define"] = null;
window["define"] = null;
}
// This is a singleton class. Use Compiler.instance() to access.

View file

@ -1,6 +1,9 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
import { debug } from "./main";
//import { debug } from "./main";
const debug = true;
import { TypedArray } from "./types";
// Internal logging for deno. Use the "debug" variable above to control

9
src/handlers.h Normal file
View file

@ -0,0 +1,9 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
#ifndef HANDLERS_H_
#define HANDLERS_H_
extern "C" {
void handle_code_fetch(const char* module_specifier,
const char* containing_file);
}
#endif // HANDLERS_H_

27
src/handlers.rs Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
// All rights reserved. MIT License.
extern crate libc;
use libc::c_char;
use std::ffi::CStr;
fn string_from_ptr(ptr: *const c_char) -> String {
let cstr = unsafe { CStr::from_ptr(ptr as *const i8) };
String::from(cstr.to_str().unwrap())
}
#[no_mangle]
pub extern "C" fn handle_code_fetch(
module_specifier: *const c_char,
containing_file: *const c_char,
) {
let module_specifier = string_from_ptr(module_specifier);
let containing_file = string_from_ptr(containing_file);
println!(
"handle_code_fetch. module_specifier = {} containing_file = {}",
module_specifier, containing_file
);
unimplemented!();
}

View file

@ -12,15 +12,17 @@
#include "deno.h"
#include "flatbuffers/flatbuffers.h"
#include "src/handlers.h"
#include "src/msg_generated.h"
#include "third_party/v8/src/base/logging.h"
namespace deno {
static char** global_argv;
static int global_argc;
void MessagesFromJS(Deno* d, const char* channel, deno_buf buf) {
printf("MessagesFromJS %s\n", channel);
// Sends StartRes message
void HandleStart(Deno* d) {
flatbuffers::FlatBufferBuilder builder;
char cwdbuf[1024];
@ -32,22 +34,56 @@ void MessagesFromJS(Deno* d, const char* channel, deno_buf buf) {
for (int i = 0; i < global_argc; ++i) {
args.push_back(builder.CreateString(global_argv[i]));
}
auto start_argv = builder.CreateVector(args);
deno::MsgBuilder msg_builder(builder);
msg_builder.add_command(deno::Command_START);
msg_builder.add_start_cwd(start_cwd);
msg_builder.add_start_argv(start_argv);
auto response = msg_builder.Finish();
builder.Finish(response);
auto start_msg = CreateStartRes(builder, start_cwd, start_argv);
auto base = CreateBase(builder, 0, Any_StartRes, start_msg.Union());
builder.Finish(base);
deno_buf bufout{reinterpret_cast<const char*>(builder.GetBufferPointer()),
builder.GetSize()};
deno_set_response(d, bufout);
}
int main(int argc, char** argv) {
void HandleCodeFetch(Deno* d, const CodeFetch* msg) {
auto module_specifier = msg->module_specifier()->c_str();
auto containing_file = msg->containing_file()->c_str();
printf("HandleCodeFetch module_specifier = %s containing_file = %s\n",
module_specifier, containing_file);
// Call into rust.
handle_code_fetch(module_specifier, containing_file);
}
void MessagesFromJS(Deno* d, const char* channel, deno_buf buf) {
auto data = reinterpret_cast<const uint8_t*>(buf.data);
flatbuffers::Verifier verifier(data, buf.len);
DCHECK(verifier.VerifyBuffer<Base>());
auto base = flatbuffers::GetRoot<Base>(buf.data);
auto msg_type = base->msg_type();
const char* msg_type_name = EnumNamesAny()[msg_type];
printf("MessagesFromJS channel %s, msg_type = %d, msg_type_name = %s\n",
channel, msg_type, msg_type_name);
switch (msg_type) {
case Any_Start:
HandleStart(d);
break;
case Any_CodeFetch:
HandleCodeFetch(d, base->msg_as_CodeFetch());
break;
case Any_NONE:
CHECK(false && "Got message with msg_type == Any_NONE");
break;
default:
printf("Unhandled message %s\n", msg_type_name);
CHECK(false && "Unhandled message");
break;
}
}
int deno_main(int argc, char** argv) {
deno_init();
deno_set_flags(&argc, argv);
@ -61,4 +97,9 @@ int main(int argc, char** argv) {
exit(1);
}
deno_delete(d);
return 0;
}
} // namespace deno
int main(int argc, char** argv) { return deno::deno_main(argc, argv); }

View file

@ -1,12 +1,101 @@
namespace deno;
enum Command: byte {
START = 0,
union Any {
Start,
StartRes,
CodeFetch,
CodeFetchRes,
CodeCache,
Exit,
TimerStart,
TimerReady,
TimerClear,
FetchReq,
FetchRes,
ReadFileSync,
ReadFileSyncRes,
WriteFileSync,
}
table Msg {
command: Command;
start_cwd: string;
start_argv: [string];
table Base {
error: string;
msg: Any;
}
struct Start {
unused: int8;
}
table StartRes {
cwd: string;
argv: [string];
debug_flag: bool;
}
table CodeFetch {
module_specifier: string;
containing_file: string;
}
table CodeFetchRes {
// If it's a non-http module, moduleName and filename will be the same.
// For http modules, moduleName is its resolved http URL, and filename
// is the location of the locally downloaded source code.
module_name: string;
filename: string;
source_code: string;
output_code: string; // Non-empty only if cached.
}
table CodeCache {
filename: string;
source_code: string;
output_code: string;
}
struct Exit {
code: int;
}
struct TimerStart {
id: uint;
interval: bool;
delay: int;
}
struct TimerReady {
id: uint;
done: bool;
}
struct TimerClear {
id: uint;
}
table FetchReq {
id: uint;
url: string;
// header_line: [string];
}
table FetchRes {
id: uint;
status: int;
header_line: [string];
body: [byte];
}
table ReadFileSync {
filename: string;
}
table ReadFileSyncRes {
data: [byte];
}
table WriteFileSync {
filename: string;
data: [byte];
perm: uint;
// perm specified by https://godoc.org/os#FileMode
}