From d672e1405dd7085a060625fc320d063f7f7970a2 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Tue, 3 Nov 2020 06:41:20 +1100 Subject: [PATCH] refactor(cli): cleanup compiler snapshot and tsc/module_graph (#8220) --- cli/Cargo.toml | 2 + cli/build.rs | 169 ++++-- cli/main.rs | 23 +- cli/{module_graph2.rs => module_graph.rs} | 53 +- cli/module_loader.rs | 2 +- cli/op_fetch_asset.rs | 115 ---- cli/ops/runtime_compiler.rs | 8 +- cli/program_state.rs | 10 +- cli/{tsc2.rs => tsc.rs} | 46 +- cli/tsc/99_main_compiler.js | 642 ++++++---------------- 10 files changed, 397 insertions(+), 673 deletions(-) rename cli/{module_graph2.rs => module_graph.rs} (98%) delete mode 100644 cli/op_fetch_asset.rs rename cli/{tsc2.rs => tsc.rs} (93%) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 731fb2efbe..ec06e1f705 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -23,6 +23,8 @@ path = "./bench/main.rs" deno_core = { path = "../core", version = "0.66.0" } deno_web = { path = "../op_crates/web", version = "0.17.0" } deno_fetch = { path = "../op_crates/fetch", version = "0.9.0" } +regex = "1.3.9" +serde = { version = "1.0.116", features = ["derive"] } [target.'cfg(windows)'.build-dependencies] winres = "0.1.11" diff --git a/cli/build.rs b/cli/build.rs index 4819988e90..6570411320 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,9 +1,13 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -mod op_fetch_asset; - +use deno_core::error::custom_error; +use deno_core::json_op_sync; +use deno_core::serde_json; +use deno_core::serde_json::json; use deno_core::JsRuntime; use deno_core::RuntimeOptions; +use regex::Regex; +use serde::Deserialize; use std::collections::HashMap; use std::env; use std::path::Path; @@ -46,46 +50,149 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec) { create_snapshot(js_runtime, snapshot_path, files); } +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + fn create_compiler_snapshot( snapshot_path: &Path, files: Vec, cwd: &Path, ) { - let mut custom_libs: HashMap = HashMap::new(); - custom_libs - .insert("lib.deno.web.d.ts".to_string(), deno_web::get_declaration()); - custom_libs.insert( - "lib.deno.fetch.d.ts".to_string(), - deno_fetch::get_declaration(), - ); - custom_libs.insert( - "lib.deno.window.d.ts".to_string(), - cwd.join("dts/lib.deno.window.d.ts"), - ); - custom_libs.insert( - "lib.deno.worker.d.ts".to_string(), - cwd.join("dts/lib.deno.worker.d.ts"), - ); - custom_libs.insert( - "lib.deno.shared_globals.d.ts".to_string(), - cwd.join("dts/lib.deno.shared_globals.d.ts"), - ); - custom_libs.insert( - "lib.deno.ns.d.ts".to_string(), - cwd.join("dts/lib.deno.ns.d.ts"), - ); - custom_libs.insert( - "lib.deno.unstable.d.ts".to_string(), - cwd.join("dts/lib.deno.unstable.d.ts"), - ); + // libs that are being provided by op crates. + let mut op_crate_libs = HashMap::new(); + op_crate_libs.insert("deno.web", deno_web::get_declaration()); + op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); + + // ensure we invalidate the build properly. + for (_, path) in op_crate_libs.iter() { + println!("cargo:rerun-if-changed={}", path.display()); + } + + // libs that should be loaded into the isolate before snapshotting. + let libs = vec![ + // Deno custom type libraries + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.ns", + "deno.unstable", + // Deno built-in type libraries + "es5", + "es2015.collection", + "es2015.core", + "es2015", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2016", + "es2017", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020", + "es2020.intl", + "es2020.promise", + "es2020.string", + "es2020.symbol.wellknown", + "esnext", + "esnext.intl", + "esnext.promise", + "esnext.string", + "esnext.weakref", + ]; + + // create a copy of the vector that includes any op crate libs to be passed + // to the JavaScript compiler to build into the snapshot + let mut build_libs = libs.clone(); + for (op_lib, _) in op_crate_libs.iter() { + build_libs.push(op_lib.to_owned()); + } + + let re_asset = Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); + let path_dts = cwd.join("dts"); + let build_specifier = "asset:///bootstrap.ts"; let mut js_runtime = JsRuntime::new(RuntimeOptions { will_snapshot: true, ..Default::default() }); js_runtime.register_op( - "op_fetch_asset", - op_fetch_asset::op_fetch_asset(custom_libs), + "op_build_info", + json_op_sync(move |_state, _args, _bufs| { + Ok(json!({ + "buildSpecifier": build_specifier, + "libs": build_libs, + })) + }), + ); + // using the same op that is used in `tsc.rs` for loading modules and reading + // files, but a slightly different implementation at build time. + js_runtime.register_op( + "op_load", + json_op_sync(move |_state, args, _bufs| { + let v: LoadArgs = serde_json::from_value(args)?; + // we need a basic file to send to tsc to warm it up. + if v.specifier == build_specifier { + Ok(json!({ + "data": r#"console.log("hello deno!");"#, + "hash": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to + // parse out just the name so we can lookup the asset. + } else if let Some(caps) = re_asset.captures(&v.specifier) { + if let Some(lib) = caps.get(1).map(|m| m.as_str()) { + // if it comes from an op crate, we were supplied with the path to the + // file. + let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { + op_crate_lib.clone() + // otherwise we are will generate the path ourself + } else { + path_dts.join(format!("lib.{}.d.ts", lib)) + }; + let data = std::fs::read_to_string(path)?; + Ok(json!({ + "data": data, + "hash": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", v.specifier), + )) + } + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", v.specifier), + )) + } + }), ); create_snapshot(js_runtime, snapshot_path, files); } diff --git a/cli/main.rs b/cli/main.rs index dc68546d58..7773f7d484 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -35,9 +35,8 @@ mod lint; mod lockfile; mod media_type; mod metrics; -mod module_graph2; +mod module_graph; mod module_loader; -mod op_fetch_asset; mod ops; mod permissions; mod program_state; @@ -49,7 +48,7 @@ mod specifier_handler; mod test_runner; mod text_encoding; mod tokio_util; -mod tsc2; +mod tsc; mod tsc_config; mod upgrade; mod version; @@ -177,7 +176,7 @@ async fn info_command( // so we allow access to all of them. Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), @@ -241,9 +240,9 @@ async fn cache_command( files: Vec, ) -> Result<(), AnyError> { let lib = if flags.unstable { - module_graph2::TypeLib::UnstableDenoWindow + module_graph::TypeLib::UnstableDenoWindow } else { - module_graph2::TypeLib::DenoWindow + module_graph::TypeLib::DenoWindow }; let program_state = ProgramState::new(flags)?; @@ -329,7 +328,7 @@ async fn bundle_command( // therefore we will allow the graph to access any module. Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), @@ -341,12 +340,12 @@ async fn bundle_command( if !flags.no_check { // TODO(@kitsonk) support bundling for workers let lib = if flags.unstable { - module_graph2::TypeLib::UnstableDenoWindow + module_graph::TypeLib::UnstableDenoWindow } else { - module_graph2::TypeLib::DenoWindow + module_graph::TypeLib::DenoWindow }; let graph = graph.clone(); - let result_info = graph.check(module_graph2::CheckOptions { + let result_info = graph.check(module_graph::CheckOptions { debug, emit: false, lib, @@ -364,7 +363,7 @@ async fn bundle_command( } let (output, stats, maybe_ignored_options) = - graph.bundle(module_graph2::BundleOptions { + graph.bundle(module_graph::BundleOptions { debug, maybe_config_path: flags.config_path, })?; @@ -563,7 +562,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> { &program_state, Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), diff --git a/cli/module_graph2.rs b/cli/module_graph.rs similarity index 98% rename from cli/module_graph2.rs rename to cli/module_graph.rs index f795a7acbe..d757aa0a12 100644 --- a/cli/module_graph2.rs +++ b/cli/module_graph.rs @@ -22,7 +22,7 @@ use crate::specifier_handler::DependencyMap; use crate::specifier_handler::Emit; use crate::specifier_handler::FetchFuture; use crate::specifier_handler::SpecifierHandler; -use crate::tsc2; +use crate::tsc; use crate::tsc_config::IgnoredCompilerOptions; use crate::tsc_config::TsConfig; use crate::version; @@ -122,12 +122,12 @@ struct BundleLoader<'a> { cm: Rc, emit_options: &'a ast::EmitOptions, globals: &'a swc_common::Globals, - graph: &'a Graph2, + graph: &'a Graph, } impl<'a> BundleLoader<'a> { pub fn new( - graph: &'a Graph2, + graph: &'a Graph, emit_options: &'a ast::EmitOptions, globals: &'a swc_common::Globals, cm: Rc, @@ -632,7 +632,7 @@ pub struct TranspileOptions { /// the builder will be loaded into the graph. Also provides an interface to /// be able to manipulate and handle the graph. #[derive(Debug, Clone)] -pub struct Graph2 { +pub struct Graph { /// A reference to the specifier handler that will retrieve and cache modules /// for the graph. handler: Rc>, @@ -658,7 +658,7 @@ pub struct Graph2 { maybe_lockfile: Option>>, } -impl Graph2 { +impl Graph { /// Create a new instance of a graph, ready to have modules loaded it. /// /// The argument `handler` is an instance of a structure that implements the @@ -668,7 +668,7 @@ impl Graph2 { handler: Rc>, maybe_lockfile: Option>>, ) -> Self { - Graph2 { + Graph { handler, maybe_tsbuildinfo: None, modules: HashMap::new(), @@ -776,9 +776,9 @@ impl Graph2 { vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; let graph = Rc::new(RefCell::new(self)); - let response = tsc2::exec( + let response = tsc::exec( js::compiler_isolate_init(), - tsc2::Request { + tsc::Request { config: config.clone(), debug: options.debug, graph: graph.clone(), @@ -897,9 +897,9 @@ impl Graph2 { vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; let graph = Rc::new(RefCell::new(self)); - let response = tsc2::exec( + let response = tsc::exec( js::compiler_isolate_init(), - tsc2::Request { + tsc::Request { config: config.clone(), debug: options.debug, graph: graph.clone(), @@ -987,7 +987,7 @@ impl Graph2 { ); let output = bundler .bundle(entries) - .context("Unable to output bundle during Graph2::bundle().")?; + .context("Unable to output bundle during Graph::bundle().")?; let mut buf = Vec::new(); { let mut emitter = swc_ecmascript::codegen::Emitter { @@ -1001,7 +1001,7 @@ impl Graph2 { emitter .emit_module(&output[0].module) - .context("Unable to emit bundle during Graph2::bundle().")?; + .context("Unable to emit bundle during Graph::bundle().")?; } String::from_utf8(buf).context("Emitted bundle is an invalid utf-8 string.") @@ -1437,7 +1437,7 @@ impl Graph2 { } } -impl swc_bundler::Resolve for Graph2 { +impl swc_bundler::Resolve for Graph { fn resolve( &self, referrer: &swc_common::FileName, @@ -1459,14 +1459,14 @@ impl swc_bundler::Resolve for Graph2 { } /// A structure for building a dependency graph of modules. -pub struct GraphBuilder2 { +pub struct GraphBuilder { fetched: HashSet, - graph: Graph2, + graph: Graph, maybe_import_map: Option>>, pending: FuturesUnordered, } -impl GraphBuilder2 { +impl GraphBuilder { pub fn new( handler: Rc>, maybe_import_map: Option, @@ -1477,8 +1477,8 @@ impl GraphBuilder2 { } else { None }; - GraphBuilder2 { - graph: Graph2::new(handler, maybe_lockfile), + GraphBuilder { + graph: Graph::new(handler, maybe_lockfile), fetched: HashSet::new(), maybe_import_map: internal_import_map, pending: FuturesUnordered::new(), @@ -1605,7 +1605,7 @@ impl GraphBuilder2 { /// Move out the graph from the builder to be utilized further. An optional /// lockfile can be provided, where if the sources in the graph do not match /// the expected lockfile, an error will be logged and the process will exit. - pub fn get_graph(self) -> Graph2 { + pub fn get_graph(self) -> Graph { self.graph.lock(); self.graph } @@ -1737,14 +1737,14 @@ pub mod tests { async fn setup( specifier: ModuleSpecifier, - ) -> (Graph2, Rc>) { + ) -> (Graph, Rc>) { let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); let fixtures = c.join("tests/module_graph"); let handler = Rc::new(RefCell::new(MockSpecifierHandler { fixtures, ..MockSpecifierHandler::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder .add(&specifier, false) .await @@ -1756,13 +1756,13 @@ pub mod tests { async fn setup_memory( specifier: ModuleSpecifier, sources: HashMap<&str, &str>, - ) -> Graph2 { + ) -> Graph { let sources: HashMap = sources .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); let handler = Rc::new(RefCell::new(MemoryHandler::new(sources))); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder .add(&specifier, false) .await @@ -1870,7 +1870,7 @@ pub mod tests { fixtures: fixtures.clone(), ..MockSpecifierHandler::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder .add(&specifier, false) .await @@ -1904,6 +1904,7 @@ pub mod tests { .expect("should have checked"); assert!(result_info.maybe_ignored_options.is_none()); assert_eq!(result_info.stats.0.len(), 12); + println!("{}", result_info.diagnostics); assert!(result_info.diagnostics.is_empty()); let h = handler.borrow(); assert_eq!(h.cache_calls.len(), 2); @@ -2084,7 +2085,7 @@ pub mod tests { fixtures, ..MockSpecifierHandler::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder .add(&specifier, false) .await @@ -2195,7 +2196,7 @@ pub mod tests { fixtures, ..MockSpecifierHandler::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, maybe_lockfile); + let mut builder = GraphBuilder::new(handler.clone(), None, maybe_lockfile); let specifier = ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") .expect("could not resolve module"); diff --git a/cli/module_loader.rs b/cli/module_loader.rs index b19476fe2d..1715f7f6b7 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,7 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::import_map::ImportMap; -use crate::module_graph2::TypeLib; +use crate::module_graph::TypeLib; use crate::permissions::Permissions; use crate::program_state::ProgramState; use deno_core::error::AnyError; diff --git a/cli/op_fetch_asset.rs b/cli/op_fetch_asset.rs deleted file mode 100644 index dcb54cde5b..0000000000 --- a/cli/op_fetch_asset.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -// Note: this module is used both in build.rs and main.rs. - -pub use deno_core::v8_set_flags; -use deno_core::BufVec; -use deno_core::Op; -use deno_core::OpState; -use std::cell::RefCell; -use std::collections::HashMap; -use std::path::PathBuf; -use std::rc::Rc; - -pub fn get_asset(name: &str) -> Option<&'static str> { - macro_rules! inc { - ($e:expr) => { - Some(include_str!(concat!("dts/", $e))) - }; - } - match name { - "bootstrap.ts" => Some("console.log(\"hello deno\");"), - "typescript.d.ts" => inc!("typescript.d.ts"), - "lib.dom.d.ts" => inc!("lib.dom.d.ts"), - "lib.dom.iterable.d.ts" => inc!("lib.dom.iterable.d.ts"), - "lib.es5.d.ts" => inc!("lib.es5.d.ts"), - "lib.es6.d.ts" => inc!("lib.es6.d.ts"), - "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"), - "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"), - "lib.es2020.full.d.ts" => inc!("lib.es2020.full.d.ts"), - "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"), - "lib.es2019.full.d.ts" => inc!("lib.es2019.full.d.ts"), - "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"), - "lib.es2018.full.d.ts" => inc!("lib.es2018.full.d.ts"), - "lib.es2017.d.ts" => inc!("lib.es2017.d.ts"), - "lib.es2017.full.d.ts" => inc!("lib.es2017.full.d.ts"), - "lib.es2016.d.ts" => inc!("lib.es2016.d.ts"), - "lib.es2016.full.d.ts" => inc!("lib.es2016.full.d.ts"), - "lib.es2015.d.ts" => inc!("lib.es2015.d.ts"), - "lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"), - "lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"), - "lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"), - "lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"), - "lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"), - "lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"), - "lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"), - "lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"), - "lib.es2015.symbol.wellknown.d.ts" => { - inc!("lib.es2015.symbol.wellknown.d.ts") - } - "lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"), - "lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"), - "lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"), - "lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"), - "lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"), - "lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"), - "lib.es2018.asyncgenerator.d.ts" => inc!("lib.es2018.asyncgenerator.d.ts"), - "lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"), - "lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"), - "lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"), - "lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"), - "lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"), - "lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"), - "lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"), - "lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"), - "lib.es2020.bigint.d.ts" => inc!("lib.es2020.bigint.d.ts"), - "lib.es2020.intl.d.ts" => inc!("lib.es2020.intl.d.ts"), - "lib.es2020.promise.d.ts" => inc!("lib.es2020.promise.d.ts"), - "lib.es2020.string.d.ts" => inc!("lib.es2020.string.d.ts"), - "lib.es2020.symbol.wellknown.d.ts" => { - inc!("lib.es2020.symbol.wellknown.d.ts") - } - "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"), - "lib.esnext.promise.d.ts" => inc!("lib.esnext.promise.d.ts"), - "lib.esnext.string.d.ts" => inc!("lib.esnext.string.d.ts"), - "lib.esnext.weakref.d.ts" => inc!("lib.esnext.weakref.d.ts"), - "lib.scripthost.d.ts" => inc!("lib.scripthost.d.ts"), - "lib.webworker.d.ts" => inc!("lib.webworker.d.ts"), - "lib.webworker.importscripts.d.ts" => { - inc!("lib.webworker.importscripts.d.ts") - } - _ => None, - } -} - -/// Warning: Returns a non-JSON op dispatcher. Must be manually attached to -/// JsRuntime. -/// -/// TODO(@kitsonk) this is only used when building the snapshot, and needs to -/// be refactored somewhere else. It is no longer used by `main.rs` and -/// therefore requires the allow unused. -#[allow(unused)] -pub fn op_fetch_asset( - custom_assets: HashMap, -) -> impl Fn(Rc>, BufVec) -> Op { - for (_, path) in custom_assets.iter() { - println!("cargo:rerun-if-changed={}", path.display()); - } - move |_state: Rc>, bufs: BufVec| -> Op { - assert_eq!(bufs.len(), 1, "Invalid number of arguments"); - let name = std::str::from_utf8(&bufs[0]).unwrap(); - - let asset_code = if let Some(source_code) = get_asset(name) { - source_code.to_string() - } else if let Some(asset_path) = custom_assets.get(name) { - let source_code_vec = - std::fs::read(&asset_path).expect("Asset not found"); - let source_code = std::str::from_utf8(&source_code_vec).unwrap(); - source_code.to_string() - } else { - panic!("fetch_asset bad asset {}", name) - }; - - let vec = asset_code.into_bytes(); - deno_core::Op::Sync(vec.into_boxed_slice()) - } -} diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index 02d0933754..f47f2fdb38 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -3,9 +3,9 @@ use crate::ast; use crate::colors; use crate::media_type::MediaType; -use crate::module_graph2::BundleType; -use crate::module_graph2::EmitOptions; -use crate::module_graph2::GraphBuilder2; +use crate::module_graph::BundleType; +use crate::module_graph::EmitOptions; +use crate::module_graph::GraphBuilder; use crate::permissions::Permissions; use crate::specifier_handler::FetchHandler; use crate::specifier_handler::MemoryHandler; @@ -65,7 +65,7 @@ async fn op_compile( runtime_permissions, )?)) }; - let mut builder = GraphBuilder2::new(handler, None, None); + let mut builder = GraphBuilder::new(handler, None, None); let specifier = ModuleSpecifier::resolve_url_or_path(&args.root_name) .context("The root specifier is invalid.")?; builder.add(&specifier, false).await?; diff --git a/cli/program_state.rs b/cli/program_state.rs index 027bbc7926..cacb64ca52 100644 --- a/cli/program_state.rs +++ b/cli/program_state.rs @@ -8,10 +8,10 @@ use crate::import_map::ImportMap; use crate::inspector::InspectorServer; use crate::lockfile::Lockfile; use crate::media_type::MediaType; -use crate::module_graph2::CheckOptions; -use crate::module_graph2::GraphBuilder2; -use crate::module_graph2::TranspileOptions; -use crate::module_graph2::TypeLib; +use crate::module_graph::CheckOptions; +use crate::module_graph::GraphBuilder; +use crate::module_graph::TranspileOptions; +use crate::module_graph::TypeLib; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::specifier_handler::FetchHandler; @@ -130,7 +130,7 @@ impl ProgramState { let handler = Rc::new(RefCell::new(FetchHandler::new(self, runtime_permissions)?)); let mut builder = - GraphBuilder2::new(handler, maybe_import_map, self.lockfile.clone()); + GraphBuilder::new(handler, maybe_import_map, self.lockfile.clone()); builder.add(&specifier, is_dynamic).await?; let mut graph = builder.get_graph(); let debug = self.flags.log_level == Some(log::Level::Debug); diff --git a/cli/tsc2.rs b/cli/tsc.rs similarity index 93% rename from cli/tsc2.rs rename to cli/tsc.rs index 25767619e4..7f90dd7b22 100644 --- a/cli/tsc2.rs +++ b/cli/tsc.rs @@ -2,10 +2,8 @@ use crate::diagnostics::Diagnostics; use crate::media_type::MediaType; -use crate::module_graph2::Graph2; -use crate::module_graph2::Stats; -// TODO(@kitsonk) this needs to be refactored when we drop MG1 -use crate::op_fetch_asset::get_asset; +use crate::module_graph::Graph; +use crate::module_graph::Stats; use crate::tsc_config::TsConfig; use deno_core::error::anyhow; @@ -26,6 +24,32 @@ use std::cell::RefCell; use std::collections::HashMap; use std::rc::Rc; +/// Provide static assets that are not preloaded in the compiler snapshot. +fn get_asset(asset: &str) -> Option<&'static str> { + macro_rules! inc { + ($e:expr) => { + Some(include_str!(concat!("dts/", $e))) + }; + } + match asset { + "lib.dom.d.ts" => inc!("lib.dom.d.ts"), + "lib.dom.iterable.d.ts" => inc!("lib.dom.iterable.d.ts"), + "lib.es6.d.ts" => inc!("lib.es6.d.ts"), + "lib.es2016.full.d.ts" => inc!("lib.es2016.full.d.ts"), + "lib.es2017.full.d.ts" => inc!("lib.es2017.full.d.ts"), + "lib.es2018.full.d.ts" => inc!("lib.es2018.full.d.ts"), + "lib.es2019.full.d.ts" => inc!("lib.es2019.full.d.ts"), + "lib.es2020.full.d.ts" => inc!("lib.es2020.full.d.ts"), + "lib.esnext.full.d.ts" => inc!("lib.esnext.full.d.ts"), + "lib.scripthost.d.ts" => inc!("lib.scripthost.d.ts"), + "lib.webworker.d.ts" => inc!("lib.webworker.d.ts"), + "lib.webworker.importscripts.d.ts" => { + inc!("lib.webworker.importscripts.d.ts") + } + _ => None, + } +} + fn get_maybe_hash( maybe_source: &Option, hash_data: &[Vec], @@ -54,7 +78,7 @@ pub struct Request { pub config: TsConfig, /// Indicates to the tsc runtime if debug logging should occur. pub debug: bool, - pub graph: Rc>, + pub graph: Rc>, pub hash_data: Vec>, pub maybe_tsbuildinfo: Option, /// A vector of strings that represent the root/entry point modules for the @@ -77,7 +101,7 @@ pub struct Response { struct State { hash_data: Vec>, emitted_files: Vec, - graph: Rc>, + graph: Rc>, maybe_tsbuildinfo: Option, maybe_response: Option, root_map: HashMap, @@ -85,7 +109,7 @@ struct State { impl State { pub fn new( - graph: Rc>, + graph: Rc>, hash_data: Vec>, maybe_tsbuildinfo: Option, root_map: HashMap, @@ -382,8 +406,8 @@ mod tests { use crate::diagnostics::Diagnostic; use crate::diagnostics::DiagnosticCategory; use crate::js; - use crate::module_graph2::tests::MockSpecifierHandler; - use crate::module_graph2::GraphBuilder2; + use crate::module_graph::tests::MockSpecifierHandler; + use crate::module_graph::GraphBuilder; use crate::tsc_config::TsConfig; use std::cell::RefCell; use std::env; @@ -404,7 +428,7 @@ mod tests { fixtures, ..MockSpecifierHandler::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder .add(&specifier, false) .await @@ -423,7 +447,7 @@ mod tests { fixtures, ..Default::default() })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); + let mut builder = GraphBuilder::new(handler.clone(), None, None); builder.add(&specifier, false).await?; let graph = Rc::new(RefCell::new(builder.get_graph())); let config = TsConfig::new(json!({ diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index e2a481d0f3..33f34b806d 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -1,19 +1,11 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // This module is the entry point for "compiler" isolate, ie. the one -// that is created when Deno needs to compile TS/WASM to JS. -// -// It provides two functions that should be called by Rust: -// - `startup` -// This functions must be called when creating isolate -// to properly setup runtime. -// - `tsCompilerOnMessage` -// This function must be called when sending a request -// to the compiler. +// that is created when Deno needs to type check TypeScript, and in some +// instances convert TypeScript to JavaScript. // Removes the `__proto__` for security reasons. This intentionally makes // Deno non compliant with ECMA-262 Annex B.2.2.1 -// delete Object.prototype.__proto__; ((window) => { @@ -22,11 +14,6 @@ delete Object.prototype.__proto__; let logDebug = false; let logSource = "JS"; - /** Instructs the host to behave in a legacy fashion, with the legacy - * pipeline for handling code. Setting the value to `true` will cause the - * host to behave in the modern way. */ - let legacy = true; - function setLogDebug(debug, source) { logDebug = debug; if (source) { @@ -57,9 +44,7 @@ delete Object.prototype.__proto__; /** @type {Map} */ const sourceFileCache = new Map(); - /** - * @param {import("../dts/typescript").DiagnosticRelatedInformation} diagnostic - */ + /** @param {ts.DiagnosticRelatedInformation} diagnostic */ function fromRelatedInformation({ start, length, @@ -96,9 +81,7 @@ delete Object.prototype.__proto__; } } - /** - * @param {import("../dts/typescript").Diagnostic[]} diagnostics - */ + /** @param {ts.Diagnostic[]} diagnostics */ function fromTypeScriptDiagnostic(diagnostics) { return diagnostics.map(({ relatedInformation: ri, source, ...diag }) => { const value = fromRelatedInformation(diag); @@ -110,426 +93,20 @@ delete Object.prototype.__proto__; }); } - // We really don't want to depend on JSON dispatch during snapshotting, so - // this op exchanges strings with Rust as raw byte arrays. - function getAsset(name) { - const opId = core.ops()["op_fetch_asset"]; - const sourceCodeBytes = core.dispatch(opId, core.encode(name)); - return core.decode(sourceCodeBytes); - } - // Using incremental compile APIs requires that all // paths must be either relative or absolute. Since // analysis in Rust operates on fully resolved URLs, // it makes sense to use the same scheme here. const ASSETS = "asset:///"; - const OUT_DIR = "deno://"; const CACHE = "cache:///"; - // This constant is passed to compiler settings when - // doing incremental compiles. Contents of this - // file are passed back to Rust and saved to $DENO_DIR. - const TS_BUILD_INFO = "cache:///tsbuildinfo.json"; - - const DEFAULT_COMPILE_OPTIONS = { - allowJs: false, - allowNonTsExtensions: true, - checkJs: false, - esModuleInterop: true, - jsx: ts.JsxEmit.React, - module: ts.ModuleKind.ESNext, - outDir: OUT_DIR, - sourceMap: true, - strict: true, - removeComments: true, - target: ts.ScriptTarget.ESNext, - }; - - const CompilerHostTarget = { - Main: "main", - Runtime: "runtime", - Worker: "worker", - }; - - // Warning! The values in this enum are duplicated in `cli/msg.rs` - // Update carefully! - const MediaType = { - 0: "JavaScript", - 1: "JSX", - 2: "TypeScript", - 3: "Dts", - 4: "TSX", - 5: "Json", - 6: "Wasm", - 7: "TsBuildInfo", - 8: "SourceMap", - 9: "Unknown", - JavaScript: 0, - JSX: 1, - TypeScript: 2, - Dts: 3, - TSX: 4, - Json: 5, - Wasm: 6, - TsBuildInfo: 7, - SourceMap: 8, - Unknown: 9, - }; - - function getExtension(fileName, mediaType) { - switch (mediaType) { - case MediaType.JavaScript: - return ts.Extension.Js; - case MediaType.JSX: - return ts.Extension.Jsx; - case MediaType.TypeScript: - return ts.Extension.Ts; - case MediaType.Dts: - return ts.Extension.Dts; - case MediaType.TSX: - return ts.Extension.Tsx; - case MediaType.Wasm: - // Custom marker for Wasm type. - return ts.Extension.Js; - case MediaType.Unknown: - default: - throw TypeError( - `Cannot resolve extension for "${fileName}" with mediaType "${ - MediaType[mediaType] - }".`, - ); - } - } - - /** A global cache of module source files that have been loaded. - * This cache will be rewritten to be populated on compiler startup - * with files provided from Rust in request message. - */ - const SOURCE_FILE_CACHE = new Map(); - /** A map of maps which cache resolved specifier for each import in a file. - * This cache is used so `resolveModuleNames` ops is called as few times - * as possible. - * - * First map's key is "referrer" URL ("file://a/b/c/mod.ts") - * Second map's key is "raw" import specifier ("./foo.ts") - * Second map's value is resolved import URL ("file:///a/b/c/foo.ts") - */ - const RESOLVED_SPECIFIER_CACHE = new Map(); - - class SourceFile { - constructor(json) { - this.processed = false; - Object.assign(this, json); - this.extension = getExtension(this.url, this.mediaType); - } - - static addToCache(json) { - if (SOURCE_FILE_CACHE.has(json.url)) { - throw new TypeError("SourceFile already exists"); - } - const sf = new SourceFile(json); - SOURCE_FILE_CACHE.set(sf.url, sf); - return sf; - } - - static getCached(url) { - return SOURCE_FILE_CACHE.get(url); - } - - static cacheResolvedUrl(resolvedUrl, rawModuleSpecifier, containingFile) { - containingFile = containingFile || ""; - let innerCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); - if (!innerCache) { - innerCache = new Map(); - RESOLVED_SPECIFIER_CACHE.set(containingFile, innerCache); - } - innerCache.set(rawModuleSpecifier, resolvedUrl); - } - - static getResolvedUrl(moduleSpecifier, containingFile) { - const containingCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); - if (containingCache) { - return containingCache.get(moduleSpecifier); - } - return undefined; - } - } - - function getAssetInternal(filename) { - const lastSegment = filename.split("/").pop(); - const url = ts.libMap.has(lastSegment) - ? ts.libMap.get(lastSegment) - : lastSegment; - const sourceFile = SourceFile.getCached(url); - if (sourceFile) { - return sourceFile; - } - const name = url.includes(".") ? url : `${url}.d.ts`; - const sourceCode = getAsset(name); - return SourceFile.addToCache({ - url, - filename: `${ASSETS}/${name}`, - mediaType: MediaType.TypeScript, - versionHash: "1", - sourceCode, - }); - } - - /** There was some private state in the legacy host, that is moved out to - * here which can then be refactored out later. */ - const legacyHostState = { - buildInfo: "", - target: CompilerHostTarget.Main, - writeFile: (_fileName, _data, _sourceFiles) => {}, - }; - - /** @type {import("../dts/typescript").CompilerHost} */ - const host = { - fileExists(fileName) { - debug(`host.fileExists("${fileName}")`); - return false; - }, - readFile(specifier) { - debug(`host.readFile("${specifier}")`); - if (legacy) { - if (specifier == TS_BUILD_INFO) { - return legacyHostState.buildInfo; - } - return unreachable(); - } else { - return core.jsonOpSync("op_load", { specifier }).data; - } - }, - getSourceFile( - specifier, - languageVersion, - onError, - shouldCreateNewSourceFile, - ) { - debug( - `host.getSourceFile("${specifier}", ${ - ts.ScriptTarget[languageVersion] - })`, - ); - if (legacy) { - try { - assert(!shouldCreateNewSourceFile); - const sourceFile = specifier.startsWith(ASSETS) - ? getAssetInternal(specifier) - : SourceFile.getCached(specifier); - assert(sourceFile != null); - if (!sourceFile.tsSourceFile) { - assert(sourceFile.sourceCode != null); - const tsSourceFileName = specifier.startsWith(ASSETS) - ? sourceFile.filename - : specifier; - - sourceFile.tsSourceFile = ts.createSourceFile( - tsSourceFileName, - sourceFile.sourceCode, - languageVersion, - ); - sourceFile.tsSourceFile.version = sourceFile.versionHash; - delete sourceFile.sourceCode; - - // This code is to support transition from the "legacy" compiler - // to the new one, by populating the new source file cache. - if ( - !sourceFileCache.has(specifier) && specifier.startsWith(ASSETS) - ) { - sourceFileCache.set(specifier, sourceFile.tsSourceFile); - } - } - return sourceFile.tsSourceFile; - } catch (e) { - if (onError) { - onError(String(e)); - } else { - throw e; - } - return undefined; - } - } else { - let sourceFile = sourceFileCache.get(specifier); - if (sourceFile) { - return sourceFile; - } - - /** @type {{ data: string; hash?: string; scriptKind: ts.ScriptKind }} */ - const { data, hash, scriptKind } = core.jsonOpSync( - "op_load", - { specifier }, - ); - assert( - data != null, - `"data" is unexpectedly null for "${specifier}".`, - ); - sourceFile = ts.createSourceFile( - specifier, - data, - languageVersion, - false, - scriptKind, - ); - sourceFile.moduleName = specifier; - sourceFile.version = hash; - sourceFileCache.set(specifier, sourceFile); - return sourceFile; - } - }, - getDefaultLibFileName() { - if (legacy) { - switch (legacyHostState.target) { - case CompilerHostTarget.Main: - case CompilerHostTarget.Runtime: - return `${ASSETS}/lib.deno.window.d.ts`; - case CompilerHostTarget.Worker: - return `${ASSETS}/lib.deno.worker.d.ts`; - } - } else { - return `${ASSETS}/lib.esnext.d.ts`; - } - }, - getDefaultLibLocation() { - return ASSETS; - }, - writeFile(fileName, data, _writeByteOrderMark, _onError, sourceFiles) { - debug(`host.writeFile("${fileName}")`); - if (legacy) { - legacyHostState.writeFile(fileName, data, sourceFiles); - } else { - let maybeSpecifiers; - if (sourceFiles) { - maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); - } - return core.jsonOpSync( - "op_emit", - { maybeSpecifiers, fileName, data }, - ); - } - }, - getCurrentDirectory() { - return CACHE; - }, - getCanonicalFileName(fileName) { - return fileName; - }, - useCaseSensitiveFileNames() { - return true; - }, - getNewLine() { - return "\n"; - }, - resolveModuleNames(specifiers, base) { - debug(`host.resolveModuleNames()`); - debug(` base: ${base}`); - debug(` specifiers: ${specifiers.join(", ")}`); - if (legacy) { - const resolved = specifiers.map((specifier) => { - const maybeUrl = SourceFile.getResolvedUrl(specifier, base); - - debug("compiler::host.resolveModuleNames maybeUrl", { - specifier, - maybeUrl, - }); - - let sourceFile = undefined; - - if (specifier.startsWith(ASSETS)) { - sourceFile = getAssetInternal(specifier); - } else if (typeof maybeUrl !== "undefined") { - sourceFile = SourceFile.getCached(maybeUrl); - } - - if (!sourceFile) { - return undefined; - } - - return { - resolvedFileName: sourceFile.url, - isExternalLibraryImport: specifier.startsWith(ASSETS), - extension: sourceFile.extension, - }; - }); - debug(resolved); - return resolved; - } else { - /** @type {Array<[string, import("../dts/typescript").Extension]>} */ - const resolved = core.jsonOpSync("op_resolve", { - specifiers, - base, - }); - let r = resolved.map(([resolvedFileName, extension]) => ({ - resolvedFileName, - extension, - isExternalLibraryImport: false, - })); - return r; - } - }, - createHash(data) { - return core.jsonOpSync("op_create_hash", { data }).hash; - }, - }; - - // This is a hacky way of adding our libs to the libs available in TypeScript() - // as these are internal APIs of TypeScript which maintain valid libs - ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals"); - ts.libMap.set("deno.ns", "lib.deno.ns.d.ts"); - ts.libMap.set("deno.web", "lib.deno.web.d.ts"); - ts.libMap.set("deno.fetch", "lib.deno.fetch.d.ts"); - ts.libMap.set("deno.window", "lib.deno.window.d.ts"); - ts.libMap.set("deno.worker", "lib.deno.worker.d.ts"); - ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts"); - ts.libMap.set("deno.unstable", "lib.deno.unstable.d.ts"); - - // TODO(@kitsonk) remove once added to TypeScript - ts.libs.push("esnext.weakref"); - ts.libMap.set("esnext.weakref", "lib.esnext.weakref.d.ts"); - - // this pre-populates the cache at snapshot time of our library files, so they - // are available in the future when needed. - host.getSourceFile( - `${ASSETS}lib.deno.ns.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.web.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.fetch.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.window.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.worker.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.shared_globals.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.unstable.d.ts`, - ts.ScriptTarget.ESNext, - ); - - // We never use this program; it's only created - // during snapshotting to hydrate and populate - // source file cache with lib declaration files. - const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ - rootNames: [`${ASSETS}bootstrap.ts`], - options: DEFAULT_COMPILE_OPTIONS, - host, - }); + /** Diagnostics that are intentionally ignored when compiling TypeScript in + * Deno, as they provide misleading or incorrect information. */ const IGNORED_DIAGNOSTICS = [ - // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is - // not a module. - 2306, + // TS1208: All files must be modules when the '--isolatedModules' flag is + // provided. We can ignore because we guarantuee that all files are + // modules. + 1208, // TS1375: 'await' expressions are only allowed at the top level of a file // when that file is a module, but this file has no imports or exports. // Consider adding an empty 'export {}' to make this file a module. @@ -537,6 +114,9 @@ delete Object.prototype.__proto__; // TS1103: 'for-await-of' statement is only allowed within an async function // or async generator. 1103, + // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is + // not a module. + 2306, // TS2691: An import path cannot end with a '.ts' extension. Consider // importing 'bad-module' instead. 2691, @@ -557,14 +137,116 @@ delete Object.prototype.__proto__; 7016, ]; - const IGNORED_COMPILE_DIAGNOSTICS = [ - // TS1208: All files must be modules when the '--isolatedModules' flag is - // provided. We can ignore because we guarantuee that all files are - // modules. - 1208, - ]; + const SNAPSHOT_COMPILE_OPTIONS = { + esModuleInterop: true, + jsx: ts.JsxEmit.React, + module: ts.ModuleKind.ESNext, + noEmit: true, + strict: true, + target: ts.ScriptTarget.ESNext, + }; - /** @type {Array<{ key: string, value: number }>} */ + /** An object literal of the incremental compiler host, which provides the + * specific "bindings" to the Deno environment that tsc needs to work. + * + * @type {ts.CompilerHost} */ + const host = { + fileExists(fileName) { + debug(`host.fileExists("${fileName}")`); + return false; + }, + readFile(specifier) { + debug(`host.readFile("${specifier}")`); + return core.jsonOpSync("op_load", { specifier }).data; + }, + getSourceFile( + specifier, + languageVersion, + _onError, + _shouldCreateNewSourceFile, + ) { + debug( + `host.getSourceFile("${specifier}", ${ + ts.ScriptTarget[languageVersion] + })`, + ); + let sourceFile = sourceFileCache.get(specifier); + if (sourceFile) { + return sourceFile; + } + + /** @type {{ data: string; hash?: string; scriptKind: ts.ScriptKind }} */ + const { data, hash, scriptKind } = core.jsonOpSync( + "op_load", + { specifier }, + ); + assert( + data != null, + `"data" is unexpectedly null for "${specifier}".`, + ); + sourceFile = ts.createSourceFile( + specifier, + data, + languageVersion, + false, + scriptKind, + ); + sourceFile.moduleName = specifier; + sourceFile.version = hash; + sourceFileCache.set(specifier, sourceFile); + return sourceFile; + }, + getDefaultLibFileName() { + return `${ASSETS}/lib.esnext.d.ts`; + }, + getDefaultLibLocation() { + return ASSETS; + }, + writeFile(fileName, data, _writeByteOrderMark, _onError, sourceFiles) { + debug(`host.writeFile("${fileName}")`); + let maybeSpecifiers; + if (sourceFiles) { + maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); + } + return core.jsonOpSync( + "op_emit", + { maybeSpecifiers, fileName, data }, + ); + }, + getCurrentDirectory() { + return CACHE; + }, + getCanonicalFileName(fileName) { + return fileName; + }, + useCaseSensitiveFileNames() { + return true; + }, + getNewLine() { + return "\n"; + }, + resolveModuleNames(specifiers, base) { + debug(`host.resolveModuleNames()`); + debug(` base: ${base}`); + debug(` specifiers: ${specifiers.join(", ")}`); + /** @type {Array<[string, ts.Extension]>} */ + const resolved = core.jsonOpSync("op_resolve", { + specifiers, + base, + }); + let r = resolved.map(([resolvedFileName, extension]) => ({ + resolvedFileName, + extension, + isExternalLibraryImport: false, + })); + return r; + }, + createHash(data) { + return core.jsonOpSync("op_create_hash", { data }).hash; + }, + }; + + /** @type {Array<[string, number]>} */ const stats = []; let statsStart = 0; @@ -579,35 +261,31 @@ delete Object.prototype.__proto__; if ("getProgram" in program) { program = program.getProgram(); } - stats.push({ key: "Files", value: program.getSourceFiles().length }); - stats.push({ key: "Nodes", value: program.getNodeCount() }); - stats.push({ key: "Identifiers", value: program.getIdentifierCount() }); - stats.push({ key: "Symbols", value: program.getSymbolCount() }); - stats.push({ key: "Types", value: program.getTypeCount() }); - stats.push({ - key: "Instantiations", - value: program.getInstantiationCount(), - }); + stats.push(["Files", program.getSourceFiles().length]); + stats.push(["Nodes", program.getNodeCount()]); + stats.push(["Identifiers", program.getIdentifierCount()]); + stats.push(["Symbols", program.getSymbolCount()]); + stats.push(["Types", program.getTypeCount()]); + stats.push(["Instantiations", program.getInstantiationCount()]); } else if (fileCount != null) { - stats.push({ key: "Files", value: fileCount }); + stats.push(["Files", fileCount]); } const programTime = ts.performance.getDuration("Program"); const bindTime = ts.performance.getDuration("Bind"); const checkTime = ts.performance.getDuration("Check"); const emitTime = ts.performance.getDuration("Emit"); - stats.push({ key: "Parse time", value: programTime }); - stats.push({ key: "Bind time", value: bindTime }); - stats.push({ key: "Check time", value: checkTime }); - stats.push({ key: "Emit time", value: emitTime }); - stats.push({ - key: "Total TS time", - value: programTime + bindTime + checkTime + emitTime, - }); + stats.push(["Parse time", programTime]); + stats.push(["Bind time", bindTime]); + stats.push(["Check time", checkTime]); + stats.push(["Emit time", emitTime]); + stats.push( + ["Total TS time", programTime + bindTime + checkTime + emitTime], + ); } function performanceEnd() { const duration = new Date() - statsStart; - stats.push({ key: "Compile time", value: duration }); + stats.push(["Compile time", duration]); return stats; } @@ -645,17 +323,12 @@ delete Object.prototype.__proto__; ...program.getGlobalDiagnostics(), ...program.getSemanticDiagnostics(), ...emitDiagnostics, - ].filter(({ code }) => - !IGNORED_DIAGNOSTICS.includes(code) && - !IGNORED_COMPILE_DIAGNOSTICS.includes(code) - ); + ].filter(({ code }) => !IGNORED_DIAGNOSTICS.includes(code)); performanceProgram({ program }); - // TODO(@kitsonk) when legacy stats are removed, convert to just tuples - let stats = performanceEnd().map(({ key, value }) => [key, value]); core.jsonOpSync("op_respond", { diagnostics: fromTypeScriptDiagnostic(diagnostics), - stats, + stats: performanceEnd(), }); debug("<<< exec stop"); } @@ -665,7 +338,7 @@ delete Object.prototype.__proto__; /** Startup the runtime environment, setting various flags. * @param {{ debugFlag?: boolean; legacyFlag?: boolean; }} msg */ - function startup({ debugFlag = false, legacyFlag = true }) { + function startup({ debugFlag = false }) { if (hasStarted) { throw new Error("The compiler runtime already started."); } @@ -673,9 +346,42 @@ delete Object.prototype.__proto__; core.ops(); core.registerErrorClass("Error", Error); setLogDebug(!!debugFlag, "TS"); - legacy = legacyFlag; } + // Setup the compiler runtime during the build process. + core.ops(); + core.registerErrorClass("Error", Error); + + // A build time only op that provides some setup information that is used to + // ensure the snapshot is setup properly. + /** @type {{ buildSpecifier: string; libs: string[] }} */ + const { buildSpecifier, libs } = core.jsonOpSync("op_build_info", {}); + for (const lib of libs) { + let specifier = `lib.${lib}.d.ts`; + // we are using internal APIs here to "inject" our custom libraries into + // tsc, so things like `"lib": [ "deno.ns" ]` are supported. + if (!ts.libs.includes(lib)) { + ts.libs.push(lib); + ts.libMap.set(lib, `lib.${lib}.d.ts`); + } + // we are caching in memory common type libraries that will be re-used by + // tsc on when the snapshot is restored + assert( + host.getSourceFile(`${ASSETS}${specifier}`, ts.ScriptTarget.ESNext), + ); + } + // this helps ensure as much as possible is in memory that is re-usable + // before the snapshotting is done, which helps unsure fast "startup" for + // subsequent uses of tsc in Deno. + const TS_SNAPSHOT_PROGRAM = ts.createProgram({ + rootNames: [buildSpecifier], + options: SNAPSHOT_COMPILE_OPTIONS, + host, + }); + ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM); + + // exposes the two functions that are called by `tsc::exec()` when type + // checking TypeScript. globalThis.startup = startup; globalThis.exec = exec; })(this);