From 1712a88e6998d058c11ee213dcf15cccc563a9b0 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 14 Jan 2023 09:36:19 -0500 Subject: [PATCH] refactor(tsc): do not store some typescript declaration file text in multiple places (#17410) --- cli/build.rs | 58 +++-------- cli/lsp/tsc.rs | 29 ++++-- cli/tsc/99_main_compiler.js | 43 +++++--- cli/tsc/mod.rs | 200 ++++++++++++++++++++---------------- 4 files changed, 173 insertions(+), 157 deletions(-) diff --git a/cli/build.rs b/cli/build.rs index 24898a7a1b..4d82630e55 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -94,6 +94,7 @@ mod ts { "es2018.regexp", "es2019.array", "es2019", + "es2019.intl", "es2019.object", "es2019.string", "es2019.symbol", @@ -116,6 +117,7 @@ mod ts { "es2022.error", "es2022.intl", "es2022.object", + "es2022.sharedmemory", "es2022.string", "esnext", "esnext.array", @@ -150,6 +152,15 @@ mod ts { build_libs.push(op_lib.to_owned()); } + // used in the tests to verify that after snapshotting it has the same number + // of lib files loaded and hasn't included any ones lazily loaded from Rust + std::fs::write( + PathBuf::from(env::var_os("OUT_DIR").unwrap()) + .join("lib_file_names.json"), + serde_json::to_string(&build_libs).unwrap(), + ) + .unwrap(); + #[op] fn op_build_info(state: &mut OpState) -> Value { let build_specifier = "asset:///bootstrap.ts"; @@ -196,7 +207,7 @@ mod ts { // we need a basic file to send to tsc to warm it up. if args.specifier == build_specifier { Ok(json!({ - "data": r#"console.log("hello deno!");"#, + "data": r#"Deno.writeTextFile("hello.txt", "hello deno!");"#, "version": "1", // this corresponds to `ts.ScriptKind.TypeScript` "scriptKind": 3 @@ -423,51 +434,6 @@ fn main() { println!("cargo:rustc-env=TS_VERSION={}", ts::version()); println!("cargo:rerun-if-env-changed=TS_VERSION"); - println!( - "cargo:rustc-env=DENO_CONSOLE_LIB_PATH={}", - deno_console::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_URL_LIB_PATH={}", - deno_url::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_WEB_LIB_PATH={}", - deno_web::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_FETCH_LIB_PATH={}", - deno_fetch::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_WEBGPU_LIB_PATH={}", - deno_webgpu_get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_WEBSOCKET_LIB_PATH={}", - deno_websocket::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_WEBSTORAGE_LIB_PATH={}", - deno_webstorage::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_CACHE_LIB_PATH={}", - deno_cache::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_CRYPTO_LIB_PATH={}", - deno_crypto::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_BROADCAST_CHANNEL_LIB_PATH={}", - deno_broadcast_channel::get_declaration().display() - ); - println!( - "cargo:rustc-env=DENO_NET_LIB_PATH={}", - deno_net::get_declaration().display() - ); - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index fabe30ac73..ced8fb6993 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -208,7 +208,7 @@ impl AssetDocument { type AssetsMap = HashMap; fn new_assets_map() -> Arc> { - let assets = tsc::STATIC_ASSETS + let assets = tsc::LAZILY_LOADED_STATIC_ASSETS .iter() .map(|(k, v)| { let url_str = format!("asset:///{}", k); @@ -3455,6 +3455,8 @@ mod tests { use crate::lsp::documents::Documents; use crate::lsp::documents::LanguageId; use crate::lsp::text::LineIndex; + use crate::tsc::AssetText; + use pretty_assertions::assert_eq; use std::path::Path; use std::path::PathBuf; use test_util::TempDir; @@ -3913,18 +3915,31 @@ mod tests { Default::default(), ) .unwrap(); - let assets = result.as_array().unwrap(); + let assets: Vec = serde_json::from_value(result).unwrap(); + let mut asset_names = assets + .iter() + .map(|a| { + a.specifier + .replace("asset:///lib.", "") + .replace(".d.ts", "") + }) + .collect::>(); + let mut expected_asset_names: Vec = serde_json::from_str( + include_str!(concat!(env!("OUT_DIR"), "/lib_file_names.json")), + ) + .unwrap(); + + asset_names.sort(); + expected_asset_names.sort(); // You might have found this assertion starts failing after upgrading TypeScript. - // Just update the new number of assets (declaration files) for this number. - assert_eq!(assets.len(), 72); + // Ensure build.rs is updated so these match. + assert_eq!(asset_names, expected_asset_names); // get some notification when the size of the assets grows let mut total_size = 0; for asset in assets { - let obj = asset.as_object().unwrap(); - let text = obj.get("text").unwrap().as_str().unwrap(); - total_size += text.len(); + total_size += asset.text.len(); } assert!(total_size > 0); assert!(total_size < 2_000_000); // currently as of TS 4.6, it's 0.7MB diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index a3cdad742e..a0219fe13b 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -385,7 +385,7 @@ delete Object.prototype.__proto__; // paths must be either relative or absolute. Since // analysis in Rust operates on fully resolved URLs, // it makes sense to use the same scheme here. - const ASSETS = "asset:///"; + const ASSETS_URL_PREFIX = "asset:///"; /** Diagnostics that are intentionally ignored when compiling TypeScript in * Deno, as they provide misleading or incorrect information. */ @@ -431,6 +431,7 @@ delete Object.prototype.__proto__; noEmit: true, strict: true, target: ts.ScriptTarget.ESNext, + lib: ["lib.deno.window.d.ts"], }; // todo(dsherret): can we remove this and just use ts.OperationCanceledException? @@ -546,10 +547,10 @@ delete Object.prototype.__proto__; return sourceFile; }, getDefaultLibFileName() { - return `${ASSETS}/lib.esnext.d.ts`; + return `${ASSETS_URL_PREFIX}lib.esnext.d.ts`; }, getDefaultLibLocation() { - return ASSETS; + return ASSETS_URL_PREFIX; }, writeFile(fileName, data, _writeByteOrderMark, _onError, _sourceFiles) { if (logDebug) { @@ -887,6 +888,20 @@ delete Object.prototype.__proto__; debug("<<< exec stop"); } + function getAssets() { + /** @type {{ specifier: string; text: string; }[]} */ + const assets = []; + for (const sourceFile of sourceFileCache.values()) { + if (sourceFile.fileName.startsWith(ASSETS_URL_PREFIX)) { + assets.push({ + specifier: sourceFile.fileName, + text: sourceFile.text, + }); + } + } + return assets; + } + /** * @param {number} id * @param {any} data @@ -935,16 +950,7 @@ delete Object.prototype.__proto__; ); } case "getAssets": { - const assets = []; - for (const sourceFile of sourceFileCache.values()) { - if (sourceFile.fileName.startsWith(ASSETS)) { - assets.push({ - specifier: sourceFile.fileName, - text: sourceFile.text, - }); - } - } - return respond(id, assets); + return respond(id, getAssets()); } case "getApplicableRefactors": { return respond( @@ -1281,7 +1287,10 @@ delete Object.prototype.__proto__; // we are caching in memory common type libraries that will be re-used by // tsc on when the snapshot is restored assert( - host.getSourceFile(`${ASSETS}${specifier}`, ts.ScriptTarget.ESNext), + host.getSourceFile( + `${ASSETS_URL_PREFIX}${specifier}`, + ts.ScriptTarget.ESNext, + ), ); } // this helps ensure as much as possible is in memory that is re-usable @@ -1292,12 +1301,16 @@ delete Object.prototype.__proto__; options: SNAPSHOT_COMPILE_OPTIONS, host, }); - ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM); + assert(ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM).length === 0); + + // remove this now that we don't need it anymore for warming up tsc + sourceFileCache.delete(buildSpecifier); // exposes the two functions that are called by `tsc::exec()` when type // checking TypeScript. globalThis.startup = startup; globalThis.exec = exec; + globalThis.getAssets = getAssets; // exposes the functions that are called when the compiler is used as a // language service. diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 6ea037522d..d540e5b33b 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -25,6 +25,7 @@ use deno_core::serde::Serializer; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::serde_json::Value; +use deno_core::serde_v8; use deno_core::Extension; use deno_core::JsRuntime; use deno_core::ModuleSpecifier; @@ -49,28 +50,6 @@ pub use self::diagnostics::DiagnosticMessageChain; pub use self::diagnostics::Diagnostics; pub use self::diagnostics::Position; -// Declaration files - -pub static DENO_NS_LIB: &str = include_str!("dts/lib.deno.ns.d.ts"); -pub static DENO_CONSOLE_LIB: &str = include_str!(env!("DENO_CONSOLE_LIB_PATH")); -pub static DENO_URL_LIB: &str = include_str!(env!("DENO_URL_LIB_PATH")); -pub static DENO_WEB_LIB: &str = include_str!(env!("DENO_WEB_LIB_PATH")); -pub static DENO_FETCH_LIB: &str = include_str!(env!("DENO_FETCH_LIB_PATH")); -pub static DENO_WEBGPU_LIB: &str = include_str!(env!("DENO_WEBGPU_LIB_PATH")); -pub static DENO_WEBSOCKET_LIB: &str = - include_str!(env!("DENO_WEBSOCKET_LIB_PATH")); -pub static DENO_WEBSTORAGE_LIB: &str = - include_str!(env!("DENO_WEBSTORAGE_LIB_PATH")); -pub static DENO_CACHE_LIB: &str = include_str!(env!("DENO_CACHE_LIB_PATH")); -pub static DENO_CRYPTO_LIB: &str = include_str!(env!("DENO_CRYPTO_LIB_PATH")); -pub static DENO_BROADCAST_CHANNEL_LIB: &str = - include_str!(env!("DENO_BROADCAST_CHANNEL_LIB_PATH")); -pub static DENO_NET_LIB: &str = include_str!(env!("DENO_NET_LIB_PATH")); -pub static SHARED_GLOBALS_LIB: &str = - include_str!("dts/lib.deno.shared_globals.d.ts"); -pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts"); -pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts"); - pub static COMPILER_SNAPSHOT: Lazy> = Lazy::new( #[cold] #[inline(never)] @@ -89,28 +68,57 @@ pub static COMPILER_SNAPSHOT: Lazy> = Lazy::new( ); pub fn get_types_declaration_file_text(unstable: bool) -> String { - let mut types = vec![ - DENO_NS_LIB, - DENO_CONSOLE_LIB, - DENO_URL_LIB, - DENO_WEB_LIB, - DENO_FETCH_LIB, - DENO_WEBGPU_LIB, - DENO_WEBSOCKET_LIB, - DENO_WEBSTORAGE_LIB, - DENO_CRYPTO_LIB, - DENO_BROADCAST_CHANNEL_LIB, - DENO_NET_LIB, - SHARED_GLOBALS_LIB, - DENO_CACHE_LIB, - WINDOW_LIB, + let mut assets = get_asset_texts_from_new_runtime() + .unwrap() + .into_iter() + .map(|a| (a.specifier, a.text)) + .collect::>(); + + let mut lib_names = vec![ + "deno.ns", + "deno.console", + "deno.url", + "deno.web", + "deno.fetch", + "deno.webgpu", + "deno.websocket", + "deno.webstorage", + "deno.crypto", + "deno.broadcast_channel", + "deno.net", + "deno.shared_globals", + "deno.cache", + "deno.window", ]; if unstable { - types.push(UNSTABLE_NS_LIB); + lib_names.push("deno.unstable"); } - types.join("\n") + lib_names + .into_iter() + .map(|name| { + let asset_url = format!("asset:///lib.{}.d.ts", name); + assets.remove(&asset_url).unwrap() + }) + .collect::>() + .join("\n") +} + +fn get_asset_texts_from_new_runtime() -> Result, AnyError> { + // the assets are stored within the typescript isolate, so take them out of there + let mut runtime = JsRuntime::new(RuntimeOptions { + startup_snapshot: Some(compiler_snapshot()), + extensions: vec![Extension::builder("deno_cli_tsc") + .ops(get_tsc_ops()) + .build()], + ..Default::default() + }); + let global = + runtime.execute_script("get_assets.js", "globalThis.getAssets()")?; + let scope = &mut runtime.handle_scope(); + let local = deno_core::v8::Local::new(scope, global); + Ok(serde_v8::from_v8::>(scope, local)?) } pub fn compiler_snapshot() -> Snapshot { @@ -124,40 +132,44 @@ macro_rules! inc { } /// Contains static assets that are not preloaded in the compiler snapshot. -pub static STATIC_ASSETS: Lazy> = - Lazy::new(|| { - ([ - ( - "lib.dom.asynciterable.d.ts", - inc!("lib.dom.asynciterable.d.ts"), - ), - ("lib.dom.d.ts", inc!("lib.dom.d.ts")), - ("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")), - ("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")), - ("lib.es6.d.ts", inc!("lib.es6.d.ts")), - ("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")), - ("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")), - ("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")), - ("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")), - ("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")), - ("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")), - ("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")), - ("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")), - ("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")), - ("lib.webworker.d.ts", inc!("lib.webworker.d.ts")), - ( - "lib.webworker.importscripts.d.ts", - inc!("lib.webworker.importscripts.d.ts"), - ), - ( - "lib.webworker.iterable.d.ts", - inc!("lib.webworker.iterable.d.ts"), - ), - ]) - .iter() - .cloned() - .collect() - }); +/// +/// We lazily load these because putting them in the compiler snapshot will +/// increase memory usage when not used (last time checked by about 0.5MB). +pub static LAZILY_LOADED_STATIC_ASSETS: Lazy< + HashMap<&'static str, &'static str>, +> = Lazy::new(|| { + ([ + ( + "lib.dom.asynciterable.d.ts", + inc!("lib.dom.asynciterable.d.ts"), + ), + ("lib.dom.d.ts", inc!("lib.dom.d.ts")), + ("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")), + ("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")), + ("lib.es6.d.ts", inc!("lib.es6.d.ts")), + ("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")), + ("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")), + ("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")), + ("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")), + ("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")), + ("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")), + ("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")), + ("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")), + ("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")), + ("lib.webworker.d.ts", inc!("lib.webworker.d.ts")), + ( + "lib.webworker.importscripts.d.ts", + inc!("lib.webworker.importscripts.d.ts"), + ), + ( + "lib.webworker.iterable.d.ts", + inc!("lib.webworker.iterable.d.ts"), + ), + ]) + .iter() + .cloned() + .collect() +}); /// A structure representing stats from a type check operation for a graph. #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -193,9 +205,16 @@ impl fmt::Display for Stats { } } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AssetText { + pub specifier: String, + pub text: String, +} + /// Retrieve a static asset that are included in the binary. -pub fn get_asset(asset: &str) -> Option<&'static str> { - STATIC_ASSETS.get(asset).map(|s| s.to_owned()) +fn get_lazily_loaded_asset(asset: &str) -> Option<&'static str> { + LAZILY_LOADED_STATIC_ASSETS.get(asset).map(|s| s.to_owned()) } fn get_maybe_hash( @@ -501,9 +520,8 @@ fn op_load(state: &mut OpState, args: Value) -> Result { hash = Some("1".to_string()); media_type = MediaType::Dts; Some(Cow::Borrowed("declare const __: any;\nexport = __;\n")) - } else if v.specifier.starts_with("asset:///") { - let name = v.specifier.replace("asset:///", ""); - let maybe_source = get_asset(&name); + } else if let Some(name) = v.specifier.strip_prefix("asset:///") { + let maybe_source = get_lazily_loaded_asset(name); hash = get_maybe_hash(maybe_source, &state.hash_data); media_type = MediaType::from(&v.specifier); maybe_source.map(Cow::Borrowed) @@ -774,16 +792,7 @@ pub fn exec(request: Request) -> Result { let mut runtime = JsRuntime::new(RuntimeOptions { startup_snapshot: Some(compiler_snapshot()), extensions: vec![Extension::builder("deno_cli_tsc") - .ops(vec![ - op_cwd::decl(), - op_create_hash::decl(), - op_emit::decl(), - op_exists::decl(), - op_is_node_file::decl(), - op_load::decl(), - op_resolve::decl(), - op_respond::decl(), - ]) + .ops(get_tsc_ops()) .state(move |state| { state.put(State::new( request.graph_data.clone(), @@ -833,6 +842,19 @@ pub fn exec(request: Request) -> Result { } } +fn get_tsc_ops() -> Vec { + vec![ + op_cwd::decl(), + op_create_hash::decl(), + op_emit::decl(), + op_exists::decl(), + op_is_node_file::decl(), + op_load::decl(), + op_resolve::decl(), + op_respond::decl(), + ] +} + #[cfg(test)] mod tests { use super::Diagnostic; @@ -1089,7 +1111,7 @@ mod tests { .expect("should have invoked op"); let actual: LoadResponse = serde_json::from_value(value).expect("failed to deserialize"); - let expected = get_asset("lib.dom.d.ts").unwrap(); + let expected = get_lazily_loaded_asset("lib.dom.d.ts").unwrap(); assert_eq!(actual.data, expected); assert!(actual.version.is_some()); assert_eq!(actual.script_kind, 3);