foster/denoland-deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-02-08 07:16:56 -05:00
Code Issues Packages 1 Wiki Activity

refactor(tsc): do not store some typescript declaration file text in multiple places (#17410)

Browse source
This commit is contained in:
David Sherret 2023-01-14 09:36:19 -05:00 • committed by GitHub
parent 429ccff657
commit 1712a88e69
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 173 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
cli/build.rs
View file

@ -94,6 +94,7 @@ mod ts {
"es2018.regexp", "es2018.regexp",
"es2019.array", "es2019.array",
"es2019", "es2019",
"es2019.intl",
"es2019.object", "es2019.object",
"es2019.string", "es2019.string",
"es2019.symbol", "es2019.symbol",
@ -116,6 +117,7 @@ mod ts {
"es2022.error", "es2022.error",
"es2022.intl", "es2022.intl",
"es2022.object", "es2022.object",
"es2022.sharedmemory",
"es2022.string", "es2022.string",
"esnext", "esnext",
"esnext.array", "esnext.array",
@ -150,6 +152,15 @@ mod ts {
build_libs.push(op_lib.to_owned()); build_libs.push(op_lib.to_owned());
} }
// used in the tests to verify that after snapshotting it has the same number
// of lib files loaded and hasn't included any ones lazily loaded from Rust
std::fs::write(
PathBuf::from(env::var_os("OUT_DIR").unwrap())
.join("lib_file_names.json"),
serde_json::to_string(&build_libs).unwrap(),
)
.unwrap();
#[op] #[op]
fn op_build_info(state: &mut OpState) -> Value { fn op_build_info(state: &mut OpState) -> Value {
let build_specifier = "asset:///bootstrap.ts"; let build_specifier = "asset:///bootstrap.ts";
@ -196,7 +207,7 @@ mod ts {
// we need a basic file to send to tsc to warm it up. // we need a basic file to send to tsc to warm it up.
if args.specifier == build_specifier { if args.specifier == build_specifier {
Ok(json!({ Ok(json!({
"data": r#"console.log("hello deno!");"#, "data": r#"Deno.writeTextFile("hello.txt", "hello deno!");"#,
"version": "1", "version": "1",
// this corresponds to `ts.ScriptKind.TypeScript` // this corresponds to `ts.ScriptKind.TypeScript`
"scriptKind": 3 "scriptKind": 3
@ -423,51 +434,6 @@ fn main() {
println!("cargo:rustc-env=TS_VERSION={}", ts::version()); println!("cargo:rustc-env=TS_VERSION={}", ts::version());
println!("cargo:rerun-if-env-changed=TS_VERSION"); println!("cargo:rerun-if-env-changed=TS_VERSION");
println!(
"cargo:rustc-env=DENO_CONSOLE_LIB_PATH={}",
deno_console::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_URL_LIB_PATH={}",
deno_url::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEB_LIB_PATH={}",
deno_web::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_FETCH_LIB_PATH={}",
deno_fetch::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBGPU_LIB_PATH={}",
deno_webgpu_get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBSOCKET_LIB_PATH={}",
deno_websocket::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_WEBSTORAGE_LIB_PATH={}",
deno_webstorage::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_CACHE_LIB_PATH={}",
deno_cache::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_CRYPTO_LIB_PATH={}",
deno_crypto::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_BROADCAST_CHANNEL_LIB_PATH={}",
deno_broadcast_channel::get_declaration().display()
);
println!(
"cargo:rustc-env=DENO_NET_LIB_PATH={}",
deno_net::get_declaration().display()
);
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());

29
cli/lsp/tsc.rs
View file

@ -208,7 +208,7 @@ impl AssetDocument {
type AssetsMap = HashMap<ModuleSpecifier, AssetDocument>; type AssetsMap = HashMap<ModuleSpecifier, AssetDocument>;
fn new_assets_map() -> Arc<Mutex<AssetsMap>> { fn new_assets_map() -> Arc<Mutex<AssetsMap>> {
let assets = tsc::STATIC_ASSETS let assets = tsc::LAZILY_LOADED_STATIC_ASSETS
.iter() .iter()
.map(|(k, v)| { .map(|(k, v)| {
let url_str = format!("asset:///{}", k); let url_str = format!("asset:///{}", k);
@ -3455,6 +3455,8 @@ mod tests {
use crate::lsp::documents::Documents; use crate::lsp::documents::Documents;
use crate::lsp::documents::LanguageId; use crate::lsp::documents::LanguageId;
use crate::lsp::text::LineIndex; use crate::lsp::text::LineIndex;
use crate::tsc::AssetText;
use pretty_assertions::assert_eq;
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use test_util::TempDir; use test_util::TempDir;
@ -3913,18 +3915,31 @@ mod tests {
Default::default(), Default::default(),
) )
.unwrap(); .unwrap();
let assets = result.as_array().unwrap(); let assets: Vec<AssetText> = serde_json::from_value(result).unwrap();
let mut asset_names = assets
.iter()
.map(|a| {
a.specifier
.replace("asset:///lib.", "")
.replace(".d.ts", "")
})
.collect::<Vec<_>>();
let mut expected_asset_names: Vec<String> = serde_json::from_str(
include_str!(concat!(env!("OUT_DIR"), "/lib_file_names.json")),
)
.unwrap();
asset_names.sort();
expected_asset_names.sort();
// You might have found this assertion starts failing after upgrading TypeScript. // You might have found this assertion starts failing after upgrading TypeScript.
// Just update the new number of assets (declaration files) for this number. // Ensure build.rs is updated so these match.
assert_eq!(assets.len(), 72); assert_eq!(asset_names, expected_asset_names);
// get some notification when the size of the assets grows // get some notification when the size of the assets grows
let mut total_size = 0; let mut total_size = 0;
for asset in assets { for asset in assets {
let obj = asset.as_object().unwrap(); total_size += asset.text.len();
let text = obj.get("text").unwrap().as_str().unwrap();
total_size += text.len();
} }
assert!(total_size > 0); assert!(total_size > 0);
assert!(total_size < 2_000_000); // currently as of TS 4.6, it's 0.7MB assert!(total_size < 2_000_000); // currently as of TS 4.6, it's 0.7MB

43
cli/tsc/99_main_compiler.js
View file

@ -385,7 +385,7 @@ delete Object.prototype.__proto__;
// paths must be either relative or absolute. Since // paths must be either relative or absolute. Since
// analysis in Rust operates on fully resolved URLs, // analysis in Rust operates on fully resolved URLs,
// it makes sense to use the same scheme here. // it makes sense to use the same scheme here.
const ASSETS = "asset:///"; const ASSETS_URL_PREFIX = "asset:///";
/** Diagnostics that are intentionally ignored when compiling TypeScript in /** Diagnostics that are intentionally ignored when compiling TypeScript in
* Deno, as they provide misleading or incorrect information. */ * Deno, as they provide misleading or incorrect information. */
@ -431,6 +431,7 @@ delete Object.prototype.__proto__;
noEmit: true, noEmit: true,
strict: true, strict: true,
target: ts.ScriptTarget.ESNext, target: ts.ScriptTarget.ESNext,
lib: ["lib.deno.window.d.ts"],
}; };
// todo(dsherret): can we remove this and just use ts.OperationCanceledException? // todo(dsherret): can we remove this and just use ts.OperationCanceledException?
@ -546,10 +547,10 @@ delete Object.prototype.__proto__;
return sourceFile; return sourceFile;
}, },
getDefaultLibFileName() { getDefaultLibFileName() {
return `${ASSETS}/lib.esnext.d.ts`; return `${ASSETS_URL_PREFIX}lib.esnext.d.ts`;
}, },
getDefaultLibLocation() { getDefaultLibLocation() {
return ASSETS; return ASSETS_URL_PREFIX;
}, },
writeFile(fileName, data, _writeByteOrderMark, _onError, _sourceFiles) { writeFile(fileName, data, _writeByteOrderMark, _onError, _sourceFiles) {
if (logDebug) { if (logDebug) {
@ -887,6 +888,20 @@ delete Object.prototype.__proto__;
debug("<<< exec stop"); debug("<<< exec stop");
} }
function getAssets() {
/** @type {{ specifier: string; text: string; }[]} */
const assets = [];
for (const sourceFile of sourceFileCache.values()) {
if (sourceFile.fileName.startsWith(ASSETS_URL_PREFIX)) {
assets.push({
specifier: sourceFile.fileName,
text: sourceFile.text,
});
}
}
return assets;
}
/** /**
* @param {number} id * @param {number} id
* @param {any} data * @param {any} data
@ -935,16 +950,7 @@ delete Object.prototype.__proto__;
); );
} }
case "getAssets": { case "getAssets": {
const assets = []; return respond(id, getAssets());
for (const sourceFile of sourceFileCache.values()) {
if (sourceFile.fileName.startsWith(ASSETS)) {
assets.push({
specifier: sourceFile.fileName,
text: sourceFile.text,
});
}
}
return respond(id, assets);
} }
case "getApplicableRefactors": { case "getApplicableRefactors": {
return respond( return respond(
@ -1281,7 +1287,10 @@ delete Object.prototype.__proto__;
// we are caching in memory common type libraries that will be re-used by // we are caching in memory common type libraries that will be re-used by
// tsc on when the snapshot is restored // tsc on when the snapshot is restored
assert( assert(
host.getSourceFile(`${ASSETS}${specifier}`, ts.ScriptTarget.ESNext), host.getSourceFile(
`${ASSETS_URL_PREFIX}${specifier}`,
ts.ScriptTarget.ESNext,
),
); );
} }
// this helps ensure as much as possible is in memory that is re-usable // this helps ensure as much as possible is in memory that is re-usable
@ -1292,12 +1301,16 @@ delete Object.prototype.__proto__;
options: SNAPSHOT_COMPILE_OPTIONS, options: SNAPSHOT_COMPILE_OPTIONS,
host, host,
}); });
ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM); assert(ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM).length === 0);
// remove this now that we don't need it anymore for warming up tsc
sourceFileCache.delete(buildSpecifier);
// exposes the two functions that are called by `tsc::exec()` when type // exposes the two functions that are called by `tsc::exec()` when type
// checking TypeScript. // checking TypeScript.
globalThis.startup = startup; globalThis.startup = startup;
globalThis.exec = exec; globalThis.exec = exec;
globalThis.getAssets = getAssets;
// exposes the functions that are called when the compiler is used as a // exposes the functions that are called when the compiler is used as a
// language service. // language service.

200
cli/tsc/mod.rs
View file

@ -25,6 +25,7 @@ use deno_core::serde::Serializer;
use deno_core::serde_json; use deno_core::serde_json;
use deno_core::serde_json::json; use deno_core::serde_json::json;
use deno_core::serde_json::Value; use deno_core::serde_json::Value;
use deno_core::serde_v8;
use deno_core::Extension; use deno_core::Extension;
use deno_core::JsRuntime; use deno_core::JsRuntime;
use deno_core::ModuleSpecifier; use deno_core::ModuleSpecifier;
@ -49,28 +50,6 @@ pub use self::diagnostics::DiagnosticMessageChain;
pub use self::diagnostics::Diagnostics; pub use self::diagnostics::Diagnostics;
pub use self::diagnostics::Position; pub use self::diagnostics::Position;
// Declaration files
pub static DENO_NS_LIB: &str = include_str!("dts/lib.deno.ns.d.ts");
pub static DENO_CONSOLE_LIB: &str = include_str!(env!("DENO_CONSOLE_LIB_PATH"));
pub static DENO_URL_LIB: &str = include_str!(env!("DENO_URL_LIB_PATH"));
pub static DENO_WEB_LIB: &str = include_str!(env!("DENO_WEB_LIB_PATH"));
pub static DENO_FETCH_LIB: &str = include_str!(env!("DENO_FETCH_LIB_PATH"));
pub static DENO_WEBGPU_LIB: &str = include_str!(env!("DENO_WEBGPU_LIB_PATH"));
pub static DENO_WEBSOCKET_LIB: &str =
include_str!(env!("DENO_WEBSOCKET_LIB_PATH"));
pub static DENO_WEBSTORAGE_LIB: &str =
include_str!(env!("DENO_WEBSTORAGE_LIB_PATH"));
pub static DENO_CACHE_LIB: &str = include_str!(env!("DENO_CACHE_LIB_PATH"));
pub static DENO_CRYPTO_LIB: &str = include_str!(env!("DENO_CRYPTO_LIB_PATH"));
pub static DENO_BROADCAST_CHANNEL_LIB: &str =
include_str!(env!("DENO_BROADCAST_CHANNEL_LIB_PATH"));
pub static DENO_NET_LIB: &str = include_str!(env!("DENO_NET_LIB_PATH"));
pub static SHARED_GLOBALS_LIB: &str =
include_str!("dts/lib.deno.shared_globals.d.ts");
pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
#[cold] #[cold]
#[inline(never)] #[inline(never)]
@ -89,28 +68,57 @@ pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
); );
pub fn get_types_declaration_file_text(unstable: bool) -> String { pub fn get_types_declaration_file_text(unstable: bool) -> String {
let mut types = vec![ let mut assets = get_asset_texts_from_new_runtime()
DENO_NS_LIB, .unwrap()
DENO_CONSOLE_LIB, .into_iter()
DENO_URL_LIB, .map(|a| (a.specifier, a.text))
DENO_WEB_LIB, .collect::<HashMap<_, _>>();
DENO_FETCH_LIB,
DENO_WEBGPU_LIB, let mut lib_names = vec![
DENO_WEBSOCKET_LIB, "deno.ns",
DENO_WEBSTORAGE_LIB, "deno.console",
DENO_CRYPTO_LIB, "deno.url",
DENO_BROADCAST_CHANNEL_LIB, "deno.web",
DENO_NET_LIB, "deno.fetch",
SHARED_GLOBALS_LIB, "deno.webgpu",
DENO_CACHE_LIB, "deno.websocket",
WINDOW_LIB, "deno.webstorage",
"deno.crypto",
"deno.broadcast_channel",
"deno.net",
"deno.shared_globals",
"deno.cache",
"deno.window",
]; ];
if unstable { if unstable {
types.push(UNSTABLE_NS_LIB); lib_names.push("deno.unstable");
} }
types.join("\n") lib_names
.into_iter()
.map(|name| {
let asset_url = format!("asset:///lib.{}.d.ts", name);
assets.remove(&asset_url).unwrap()
})
.collect::<Vec<_>>()
.join("\n")
}
fn get_asset_texts_from_new_runtime() -> Result<Vec<AssetText>, AnyError> {
// the assets are stored within the typescript isolate, so take them out of there
let mut runtime = JsRuntime::new(RuntimeOptions {
startup_snapshot: Some(compiler_snapshot()),
extensions: vec![Extension::builder("deno_cli_tsc")
.ops(get_tsc_ops())
.build()],
..Default::default()
});
let global =
runtime.execute_script("get_assets.js", "globalThis.getAssets()")?;
let scope = &mut runtime.handle_scope();
let local = deno_core::v8::Local::new(scope, global);
Ok(serde_v8::from_v8::<Vec<AssetText>>(scope, local)?)
} }
pub fn compiler_snapshot() -> Snapshot { pub fn compiler_snapshot() -> Snapshot {
@ -124,40 +132,44 @@ macro_rules! inc {
} }
/// Contains static assets that are not preloaded in the compiler snapshot. /// Contains static assets that are not preloaded in the compiler snapshot.
pub static STATIC_ASSETS: Lazy<HashMap<&'static str, &'static str>> = ///
Lazy::new(|| { /// We lazily load these because putting them in the compiler snapshot will
([ /// increase memory usage when not used (last time checked by about 0.5MB).
( pub static LAZILY_LOADED_STATIC_ASSETS: Lazy<
"lib.dom.asynciterable.d.ts", HashMap<&'static str, &'static str>,
inc!("lib.dom.asynciterable.d.ts"), > = Lazy::new(|| {
), ([
("lib.dom.d.ts", inc!("lib.dom.d.ts")), (
("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")), "lib.dom.asynciterable.d.ts",
("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")), inc!("lib.dom.asynciterable.d.ts"),
("lib.es6.d.ts", inc!("lib.es6.d.ts")), ),
("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")), ("lib.dom.d.ts", inc!("lib.dom.d.ts")),
("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")), ("lib.dom.extras.d.ts", inc!("lib.dom.extras.d.ts")),
("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")), ("lib.dom.iterable.d.ts", inc!("lib.dom.iterable.d.ts")),
("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")), ("lib.es6.d.ts", inc!("lib.es6.d.ts")),
("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")), ("lib.es2016.full.d.ts", inc!("lib.es2016.full.d.ts")),
("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")), ("lib.es2017.full.d.ts", inc!("lib.es2017.full.d.ts")),
("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")), ("lib.es2018.full.d.ts", inc!("lib.es2018.full.d.ts")),
("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")), ("lib.es2019.full.d.ts", inc!("lib.es2019.full.d.ts")),
("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")), ("lib.es2020.full.d.ts", inc!("lib.es2020.full.d.ts")),
("lib.webworker.d.ts", inc!("lib.webworker.d.ts")), ("lib.es2021.full.d.ts", inc!("lib.es2021.full.d.ts")),
( ("lib.es2022.full.d.ts", inc!("lib.es2022.full.d.ts")),
"lib.webworker.importscripts.d.ts", ("lib.esnext.full.d.ts", inc!("lib.esnext.full.d.ts")),
inc!("lib.webworker.importscripts.d.ts"), ("lib.scripthost.d.ts", inc!("lib.scripthost.d.ts")),
), ("lib.webworker.d.ts", inc!("lib.webworker.d.ts")),
( (
"lib.webworker.iterable.d.ts", "lib.webworker.importscripts.d.ts",
inc!("lib.webworker.iterable.d.ts"), inc!("lib.webworker.importscripts.d.ts"),
), ),
]) (
.iter() "lib.webworker.iterable.d.ts",
.cloned() inc!("lib.webworker.iterable.d.ts"),
.collect() ),
}); ])
.iter()
.cloned()
.collect()
});
/// A structure representing stats from a type check operation for a graph. /// A structure representing stats from a type check operation for a graph.
#[derive(Clone, Debug, Default, Eq, PartialEq)] #[derive(Clone, Debug, Default, Eq, PartialEq)]
@ -193,9 +205,16 @@ impl fmt::Display for Stats {
} }
} }
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssetText {
pub specifier: String,
pub text: String,
}
/// Retrieve a static asset that are included in the binary. /// Retrieve a static asset that are included in the binary.
pub fn get_asset(asset: &str) -> Option<&'static str> { fn get_lazily_loaded_asset(asset: &str) -> Option<&'static str> {
STATIC_ASSETS.get(asset).map(|s| s.to_owned()) LAZILY_LOADED_STATIC_ASSETS.get(asset).map(|s| s.to_owned())
} }
fn get_maybe_hash( fn get_maybe_hash(
@ -501,9 +520,8 @@ fn op_load(state: &mut OpState, args: Value) -> Result<Value, AnyError> {
hash = Some("1".to_string()); hash = Some("1".to_string());
media_type = MediaType::Dts; media_type = MediaType::Dts;
Some(Cow::Borrowed("declare const __: any;\nexport = __;\n")) Some(Cow::Borrowed("declare const __: any;\nexport = __;\n"))
} else if v.specifier.starts_with("asset:///") { } else if let Some(name) = v.specifier.strip_prefix("asset:///") {
let name = v.specifier.replace("asset:///", ""); let maybe_source = get_lazily_loaded_asset(name);
let maybe_source = get_asset(&name);
hash = get_maybe_hash(maybe_source, &state.hash_data); hash = get_maybe_hash(maybe_source, &state.hash_data);
media_type = MediaType::from(&v.specifier); media_type = MediaType::from(&v.specifier);
maybe_source.map(Cow::Borrowed) maybe_source.map(Cow::Borrowed)
@ -774,16 +792,7 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
let mut runtime = JsRuntime::new(RuntimeOptions { let mut runtime = JsRuntime::new(RuntimeOptions {
startup_snapshot: Some(compiler_snapshot()), startup_snapshot: Some(compiler_snapshot()),
extensions: vec![Extension::builder("deno_cli_tsc") extensions: vec![Extension::builder("deno_cli_tsc")
.ops(vec![ .ops(get_tsc_ops())
op_cwd::decl(),
op_create_hash::decl(),
op_emit::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_resolve::decl(),
op_respond::decl(),
])
.state(move |state| { .state(move |state| {
state.put(State::new( state.put(State::new(
request.graph_data.clone(), request.graph_data.clone(),
@ -833,6 +842,19 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
} }
} }
fn get_tsc_ops() -> Vec<deno_core::OpDecl> {
vec![
op_cwd::decl(),
op_create_hash::decl(),
op_emit::decl(),
op_exists::decl(),
op_is_node_file::decl(),
op_load::decl(),
op_resolve::decl(),
op_respond::decl(),
]
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::Diagnostic; use super::Diagnostic;
@ -1089,7 +1111,7 @@ mod tests {
.expect("should have invoked op"); .expect("should have invoked op");
let actual: LoadResponse = let actual: LoadResponse =
serde_json::from_value(value).expect("failed to deserialize"); serde_json::from_value(value).expect("failed to deserialize");
let expected = get_asset("lib.dom.d.ts").unwrap(); let expected = get_lazily_loaded_asset("lib.dom.d.ts").unwrap();
assert_eq!(actual.data, expected); assert_eq!(actual.data, expected);
assert!(actual.version.is_some()); assert!(actual.version.is_some());
assert_eq!(actual.script_kind, 3); assert_eq!(actual.script_kind, 3);