From 8799855fdc97960b18fbbb3450ed132e352607c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 20 May 2020 16:25:40 +0200 Subject: [PATCH] refactor: reorganize TS compiler (#5603) --- cli/file_fetcher.rs | 2 +- cli/js/compiler.ts | 89 +++++---------- cli/main.rs | 31 +++++- cli/module_graph.rs | 6 +- cli/tsc.rs | 258 ++++++++++++++++++++++---------------------- 5 files changed, 187 insertions(+), 199 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index cbfb340ddc..375142e071 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -535,7 +535,7 @@ impl SourceFileFetcher { } } -fn map_file_extension(path: &Path) -> msg::MediaType { +pub fn map_file_extension(path: &Path) -> msg::MediaType { match path.extension() { None => msg::MediaType::Unknown, Some(os_str) => match os_str.to_str() { diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index fa4a7526e7..a1189dc202 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -18,20 +18,15 @@ import { Diagnostic, DiagnosticItem } from "./diagnostics.ts"; import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts"; import { TranspileOnlyResult } from "./ops/runtime_compiler.ts"; import { bootstrapWorkerRuntime } from "./runtime_worker.ts"; -import { assert } from "./util.ts"; -import * as util from "./util.ts"; -import { TextDecoder, TextEncoder } from "./web/text_encoding.ts"; +import { assert, log, notImplemented } from "./util.ts"; import { core } from "./core.ts"; -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - +// We really don't want to depend on JSON dispatch during snapshotting, so +// this op exchanges strings with Rust as raw byte arrays. function getAsset(name: string): string { const opId = core.ops()["op_fetch_asset"]; - // We really don't want to depend on JSON dispatch during snapshotting, so - // this op exchanges strings with Rust as raw byte arrays. - const sourceCodeBytes = core.dispatch(opId, encoder.encode(name)); - return decoder.decode(sourceCodeBytes!); + const sourceCodeBytes = core.dispatch(opId, core.encode(name)); + return core.decode(sourceCodeBytes!); } // Constants used by `normalizeString` and `resolvePath` @@ -194,18 +189,6 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension { } } -/** Because we support providing types for JS files as well as X-TypeScript-Types - * header we might be feeding TS compiler with different files than import specifiers - * suggest. To accomplish that we keep track of two different specifiers: - * - original - the one in import statement (import "./foo.js") - * - mapped - if there's no type directive it's the same as original, otherwise - * it's unresolved specifier for type directive (/// @deno-types="./foo.d.ts") - */ -interface SourceFileSpecifierMap { - original: string; - mapped: string; -} - /** A global cache of module source files that have been loaded. * This cache will be rewritten to be populated on compiler startup * with files provided from Rust in request message. @@ -329,7 +312,7 @@ class Host implements ts.CompilerHost { path: string, configurationText: string ): ConfigureResponse { - util.log("compiler::host.configure", path); + log("compiler::host.configure", path); assert(configurationText); const { config, error } = ts.parseConfigFileTextToJson( path, @@ -367,7 +350,7 @@ class Host implements ts.CompilerHost { /* TypeScript CompilerHost APIs */ fileExists(_fileName: string): boolean { - return util.notImplemented(); + return notImplemented(); } getCanonicalFileName(fileName: string): string { @@ -375,7 +358,7 @@ class Host implements ts.CompilerHost { } getCompilationSettings(): ts.CompilerOptions { - util.log("compiler::host.getCompilationSettings()"); + log("compiler::host.getCompilationSettings()"); return this.#options; } @@ -384,7 +367,7 @@ class Host implements ts.CompilerHost { } getDefaultLibFileName(_options: ts.CompilerOptions): string { - util.log("compiler::host.getDefaultLibFileName()"); + log("compiler::host.getDefaultLibFileName()"); switch (this.#target) { case CompilerHostTarget.Main: case CompilerHostTarget.Runtime: @@ -404,7 +387,7 @@ class Host implements ts.CompilerHost { onError?: (message: string) => void, shouldCreateNewSourceFile?: boolean ): ts.SourceFile | undefined { - util.log("compiler::host.getSourceFile", fileName); + log("compiler::host.getSourceFile", fileName); try { assert(!shouldCreateNewSourceFile); const sourceFile = fileName.startsWith(ASSETS) @@ -436,21 +419,21 @@ class Host implements ts.CompilerHost { } readFile(_fileName: string): string | undefined { - return util.notImplemented(); + return notImplemented(); } resolveModuleNames( moduleNames: string[], containingFile: string ): Array { - util.log("compiler::host.resolveModuleNames", { + log("compiler::host.resolveModuleNames", { moduleNames, containingFile, }); return moduleNames.map((specifier) => { const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile); - util.log("compiler::host.resolveModuleNames maybeUrl", { + log("compiler::host.resolveModuleNames maybeUrl", { specifier, containingFile, maybeUrl, @@ -488,7 +471,7 @@ class Host implements ts.CompilerHost { _onError?: (message: string) => void, sourceFiles?: readonly ts.SourceFile[] ): void { - util.log("compiler::host.writeFile", fileName); + log("compiler::host.writeFile", fileName); this.#writeFile(fileName, data, sourceFiles); } } @@ -546,30 +529,6 @@ const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ // This function is called only during snapshotting process const SYSTEM_LOADER = getAsset("system_loader.js"); -function getMediaType(filename: string): MediaType { - const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); - if (!maybeExtension) { - util.log(`!!! Could not identify valid extension: "${filename}"`); - return MediaType.Unknown; - } - const [, extension] = maybeExtension; - switch (extension.toLowerCase()) { - case "js": - return MediaType.JavaScript; - case "jsx": - return MediaType.JSX; - case "ts": - return MediaType.TypeScript; - case "tsx": - return MediaType.TSX; - case "wasm": - return MediaType.Wasm; - default: - util.log(`!!! Unknown extension: "${extension}"`); - return MediaType.Unknown; - } -} - function buildLocalSourceFileCache( sourceFileMap: Record ): void { @@ -578,7 +537,7 @@ function buildLocalSourceFileCache( SourceFile.addToCache({ url: entry.url, filename: entry.url, - mediaType: getMediaType(entry.url), + mediaType: entry.mediaType, sourceCode: entry.sourceCode, }); @@ -673,7 +632,7 @@ function buildSourceFileCache( } } -interface EmmitedSource { +interface EmittedSource { // original filename filename: string; // compiled contents @@ -692,7 +651,7 @@ interface WriteFileState { bundleOutput?: string; host?: Host; rootNames: string[]; - emitMap?: Record; + emitMap?: Record; sources?: Record; } @@ -1115,13 +1074,13 @@ type CompilerRequest = | CompilerRequestRuntimeTranspile; interface CompileResult { - emitMap?: Record; + emitMap?: Record; bundleOutput?: string; diagnostics: Diagnostic; } interface RuntimeCompileResult { - emitMap: Record; + emitMap: Record; diagnostics: DiagnosticItem[]; } @@ -1141,7 +1100,7 @@ function compile(request: CompilerRequestCompile): CompileResult { cwd, sourceFileMap, } = request; - util.log(">>> compile start", { + log(">>> compile start", { rootNames, type: CompilerRequestType[request.type], }); @@ -1221,7 +1180,7 @@ function compile(request: CompilerRequestCompile): CompileResult { diagnostics: fromTypeScriptDiagnostic(diagnostics), }; - util.log("<<< compile end", { + log("<<< compile end", { rootNames, type: CompilerRequestType[request.type], }); @@ -1241,7 +1200,7 @@ function runtimeCompile( sourceFileMap, } = request; - util.log(">>> runtime compile start", { + log(">>> runtime compile start", { rootNames, bundle, }); @@ -1312,7 +1271,7 @@ function runtimeCompile( assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); assert(state.emitMap); - util.log("<<< runtime compile finish", { + log("<<< runtime compile finish", { rootNames, bundle, emitMap: Object.keys(state.emitMap), @@ -1385,7 +1344,7 @@ async function tsCompilerOnMessage({ break; } default: - util.log( + log( `!!! unhandled CompilerRequestType: ${ (request as CompilerRequest).type } (${CompilerRequestType[(request as CompilerRequest).type]})` diff --git a/cli/main.rs b/cli/main.rs index 5985a1040a..68ba5e7763 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -72,10 +72,12 @@ use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::fs as deno_fs; use crate::global_state::GlobalState; +use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::op_error::OpError; use crate::ops::io::get_stdio; use crate::permissions::Permissions; +use crate::state::exit_unstable; use crate::state::State; use crate::tsc::TargetLib; use crate::worker::MainWorker; @@ -396,12 +398,31 @@ async fn bundle_command( module_name = ModuleSpecifier::from(u) } - let global_state = GlobalState::new(flags)?; debug!(">>>>> bundle START"); - let bundle_result = global_state - .ts_compiler - .bundle(global_state.clone(), module_name, out_file) - .await; + let compiler_config = tsc::CompilerConfig::load(flags.config_path.clone())?; + + let maybe_import_map = match flags.import_map_path.as_ref() { + None => None, + Some(file_path) => { + if !flags.unstable { + exit_unstable("--importmap") + } + Some(ImportMap::load(file_path)?) + } + }; + + let global_state = GlobalState::new(flags)?; + + let bundle_result = tsc::bundle( + &global_state, + compiler_config, + module_name, + maybe_import_map, + out_file, + global_state.flags.unstable, + ) + .await; + debug!(">>>>> bundle END"); bundle_result } diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 23a9fe26b2..21e575cfd9 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -1,5 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; @@ -19,6 +20,7 @@ use serde::Serialize; use serde::Serializer; use std::collections::HashMap; use std::hash::BuildHasher; +use std::path::PathBuf; use std::pin::Pin; fn serialize_module_specifier( @@ -258,9 +260,9 @@ impl ModuleGraphLoader { ModuleGraphFile { specifier: specifier.to_string(), url: specifier.to_string(), + media_type: map_file_extension(&PathBuf::from(specifier.clone())) + as i32, filename: specifier, - // ignored, it's set in TS worker - media_type: MediaType::JavaScript as i32, source_code, imports, referenced_files, diff --git a/cli/tsc.rs b/cli/tsc.rs index 41eb7e43cf..0bfcf62667 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -165,6 +165,33 @@ lazy_static! { Regex::new(r#""checkJs"\s*?:\s*?true"#).unwrap(); } +/// Create a new worker with snapshot of TS compiler and setup compiler's +/// runtime. +fn create_compiler_worker( + global_state: GlobalState, + permissions: Permissions, +) -> CompilerWorker { + // TODO(bartlomieju): these $deno$ specifiers should be unified for all subcommands + // like 'eval', 'repl' + let entry_point = + ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap(); + let worker_state = + State::new(global_state.clone(), Some(permissions), entry_point, true) + .expect("Unable to create worker state"); + + // TODO(bartlomieju): this metric is never used anywhere + // Count how many times we start the compiler worker. + global_state.compiler_starts.fetch_add(1, Ordering::SeqCst); + + let mut worker = CompilerWorker::new( + "TS".to_string(), + startup_data::compiler_isolate_init(), + worker_state, + ); + worker.execute("bootstrap.tsCompilerRuntime()").unwrap(); + worker +} + #[derive(Clone)] pub enum TargetLib { Main, @@ -312,7 +339,7 @@ struct EmittedSource { #[serde(rename_all = "camelCase")] struct BundleResponse { diagnostics: Diagnostic, - bundle_output: String, + bundle_output: Option, } #[derive(Deserialize)] @@ -356,126 +383,6 @@ impl TsCompiler { }))) } - /// Create a new V8 worker with snapshot of TS compiler and setup compiler's - /// runtime. - fn setup_worker( - global_state: GlobalState, - permissions: Permissions, - ) -> CompilerWorker { - let entry_point = - ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap(); - let worker_state = - State::new(global_state.clone(), Some(permissions), entry_point, true) - .expect("Unable to create worker state"); - - // Count how many times we start the compiler worker. - global_state.compiler_starts.fetch_add(1, Ordering::SeqCst); - - let mut worker = CompilerWorker::new( - "TS".to_string(), - startup_data::compiler_isolate_init(), - worker_state, - ); - worker.execute("bootstrap.tsCompilerRuntime()").unwrap(); - worker - } - - pub async fn bundle( - &self, - global_state: GlobalState, - module_specifier: ModuleSpecifier, - out_file: Option, - ) -> Result<(), ErrBox> { - debug!( - "Invoking the compiler to bundle. module_name: {}", - module_specifier.to_string() - ); - eprintln!("Bundling {}", module_specifier.to_string()); - - let import_map: Option = - match global_state.flags.import_map_path.as_ref() { - None => None, - Some(file_path) => { - if !global_state.flags.unstable { - exit_unstable("--importmap") - } - Some(ImportMap::load(file_path)?) - } - }; - let permissions = Permissions::allow_all(); - let mut module_graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - import_map, - permissions.clone(), - false, - true, - ); - module_graph_loader.add_to_graph(&module_specifier).await?; - let module_graph = module_graph_loader.get_graph(); - let module_graph_json = - serde_json::to_value(module_graph).expect("Failed to serialize data"); - - let root_names = vec![module_specifier.to_string()]; - let bundle = true; - let target = "main"; - let unstable = global_state.flags.unstable; - let compiler_config = self.config.clone(); - let cwd = std::env::current_dir().unwrap(); - let j = match (compiler_config.path, compiler_config.content) { - (Some(config_path), Some(config_data)) => json!({ - "type": msg::CompilerRequestType::Compile as i32, - "target": target, - "rootNames": root_names, - "bundle": bundle, - "unstable": unstable, - "configPath": config_path, - "config": str::from_utf8(&config_data).unwrap(), - "cwd": cwd, - "sourceFileMap": module_graph_json, - }), - _ => json!({ - "type": msg::CompilerRequestType::Compile as i32, - "target": target, - "rootNames": root_names, - "bundle": bundle, - "unstable": unstable, - "cwd": cwd, - "sourceFileMap": module_graph_json, - }), - }; - - let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); - - let msg = - execute_in_same_thread(global_state.clone(), permissions, req_msg) - .await?; - let json_str = std::str::from_utf8(&msg).unwrap(); - debug!("Message: {}", json_str); - - let bundle_response: BundleResponse = serde_json::from_str(json_str)?; - - if !bundle_response.diagnostics.items.is_empty() { - return Err(ErrBox::from(bundle_response.diagnostics)); - } - - let output_string = fmt::format_text(&bundle_response.bundle_output)?; - - if let Some(out_file_) = out_file.as_ref() { - eprintln!("Emitting bundle to {:?}", out_file_); - - let output_bytes = output_string.as_bytes(); - let output_len = output_bytes.len(); - - deno_fs::write_file(out_file_, output_bytes, 0o666)?; - // TODO(bartlomieju): add "humanFileSize" method - eprintln!("{} bytes emitted.", output_len); - } else { - println!("{}", output_string); - } - - Ok(()) - } - /// Mark given module URL as compiled to avoid multiple compilations of same /// module in single run. fn mark_compiled(&self, url: &Url) { @@ -914,7 +821,7 @@ async fn execute_in_same_thread( permissions: Permissions, req: Buf, ) -> Result { - let mut worker = TsCompiler::setup_worker(global_state.clone(), permissions); + let mut worker = create_compiler_worker(global_state.clone(), permissions); let handle = worker.thread_safe_handle(); handle.post_message(req)?; @@ -943,6 +850,100 @@ async fn execute_in_same_thread( } } +pub async fn bundle( + global_state: &GlobalState, + compiler_config: CompilerConfig, + module_specifier: ModuleSpecifier, + maybe_import_map: Option, + out_file: Option, + unstable: bool, +) -> Result<(), ErrBox> { + debug!( + "Invoking the compiler to bundle. module_name: {}", + module_specifier.to_string() + ); + eprintln!("Bundling {}", module_specifier.to_string()); + + let permissions = Permissions::allow_all(); + let mut module_graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + maybe_import_map, + permissions.clone(), + false, + true, + ); + module_graph_loader.add_to_graph(&module_specifier).await?; + let module_graph = module_graph_loader.get_graph(); + let module_graph_json = + serde_json::to_value(module_graph).expect("Failed to serialize data"); + + let root_names = vec![module_specifier.to_string()]; + let bundle = true; + let target = "main"; + let cwd = std::env::current_dir().unwrap(); + + // TODO(bartlomieju): this is non-sense; CompilerConfig's `path` and `content` should + // be optional + let j = match (compiler_config.path, compiler_config.content) { + (Some(config_path), Some(config_data)) => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "configPath": config_path, + "config": str::from_utf8(&config_data).unwrap(), + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + _ => json!({ + "type": msg::CompilerRequestType::Compile as i32, + "target": target, + "rootNames": root_names, + "bundle": bundle, + "unstable": unstable, + "cwd": cwd, + "sourceFileMap": module_graph_json, + }), + }; + + let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); + + let msg = + execute_in_same_thread(global_state.clone(), permissions, req_msg).await?; + let json_str = std::str::from_utf8(&msg).unwrap(); + debug!("Message: {}", json_str); + + let bundle_response: BundleResponse = serde_json::from_str(json_str)?; + + if !bundle_response.diagnostics.items.is_empty() { + return Err(ErrBox::from(bundle_response.diagnostics)); + } + + assert!(bundle_response.bundle_output.is_some()); + let output = bundle_response.bundle_output.unwrap(); + + // TODO(bartlomieju): the rest of this function should be handled + // in `main.rs` - it has nothing to do with TypeScript... + let output_string = fmt::format_text(&output)?; + + if let Some(out_file_) = out_file.as_ref() { + eprintln!("Emitting bundle to {:?}", out_file_); + + let output_bytes = output_string.as_bytes(); + let output_len = output_bytes.len(); + + deno_fs::write_file(out_file_, output_bytes, 0o666)?; + // TODO(bartlomieju): do we really need to show this info? (it doesn't respect --quiet flag) + // TODO(bartlomieju): add "humanFileSize" method + eprintln!("{} bytes emitted.", output_len); + } else { + println!("{}", output_string); + } + + Ok(()) +} + /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. pub async fn runtime_compile( global_state: GlobalState, @@ -1138,10 +1139,15 @@ mod tests { String::from("$deno$/bundle.js"), ]); - let result = state - .ts_compiler - .bundle(state.clone(), module_name, None) - .await; + let result = bundle( + &state, + CompilerConfig::load(None).unwrap(), + module_name, + None, + None, + false, + ) + .await; assert!(result.is_ok()); }