diff --git a/cli/ops/compiler.rs b/cli/ops/compiler.rs index 40d25aa742..975e01ee14 100644 --- a/cli/ops/compiler.rs +++ b/cli/ops/compiler.rs @@ -33,39 +33,49 @@ pub fn op_cache( } #[derive(Deserialize)] -struct FetchSourceFileArgs { - specifier: String, +struct FetchSourceFilesArgs { + specifiers: Vec, referrer: String, } -pub fn op_fetch_source_file( +pub fn op_fetch_source_files( state: &ThreadSafeState, args: Value, _zero_copy: Option, ) -> Result { - let args: FetchSourceFileArgs = serde_json::from_value(args)?; + let args: FetchSourceFilesArgs = serde_json::from_value(args)?; // TODO(ry) Maybe a security hole. Only the compiler worker should have access // to this. Need a test to demonstrate the hole. let is_dyn_import = false; - let resolved_specifier = - state.resolve(&args.specifier, &args.referrer, false, is_dyn_import)?; - - let fut = state - .file_fetcher - .fetch_source_file_async(&resolved_specifier); + let mut futures = vec![]; + for specifier in &args.specifiers { + let resolved_specifier = + state.resolve(specifier, &args.referrer, false, is_dyn_import)?; + let fut = state + .file_fetcher + .fetch_source_file_async(&resolved_specifier); + futures.push(fut); + } // WARNING: Here we use tokio_util::block_on() which starts a new Tokio - // runtime for executing the future. This is so we don't inadvernently run + // runtime for executing the future. This is so we don't inadvertently run // out of threads in the main runtime. - let out = tokio_util::block_on(fut)?; - Ok(JsonOp::Sync(json!({ - "moduleName": out.url.to_string(), - "filename": out.filename.to_str().unwrap(), - "mediaType": out.media_type as i32, - "sourceCode": String::from_utf8(out.source_code).unwrap(), - }))) + let files = tokio_util::block_on(futures::future::join_all(futures))?; + let res: Vec = files + .into_iter() + .map(|file| { + json!({ + "moduleName": file.url.to_string(), + "filename": file.filename.to_str().unwrap(), + "mediaType": file.media_type as i32, + "sourceCode": String::from_utf8(file.source_code).unwrap(), + }) + }) + .collect(); + + Ok(JsonOp::Sync(json!(res))) } #[derive(Deserialize)] diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index 1f07acc658..7a2e9c9f46 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -37,7 +37,7 @@ pub const OP_START: OpId = 10; pub const OP_APPLY_SOURCE_MAP: OpId = 11; pub const OP_FORMAT_ERROR: OpId = 12; pub const OP_CACHE: OpId = 13; -pub const OP_FETCH_SOURCE_FILE: OpId = 14; +pub const OP_FETCH_SOURCE_FILES: OpId = 14; pub const OP_OPEN: OpId = 15; pub const OP_CLOSE: OpId = 16; pub const OP_SEEK: OpId = 17; @@ -133,8 +133,8 @@ pub fn dispatch( OP_CACHE => { dispatch_json::dispatch(compiler::op_cache, state, control, zero_copy) } - OP_FETCH_SOURCE_FILE => dispatch_json::dispatch( - compiler::op_fetch_source_file, + OP_FETCH_SOURCE_FILES => dispatch_json::dispatch( + compiler::op_fetch_source_files, state, control, zero_copy, diff --git a/js/compiler.ts b/js/compiler.ts index 41f23bcd21..1e1fe3dd39 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -136,18 +136,25 @@ function fetchAsset(name: string): string { return sendSync(dispatch.OP_FETCH_ASSET, { name }); } -/** Ops to Rust to resolve and fetch a modules meta data. */ -function fetchSourceFile(specifier: string, referrer: string): SourceFile { - util.log("compiler.fetchSourceFile", { specifier, referrer }); - const res = sendSync(dispatch.OP_FETCH_SOURCE_FILE, { - specifier, +/** Ops to Rust to resolve and fetch modules meta data. */ +function fetchSourceFiles( + specifiers: string[], + referrer: string +): SourceFile[] { + util.log("compiler.fetchSourceFiles", { specifiers, referrer }); + const res = sendSync(dispatch.OP_FETCH_SOURCE_FILES, { + specifiers, referrer }); - return { - ...res, - typeDirectives: parseTypeDirectives(res.sourceCode) - }; + return res.map( + (sourceFile: SourceFile): SourceFile => { + return { + ...sourceFile, + typeDirectives: parseTypeDirectives(sourceFile.sourceCode) + }; + } + ); } /** Utility function to turn the number of bytes into a human readable @@ -219,36 +226,71 @@ class Host implements ts.CompilerHost { private _sourceFileCache: Record = {}; - private _resolveModule(specifier: string, referrer: string): SourceFile { - util.log("host._resolveModule", { specifier, referrer }); - // Handle built-in assets specially. - if (specifier.startsWith(ASSETS)) { - const moduleName = specifier.split("/").pop()!; - if (moduleName in this._sourceFileCache) { - return this._sourceFileCache[moduleName]; - } - const assetName = moduleName.includes(".") - ? moduleName - : `${moduleName}.d.ts`; - const sourceCode = fetchAsset(assetName); - const sourceFile = { - moduleName, - filename: specifier, - mediaType: MediaType.TypeScript, - sourceCode - }; - this._sourceFileCache[moduleName] = sourceFile; - return sourceFile; - } - const sourceFile = fetchSourceFile(specifier, referrer); - assert(sourceFile.moduleName != null); - const { moduleName } = sourceFile; - if (!(moduleName! in this._sourceFileCache)) { - this._sourceFileCache[moduleName!] = sourceFile; + private _getAsset(specifier: string): SourceFile { + const moduleName = specifier.split("/").pop()!; + if (moduleName in this._sourceFileCache) { + return this._sourceFileCache[moduleName]; } + const assetName = moduleName.includes(".") + ? moduleName + : `${moduleName}.d.ts`; + const sourceCode = fetchAsset(assetName); + const sourceFile = { + moduleName, + filename: specifier, + mediaType: MediaType.TypeScript, + sourceCode + }; + this._sourceFileCache[moduleName] = sourceFile; return sourceFile; } + private _resolveModule(specifier: string, referrer: string): SourceFile { + return this._resolveModules([specifier], referrer)[0]; + } + + private _resolveModules( + specifiers: string[], + referrer: string + ): SourceFile[] { + util.log("host._resolveModules", { specifiers, referrer }); + const resolvedModules: Array = []; + const modulesToRequest = []; + + for (const specifier of specifiers) { + // Firstly built-in assets are handled specially, so they should + // be removed from array of files that we'll be requesting from Rust. + if (specifier.startsWith(ASSETS)) { + const assetFile = this._getAsset(specifier); + resolvedModules.push(assetFile); + } else if (specifier in this._sourceFileCache) { + const module = this._sourceFileCache[specifier]; + resolvedModules.push(module); + } else { + // Temporarily fill with undefined, after fetching file from + // Rust it will be filled with proper value. + resolvedModules.push(undefined); + modulesToRequest.push(specifier); + } + } + + // Now get files from Rust. + const sourceFiles = fetchSourceFiles(modulesToRequest, referrer); + + for (const sourceFile of sourceFiles) { + assert(sourceFile.moduleName != null); + const { moduleName } = sourceFile; + if (!(moduleName! in this._sourceFileCache)) { + this._sourceFileCache[moduleName!] = sourceFile; + } + // And fill temporary `undefined`s with actual files. + const index = resolvedModules.indexOf(undefined); + resolvedModules[index] = sourceFile; + } + + return resolvedModules as SourceFile[]; + } + /* Deno specific APIs */ /** Provides the `ts.HostCompiler` interface for Deno. @@ -371,22 +413,25 @@ class Host implements ts.CompilerHost { containingFile in this._sourceFileCache ? this._sourceFileCache[containingFile].typeDirectives : undefined; - return moduleNames.map( - (moduleName): ts.ResolvedModuleFull | undefined => { - const mappedModuleName = getMappedModuleName( - moduleName, - containingFile, - typeDirectives - ); - const sourceFile = this._resolveModule( - mappedModuleName, - containingFile - ); + + const mappedModuleNames = moduleNames.map( + (moduleName: string): string => { + return getMappedModuleName(moduleName, containingFile, typeDirectives); + } + ); + + return this._resolveModules(mappedModuleNames, containingFile).map( + ( + sourceFile: SourceFile, + index: number + ): ts.ResolvedModuleFull | undefined => { if (sourceFile.moduleName) { const resolvedFileName = sourceFile.moduleName; // This flags to the compiler to not go looking to transpile functional // code, anything that is in `/$asset$/` is just library code - const isExternalLibraryImport = moduleName.startsWith(ASSETS); + const isExternalLibraryImport = mappedModuleNames[index].startsWith( + ASSETS + ); const extension = getExtension( resolvedFileName, sourceFile.mediaType diff --git a/js/dispatch.ts b/js/dispatch.ts index 1a60a53633..a15da69f48 100644 --- a/js/dispatch.ts +++ b/js/dispatch.ts @@ -16,7 +16,7 @@ export const OP_START = 10; export const OP_APPLY_SOURCE_MAP = 11; export const OP_FORMAT_ERROR = 12; export const OP_CACHE = 13; -export const OP_FETCH_SOURCE_FILE = 14; +export const OP_FETCH_SOURCE_FILES = 14; export const OP_OPEN = 15; export const OP_CLOSE = 16; export const OP_SEEK = 17; diff --git a/tests/error_004_missing_module.ts.out b/tests/error_004_missing_module.ts.out index db56f51a57..c2a0d02082 100644 --- a/tests/error_004_missing_module.ts.out +++ b/tests/error_004_missing_module.ts.out @@ -3,10 +3,10 @@ at DenoError ([WILDCARD]errors.ts:[WILDCARD]) at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD]) - at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD]) - at [WILDCARD]compiler.ts:[WILDCARD] + at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD]) + at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD]) at resolveModuleNames ([WILDCARD]compiler.ts:[WILDCARD]) at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at processImportedModules ([WILDCARD]typescript.js:[WILDCARD]) + at findSourceFile ([WILDCARD]typescript.js:[WILDCARD]) diff --git a/tests/error_005_missing_dynamic_import.ts.out b/tests/error_005_missing_dynamic_import.ts.out index eb1d7b7b11..ec1468b09b 100644 --- a/tests/error_005_missing_dynamic_import.ts.out +++ b/tests/error_005_missing_dynamic_import.ts.out @@ -3,9 +3,10 @@ at DenoError ([WILDCARD]errors.ts:[WILDCARD]) at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD]) - at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD]) + at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD]) + at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD]) at [WILDCARD]compiler.ts:[WILDCARD] at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at processImportedModules ([WILDCARD]typescript.js:[WILDCARD]) + at findSourceFile ([WILDCARD]typescript.js:[WILDCARD]) diff --git a/tests/error_006_import_ext_failure.ts.out b/tests/error_006_import_ext_failure.ts.out index d0e14520b4..aa82c10aaf 100644 --- a/tests/error_006_import_ext_failure.ts.out +++ b/tests/error_006_import_ext_failure.ts.out @@ -3,9 +3,10 @@ at DenoError ([WILDCARD]errors.ts:[WILDCARD]) at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD]) - at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD]) + at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD]) + at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD]) at [WILDCARD]compiler.ts:[WILDCARD] at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at processImportedModules ([WILDCARD]typescript.js:[WILDCARD]) + at findSourceFile ([WILDCARD]typescript.js:[WILDCARD]) diff --git a/tests/error_011_bad_module_specifier.ts.out b/tests/error_011_bad_module_specifier.ts.out index 9918c503c8..97f59f2caf 100644 --- a/tests/error_011_bad_module_specifier.ts.out +++ b/tests/error_011_bad_module_specifier.ts.out @@ -3,9 +3,10 @@ at DenoError ([WILDCARD]errors.ts:[WILDCARD]) at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD]) - at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD]) + at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD]) + at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD]) at [WILDCARD]compiler.ts:[WILDCARD] at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at processImportedModules ([WILDCARD]typescript.js:[WILDCARD]) + at findSourceFile ([WILDCARD]typescript.js:[WILDCARD]) diff --git a/tests/error_012_bad_dynamic_import_specifier.ts.out b/tests/error_012_bad_dynamic_import_specifier.ts.out index 9918c503c8..97f59f2caf 100644 --- a/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -3,9 +3,10 @@ at DenoError ([WILDCARD]errors.ts:[WILDCARD]) at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at sendSync[WILDCARD] ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at fetchSourceFile ([WILDCARD]compiler.ts:[WILDCARD]) - at _resolveModule ([WILDCARD]compiler.ts:[WILDCARD]) + at fetchSourceFiles ([WILDCARD]compiler.ts:[WILDCARD]) + at _resolveModules ([WILDCARD]compiler.ts:[WILDCARD]) at [WILDCARD]compiler.ts:[WILDCARD] at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at processImportedModules ([WILDCARD]typescript.js:[WILDCARD]) + at findSourceFile ([WILDCARD]typescript.js:[WILDCARD])