From 2ecdbb62ae8f925b01e3e49f397d439a2f464a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 6 May 2020 00:19:18 +0200 Subject: [PATCH] refactor: merge TS compiler into single file (#5091) --- cli/js/compiler.ts | 1193 ++++++++++++++++- cli/js/compiler/bootstrap.ts | 42 - cli/js/compiler/bundler.ts | 93 -- cli/js/compiler/host.ts | 329 ----- cli/js/compiler/imports.ts | 130 -- cli/js/compiler/sourcefile.ts | 165 --- cli/js/compiler/type_directives.ts | 71 - cli/js/compiler/util.ts | 329 ----- cli/js/{compiler/api.ts => compiler_api.ts} | 142 +- cli/js/compiler_options.ts | 133 ++ cli/js/deno_unstable.ts | 2 +- cli/js/ops/compiler.ts | 40 - cli/js/{compiler => }/ts_global.d.ts | 0 cli/tests/020_json_modules.ts.out | 8 +- cli/tests/error_004_missing_module.ts.out | 4 +- .../error_005_missing_dynamic_import.ts.out | 4 +- cli/tests/error_006_import_ext_failure.ts.out | 4 +- .../error_011_bad_module_specifier.ts.out | 6 +- ...or_012_bad_dynamic_import_specifier.ts.out | 6 +- ...ror_local_static_import_from_remote.ts.out | 4 +- cli/tests/error_type_definitions.ts.out | 9 +- 21 files changed, 1320 insertions(+), 1394 deletions(-) delete mode 100644 cli/js/compiler/bootstrap.ts delete mode 100644 cli/js/compiler/bundler.ts delete mode 100644 cli/js/compiler/host.ts delete mode 100644 cli/js/compiler/imports.ts delete mode 100644 cli/js/compiler/sourcefile.ts delete mode 100644 cli/js/compiler/type_directives.ts delete mode 100644 cli/js/compiler/util.ts rename cli/js/{compiler/api.ts => compiler_api.ts} (51%) create mode 100644 cli/js/compiler_options.ts delete mode 100644 cli/js/ops/compiler.ts rename cli/js/{compiler => }/ts_global.d.ts (100%) diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index f1d3393168..99c168c5ee 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -11,40 +11,1165 @@ // to properly setup runtime. // NOTE: this import has side effects! -import "./compiler/ts_global.d.ts"; +import "./ts_global.d.ts"; -import { TranspileOnlyResult } from "./compiler/api.ts"; -import { TS_SNAPSHOT_PROGRAM } from "./compiler/bootstrap.ts"; -import { setRootExports } from "./compiler/bundler.ts"; -import { - CompilerHostTarget, - defaultBundlerOptions, - defaultRuntimeCompileOptions, - defaultTranspileOptions, - Host, -} from "./compiler/host.ts"; -import { - processImports, - processLocalImports, - resolveModules, -} from "./compiler/imports.ts"; -import { - EmmitedSource, - WriteFileCallback, - createCompileWriteFile, - createBundleWriteFile, - CompilerRequestType, - convertCompilerOptions, - ignoredDiagnostics, - WriteFileState, - processConfigureResponse, - base64ToUint8Array, -} from "./compiler/util.ts"; +import { bold, cyan, yellow } from "./colors.ts"; +import { CompilerOptions } from "./compiler_options.ts"; import { Diagnostic, DiagnosticItem } from "./diagnostics.ts"; import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts"; -import { assert } from "./util.ts"; -import * as util from "./util.ts"; +import { TranspileOnlyResult } from "./ops/runtime_compiler.ts"; +import { sendAsync, sendSync } from "./ops/dispatch_json.ts"; import { bootstrapWorkerRuntime } from "./runtime_worker.ts"; +import { assert, log } from "./util.ts"; +import * as util from "./util.ts"; +import { atob } from "./web/text_encoding.ts"; +import { TextDecoder, TextEncoder } from "./web/text_encoding.ts"; +import { core } from "./core.ts"; + +export function resolveModules( + specifiers: string[], + referrer?: string +): string[] { + util.log("compiler::resolveModules", { specifiers, referrer }); + return sendSync("op_resolve_modules", { specifiers, referrer }); +} + +export function fetchSourceFiles( + specifiers: string[], + referrer?: string +): Promise< + Array<{ + url: string; + filename: string; + mediaType: number; + sourceCode: string; + }> +> { + util.log("compiler::fetchSourceFiles", { specifiers, referrer }); + return sendAsync("op_fetch_source_files", { + specifiers, + referrer, + }); +} + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function getAsset(name: string): string { + const opId = core.ops()["op_fetch_asset"]; + // We really don't want to depend on JSON dispatch during snapshotting, so + // this op exchanges strings with Rust as raw byte arrays. + const sourceCodeBytes = core.dispatch(opId, encoder.encode(name)); + return decoder.decode(sourceCodeBytes!); +} + +// Constants used by `normalizeString` and `resolvePath` +const CHAR_DOT = 46; /* . */ +const CHAR_FORWARD_SLASH = 47; /* / */ +const ASSETS = "$asset$"; +const OUT_DIR = "$deno$"; + +// TODO(Bartlomieju): this check should be done in Rust +const IGNORED_COMPILER_OPTIONS: readonly string[] = [ + "allowSyntheticDefaultImports", + "baseUrl", + "build", + "composite", + "declaration", + "declarationDir", + "declarationMap", + "diagnostics", + "downlevelIteration", + "emitBOM", + "emitDeclarationOnly", + "esModuleInterop", + "extendedDiagnostics", + "forceConsistentCasingInFileNames", + "help", + "importHelpers", + "incremental", + "inlineSourceMap", + "inlineSources", + "init", + "isolatedModules", + "listEmittedFiles", + "listFiles", + "mapRoot", + "maxNodeModuleJsDepth", + "module", + "moduleResolution", + "newLine", + "noEmit", + "noEmitHelpers", + "noEmitOnError", + "noLib", + "noResolve", + "out", + "outDir", + "outFile", + "paths", + "preserveSymlinks", + "preserveWatchOutput", + "pretty", + "rootDir", + "rootDirs", + "showConfig", + "skipDefaultLibCheck", + "skipLibCheck", + "sourceMap", + "sourceRoot", + "stripInternal", + "target", + "traceResolution", + "tsBuildInfoFile", + "types", + "typeRoots", + "version", + "watch", +]; + +const DEFAULT_BUNDLER_OPTIONS: ts.CompilerOptions = { + allowJs: true, + inlineSourceMap: false, + module: ts.ModuleKind.System, + outDir: undefined, + outFile: `${OUT_DIR}/bundle.js`, + // disabled until we have effective way to modify source maps + sourceMap: false, +}; + +const DEFAULT_COMPILE_OPTIONS: ts.CompilerOptions = { + allowJs: false, + allowNonTsExtensions: true, + checkJs: false, + esModuleInterop: true, + jsx: ts.JsxEmit.React, + module: ts.ModuleKind.ESNext, + outDir: OUT_DIR, + resolveJsonModule: true, + sourceMap: true, + strict: true, + stripComments: true, + target: ts.ScriptTarget.ESNext, +}; + +const DEFAULT_RUNTIME_COMPILE_OPTIONS: ts.CompilerOptions = { + outDir: undefined, +}; + +const DEFAULT_RUNTIME_TRANSPILE_OPTIONS: ts.CompilerOptions = { + esModuleInterop: true, + module: ts.ModuleKind.ESNext, + sourceMap: true, + scriptComments: true, + target: ts.ScriptTarget.ESNext, +}; + +enum CompilerHostTarget { + Main = "main", + Runtime = "runtime", + Worker = "worker", +} + +interface CompilerHostOptions { + bundle?: boolean; + target: CompilerHostTarget; + unstable?: boolean; + writeFile: WriteFileCallback; +} + +interface ConfigureResponse { + ignoredOptions?: string[]; + diagnostics?: ts.Diagnostic[]; +} + +// Warning! The values in this enum are duplicated in `cli/msg.rs` +// Update carefully! +enum MediaType { + JavaScript = 0, + JSX = 1, + TypeScript = 2, + TSX = 3, + Json = 4, + Wasm = 5, + Unknown = 6, +} + +interface SourceFileJson { + url: string; + filename: string; + mediaType: MediaType; + sourceCode: string; +} + +function getExtension(fileName: string, mediaType: MediaType): ts.Extension { + switch (mediaType) { + case MediaType.JavaScript: + return ts.Extension.Js; + case MediaType.JSX: + return ts.Extension.Jsx; + case MediaType.TypeScript: + return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; + case MediaType.TSX: + return ts.Extension.Tsx; + case MediaType.Wasm: + // Custom marker for Wasm type. + return ts.Extension.Js; + case MediaType.Unknown: + default: + throw TypeError( + `Cannot resolve extension for "${fileName}" with mediaType "${MediaType[mediaType]}".` + ); + } +} + +/** A global cache of module source files that have been loaded. */ +const moduleCache: Map = new Map(); +/** A map of maps which cache source files for quicker modules resolution. */ +const specifierCache: Map> = new Map(); + +class SourceFile { + extension!: ts.Extension; + filename!: string; + + mediaType!: MediaType; + processed = false; + sourceCode?: string; + tsSourceFile?: ts.SourceFile; + url!: string; + + constructor(json: SourceFileJson) { + if (moduleCache.has(json.url)) { + throw new TypeError("SourceFile already exists"); + } + Object.assign(this, json); + this.extension = getExtension(this.url, this.mediaType); + moduleCache.set(this.url, this); + } + + cache(moduleSpecifier: string, containingFile?: string): void { + containingFile = containingFile || ""; + let innerCache = specifierCache.get(containingFile); + if (!innerCache) { + innerCache = new Map(); + specifierCache.set(containingFile, innerCache); + } + innerCache.set(moduleSpecifier, this); + } + + imports(processJsImports: boolean): Array<[string, string]> { + if (this.processed) { + throw new Error("SourceFile has already been processed."); + } + assert(this.sourceCode != null); + // we shouldn't process imports for files which contain the nocheck pragma + // (like bundles) + if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) { + log(`Skipping imports for "${this.filename}"`); + return []; + } + + const readImportFiles = true; + const detectJsImports = + this.mediaType === MediaType.JavaScript || + this.mediaType === MediaType.JSX; + + const preProcessedFileInfo = ts.preProcessFile( + this.sourceCode, + readImportFiles, + detectJsImports + ); + this.processed = true; + const files: Array<[string, string]> = []; + + function process(references: Array<{ fileName: string }>): void { + for (const { fileName } of references) { + files.push([fileName, fileName]); + } + } + + const { + importedFiles, + referencedFiles, + libReferenceDirectives, + typeReferenceDirectives, + } = preProcessedFileInfo; + const typeDirectives = parseTypeDirectives(this.sourceCode); + if (typeDirectives) { + for (const importedFile of importedFiles) { + files.push([ + importedFile.fileName, + getMappedModuleName(importedFile, typeDirectives), + ]); + } + } else if ( + !( + !processJsImports && + (this.mediaType === MediaType.JavaScript || + this.mediaType === MediaType.JSX) + ) + ) { + process(importedFiles); + } + process(referencedFiles); + // built in libs comes across as `"dom"` for example, and should be filtered + // out during pre-processing as they are either already cached or they will + // be lazily fetched by the compiler host. Ones that contain full files are + // not filtered out and will be fetched as normal. + process( + libReferenceDirectives.filter( + ({ fileName }) => !ts.libMap.has(fileName.toLowerCase()) + ) + ); + process(typeReferenceDirectives); + return files; + } + + static getUrl( + moduleSpecifier: string, + containingFile: string + ): string | undefined { + const containingCache = specifierCache.get(containingFile); + if (containingCache) { + const sourceFile = containingCache.get(moduleSpecifier); + return sourceFile && sourceFile.url; + } + return undefined; + } + + static get(url: string): SourceFile | undefined { + return moduleCache.get(url); + } +} + +function getAssetInternal(filename: string): SourceFile { + const lastSegment = filename.split("/").pop()!; + const url = ts.libMap.has(lastSegment) + ? ts.libMap.get(lastSegment)! + : lastSegment; + const sourceFile = SourceFile.get(url); + if (sourceFile) { + return sourceFile; + } + const name = url.includes(".") ? url : `${url}.d.ts`; + const sourceCode = getAsset(name); + return new SourceFile({ + url, + filename: `${ASSETS}/${name}`, + mediaType: MediaType.TypeScript, + sourceCode, + }); +} + +class Host implements ts.CompilerHost { + readonly #options = DEFAULT_COMPILE_OPTIONS; + #target: CompilerHostTarget; + #writeFile: WriteFileCallback; + + /* Deno specific APIs */ + + constructor({ + bundle = false, + target, + unstable, + writeFile, + }: CompilerHostOptions) { + this.#target = target; + this.#writeFile = writeFile; + if (bundle) { + // options we need to change when we are generating a bundle + Object.assign(this.#options, DEFAULT_BUNDLER_OPTIONS); + } + if (unstable) { + this.#options.lib = [ + target === CompilerHostTarget.Worker + ? "lib.deno.worker.d.ts" + : "lib.deno.window.d.ts", + "lib.deno.unstable.d.ts", + ]; + } + } + + configure( + cwd: string, + path: string, + configurationText: string + ): ConfigureResponse { + util.log("compiler::host.configure", path); + assert(configurationText); + const { config, error } = ts.parseConfigFileTextToJson( + path, + configurationText + ); + if (error) { + return { diagnostics: [error] }; + } + const { options, errors } = ts.convertCompilerOptionsFromJson( + config.compilerOptions, + cwd + ); + const ignoredOptions: string[] = []; + for (const key of Object.keys(options)) { + if ( + IGNORED_COMPILER_OPTIONS.includes(key) && + (!(key in this.#options) || options[key] !== this.#options[key]) + ) { + ignoredOptions.push(key); + delete options[key]; + } + } + Object.assign(this.#options, options); + return { + ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined, + diagnostics: errors.length ? errors : undefined, + }; + } + + mergeOptions(...options: ts.CompilerOptions[]): ts.CompilerOptions { + Object.assign(this.#options, ...options); + return Object.assign({}, this.#options); + } + + /* TypeScript CompilerHost APIs */ + + fileExists(_fileName: string): boolean { + return util.notImplemented(); + } + + getCanonicalFileName(fileName: string): string { + return fileName; + } + + getCompilationSettings(): ts.CompilerOptions { + util.log("compiler::host.getCompilationSettings()"); + return this.#options; + } + + getCurrentDirectory(): string { + return ""; + } + + getDefaultLibFileName(_options: ts.CompilerOptions): string { + util.log("compiler::host.getDefaultLibFileName()"); + switch (this.#target) { + case CompilerHostTarget.Main: + case CompilerHostTarget.Runtime: + return `${ASSETS}/lib.deno.window.d.ts`; + case CompilerHostTarget.Worker: + return `${ASSETS}/lib.deno.worker.d.ts`; + } + } + + getNewLine(): string { + return "\n"; + } + + getSourceFile( + fileName: string, + languageVersion: ts.ScriptTarget, + onError?: (message: string) => void, + shouldCreateNewSourceFile?: boolean + ): ts.SourceFile | undefined { + util.log("compiler::host.getSourceFile", fileName); + try { + assert(!shouldCreateNewSourceFile); + const sourceFile = fileName.startsWith(ASSETS) + ? getAssetInternal(fileName) + : SourceFile.get(fileName); + assert(sourceFile != null); + if (!sourceFile.tsSourceFile) { + assert(sourceFile.sourceCode != null); + const tsSourceFileName = fileName.startsWith(ASSETS) + ? sourceFile.filename + : fileName; + + sourceFile.tsSourceFile = ts.createSourceFile( + tsSourceFileName, + sourceFile.sourceCode, + languageVersion + ); + delete sourceFile.sourceCode; + } + return sourceFile.tsSourceFile; + } catch (e) { + if (onError) { + onError(String(e)); + } else { + throw e; + } + return undefined; + } + } + + readFile(_fileName: string): string | undefined { + return util.notImplemented(); + } + + resolveModuleNames( + moduleNames: string[], + containingFile: string + ): Array { + util.log("compiler::host.resolveModuleNames", { + moduleNames, + containingFile, + }); + return moduleNames.map((specifier) => { + const maybeUrl = SourceFile.getUrl(specifier, containingFile); + + let sourceFile: SourceFile | undefined = undefined; + + if (specifier.startsWith(ASSETS)) { + sourceFile = getAssetInternal(specifier); + } else if (typeof maybeUrl !== "undefined") { + sourceFile = SourceFile.get(maybeUrl); + } + + if (!sourceFile) { + return undefined; + } + + return { + resolvedFileName: sourceFile.url, + isExternalLibraryImport: specifier.startsWith(ASSETS), + extension: sourceFile.extension, + }; + }); + } + + useCaseSensitiveFileNames(): boolean { + return true; + } + + writeFile( + fileName: string, + data: string, + _writeByteOrderMark: boolean, + _onError?: (message: string) => void, + sourceFiles?: readonly ts.SourceFile[] + ): void { + util.log("compiler::host.writeFile", fileName); + this.#writeFile(fileName, data, sourceFiles); + } +} + +// NOTE: target doesn't really matter here, +// this is in fact a mock host created just to +// load all type definitions and snapshot them. +const SNAPSHOT_HOST = new Host({ + target: CompilerHostTarget.Main, + writeFile(): void {}, +}); +const SNAPSHOT_COMPILER_OPTIONS = SNAPSHOT_HOST.getCompilationSettings(); + +// This is a hacky way of adding our libs to the libs available in TypeScript() +// as these are internal APIs of TypeScript which maintain valid libs +ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals"); +ts.libMap.set("deno.ns", "lib.deno.ns.d.ts"); +ts.libMap.set("deno.window", "lib.deno.window.d.ts"); +ts.libMap.set("deno.worker", "lib.deno.worker.d.ts"); +ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts"); +ts.libMap.set("deno.unstable", "lib.deno.unstable.d.ts"); + +// this pre-populates the cache at snapshot time of our library files, so they +// are available in the future when needed. +SNAPSHOT_HOST.getSourceFile( + `${ASSETS}/lib.deno.ns.d.ts`, + ts.ScriptTarget.ESNext +); +SNAPSHOT_HOST.getSourceFile( + `${ASSETS}/lib.deno.window.d.ts`, + ts.ScriptTarget.ESNext +); +SNAPSHOT_HOST.getSourceFile( + `${ASSETS}/lib.deno.worker.d.ts`, + ts.ScriptTarget.ESNext +); +SNAPSHOT_HOST.getSourceFile( + `${ASSETS}/lib.deno.shared_globals.d.ts`, + ts.ScriptTarget.ESNext +); +SNAPSHOT_HOST.getSourceFile( + `${ASSETS}/lib.deno.unstable.d.ts`, + ts.ScriptTarget.ESNext +); + +const TS_SNAPSHOT_PROGRAM = ts.createProgram({ + rootNames: [`${ASSETS}/bootstrap.ts`], + options: SNAPSHOT_COMPILER_OPTIONS, + host: SNAPSHOT_HOST, +}); + +// This function is called only during snapshotting process +const SYSTEM_LOADER = getAsset("system_loader.js"); + +function resolveSpecifier(specifier: string, referrer: string): string { + // The resolveModules op only handles fully qualified URLs for referrer. + // However we will have cases where referrer is "/foo.ts". We add this dummy + // prefix "file://" in order to use the op. + // TODO(ry) Maybe we should perhaps ModuleSpecifier::resolve_import() to + // handle this situation. + let dummyPrefix = false; + const prefix = "file://"; + if (referrer.startsWith("/")) { + dummyPrefix = true; + referrer = prefix + referrer; + } + let r = resolveModules([specifier], referrer)[0]; + if (dummyPrefix) { + r = r.replace(prefix, ""); + } + return r; +} + +function getMediaType(filename: string): MediaType { + const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); + if (!maybeExtension) { + util.log(`!!! Could not identify valid extension: "${filename}"`); + return MediaType.Unknown; + } + const [, extension] = maybeExtension; + switch (extension.toLowerCase()) { + case "js": + return MediaType.JavaScript; + case "jsx": + return MediaType.JSX; + case "ts": + return MediaType.TypeScript; + case "tsx": + return MediaType.TSX; + case "wasm": + return MediaType.Wasm; + default: + util.log(`!!! Unknown extension: "${extension}"`); + return MediaType.Unknown; + } +} + +function processLocalImports( + sources: Record, + specifiers: Array<[string, string]>, + referrer?: string, + processJsImports = false +): string[] { + if (!specifiers.length) { + return []; + } + const moduleNames = specifiers.map( + referrer + ? ([, specifier]): string => resolveSpecifier(specifier, referrer) + : ([, specifier]): string => specifier + ); + for (let i = 0; i < moduleNames.length; i++) { + const moduleName = moduleNames[i]; + assert(moduleName in sources, `Missing module in sources: "${moduleName}"`); + const sourceFile = + SourceFile.get(moduleName) || + new SourceFile({ + url: moduleName, + filename: moduleName, + sourceCode: sources[moduleName], + mediaType: getMediaType(moduleName), + }); + sourceFile.cache(specifiers[i][0], referrer); + if (!sourceFile.processed) { + processLocalImports( + sources, + sourceFile.imports(processJsImports), + sourceFile.url, + processJsImports + ); + } + } + return moduleNames; +} + +async function processImports( + specifiers: Array<[string, string]>, + referrer?: string, + processJsImports = false +): Promise { + if (!specifiers.length) { + return []; + } + const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier); + const resolvedSources = resolveModules(sources, referrer); + const sourceFiles = await fetchSourceFiles(resolvedSources, referrer); + assert(sourceFiles.length === specifiers.length); + for (let i = 0; i < sourceFiles.length; i++) { + const sourceFileJson = sourceFiles[i]; + const sourceFile = + SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson); + sourceFile.cache(specifiers[i][0], referrer); + if (!sourceFile.processed) { + const sourceFileImports = sourceFile.imports(processJsImports); + await processImports(sourceFileImports, sourceFile.url, processJsImports); + } + } + return resolvedSources; +} + +interface FileReference { + fileName: string; + pos: number; + end: number; +} + +function getMappedModuleName( + source: FileReference, + typeDirectives: Map +): string { + const { fileName: sourceFileName, pos: sourcePos } = source; + for (const [{ fileName, pos }, value] of typeDirectives.entries()) { + if (sourceFileName === fileName && sourcePos === pos) { + return value; + } + } + return source.fileName; +} + +const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi; + +const importExportRegEx = /(?:import|export)(?:\s+|\s+[\s\S]*?from\s+)?(["'])((?:(?=(\\?))\3.)*?)\1/; + +function parseTypeDirectives( + sourceCode: string | undefined +): Map | undefined { + if (!sourceCode) { + return; + } + + // collect all the directives in the file and their start and end positions + const directives: FileReference[] = []; + let maybeMatch: RegExpExecArray | null = null; + while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) { + const [matchString, , fileName] = maybeMatch; + const { index: pos } = maybeMatch; + directives.push({ + fileName, + pos, + end: pos + matchString.length, + }); + } + if (!directives.length) { + return; + } + + // work from the last directive backwards for the next `import`/`export` + // statement + directives.reverse(); + const results = new Map(); + for (const { end, fileName, pos } of directives) { + const searchString = sourceCode.substring(end); + const maybeMatch = importExportRegEx.exec(searchString); + if (maybeMatch) { + const [matchString, , targetFileName] = maybeMatch; + const targetPos = + end + maybeMatch.index + matchString.indexOf(targetFileName) - 1; + const target: FileReference = { + fileName: targetFileName, + pos: targetPos, + end: targetPos + targetFileName.length, + }; + results.set(target, fileName); + } + sourceCode = sourceCode.substring(0, pos); + } + + return results; +} + +interface EmmitedSource { + // original filename + filename: string; + // compiled contents + contents: string; +} + +type WriteFileCallback = ( + fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] +) => void; + +interface WriteFileState { + type: CompilerRequestType; + bundle?: boolean; + bundleOutput?: string; + host?: Host; + rootNames: string[]; + emitMap?: Record; + sources?: Record; +} + +// Warning! The values in this enum are duplicated in `cli/msg.rs` +// Update carefully! +enum CompilerRequestType { + Compile = 0, + RuntimeCompile = 1, + RuntimeTranspile = 2, +} + +// TODO(bartlomieju): probably could be defined inline? +function createBundleWriteFile(state: WriteFileState): WriteFileCallback { + return function writeFile( + _fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] + ): void { + assert(sourceFiles != null); + assert(state.host); + assert(state.emitMap); + assert(state.bundle); + // we only support single root names for bundles + assert(state.rootNames.length === 1); + state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles); + }; +} + +// TODO(bartlomieju): probably could be defined inline? +function createCompileWriteFile(state: WriteFileState): WriteFileCallback { + return function writeFile( + fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] + ): void { + assert(sourceFiles != null); + assert(state.host); + assert(state.emitMap); + assert(!state.bundle); + assert(sourceFiles.length === 1); + state.emitMap[fileName] = { + filename: sourceFiles[0].fileName, + contents: data, + }; + }; +} + +interface ConvertCompilerOptionsResult { + files?: string[]; + options: ts.CompilerOptions; +} + +function convertCompilerOptions(str: string): ConvertCompilerOptionsResult { + const options: CompilerOptions = JSON.parse(str); + const out: Record = {}; + const keys = Object.keys(options) as Array; + const files: string[] = []; + for (const key of keys) { + switch (key) { + case "jsx": + const value = options[key]; + if (value === "preserve") { + out[key] = ts.JsxEmit.Preserve; + } else if (value === "react") { + out[key] = ts.JsxEmit.React; + } else { + out[key] = ts.JsxEmit.ReactNative; + } + break; + case "module": + switch (options[key]) { + case "amd": + out[key] = ts.ModuleKind.AMD; + break; + case "commonjs": + out[key] = ts.ModuleKind.CommonJS; + break; + case "es2015": + case "es6": + out[key] = ts.ModuleKind.ES2015; + break; + case "esnext": + out[key] = ts.ModuleKind.ESNext; + break; + case "none": + out[key] = ts.ModuleKind.None; + break; + case "system": + out[key] = ts.ModuleKind.System; + break; + case "umd": + out[key] = ts.ModuleKind.UMD; + break; + default: + throw new TypeError("Unexpected module type"); + } + break; + case "target": + switch (options[key]) { + case "es3": + out[key] = ts.ScriptTarget.ES3; + break; + case "es5": + out[key] = ts.ScriptTarget.ES5; + break; + case "es6": + case "es2015": + out[key] = ts.ScriptTarget.ES2015; + break; + case "es2016": + out[key] = ts.ScriptTarget.ES2016; + break; + case "es2017": + out[key] = ts.ScriptTarget.ES2017; + break; + case "es2018": + out[key] = ts.ScriptTarget.ES2018; + break; + case "es2019": + out[key] = ts.ScriptTarget.ES2019; + break; + case "es2020": + out[key] = ts.ScriptTarget.ES2020; + break; + case "esnext": + out[key] = ts.ScriptTarget.ESNext; + break; + default: + throw new TypeError("Unexpected emit target."); + } + break; + case "types": + const types = options[key]; + assert(types); + files.push(...types); + break; + default: + out[key] = options[key]; + } + } + return { + options: out as ts.CompilerOptions, + files: files.length ? files : undefined, + }; +} + +const ignoredDiagnostics = [ + // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is + // not a module. + 2306, + // TS1375: 'await' expressions are only allowed at the top level of a file + // when that file is a module, but this file has no imports or exports. + // Consider adding an empty 'export {}' to make this file a module. + 1375, + // TS1103: 'for-await-of' statement is only allowed within an async function + // or async generator. + 1103, + // TS2691: An import path cannot end with a '.ts' extension. Consider + // importing 'bad-module' instead. + 2691, + // TS5009: Cannot find the common subdirectory path for the input files. + 5009, + // TS5055: Cannot write file + // 'http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js' + // because it would overwrite input file. + 5055, + // TypeScript is overly opinionated that only CommonJS modules kinds can + // support JSON imports. Allegedly this was fixed in + // Microsoft/TypeScript#26825 but that doesn't seem to be working here, + // so we will ignore complaints about this compiler setting. + 5070, + // TS7016: Could not find a declaration file for module '...'. '...' + // implicitly has an 'any' type. This is due to `allowJs` being off by + // default but importing of a JavaScript module. + 7016, +]; + +// TODO(Bartlomieju): this check should be done in Rust; there should be no +// console.log here +function processConfigureResponse( + configResult: ConfigureResponse, + configPath: string +): ts.Diagnostic[] | undefined { + const { ignoredOptions, diagnostics } = configResult; + if (ignoredOptions) { + console.warn( + yellow(`Unsupported compiler options in "${configPath}"\n`) + + cyan(` The following options were ignored:\n`) + + ` ${ignoredOptions.map((value): string => bold(value)).join(", ")}` + ); + } + return diagnostics; +} + +function normalizeString(path: string): string { + let res = ""; + let lastSegmentLength = 0; + let lastSlash = -1; + let dots = 0; + let code: number; + for (let i = 0, len = path.length; i <= len; ++i) { + if (i < len) code = path.charCodeAt(i); + else if (code! === CHAR_FORWARD_SLASH) break; + else code = CHAR_FORWARD_SLASH; + + if (code === CHAR_FORWARD_SLASH) { + if (lastSlash === i - 1 || dots === 1) { + // NOOP + } else if (lastSlash !== i - 1 && dots === 2) { + if ( + res.length < 2 || + lastSegmentLength !== 2 || + res.charCodeAt(res.length - 1) !== CHAR_DOT || + res.charCodeAt(res.length - 2) !== CHAR_DOT + ) { + if (res.length > 2) { + const lastSlashIndex = res.lastIndexOf("/"); + if (lastSlashIndex === -1) { + res = ""; + lastSegmentLength = 0; + } else { + res = res.slice(0, lastSlashIndex); + lastSegmentLength = res.length - 1 - res.lastIndexOf("/"); + } + lastSlash = i; + dots = 0; + continue; + } else if (res.length === 2 || res.length === 1) { + res = ""; + lastSegmentLength = 0; + lastSlash = i; + dots = 0; + continue; + } + } + } else { + if (res.length > 0) res += "/" + path.slice(lastSlash + 1, i); + else res = path.slice(lastSlash + 1, i); + lastSegmentLength = i - lastSlash - 1; + } + lastSlash = i; + dots = 0; + } else if (code === CHAR_DOT && dots !== -1) { + ++dots; + } else { + dots = -1; + } + } + return res; +} + +function commonPath(paths: string[], sep = "/"): string { + const [first = "", ...remaining] = paths; + if (first === "" || remaining.length === 0) { + return first.substring(0, first.lastIndexOf(sep) + 1); + } + const parts = first.split(sep); + + let endOfPrefix = parts.length; + for (const path of remaining) { + const compare = path.split(sep); + for (let i = 0; i < endOfPrefix; i++) { + if (compare[i] !== parts[i]) { + endOfPrefix = i; + } + } + + if (endOfPrefix === 0) { + return ""; + } + } + const prefix = parts.slice(0, endOfPrefix).join(sep); + return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`; +} + +function base64ToUint8Array(data: string): Uint8Array { + const binString = atob(data); + const size = binString.length; + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) { + bytes[i] = binString.charCodeAt(i); + } + return bytes; +} + +let rootExports: string[] | undefined; + +function normalizeUrl(rootName: string): string { + const match = /^(\S+:\/{2,3})(.+)$/.exec(rootName); + if (match) { + const [, protocol, path] = match; + return `${protocol}${normalizeString(path)}`; + } else { + return rootName; + } +} + +function buildBundle( + rootName: string, + data: string, + sourceFiles: readonly ts.SourceFile[] +): string { + // when outputting to AMD and a single outfile, TypeScript makes up the module + // specifiers which are used to define the modules, and doesn't expose them + // publicly, so we have to try to replicate + const sources = sourceFiles.map((sf) => sf.fileName); + const sharedPath = commonPath(sources); + rootName = normalizeUrl(rootName) + .replace(sharedPath, "") + .replace(/\.\w+$/i, ""); + // If one of the modules requires support for top-level-await, TypeScript will + // emit the execute function as an async function. When this is the case we + // need to bubble up the TLA to the instantiation, otherwise we instantiate + // synchronously. + const hasTla = data.match(/execute:\sasync\sfunction\s/); + let instantiate: string; + if (rootExports && rootExports.length) { + instantiate = hasTla + ? `const __exp = await __instantiateAsync("${rootName}");\n` + : `const __exp = __instantiate("${rootName}");\n`; + for (const rootExport of rootExports) { + if (rootExport === "default") { + instantiate += `export default __exp["${rootExport}"];\n`; + } else { + instantiate += `export const ${rootExport} = __exp["${rootExport}"];\n`; + } + } + } else { + instantiate = hasTla + ? `await __instantiateAsync("${rootName}");\n` + : `__instantiate("${rootName}");\n`; + } + return `${SYSTEM_LOADER}\n${data}\n${instantiate}`; +} + +function setRootExports(program: ts.Program, rootModule: string): void { + // get a reference to the type checker, this will let us find symbols from + // the AST. + const checker = program.getTypeChecker(); + // get a reference to the main source file for the bundle + const mainSourceFile = program.getSourceFile(rootModule); + assert(mainSourceFile); + // retrieve the internal TypeScript symbol for this AST node + const mainSymbol = checker.getSymbolAtLocation(mainSourceFile); + if (!mainSymbol) { + return; + } + rootExports = checker + .getExportsOfModule(mainSymbol) + // .getExportsOfModule includes type only symbols which are exported from + // the module, so we need to try to filter those out. While not critical + // someone looking at the bundle would think there is runtime code behind + // that when there isn't. There appears to be no clean way of figuring that + // out, so inspecting SymbolFlags that might be present that are type only + .filter( + (sym) => + sym.flags & ts.SymbolFlags.Class || + !( + sym.flags & ts.SymbolFlags.Interface || + sym.flags & ts.SymbolFlags.TypeLiteral || + sym.flags & ts.SymbolFlags.Signature || + sym.flags & ts.SymbolFlags.TypeParameter || + sym.flags & ts.SymbolFlags.TypeAlias || + sym.flags & ts.SymbolFlags.Type || + sym.flags & ts.SymbolFlags.Namespace || + sym.flags & ts.SymbolFlags.InterfaceExcludes || + sym.flags & ts.SymbolFlags.TypeParameterExcludes || + sym.flags & ts.SymbolFlags.TypeAliasExcludes + ) + ) + .map((sym) => sym.getName()); +} interface CompilerRequestCompile { type: CompilerRequestType.Compile; @@ -283,7 +1408,7 @@ async function runtimeCompile( target, writeFile, })); - const compilerOptions = [defaultRuntimeCompileOptions]; + const compilerOptions = [DEFAULT_RUNTIME_COMPILE_OPTIONS]; if (convertedOptions) { compilerOptions.push(convertedOptions); } @@ -296,7 +1421,7 @@ async function runtimeCompile( }); } if (bundle) { - compilerOptions.push(defaultBundlerOptions); + compilerOptions.push(DEFAULT_BUNDLER_OPTIONS); } host.mergeOptions(...compilerOptions); @@ -352,10 +1477,10 @@ function runtimeTranspile( const compilerOptions = options ? Object.assign( {}, - defaultTranspileOptions, + DEFAULT_RUNTIME_TRANSPILE_OPTIONS, convertCompilerOptions(options).options ) - : defaultTranspileOptions; + : DEFAULT_RUNTIME_TRANSPILE_OPTIONS; for (const [fileName, inputText] of Object.entries(sources)) { const { outputText: source, sourceMapText: map } = ts.transpileModule( diff --git a/cli/js/compiler/bootstrap.ts b/cli/js/compiler/bootstrap.ts deleted file mode 100644 index 5f9dfad92a..0000000000 --- a/cli/js/compiler/bootstrap.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { CompilerHostTarget, Host } from "./host.ts"; -import { ASSETS } from "./sourcefile.ts"; -import { getAsset } from "./util.ts"; - -// NOTE: target doesn't really matter here, -// this is in fact a mock host created just to -// load all type definitions and snapshot them. -const host = new Host({ - target: CompilerHostTarget.Main, - writeFile(): void {}, -}); -const options = host.getCompilationSettings(); - -// This is a hacky way of adding our libs to the libs available in TypeScript() -// as these are internal APIs of TypeScript which maintain valid libs -ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals"); -ts.libMap.set("deno.ns", "lib.deno.ns.d.ts"); -ts.libMap.set("deno.window", "lib.deno.window.d.ts"); -ts.libMap.set("deno.worker", "lib.deno.worker.d.ts"); -ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts"); -ts.libMap.set("deno.unstable", "lib.deno.unstable.d.ts"); - -// this pre-populates the cache at snapshot time of our library files, so they -// are available in the future when needed. -host.getSourceFile(`${ASSETS}/lib.deno.ns.d.ts`, ts.ScriptTarget.ESNext); -host.getSourceFile(`${ASSETS}/lib.deno.window.d.ts`, ts.ScriptTarget.ESNext); -host.getSourceFile(`${ASSETS}/lib.deno.worker.d.ts`, ts.ScriptTarget.ESNext); -host.getSourceFile( - `${ASSETS}/lib.deno.shared_globals.d.ts`, - ts.ScriptTarget.ESNext -); -host.getSourceFile(`${ASSETS}/lib.deno.unstable.d.ts`, ts.ScriptTarget.ESNext); - -export const TS_SNAPSHOT_PROGRAM = ts.createProgram({ - rootNames: [`${ASSETS}/bootstrap.ts`], - options, - host, -}); - -export const SYSTEM_LOADER = getAsset("system_loader.js"); diff --git a/cli/js/compiler/bundler.ts b/cli/js/compiler/bundler.ts deleted file mode 100644 index 3f05a3be3d..0000000000 --- a/cli/js/compiler/bundler.ts +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { SYSTEM_LOADER } from "./bootstrap.ts"; -import { commonPath, normalizeString } from "./util.ts"; -import { assert } from "../util.ts"; - -let rootExports: string[] | undefined; - -function normalizeUrl(rootName: string): string { - const match = /^(\S+:\/{2,3})(.+)$/.exec(rootName); - if (match) { - const [, protocol, path] = match; - return `${protocol}${normalizeString(path)}`; - } else { - return rootName; - } -} - -export function buildBundle( - rootName: string, - data: string, - sourceFiles: readonly ts.SourceFile[] -): string { - // when outputting to AMD and a single outfile, TypeScript makes up the module - // specifiers which are used to define the modules, and doesn't expose them - // publicly, so we have to try to replicate - const sources = sourceFiles.map((sf) => sf.fileName); - const sharedPath = commonPath(sources); - rootName = normalizeUrl(rootName) - .replace(sharedPath, "") - .replace(/\.\w+$/i, ""); - // If one of the modules requires support for top-level-await, TypeScript will - // emit the execute function as an async function. When this is the case we - // need to bubble up the TLA to the instantiation, otherwise we instantiate - // synchronously. - const hasTla = data.match(/execute:\sasync\sfunction\s/); - let instantiate: string; - if (rootExports && rootExports.length) { - instantiate = hasTla - ? `const __exp = await __instantiateAsync("${rootName}");\n` - : `const __exp = __instantiate("${rootName}");\n`; - for (const rootExport of rootExports) { - if (rootExport === "default") { - instantiate += `export default __exp["${rootExport}"];\n`; - } else { - instantiate += `export const ${rootExport} = __exp["${rootExport}"];\n`; - } - } - } else { - instantiate = hasTla - ? `await __instantiateAsync("${rootName}");\n` - : `__instantiate("${rootName}");\n`; - } - return `${SYSTEM_LOADER}\n${data}\n${instantiate}`; -} - -export function setRootExports(program: ts.Program, rootModule: string): void { - // get a reference to the type checker, this will let us find symbols from - // the AST. - const checker = program.getTypeChecker(); - // get a reference to the main source file for the bundle - const mainSourceFile = program.getSourceFile(rootModule); - assert(mainSourceFile); - // retrieve the internal TypeScript symbol for this AST node - const mainSymbol = checker.getSymbolAtLocation(mainSourceFile); - if (!mainSymbol) { - return; - } - rootExports = checker - .getExportsOfModule(mainSymbol) - // .getExportsOfModule includes type only symbols which are exported from - // the module, so we need to try to filter those out. While not critical - // someone looking at the bundle would think there is runtime code behind - // that when there isn't. There appears to be no clean way of figuring that - // out, so inspecting SymbolFlags that might be present that are type only - .filter( - (sym) => - sym.flags & ts.SymbolFlags.Class || - !( - sym.flags & ts.SymbolFlags.Interface || - sym.flags & ts.SymbolFlags.TypeLiteral || - sym.flags & ts.SymbolFlags.Signature || - sym.flags & ts.SymbolFlags.TypeParameter || - sym.flags & ts.SymbolFlags.TypeAlias || - sym.flags & ts.SymbolFlags.Type || - sym.flags & ts.SymbolFlags.Namespace || - sym.flags & ts.SymbolFlags.InterfaceExcludes || - sym.flags & ts.SymbolFlags.TypeParameterExcludes || - sym.flags & ts.SymbolFlags.TypeAliasExcludes - ) - ) - .map((sym) => sym.getName()); -} diff --git a/cli/js/compiler/host.ts b/cli/js/compiler/host.ts deleted file mode 100644 index 64b5e0245f..0000000000 --- a/cli/js/compiler/host.ts +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { ASSETS, MediaType, SourceFile } from "./sourcefile.ts"; -import { OUT_DIR, WriteFileCallback, getAsset } from "./util.ts"; -import { assert, notImplemented } from "../util.ts"; -import * as util from "../util.ts"; - -export enum CompilerHostTarget { - Main = "main", - Runtime = "runtime", - Worker = "worker", -} - -export interface CompilerHostOptions { - bundle?: boolean; - target: CompilerHostTarget; - unstable?: boolean; - writeFile: WriteFileCallback; -} - -export interface ConfigureResponse { - ignoredOptions?: string[]; - diagnostics?: ts.Diagnostic[]; -} - -export const defaultBundlerOptions: ts.CompilerOptions = { - allowJs: true, - inlineSourceMap: false, - module: ts.ModuleKind.System, - outDir: undefined, - outFile: `${OUT_DIR}/bundle.js`, - // disabled until we have effective way to modify source maps - sourceMap: false, -}; - -export const defaultCompileOptions: ts.CompilerOptions = { - allowJs: false, - allowNonTsExtensions: true, - checkJs: false, - esModuleInterop: true, - jsx: ts.JsxEmit.React, - module: ts.ModuleKind.ESNext, - outDir: OUT_DIR, - resolveJsonModule: true, - sourceMap: true, - strict: true, - stripComments: true, - target: ts.ScriptTarget.ESNext, -}; - -export const defaultRuntimeCompileOptions: ts.CompilerOptions = { - outDir: undefined, -}; - -export const defaultTranspileOptions: ts.CompilerOptions = { - esModuleInterop: true, - module: ts.ModuleKind.ESNext, - sourceMap: true, - scriptComments: true, - target: ts.ScriptTarget.ESNext, -}; - -const ignoredCompilerOptions: readonly string[] = [ - "allowSyntheticDefaultImports", - "baseUrl", - "build", - "composite", - "declaration", - "declarationDir", - "declarationMap", - "diagnostics", - "downlevelIteration", - "emitBOM", - "emitDeclarationOnly", - "esModuleInterop", - "extendedDiagnostics", - "forceConsistentCasingInFileNames", - "help", - "importHelpers", - "incremental", - "inlineSourceMap", - "inlineSources", - "init", - "isolatedModules", - "listEmittedFiles", - "listFiles", - "mapRoot", - "maxNodeModuleJsDepth", - "module", - "moduleResolution", - "newLine", - "noEmit", - "noEmitHelpers", - "noEmitOnError", - "noLib", - "noResolve", - "out", - "outDir", - "outFile", - "paths", - "preserveSymlinks", - "preserveWatchOutput", - "pretty", - "rootDir", - "rootDirs", - "showConfig", - "skipDefaultLibCheck", - "skipLibCheck", - "sourceMap", - "sourceRoot", - "stripInternal", - "target", - "traceResolution", - "tsBuildInfoFile", - "types", - "typeRoots", - "version", - "watch", -]; - -function getAssetInternal(filename: string): SourceFile { - const lastSegment = filename.split("/").pop()!; - const url = ts.libMap.has(lastSegment) - ? ts.libMap.get(lastSegment)! - : lastSegment; - const sourceFile = SourceFile.get(url); - if (sourceFile) { - return sourceFile; - } - const name = url.includes(".") ? url : `${url}.d.ts`; - const sourceCode = getAsset(name); - return new SourceFile({ - url, - filename: `${ASSETS}/${name}`, - mediaType: MediaType.TypeScript, - sourceCode, - }); -} - -export class Host implements ts.CompilerHost { - readonly #options = defaultCompileOptions; - #target: CompilerHostTarget; - #writeFile: WriteFileCallback; - - /* Deno specific APIs */ - - constructor({ - bundle = false, - target, - unstable, - writeFile, - }: CompilerHostOptions) { - this.#target = target; - this.#writeFile = writeFile; - if (bundle) { - // options we need to change when we are generating a bundle - Object.assign(this.#options, defaultBundlerOptions); - } - if (unstable) { - this.#options.lib = [ - target === CompilerHostTarget.Worker - ? "lib.deno.worker.d.ts" - : "lib.deno.window.d.ts", - "lib.deno.unstable.d.ts", - ]; - } - } - - configure( - cwd: string, - path: string, - configurationText: string - ): ConfigureResponse { - util.log("compiler::host.configure", path); - assert(configurationText); - const { config, error } = ts.parseConfigFileTextToJson( - path, - configurationText - ); - if (error) { - return { diagnostics: [error] }; - } - const { options, errors } = ts.convertCompilerOptionsFromJson( - config.compilerOptions, - cwd - ); - const ignoredOptions: string[] = []; - for (const key of Object.keys(options)) { - if ( - ignoredCompilerOptions.includes(key) && - (!(key in this.#options) || options[key] !== this.#options[key]) - ) { - ignoredOptions.push(key); - delete options[key]; - } - } - Object.assign(this.#options, options); - return { - ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined, - diagnostics: errors.length ? errors : undefined, - }; - } - - mergeOptions(...options: ts.CompilerOptions[]): ts.CompilerOptions { - Object.assign(this.#options, ...options); - return Object.assign({}, this.#options); - } - - /* TypeScript CompilerHost APIs */ - - fileExists(_fileName: string): boolean { - return notImplemented(); - } - - getCanonicalFileName(fileName: string): string { - return fileName; - } - - getCompilationSettings(): ts.CompilerOptions { - util.log("compiler::host.getCompilationSettings()"); - return this.#options; - } - - getCurrentDirectory(): string { - return ""; - } - - getDefaultLibFileName(_options: ts.CompilerOptions): string { - util.log("compiler::host.getDefaultLibFileName()"); - switch (this.#target) { - case CompilerHostTarget.Main: - case CompilerHostTarget.Runtime: - return `${ASSETS}/lib.deno.window.d.ts`; - case CompilerHostTarget.Worker: - return `${ASSETS}/lib.deno.worker.d.ts`; - } - } - - getNewLine(): string { - return "\n"; - } - - getSourceFile( - fileName: string, - languageVersion: ts.ScriptTarget, - onError?: (message: string) => void, - shouldCreateNewSourceFile?: boolean - ): ts.SourceFile | undefined { - util.log("compiler::host.getSourceFile", fileName); - try { - assert(!shouldCreateNewSourceFile); - const sourceFile = fileName.startsWith(ASSETS) - ? getAssetInternal(fileName) - : SourceFile.get(fileName); - assert(sourceFile != null); - if (!sourceFile.tsSourceFile) { - assert(sourceFile.sourceCode != null); - const tsSourceFileName = fileName.startsWith(ASSETS) - ? sourceFile.filename - : fileName; - - sourceFile.tsSourceFile = ts.createSourceFile( - tsSourceFileName, - sourceFile.sourceCode, - languageVersion - ); - delete sourceFile.sourceCode; - } - return sourceFile.tsSourceFile; - } catch (e) { - if (onError) { - onError(String(e)); - } else { - throw e; - } - return undefined; - } - } - - readFile(_fileName: string): string | undefined { - return notImplemented(); - } - - resolveModuleNames( - moduleNames: string[], - containingFile: string - ): Array { - util.log("compiler::host.resolveModuleNames", { - moduleNames, - containingFile, - }); - return moduleNames.map((specifier) => { - const maybeUrl = SourceFile.getUrl(specifier, containingFile); - - let sourceFile: SourceFile | undefined = undefined; - - if (specifier.startsWith(ASSETS)) { - sourceFile = getAssetInternal(specifier); - } else if (typeof maybeUrl !== "undefined") { - sourceFile = SourceFile.get(maybeUrl); - } - - if (!sourceFile) { - return undefined; - } - - return { - resolvedFileName: sourceFile.url, - isExternalLibraryImport: specifier.startsWith(ASSETS), - extension: sourceFile.extension, - }; - }); - } - - useCaseSensitiveFileNames(): boolean { - return true; - } - - writeFile( - fileName: string, - data: string, - _writeByteOrderMark: boolean, - _onError?: (message: string) => void, - sourceFiles?: readonly ts.SourceFile[] - ): void { - util.log("compiler::host.writeFile", fileName); - this.#writeFile(fileName, data, sourceFiles); - } -} diff --git a/cli/js/compiler/imports.ts b/cli/js/compiler/imports.ts deleted file mode 100644 index a811075bef..0000000000 --- a/cli/js/compiler/imports.ts +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { MediaType, SourceFile, SourceFileJson } from "./sourcefile.ts"; -import { assert } from "../util.ts"; -import * as util from "../util.ts"; -import * as compilerOps from "../ops/compiler.ts"; - -function resolveSpecifier(specifier: string, referrer: string): string { - // The resolveModules op only handles fully qualified URLs for referrer. - // However we will have cases where referrer is "/foo.ts". We add this dummy - // prefix "file://" in order to use the op. - // TODO(ry) Maybe we should perhaps ModuleSpecifier::resolve_import() to - // handle this situation. - let dummyPrefix = false; - const prefix = "file://"; - if (referrer.startsWith("/")) { - dummyPrefix = true; - referrer = prefix + referrer; - } - let r = resolveModules([specifier], referrer)[0]; - if (dummyPrefix) { - r = r.replace(prefix, ""); - } - return r; -} - -// TODO(ry) Remove. Unnecessary redirection to compilerOps.resolveModules. -export function resolveModules( - specifiers: string[], - referrer?: string -): string[] { - util.log("compiler_imports::resolveModules", { specifiers, referrer }); - return compilerOps.resolveModules(specifiers, referrer); -} - -// TODO(ry) Remove. Unnecessary redirection to compilerOps.fetchSourceFiles. -function fetchSourceFiles( - specifiers: string[], - referrer?: string -): Promise { - util.log("compiler_imports::fetchSourceFiles", { specifiers, referrer }); - return compilerOps.fetchSourceFiles(specifiers, referrer); -} - -function getMediaType(filename: string): MediaType { - const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); - if (!maybeExtension) { - util.log(`!!! Could not identify valid extension: "${filename}"`); - return MediaType.Unknown; - } - const [, extension] = maybeExtension; - switch (extension.toLowerCase()) { - case "js": - return MediaType.JavaScript; - case "jsx": - return MediaType.JSX; - case "ts": - return MediaType.TypeScript; - case "tsx": - return MediaType.TSX; - case "wasm": - return MediaType.Wasm; - default: - util.log(`!!! Unknown extension: "${extension}"`); - return MediaType.Unknown; - } -} - -export function processLocalImports( - sources: Record, - specifiers: Array<[string, string]>, - referrer?: string, - processJsImports = false -): string[] { - if (!specifiers.length) { - return []; - } - const moduleNames = specifiers.map( - referrer - ? ([, specifier]): string => resolveSpecifier(specifier, referrer) - : ([, specifier]): string => specifier - ); - for (let i = 0; i < moduleNames.length; i++) { - const moduleName = moduleNames[i]; - assert(moduleName in sources, `Missing module in sources: "${moduleName}"`); - const sourceFile = - SourceFile.get(moduleName) || - new SourceFile({ - url: moduleName, - filename: moduleName, - sourceCode: sources[moduleName], - mediaType: getMediaType(moduleName), - }); - sourceFile.cache(specifiers[i][0], referrer); - if (!sourceFile.processed) { - processLocalImports( - sources, - sourceFile.imports(processJsImports), - sourceFile.url, - processJsImports - ); - } - } - return moduleNames; -} - -export async function processImports( - specifiers: Array<[string, string]>, - referrer?: string, - processJsImports = false -): Promise { - if (!specifiers.length) { - return []; - } - const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier); - const resolvedSources = resolveModules(sources, referrer); - const sourceFiles = await fetchSourceFiles(resolvedSources, referrer); - assert(sourceFiles.length === specifiers.length); - for (let i = 0; i < sourceFiles.length; i++) { - const sourceFileJson = sourceFiles[i]; - const sourceFile = - SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson); - sourceFile.cache(specifiers[i][0], referrer); - if (!sourceFile.processed) { - const sourceFileImports = sourceFile.imports(processJsImports); - await processImports(sourceFileImports, sourceFile.url, processJsImports); - } - } - return resolvedSources; -} diff --git a/cli/js/compiler/sourcefile.ts b/cli/js/compiler/sourcefile.ts deleted file mode 100644 index d390c3f562..0000000000 --- a/cli/js/compiler/sourcefile.ts +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { getMappedModuleName, parseTypeDirectives } from "./type_directives.ts"; -import { assert, log } from "../util.ts"; - -// Warning! The values in this enum are duplicated in `cli/msg.rs` -// Update carefully! -export enum MediaType { - JavaScript = 0, - JSX = 1, - TypeScript = 2, - TSX = 3, - Json = 4, - Wasm = 5, - Unknown = 6, -} - -export interface SourceFileJson { - url: string; - filename: string; - mediaType: MediaType; - sourceCode: string; -} - -export const ASSETS = "$asset$"; - -function getExtension(fileName: string, mediaType: MediaType): ts.Extension { - switch (mediaType) { - case MediaType.JavaScript: - return ts.Extension.Js; - case MediaType.JSX: - return ts.Extension.Jsx; - case MediaType.TypeScript: - return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; - case MediaType.TSX: - return ts.Extension.Tsx; - case MediaType.Wasm: - // Custom marker for Wasm type. - return ts.Extension.Js; - case MediaType.Unknown: - default: - throw TypeError( - `Cannot resolve extension for "${fileName}" with mediaType "${MediaType[mediaType]}".` - ); - } -} - -/** A global cache of module source files that have been loaded. */ -const moduleCache: Map = new Map(); -/** A map of maps which cache source files for quicker modules resolution. */ -const specifierCache: Map> = new Map(); - -export class SourceFile { - extension!: ts.Extension; - filename!: string; - - mediaType!: MediaType; - processed = false; - sourceCode?: string; - tsSourceFile?: ts.SourceFile; - url!: string; - - constructor(json: SourceFileJson) { - if (moduleCache.has(json.url)) { - throw new TypeError("SourceFile already exists"); - } - Object.assign(this, json); - this.extension = getExtension(this.url, this.mediaType); - moduleCache.set(this.url, this); - } - - cache(moduleSpecifier: string, containingFile?: string): void { - containingFile = containingFile || ""; - let innerCache = specifierCache.get(containingFile); - if (!innerCache) { - innerCache = new Map(); - specifierCache.set(containingFile, innerCache); - } - innerCache.set(moduleSpecifier, this); - } - - imports(processJsImports: boolean): Array<[string, string]> { - if (this.processed) { - throw new Error("SourceFile has already been processed."); - } - assert(this.sourceCode != null); - // we shouldn't process imports for files which contain the nocheck pragma - // (like bundles) - if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) { - log(`Skipping imports for "${this.filename}"`); - return []; - } - - const readImportFiles = true; - const detectJsImports = - this.mediaType === MediaType.JavaScript || - this.mediaType === MediaType.JSX; - - const preProcessedFileInfo = ts.preProcessFile( - this.sourceCode, - readImportFiles, - detectJsImports - ); - this.processed = true; - const files: Array<[string, string]> = []; - - function process(references: Array<{ fileName: string }>): void { - for (const { fileName } of references) { - files.push([fileName, fileName]); - } - } - - const { - importedFiles, - referencedFiles, - libReferenceDirectives, - typeReferenceDirectives, - } = preProcessedFileInfo; - const typeDirectives = parseTypeDirectives(this.sourceCode); - if (typeDirectives) { - for (const importedFile of importedFiles) { - files.push([ - importedFile.fileName, - getMappedModuleName(importedFile, typeDirectives), - ]); - } - } else if ( - !( - !processJsImports && - (this.mediaType === MediaType.JavaScript || - this.mediaType === MediaType.JSX) - ) - ) { - process(importedFiles); - } - process(referencedFiles); - // built in libs comes across as `"dom"` for example, and should be filtered - // out during pre-processing as they are either already cached or they will - // be lazily fetched by the compiler host. Ones that contain full files are - // not filtered out and will be fetched as normal. - process( - libReferenceDirectives.filter( - ({ fileName }) => !ts.libMap.has(fileName.toLowerCase()) - ) - ); - process(typeReferenceDirectives); - return files; - } - - static getUrl( - moduleSpecifier: string, - containingFile: string - ): string | undefined { - const containingCache = specifierCache.get(containingFile); - if (containingCache) { - const sourceFile = containingCache.get(moduleSpecifier); - return sourceFile && sourceFile.url; - } - return undefined; - } - - static get(url: string): SourceFile | undefined { - return moduleCache.get(url); - } -} diff --git a/cli/js/compiler/type_directives.ts b/cli/js/compiler/type_directives.ts deleted file mode 100644 index 5f33f4b332..0000000000 --- a/cli/js/compiler/type_directives.ts +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -interface FileReference { - fileName: string; - pos: number; - end: number; -} - -export function getMappedModuleName( - source: FileReference, - typeDirectives: Map -): string { - const { fileName: sourceFileName, pos: sourcePos } = source; - for (const [{ fileName, pos }, value] of typeDirectives.entries()) { - if (sourceFileName === fileName && sourcePos === pos) { - return value; - } - } - return source.fileName; -} - -const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi; - -const importExportRegEx = /(?:import|export)(?:\s+|\s+[\s\S]*?from\s+)?(["'])((?:(?=(\\?))\3.)*?)\1/; - -export function parseTypeDirectives( - sourceCode: string | undefined -): Map | undefined { - if (!sourceCode) { - return; - } - - // collect all the directives in the file and their start and end positions - const directives: FileReference[] = []; - let maybeMatch: RegExpExecArray | null = null; - while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) { - const [matchString, , fileName] = maybeMatch; - const { index: pos } = maybeMatch; - directives.push({ - fileName, - pos, - end: pos + matchString.length, - }); - } - if (!directives.length) { - return; - } - - // work from the last directive backwards for the next `import`/`export` - // statement - directives.reverse(); - const results = new Map(); - for (const { end, fileName, pos } of directives) { - const searchString = sourceCode.substring(end); - const maybeMatch = importExportRegEx.exec(searchString); - if (maybeMatch) { - const [matchString, , targetFileName] = maybeMatch; - const targetPos = - end + maybeMatch.index + matchString.indexOf(targetFileName) - 1; - const target: FileReference = { - fileName: targetFileName, - pos: targetPos, - end: targetPos + targetFileName.length, - }; - results.set(target, fileName); - } - sourceCode = sourceCode.substring(0, pos); - } - - return results; -} diff --git a/cli/js/compiler/util.ts b/cli/js/compiler/util.ts deleted file mode 100644 index f3cbe5566e..0000000000 --- a/cli/js/compiler/util.ts +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { bold, cyan, yellow } from "../colors.ts"; -import { CompilerOptions } from "./api.ts"; -import { buildBundle } from "./bundler.ts"; -import { ConfigureResponse, Host } from "./host.ts"; -import { atob } from "../web/text_encoding.ts"; -import * as compilerOps from "../ops/compiler.ts"; -import { assert } from "../util.ts"; - -export interface EmmitedSource { - // original filename - filename: string; - // compiled contents - contents: string; -} - -export type WriteFileCallback = ( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] -) => void; - -export interface WriteFileState { - type: CompilerRequestType; - bundle?: boolean; - bundleOutput?: string; - host?: Host; - rootNames: string[]; - emitMap?: Record; - sources?: Record; -} - -// Warning! The values in this enum are duplicated in `cli/msg.rs` -// Update carefully! -export enum CompilerRequestType { - Compile = 0, - RuntimeCompile = 1, - RuntimeTranspile = 2, -} - -export const OUT_DIR = "$deno$"; - -export function getAsset(name: string): string { - return compilerOps.getAsset(name); -} - -// TODO(bartlomieju): probably could be defined inline? -export function createBundleWriteFile( - state: WriteFileState -): WriteFileCallback { - return function writeFile( - _fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] - ): void { - assert(sourceFiles != null); - assert(state.host); - assert(state.emitMap); - assert(state.bundle); - // we only support single root names for bundles - assert(state.rootNames.length === 1); - state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles); - }; -} - -// TODO(bartlomieju): probably could be defined inline? -export function createCompileWriteFile( - state: WriteFileState -): WriteFileCallback { - return function writeFile( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] - ): void { - assert(sourceFiles != null); - assert(state.host); - assert(state.emitMap); - assert(!state.bundle); - assert(sourceFiles.length === 1); - state.emitMap[fileName] = { - filename: sourceFiles[0].fileName, - contents: data, - }; - }; -} - -export interface ConvertCompilerOptionsResult { - files?: string[]; - options: ts.CompilerOptions; -} - -export function convertCompilerOptions( - str: string -): ConvertCompilerOptionsResult { - const options: CompilerOptions = JSON.parse(str); - const out: Record = {}; - const keys = Object.keys(options) as Array; - const files: string[] = []; - for (const key of keys) { - switch (key) { - case "jsx": - const value = options[key]; - if (value === "preserve") { - out[key] = ts.JsxEmit.Preserve; - } else if (value === "react") { - out[key] = ts.JsxEmit.React; - } else { - out[key] = ts.JsxEmit.ReactNative; - } - break; - case "module": - switch (options[key]) { - case "amd": - out[key] = ts.ModuleKind.AMD; - break; - case "commonjs": - out[key] = ts.ModuleKind.CommonJS; - break; - case "es2015": - case "es6": - out[key] = ts.ModuleKind.ES2015; - break; - case "esnext": - out[key] = ts.ModuleKind.ESNext; - break; - case "none": - out[key] = ts.ModuleKind.None; - break; - case "system": - out[key] = ts.ModuleKind.System; - break; - case "umd": - out[key] = ts.ModuleKind.UMD; - break; - default: - throw new TypeError("Unexpected module type"); - } - break; - case "target": - switch (options[key]) { - case "es3": - out[key] = ts.ScriptTarget.ES3; - break; - case "es5": - out[key] = ts.ScriptTarget.ES5; - break; - case "es6": - case "es2015": - out[key] = ts.ScriptTarget.ES2015; - break; - case "es2016": - out[key] = ts.ScriptTarget.ES2016; - break; - case "es2017": - out[key] = ts.ScriptTarget.ES2017; - break; - case "es2018": - out[key] = ts.ScriptTarget.ES2018; - break; - case "es2019": - out[key] = ts.ScriptTarget.ES2019; - break; - case "es2020": - out[key] = ts.ScriptTarget.ES2020; - break; - case "esnext": - out[key] = ts.ScriptTarget.ESNext; - break; - default: - throw new TypeError("Unexpected emit target."); - } - break; - case "types": - const types = options[key]; - assert(types); - files.push(...types); - break; - default: - out[key] = options[key]; - } - } - return { - options: out as ts.CompilerOptions, - files: files.length ? files : undefined, - }; -} - -export const ignoredDiagnostics = [ - // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is - // not a module. - 2306, - // TS1375: 'await' expressions are only allowed at the top level of a file - // when that file is a module, but this file has no imports or exports. - // Consider adding an empty 'export {}' to make this file a module. - 1375, - // TS1103: 'for-await-of' statement is only allowed within an async function - // or async generator. - 1103, - // TS2691: An import path cannot end with a '.ts' extension. Consider - // importing 'bad-module' instead. - 2691, - // TS5009: Cannot find the common subdirectory path for the input files. - 5009, - // TS5055: Cannot write file - // 'http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js' - // because it would overwrite input file. - 5055, - // TypeScript is overly opinionated that only CommonJS modules kinds can - // support JSON imports. Allegedly this was fixed in - // Microsoft/TypeScript#26825 but that doesn't seem to be working here, - // so we will ignore complaints about this compiler setting. - 5070, - // TS7016: Could not find a declaration file for module '...'. '...' - // implicitly has an 'any' type. This is due to `allowJs` being off by - // default but importing of a JavaScript module. - 7016, -]; - -export function processConfigureResponse( - configResult: ConfigureResponse, - configPath: string -): ts.Diagnostic[] | undefined { - const { ignoredOptions, diagnostics } = configResult; - if (ignoredOptions) { - console.warn( - yellow(`Unsupported compiler options in "${configPath}"\n`) + - cyan(` The following options were ignored:\n`) + - ` ${ignoredOptions.map((value): string => bold(value)).join(", ")}` - ); - } - return diagnostics; -} - -// Constants used by `normalizeString` and `resolvePath` -export const CHAR_DOT = 46; /* . */ -export const CHAR_FORWARD_SLASH = 47; /* / */ - -export function normalizeString(path: string): string { - let res = ""; - let lastSegmentLength = 0; - let lastSlash = -1; - let dots = 0; - let code: number; - for (let i = 0, len = path.length; i <= len; ++i) { - if (i < len) code = path.charCodeAt(i); - else if (code! === CHAR_FORWARD_SLASH) break; - else code = CHAR_FORWARD_SLASH; - - if (code === CHAR_FORWARD_SLASH) { - if (lastSlash === i - 1 || dots === 1) { - // NOOP - } else if (lastSlash !== i - 1 && dots === 2) { - if ( - res.length < 2 || - lastSegmentLength !== 2 || - res.charCodeAt(res.length - 1) !== CHAR_DOT || - res.charCodeAt(res.length - 2) !== CHAR_DOT - ) { - if (res.length > 2) { - const lastSlashIndex = res.lastIndexOf("/"); - if (lastSlashIndex === -1) { - res = ""; - lastSegmentLength = 0; - } else { - res = res.slice(0, lastSlashIndex); - lastSegmentLength = res.length - 1 - res.lastIndexOf("/"); - } - lastSlash = i; - dots = 0; - continue; - } else if (res.length === 2 || res.length === 1) { - res = ""; - lastSegmentLength = 0; - lastSlash = i; - dots = 0; - continue; - } - } - } else { - if (res.length > 0) res += "/" + path.slice(lastSlash + 1, i); - else res = path.slice(lastSlash + 1, i); - lastSegmentLength = i - lastSlash - 1; - } - lastSlash = i; - dots = 0; - } else if (code === CHAR_DOT && dots !== -1) { - ++dots; - } else { - dots = -1; - } - } - return res; -} - -export function commonPath(paths: string[], sep = "/"): string { - const [first = "", ...remaining] = paths; - if (first === "" || remaining.length === 0) { - return first.substring(0, first.lastIndexOf(sep) + 1); - } - const parts = first.split(sep); - - let endOfPrefix = parts.length; - for (const path of remaining) { - const compare = path.split(sep); - for (let i = 0; i < endOfPrefix; i++) { - if (compare[i] !== parts[i]) { - endOfPrefix = i; - } - } - - if (endOfPrefix === 0) { - return ""; - } - } - const prefix = parts.slice(0, endOfPrefix).join(sep); - return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`; -} - -// @internal -export function base64ToUint8Array(data: string): Uint8Array { - const binString = atob(data); - const size = binString.length; - const bytes = new Uint8Array(size); - for (let i = 0; i < size; i++) { - bytes[i] = binString.charCodeAt(i); - } - return bytes; -} diff --git a/cli/js/compiler/api.ts b/cli/js/compiler_api.ts similarity index 51% rename from cli/js/compiler/api.ts rename to cli/js/compiler_api.ts index a7d1e57a8f..88d99e5b41 100644 --- a/cli/js/compiler/api.ts +++ b/cli/js/compiler_api.ts @@ -3,143 +3,11 @@ // This file contains the runtime APIs which will dispatch work to the internal // compiler within Deno. -import { DiagnosticItem } from "../diagnostics.ts"; -import * as util from "../util.ts"; -import * as runtimeCompilerOps from "../ops/runtime_compiler.ts"; -import { TranspileOnlyResult } from "../ops/runtime_compiler.ts"; -export { TranspileOnlyResult } from "../ops/runtime_compiler.ts"; - -export interface CompilerOptions { - allowJs?: boolean; - - allowSyntheticDefaultImports?: boolean; - - allowUmdGlobalAccess?: boolean; - - allowUnreachableCode?: boolean; - - allowUnusedLabels?: boolean; - - alwaysStrict?: boolean; - - baseUrl?: string; - - checkJs?: boolean; - - declaration?: boolean; - - declarationDir?: string; - - declarationMap?: boolean; - - downlevelIteration?: boolean; - - emitBOM?: boolean; - - emitDeclarationOnly?: boolean; - - emitDecoratorMetadata?: boolean; - - esModuleInterop?: boolean; - - experimentalDecorators?: boolean; - - inlineSourceMap?: boolean; - - inlineSources?: boolean; - - isolatedModules?: boolean; - - jsx?: "react" | "preserve" | "react-native"; - - jsxFactory?: string; - - keyofStringsOnly?: string; - - useDefineForClassFields?: boolean; - - lib?: string[]; - - locale?: string; - - mapRoot?: string; - - module?: - | "none" - | "commonjs" - | "amd" - | "system" - | "umd" - | "es6" - | "es2015" - | "esnext"; - - noEmitHelpers?: boolean; - - noFallthroughCasesInSwitch?: boolean; - - noImplicitAny?: boolean; - - noImplicitReturns?: boolean; - - noImplicitThis?: boolean; - - noImplicitUseStrict?: boolean; - - noResolve?: boolean; - - noStrictGenericChecks?: boolean; - - noUnusedLocals?: boolean; - - noUnusedParameters?: boolean; - - outDir?: string; - - paths?: Record; - - preserveConstEnums?: boolean; - - removeComments?: boolean; - - resolveJsonModule?: boolean; - - rootDir?: string; - - rootDirs?: string[]; - - sourceMap?: boolean; - - sourceRoot?: string; - - strict?: boolean; - - strictBindCallApply?: boolean; - - strictFunctionTypes?: boolean; - - strictPropertyInitialization?: boolean; - - strictNullChecks?: boolean; - - suppressExcessPropertyErrors?: boolean; - - suppressImplicitAnyIndexErrors?: boolean; - - target?: - | "es3" - | "es5" - | "es6" - | "es2015" - | "es2016" - | "es2017" - | "es2018" - | "es2019" - | "es2020" - | "esnext"; - - types?: string[]; -} +import { DiagnosticItem } from "./diagnostics.ts"; +import * as util from "./util.ts"; +import * as runtimeCompilerOps from "./ops/runtime_compiler.ts"; +import { TranspileOnlyResult } from "./ops/runtime_compiler.ts"; +import { CompilerOptions } from "./compiler_options.ts"; function checkRelative(specifier: string): string { return specifier.match(/^([\.\/\\]|https?:\/{2}|file:\/{2})/) diff --git a/cli/js/compiler_options.ts b/cli/js/compiler_options.ts new file mode 100644 index 0000000000..dd1a0a9f2e --- /dev/null +++ b/cli/js/compiler_options.ts @@ -0,0 +1,133 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +export interface CompilerOptions { + allowJs?: boolean; + + allowSyntheticDefaultImports?: boolean; + + allowUmdGlobalAccess?: boolean; + + allowUnreachableCode?: boolean; + + allowUnusedLabels?: boolean; + + alwaysStrict?: boolean; + + baseUrl?: string; + + checkJs?: boolean; + + declaration?: boolean; + + declarationDir?: string; + + declarationMap?: boolean; + + downlevelIteration?: boolean; + + emitBOM?: boolean; + + emitDeclarationOnly?: boolean; + + emitDecoratorMetadata?: boolean; + + esModuleInterop?: boolean; + + experimentalDecorators?: boolean; + + inlineSourceMap?: boolean; + + inlineSources?: boolean; + + isolatedModules?: boolean; + + jsx?: "react" | "preserve" | "react-native"; + + jsxFactory?: string; + + keyofStringsOnly?: string; + + useDefineForClassFields?: boolean; + + lib?: string[]; + + locale?: string; + + mapRoot?: string; + + module?: + | "none" + | "commonjs" + | "amd" + | "system" + | "umd" + | "es6" + | "es2015" + | "esnext"; + + noEmitHelpers?: boolean; + + noFallthroughCasesInSwitch?: boolean; + + noImplicitAny?: boolean; + + noImplicitReturns?: boolean; + + noImplicitThis?: boolean; + + noImplicitUseStrict?: boolean; + + noResolve?: boolean; + + noStrictGenericChecks?: boolean; + + noUnusedLocals?: boolean; + + noUnusedParameters?: boolean; + + outDir?: string; + + paths?: Record; + + preserveConstEnums?: boolean; + + removeComments?: boolean; + + resolveJsonModule?: boolean; + + rootDir?: string; + + rootDirs?: string[]; + + sourceMap?: boolean; + + sourceRoot?: string; + + strict?: boolean; + + strictBindCallApply?: boolean; + + strictFunctionTypes?: boolean; + + strictPropertyInitialization?: boolean; + + strictNullChecks?: boolean; + + suppressExcessPropertyErrors?: boolean; + + suppressImplicitAnyIndexErrors?: boolean; + + target?: + | "es3" + | "es5" + | "es6" + | "es2015" + | "es2016" + | "es2017" + | "es2018" + | "es2019" + | "es2020" + | "esnext"; + + types?: string[]; +} diff --git a/cli/js/deno_unstable.ts b/cli/js/deno_unstable.ts index 98e60f665d..57d1a0970f 100644 --- a/cli/js/deno_unstable.ts +++ b/cli/js/deno_unstable.ts @@ -7,7 +7,7 @@ export { linkSync, link } from "./ops/fs/link.ts"; export { symlinkSync, symlink } from "./ops/fs/symlink.ts"; export { dir, loadavg, osRelease } from "./ops/os.ts"; export { openPlugin } from "./ops/plugins.ts"; -export { transpileOnly, compile, bundle } from "./compiler/api.ts"; +export { transpileOnly, compile, bundle } from "./compiler_api.ts"; export { applySourceMap, formatDiagnostics } from "./ops/errors.ts"; export { signal, signals, Signal, SignalStream } from "./signals.ts"; export { setRaw } from "./ops/tty.ts"; diff --git a/cli/js/ops/compiler.ts b/cli/js/ops/compiler.ts deleted file mode 100644 index 60f8147417..0000000000 --- a/cli/js/ops/compiler.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -import { sendAsync, sendSync } from "./dispatch_json.ts"; -import { TextDecoder, TextEncoder } from "../web/text_encoding.ts"; -import { core } from "../core.ts"; - -export function resolveModules( - specifiers: string[], - referrer?: string -): string[] { - return sendSync("op_resolve_modules", { specifiers, referrer }); -} - -export function fetchSourceFiles( - specifiers: string[], - referrer?: string -): Promise< - Array<{ - url: string; - filename: string; - mediaType: number; - sourceCode: string; - }> -> { - return sendAsync("op_fetch_source_files", { - specifiers, - referrer, - }); -} - -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -export function getAsset(name: string): string { - const opId = core.ops()["op_fetch_asset"]; - // We really don't want to depend on JSON dispatch during snapshotting, so - // this op exchanges strings with Rust as raw byte arrays. - const sourceCodeBytes = core.dispatch(opId, encoder.encode(name)); - return decoder.decode(sourceCodeBytes!); -} diff --git a/cli/js/compiler/ts_global.d.ts b/cli/js/ts_global.d.ts similarity index 100% rename from cli/js/compiler/ts_global.d.ts rename to cli/js/ts_global.d.ts diff --git a/cli/tests/020_json_modules.ts.out b/cli/tests/020_json_modules.ts.out index cb923561b4..cda28038e4 100644 --- a/cli/tests/020_json_modules.ts.out +++ b/cli/tests/020_json_modules.ts.out @@ -1,9 +1,9 @@ [WILDCARD] error: Uncaught TypeError: Cannot resolve extension for "[WILDCARD]config.json" with mediaType "Json". - at getExtension ($deno$/compiler/sourcefile.ts:[WILDCARD]) - at new SourceFile ($deno$/compiler/sourcefile.ts:[WILDCARD]) - at processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at getExtension ($deno$/compiler.ts:[WILDCARD]) + at new SourceFile ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_004_missing_module.ts.out b/cli/tests/error_004_missing_module.ts.out index 1c77c06266..b93f2e6132 100644 --- a/cli/tests/error_004_missing_module.ts.out +++ b/cli/tests/error_004_missing_module.ts.out @@ -1,8 +1,8 @@ [WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_004_missing_module.ts" at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out index b05038a862..e8c8f8cd09 100644 --- a/cli/tests/error_005_missing_dynamic_import.ts.out +++ b/cli/tests/error_005_missing_dynamic_import.ts.out @@ -1,8 +1,8 @@ [WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_006_import_ext_failure.ts.out b/cli/tests/error_006_import_ext_failure.ts.out index 25e772a05c..5c89adeff8 100644 --- a/cli/tests/error_006_import_ext_failure.ts.out +++ b/cli/tests/error_006_import_ext_failure.ts.out @@ -1,8 +1,8 @@ [WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts" at unwrapResponse ([WILDCARD]dispatch_json.ts:[WILDCARD]) at Object.sendAsync ([WILDCARD]dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_011_bad_module_specifier.ts.out b/cli/tests/error_011_bad_module_specifier.ts.out index 984a1e4afb..39726d5c63 100644 --- a/cli/tests/error_011_bad_module_specifier.ts.out +++ b/cli/tests/error_011_bad_module_specifier.ts.out @@ -1,9 +1,9 @@ [WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_011_bad_module_specifier.ts" at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler/imports.ts:[WILDCARD]) - at processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at resolveModules ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out index 6058e1c5af..900b8f52da 100644 --- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -1,9 +1,9 @@ [WILDCARD]error: Uncaught URIError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler/imports.ts:[WILDCARD]) - at processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at resolveModules ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_local_static_import_from_remote.ts.out b/cli/tests/error_local_static_import_from_remote.ts.out index 841fdb6e31..af3c8852fe 100644 --- a/cli/tests/error_local_static_import_from_remote.ts.out +++ b/cli/tests/error_local_static_import_from_remote.ts.out @@ -2,8 +2,8 @@ error: Uncaught PermissionDenied: Remote module are not allowed to statically import local modules. Use dynamic import instead. at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) at Object.sendAsync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at async processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ($deno$/runtime_worker.ts:[WILDCARD]) diff --git a/cli/tests/error_type_definitions.ts.out b/cli/tests/error_type_definitions.ts.out index e2791a6c0d..bacca4f80e 100644 --- a/cli/tests/error_type_definitions.ts.out +++ b/cli/tests/error_type_definitions.ts.out @@ -1,11 +1,10 @@ [WILDCARD]error: Uncaught URIError: relative import path "baz" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/type_definitions/bar.d.ts" at unwrapResponse ($deno$/ops/dispatch_json.ts:[WILDCARD]) at Object.sendSync ($deno$/ops/dispatch_json.ts:[WILDCARD]) - at Object.resolveModules ($deno$/ops/compiler.ts:[WILDCARD]) - at resolveModules ($deno$/compiler/imports.ts:[WILDCARD]) - at processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at processImports ($deno$/compiler/imports.ts:[WILDCARD]) - at async Object.processImports ($deno$/compiler/imports.ts:[WILDCARD]) + at resolveModules ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) + at processImports ($deno$/compiler.ts:[WILDCARD]) + at async processImports ($deno$/compiler.ts:[WILDCARD]) at async compile ($deno$/compiler.ts:[WILDCARD]) at async tsCompilerOnMessage ($deno$/compiler.ts:[WILDCARD]) at async workerMessageRecvCallback ([WILDCARD]runtime_worker.ts:[WILDCARD])