diff --git a/BUILD.gn b/BUILD.gn index 16941c003f..1696c2483b 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -107,7 +107,6 @@ ts_sources = [ "js/v8_source_maps.ts", "js/write_file.ts", - "js/tsconfig.declarations.json", "tsconfig.json", # Listing package.json and yarn.lock as sources ensures the bundle is rebuilt @@ -246,26 +245,33 @@ executable("snapshot_creator") { configs += [ ":deno_config" ] } -# Generates type declarations for files that need to be included -# in the runtime bundle -run_node("gen_declarations") { +# Generates the core TypeScript type library for deno that will be +# included in the runtime bundle +run_node("deno_runtime_declaration") { out_dir = target_gen_dir sources = ts_sources outputs = [ - "$out_dir/types/globals.d.ts", + "$out_dir/lib/lib.deno_runtime.d.ts", ] deps = [ ":msg_ts", ] args = [ - "./node_modules/typescript/bin/tsc", - "-p", - rebase_path("js/tsconfig.declarations.json", root_build_dir), - "--baseUrl", + rebase_path("node_modules/.bin/ts-node", root_build_dir), + "--project", + rebase_path("tools/ts_library_builder/tsconfig.json"), + rebase_path("tools/ts_library_builder/main.ts", root_build_dir), + "--basePath", + rebase_path(".", root_build_dir), + "--buildPath", rebase_path(root_build_dir, root_build_dir), "--outFile", - rebase_path("$out_dir/types/globals.js", root_build_dir), + rebase_path("$out_dir/lib/lib.deno_runtime.d.ts", root_build_dir), + "--silent", ] + if (is_debug) { + args += [ "--debug" ] + } } run_node("bundle") { @@ -276,7 +282,7 @@ run_node("bundle") { out_dir + "main.js.map", ] deps = [ - ":gen_declarations", + ":deno_runtime_declaration", ":msg_ts", ] args = [ diff --git a/js/assets.ts b/js/assets.ts index c860555c4f..481c48239d 100644 --- a/js/assets.ts +++ b/js/assets.ts @@ -7,7 +7,7 @@ // tslint:disable:max-line-length // Generated default library -import globalsDts from "gen/types/globals.d.ts!string"; +import libDts from "gen/lib/lib.deno_runtime.d.ts!string"; // Static libraries import libEs2015Dts from "/third_party/node_modules/typescript/lib/lib.es2015.d.ts!string"; @@ -40,7 +40,6 @@ import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnex import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string"; // Static definitions -import flatbuffersDts from "/third_party/node_modules/@types/flatbuffers/index.d.ts!string"; import textEncodingDts from "/third_party/node_modules/@types/text-encoding/index.d.ts!string"; import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d.ts!string"; // tslint:enable:max-line-length @@ -48,7 +47,7 @@ import typescriptDts from "/third_party/node_modules/typescript/lib/typescript.d // @internal export const assetSourceCode: { [key: string]: string } = { // Generated library - "globals.d.ts": globalsDts, + "lib.deno_runtime.d.ts": libDts, // Static libraries "lib.es2015.collection.d.ts": libEs2015CollectionDts, @@ -81,7 +80,6 @@ export const assetSourceCode: { [key: string]: string } = { "lib.esnext.symbol.d.ts": libEsnextSymbolDts, // Static definitions - "flatbuffers.d.ts": flatbuffersDts, "text-encoding.d.ts": textEncodingDts, "typescript.d.ts": typescriptDts }; diff --git a/js/compiler.ts b/js/compiler.ts index 7b577a8b94..c398f181d3 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -13,6 +13,7 @@ import * as sourceMaps from "./v8_source_maps"; const EOL = "\n"; const ASSETS = "$asset$"; +const LIB_RUNTIME = "lib.deno_runtime.d.ts"; // tslint:disable:no-any type AmdCallback = (...args: any[]) => void; @@ -619,7 +620,7 @@ export class DenoCompiler getDefaultLibFileName(): string { this._log("getDefaultLibFileName()"); - const moduleSpecifier = "globals.d.ts"; + const moduleSpecifier = LIB_RUNTIME; const moduleMetaData = this.resolveModule(moduleSpecifier, ASSETS); return moduleMetaData.fileName; } @@ -649,8 +650,8 @@ export class DenoCompiler return moduleNames.map(name => { let resolvedFileName; if (name === "deno") { - // builtin modules are part of `globals.d.ts` - resolvedFileName = this._resolveModuleName("globals.d.ts", ASSETS); + // builtin modules are part of the runtime lib + resolvedFileName = this._resolveModuleName(LIB_RUNTIME, ASSETS); } else if (name === "typescript") { resolvedFileName = this._resolveModuleName("typescript.d.ts", ASSETS); } else { diff --git a/js/compiler_test.ts b/js/compiler_test.ts index dcb8c12852..f05a96e52b 100644 --- a/js/compiler_test.ts +++ b/js/compiler_test.ts @@ -546,7 +546,10 @@ test(function compilerGetCurrentDirectory() { test(function compilerGetDefaultLibFileName() { setup(); - assertEqual(compilerInstance.getDefaultLibFileName(), "$asset$/globals.d.ts"); + assertEqual( + compilerInstance.getDefaultLibFileName(), + "$asset$/lib.deno_runtime.d.ts" + ); teardown(); }); @@ -572,7 +575,7 @@ test(function compilerFileExists() { "/root/project" ); assert(compilerInstance.fileExists(moduleMetaData.fileName)); - assert(compilerInstance.fileExists("$asset$/globals.d.ts")); + assert(compilerInstance.fileExists("$asset$/lib.deno_runtime.d.ts")); assertEqual( compilerInstance.fileExists("/root/project/unknown-module.ts"), false @@ -590,7 +593,7 @@ test(function compilerResolveModuleNames() { const fixtures: Array<[string, boolean]> = [ ["/root/project/foo/bar.ts", false], ["/root/project/foo/baz.ts", false], - ["$asset$/globals.d.ts", true] + ["$asset$/lib.deno_runtime.d.ts", true] ]; for (let i = 0; i < results.length; i++) { const result = results[i]; diff --git a/js/globals.ts b/js/globals.ts index 2e6b24305e..11a1726333 100644 --- a/js/globals.ts +++ b/js/globals.ts @@ -1,55 +1,22 @@ // Copyright 2018 the Deno authors. All rights reserved. MIT license. -import { Console } from "./console"; -import * as timers from "./timers"; -import * as textEncoding from "./text_encoding"; +import * as blob from "./blob"; +import * as console from "./console"; import * as fetch_ from "./fetch"; -import { libdeno } from "./libdeno"; import { globalEval } from "./global_eval"; -import { DenoHeaders } from "./fetch"; -import { DenoBlob } from "./blob"; +import { libdeno } from "./libdeno"; +import * as textEncoding from "./text_encoding"; +import * as timers from "./timers"; + +// During the build process, augmentations to the variable `window` in this +// file are tracked and created as part of default library that is built into +// deno, we only need to declare the enough to compile deno. declare global { - interface Window { - define: Readonly; - - clearTimeout: typeof clearTimeout; - clearInterval: typeof clearInterval; - setTimeout: typeof setTimeout; - setInterval: typeof setInterval; - - console: typeof console; - window: typeof window; - - fetch: typeof fetch; - - TextEncoder: typeof TextEncoder; - TextDecoder: typeof TextDecoder; - atob: typeof atob; - btoa: typeof btoa; - - Headers: typeof Headers; - Blob: typeof Blob; - } - - const clearTimeout: typeof timers.clearTimer; - const clearInterval: typeof timers.clearTimer; + const console: console.Console; const setTimeout: typeof timers.setTimeout; - const setInterval: typeof timers.setInterval; - - const console: Console; - const window: Window; - - const fetch: typeof fetch_.fetch; - - // tslint:disable:variable-name + // tslint:disable-next-line:variable-name const TextEncoder: typeof textEncoding.TextEncoder; - const TextDecoder: typeof textEncoding.TextDecoder; - const atob: typeof textEncoding.atob; - const btoa: typeof textEncoding.btoa; - const Headers: typeof DenoHeaders; - const Blob: typeof DenoBlob; - // tslint:enable:variable-name } // A reference to the global object. @@ -61,7 +28,7 @@ window.setInterval = timers.setInterval; window.clearTimeout = timers.clearTimer; window.clearInterval = timers.clearTimer; -window.console = new Console(libdeno.print); +window.console = new console.Console(libdeno.print); window.TextEncoder = textEncoding.TextEncoder; window.TextDecoder = textEncoding.TextDecoder; window.atob = textEncoding.atob; @@ -69,5 +36,5 @@ window.btoa = textEncoding.btoa; window.fetch = fetch_.fetch; -window.Headers = DenoHeaders; -window.Blob = DenoBlob; +window.Headers = fetch_.DenoHeaders; +window.Blob = blob.DenoBlob; diff --git a/js/testing/testing.ts b/js/testing/testing.ts index ff91ddebc5..437cb0f38f 100644 --- a/js/testing/testing.ts +++ b/js/testing/testing.ts @@ -13,7 +13,7 @@ limitations under the License. */ -export { assert, assertEqual, equal } from "./util.ts"; +export { assert, assertEqual, equal } from "./util"; export type TestFunction = () => void | Promise; diff --git a/js/tsconfig.declarations.json b/js/tsconfig.declarations.json deleted file mode 100644 index 6371182b65..0000000000 --- a/js/tsconfig.declarations.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // This configuration file provides the tsc configuration for generating - // definitions for the runtime, which are then inlined via the `js/assets.ts` - // module into the bundle to be available for type checking at runtime - // See also gen_declarations in //BUILD.gn - "extends": "../tsconfig.json", - "compilerOptions": { - "declaration": true, - "emitDeclarationOnly": true, - "module": "amd", - "removeComments": false, - "stripInternal": true - }, - "files": [ - "../node_modules/typescript/lib/lib.esnext.d.ts", - "./deno.ts", - "./globals.ts" - ] -} diff --git a/package.json b/package.json index fd2b56c4e9..c25c94ee38 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "devDependencies": { "@types/base64-js": "^1.2.5", "@types/flatbuffers": "^1.9.0", + "@types/prettier": "^1.13.2", "@types/source-map-support": "^0.4.1", "@types/text-encoding": "0.0.33", "base64-js": "^1.3.0", @@ -20,6 +21,8 @@ "rollup-pluginutils": "^2.3.0", "source-map-support": "^0.5.6", "text-encoding": "0.6.4", + "ts-node": "^7.0.1", + "ts-simple-ast": "^16.0.4", "tslint": "^5.10.0", "tslint-eslint-rules": "^5.3.1", "tslint-no-circular-imports": "^0.5.0", diff --git a/rollup.config.js b/rollup.config.js index 4cb3197f5b..55f8325855 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -26,12 +26,6 @@ const tsconfigOverride = { } }; -// this is a preamble for the `globals.d.ts` file to allow it to be the default -// lib for deno. -const libPreamble = `/// -/// -`; - // this is a rollup plugin which will look for imports ending with `!string` and resolve // them with a module that will inline the contents of the file as a string. Needed to // support `js/assets.ts`. @@ -70,9 +64,7 @@ function strings({ include, exclude } = {}) { transform(code, id) { if (filter(id)) { return { - code: `export default ${JSON.stringify( - id.endsWith("globals.d.ts") ? libPreamble + code : code - )};`, + code: `export default ${JSON.stringify(code)};`, map: { mappings: "" } }; } diff --git a/tests/error_003_typescript.ts.out b/tests/error_003_typescript.ts.out index 1423ae1d42..438b65609d 100644 --- a/tests/error_003_typescript.ts.out +++ b/tests/error_003_typescript.ts.out @@ -3,8 +3,8 @@ [WILDCARD][0m consol.log("hello world!");   ~~~~~~ - $asset$/globals.d.tsILDCARD] -[WILDCARD]const console: Console; + $asset$/lib.deno_runtime.d.tsILDCARD] +[WILDCARD]const console: console.Console; [WILDCARD]~~~~~~~ [WILDCARD]'console' is declared here. diff --git a/third_party b/third_party index 8401dc99b9..a133fa714b 160000 --- a/third_party +++ b/third_party @@ -1 +1 @@ -Subproject commit 8401dc99b953c61c2a79c2f85294d8c7f3c55c9d +Subproject commit a133fa714b960d8f88c55188ccc1a41882961e6e diff --git a/tools/format.py b/tools/format.py index 272f2c82b3..5b6afd1c36 100755 --- a/tools/format.py +++ b/tools/format.py @@ -34,6 +34,7 @@ run(["node", prettier, "--write"] + find_exts(".github/", ".md") + find_exts("js/", ".js", ".ts", ".md") + find_exts("tests/", ".js", ".ts", ".md") + + find_exts("tools/", ".js", ".json", ".ts", ".md") + find_exts("website/", ".js", ".ts", ".md")) # yapf: enable diff --git a/tools/ts_library_builder/README.md b/tools/ts_library_builder/README.md new file mode 100644 index 0000000000..de1305dca9 --- /dev/null +++ b/tools/ts_library_builder/README.md @@ -0,0 +1,95 @@ +# ts_library_builder + +This tool allows us to produce a single TypeScript declaration file that +describes the complete Deno runtime, including global variables and the built-in +`deno` module. The output of this tool, `lib.deno_runtime.d.ts`, serves several +purposes: + +1. It is passed to the TypeScript compiler `js/compiler.ts`, so that TypeScript + knows what types to expect and can validate code against the runtime + environment. +2. It is outputted to stdout by `deno --types`, so that users can easily have + access to the complete declaration file. Editors can use this in the future + to perform type checking. +3. Because JSDocs are maintained, this serves as a simple documentation page for + Deno. We will use this file to generate HTML docs in the future. + +The tool depends upon a couple libraries: + +- [`ts-node`](https://www.npmjs.com/package/ts-node) to provide just in time + transpiling of TypeScript for the tool itself. +- [`ts-simple-ast`](https://www.npmjs.com/package/ts-simple-ast) which provides + a more rational and functional interface to the TypeScript AST to make + manipulations easier. +- [`prettier`](https://www.npmjs.com/package/prettier) and + [`@types/prettier`](https://www.npmjs.com/package/@types/prettier) to format + the output. + +## Design + +Ideally we wouldn't have to build this tool at all, and could simply use `tsc` +to output this declaration file. While, `--emitDeclarationsOnly`, `--outFile` +and `--module AMD` generates a single declaration file, it isn't clean. It was +never designed for a library generation, where what is available in a runtime +environment significantly differs from the code that creates that environment's +structure. + +Therefore this tool injects some of the knowledge of what occurs in the Deno +runtime environment as well as ensures that the output file is more clean and +logical for an end user. In the deno runtime, code runs in a global scope that +is defined in `js/global.ts`. This contains global scope items that one +reasonably expects in a JavaScript runtime, like `console`. It also defines the +global scope on a self-reflective `window` variable. There is currently only one +module of Deno specific APIs which is available to the user. This is defined in +`js/deno.ts`. + +This tool takes advantage of an experimental feature of TypeScript that items +that are not really intended to be part of the public API are marked with a +comment pragma of `@internal` and then are not emitted when generating type +definitions. In addition TypeScript will _tree-shake_ any dependencies tied to +that "hidden" API and elide them as well. This really helps keep the public API +clean and as minimal as needed. + +In order to create the default type library, the process at a high-level looks +like this: + +- We read in all of the runtime environment definition code into TypeScript AST + parser "project". +- We emit the TypeScript type definitions only into another AST parser + "project". +- We process the `deno` namespace/module, by "flattening" the type definition + file. + - We determine the exported symbols for `js/deno.ts`. + - We create a custom extraction of the `gen/msg_generated.ts` which is + generated during the build process and contains the type information related + to flatbuffer structures that communicate between the privileged part of + deno and the user land. Currently, the tool doesn't do full complex + dependency analysis to be able to determine what is required out of this + file, so we explicitly extract the type information we need. + - We recurse over all imports/exports of the modules, only exporting those + symbols which are finally exported by `js/deno.ts`. + - We replace the import/export with the type information from the source file. + - This process assumes that all the modules that feed `js/deno.ts` will have a + public type API that does not have name conflicts. +- We process the `js/globals.ts` file to generate the global namespace. + - Currently we create a `"globals"` module which will contain the type + definitions. + - We create a `Window` interface and a `global` scope augmentation namespace. + - We iterate over augmentations to the `window` variable declared in the file, + extract the type information and apply it to both a global variable + declaration and a property on the `Window` interface. +- We take each namespace import to `js/globals.ts`, we resolve the emitted + declaration `.d.ts` file and create it as its own namespace withing the + `"globals"` module. It is unsafe to just flatten these, because there is a + high risk of collisions, but also, it makes authoring the types easier within + the generated interface and variable declarations. +- We then validate the resulting definition file and write it out to the + appropriate build path. + +## TODO + +- The tool does not _tree-shake_ when flattening imports. This means there are + extraneous types that get included that are not really needed and it means + that `gen/msg_generated.ts` has to be explicitly carved down. +- Complete the tests... we have some coverage, but not a lot of what is in + `ast_util_test` which is being tested implicitly. diff --git a/tools/ts_library_builder/ast_util.ts b/tools/ts_library_builder/ast_util.ts new file mode 100644 index 0000000000..c13195b08b --- /dev/null +++ b/tools/ts_library_builder/ast_util.ts @@ -0,0 +1,331 @@ +import { relative } from "path"; +import { readFileSync } from "fs"; +import { EOL } from "os"; +import { + ExportDeclaration, + ImportDeclaration, + InterfaceDeclaration, + JSDoc, + Project, + PropertySignature, + SourceFile, + StatementedNode, + ts, + TypeGuards, + VariableStatement, + VariableDeclarationKind +} from "ts-simple-ast"; + +/** Add a property to an interface */ +export function addInterfaceProperty( + interfaceDeclaration: InterfaceDeclaration, + name: string, + type: string, + jsdocs?: JSDoc[] +): PropertySignature { + return interfaceDeclaration.addProperty({ + name, + type, + docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()) + }); +} + +/** Add `@url` comment to node. */ +export function addSourceComment( + node: StatementedNode, + sourceFile: SourceFile, + rootPath: string +): void { + node.insertStatements( + 0, + `// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n` + ); +} + +/** Add a declaration of a variable to a node */ +export function addVariableDeclaration( + node: StatementedNode, + name: string, + type: string, + jsdocs?: JSDoc[] +): VariableStatement { + return node.addVariableStatement({ + declarationKind: VariableDeclarationKind.Const, + declarations: [{ name, type }], + docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()) + }); +} + +/** Check diagnostics, and if any exist, exit the process */ +export function checkDiagnostics(project: Project, onlyFor?: string[]) { + const program = project.getProgram(); + const diagnostics = [ + ...program.getGlobalDiagnostics(), + ...program.getSyntacticDiagnostics(), + ...program.getSemanticDiagnostics(), + ...program.getDeclarationDiagnostics() + ] + .filter(diagnostic => { + const sourceFile = diagnostic.getSourceFile(); + return onlyFor && sourceFile + ? onlyFor.includes(sourceFile.getFilePath()) + : true; + }) + .map(diagnostic => diagnostic.compilerObject); + + if (diagnostics.length) { + console.log( + ts.formatDiagnosticsWithColorAndContext(diagnostics, formatDiagnosticHost) + ); + process.exit(1); + } +} + +export interface FlattenNamespaceOptions { + customSources?: { [sourceFilePath: string]: string }; + debug?: boolean; + rootPath: string; + sourceFile: SourceFile; +} + +/** Take a namespace and flatten all exports. */ +export function flattenNamespace({ + customSources, + debug, + rootPath, + sourceFile +}: FlattenNamespaceOptions): string { + const sourceFiles = new Set(); + let output = ""; + const exportedSymbols = getExportedSymbols(sourceFile); + + function flattenDeclarations( + declaration: ImportDeclaration | ExportDeclaration + ) { + const declarationSourceFile = declaration.getModuleSpecifierSourceFile(); + if (declarationSourceFile) { + processSourceFile(declarationSourceFile); + declaration.remove(); + } + } + + function rectifyNodes(currentSourceFile: SourceFile) { + currentSourceFile.forEachChild(node => { + if (TypeGuards.isAmbientableNode(node)) { + node.setHasDeclareKeyword(false); + } + if (TypeGuards.isExportableNode(node)) { + const nodeSymbol = node.getSymbol(); + if ( + nodeSymbol && + !exportedSymbols.has(nodeSymbol.getFullyQualifiedName()) + ) { + node.setIsExported(false); + } + } + }); + } + + function processSourceFile(currentSourceFile: SourceFile) { + if (sourceFiles.has(currentSourceFile)) { + return; + } + sourceFiles.add(currentSourceFile); + + const currentSourceFilePath = currentSourceFile.getFilePath(); + if (customSources && currentSourceFilePath in customSources) { + output += customSources[currentSourceFilePath]; + return; + } + + currentSourceFile.getImportDeclarations().forEach(flattenDeclarations); + currentSourceFile.getExportDeclarations().forEach(flattenDeclarations); + + rectifyNodes(currentSourceFile); + + output += + (debug ? getSourceComment(currentSourceFile, rootPath) : "") + + currentSourceFile.print(); + } + + sourceFile.getExportDeclarations().forEach(exportDeclaration => { + processSourceFile(exportDeclaration.getModuleSpecifierSourceFileOrThrow()); + exportDeclaration.remove(); + }); + + rectifyNodes(sourceFile); + + return ( + output + + (debug ? getSourceComment(sourceFile, rootPath) : "") + + sourceFile.print() + ); +} + +/** Used when formatting diagnostics */ +const formatDiagnosticHost: ts.FormatDiagnosticsHost = { + getCurrentDirectory() { + return process.cwd(); + }, + getCanonicalFileName(path: string) { + return path; + }, + getNewLine() { + return EOL; + } +}; + +/** Return a set of fully qualified symbol names for the files exports */ +function getExportedSymbols(sourceFile: SourceFile): Set { + const exportedSymbols = new Set(); + const exportDeclarations = sourceFile.getExportDeclarations(); + for (const exportDeclaration of exportDeclarations) { + const exportSpecifiers = exportDeclaration.getNamedExports(); + for (const exportSpecifier of exportSpecifiers) { + const aliasedSymbol = exportSpecifier + .getSymbolOrThrow() + .getAliasedSymbol(); + if (aliasedSymbol) { + exportedSymbols.add(aliasedSymbol.getFullyQualifiedName()); + } + } + } + return exportedSymbols; +} + +/** Returns a string which indicates the source file as the source */ +export function getSourceComment( + sourceFile: SourceFile, + rootPath: string +): string { + return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`; +} + +/** + * Load and write to a virtual file system all the default libs needed to + * resolve types on project. + */ +export function loadDtsFiles(project: Project) { + loadFiles( + project, + [ + "lib.es2015.collection.d.ts", + "lib.es2015.core.d.ts", + "lib.es2015.d.ts", + "lib.es2015.generator.d.ts", + "lib.es2015.iterable.d.ts", + "lib.es2015.promise.d.ts", + "lib.es2015.proxy.d.ts", + "lib.es2015.reflect.d.ts", + "lib.es2015.symbol.d.ts", + "lib.es2015.symbol.wellknown.d.ts", + "lib.es2016.array.include.d.ts", + "lib.es2016.d.ts", + "lib.es2017.d.ts", + "lib.es2017.intl.d.ts", + "lib.es2017.object.d.ts", + "lib.es2017.sharedmemory.d.ts", + "lib.es2017.string.d.ts", + "lib.es2017.typedarrays.d.ts", + "lib.es2018.d.ts", + "lib.es2018.intl.d.ts", + "lib.es2018.promise.d.ts", + "lib.es2018.regexp.d.ts", + "lib.es5.d.ts", + "lib.esnext.d.ts", + "lib.esnext.array.d.ts", + "lib.esnext.asynciterable.d.ts", + "lib.esnext.intl.d.ts", + "lib.esnext.symbol.d.ts" + ].map(fileName => `node_modules/typescript/lib/${fileName}`) + ); +} + +/** Load a set of files into a file system host. */ +export function loadFiles(project: Project, filePaths: string[]) { + const fileSystem = project.getFileSystem(); + for (const filePath of filePaths) { + const fileText = readFileSync(filePath, { + encoding: "utf8" + }); + fileSystem.writeFileSync(filePath, fileText); + } +} + +export interface NamespaceSourceFileOptions { + debug?: boolean; + namespace?: string; + rootPath: string; + sourceFileMap: Map; +} + +/** + * Take a source file (`.d.ts`) and convert it to a namespace, resolving any + * imports as their own namespaces. + */ +export function namespaceSourceFile( + sourceFile: SourceFile, + { debug, namespace, rootPath, sourceFileMap }: NamespaceSourceFileOptions +): string { + if (sourceFileMap.has(sourceFile)) { + return ""; + } + if (!namespace) { + namespace = sourceFile.getBaseNameWithoutExtension(); + } + sourceFileMap.set(sourceFile, namespace); + + sourceFile.forEachChild(node => { + if (TypeGuards.isAmbientableNode(node)) { + node.setHasDeclareKeyword(false); + } + }); + + const globalNamespace = sourceFile.getNamespace("global"); + const globalNamespaceText = globalNamespace && globalNamespace.print(); + if (globalNamespace) { + globalNamespace.remove(); + } + + const output = sourceFile + .getImportDeclarations() + .map(declaration => { + if ( + declaration.getNamedImports().length || + !declaration.getNamespaceImport() + ) { + throw new Error( + "Unsupported import clause.\n" + + ` In: "${declaration.getSourceFile().getFilePath()}"\n` + + ` Text: "${declaration.getText()}"` + ); + } + const text = namespaceSourceFile( + declaration.getModuleSpecifierSourceFileOrThrow(), + { + debug, + namespace: declaration.getNamespaceImportOrThrow().getText(), + rootPath, + sourceFileMap + } + ); + declaration.remove(); + return text; + }) + .join("\n"); + sourceFile + .getExportDeclarations() + .forEach(declaration => declaration.remove()); + + return `${output} + ${globalNamespaceText || ""} + namespace ${namespace} { + ${debug ? getSourceComment(sourceFile, rootPath) : ""} + ${sourceFile.getText()} + }`; +} + +/** Mirrors TypeScript's handling of paths */ +export function normalizeSlashes(path: string): string { + return path.replace(/\\/g, "/"); +} diff --git a/tools/ts_library_builder/build_library.ts b/tools/ts_library_builder/build_library.ts new file mode 100644 index 0000000000..589e26a822 --- /dev/null +++ b/tools/ts_library_builder/build_library.ts @@ -0,0 +1,453 @@ +import { writeFileSync } from "fs"; +import * as prettier from "prettier"; +import { + ExpressionStatement, + ModuleKind, + ModuleResolutionKind, + NamespaceDeclarationKind, + Project, + ScriptTarget, + SourceFile, + Type, + TypeGuards +} from "ts-simple-ast"; +import { + addInterfaceProperty, + addSourceComment, + addVariableDeclaration, + checkDiagnostics, + flattenNamespace, + getSourceComment, + loadDtsFiles, + loadFiles, + namespaceSourceFile, + normalizeSlashes +} from "./ast_util"; + +export interface BuildLibraryOptions { + /** + * The path to the root of the deno repository + */ + basePath: string; + + /** + * The path to the current build path + */ + buildPath: string; + + /** + * Denotes if the library should be built with debug information (comments + * that indicate the source of the types) + */ + debug?: boolean; + + /** + * The path to the output library + */ + outFile: string; + + /** + * Execute in silent mode or not + */ + silent?: boolean; +} + +/** + * A preamble which is appended to the start of the library. + */ +// tslint:disable-next-line:max-line-length +const libPreamble = `// Copyright 2018 the Deno authors. All rights reserved. MIT license. + +/// +/// + +`; + +// The path to the msg_generated file relative to the build path +const MSG_GENERATED_PATH = "/gen/msg_generated.ts"; + +// An array of enums we want to expose pub +const MSG_GENERATED_ENUMS = ["ErrorKind"]; + +/** Extracts enums from a source file */ +function extract(sourceFile: SourceFile, enumNames: string[]): string { + // Copy specified enums from msg_generated + let output = ""; + for (const enumName of enumNames) { + const enumDeclaration = sourceFile.getEnumOrThrow(enumName); + enumDeclaration.setHasDeclareKeyword(false); + // we are not copying JSDocs or other trivia here because msg_generated only + // contains some non-useful JSDocs and comments that are not ideal to copy + // over + output += enumDeclaration.getText(); + } + return output; +} + +interface FlattenOptions { + basePath: string; + customSources: { [filePath: string]: string }; + filePath: string; + debug?: boolean; + declarationProject: Project; + namespaceName: string; + targetSourceFile: SourceFile; +} + +/** Flatten a module */ +export function flatten({ + basePath, + customSources, + filePath, + debug, + declarationProject, + namespaceName, + targetSourceFile +}: FlattenOptions): void { + // Flatten the source file into a single module declaration + const statements = flattenNamespace({ + sourceFile: declarationProject.getSourceFileOrThrow(filePath), + rootPath: basePath, + customSources, + debug + }); + + // Create the module in the target file + const namespace = targetSourceFile.addNamespace({ + name: namespaceName, + hasDeclareKeyword: true, + declarationKind: NamespaceDeclarationKind.Module + }); + + // Add the output of the flattening to the namespace + namespace.addStatements(statements); +} + +interface MergeOptions { + basePath: string; + declarationProject: Project; + debug?: boolean; + globalVarName: string; + filePath: string; + inputProject: Project; + interfaceName: string; + namespaceName: string; + targetSourceFile: SourceFile; +} + +/** Take a module and merge into into a single namespace */ +export function merge({ + basePath, + declarationProject, + debug, + globalVarName, + filePath, + inputProject, + interfaceName, + namespaceName, + targetSourceFile +}: MergeOptions) { + // We have to build the module/namespace in small pieces which will reflect + // how the global runtime environment will be for Deno + + // We need to add a module named `"globals"` which will contain all the global + // runtime context + const mergedModule = targetSourceFile.addNamespace({ + name: namespaceName, + hasDeclareKeyword: true, + declarationKind: NamespaceDeclarationKind.Module + }); + + // Add the global Window interface + const interfaceDeclaration = mergedModule.addInterface({ + name: interfaceName + }); + + // Add the global scope augmentation module of the "globals" module + const mergedGlobalNamespace = mergedModule.addNamespace({ + name: "global", + declarationKind: NamespaceDeclarationKind.Global + }); + + // Declare the global variable + addVariableDeclaration(mergedGlobalNamespace, globalVarName, interfaceName); + + // Add self reference to the global variable + addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName); + + // Retrieve source file from the input project + const sourceFile = inputProject.getSourceFileOrThrow(filePath); + + // we are going to create a map of variables + const globalVariables = new Map< + string, + { + type: Type; + node: ExpressionStatement; + } + >(); + + // For every augmentation of the global variable in source file, we want + // to extract the type and add it to the global variable map + sourceFile.forEachChild(node => { + if (TypeGuards.isExpressionStatement(node)) { + const firstChild = node.getFirstChild(); + if (!firstChild) { + return; + } + if (TypeGuards.isBinaryExpression(firstChild)) { + const leftExpression = firstChild.getLeft(); + if ( + TypeGuards.isPropertyAccessExpression(leftExpression) && + leftExpression.getExpression().getText() === globalVarName + ) { + const windowProperty = leftExpression.getName(); + if (windowProperty !== globalVarName) { + globalVariables.set(windowProperty, { + type: firstChild.getType(), + node + }); + } + } + } + } + }); + + // A set of source files that the types we are using are dependent on us + // importing + const dependentSourceFiles = new Set(); + + // Create a global variable and add the property to the `Window` interface + // for each mutation of the `window` variable we observed in `globals.ts` + for (const [property, info] of globalVariables) { + const type = info.type.getText(info.node); + const typeSymbol = info.type.getSymbol(); + if (typeSymbol) { + const valueDeclaration = typeSymbol.getValueDeclaration(); + if (valueDeclaration) { + dependentSourceFiles.add(valueDeclaration.getSourceFile()); + } + } + addVariableDeclaration(mergedGlobalNamespace, property, type); + addInterfaceProperty(interfaceDeclaration, property, type); + } + + // We need to ensure that we only namespace each source file once, so we + // will use this map for tracking that. + const sourceFileMap = new Map(); + + // For each import declaration in source file we will want to convert the + // declaration source file into a namespace that exists within the merged + // namespace + const importDeclarations = sourceFile.getImportDeclarations(); + for (const declaration of importDeclarations) { + const declarationSourceFile = declaration.getModuleSpecifierSourceFile(); + if ( + declarationSourceFile && + dependentSourceFiles.has(declarationSourceFile) + ) { + // the source file will resolve to the original `.ts` file, but the + // information we really want is in the emitted `.d.ts` file, so we will + // resolve to that file + const dtsFilePath = declarationSourceFile + .getFilePath() + .replace(/\.ts$/, ".d.ts"); + const dtsSourceFile = declarationProject.getSourceFileOrThrow( + dtsFilePath + ); + mergedModule.addStatements( + namespaceSourceFile(dtsSourceFile, { + debug, + namespace: declaration.getNamespaceImportOrThrow().getText(), + rootPath: basePath, + sourceFileMap + }) + ); + } + } + + if (debug) { + addSourceComment(mergedModule, sourceFile, basePath); + } +} + +/** + * Generate the runtime library for Deno and write it to the supplied out file + * name. + */ +export function main({ + basePath, + buildPath, + debug, + outFile, + silent +}: BuildLibraryOptions) { + if (!silent) { + console.log("-----"); + console.log("build_lib"); + console.log(); + console.log(`basePath: "${basePath}"`); + console.log(`buildPath: "${buildPath}"`); + console.log(`debug: ${!!debug}`); + console.log(`outFile: "${outFile}"`); + console.log(); + } + + // the inputProject will take in the TypeScript files that are internal + // to Deno to be used to generate the library + const inputProject = new Project({ + compilerOptions: { + baseUrl: basePath, + declaration: true, + emitDeclarationOnly: true, + lib: [], + module: ModuleKind.AMD, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + paths: { + "*": ["*", `${buildPath}/*`] + }, + preserveConstEnums: true, + strict: true, + stripInternal: true, + target: ScriptTarget.ESNext + } + }); + + // Add the input files we will need to generate the declarations, `globals` + // plus any modules that are importable in the runtime need to be added here + // plus the `lib.esnext` which is used as the base library + inputProject.addExistingSourceFiles([ + `${basePath}/node_modules/typescript/lib/lib.esnext.d.ts`, + `${basePath}/js/deno.ts`, + `${basePath}/js/globals.ts` + ]); + + // emit the project, which will be only the declaration files + const inputEmitResult = inputProject.emitToMemory(); + + // the declaration project will be the target for the emitted files from + // the input project, these will be used to transfer information over to + // the final library file + const declarationProject = new Project({ + compilerOptions: { + baseUrl: basePath, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + paths: { + "*": ["*", `${buildPath}/*`] + }, + strict: true, + target: ScriptTarget.ESNext + }, + useVirtualFileSystem: true + }); + + // we don't want to add to the declaration project any of the original + // `.ts` source files, so we need to filter those out + const jsPath = normalizeSlashes(`${basePath}/js`); + const inputProjectFiles = inputProject + .getSourceFiles() + .map(sourceFile => sourceFile.getFilePath()) + .filter(filePath => !filePath.startsWith(jsPath)); + loadFiles(declarationProject, inputProjectFiles); + + // now we add the emitted declaration files from the input project + for (const { filePath, text } of inputEmitResult.getFiles()) { + declarationProject.createSourceFile(filePath, text); + } + + // the outputProject will contain the final library file we are looking to + // build + const outputProject = new Project({ + compilerOptions: { + baseUrl: buildPath, + moduleResolution: ModuleResolutionKind.NodeJs, + noLib: true, + strict: true, + target: ScriptTarget.ESNext, + types: ["text-encoding"] + }, + useVirtualFileSystem: true + }); + + // There are files we need to load into memory, so that the project "compiles" + loadDtsFiles(outputProject); + // tslint:disable-next-line:max-line-length + const textEncodingFilePath = `${buildPath}/node_modules/@types/text-encoding/index.d.ts`; + loadFiles(outputProject, [textEncodingFilePath]); + outputProject.addExistingSourceFileIfExists(textEncodingFilePath); + + // libDts is the final output file we are looking to build and we are not + // actually creating it, only in memory at this stage. + const libDTs = outputProject.createSourceFile(outFile); + + // Deal with `js/deno.ts` + + // `gen/msg_generated.d.ts` contains too much exported information that is not + // part of the public API surface of Deno, so we are going to extract just the + // information we need. + const msgGeneratedDts = inputProject.getSourceFileOrThrow( + `${buildPath}${MSG_GENERATED_PATH}` + ); + const msgGeneratedDtsText = extract(msgGeneratedDts, MSG_GENERATED_ENUMS); + + // Generate a object hash of substitutions of modules to use when flattening + const customSources = { + [msgGeneratedDts.getFilePath()]: `${ + debug ? getSourceComment(msgGeneratedDts, basePath) : "" + }${msgGeneratedDtsText}\n` + }; + + flatten({ + basePath, + customSources, + debug, + declarationProject, + filePath: `${basePath}/js/deno.d.ts`, + namespaceName: `"deno"`, + targetSourceFile: libDTs + }); + + if (!silent) { + console.log(`Created module "deno".`); + } + + merge({ + basePath, + declarationProject, + debug, + globalVarName: "window", + filePath: `${basePath}/js/globals.ts`, + inputProject, + interfaceName: "Window", + namespaceName: `"globals"`, + targetSourceFile: libDTs + }); + + if (!silent) { + console.log(`Created module "globals".`); + } + + // Add the preamble + libDTs.insertStatements(0, libPreamble); + + // Check diagnostics + checkDiagnostics(outputProject); + + // Output the final library file + libDTs.saveSync(); + const libDTsText = prettier.format( + outputProject.getFileSystem().readFileSync(outFile, "utf8"), + { parser: "typescript" } + ); + if (!silent) { + console.log(`Outputting library to: "${outFile}"`); + console.log(` Length: ${libDTsText.length}`); + } + writeFileSync(outFile, libDTsText, { encoding: "utf8" }); + if (!silent) { + console.log("-----"); + console.log(); + } +} diff --git a/tools/ts_library_builder/main.ts b/tools/ts_library_builder/main.ts new file mode 100644 index 0000000000..e6662577f1 --- /dev/null +++ b/tools/ts_library_builder/main.ts @@ -0,0 +1,39 @@ +import * as path from "path"; +import { main as buildRuntimeLib } from "./build_library"; + +// this is very simplistic argument parsing, just enough to integrate into +// the build scripts, versus being very robust +let basePath = process.cwd(); +let buildPath = path.join(basePath, "out", "debug"); +let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts"); +let debug = false; +let silent = false; + +process.argv.forEach((arg, i, argv) => { + // tslint:disable-next-line:switch-default + switch (arg) { + case "--basePath": + basePath = path.resolve(argv[i + 1]); + break; + case "--buildPath": + buildPath = path.resolve(argv[i + 1]); + break; + case "--outFile": + outFile = path.resolve(argv[i + 1]); + break; + case "--debug": + debug = true; + break; + case "--silent": + silent = true; + break; + } +}); + +buildRuntimeLib({ + basePath, + buildPath, + debug, + outFile, + silent +}); diff --git a/tools/ts_library_builder/test.ts b/tools/ts_library_builder/test.ts new file mode 100644 index 0000000000..e5317b3939 --- /dev/null +++ b/tools/ts_library_builder/test.ts @@ -0,0 +1,159 @@ +// Run this manually with: +// +// ./node_modules/.bin/ts-node --project tools/ts_library_builder/tsconfig.json tools/ts_library_builder/test.ts + +import { + ModuleKind, + ModuleResolutionKind, + Project, + ScriptTarget +} from "ts-simple-ast"; +import { assert, assertEqual, test } from "../../js/testing/testing"; +import { flatten, merge } from "./build_library"; +import { loadDtsFiles } from "./ast_util"; + +/** setups and returns the fixtures for testing */ +function setupFixtures() { + const basePath = process.cwd(); + const buildPath = `${basePath}/tools/ts_library_builder/testdata`; + const outputFile = `${buildPath}/lib.output.d.ts`; + const inputProject = new Project({ + compilerOptions: { + baseUrl: basePath, + declaration: true, + emitDeclarationOnly: true, + module: ModuleKind.AMD, + moduleResolution: ModuleResolutionKind.NodeJs, + strict: true, + stripInternal: true, + target: ScriptTarget.ESNext + } + }); + inputProject.addExistingSourceFiles([ + `${buildPath}/globals.ts`, + `${buildPath}/api.ts` + ]); + const declarationProject = new Project({ + compilerOptions: {}, + useVirtualFileSystem: true + }); + loadDtsFiles(declarationProject); + for (const { filePath, text } of inputProject.emitToMemory().getFiles()) { + declarationProject.createSourceFile(filePath, text); + } + const outputProject = new Project({ + compilerOptions: {}, + useVirtualFileSystem: true + }); + loadDtsFiles(outputProject); + const outputSourceFile = outputProject.createSourceFile(outputFile); + const debug = true; + + return { + basePath, + buildPath, + inputProject, + outputFile, + declarationProject, + outputProject, + outputSourceFile, + debug + }; +} + +test(function buildLibraryFlatten() { + const { + basePath, + buildPath, + debug, + declarationProject, + outputSourceFile: targetSourceFile + } = setupFixtures(); + + flatten({ + basePath, + customSources: {}, + debug, + declarationProject, + filePath: `${buildPath}/api.d.ts`, + namespaceName: `"api"`, + targetSourceFile + }); + + assert(targetSourceFile.getNamespace(`"api"`) != null); + assertEqual(targetSourceFile.getNamespaces().length, 1); + const namespaceApi = targetSourceFile.getNamespaceOrThrow(`"api"`); + const functions = namespaceApi.getFunctions(); + assertEqual(functions[0].getName(), "foo"); + assertEqual( + functions[0] + .getJsDocs() + .map(jsdoc => jsdoc.getInnerText()) + .join("\n"), + "jsdoc for foo" + ); + assertEqual(functions[1].getName(), "bar"); + assertEqual( + functions[1] + .getJsDocs() + .map(jsdoc => jsdoc.getInnerText()) + .join("\n"), + "" + ); + assertEqual(functions.length, 2); + const classes = namespaceApi.getClasses(); + assertEqual(classes[0].getName(), "Foo"); + assertEqual(classes.length, 1); + const variableDeclarations = namespaceApi.getVariableDeclarations(); + assertEqual(variableDeclarations[0].getName(), "arr"); + assertEqual(variableDeclarations.length, 1); +}); + +test(function buildLibraryMerge() { + const { + basePath, + buildPath, + declarationProject, + debug, + inputProject, + outputSourceFile: targetSourceFile + } = setupFixtures(); + + merge({ + basePath, + declarationProject, + debug, + globalVarName: "foobarbaz", + filePath: `${buildPath}/globals.ts`, + inputProject, + interfaceName: "FooBar", + namespaceName: `"bazqat"`, + targetSourceFile + }); + + assert(targetSourceFile.getNamespace(`"bazqat"`) != null); + assertEqual(targetSourceFile.getNamespaces().length, 1); + const namespaceBazqat = targetSourceFile.getNamespaceOrThrow(`"bazqat"`); + assert(namespaceBazqat.getNamespace("global") != null); + assert(namespaceBazqat.getNamespace("moduleC") != null); + assertEqual(namespaceBazqat.getNamespaces().length, 2); + assert(namespaceBazqat.getInterface("FooBar") != null); + assertEqual(namespaceBazqat.getInterfaces().length, 1); + const globalNamespace = namespaceBazqat.getNamespaceOrThrow("global"); + const variableDeclarations = globalNamespace.getVariableDeclarations(); + assertEqual( + variableDeclarations[0].getType().getText(), + `import("bazqat").FooBar` + ); + assertEqual( + variableDeclarations[1].getType().getText(), + `import("bazqat").moduleC.Bar` + ); + assertEqual( + variableDeclarations[2].getType().getText(), + `typeof import("bazqat").moduleC.qat` + ); + assertEqual(variableDeclarations.length, 3); +}); + +// TODO author unit tests for `ast_util.ts` diff --git a/tools/ts_library_builder/testdata/api.ts b/tools/ts_library_builder/testdata/api.ts new file mode 100644 index 0000000000..f282a84141 --- /dev/null +++ b/tools/ts_library_builder/testdata/api.ts @@ -0,0 +1,4 @@ +export { foo, bar } from "./moduleA"; +export { Foo } from "./moduleB"; +/** jsdoc for arr */ +export const arr: string[] = []; diff --git a/tools/ts_library_builder/testdata/globals.ts b/tools/ts_library_builder/testdata/globals.ts new file mode 100644 index 0000000000..41a86bdf88 --- /dev/null +++ b/tools/ts_library_builder/testdata/globals.ts @@ -0,0 +1,6 @@ +import * as moduleC from "./moduleC"; + +// tslint:disable-next-line:no-any +const foobarbaz: any = {}; +foobarbaz.bar = new moduleC.Bar(); +foobarbaz.qat = moduleC.qat; diff --git a/tools/ts_library_builder/testdata/moduleA.ts b/tools/ts_library_builder/testdata/moduleA.ts new file mode 100644 index 0000000000..a5bd07e2fc --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleA.ts @@ -0,0 +1,9 @@ +/** jsdoc for foo */ +export function foo(a: string, b: string) { + console.log(a, b); +} + +// no jsdoc for bar +export async function bar(promise: Promise): Promise { + return promise.then(() => {}); +} diff --git a/tools/ts_library_builder/testdata/moduleB.ts b/tools/ts_library_builder/testdata/moduleB.ts new file mode 100644 index 0000000000..91f5ea8757 --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleB.ts @@ -0,0 +1,9 @@ +/** jsdoc about Foo */ +export class Foo { + private _foo = "foo"; + /** jsdoc about Foo.log() */ + log() { + console.log(this._foo); + return this._foo; + } +} diff --git a/tools/ts_library_builder/testdata/moduleC.ts b/tools/ts_library_builder/testdata/moduleC.ts new file mode 100644 index 0000000000..b998c9e9db --- /dev/null +++ b/tools/ts_library_builder/testdata/moduleC.ts @@ -0,0 +1,18 @@ +/** jsdoc for Bar */ +export class Bar { + private _bar: string; + /** jsdoc for Bar.log() */ + log() { + console.log(this._bar); + return this.log; + } +} + +/** + * jsdoc for qat + * @param a jsdoc for qat(a) + * @param b jsdoc for qat(b) + */ +export function qat(a: string, b: string) { + return a + b; +} diff --git a/tools/ts_library_builder/tsconfig.json b/tools/ts_library_builder/tsconfig.json new file mode 100644 index 0000000000..6d010335c2 --- /dev/null +++ b/tools/ts_library_builder/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "strict": true, + "target": "esnext" + }, + "files": ["./build_library.ts"] +}