diff --git a/cli/js/deno.ts b/cli/js/deno.ts index b22f076ef7..6a493faf8e 100644 --- a/cli/js/deno.ts +++ b/cli/js/deno.ts @@ -118,7 +118,7 @@ export { utimeSync, utime } from "./ops/fs/utime.ts"; export { version } from "./version.ts"; export { writeFileSync, writeFile, WriteFileOptions } from "./write_file.ts"; export const args: string[] = []; -export { test, runTests } from "./testing.ts"; +export { test, runTests, TestEvent, ConsoleTestReporter } from "./testing.ts"; // These are internal Deno APIs. We are marking them as internal so they do not // appear in the runtime type library. diff --git a/cli/js/lib.deno.ns.d.ts b/cli/js/lib.deno.ns.d.ts index bf10049bb4..751e4452b3 100644 --- a/cli/js/lib.deno.ns.d.ts +++ b/cli/js/lib.deno.ns.d.ts @@ -32,6 +32,59 @@ declare namespace Deno { * when `Deno.runTests` is used */ export function test(name: string, fn: TestFunction): void; + interface TestResult { + passed: boolean; + name: string; + skipped: boolean; + hasRun: boolean; + duration: number; + error?: Error; + } + + interface TestStats { + filtered: number; + ignored: number; + measured: number; + passed: number; + failed: number; + } + + export enum TestEvent { + Start = "start", + Result = "result", + End = "end" + } + + interface TestEventStart { + kind: TestEvent.Start; + tests: number; + } + + interface TestEventResult { + kind: TestEvent.Result; + result: TestResult; + } + + interface TestEventEnd { + kind: TestEvent.End; + stats: TestStats; + duration: number; + results: TestResult[]; + } + + interface TestReporter { + start(event: TestEventStart): Promise; + result(event: TestEventResult): Promise; + end(event: TestEventEnd): Promise; + } + + export class ConsoleTestReporter implements TestReporter { + constructor(); + start(event: TestEventStart): Promise; + result(event: TestEventResult): Promise; + end(event: TestEventEnd): Promise; + } + export interface RunTestsOptions { /** If `true`, Deno will exit with status code 1 if there was * test failure. Defaults to `true`. */ @@ -46,11 +99,19 @@ declare namespace Deno { skip?: string | RegExp; /** Disable logging of the results. Defaults to `false`. */ disableLog?: boolean; + /** Custom reporter class. If not provided uses console reporter. */ + reporter?: TestReporter; } /** Run any tests which have been registered. Always resolves * asynchronously. */ - export function runTests(opts?: RunTestsOptions): Promise; + export function runTests( + opts?: RunTestsOptions + ): Promise<{ + results: TestResult[]; + stats: TestStats; + duration: number; + }>; /** Get the `loadavg`. Requires `allow-env` permission. * diff --git a/cli/js/testing.ts b/cli/js/testing.ts index f1318f0ce4..a2944aff41 100644 --- a/cli/js/testing.ts +++ b/cli/js/testing.ts @@ -3,17 +3,16 @@ import { red, green, bgRed, gray, italic } from "./colors.ts"; import { exit } from "./ops/os.ts"; import { Console } from "./web/console.ts"; +const RED_FAILED = red("FAILED"); +const GREEN_OK = green("OK"); +const RED_BG_FAIL = bgRed(" FAIL "); +const disabledConsole = new Console((_x: string, _isErr?: boolean): void => {}); + function formatDuration(time = 0): string { const timeStr = `(${time}ms)`; return gray(italic(timeStr)); } -function defer(n: number): Promise { - return new Promise((resolve: () => void, _) => { - setTimeout(resolve, n); - }); -} - export type TestFunction = () => void | Promise; export interface TestDefinition { @@ -70,27 +69,137 @@ interface TestStats { failed: number; } -interface TestCase { - name: string; - fn: TestFunction; - timeElapsed?: number; - error?: Error; -} - export interface RunTestsOptions { exitOnFail?: boolean; failFast?: boolean; only?: string | RegExp; skip?: string | RegExp; disableLog?: boolean; + reporter?: TestReporter; } -function filterTests( - tests: TestDefinition[], +interface TestResult { + passed: boolean; + name: string; + skipped: boolean; + hasRun: boolean; + duration: number; + error?: Error; +} + +interface TestCase { + result: TestResult; + fn: TestFunction; +} + +export enum TestEvent { + Start = "start", + Result = "result", + End = "end" +} + +interface TestEventStart { + kind: TestEvent.Start; + tests: number; +} + +interface TestEventResult { + kind: TestEvent.Result; + result: TestResult; +} + +interface TestEventEnd { + kind: TestEvent.End; + stats: TestStats; + duration: number; + results: TestResult[]; +} + +function testDefinitionToTestCase(def: TestDefinition): TestCase { + return { + fn: def.fn, + result: { + name: def.name, + passed: false, + skipped: false, + hasRun: false, + duration: 0 + } + }; +} + +// TODO: already implements AsyncGenerator, but add as "implements to class" +// TODO: implements PromiseLike +class TestApi { + readonly testsToRun: TestDefinition[]; + readonly testCases: TestCase[]; + readonly stats: TestStats = { + filtered: 0, + ignored: 0, + measured: 0, + passed: 0, + failed: 0 + }; + + constructor( + public tests: TestDefinition[], + public filterFn: (def: TestDefinition) => boolean, + public failFast: boolean + ) { + this.testsToRun = tests.filter(filterFn); + this.stats.filtered = tests.length - this.testsToRun.length; + this.testCases = this.testsToRun.map(testDefinitionToTestCase); + } + + async *[Symbol.asyncIterator](): AsyncIterator< + TestEventStart | TestEventResult | TestEventEnd + > { + yield { + kind: TestEvent.Start, + tests: this.testsToRun.length + }; + + const suiteStart = +new Date(); + for (const testCase of this.testCases) { + const { fn, result } = testCase; + let shouldBreak = false; + try { + const start = +new Date(); + await fn(); + result.duration = +new Date() - start; + result.passed = true; + this.stats.passed++; + } catch (err) { + result.passed = false; + result.error = err; + this.stats.failed++; + shouldBreak = this.failFast; + } finally { + result.hasRun = true; + yield { kind: TestEvent.Result, result }; + if (shouldBreak) { + break; + } + } + } + + const duration = +new Date() - suiteStart; + const results = this.testCases.map(r => r.result); + + yield { + kind: TestEvent.End, + stats: this.stats, + results, + duration + }; + } +} + +function createFilterFn( only: undefined | string | RegExp, skip: undefined | string | RegExp -): TestDefinition[] { - return tests.filter((def: TestDefinition): boolean => { +): (def: TestDefinition) => boolean { + return (def: TestDefinition): boolean => { let passes = true; if (only) { @@ -110,7 +219,49 @@ function filterTests( } return passes; - }); + }; +} + +interface TestReporter { + start(msg: TestEventStart): Promise; + result(msg: TestEventResult): Promise; + end(msg: TestEventEnd): Promise; +} + +export class ConsoleTestReporter implements TestReporter { + private console: Console; + constructor() { + this.console = globalThis.console as Console; + } + + async start(event: TestEventStart): Promise { + this.console.log(`running ${event.tests} tests`); + } + + async result(event: TestEventResult): Promise { + const { result } = event; + + if (result.passed) { + this.console.log( + `${GREEN_OK} ${result.name} ${formatDuration(result.duration)}` + ); + } else { + this.console.log(`${RED_FAILED} ${result.name}`); + this.console.log(result.error!); + } + } + + async end(event: TestEventEnd): Promise { + const { stats, duration } = event; + // Attempting to match the output of Rust's test runner. + this.console.log( + `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` + + `${stats.passed} passed; ${stats.failed} failed; ` + + `${stats.ignored} ignored; ${stats.measured} measured; ` + + `${stats.filtered} filtered out ` + + `${formatDuration(duration)}\n` + ); + } } export async function runTests({ @@ -118,104 +269,54 @@ export async function runTests({ failFast = false, only = undefined, skip = undefined, - disableLog = false -}: RunTestsOptions = {}): Promise { - const testsToRun = filterTests(TEST_REGISTRY, only, skip); + disableLog = false, + reporter = undefined +}: RunTestsOptions = {}): Promise<{ + results: TestResult[]; + stats: TestStats; + duration: number; +}> { + const filterFn = createFilterFn(only, skip); + const testApi = new TestApi(TEST_REGISTRY, filterFn, failFast); - const stats: TestStats = { - measured: 0, - ignored: 0, - filtered: 0, - passed: 0, - failed: 0 - }; - - const testCases = testsToRun.map( - ({ name, fn }): TestCase => { - return { - name, - fn, - timeElapsed: 0, - error: undefined - }; - } - ); + if (!reporter) { + reporter = new ConsoleTestReporter(); + } // @ts-ignore const originalConsole = globalThis.console; - // TODO(bartlomieju): add option to capture output of test - // cases and display it if test fails (like --nopcature in Rust) - const disabledConsole = new Console( - (_x: string, _isErr?: boolean): void => {} - ); if (disableLog) { // @ts-ignore globalThis.console = disabledConsole; } - const RED_FAILED = red("FAILED"); - const GREEN_OK = green("OK"); - const RED_BG_FAIL = bgRed(" FAIL "); + let endMsg: TestEventEnd; - originalConsole.log(`running ${testsToRun.length} tests`); - const suiteStart = +new Date(); - - for (const testCase of testCases) { - try { - const start = +new Date(); - await testCase.fn(); - testCase.timeElapsed = +new Date() - start; - originalConsole.log( - `${GREEN_OK} ${testCase.name} ${formatDuration( - testCase.timeElapsed - )}` - ); - stats.passed++; - } catch (err) { - testCase.error = err; - originalConsole.log(`${RED_FAILED} ${testCase.name}`); - originalConsole.log(err.stack); - stats.failed++; - if (failFast) { - break; - } + for await (const testMsg of testApi) { + switch (testMsg.kind) { + case TestEvent.Start: + await reporter.start(testMsg); + continue; + case TestEvent.Result: + await reporter.result(testMsg); + continue; + case TestEvent.End: + endMsg = testMsg; + delete endMsg!.kind; + await reporter.end(testMsg); + continue; } } - const suiteDuration = +new Date() - suiteStart; - if (disableLog) { // @ts-ignore globalThis.console = originalConsole; } - // Attempting to match the output of Rust's test runner. - originalConsole.log( - `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` + - `${stats.passed} passed; ${stats.failed} failed; ` + - `${stats.ignored} ignored; ${stats.measured} measured; ` + - `${stats.filtered} filtered out ` + - `${formatDuration(suiteDuration)}\n` - ); - - // TODO(bartlomieju): is `defer` really needed? Shouldn't unhandled - // promise rejection be handled per test case? - // Use defer to avoid the error being ignored due to unhandled - // promise rejections being swallowed. - await defer(0); - - if (stats.failed > 0) { - originalConsole.error(`There were ${stats.failed} test failures.`); - testCases - .filter(testCase => !!testCase.error) - .forEach(testCase => { - originalConsole.error(`${RED_BG_FAIL} ${red(testCase.name)}`); - originalConsole.error(testCase.error); - }); - - if (exitOnFail) { - exit(1); - } + if (endMsg!.stats.failed > 0 && exitOnFail) { + exit(1); } + + return endMsg!; } diff --git a/cli/js/tests/location_test.ts b/cli/js/tests/location_test.ts index 78ecb55b39..2d2faf0c21 100644 --- a/cli/js/tests/location_test.ts +++ b/cli/js/tests/location_test.ts @@ -3,5 +3,5 @@ import { unitTest, assert } from "./test_util.ts"; unitTest(function locationBasic(): void { // location example: file:///Users/rld/src/deno/js/unit_tests.ts - assert(window.location.toString().endsWith("unit_tests.ts")); + assert(window.location.toString().endsWith("unit_test_runner.ts")); }); diff --git a/cli/js/tests/resources_test.ts b/cli/js/tests/resources_test.ts index 84b713a6d5..680fac8b73 100644 --- a/cli/js/tests/resources_test.ts +++ b/cli/js/tests/resources_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { unitTest, assertEquals } from "./test_util.ts"; +import { unitTest, assertEquals, assert } from "./test_util.ts"; unitTest(function resourcesStdio(): void { const res = Deno.resources(); @@ -21,10 +21,10 @@ unitTest({ perms: { net: true } }, async function resourcesNet(): Promise< Object.values(res).filter((r): boolean => r === "tcpListener").length, 1 ); - assertEquals( - Object.values(res).filter((r): boolean => r === "tcpStream").length, - 2 + const tcpStreams = Object.values(res).filter( + (r): boolean => r === "tcpStream" ); + assert(tcpStreams.length >= 2); listenerConn.close(); dialerConn.close(); diff --git a/cli/js/tests/test_util.ts b/cli/js/tests/test_util.ts index c8f28437df..66edd6681e 100644 --- a/cli/js/tests/test_util.ts +++ b/cli/js/tests/test_util.ts @@ -1,13 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -// -// We want to test many ops in deno which have different behavior depending on -// the permissions set. These tests can specify which permissions they expect, -// which appends a special string like "permW1N0" to the end of the test name. -// Here we run several copies of deno with different permissions, filtering the -// tests by the special string. permW1N0 means allow-write but not allow-net. -// See tools/unit_tests.py for more details. -import { readLines } from "../../../std/io/bufio.ts"; import { assert, assertEquals } from "../../../std/testing/asserts.ts"; export { assert, @@ -20,16 +12,7 @@ export { unreachable, fail } from "../../../std/testing/asserts.ts"; - -interface TestPermissions { - read?: boolean; - write?: boolean; - net?: boolean; - env?: boolean; - run?: boolean; - plugin?: boolean; - hrtime?: boolean; -} +export { readLines } from "../../../std/io/bufio.ts"; export interface Permissions { read: boolean; @@ -41,10 +24,22 @@ export interface Permissions { hrtime: boolean; } +export function fmtPerms(perms: Permissions): string { + const p = Object.keys(perms) + .filter((e): boolean => perms[e as keyof Permissions] === true) + .map(key => `--allow-${key}`); + + if (p.length) { + return p.join(" "); + } + + return ""; +} + const isGranted = async (name: Deno.PermissionName): Promise => (await Deno.permissions.query({ name })).state === "granted"; -async function getProcessPermissions(): Promise { +export async function getProcessPermissions(): Promise { return { run: await isGranted("run"), read: await isGranted("read"), @@ -56,9 +51,7 @@ async function getProcessPermissions(): Promise { }; } -const processPerms = await getProcessPermissions(); - -function permissionsMatch( +export function permissionsMatch( processPerms: Permissions, requiredPerms: Permissions ): boolean { @@ -94,7 +87,23 @@ function registerPermCombination(perms: Permissions): void { } } -function normalizeTestPermissions(perms: TestPermissions): Permissions { +export async function registerUnitTests(): Promise { + const processPerms = await getProcessPermissions(); + + for (const unitTestDefinition of REGISTERED_UNIT_TESTS) { + if (unitTestDefinition.skip) { + continue; + } + + if (!permissionsMatch(processPerms, unitTestDefinition.perms)) { + continue; + } + + Deno.test(unitTestDefinition); + } +} + +function normalizeTestPermissions(perms: UnitTestPermissions): Permissions { return { read: !!perms.read, write: !!perms.write, @@ -147,11 +156,30 @@ function assertResources(fn: Deno.TestFunction): Deno.TestFunction { }; } +interface UnitTestPermissions { + read?: boolean; + write?: boolean; + net?: boolean; + env?: boolean; + run?: boolean; + plugin?: boolean; + hrtime?: boolean; +} + interface UnitTestOptions { skip?: boolean; - perms?: TestPermissions; + perms?: UnitTestPermissions; } +interface UnitTestDefinition { + name: string; + fn: Deno.TestFunction; + skip?: boolean; + perms: Permissions; +} + +export const REGISTERED_UNIT_TESTS: UnitTestDefinition[] = []; + export function unitTest(fn: Deno.TestFunction): void; export function unitTest(options: UnitTestOptions, fn: Deno.TestFunction): void; export function unitTest( @@ -187,53 +215,15 @@ export function unitTest( const normalizedPerms = normalizeTestPermissions(options.perms || {}); registerPermCombination(normalizedPerms); - if (!permissionsMatch(processPerms, normalizedPerms)) { - return; - } - const testDefinition: Deno.TestDefinition = { + const unitTestDefinition: UnitTestDefinition = { name, - fn: assertResources(assertOps(fn)) + fn: assertResources(assertOps(fn)), + skip: !!options.skip, + perms: normalizedPerms }; - Deno.test(testDefinition); -} -function extractNumber(re: RegExp, str: string): number | undefined { - const match = str.match(re); - - if (match) { - return Number.parseInt(match[1]); - } -} - -export async function parseUnitTestOutput( - reader: Deno.Reader, - print: boolean -): Promise<{ actual?: number; expected?: number; resultOutput?: string }> { - let expected, actual, result; - - for await (const line of readLines(reader)) { - if (!expected) { - // expect "running 30 tests" - expected = extractNumber(/running (\d+) tests/, line); - } else if (line.indexOf("test result:") !== -1) { - result = line; - } - - if (print) { - console.log(line); - } - } - - // Check that the number of expected tests equals what was reported at the - // bottom. - if (result) { - // result should be a string like this: - // "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..." - actual = extractNumber(/(\d+) passed/, result); - } - - return { actual, expected, resultOutput: result }; + REGISTERED_UNIT_TESTS.push(unitTestDefinition); } export interface ResolvableMethods { @@ -254,6 +244,45 @@ export function createResolvable(): Resolvable { return Object.assign(promise, methods!) as Resolvable; } +export class SocketReporter implements Deno.TestReporter { + private encoder: TextEncoder; + + constructor(private conn: Deno.Conn) { + this.encoder = new TextEncoder(); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async write(msg: any): Promise { + const encodedMsg = this.encoder.encode(`${JSON.stringify(msg)}\n`); + await Deno.writeAll(this.conn, encodedMsg); + } + + async start(msg: Deno.TestEventStart): Promise { + await this.write(msg); + } + + async result(msg: Deno.TestEventResult): Promise { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const serializedMsg: any = { ...msg }; + + // Error is a JS object, so we need to turn it into string to + // send over socket. + if (serializedMsg.result.error) { + serializedMsg.result.error = String(serializedMsg.result.error.stack); + } + + await this.write(serializedMsg); + } + + async end(msg: Deno.TestEventEnd): Promise { + await this.write(msg); + } + + close(): void { + this.conn.close(); + } +} + unitTest(function permissionsMatches(): void { assert( permissionsMatch( @@ -341,43 +370,6 @@ unitTest(function permissionsMatches(): void { ); }); -unitTest( - { perms: { read: true } }, - async function parsingUnitTestOutput(): Promise { - const cwd = Deno.cwd(); - const testDataPath = `${cwd}/tools/testdata/`; - - let result; - - // This is an example of a successful unit test output. - const f1 = await Deno.open(`${testDataPath}/unit_test_output1.txt`); - result = await parseUnitTestOutput(f1, false); - assertEquals(result.actual, 96); - assertEquals(result.expected, 96); - f1.close(); - - // This is an example of a silently dying unit test. - const f2 = await Deno.open(`${testDataPath}/unit_test_output2.txt`); - result = await parseUnitTestOutput(f2, false); - assertEquals(result.actual, undefined); - assertEquals(result.expected, 96); - f2.close(); - - // This is an example of compiling before successful unit tests. - const f3 = await Deno.open(`${testDataPath}/unit_test_output3.txt`); - result = await parseUnitTestOutput(f3, false); - assertEquals(result.actual, 96); - assertEquals(result.expected, 96); - f3.close(); - - // Check what happens on empty output. - const f = new Deno.Buffer(new TextEncoder().encode("\n\n\n")); - result = await parseUnitTestOutput(f, false); - assertEquals(result.actual, undefined); - assertEquals(result.expected, undefined); - } -); - /* * Ensure all unit test files (e.g. xxx_test.ts) are present as imports in * cli/js/tests/unit_tests.ts as it is easy to miss this out diff --git a/cli/js/tests/unit_test_runner.ts b/cli/js/tests/unit_test_runner.ts index a5b7c3a487..f018fb59e5 100755 --- a/cli/js/tests/unit_test_runner.ts +++ b/cli/js/tests/unit_test_runner.ts @@ -2,42 +2,187 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import "./unit_tests.ts"; import { + assert, + readLines, permissionCombinations, - parseUnitTestOutput, - Permissions + Permissions, + registerUnitTests, + SocketReporter, + fmtPerms } from "./test_util.ts"; -interface TestResult { - perms: string; - output?: string; - result: number; +interface PermissionSetTestResult { + perms: Permissions; + passed: boolean; + stats: Deno.TestStats; + permsStr: string; + duration: number; } -function permsToCliFlags(perms: Permissions): string[] { - return Object.keys(perms) - .map(key => { - if (!perms[key as keyof Permissions]) return ""; +const PERMISSIONS: Deno.PermissionName[] = [ + "read", + "write", + "net", + "env", + "run", + "plugin", + "hrtime" +]; - const cliFlag = key.replace( - /\.?([A-Z])/g, - (x, y): string => `-${y.toLowerCase()}` - ); - return `--allow-${cliFlag}`; +/** + * Take a list of permissions and revoke missing permissions. + */ +async function dropWorkerPermissions( + requiredPermissions: Deno.PermissionName[] +): Promise { + const permsToDrop = PERMISSIONS.filter((p): boolean => { + return !requiredPermissions.includes(p); + }); + + for (const perm of permsToDrop) { + await Deno.permissions.revoke({ name: perm }); + } +} + +async function workerRunnerMain(args: string[]): Promise { + const addrArg = args.find(e => e.includes("--addr")); + assert(typeof addrArg === "string", "Missing --addr argument"); + const addrStr = addrArg.split("=")[1]; + const [hostname, port] = addrStr.split(":"); + const addr = { hostname, port: Number(port) }; + + let perms: Deno.PermissionName[] = []; + const permsArg = args.find(e => e.includes("--perms")); + assert(typeof permsArg === "string", "Missing --perms argument"); + const permsStr = permsArg.split("=")[1]; + if (permsStr.length > 0) { + perms = permsStr.split(",") as Deno.PermissionName[]; + } + // Setup reporter + const conn = await Deno.connect(addr); + const socketReporter = new SocketReporter(conn); + // Drop current process permissions to requested set + await dropWorkerPermissions(perms); + // Register unit tests that match process permissions + await registerUnitTests(); + // Execute tests + await Deno.runTests({ + failFast: false, + exitOnFail: false, + reporter: socketReporter + }); + // Notify parent process we're done + socketReporter.close(); +} + +function spawnWorkerRunner(addr: string, perms: Permissions): Deno.Process { + // run subsequent tests using same deno executable + const permStr = Object.keys(perms) + .filter((permName): boolean => { + return perms[permName as Deno.PermissionName] === true; }) - .filter((e): boolean => e.length > 0); + .join(","); + + const args = [ + Deno.execPath(), + "run", + "-A", + "cli/js/tests/unit_test_runner.ts", + "--", + "--worker", + `--addr=${addr}`, + `--perms=${permStr}` + ]; + + const p = Deno.run({ + args, + stdin: "null", + stdout: "piped", + stderr: "null" + }); + + return p; } -function fmtPerms(perms: Permissions): string { - let fmt = permsToCliFlags(perms).join(" "); +async function runTestsForPermissionSet( + reporter: Deno.ConsoleTestReporter, + perms: Permissions +): Promise { + const permsFmt = fmtPerms(perms); + console.log(`Running tests for: ${permsFmt}`); + const addr = { hostname: "127.0.0.1", port: 4510 }; + const addrStr = `${addr.hostname}:${addr.port}`; + const workerListener = Deno.listen(addr); - if (!fmt) { - fmt = ""; + const workerProcess = spawnWorkerRunner(addrStr, perms); + + // Wait for worker subprocess to go online + const conn = await workerListener.accept(); + + let err; + let hasThrown = false; + let expectedPassedTests; + let endEvent; + + try { + for await (const line of readLines(conn)) { + const msg = JSON.parse(line); + + if (msg.kind === Deno.TestEvent.Start) { + expectedPassedTests = msg.tests; + await reporter.start(msg); + continue; + } + + if (msg.kind === Deno.TestEvent.Result) { + await reporter.result(msg); + continue; + } + + endEvent = msg; + await reporter.end(msg); + break; + } + } catch (e) { + hasThrown = true; + err = e; + } finally { + workerListener.close(); } - return fmt; + if (hasThrown) { + throw err; + } + + if (typeof expectedPassedTests === "undefined") { + throw new Error("Worker runner didn't report start"); + } + + if (typeof endEvent === "undefined") { + throw new Error("Worker runner didn't report end"); + } + + const workerStatus = await workerProcess.status(); + if (!workerStatus.success) { + throw new Error( + `Worker runner exited with status code: ${workerStatus.code}` + ); + } + + workerProcess.close(); + + const passed = expectedPassedTests === endEvent.stats.passed; + + return { + perms, + passed, + permsStr: permsFmt, + duration: endEvent.duration, + stats: endEvent.stats + }; } -async function main(): Promise { +async function masterRunnerMain(): Promise { console.log( "Discovered permission combinations for tests:", permissionCombinations.size @@ -47,57 +192,31 @@ async function main(): Promise { console.log("\t" + fmtPerms(perms)); } - const testResults = new Set(); + const testResults = new Set(); + const consoleReporter = new Deno.ConsoleTestReporter(); for (const perms of permissionCombinations.values()) { - const permsFmt = fmtPerms(perms); - console.log(`Running tests for: ${permsFmt}`); - const cliPerms = permsToCliFlags(perms); - // run subsequent tests using same deno executable - const args = [ - Deno.execPath(), - "run", - ...cliPerms, - "cli/js/tests/unit_tests.ts" - ]; - - const p = Deno.run({ - args, - stdout: "piped" - }); - - const { actual, expected, resultOutput } = await parseUnitTestOutput( - p.stdout!, - true - ); - - let result = 0; - - if (!actual && !expected) { - console.error("Bad cli/js/tests/unit_test.ts output"); - result = 1; - } else if (expected !== actual) { - result = 1; - } - - testResults.add({ - perms: permsFmt, - output: resultOutput, - result - }); + const result = await runTestsForPermissionSet(consoleReporter, perms); + testResults.add(result); } // if any run tests returned non-zero status then whole test // run should fail - let testsFailed = false; + let testsPassed = true; for (const testResult of testResults) { - console.log(`Summary for ${testResult.perms}`); - console.log(testResult.output + "\n"); - testsFailed = testsFailed || Boolean(testResult.result); + const { permsStr, stats, duration } = testResult; + console.log(`Summary for ${permsStr}`); + await consoleReporter.end({ + kind: Deno.TestEvent.End, + stats, + duration, + results: [] + }); + testsPassed = testsPassed && testResult.passed; } - if (testsFailed) { + if (!testsPassed) { console.error("Unit tests failed"); Deno.exit(1); } @@ -105,4 +224,16 @@ async function main(): Promise { console.log("Unit tests passed"); } +async function main(): Promise { + const args = Deno.args; + + const isWorker = args.includes("--worker"); + + if (isWorker) { + return await workerRunnerMain(args); + } + + return await masterRunnerMain(); +} + main(); diff --git a/cli/js/tests/unit_tests.ts b/cli/js/tests/unit_tests.ts index 9c80859d6e..4cff3d1d87 100644 --- a/cli/js/tests/unit_tests.ts +++ b/cli/js/tests/unit_tests.ts @@ -1,7 +1,8 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -// This test is executed as part of tools/test.py -// But it can also be run manually: ./target/debug/deno cli/js/tests/unit_tests.ts +// This test is executed as part of unit test suite. +// +// Test runner automatically spawns subprocesses for each required permissions combination. import "./blob_test.ts"; import "./body_test.ts"; @@ -63,7 +64,3 @@ import "./utime_test.ts"; import "./write_file_test.ts"; import "./performance_test.ts"; import "./version_test.ts"; - -if (import.meta.main) { - await Deno.runTests(); -} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index ce08c6b61a..f90e434da7 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -272,8 +272,7 @@ fn js_unit_tests() { .current_dir(util::root_path()) .arg("run") .arg("--reload") - .arg("--allow-run") - .arg("--allow-env") + .arg("-A") .arg("cli/js/tests/unit_test_runner.ts") .spawn() .expect("failed to spawn script"); diff --git a/tools/testdata/unit_test_output1.txt b/tools/testdata/unit_test_output1.txt deleted file mode 100644 index 8a4cfdfd3b..0000000000 --- a/tools/testdata/unit_test_output1.txt +++ /dev/null @@ -1,238 +0,0 @@ -running 96 tests -test permSerialization_permW0N0E0 -... ok -test permFromStringThrows_permW0N0E0 -... ok -test compilerInstance_permW0N0E0 -... ok -test compilerRun_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRunMultiModule_permW0N0E0 -... ok -test compilerRunCircularDependency_permW0N0E0 -Compiling modA -Compiling modB -... ok -test compilerResolveModule_permW0N0E0 -... ok -test compilerGetModuleDependencies_permW0N0E0 -... ok -test compilerGetCompilationSettings_permW0N0E0 -... ok -test compilerGetNewLine_permW0N0E0 -... ok -test compilerGetScriptFileNames_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRecompileFlag_permW0N0E0 -Compiling /root/project/foo/bar.ts -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptKind_permW0N0E0 -... ok -test compilerGetScriptVersion_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptVersionUnknown_permW0N0E0 -... ok -test compilerGetScriptSnapshot_permW0N0E0 -... ok -test compilerGetCurrentDirectory_permW0N0E0 -... ok -test compilerGetDefaultLibFileName_permW0N0E0 -... ok -test compilerUseCaseSensitiveFileNames_permW0N0E0 -... ok -test compilerReadFile_permW0N0E0 -... ok -test compilerFileExists_permW0N0E0 -... ok -test compilerResolveModuleNames_permW0N0E0 -... ok -test consoleTestAssert_permW0N0E0 -... ok -test consoleTestStringifyComplexObjects_permW0N0E0 -... ok -test consoleTestStringifyCircular_permW0N0E0 -... ok -test consoleTestStringifyWithDepth_permW0N0E0 -... ok -test consoleTestError_permW0N0E0 -... ok -test consoleDetachedLog_permW0N0E0 -Hello world -Hello world -Hello world -Hello world -Hello world -Hello world -... ok -test fetchPerm_permW0N0E0 -... ok -test headersAppend_permW0N0E0 -... ok -test newHeaderTest_permW0N0E0 -... ok -test newHeaderWithSequence_permW0N0E0 -... ok -test newHeaderWithRecord_permW0N0E0 -... ok -test newHeaderWithHeadersInstance_permW0N0E0 -... ok -test headerAppendSuccess_permW0N0E0 -... ok -test headerSetSuccess_permW0N0E0 -... ok -test headerHasSuccess_permW0N0E0 -... ok -test headerDeleteSuccess_permW0N0E0 -... ok -test headerGetSuccess_permW0N0E0 -... ok -test headerForEachSuccess_permW0N0E0 -... ok -test envFailure_permW0N0E0 -... ok -test filesStdioFileDescriptors_permW0N0E0 -... ok -test filesCopyToStdout_permW0N0E0 -{ - "name": "deno", - "devDependencies": { - "@types/base64-js": "^1.2.5", - "@types/flatbuffers": "^1.9.0", - "@types/source-map-support": "^0.4.1", - "@types/text-encoding": "0.0.33", - "base64-js": "^1.3.0", - "flatbuffers": "^1.9.0", - "magic-string": "^0.22.5", - "prettier": "^1.14.0", - "rollup": "^0.63.2", - "rollup-plugin-alias": "^1.4.0", - "rollup-plugin-analyzer": "^2.1.0", - "rollup-plugin-commonjs": "^9.1.3", - "rollup-plugin-node-globals": "^1.2.1", - "rollup-plugin-node-resolve": "^3.3.0", - "rollup-plugin-string": "^2.0.2", - "rollup-plugin-typescript2": "^0.16.1", - "rollup-pluginutils": "^2.3.0", - "source-map-support": "^0.5.6", - "text-encoding": "0.6.4", - "tslint": "^5.10.0", - "tslint-eslint-rules": "^5.3.1", - "tslint-no-circular-imports": "^0.5.0", - "typescript": "3.0.3" - } -} -bytes written 860 -... ok -test readFileSyncSuccess_permW0N0E0 -... ok -test readFileSyncNotFound_permW0N0E0 -... ok -test readFileSuccess_permW0N0E0 -... ok -test readdirSyncNotDir_permW0N0E0 -... ok -test readdirSyncNotFound_permW0N0E0 -... ok -test writeFileSyncPerm_permW0N0E0 -... ok -test writeFilePerm_permW0N0E0 -... ok -test copyFileSyncPerm_permW0N0E0 -... ok -test copyFilePerm_permW0N0E0 -... ok -test mkdirSyncPerm_permW0N0E0 -... ok -test makeTempDirSyncPerm_permW0N0E0 -... ok -test statSyncSuccess_permW0N0E0 -... ok -test statSyncNotFound_permW0N0E0 -... ok -test lstatSyncSuccess_permW0N0E0 -... ok -test lstatSyncNotFound_permW0N0E0 -... ok -test statSuccess_permW0N0E0 -... ok -test statNotFound_permW0N0E0 -... ok -test lstatSuccess_permW0N0E0 -... ok -test lstatNotFound_permW0N0E0 -... ok -test renameSyncPerm_permW0N0E0 -... ok -test readlinkSyncNotFound_permW0N0E0 -... ok -test blobString_permW0N0E0 -... ok -test blobBuffer_permW0N0E0 -... ok -test blobSlice_permW0N0E0 -... ok -test timeoutSuccess_permW0N0E0 -... ok -test timeoutArgs_permW0N0E0 -... ok -test timeoutCancelSuccess_permW0N0E0 -... ok -test timeoutCancelMultiple_permW0N0E0 -... ok -test timeoutCancelInvalidSilentFail_permW0N0E0 -... ok -test intervalSuccess_permW0N0E0 -... ok -test intervalCancelSuccess_permW0N0E0 -... ok -test intervalOrdering_permW0N0E0 -... ok -test intervalCancelInvalidSilentFail_permW0N0E0 -... ok -test symlinkSyncPerm_permW0N0E0 -... ok -test platformTransform_permW0N0E0 -... ok -test atobSuccess_permW0N0E0 -... ok -test btoaSuccess_permW0N0E0 -... ok -test btoaFailed_permW0N0E0 -... ok -test truncateSyncPerm_permW0N0E0 -... ok -test truncatePerm_permW0N0E0 -... ok -test evalErrorFormatted_permW0N0E0 -... ok -test createExecTimeColumnsRegularData_permW0N0E0 -... ok -test createExecTimeColumnsIrregularData_permW0N0E0 -... ok -test createBinarySizeColumnsRegularData_permW0N0E0 -... ok -test createBinarySizeColumnsIrregularData_permW0N0E0 -... ok -test createThreadCountColumnsRegularData_permW0N0E0 -... ok -test createThreadCountColumnsIrregularData_permW0N0E0 -... ok -test createSyscallCountColumnsRegularData_permW0N0E0 -... ok -test createSyscallCountColumnsIrregularData_permW0N0E0 -... ok -test createSha1ListRegularData_permW0N0E0 -... ok -test formatBytesPatterns_permW0N0E0 -... ok -test formatSecondsPatterns_permW0N0E0 -... ok -test getTravisDataSuccess_permW0N0E0 -... ok - -test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out - diff --git a/tools/testdata/unit_test_output2.txt b/tools/testdata/unit_test_output2.txt deleted file mode 100644 index 5913d3b90e..0000000000 --- a/tools/testdata/unit_test_output2.txt +++ /dev/null @@ -1,71 +0,0 @@ -running 96 tests -test permSerialization_permW0N0E0 -... ok -test permFromStringThrows_permW0N0E0 -... ok -test compilerInstance_permW0N0E0 -... ok -test compilerRun_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRunMultiModule_permW0N0E0 -... ok -test compilerRunCircularDependency_permW0N0E0 -Compiling modA -Compiling modB -... ok -test compilerResolveModule_permW0N0E0 -... ok -test compilerGetModuleDependencies_permW0N0E0 -... ok -test compilerGetCompilationSettings_permW0N0E0 -... ok -test compilerGetNewLine_permW0N0E0 -... ok -test compilerGetScriptFileNames_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRecompileFlag_permW0N0E0 -Compiling /root/project/foo/bar.ts -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptKind_permW0N0E0 -... ok -test compilerGetScriptVersion_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptVersionUnknown_permW0N0E0 -... ok -test compilerGetScriptSnapshot_permW0N0E0 -... ok -test compilerGetCurrentDirectory_permW0N0E0 -... ok -test compilerGetDefaultLibFileName_permW0N0E0 -... ok -test compilerUseCaseSensitiveFileNames_permW0N0E0 -... ok -test compilerReadFile_permW0N0E0 -... ok -test compilerFileExists_permW0N0E0 -... ok -test compilerResolveModuleNames_permW0N0E0 -... ok -test consoleTestAssert_permW0N0E0 -... ok -test consoleTestStringifyComplexObjects_permW0N0E0 -... ok -test consoleTestStringifyCircular_permW0N0E0 -... ok -test consoleTestStringifyWithDepth_permW0N0E0 -... ok -test consoleTestError_permW0N0E0 -... ok -test consoleDetachedLog_permW0N0E0 -Hello world -Hello world -Hello world -Hello world -Hello world -Hello world -... ok -test fetchPerm_permW0N0E0 diff --git a/tools/testdata/unit_test_output3.txt b/tools/testdata/unit_test_output3.txt deleted file mode 100644 index f1dd7078e9..0000000000 --- a/tools/testdata/unit_test_output3.txt +++ /dev/null @@ -1,268 +0,0 @@ -Compiling /Users/rld/src/deno/js/unit_tests.ts -Compiling /Users/rld/src/deno/js/compiler_test.ts -Compiling /Users/rld/src/deno/js/test_util.ts -Compiling /Users/rld/src/deno/js/testing/testing.ts -Compiling /Users/rld/src/deno/js/testing/util.ts -Compiling /Users/rld/src/deno/js/console_test.ts -Compiling /Users/rld/src/deno/js/console.ts -Compiling /Users/rld/src/deno/js/fetch_test.ts -Compiling /Users/rld/src/deno/js/os_test.ts -Compiling /Users/rld/src/deno/js/files_test.ts -Compiling /Users/rld/src/deno/js/read_file_test.ts -Compiling /Users/rld/src/deno/js/read_dir_test.ts -Compiling /Users/rld/src/deno/js/write_file_test.ts -Compiling /Users/rld/src/deno/js/copy_file_test.ts -Compiling /Users/rld/src/deno/js/mkdir_test.ts -Compiling /Users/rld/src/deno/js/make_temp_dir_test.ts -Compiling /Users/rld/src/deno/js/stat_test.ts -Compiling /Users/rld/src/deno/js/rename_test.ts -Compiling /Users/rld/src/deno/js/read_link_test.ts -Compiling /Users/rld/src/deno/js/blob_test.ts -Compiling /Users/rld/src/deno/js/timers_test.ts -Compiling /Users/rld/src/deno/js/symlink_test.ts -Compiling /Users/rld/src/deno/js/platform_test.ts -Compiling /Users/rld/src/deno/js/text_encoding_test.ts -Compiling /Users/rld/src/deno/js/net_test.ts -Compiling /Users/rld/src/deno/js/trace_test.ts -Compiling /Users/rld/src/deno/js/truncate_test.ts -Compiling /Users/rld/src/deno/js/v8_source_maps_test.ts -Compiling /Users/rld/src/deno/website/app_test.js -Compiling /Users/rld/src/deno/website/app.js -running 96 tests -test permSerialization_permW0N0E0 -... ok -test permFromStringThrows_permW0N0E0 -... ok -test compilerInstance_permW0N0E0 -... ok -test compilerRun_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRunMultiModule_permW0N0E0 -... ok -test compilerRunCircularDependency_permW0N0E0 -Compiling modA -Compiling modB -... ok -test compilerResolveModule_permW0N0E0 -... ok -test compilerGetModuleDependencies_permW0N0E0 -... ok -test compilerGetCompilationSettings_permW0N0E0 -... ok -test compilerGetNewLine_permW0N0E0 -... ok -test compilerGetScriptFileNames_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerRecompileFlag_permW0N0E0 -Compiling /root/project/foo/bar.ts -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptKind_permW0N0E0 -... ok -test compilerGetScriptVersion_permW0N0E0 -Compiling /root/project/foo/bar.ts -... ok -test compilerGetScriptVersionUnknown_permW0N0E0 -... ok -test compilerGetScriptSnapshot_permW0N0E0 -... ok -test compilerGetCurrentDirectory_permW0N0E0 -... ok -test compilerGetDefaultLibFileName_permW0N0E0 -... ok -test compilerUseCaseSensitiveFileNames_permW0N0E0 -... ok -test compilerReadFile_permW0N0E0 -... ok -test compilerFileExists_permW0N0E0 -... ok -test compilerResolveModuleNames_permW0N0E0 -... ok -test consoleTestAssert_permW0N0E0 -... ok -test consoleTestStringifyComplexObjects_permW0N0E0 -... ok -test consoleTestStringifyCircular_permW0N0E0 -... ok -test consoleTestStringifyWithDepth_permW0N0E0 -... ok -test consoleTestError_permW0N0E0 -... ok -test consoleDetachedLog_permW0N0E0 -Hello world -Hello world -Hello world -Hello world -Hello world -Hello world -... ok -test fetchPerm_permW0N0E0 -... ok -test headersAppend_permW0N0E0 -... ok -test newHeaderTest_permW0N0E0 -... ok -test newHeaderWithSequence_permW0N0E0 -... ok -test newHeaderWithRecord_permW0N0E0 -... ok -test newHeaderWithHeadersInstance_permW0N0E0 -... ok -test headerAppendSuccess_permW0N0E0 -... ok -test headerSetSuccess_permW0N0E0 -... ok -test headerHasSuccess_permW0N0E0 -... ok -test headerDeleteSuccess_permW0N0E0 -... ok -test headerGetSuccess_permW0N0E0 -... ok -test headerForEachSuccess_permW0N0E0 -... ok -test envFailure_permW0N0E0 -... ok -test filesStdioFileDescriptors_permW0N0E0 -... ok -test filesCopyToStdout_permW0N0E0 -{ - "name": "deno", - "devDependencies": { - "@types/base64-js": "^1.2.5", - "@types/flatbuffers": "^1.9.0", - "@types/source-map-support": "^0.4.1", - "@types/text-encoding": "0.0.33", - "base64-js": "^1.3.0", - "flatbuffers": "^1.9.0", - "magic-string": "^0.22.5", - "prettier": "^1.14.0", - "rollup": "^0.63.2", - "rollup-plugin-alias": "^1.4.0", - "rollup-plugin-analyzer": "^2.1.0", - "rollup-plugin-commonjs": "^9.1.3", - "rollup-plugin-node-globals": "^1.2.1", - "rollup-plugin-node-resolve": "^3.3.0", - "rollup-plugin-string": "^2.0.2", - "rollup-plugin-typescript2": "^0.16.1", - "rollup-pluginutils": "^2.3.0", - "source-map-support": "^0.5.6", - "text-encoding": "0.6.4", - "tslint": "^5.10.0", - "tslint-eslint-rules": "^5.3.1", - "tslint-no-circular-imports": "^0.5.0", - "typescript": "3.0.3" - } -} -bytes written 860 -... ok -test readFileSyncSuccess_permW0N0E0 -... ok -test readFileSyncNotFound_permW0N0E0 -... ok -test readFileSuccess_permW0N0E0 -... ok -test readdirSyncNotDir_permW0N0E0 -... ok -test readdirSyncNotFound_permW0N0E0 -... ok -test writeFileSyncPerm_permW0N0E0 -... ok -test writeFilePerm_permW0N0E0 -... ok -test copyFileSyncPerm_permW0N0E0 -... ok -test copyFilePerm_permW0N0E0 -... ok -test mkdirSyncPerm_permW0N0E0 -... ok -test makeTempDirSyncPerm_permW0N0E0 -... ok -test statSyncSuccess_permW0N0E0 -... ok -test statSyncNotFound_permW0N0E0 -... ok -test lstatSyncSuccess_permW0N0E0 -... ok -test lstatSyncNotFound_permW0N0E0 -... ok -test statSuccess_permW0N0E0 -... ok -test statNotFound_permW0N0E0 -... ok -test lstatSuccess_permW0N0E0 -... ok -test lstatNotFound_permW0N0E0 -... ok -test renameSyncPerm_permW0N0E0 -... ok -test readlinkSyncNotFound_permW0N0E0 -... ok -test blobString_permW0N0E0 -... ok -test blobBuffer_permW0N0E0 -... ok -test blobSlice_permW0N0E0 -... ok -test timeoutSuccess_permW0N0E0 -... ok -test timeoutArgs_permW0N0E0 -... ok -test timeoutCancelSuccess_permW0N0E0 -... ok -test timeoutCancelMultiple_permW0N0E0 -... ok -test timeoutCancelInvalidSilentFail_permW0N0E0 -... ok -test intervalSuccess_permW0N0E0 -... ok -test intervalCancelSuccess_permW0N0E0 -... ok -test intervalOrdering_permW0N0E0 -... ok -test intervalCancelInvalidSilentFail_permW0N0E0 -... ok -test symlinkSyncPerm_permW0N0E0 -... ok -test platformTransform_permW0N0E0 -... ok -test atobSuccess_permW0N0E0 -... ok -test btoaSuccess_permW0N0E0 -... ok -test btoaFailed_permW0N0E0 -... ok -test truncateSyncPerm_permW0N0E0 -... ok -test truncatePerm_permW0N0E0 -... ok -test evalErrorFormatted_permW0N0E0 -... ok -test createExecTimeColumnsRegularData_permW0N0E0 -... ok -test createExecTimeColumnsIrregularData_permW0N0E0 -... ok -test createBinarySizeColumnsRegularData_permW0N0E0 -... ok -test createBinarySizeColumnsIrregularData_permW0N0E0 -... ok -test createThreadCountColumnsRegularData_permW0N0E0 -... ok -test createThreadCountColumnsIrregularData_permW0N0E0 -... ok -test createSyscallCountColumnsRegularData_permW0N0E0 -... ok -test createSyscallCountColumnsIrregularData_permW0N0E0 -... ok -test createSha1ListRegularData_permW0N0E0 -... ok -test formatBytesPatterns_permW0N0E0 -... ok -test formatSecondsPatterns_permW0N0E0 -... ok -test getTravisDataSuccess_permW0N0E0 -... ok - -test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out -