BREAKING: remove Deno.runTests() API (#4922)

Deno.runTests() interface is not yet good enough to be exposed
publicly with stability guarantees.

This commit removes public API related to testing: Deno.runTests()
and Deno.TestMessage, but keeps them exposed on Deno.internal object
so they can be used with "deno test" subcommand.

cli/js/deno.ts

@@ -114,13 +114,7 @@ export { utimeSync, utime } from "./ops/fs/utime.ts";
 export { version } from "./version.ts";
 export { writeFileSync, writeFile, WriteFileOptions } from "./write_file.ts";
 export const args: string[] = [];
-export {
-  RunTestsOptions,
-  TestDefinition,
-  TestMessage,
-  runTests,
-  test,
-} from "./testing.ts";
+export { TestDefinition, test } from "./testing.ts";
 
 // These are internal Deno APIs.  We are marking them as internal so they do not
 // appear in the runtime type library.

cli/js/lib.deno.ns.d.ts

@@ -21,8 +21,8 @@ declare namespace Deno {
   }
 
   /** Register a test which will be run when `deno test` is used on the command
-   * line and the containing module looks like a test module, or explicitly
-   * when `Deno.runTests` is used.  `fn` can be async if required.
+   * line and the containing module looks like a test module.
+   * `fn` can be async if required.
    *
    *          import {assert, fail, assertEquals} from "https://deno.land/std/testing/asserts.ts";
    *
@@ -53,8 +53,8 @@ declare namespace Deno {
   export function test(t: TestDefinition): void;
 
   /** Register a test which will be run when `deno test` is used on the command
-   * line and the containing module looks like a test module, or explicitly
-   * when `Deno.runTests` is used
+   * line and the containing module looks like a test module.
+   * `fn` can be async if required.
    *
    *        import {assert, fail, assertEquals} from "https://deno.land/std/testing/asserts.ts";
    *
@@ -71,8 +71,8 @@ declare namespace Deno {
   export function test(fn: () => void | Promise<void>): void;
 
   /** Register a test which will be run when `deno test` is used on the command
-   * line and the containing module looks like a test module, or explicitly
-   * when `Deno.runTests` is used
+   * line and the containing module looks like a test module.
+   * `fn` can be async if required.
    *
    *        import {assert, fail, assertEquals} from "https://deno.land/std/testing/asserts.ts";
    *
@@ -88,72 +88,6 @@ declare namespace Deno {
    * */
   export function test(name: string, fn: () => void | Promise<void>): void;
 
-  export interface TestMessage {
-    start?: {
-      tests: TestDefinition[];
-    };
-    testStart?: {
-      [P in keyof TestDefinition]: TestDefinition[P];
-    };
-    testEnd?: {
-      name: string;
-      status: "passed" | "failed" | "ignored";
-      duration: number;
-      error?: Error;
-    };
-    end?: {
-      filtered: number;
-      ignored: number;
-      measured: number;
-      passed: number;
-      failed: number;
-      duration: number;
-      results: Array<TestMessage["testEnd"] & {}>;
-    };
-  }
-
-  export interface RunTestsOptions {
-    /** If `true`, Deno will exit with status code 1 if there was
-     * test failure. Defaults to `true`. */
-    exitOnFail?: boolean;
-    /** If `true`, Deno will exit upon first test failure. Defaults to `false`. */
-    failFast?: boolean;
-    /** String or RegExp used to filter test to run. Only test with names
-     * matching provided `String` or `RegExp` will be run. */
-    filter?: string | RegExp;
-    /** String or RegExp used to skip tests to run. Tests with names
-     * matching provided `String` or `RegExp` will not be run. */
-    skip?: string | RegExp;
-    /** Disable logging of the results. Defaults to `false`. */
-    disableLog?: boolean;
-    /** If true, report results to the console as is done for `deno test`. Defaults to `true`. */
-    reportToConsole?: boolean;
-    /** Called for each message received from the test run. */
-    onMessage?: (message: TestMessage) => void | Promise<void>;
-  }
-
-  /** Run any tests which have been registered via `Deno.test()`. Always resolves
-   * asynchronously.
-   *
-   *        // Register test
-   *        Deno.test({
-   *          name: "example test",
-   *          fn(): void {
-   *            assertEquals("world", "world");
-   *            assertEquals({ hello: "world" }, { hello: "world" });
-   *          },
-   *        });
-   *
-   *        // Run tests
-   *        const runInfo = await Deno.runTests();
-   *        console.log(runInfo.duration);  // all tests duration, e.g. "5" (in ms)
-   *        console.log(runInfo.stats.passed);  // e.g. 1
-   *        console.log(runInfo.results[0].name);  // e.g. "example test"
-   */
-  export function runTests(
-    opts?: RunTestsOptions
-  ): Promise<TestMessage["end"]> & {};
-
   /** Returns an array containing the 1, 5, and 15 minute load averages. The
    * load average is a measure of CPU and IO utilization of the last one, five,
    * and 15 minute periods expressed as a fractional number.  Zero means there

cli/js/testing.ts

@@ -142,7 +142,7 @@ export function test(
   TEST_REGISTRY.push(testDef);
 }
 
-export interface TestMessage {
+interface TestMessage {
   start?: {
     tests: TestDefinition[];
   };
@@ -317,7 +317,7 @@ function createFilterFn(
   };
 }
 
-export interface RunTestsOptions {
+interface RunTestsOptions {
   exitOnFail?: boolean;
   failFast?: boolean;
   filter?: string | RegExp;
@@ -327,7 +327,7 @@ export interface RunTestsOptions {
   onMessage?: (message: TestMessage) => void | Promise<void>;
 }
 
-export async function runTests({
+async function runTests({
   exitOnFail = true,
   failFast = false,
   filter = undefined,
@@ -372,3 +372,5 @@ export async function runTests({
 
   return endMsg!;
 }
+
+exposeForTest("runTests", runTests);

cli/js/tests/test_util.ts

@@ -200,11 +200,14 @@ const encoder = new TextEncoder();
 
 // Replace functions with null, errors with their stack strings, and JSONify.
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-function serializeTestMessage(message: Deno.TestMessage): string {
+function serializeTestMessage(message: any): string {
   return JSON.stringify({
     start: message.start && {
       ...message.start,
-      tests: message.start.tests.map((test) => ({ ...test, fn: null })),
+      tests: message.start.tests.map((test: Deno.TestDefinition) => ({
+        ...test,
+        fn: null,
+      })),
     },
     testStart: message.testStart && { ...message.testStart, fn: null },
     testEnd: message.testEnd && {
@@ -213,7 +216,8 @@ function serializeTestMessage(message: Deno.TestMessage): string {
     },
     end: message.end && {
       ...message.end,
-      results: message.end.results.map((result) => ({
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      results: message.end.results.map((result: any) => ({
         ...result,
         error: result.error?.stack,
       })),
@@ -223,7 +227,8 @@ function serializeTestMessage(message: Deno.TestMessage): string {
 
 export async function reportToConn(
   conn: Deno.Conn,
-  message: Deno.TestMessage
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  message: any
 ): Promise<void> {
   const line = serializeTestMessage(message);
   const encodedMsg = encoder.encode(line + (message.end == null ? "\n" : ""));

cli/js/tests/unit_test_runner.ts

@@ -12,13 +12,17 @@ import {
 } from "./test_util.ts";
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-const reportToConsole = (Deno as any)[Deno.symbols.internal]
-  .reportToConsole as (message: Deno.TestMessage) => void;
+const internalObj = (Deno as any)[Deno.symbols.internal];
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const reportToConsole = internalObj.reportToConsole as (message: any) => void;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const runTests = internalObj.runTests as (options: any) => Promise<any>;
 
 interface PermissionSetTestResult {
   perms: Permissions;
   passed: boolean;
-  endMessage: Deno.TestMessage["end"];
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  endMessage: any;
   permsStr: string;
 }
 
@@ -66,7 +70,7 @@ async function workerRunnerMain(
   // Register unit tests that match process permissions
   await registerUnitTests();
   // Execute tests
-  await Deno.runTests({
+  await runTests({
     exitOnFail: false,
     filter,
     reportToConsole: false,
@@ -129,11 +133,13 @@ async function runTestsForPermissionSet(
   const conn = await listener.accept();
 
   let expectedPassedTests;
-  let endMessage: Deno.TestMessage["end"];
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let endMessage: any;
 
   try {
     for await (const line of readLines(conn)) {
-      const message = JSON.parse(line) as Deno.TestMessage;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const message = JSON.parse(line) as any;
       reportToConsole(message);
       if (message.start != null) {
         expectedPassedTests = message.start.tests.length;
@@ -297,7 +303,7 @@ async function main(): Promise<void> {
 
   // Running tests matching current process permissions
   await registerUnitTests();
-  await Deno.runTests({ filter });
+  await runTests({ filter });
 }
 
 main();

cli/test_runner.rs

@@ -78,7 +78,10 @@ pub fn render_test_file(
     json!({ "failFast": fail_fast, "reportToConsole": !quiet, "disableLog": quiet })
   };
 
-  let run_tests_cmd = format!("Deno.runTests({});\n", options);
+  let run_tests_cmd = format!(
+    "(Deno as any)[Deno.symbols.internal].runTests({});\n",
+    options
+  );
   test_file.push_str(&run_tests_cmd);
 
   test_file