refactor: rewrite deno test, add Deno.test() (#3865)

* rewrite test runner in Rust
* migrate "test" and "runTests" functions from std to "Deno" namespace
* use "Deno.test()" to run internal JS unit tests
* remove std downloads for Deno subcommands

cli/flags.rs

@@ -22,15 +22,6 @@ macro_rules! sset {
   }}
 }
 
-macro_rules! std_url {
-  ($x:expr) => {
-    concat!("https://deno.land/std@v0.29.0/", $x)
-  };
-}
-
-/// Used for `deno test...` subcommand
-const TEST_RUNNER_URL: &str = std_url!("testing/runner.ts");
-
 #[derive(Clone, Debug, PartialEq)]
 pub enum DenoSubcommand {
   Bundle {
@@ -65,6 +56,12 @@ pub enum DenoSubcommand {
   Run {
     script: String,
   },
+  Test {
+    fail_fast: bool,
+    quiet: bool,
+    allow_none: bool,
+    include: Option<Vec<String>>,
+  },
   Types,
 }
 
@@ -495,40 +492,31 @@ fn run_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
 }
 
 fn test_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
-  flags.subcommand = DenoSubcommand::Run {
-    script: TEST_RUNNER_URL.to_string(),
-  };
   flags.allow_read = true;
 
   run_test_args_parse(flags, matches);
 
-  if matches.is_present("quiet") {
-    flags.argv.push("--quiet".to_string());
-  }
+  let quiet = matches.is_present("quiet");
+  let failfast = matches.is_present("failfast");
+  let allow_none = matches.is_present("allow_none");
 
-  if matches.is_present("failfast") {
-    flags.argv.push("--failfast".to_string());
-  }
-
-  if matches.is_present("exclude") {
-    flags.argv.push("--exclude".to_string());
-    let exclude: Vec<String> = matches
-      .values_of("exclude")
-      .unwrap()
-      .map(String::from)
-      .collect();
-    flags.argv.extend(exclude);
-  }
-
-  if matches.is_present("files") {
-    flags.argv.push("--".to_string());
+  let include = if matches.is_present("files") {
     let files: Vec<String> = matches
       .values_of("files")
       .unwrap()
       .map(String::from)
       .collect();
-    flags.argv.extend(files);
-  }
+    Some(files)
+  } else {
+    None
+  };
+
+  flags.subcommand = DenoSubcommand::Test {
+    quiet,
+    fail_fast: failfast,
+    include,
+    allow_none,
+  };
 }
 
 fn types_subcommand<'a, 'b>() -> App<'a, 'b> {
@@ -857,11 +845,10 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
         .takes_value(false),
     )
     .arg(
-      Arg::with_name("exclude")
-        .short("e")
-        .long("exclude")
-        .help("List of file names to exclude from run")
-        .takes_value(true),
+      Arg::with_name("allow_none")
+        .long("allow-none")
+        .help("Don't return error code if no test files are found")
+        .takes_value(false),
     )
     .arg(
       Arg::with_name("files")
@@ -2041,45 +2028,25 @@ mod tests {
     );
   }
 
-  #[test]
-  fn test_with_exclude() {
-    let r = flags_from_vec_safe(svec![
-      "deno",
-      "test",
-      "--exclude",
-      "some_dir/",
-      "dir1/",
-      "dir2/"
-    ]);
-    assert_eq!(
-      r.unwrap(),
-      DenoFlags {
-        subcommand: DenoSubcommand::Run {
-          script: TEST_RUNNER_URL.to_string(),
-        },
-        argv: svec!["--exclude", "some_dir/", "--", "dir1/", "dir2/"],
-        allow_read: true,
-        ..DenoFlags::default()
-      }
-    );
-  }
-
   #[test]
   fn test_with_allow_net() {
     let r = flags_from_vec_safe(svec![
       "deno",
       "test",
       "--allow-net",
+      "--allow-none",
       "dir1/",
       "dir2/"
     ]);
     assert_eq!(
       r.unwrap(),
       DenoFlags {
-        subcommand: DenoSubcommand::Run {
-          script: TEST_RUNNER_URL.to_string(),
+        subcommand: DenoSubcommand::Test {
+          fail_fast: false,
+          quiet: false,
+          allow_none: true,
+          include: Some(svec!["dir1/", "dir2/"]),
         },
-        argv: svec!["--", "dir1/", "dir2/"],
         allow_read: true,
         allow_net: true,
         ..DenoFlags::default()

cli/installer.rs

@@ -22,7 +22,7 @@ lazy_static! {
     ).case_insensitive(true).build().unwrap();
 }
 
-fn is_remote_url(module_url: &str) -> bool {
+pub fn is_remote_url(module_url: &str) -> bool {
   module_url.starts_with("http://") || module_url.starts_with("https://")
 }
 

cli/js/colors.ts

@@ -31,6 +31,10 @@ export function bold(str: string): string {
   return run(str, code(1, 22));
 }
 
+export function italic(str: string): string {
+  return run(str, code(3, 23));
+}
+
 export function yellow(str: string): string {
   return run(str, code(33, 39));
 }
@@ -38,3 +42,23 @@ export function yellow(str: string): string {
 export function cyan(str: string): string {
   return run(str, code(36, 39));
 }
+
+export function red(str: string): string {
+  return run(str, code(31, 39));
+}
+
+export function green(str: string): string {
+  return run(str, code(32, 39));
+}
+
+export function bgRed(str: string): string {
+  return run(str, code(41, 49));
+}
+
+export function white(str: string): string {
+  return run(str, code(37, 39));
+}
+
+export function gray(str: string): string {
+  return run(str, code(90, 39));
+}

cli/js/deno.ts

@@ -111,6 +111,7 @@ export { utimeSync, utime } from "./utime.ts";
 export { version } from "./version.ts";
 export { writeFileSync, writeFile, WriteFileOptions } from "./write_file.ts";
 export const args: string[] = [];
+export { test, runTests } from "./testing.ts";
 
 // These are internal Deno APIs.  We are marking them as internal so they do not
 // appear in the runtime type library.

cli/js/globals.ts

@@ -59,6 +59,11 @@ declare global {
     thrown: any;
   }
 
+  interface ImportMeta {
+    url: string;
+    main: boolean;
+  }
+
   interface DenoCore {
     print(s: string, isErr?: boolean): void;
     dispatch(
@@ -137,6 +142,9 @@ declare global {
   // Assigned to `self` global - compiler
   var bootstrapTsCompilerRuntime: (() => void) | undefined;
   var bootstrapWasmCompilerRuntime: (() => void) | undefined;
+
+  var performance: performanceUtil.Performance;
+  var setTimeout: typeof timers.setTimeout;
   /* eslint-enable */
 }
 

cli/js/lib.deno.ns.d.ts

@@ -10,6 +10,26 @@ declare namespace Deno {
   /** Reflects the NO_COLOR environment variable: https://no-color.org/ */
   export let noColor: boolean;
 
+  export type TestFunction = () => void | Promise<void>;
+
+  export interface TestDefinition {
+    fn: TestFunction;
+    name: string;
+  }
+
+  export function test(t: TestDefinition): void;
+  export function test(fn: TestFunction): void;
+  export function test(name: string, fn: TestFunction): void;
+
+  export interface RunTestsOptions {
+    exitOnFail?: boolean;
+    only?: RegExp;
+    skip?: RegExp;
+    disableLog?: boolean;
+  }
+
+  export function runTests(opts?: RunTestsOptions): Promise<void>;
+
   /** Check if running in terminal.
    *
    *       console.log(Deno.isTTY().stdout);

cli/js/net_test.ts

@@ -1,6 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 import { testPerm, assert, assertEquals } from "./test_util.ts";
-import { runIfMain } from "../../std/testing/mod.ts";
 
 testPerm({ net: true }, function netListenClose(): void {
   const listener = Deno.listen({ hostname: "127.0.0.1", port: 4500 });
@@ -240,5 +239,3 @@ testPerm({ net: true }, async function netDoubleCloseWrite() {
   conn.close();
 });
 */
-
-runIfMain(import.meta);

cli/js/test_util.ts

@@ -7,7 +7,6 @@
 // tests by the special string. permW1N0 means allow-write but not allow-net.
 // See tools/unit_tests.py for more details.
 
-import * as testing from "../../std/testing/mod.ts";
 import { assert, assertEquals } from "../../std/testing/asserts.ts";
 export {
   assert,
@@ -103,10 +102,7 @@ function normalizeTestPermissions(perms: TestPermissions): Permissions {
   };
 }
 
-export function testPerm(
-  perms: TestPermissions,
-  fn: testing.TestFunction
-): void {
+export function testPerm(perms: TestPermissions, fn: Deno.TestFunction): void {
   const normalizedPerms = normalizeTestPermissions(perms);
 
   registerPermCombination(normalizedPerms);
@@ -115,10 +111,10 @@ export function testPerm(
     return;
   }
 
-  testing.test(fn);
+  Deno.test(fn);
 }
 
-export function test(fn: testing.TestFunction): void {
+export function test(fn: Deno.TestFunction): void {
   testPerm(
     {
       read: false,

cli/js/testing.ts

@@ -0,0 +1,207 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+import { red, green, bgRed, bold, white, gray, italic } from "./colors.ts";
+import { exit } from "./os.ts";
+import { Console } from "./console.ts";
+
+function formatTestTime(time = 0): string {
+  return `${time.toFixed(2)}ms`;
+}
+
+function promptTestTime(time = 0, displayWarning = false): string {
+  // if time > 5s we display a warning
+  // only for test time, not the full runtime
+  if (displayWarning && time >= 5000) {
+    return bgRed(white(bold(`(${formatTestTime(time)})`)));
+  } else {
+    return gray(italic(`(${formatTestTime(time)})`));
+  }
+}
+
+export type TestFunction = () => void | Promise<void>;
+
+export interface TestDefinition {
+  fn: TestFunction;
+  name: string;
+}
+
+declare global {
+  // Only `var` variables show up in the `globalThis` type when doing a global
+  // scope augmentation.
+  // eslint-disable-next-line no-var
+  var __DENO_TEST_REGISTRY: TestDefinition[];
+}
+
+let TEST_REGISTRY: TestDefinition[] = [];
+if (globalThis["__DENO_TEST_REGISTRY"]) {
+  TEST_REGISTRY = globalThis.__DENO_TEST_REGISTRY as TestDefinition[];
+} else {
+  Object.defineProperty(globalThis, "__DENO_TEST_REGISTRY", {
+    enumerable: false,
+    value: TEST_REGISTRY
+  });
+}
+
+export function test(t: TestDefinition): void;
+export function test(fn: TestFunction): void;
+export function test(name: string, fn: TestFunction): void;
+// Main test function provided by Deno, as you can see it merely
+// creates a new object with "name" and "fn" fields.
+export function test(
+  t: string | TestDefinition | TestFunction,
+  fn?: TestFunction
+): void {
+  let name: string;
+
+  if (typeof t === "string") {
+    if (!fn) {
+      throw new Error("Missing test function");
+    }
+    name = t;
+    if (!name) {
+      throw new Error("The name of test case can't be empty");
+    }
+  } else if (typeof t === "function") {
+    fn = t;
+    name = t.name;
+    if (!name) {
+      throw new Error("Test function can't be anonymous");
+    }
+  } else {
+    fn = t.fn;
+    if (!fn) {
+      throw new Error("Missing test function");
+    }
+    name = t.name;
+    if (!name) {
+      throw new Error("The name of test case can't be empty");
+    }
+  }
+
+  TEST_REGISTRY.push({ fn, name });
+}
+
+interface TestStats {
+  filtered: number;
+  ignored: number;
+  measured: number;
+  passed: number;
+  failed: number;
+}
+
+interface TestCase {
+  name: string;
+  fn: TestFunction;
+  timeElapsed?: number;
+  error?: Error;
+}
+
+export interface RunTestsOptions {
+  exitOnFail?: boolean;
+  only?: RegExp;
+  skip?: RegExp;
+  disableLog?: boolean;
+}
+
+export async function runTests({
+  exitOnFail = false,
+  only = /[^\s]/,
+  skip = /^\s*$/,
+  disableLog = false
+}: RunTestsOptions = {}): Promise<void> {
+  const testsToRun = TEST_REGISTRY.filter(
+    ({ name }): boolean => only.test(name) && !skip.test(name)
+  );
+
+  const stats: TestStats = {
+    measured: 0,
+    ignored: 0,
+    filtered: 0,
+    passed: 0,
+    failed: 0
+  };
+
+  const testCases = testsToRun.map(
+    ({ name, fn }): TestCase => {
+      return {
+        name,
+        fn,
+        timeElapsed: 0,
+        error: undefined
+      };
+    }
+  );
+
+  // @ts-ignore
+  const originalConsole = globalThis.console;
+  // TODO(bartlomieju): add option to capture output of test
+  // cases and display it if test fails (like --nopcature in Rust)
+  const disabledConsole = new Console(
+    (_x: string, _isErr?: boolean): void => {}
+  );
+
+  if (disableLog) {
+    // @ts-ignore
+    globalThis.console = disabledConsole;
+  }
+
+  const RED_FAILED = red("FAILED");
+  const GREEN_OK = green("OK");
+  const RED_BG_FAIL = bgRed(" FAIL ");
+
+  originalConsole.log(`running ${testsToRun.length} tests`);
+  const suiteStart = performance.now();
+
+  for (const testCase of testCases) {
+    try {
+      const start = performance.now();
+      await testCase.fn();
+      const end = performance.now();
+      testCase.timeElapsed = end - start;
+      originalConsole.log(
+        `${GREEN_OK}     ${testCase.name} ${promptTestTime(end - start, true)}`
+      );
+      stats.passed++;
+    } catch (err) {
+      testCase.error = err;
+      originalConsole.log(`${RED_FAILED} ${testCase.name}`);
+      originalConsole.log(err.stack);
+      stats.failed++;
+      if (exitOnFail) {
+        break;
+      }
+    }
+  }
+
+  const suiteEnd = performance.now();
+
+  if (disableLog) {
+    // @ts-ignore
+    globalThis.console = originalConsole;
+  }
+
+  // Attempting to match the output of Rust's test runner.
+  originalConsole.log(
+    `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` +
+      `${stats.passed} passed; ${stats.failed} failed; ` +
+      `${stats.ignored} ignored; ${stats.measured} measured; ` +
+      `${stats.filtered} filtered out ` +
+      `${promptTestTime(suiteEnd - suiteStart)}\n`
+  );
+
+  // TODO(bartlomieju): what's it for? Do we really need, maybe add handler for unhandled
+  // promise to avoid such shenanigans
+  if (stats.failed) {
+    // Use setTimeout to avoid the error being ignored due to unhandled
+    // promise rejections being swallowed.
+    setTimeout((): void => {
+      originalConsole.error(`There were ${stats.failed} test failures.`);
+      testCases
+        .filter(testCase => !!testCase.error)
+        .forEach(testCase => {
+          originalConsole.error(`${RED_BG_FAIL} ${red(testCase.name)}`);
+          originalConsole.error(testCase.error);
+        });
+      exit(1);
+    }, 0);
+  }
+}

cli/js/tls_test.ts

@@ -2,7 +2,6 @@
 import { test, testPerm, assert, assertEquals } from "./test_util.ts";
 import { BufWriter, BufReader } from "../../std/io/bufio.ts";
 import { TextProtoReader } from "../../std/textproto/mod.ts";
-import { runIfMain } from "../../std/testing/mod.ts";
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
@@ -202,5 +201,3 @@ testPerm({ read: true, net: true }, async function dialAndListenTLS(): Promise<
   assertEquals(decoder.decode(bodyBuf), "Hello World\n");
   conn.close();
 });
-
-runIfMain(import.meta);

cli/js/unit_tests.ts

@@ -61,11 +61,6 @@ import "./permissions_test.ts";
 import "./version_test.ts";
 import "./workers_test.ts";
 
-import { runIfMain } from "../../std/testing/mod.ts";
-
-async function main(): Promise<void> {
-  // Testing entire test suite serially
-  runIfMain(import.meta);
+if (import.meta.main) {
+  await Deno.runTests();
 }
-
-main();

cli/lib.rs

@@ -50,6 +50,7 @@ pub mod signal;
 pub mod source_maps;
 mod startup_data;
 pub mod state;
+mod test_runner;
 pub mod test_util;
 mod tokio_util;
 pub mod version;
@@ -59,6 +60,7 @@ pub mod worker;
 use crate::compilers::TargetLib;
 use crate::deno_error::js_check;
 use crate::deno_error::{print_err_and_exit, print_msg_and_exit};
+use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
 use crate::ops::io::get_stdio;
 use crate::state::State;
@@ -72,6 +74,7 @@ use log::Level;
 use log::Metadata;
 use log::Record;
 use std::env;
+use std::fs as std_fs;
 use std::path::PathBuf;
 
 static LOGGER: Logger = Logger;
@@ -374,18 +377,10 @@ async fn run_repl(flags: DenoFlags) {
   }
 }
 
-async fn run_script(flags: DenoFlags, script: String) {
-  let main_module = ModuleSpecifier::resolve_url_or_path(&script).unwrap();
-  let global_state = create_global_state(flags);
-  let mut worker =
-    create_main_worker(global_state.clone(), main_module.clone());
-
-  // Setup runtime.
-  js_check(worker.execute("bootstrapMainRuntime()"));
-  debug!("main_module {}", main_module);
-
-  let mod_result = worker.execute_mod_async(&main_module, None, false).await;
-  if let Err(err) = mod_result {
+async fn run_command(flags: DenoFlags, script: String) {
+  let global_state = create_global_state(flags.clone());
+  let result = run_script(global_state.clone(), script).await;
+  if let Err(err) = result {
     print_err_and_exit(err);
   }
   if global_state.flags.lock_write {
@@ -399,10 +394,23 @@ async fn run_script(flags: DenoFlags, script: String) {
       std::process::exit(11);
     }
   }
-  js_check(worker.execute("window.dispatchEvent(new Event('load'))"));
-  let result = (&mut *worker).await;
-  js_check(result);
-  js_check(worker.execute("window.dispatchEvent(new Event('unload'))"));
+}
+
+async fn run_script(
+  global_state: GlobalState,
+  script: String,
+) -> Result<(), ErrBox> {
+  let main_module = ModuleSpecifier::resolve_url_or_path(&script).unwrap();
+  let mut worker =
+    create_main_worker(global_state.clone(), main_module.clone());
+  // Setup runtime.
+  worker.execute("bootstrapMainRuntime()")?;
+  debug!("main_module {}", main_module);
+  worker.execute_mod_async(&main_module, None, false).await?;
+  worker.execute("window.dispatchEvent(new Event('load'))")?;
+  (&mut *worker).await?;
+  worker.execute("window.dispatchEvent(new Event('unload'))")?;
+  Ok(())
 }
 
 async fn fmt_command(files: Option<Vec<PathBuf>>, check: bool) {
@@ -411,6 +419,49 @@ async fn fmt_command(files: Option<Vec<PathBuf>>, check: bool) {
   }
 }
 
+async fn test_command(
+  flags: DenoFlags,
+  include: Option<Vec<String>>,
+  fail_fast: bool,
+  _quiet: bool,
+  allow_none: bool,
+) {
+  let global_state = create_global_state(flags.clone());
+  let cwd = std::env::current_dir().expect("No current directory");
+  let include = include.unwrap_or_else(|| vec![".".to_string()]);
+  let res = test_runner::prepare_test_modules_urls(include, cwd.clone());
+
+  let test_modules = match res {
+    Ok(modules) => modules,
+    Err(e) => return print_err_and_exit(e),
+  };
+  if test_modules.is_empty() {
+    println!("No matching test modules found");
+    if !allow_none {
+      std::process::exit(1);
+    }
+    return;
+  }
+
+  let test_file = test_runner::render_test_file(test_modules, fail_fast);
+  let test_file_path = cwd.join(".deno.test.ts");
+  deno_fs::write_file(&test_file_path, test_file.as_bytes(), 0o666)
+    .expect("Can't write test file");
+
+  let mut flags = flags.clone();
+  flags
+    .argv
+    .push(test_file_path.to_string_lossy().to_string());
+
+  let result =
+    run_script(global_state, test_file_path.to_string_lossy().to_string())
+      .await;
+  std_fs::remove_file(&test_file_path).expect("Failed to remove temp file");
+  if let Err(err) = result {
+    print_err_and_exit(err);
+  }
+}
+
 pub fn main() {
   #[cfg(windows)]
   ansi_term::enable_ansi_support().ok(); // For Windows 10
@@ -454,7 +505,13 @@ pub fn main() {
         force,
       } => install_command(flags, dir, exe_name, module_url, args, force).await,
       DenoSubcommand::Repl => run_repl(flags).await,
-      DenoSubcommand::Run { script } => run_script(flags, script).await,
+      DenoSubcommand::Run { script } => run_command(flags, script).await,
+      DenoSubcommand::Test {
+        quiet,
+        fail_fast,
+        include,
+        allow_none,
+      } => test_command(flags, include, fail_fast, quiet, allow_none).await,
       DenoSubcommand::Types => types_command(),
       _ => panic!("bad subcommand"),
     }

cli/test_runner.rs

@@ -0,0 +1,81 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+use crate::installer::is_remote_url;
+use deno_core::ErrBox;
+use std;
+use std::path::PathBuf;
+use url::Url;
+
+pub fn prepare_test_modules_urls(
+  include: Vec<String>,
+  root_path: PathBuf,
+) -> Result<Vec<Url>, ErrBox> {
+  let (include_paths, include_urls): (Vec<String>, Vec<String>) =
+    include.into_iter().partition(|n| !is_remote_url(n));
+
+  let mut prepared = vec![];
+
+  for path in include_paths {
+    let p = root_path.join(path).canonicalize()?;
+    let url = Url::from_file_path(p).unwrap();
+    prepared.push(url);
+  }
+
+  for remote_url in include_urls {
+    let url = Url::parse(&remote_url)?;
+    prepared.push(url);
+  }
+
+  Ok(prepared)
+}
+
+pub fn render_test_file(modules: Vec<Url>, fail_fast: bool) -> String {
+  let mut test_file = "".to_string();
+
+  for module in modules {
+    test_file.push_str(&format!("import \"{}\";\n", module.to_string()));
+  }
+
+  let run_tests_cmd =
+    format!("Deno.runTests({{ exitOnFail: {} }});\n", fail_fast);
+  test_file.push_str(&run_tests_cmd);
+
+  test_file
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::test_util;
+
+  #[test]
+  fn test_prepare_test_modules_urls() {
+    let test_data_path = test_util::root_path().join("cli/tests/subdir");
+    let mut matched_urls = prepare_test_modules_urls(
+      vec![
+        "https://example.com/colors_test.ts".to_string(),
+        "./mod1.ts".to_string(),
+        "./mod3.js".to_string(),
+        "subdir2/mod2.ts".to_string(),
+        "http://example.com/printf_test.ts".to_string(),
+      ],
+      test_data_path.clone(),
+    )
+    .unwrap();
+    let test_data_url =
+      Url::from_file_path(test_data_path).unwrap().to_string();
+
+    let expected: Vec<Url> = vec![
+      format!("{}/mod1.ts", test_data_url),
+      format!("{}/mod3.js", test_data_url),
+      format!("{}/subdir2/mod2.ts", test_data_url),
+      "http://example.com/printf_test.ts".to_string(),
+      "https://example.com/colors_test.ts".to_string(),
+    ]
+    .into_iter()
+    .map(|f| Url::parse(&f).unwrap())
+    .collect();
+    matched_urls.sort();
+    assert_eq!(matched_urls, expected);
+  }
+}