diff --git a/cli/flags.rs b/cli/flags.rs index 7f02d70661..5326dc2874 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -22,15 +22,6 @@ macro_rules! sset { }} } -macro_rules! std_url { - ($x:expr) => { - concat!("https://deno.land/std@v0.29.0/", $x) - }; -} - -/// Used for `deno test...` subcommand -const TEST_RUNNER_URL: &str = std_url!("testing/runner.ts"); - #[derive(Clone, Debug, PartialEq)] pub enum DenoSubcommand { Bundle { @@ -65,6 +56,12 @@ pub enum DenoSubcommand { Run { script: String, }, + Test { + fail_fast: bool, + quiet: bool, + allow_none: bool, + include: Option>, + }, Types, } @@ -495,40 +492,31 @@ fn run_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { } fn test_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) { - flags.subcommand = DenoSubcommand::Run { - script: TEST_RUNNER_URL.to_string(), - }; flags.allow_read = true; run_test_args_parse(flags, matches); - if matches.is_present("quiet") { - flags.argv.push("--quiet".to_string()); - } + let quiet = matches.is_present("quiet"); + let failfast = matches.is_present("failfast"); + let allow_none = matches.is_present("allow_none"); - if matches.is_present("failfast") { - flags.argv.push("--failfast".to_string()); - } - - if matches.is_present("exclude") { - flags.argv.push("--exclude".to_string()); - let exclude: Vec = matches - .values_of("exclude") - .unwrap() - .map(String::from) - .collect(); - flags.argv.extend(exclude); - } - - if matches.is_present("files") { - flags.argv.push("--".to_string()); + let include = if matches.is_present("files") { let files: Vec = matches .values_of("files") .unwrap() .map(String::from) .collect(); - flags.argv.extend(files); - } + Some(files) + } else { + None + }; + + flags.subcommand = DenoSubcommand::Test { + quiet, + fail_fast: failfast, + include, + allow_none, + }; } fn types_subcommand<'a, 'b>() -> App<'a, 'b> { @@ -857,11 +845,10 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> { .takes_value(false), ) .arg( - Arg::with_name("exclude") - .short("e") - .long("exclude") - .help("List of file names to exclude from run") - .takes_value(true), + Arg::with_name("allow_none") + .long("allow-none") + .help("Don't return error code if no test files are found") + .takes_value(false), ) .arg( Arg::with_name("files") @@ -2041,45 +2028,25 @@ mod tests { ); } - #[test] - fn test_with_exclude() { - let r = flags_from_vec_safe(svec![ - "deno", - "test", - "--exclude", - "some_dir/", - "dir1/", - "dir2/" - ]); - assert_eq!( - r.unwrap(), - DenoFlags { - subcommand: DenoSubcommand::Run { - script: TEST_RUNNER_URL.to_string(), - }, - argv: svec!["--exclude", "some_dir/", "--", "dir1/", "dir2/"], - allow_read: true, - ..DenoFlags::default() - } - ); - } - #[test] fn test_with_allow_net() { let r = flags_from_vec_safe(svec![ "deno", "test", "--allow-net", + "--allow-none", "dir1/", "dir2/" ]); assert_eq!( r.unwrap(), DenoFlags { - subcommand: DenoSubcommand::Run { - script: TEST_RUNNER_URL.to_string(), + subcommand: DenoSubcommand::Test { + fail_fast: false, + quiet: false, + allow_none: true, + include: Some(svec!["dir1/", "dir2/"]), }, - argv: svec!["--", "dir1/", "dir2/"], allow_read: true, allow_net: true, ..DenoFlags::default() diff --git a/cli/installer.rs b/cli/installer.rs index b1a795f99c..c7f32ce21d 100644 --- a/cli/installer.rs +++ b/cli/installer.rs @@ -22,7 +22,7 @@ lazy_static! { ).case_insensitive(true).build().unwrap(); } -fn is_remote_url(module_url: &str) -> bool { +pub fn is_remote_url(module_url: &str) -> bool { module_url.starts_with("http://") || module_url.starts_with("https://") } diff --git a/cli/js/colors.ts b/cli/js/colors.ts index 21539c83ec..372e90ba51 100644 --- a/cli/js/colors.ts +++ b/cli/js/colors.ts @@ -31,6 +31,10 @@ export function bold(str: string): string { return run(str, code(1, 22)); } +export function italic(str: string): string { + return run(str, code(3, 23)); +} + export function yellow(str: string): string { return run(str, code(33, 39)); } @@ -38,3 +42,23 @@ export function yellow(str: string): string { export function cyan(str: string): string { return run(str, code(36, 39)); } + +export function red(str: string): string { + return run(str, code(31, 39)); +} + +export function green(str: string): string { + return run(str, code(32, 39)); +} + +export function bgRed(str: string): string { + return run(str, code(41, 49)); +} + +export function white(str: string): string { + return run(str, code(37, 39)); +} + +export function gray(str: string): string { + return run(str, code(90, 39)); +} diff --git a/cli/js/deno.ts b/cli/js/deno.ts index 077b198c4e..c52e6dc2d4 100644 --- a/cli/js/deno.ts +++ b/cli/js/deno.ts @@ -111,6 +111,7 @@ export { utimeSync, utime } from "./utime.ts"; export { version } from "./version.ts"; export { writeFileSync, writeFile, WriteFileOptions } from "./write_file.ts"; export const args: string[] = []; +export { test, runTests } from "./testing.ts"; // These are internal Deno APIs. We are marking them as internal so they do not // appear in the runtime type library. diff --git a/cli/js/globals.ts b/cli/js/globals.ts index 53eb696ac0..9a7161ff05 100644 --- a/cli/js/globals.ts +++ b/cli/js/globals.ts @@ -59,6 +59,11 @@ declare global { thrown: any; } + interface ImportMeta { + url: string; + main: boolean; + } + interface DenoCore { print(s: string, isErr?: boolean): void; dispatch( @@ -137,6 +142,9 @@ declare global { // Assigned to `self` global - compiler var bootstrapTsCompilerRuntime: (() => void) | undefined; var bootstrapWasmCompilerRuntime: (() => void) | undefined; + + var performance: performanceUtil.Performance; + var setTimeout: typeof timers.setTimeout; /* eslint-enable */ } diff --git a/cli/js/lib.deno.ns.d.ts b/cli/js/lib.deno.ns.d.ts index 4bbbf63202..b96e108c3b 100644 --- a/cli/js/lib.deno.ns.d.ts +++ b/cli/js/lib.deno.ns.d.ts @@ -10,6 +10,26 @@ declare namespace Deno { /** Reflects the NO_COLOR environment variable: https://no-color.org/ */ export let noColor: boolean; + export type TestFunction = () => void | Promise; + + export interface TestDefinition { + fn: TestFunction; + name: string; + } + + export function test(t: TestDefinition): void; + export function test(fn: TestFunction): void; + export function test(name: string, fn: TestFunction): void; + + export interface RunTestsOptions { + exitOnFail?: boolean; + only?: RegExp; + skip?: RegExp; + disableLog?: boolean; + } + + export function runTests(opts?: RunTestsOptions): Promise; + /** Check if running in terminal. * * console.log(Deno.isTTY().stdout); diff --git a/cli/js/net_test.ts b/cli/js/net_test.ts index 7f5334df33..e4d0be81f9 100644 --- a/cli/js/net_test.ts +++ b/cli/js/net_test.ts @@ -1,6 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { testPerm, assert, assertEquals } from "./test_util.ts"; -import { runIfMain } from "../../std/testing/mod.ts"; testPerm({ net: true }, function netListenClose(): void { const listener = Deno.listen({ hostname: "127.0.0.1", port: 4500 }); @@ -240,5 +239,3 @@ testPerm({ net: true }, async function netDoubleCloseWrite() { conn.close(); }); */ - -runIfMain(import.meta); diff --git a/cli/js/test_util.ts b/cli/js/test_util.ts index a546fa5c5e..dbb7bf2c47 100644 --- a/cli/js/test_util.ts +++ b/cli/js/test_util.ts @@ -7,7 +7,6 @@ // tests by the special string. permW1N0 means allow-write but not allow-net. // See tools/unit_tests.py for more details. -import * as testing from "../../std/testing/mod.ts"; import { assert, assertEquals } from "../../std/testing/asserts.ts"; export { assert, @@ -103,10 +102,7 @@ function normalizeTestPermissions(perms: TestPermissions): Permissions { }; } -export function testPerm( - perms: TestPermissions, - fn: testing.TestFunction -): void { +export function testPerm(perms: TestPermissions, fn: Deno.TestFunction): void { const normalizedPerms = normalizeTestPermissions(perms); registerPermCombination(normalizedPerms); @@ -115,10 +111,10 @@ export function testPerm( return; } - testing.test(fn); + Deno.test(fn); } -export function test(fn: testing.TestFunction): void { +export function test(fn: Deno.TestFunction): void { testPerm( { read: false, diff --git a/cli/js/testing.ts b/cli/js/testing.ts new file mode 100644 index 0000000000..b4c86e8b88 --- /dev/null +++ b/cli/js/testing.ts @@ -0,0 +1,207 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { red, green, bgRed, bold, white, gray, italic } from "./colors.ts"; +import { exit } from "./os.ts"; +import { Console } from "./console.ts"; + +function formatTestTime(time = 0): string { + return `${time.toFixed(2)}ms`; +} + +function promptTestTime(time = 0, displayWarning = false): string { + // if time > 5s we display a warning + // only for test time, not the full runtime + if (displayWarning && time >= 5000) { + return bgRed(white(bold(`(${formatTestTime(time)})`))); + } else { + return gray(italic(`(${formatTestTime(time)})`)); + } +} + +export type TestFunction = () => void | Promise; + +export interface TestDefinition { + fn: TestFunction; + name: string; +} + +declare global { + // Only `var` variables show up in the `globalThis` type when doing a global + // scope augmentation. + // eslint-disable-next-line no-var + var __DENO_TEST_REGISTRY: TestDefinition[]; +} + +let TEST_REGISTRY: TestDefinition[] = []; +if (globalThis["__DENO_TEST_REGISTRY"]) { + TEST_REGISTRY = globalThis.__DENO_TEST_REGISTRY as TestDefinition[]; +} else { + Object.defineProperty(globalThis, "__DENO_TEST_REGISTRY", { + enumerable: false, + value: TEST_REGISTRY + }); +} + +export function test(t: TestDefinition): void; +export function test(fn: TestFunction): void; +export function test(name: string, fn: TestFunction): void; +// Main test function provided by Deno, as you can see it merely +// creates a new object with "name" and "fn" fields. +export function test( + t: string | TestDefinition | TestFunction, + fn?: TestFunction +): void { + let name: string; + + if (typeof t === "string") { + if (!fn) { + throw new Error("Missing test function"); + } + name = t; + if (!name) { + throw new Error("The name of test case can't be empty"); + } + } else if (typeof t === "function") { + fn = t; + name = t.name; + if (!name) { + throw new Error("Test function can't be anonymous"); + } + } else { + fn = t.fn; + if (!fn) { + throw new Error("Missing test function"); + } + name = t.name; + if (!name) { + throw new Error("The name of test case can't be empty"); + } + } + + TEST_REGISTRY.push({ fn, name }); +} + +interface TestStats { + filtered: number; + ignored: number; + measured: number; + passed: number; + failed: number; +} + +interface TestCase { + name: string; + fn: TestFunction; + timeElapsed?: number; + error?: Error; +} + +export interface RunTestsOptions { + exitOnFail?: boolean; + only?: RegExp; + skip?: RegExp; + disableLog?: boolean; +} + +export async function runTests({ + exitOnFail = false, + only = /[^\s]/, + skip = /^\s*$/, + disableLog = false +}: RunTestsOptions = {}): Promise { + const testsToRun = TEST_REGISTRY.filter( + ({ name }): boolean => only.test(name) && !skip.test(name) + ); + + const stats: TestStats = { + measured: 0, + ignored: 0, + filtered: 0, + passed: 0, + failed: 0 + }; + + const testCases = testsToRun.map( + ({ name, fn }): TestCase => { + return { + name, + fn, + timeElapsed: 0, + error: undefined + }; + } + ); + + // @ts-ignore + const originalConsole = globalThis.console; + // TODO(bartlomieju): add option to capture output of test + // cases and display it if test fails (like --nopcature in Rust) + const disabledConsole = new Console( + (_x: string, _isErr?: boolean): void => {} + ); + + if (disableLog) { + // @ts-ignore + globalThis.console = disabledConsole; + } + + const RED_FAILED = red("FAILED"); + const GREEN_OK = green("OK"); + const RED_BG_FAIL = bgRed(" FAIL "); + + originalConsole.log(`running ${testsToRun.length} tests`); + const suiteStart = performance.now(); + + for (const testCase of testCases) { + try { + const start = performance.now(); + await testCase.fn(); + const end = performance.now(); + testCase.timeElapsed = end - start; + originalConsole.log( + `${GREEN_OK} ${testCase.name} ${promptTestTime(end - start, true)}` + ); + stats.passed++; + } catch (err) { + testCase.error = err; + originalConsole.log(`${RED_FAILED} ${testCase.name}`); + originalConsole.log(err.stack); + stats.failed++; + if (exitOnFail) { + break; + } + } + } + + const suiteEnd = performance.now(); + + if (disableLog) { + // @ts-ignore + globalThis.console = originalConsole; + } + + // Attempting to match the output of Rust's test runner. + originalConsole.log( + `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` + + `${stats.passed} passed; ${stats.failed} failed; ` + + `${stats.ignored} ignored; ${stats.measured} measured; ` + + `${stats.filtered} filtered out ` + + `${promptTestTime(suiteEnd - suiteStart)}\n` + ); + + // TODO(bartlomieju): what's it for? Do we really need, maybe add handler for unhandled + // promise to avoid such shenanigans + if (stats.failed) { + // Use setTimeout to avoid the error being ignored due to unhandled + // promise rejections being swallowed. + setTimeout((): void => { + originalConsole.error(`There were ${stats.failed} test failures.`); + testCases + .filter(testCase => !!testCase.error) + .forEach(testCase => { + originalConsole.error(`${RED_BG_FAIL} ${red(testCase.name)}`); + originalConsole.error(testCase.error); + }); + exit(1); + }, 0); + } +} diff --git a/cli/js/tls_test.ts b/cli/js/tls_test.ts index de841f5a1a..1273da34f7 100644 --- a/cli/js/tls_test.ts +++ b/cli/js/tls_test.ts @@ -2,7 +2,6 @@ import { test, testPerm, assert, assertEquals } from "./test_util.ts"; import { BufWriter, BufReader } from "../../std/io/bufio.ts"; import { TextProtoReader } from "../../std/textproto/mod.ts"; -import { runIfMain } from "../../std/testing/mod.ts"; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -202,5 +201,3 @@ testPerm({ read: true, net: true }, async function dialAndListenTLS(): Promise< assertEquals(decoder.decode(bodyBuf), "Hello World\n"); conn.close(); }); - -runIfMain(import.meta); diff --git a/cli/js/unit_tests.ts b/cli/js/unit_tests.ts index a6435d1831..3c808fe462 100644 --- a/cli/js/unit_tests.ts +++ b/cli/js/unit_tests.ts @@ -61,11 +61,6 @@ import "./permissions_test.ts"; import "./version_test.ts"; import "./workers_test.ts"; -import { runIfMain } from "../../std/testing/mod.ts"; - -async function main(): Promise { - // Testing entire test suite serially - runIfMain(import.meta); +if (import.meta.main) { + await Deno.runTests(); } - -main(); diff --git a/cli/lib.rs b/cli/lib.rs index f6d12d21d6..f89eb4bd74 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -50,6 +50,7 @@ pub mod signal; pub mod source_maps; mod startup_data; pub mod state; +mod test_runner; pub mod test_util; mod tokio_util; pub mod version; @@ -59,6 +60,7 @@ pub mod worker; use crate::compilers::TargetLib; use crate::deno_error::js_check; use crate::deno_error::{print_err_and_exit, print_msg_and_exit}; +use crate::fs as deno_fs; use crate::global_state::GlobalState; use crate::ops::io::get_stdio; use crate::state::State; @@ -72,6 +74,7 @@ use log::Level; use log::Metadata; use log::Record; use std::env; +use std::fs as std_fs; use std::path::PathBuf; static LOGGER: Logger = Logger; @@ -374,18 +377,10 @@ async fn run_repl(flags: DenoFlags) { } } -async fn run_script(flags: DenoFlags, script: String) { - let main_module = ModuleSpecifier::resolve_url_or_path(&script).unwrap(); - let global_state = create_global_state(flags); - let mut worker = - create_main_worker(global_state.clone(), main_module.clone()); - - // Setup runtime. - js_check(worker.execute("bootstrapMainRuntime()")); - debug!("main_module {}", main_module); - - let mod_result = worker.execute_mod_async(&main_module, None, false).await; - if let Err(err) = mod_result { +async fn run_command(flags: DenoFlags, script: String) { + let global_state = create_global_state(flags.clone()); + let result = run_script(global_state.clone(), script).await; + if let Err(err) = result { print_err_and_exit(err); } if global_state.flags.lock_write { @@ -399,10 +394,23 @@ async fn run_script(flags: DenoFlags, script: String) { std::process::exit(11); } } - js_check(worker.execute("window.dispatchEvent(new Event('load'))")); - let result = (&mut *worker).await; - js_check(result); - js_check(worker.execute("window.dispatchEvent(new Event('unload'))")); +} + +async fn run_script( + global_state: GlobalState, + script: String, +) -> Result<(), ErrBox> { + let main_module = ModuleSpecifier::resolve_url_or_path(&script).unwrap(); + let mut worker = + create_main_worker(global_state.clone(), main_module.clone()); + // Setup runtime. + worker.execute("bootstrapMainRuntime()")?; + debug!("main_module {}", main_module); + worker.execute_mod_async(&main_module, None, false).await?; + worker.execute("window.dispatchEvent(new Event('load'))")?; + (&mut *worker).await?; + worker.execute("window.dispatchEvent(new Event('unload'))")?; + Ok(()) } async fn fmt_command(files: Option>, check: bool) { @@ -411,6 +419,49 @@ async fn fmt_command(files: Option>, check: bool) { } } +async fn test_command( + flags: DenoFlags, + include: Option>, + fail_fast: bool, + _quiet: bool, + allow_none: bool, +) { + let global_state = create_global_state(flags.clone()); + let cwd = std::env::current_dir().expect("No current directory"); + let include = include.unwrap_or_else(|| vec![".".to_string()]); + let res = test_runner::prepare_test_modules_urls(include, cwd.clone()); + + let test_modules = match res { + Ok(modules) => modules, + Err(e) => return print_err_and_exit(e), + }; + if test_modules.is_empty() { + println!("No matching test modules found"); + if !allow_none { + std::process::exit(1); + } + return; + } + + let test_file = test_runner::render_test_file(test_modules, fail_fast); + let test_file_path = cwd.join(".deno.test.ts"); + deno_fs::write_file(&test_file_path, test_file.as_bytes(), 0o666) + .expect("Can't write test file"); + + let mut flags = flags.clone(); + flags + .argv + .push(test_file_path.to_string_lossy().to_string()); + + let result = + run_script(global_state, test_file_path.to_string_lossy().to_string()) + .await; + std_fs::remove_file(&test_file_path).expect("Failed to remove temp file"); + if let Err(err) = result { + print_err_and_exit(err); + } +} + pub fn main() { #[cfg(windows)] ansi_term::enable_ansi_support().ok(); // For Windows 10 @@ -454,7 +505,13 @@ pub fn main() { force, } => install_command(flags, dir, exe_name, module_url, args, force).await, DenoSubcommand::Repl => run_repl(flags).await, - DenoSubcommand::Run { script } => run_script(flags, script).await, + DenoSubcommand::Run { script } => run_command(flags, script).await, + DenoSubcommand::Test { + quiet, + fail_fast, + include, + allow_none, + } => test_command(flags, include, fail_fast, quiet, allow_none).await, DenoSubcommand::Types => types_command(), _ => panic!("bad subcommand"), } diff --git a/cli/test_runner.rs b/cli/test_runner.rs new file mode 100644 index 0000000000..e05013889d --- /dev/null +++ b/cli/test_runner.rs @@ -0,0 +1,81 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::installer::is_remote_url; +use deno_core::ErrBox; +use std; +use std::path::PathBuf; +use url::Url; + +pub fn prepare_test_modules_urls( + include: Vec, + root_path: PathBuf, +) -> Result, ErrBox> { + let (include_paths, include_urls): (Vec, Vec) = + include.into_iter().partition(|n| !is_remote_url(n)); + + let mut prepared = vec![]; + + for path in include_paths { + let p = root_path.join(path).canonicalize()?; + let url = Url::from_file_path(p).unwrap(); + prepared.push(url); + } + + for remote_url in include_urls { + let url = Url::parse(&remote_url)?; + prepared.push(url); + } + + Ok(prepared) +} + +pub fn render_test_file(modules: Vec, fail_fast: bool) -> String { + let mut test_file = "".to_string(); + + for module in modules { + test_file.push_str(&format!("import \"{}\";\n", module.to_string())); + } + + let run_tests_cmd = + format!("Deno.runTests({{ exitOnFail: {} }});\n", fail_fast); + test_file.push_str(&run_tests_cmd); + + test_file +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_util; + + #[test] + fn test_prepare_test_modules_urls() { + let test_data_path = test_util::root_path().join("cli/tests/subdir"); + let mut matched_urls = prepare_test_modules_urls( + vec![ + "https://example.com/colors_test.ts".to_string(), + "./mod1.ts".to_string(), + "./mod3.js".to_string(), + "subdir2/mod2.ts".to_string(), + "http://example.com/printf_test.ts".to_string(), + ], + test_data_path.clone(), + ) + .unwrap(); + let test_data_url = + Url::from_file_path(test_data_path).unwrap().to_string(); + + let expected: Vec = vec![ + format!("{}/mod1.ts", test_data_url), + format!("{}/mod3.js", test_data_url), + format!("{}/subdir2/mod2.ts", test_data_url), + "http://example.com/printf_test.ts".to_string(), + "https://example.com/colors_test.ts".to_string(), + ] + .into_iter() + .map(|f| Url::parse(&f).unwrap()) + .collect(); + matched_urls.sort(); + assert_eq!(matched_urls, expected); + } +}