From 32855f2c853b7f30a92d4cb95549cba8abfd44e5 Mon Sep 17 00:00:00 2001 From: "Yasser A.Idrissi" Date: Mon, 12 Jul 2021 11:55:42 +0100 Subject: [PATCH] feat: Add support for "deno test --fail-fast=N" (#11316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for specifying threshold in the "--fail-fast" flag for "deno test" subcommand. Previously using "--fail-fast" stopped running the test suite after first failure and with this change users may specify number of failed tests that will cause the suite to be interrupted. Co-authored-by: Bartek Iwańczuk --- cli/flags.rs | 57 ++++++++++++++++++++++++--- cli/main.rs | 2 +- cli/tests/test/fail_fast_with_val.out | 23 +++++++++++ cli/tests/test/fail_fast_with_val.ts | 30 ++++++++++++++ cli/tools/test_runner.rs | 10 +++-- 5 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 cli/tests/test/fail_fast_with_val.out create mode 100644 cli/tests/test/fail_fast_with_val.ts diff --git a/cli/flags.rs b/cli/flags.rs index 400798cbd1..2a66cb0ea4 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -98,7 +98,7 @@ pub enum DenoSubcommand { Test { doc: bool, no_run: bool, - fail_fast: bool, + fail_fast: Option, quiet: bool, allow_none: bool, include: Option>, @@ -1001,8 +1001,23 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> { Arg::with_name("fail-fast") .long("fail-fast") .alias("failfast") - .help("Stop on first error") - .takes_value(false), + .help("Stop after N errors. Defaults to stopping after first failure.") + .min_values(0) + .required(false) + .takes_value(true) + .require_equals(true) + .value_name("N") + .validator(|val: String| match val.parse::() { + Ok(val) => { + if val == 0 { + return Err( + "fail-fast should be an number greater than 0".to_string(), + ); + } + Ok(()) + } + Err(_) => Err("fail-fast should be a number".to_string()), + }), ) .arg( Arg::with_name("allow-none") @@ -1696,11 +1711,20 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let no_run = matches.is_present("no-run"); let doc = matches.is_present("doc"); - let fail_fast = matches.is_present("fail-fast"); let allow_none = matches.is_present("allow-none"); let quiet = matches.is_present("quiet"); let filter = matches.value_of("filter").map(String::from); + let fail_fast = if matches.is_present("fail-fast") { + if let Some(value) = matches.value_of("fail-fast") { + Some(value.parse().unwrap()) + } else { + Some(1) + } + } else { + None + }; + let shuffle = if matches.is_present("shuffle") { let value = if let Some(value) = matches.value_of("shuffle") { value.parse::().unwrap() @@ -3387,7 +3411,7 @@ mod tests { subcommand: DenoSubcommand::Test { no_run: true, doc: false, - fail_fast: false, + fail_fast: None, filter: Some("- foo".to_string()), allow_none: true, quiet: false, @@ -3426,6 +3450,29 @@ mod tests { ); } + #[test] + fn test_with_fail_fast() { + let r = flags_from_vec(svec!["deno", "test", "--fail-fast=3"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Test { + no_run: false, + doc: false, + fail_fast: Some(3), + filter: None, + allow_none: false, + quiet: false, + shuffle: None, + include: None, + concurrent_jobs: 1, + terse: false + }, + ..Flags::default() + } + ); + } + #[test] fn bundle_with_cafile() { let r = flags_from_vec(svec![ diff --git a/cli/main.rs b/cli/main.rs index 77f8b34ba4..2170422370 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -987,7 +987,7 @@ async fn test_command( include: Option>, no_run: bool, doc: bool, - fail_fast: bool, + fail_fast: Option, quiet: bool, allow_none: bool, filter: Option, diff --git a/cli/tests/test/fail_fast_with_val.out b/cli/tests/test/fail_fast_with_val.out new file mode 100644 index 0000000000..a902e4c7bc --- /dev/null +++ b/cli/tests/test/fail_fast_with_val.out @@ -0,0 +1,23 @@ +[WILDCARD] +running 10 tests from [WILDCARD]/test/fail_fast_with_val.ts +test test 1 ... FAILED ([WILDCARD]) +test test 2 ... FAILED ([WILDCARD]) + +failures: + +test 1 +Error + at [WILDCARD]/test/fail_fast_with_val.ts:2:9 + at [WILDCARD] + +test 2 +Error + at [WILDCARD]/test/fail_fast_with_val.ts:5:9 + at [WILDCARD] + +failures: + + test 1 + test 2 + +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD]) diff --git a/cli/tests/test/fail_fast_with_val.ts b/cli/tests/test/fail_fast_with_val.ts new file mode 100644 index 0000000000..637e825ece --- /dev/null +++ b/cli/tests/test/fail_fast_with_val.ts @@ -0,0 +1,30 @@ +Deno.test("test 1", () => { + throw new Error(); +}); +Deno.test("test 2", () => { + throw new Error(); +}); +Deno.test("test 3", () => { + throw new Error(); +}); +Deno.test("test 4", () => { + throw new Error(); +}); +Deno.test("test 5", () => { + throw new Error(); +}); +Deno.test("test 6", () => { + throw new Error(); +}); +Deno.test("test 7", () => { + throw new Error(); +}); +Deno.test("test 8", () => { + throw new Error(); +}); +Deno.test("test 9", () => { + throw new Error(); +}); +Deno.test("test 0", () => { + throw new Error(); +}); diff --git a/cli/tools/test_runner.rs b/cli/tools/test_runner.rs index 432935aef2..733a3edd52 100644 --- a/cli/tools/test_runner.rs +++ b/cli/tools/test_runner.rs @@ -340,7 +340,7 @@ pub async fn run_tests( doc_modules: Vec, test_modules: Vec, no_run: bool, - fail_fast: bool, + fail_fast: Option, quiet: bool, allow_none: bool, filter: Option, @@ -515,6 +515,7 @@ pub async fn run_tests( let mut has_error = false; let mut planned = 0; let mut reported = 0; + let mut failed = 0; for event in receiver.iter() { match event.message.clone() { @@ -538,6 +539,7 @@ pub async fn run_tests( if let TestResult::Failed(_) = result { has_error = true; + failed += 1; } } _ => {} @@ -545,8 +547,10 @@ pub async fn run_tests( reporter.visit_event(event); - if has_error && fail_fast { - break; + if let Some(x) = fail_fast { + if failed >= x { + break; + } } }