fix: std/testing/runner.ts and deno test (#4392)

After splitting "failFast" and "exitOnFail" arguments, there was a situation where failing tests did not exit with code 1.

* fixed argument value passed to Deno.runTests() in deno test
* fixed argument value passed to Deno.runTests() in std/testing/runner.ts
* added integration tests for deno test to ensure failFast and exitOnFail work as expected
* don't write test file to file system, but keep it in memory

cli/lib.rs

@@ -58,7 +58,6 @@ pub mod worker;
 
 use crate::compilers::TargetLib;
 use crate::file_fetcher::SourceFile;
-use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
 use crate::msg::MediaType;
 use crate::ops::io::get_stdio;
@@ -74,7 +73,6 @@ use log::Level;
 use log::Metadata;
 use log::Record;
 use std::env;
-use std::fs as std_fs;
 use std::io::Write;
 use std::path::PathBuf;
 use url::Url;
@@ -392,23 +390,31 @@ async fn test_command(
     return Ok(());
   }
 
-  let test_file = test_runner::render_test_file(test_modules, fail_fast);
   let test_file_path = cwd.join(".deno.test.ts");
   let test_file_url =
     Url::from_file_path(&test_file_path).expect("Should be valid file url");
+  let test_file = test_runner::render_test_file(test_modules, fail_fast);
   let main_module =
     ModuleSpecifier::resolve_url(&test_file_url.to_string()).unwrap();
-  // First create worker with specified test file and only then write
-  // file to disk. Then test file will be executed and removed
-  // immediately after. That way even if compilation/tests fail test
-  // file can be cleaned up.
   let mut worker =
     create_main_worker(global_state.clone(), main_module.clone())?;
-  deno_fs::write_file(&test_file_path, test_file.as_bytes(), 0o666)
-    .expect("Can't write test file");
+  // Create a dummy source file.
+  let source_file = SourceFile {
+    filename: test_file_url.to_file_path().unwrap(),
+    url: test_file_url,
+    types_url: None,
+    media_type: MediaType::TypeScript,
+    source_code: test_file.clone().into_bytes(),
+  };
+  // Save our fake file into file fetcher cache
+  // to allow module access by TS compiler (e.g. op_fetch_source_files)
+  worker
+    .state
+    .borrow()
+    .global_state
+    .file_fetcher
+    .save_source_file_in_cache(&main_module, source_file);
   let execute_result = worker.execute_module(&main_module).await;
-  // Remove temporary test file
-  std_fs::remove_file(&test_file_path).expect("Failed to remove temp file");
   execute_result?;
   worker.execute("window.dispatchEvent(new Event('load'))")?;
   (&mut *worker).await?;

cli/test_runner.rs

@@ -69,7 +69,7 @@ pub fn render_test_file(modules: Vec<Url>, fail_fast: bool) -> String {
   }
 
   let run_tests_cmd =
-    format!("Deno.runTests({{ exitOnFail: {} }});\n", fail_fast);
+    format!("Deno.runTests({{ failFast: {} }});\n", fail_fast);
   test_file.push_str(&run_tests_cmd);
 
   test_file

cli/tests/deno_test.out

@@ -0,0 +1,25 @@
+running 4 tests
+test fail1 ... FAILED [WILDCARD]
+test fail2 ... FAILED [WILDCARD]
+test success1 ... ok [WILDCARD]
+test fail3 ... FAILED [WILDCARD]
+
+failures:
+
+fail1
+AssertionError: fail1 assertion
+[WILDCARD]
+
+fail2
+AssertionError: fail2 assertion
+[WILDCARD]
+
+fail3
+AssertionError: fail3 assertion
+[WILDCARD]
+
+failures:
+[WILDCARD]
+
+test result: FAILED. 1 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
+

cli/tests/deno_test_fail_fast.out

@@ -0,0 +1,14 @@
+running 4 tests
+test fail1 ... FAILED [WILDCARD]
+
+failures:
+
+fail1
+AssertionError: fail1 assertion
+[WILDCARD]
+
+failures:
+[WILDCARD]
+
+test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
+

cli/tests/integration_tests.rs

@@ -899,6 +899,18 @@ itest!(_026_redirect_javascript {
   http_server: true,
 });
 
+itest!(deno_test_fail_fast {
+  args: "test --failfast test_runner_test.ts",
+  exit_code: 1,
+  output: "deno_test_fail_fast.out",
+});
+
+itest!(deno_test {
+  args: "test test_runner_test.ts",
+  exit_code: 1,
+  output: "deno_test.out",
+});
+
 itest!(workers {
   args: "test --reload --allow-net workers_test.ts",
   http_server: true,

cli/tests/test_runner_test.ts

@@ -0,0 +1,19 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+import { assert } from "../../std/testing/asserts.ts";
+
+Deno.test(function fail1() {
+  assert(false, "fail1 assertion");
+});
+
+Deno.test(function fail2() {
+  assert(false, "fail2 assertion");
+});
+
+Deno.test(function success1() {
+  assert(true);
+});
+
+Deno.test(function fail3() {
+  assert(false, "fail3 assertion");
+});

std/testing/runner.ts

@@ -269,7 +269,6 @@ async function main(): Promise<void> {
   const exclude =
     parsedArgs.exclude != null ? (parsedArgs.exclude as string).split(",") : [];
   const allowNone = parsedArgs["allow-none"];
-  const exitOnFail = parsedArgs.failfast;
   const disableLog = parsedArgs.quiet;
 
   try {
@@ -277,8 +276,8 @@ async function main(): Promise<void> {
       include,
       exclude,
       allowNone,
-      exitOnFail,
-      disableLog
+      disableLog,
+      exitOnFail: true
     });
   } catch (error) {
     if (!disableLog) {