feat(unstable): deno test --coverage (#6901)

This commit adds basic support for collecting coverage
data using "deno test". 

Currently the report is only a text added to the end
of output from "deno test".

cli/coverage.rs

@@ -0,0 +1,339 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+use crate::colors;
+use crate::file_fetcher::SourceFile;
+use crate::global_state::GlobalState;
+use crate::inspector::DenoInspector;
+use crate::permissions::Permissions;
+use deno_core::v8;
+use deno_core::ErrBox;
+use deno_core::ModuleSpecifier;
+use serde::Deserialize;
+use std::collections::VecDeque;
+use std::mem::MaybeUninit;
+use std::ops::Deref;
+use std::ops::DerefMut;
+use std::ptr;
+use std::sync::Arc;
+use url::Url;
+
+pub struct CoverageCollector {
+  v8_channel: v8::inspector::ChannelBase,
+  v8_session: v8::UniqueRef<v8::inspector::V8InspectorSession>,
+  response_queue: VecDeque<String>,
+}
+
+impl Deref for CoverageCollector {
+  type Target = v8::inspector::V8InspectorSession;
+  fn deref(&self) -> &Self::Target {
+    &self.v8_session
+  }
+}
+
+impl DerefMut for CoverageCollector {
+  fn deref_mut(&mut self) -> &mut Self::Target {
+    &mut self.v8_session
+  }
+}
+
+impl v8::inspector::ChannelImpl for CoverageCollector {
+  fn base(&self) -> &v8::inspector::ChannelBase {
+    &self.v8_channel
+  }
+
+  fn base_mut(&mut self) -> &mut v8::inspector::ChannelBase {
+    &mut self.v8_channel
+  }
+
+  fn send_response(
+    &mut self,
+    _call_id: i32,
+    message: v8::UniquePtr<v8::inspector::StringBuffer>,
+  ) {
+    let message = message.unwrap().string().to_string();
+    self.response_queue.push_back(message);
+  }
+
+  fn send_notification(
+    &mut self,
+    _message: v8::UniquePtr<v8::inspector::StringBuffer>,
+  ) {
+  }
+
+  fn flush_protocol_notifications(&mut self) {}
+}
+
+impl CoverageCollector {
+  const CONTEXT_GROUP_ID: i32 = 1;
+
+  pub fn new(inspector_ptr: *mut DenoInspector) -> Box<Self> {
+    new_box_with(move |self_ptr| {
+      let v8_channel = v8::inspector::ChannelBase::new::<Self>();
+      let v8_session = unsafe { &mut *inspector_ptr }.connect(
+        Self::CONTEXT_GROUP_ID,
+        unsafe { &mut *self_ptr },
+        v8::inspector::StringView::empty(),
+      );
+
+      let response_queue = VecDeque::with_capacity(10);
+
+      Self {
+        v8_channel,
+        v8_session,
+        response_queue,
+      }
+    })
+  }
+
+  async fn dispatch(&mut self, message: String) -> Result<String, ErrBox> {
+    let message = v8::inspector::StringView::from(message.as_bytes());
+    self.v8_session.dispatch_protocol_message(message);
+
+    let response = self.response_queue.pop_back();
+    Ok(response.unwrap())
+  }
+
+  pub async fn start_collecting(&mut self) -> Result<(), ErrBox> {
+    self
+      .dispatch(r#"{"id":1,"method":"Runtime.enable"}"#.into())
+      .await?;
+    self
+      .dispatch(r#"{"id":2,"method":"Profiler.enable"}"#.into())
+      .await?;
+
+    self
+        .dispatch(r#"{"id":3,"method":"Profiler.startPreciseCoverage", "params": {"callCount": true, "detailed": true}}"#.into())
+        .await?;
+
+    Ok(())
+  }
+
+  pub async fn take_precise_coverage(
+    &mut self,
+  ) -> Result<Vec<ScriptCoverage>, ErrBox> {
+    let response = self
+      .dispatch(r#"{"id":4,"method":"Profiler.takePreciseCoverage" }"#.into())
+      .await?;
+
+    let coverage_result: TakePreciseCoverageResponse =
+      serde_json::from_str(&response).unwrap();
+
+    Ok(coverage_result.result.result)
+  }
+
+  pub async fn stop_collecting(&mut self) -> Result<(), ErrBox> {
+    self
+      .dispatch(r#"{"id":5,"method":"Profiler.stopPreciseCoverage"}"#.into())
+      .await?;
+
+    self
+      .dispatch(r#"{"id":6,"method":"Profiler.disable"}"#.into())
+      .await?;
+
+    self
+      .dispatch(r#"{"id":7,"method":"Runtime.disable"}"#.into())
+      .await?;
+
+    Ok(())
+  }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CoverageRange {
+  pub start_offset: usize,
+  pub end_offset: usize,
+  pub count: usize,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FunctionCoverage {
+  pub function_name: String,
+  pub ranges: Vec<CoverageRange>,
+  pub is_block_coverage: bool,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ScriptCoverage {
+  pub script_id: String,
+  pub url: String,
+  pub functions: Vec<FunctionCoverage>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TakePreciseCoverageResult {
+  result: Vec<ScriptCoverage>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TakePreciseCoverageResponse {
+  id: usize,
+  result: TakePreciseCoverageResult,
+}
+
+pub struct PrettyCoverageReporter {
+  coverages: Vec<ScriptCoverage>,
+  global_state: Arc<GlobalState>,
+}
+
+// TODO(caspervonb) add support for lcov output (see geninfo(1) for format spec).
+impl PrettyCoverageReporter {
+  pub fn new(
+    global_state: Arc<GlobalState>,
+    coverages: Vec<ScriptCoverage>,
+  ) -> PrettyCoverageReporter {
+    PrettyCoverageReporter {
+      global_state,
+      coverages,
+    }
+  }
+
+  pub fn get_report(&self) -> String {
+    let mut report = String::from("test coverage:\n");
+
+    for script_coverage in &self.coverages {
+      if let Some(script_report) = self.get_script_report(script_coverage) {
+        report.push_str(&format!("{}\n", script_report))
+      }
+    }
+
+    report
+  }
+
+  fn get_source_file_for_script(
+    &self,
+    script_coverage: &ScriptCoverage,
+  ) -> Option<SourceFile> {
+    let module_specifier =
+      ModuleSpecifier::resolve_url_or_path(&script_coverage.url).ok()?;
+
+    let maybe_source_file = self
+      .global_state
+      .ts_compiler
+      .get_compiled_source_file(&module_specifier.as_url())
+      .or_else(|_| {
+        self
+          .global_state
+          .file_fetcher
+          .fetch_cached_source_file(&module_specifier, Permissions::allow_all())
+          .ok_or_else(|| ErrBox::error("unable to fetch source file"))
+      })
+      .ok();
+
+    maybe_source_file
+  }
+
+  fn get_script_report(
+    &self,
+    script_coverage: &ScriptCoverage,
+  ) -> Option<String> {
+    let source_file = match self.get_source_file_for_script(script_coverage) {
+      Some(sf) => sf,
+      None => return None,
+    };
+
+    let mut total_lines = 0;
+    let mut covered_lines = 0;
+
+    let mut line_offset = 0;
+    let source_string = source_file.source_code.to_string().unwrap();
+
+    for line in source_string.lines() {
+      let line_start_offset = line_offset;
+      let line_end_offset = line_start_offset + line.len();
+
+      let mut count = 0;
+      for function in &script_coverage.functions {
+        for range in &function.ranges {
+          if range.start_offset <= line_start_offset
+            && range.end_offset >= line_end_offset
+          {
+            count += range.count;
+            if range.count == 0 {
+              count = 0;
+              break;
+            }
+          }
+        }
+      }
+
+      if count > 0 {
+        covered_lines += 1;
+      }
+
+      total_lines += 1;
+      line_offset += line.len();
+    }
+
+    let line_ratio = covered_lines as f32 / total_lines as f32;
+    let line_coverage = format!("{:.3}%", line_ratio * 100.0);
+
+    let line = if line_ratio >= 0.9 {
+      format!(
+        "{} {}",
+        source_file.url.to_string(),
+        colors::green(&line_coverage)
+      )
+    } else if line_ratio >= 0.75 {
+      format!(
+        "{} {}",
+        source_file.url.to_string(),
+        colors::yellow(&line_coverage)
+      )
+    } else {
+      format!(
+        "{} {}",
+        source_file.url.to_string(),
+        colors::red(&line_coverage)
+      )
+    };
+
+    Some(line)
+  }
+}
+
+fn new_box_with<T>(new_fn: impl FnOnce(*mut T) -> T) -> Box<T> {
+  let b = Box::new(MaybeUninit::<T>::uninit());
+  let p = Box::into_raw(b) as *mut T;
+  unsafe { ptr::write(p, new_fn(p)) };
+  unsafe { Box::from_raw(p) }
+}
+
+pub fn filter_script_coverages(
+  coverages: Vec<ScriptCoverage>,
+  test_file_url: Url,
+  test_modules: Vec<Url>,
+) -> Vec<ScriptCoverage> {
+  coverages
+    .into_iter()
+    .filter(|e| {
+      if let Ok(url) = Url::parse(&e.url) {
+        if url == test_file_url {
+          return false;
+        }
+
+        for test_module_url in &test_modules {
+          if &url == test_module_url {
+            return false;
+          }
+        }
+
+        if let Ok(path) = url.to_file_path() {
+          for test_module_url in &test_modules {
+            if let Ok(test_module_path) = test_module_url.to_file_path() {
+              if path.starts_with(test_module_path.parent().unwrap()) {
+                return true;
+              }
+            }
+          }
+        }
+      }
+
+      false
+    })
+    .collect::<Vec<ScriptCoverage>>()
+}

cli/flags.rs

@@ -71,6 +71,7 @@ pub enum DenoSubcommand {
     allow_none: bool,
     include: Option<Vec<String>>,
     filter: Option<String>,
+    coverage: bool,
   },
   Types,
   Upgrade {
@@ -573,6 +574,13 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   let allow_none = matches.is_present("allow_none");
   let quiet = matches.is_present("quiet");
   let filter = matches.value_of("filter").map(String::from);
+  let coverage = matches.is_present("coverage");
+
+  // Coverage implies `--inspect`
+  if coverage {
+    flags.inspect = Some("127.0.0.1:9229".parse::<SocketAddr>().unwrap());
+  }
+
   let include = if matches.is_present("files") {
     let files: Vec<String> = matches
       .values_of("files")
@@ -590,6 +598,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
     include,
     filter,
     allow_none,
+    coverage,
   };
 }
 
@@ -1205,6 +1214,15 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
         .takes_value(true)
         .help("Run tests with this string or pattern in the test name"),
     )
+    .arg(
+      Arg::with_name("coverage")
+        .long("coverage")
+        .takes_value(false)
+        .requires("unstable")
+        .conflicts_with("inspect")
+        .conflicts_with("inspect-brk")
+        .help("Collect coverage information"),
+    )
     .arg(
       Arg::with_name("files")
         .help("List of file names to run")
@@ -2873,6 +2891,7 @@ mod tests {
           allow_none: true,
           quiet: false,
           include: Some(svec!["dir1/", "dir2/"]),
+          coverage: false,
         },
         allow_net: true,
         ..Flags::default()
@@ -2892,6 +2911,7 @@ mod tests {
           quiet: false,
           filter: Some("foo".to_string()),
           include: Some(svec!["dir1"]),
+          coverage: false,
         },
         ..Flags::default()
       }
@@ -2911,12 +2931,40 @@ mod tests {
           quiet: false,
           filter: Some("- foo".to_string()),
           include: Some(svec!["dir1"]),
+          coverage: false,
         },
         ..Flags::default()
       }
     );
   }
 
+  #[test]
+  fn test_coverage() {
+    let r = flags_from_vec_safe(svec![
+      "deno",
+      "test",
+      "--unstable",
+      "--coverage",
+      "dir1"
+    ]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Test {
+          fail_fast: false,
+          allow_none: false,
+          quiet: false,
+          filter: None,
+          include: Some(svec!["dir1"]),
+          coverage: true,
+        },
+        inspect: Some("127.0.0.1:9229".parse::<SocketAddr>().unwrap()),
+        unstable: true,
+        ..Flags::default()
+      }
+    );
+  }
+
   #[test]
   fn run_with_cafile() {
     let r = flags_from_vec_safe(svec![

cli/main.rs

@@ -24,6 +24,7 @@ extern crate url;
 
 mod checksum;
 pub mod colors;
+mod coverage;
 pub mod deno_dir;
 pub mod diagnostics;
 mod diff;
@@ -69,6 +70,8 @@ pub mod version;
 mod web_worker;
 pub mod worker;
 
+use crate::coverage::CoverageCollector;
+use crate::coverage::PrettyCoverageReporter;
 use crate::file_fetcher::map_file_extension;
 use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
@@ -540,6 +543,7 @@ async fn test_command(
   quiet: bool,
   allow_none: bool,
   filter: Option<String>,
+  coverage: bool,
 ) -> Result<(), ErrBox> {
   let global_state = GlobalState::new(flags.clone())?;
   let cwd = std::env::current_dir().expect("No current directory");
@@ -557,15 +561,19 @@ async fn test_command(
   let test_file_path = cwd.join(".deno.test.ts");
   let test_file_url =
     Url::from_file_path(&test_file_path).expect("Should be valid file url");
-  let test_file =
-    test_runner::render_test_file(test_modules, fail_fast, quiet, filter);
+  let test_file = test_runner::render_test_file(
+    test_modules.clone(),
+    fail_fast,
+    quiet,
+    filter,
+  );
   let main_module =
     ModuleSpecifier::resolve_url(&test_file_url.to_string()).unwrap();
   let mut worker = MainWorker::create(&global_state, main_module.clone())?;
   // Create a dummy source file.
   let source_file = SourceFile {
     filename: test_file_url.to_file_path().unwrap(),
-    url: test_file_url,
+    url: test_file_url.clone(),
     types_header: None,
     media_type: MediaType::TypeScript,
     source_code: TextDocument::new(
@@ -578,11 +586,45 @@ async fn test_command(
   global_state
     .file_fetcher
     .save_source_file_in_cache(&main_module, source_file);
+
+  let mut maybe_coverage_collector = if coverage {
+    let inspector = worker
+      .inspector
+      .as_mut()
+      .expect("Inspector is not created.");
+
+    let mut coverage_collector = CoverageCollector::new(&mut **inspector);
+    coverage_collector.start_collecting().await?;
+
+    Some(coverage_collector)
+  } else {
+    None
+  };
+
   let execute_result = worker.execute_module(&main_module).await;
   execute_result?;
   worker.execute("window.dispatchEvent(new Event('load'))")?;
   (&mut *worker).await?;
-  worker.execute("window.dispatchEvent(new Event('unload'))")
+  worker.execute("window.dispatchEvent(new Event('unload'))")?;
+  (&mut *worker).await?;
+
+  if let Some(coverage_collector) = maybe_coverage_collector.as_mut() {
+    let script_coverage = coverage_collector.take_precise_coverage().await?;
+    coverage_collector.stop_collecting().await?;
+
+    let filtered_coverage = coverage::filter_script_coverages(
+      script_coverage,
+      test_file_url,
+      test_modules,
+    );
+
+    let pretty_coverage_reporter =
+      PrettyCoverageReporter::new(global_state, filtered_coverage);
+    let report = pretty_coverage_reporter.get_report();
+    print!("{}", report)
+  }
+
+  Ok(())
 }
 
 pub fn main() {
@@ -694,8 +736,11 @@ pub fn main() {
       include,
       allow_none,
       filter,
-    } => test_command(flags, include, fail_fast, quiet, allow_none, filter)
-      .boxed_local(),
+      coverage,
+    } => test_command(
+      flags, include, fail_fast, quiet, allow_none, filter, coverage,
+    )
+    .boxed_local(),
     DenoSubcommand::Completions { buf } => {
       if let Err(e) = write_to_stdout_ignore_sigpipe(&buf) {
         eprintln!("{}", e);

cli/tests/integration_tests.rs

@@ -2348,6 +2348,12 @@ itest!(proto_exploit {
   output: "proto_exploit.js.out",
 });
 
+itest!(deno_test_coverage {
+  args: "test --coverage --unstable test_coverage.ts",
+  output: "test_coverage.out",
+  exit_code: 0,
+});
+
 itest!(deno_lint {
   args: "lint --unstable lint/file1.js lint/file2.ts lint/ignored_file.ts",
   output: "lint/expected.out",

cli/tests/test_coverage.out

@@ -0,0 +1,10 @@
+[WILDCARD]
+running 1 tests
+test returnsHiSuccess ... ok ([WILDCARD])
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
+test coverage:
+file://[WILDCARD]/cli/tests/subdir/mod1.ts 57.143%
+file://[WILDCARD]/cli/tests/subdir/subdir2/mod2.ts 50.000%
+file://[WILDCARD]/cli/tests/subdir/print_hello.ts 50.000%

cli/tests/test_coverage.ts

@@ -0,0 +1,5 @@
+import { returnsHi } from "./subdir/mod1.ts";
+
+Deno.test("returnsHiSuccess", function () {
+  returnsHi();
+});