diff --git a/cli/coverage.rs b/cli/coverage.rs new file mode 100644 index 0000000000..a2ccdc16ab --- /dev/null +++ b/cli/coverage.rs @@ -0,0 +1,339 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::colors; +use crate::file_fetcher::SourceFile; +use crate::global_state::GlobalState; +use crate::inspector::DenoInspector; +use crate::permissions::Permissions; +use deno_core::v8; +use deno_core::ErrBox; +use deno_core::ModuleSpecifier; +use serde::Deserialize; +use std::collections::VecDeque; +use std::mem::MaybeUninit; +use std::ops::Deref; +use std::ops::DerefMut; +use std::ptr; +use std::sync::Arc; +use url::Url; + +pub struct CoverageCollector { + v8_channel: v8::inspector::ChannelBase, + v8_session: v8::UniqueRef, + response_queue: VecDeque, +} + +impl Deref for CoverageCollector { + type Target = v8::inspector::V8InspectorSession; + fn deref(&self) -> &Self::Target { + &self.v8_session + } +} + +impl DerefMut for CoverageCollector { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.v8_session + } +} + +impl v8::inspector::ChannelImpl for CoverageCollector { + fn base(&self) -> &v8::inspector::ChannelBase { + &self.v8_channel + } + + fn base_mut(&mut self) -> &mut v8::inspector::ChannelBase { + &mut self.v8_channel + } + + fn send_response( + &mut self, + _call_id: i32, + message: v8::UniquePtr, + ) { + let message = message.unwrap().string().to_string(); + self.response_queue.push_back(message); + } + + fn send_notification( + &mut self, + _message: v8::UniquePtr, + ) { + } + + fn flush_protocol_notifications(&mut self) {} +} + +impl CoverageCollector { + const CONTEXT_GROUP_ID: i32 = 1; + + pub fn new(inspector_ptr: *mut DenoInspector) -> Box { + new_box_with(move |self_ptr| { + let v8_channel = v8::inspector::ChannelBase::new::(); + let v8_session = unsafe { &mut *inspector_ptr }.connect( + Self::CONTEXT_GROUP_ID, + unsafe { &mut *self_ptr }, + v8::inspector::StringView::empty(), + ); + + let response_queue = VecDeque::with_capacity(10); + + Self { + v8_channel, + v8_session, + response_queue, + } + }) + } + + async fn dispatch(&mut self, message: String) -> Result { + let message = v8::inspector::StringView::from(message.as_bytes()); + self.v8_session.dispatch_protocol_message(message); + + let response = self.response_queue.pop_back(); + Ok(response.unwrap()) + } + + pub async fn start_collecting(&mut self) -> Result<(), ErrBox> { + self + .dispatch(r#"{"id":1,"method":"Runtime.enable"}"#.into()) + .await?; + self + .dispatch(r#"{"id":2,"method":"Profiler.enable"}"#.into()) + .await?; + + self + .dispatch(r#"{"id":3,"method":"Profiler.startPreciseCoverage", "params": {"callCount": true, "detailed": true}}"#.into()) + .await?; + + Ok(()) + } + + pub async fn take_precise_coverage( + &mut self, + ) -> Result, ErrBox> { + let response = self + .dispatch(r#"{"id":4,"method":"Profiler.takePreciseCoverage" }"#.into()) + .await?; + + let coverage_result: TakePreciseCoverageResponse = + serde_json::from_str(&response).unwrap(); + + Ok(coverage_result.result.result) + } + + pub async fn stop_collecting(&mut self) -> Result<(), ErrBox> { + self + .dispatch(r#"{"id":5,"method":"Profiler.stopPreciseCoverage"}"#.into()) + .await?; + + self + .dispatch(r#"{"id":6,"method":"Profiler.disable"}"#.into()) + .await?; + + self + .dispatch(r#"{"id":7,"method":"Runtime.disable"}"#.into()) + .await?; + + Ok(()) + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CoverageRange { + pub start_offset: usize, + pub end_offset: usize, + pub count: usize, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FunctionCoverage { + pub function_name: String, + pub ranges: Vec, + pub is_block_coverage: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptCoverage { + pub script_id: String, + pub url: String, + pub functions: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TakePreciseCoverageResult { + result: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TakePreciseCoverageResponse { + id: usize, + result: TakePreciseCoverageResult, +} + +pub struct PrettyCoverageReporter { + coverages: Vec, + global_state: Arc, +} + +// TODO(caspervonb) add support for lcov output (see geninfo(1) for format spec). +impl PrettyCoverageReporter { + pub fn new( + global_state: Arc, + coverages: Vec, + ) -> PrettyCoverageReporter { + PrettyCoverageReporter { + global_state, + coverages, + } + } + + pub fn get_report(&self) -> String { + let mut report = String::from("test coverage:\n"); + + for script_coverage in &self.coverages { + if let Some(script_report) = self.get_script_report(script_coverage) { + report.push_str(&format!("{}\n", script_report)) + } + } + + report + } + + fn get_source_file_for_script( + &self, + script_coverage: &ScriptCoverage, + ) -> Option { + let module_specifier = + ModuleSpecifier::resolve_url_or_path(&script_coverage.url).ok()?; + + let maybe_source_file = self + .global_state + .ts_compiler + .get_compiled_source_file(&module_specifier.as_url()) + .or_else(|_| { + self + .global_state + .file_fetcher + .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) + .ok_or_else(|| ErrBox::error("unable to fetch source file")) + }) + .ok(); + + maybe_source_file + } + + fn get_script_report( + &self, + script_coverage: &ScriptCoverage, + ) -> Option { + let source_file = match self.get_source_file_for_script(script_coverage) { + Some(sf) => sf, + None => return None, + }; + + let mut total_lines = 0; + let mut covered_lines = 0; + + let mut line_offset = 0; + let source_string = source_file.source_code.to_string().unwrap(); + + for line in source_string.lines() { + let line_start_offset = line_offset; + let line_end_offset = line_start_offset + line.len(); + + let mut count = 0; + for function in &script_coverage.functions { + for range in &function.ranges { + if range.start_offset <= line_start_offset + && range.end_offset >= line_end_offset + { + count += range.count; + if range.count == 0 { + count = 0; + break; + } + } + } + } + + if count > 0 { + covered_lines += 1; + } + + total_lines += 1; + line_offset += line.len(); + } + + let line_ratio = covered_lines as f32 / total_lines as f32; + let line_coverage = format!("{:.3}%", line_ratio * 100.0); + + let line = if line_ratio >= 0.9 { + format!( + "{} {}", + source_file.url.to_string(), + colors::green(&line_coverage) + ) + } else if line_ratio >= 0.75 { + format!( + "{} {}", + source_file.url.to_string(), + colors::yellow(&line_coverage) + ) + } else { + format!( + "{} {}", + source_file.url.to_string(), + colors::red(&line_coverage) + ) + }; + + Some(line) + } +} + +fn new_box_with(new_fn: impl FnOnce(*mut T) -> T) -> Box { + let b = Box::new(MaybeUninit::::uninit()); + let p = Box::into_raw(b) as *mut T; + unsafe { ptr::write(p, new_fn(p)) }; + unsafe { Box::from_raw(p) } +} + +pub fn filter_script_coverages( + coverages: Vec, + test_file_url: Url, + test_modules: Vec, +) -> Vec { + coverages + .into_iter() + .filter(|e| { + if let Ok(url) = Url::parse(&e.url) { + if url == test_file_url { + return false; + } + + for test_module_url in &test_modules { + if &url == test_module_url { + return false; + } + } + + if let Ok(path) = url.to_file_path() { + for test_module_url in &test_modules { + if let Ok(test_module_path) = test_module_url.to_file_path() { + if path.starts_with(test_module_path.parent().unwrap()) { + return true; + } + } + } + } + } + + false + }) + .collect::>() +} diff --git a/cli/flags.rs b/cli/flags.rs index 9dcfad43d2..ebaf63cb2d 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -71,6 +71,7 @@ pub enum DenoSubcommand { allow_none: bool, include: Option>, filter: Option, + coverage: bool, }, Types, Upgrade { @@ -573,6 +574,13 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let allow_none = matches.is_present("allow_none"); let quiet = matches.is_present("quiet"); let filter = matches.value_of("filter").map(String::from); + let coverage = matches.is_present("coverage"); + + // Coverage implies `--inspect` + if coverage { + flags.inspect = Some("127.0.0.1:9229".parse::().unwrap()); + } + let include = if matches.is_present("files") { let files: Vec = matches .values_of("files") @@ -590,6 +598,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { include, filter, allow_none, + coverage, }; } @@ -1205,6 +1214,15 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .help("Run tests with this string or pattern in the test name"), ) + .arg( + Arg::with_name("coverage") + .long("coverage") + .takes_value(false) + .requires("unstable") + .conflicts_with("inspect") + .conflicts_with("inspect-brk") + .help("Collect coverage information"), + ) .arg( Arg::with_name("files") .help("List of file names to run") @@ -2873,6 +2891,7 @@ mod tests { allow_none: true, quiet: false, include: Some(svec!["dir1/", "dir2/"]), + coverage: false, }, allow_net: true, ..Flags::default() @@ -2892,6 +2911,7 @@ mod tests { quiet: false, filter: Some("foo".to_string()), include: Some(svec!["dir1"]), + coverage: false, }, ..Flags::default() } @@ -2911,12 +2931,40 @@ mod tests { quiet: false, filter: Some("- foo".to_string()), include: Some(svec!["dir1"]), + coverage: false, }, ..Flags::default() } ); } + #[test] + fn test_coverage() { + let r = flags_from_vec_safe(svec![ + "deno", + "test", + "--unstable", + "--coverage", + "dir1" + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Test { + fail_fast: false, + allow_none: false, + quiet: false, + filter: None, + include: Some(svec!["dir1"]), + coverage: true, + }, + inspect: Some("127.0.0.1:9229".parse::().unwrap()), + unstable: true, + ..Flags::default() + } + ); + } + #[test] fn run_with_cafile() { let r = flags_from_vec_safe(svec![ diff --git a/cli/main.rs b/cli/main.rs index 4584f0c304..51e768caf8 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -24,6 +24,7 @@ extern crate url; mod checksum; pub mod colors; +mod coverage; pub mod deno_dir; pub mod diagnostics; mod diff; @@ -69,6 +70,8 @@ pub mod version; mod web_worker; pub mod worker; +use crate::coverage::CoverageCollector; +use crate::coverage::PrettyCoverageReporter; use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; @@ -540,6 +543,7 @@ async fn test_command( quiet: bool, allow_none: bool, filter: Option, + coverage: bool, ) -> Result<(), ErrBox> { let global_state = GlobalState::new(flags.clone())?; let cwd = std::env::current_dir().expect("No current directory"); @@ -557,15 +561,19 @@ async fn test_command( let test_file_path = cwd.join(".deno.test.ts"); let test_file_url = Url::from_file_path(&test_file_path).expect("Should be valid file url"); - let test_file = - test_runner::render_test_file(test_modules, fail_fast, quiet, filter); + let test_file = test_runner::render_test_file( + test_modules.clone(), + fail_fast, + quiet, + filter, + ); let main_module = ModuleSpecifier::resolve_url(&test_file_url.to_string()).unwrap(); let mut worker = MainWorker::create(&global_state, main_module.clone())?; // Create a dummy source file. let source_file = SourceFile { filename: test_file_url.to_file_path().unwrap(), - url: test_file_url, + url: test_file_url.clone(), types_header: None, media_type: MediaType::TypeScript, source_code: TextDocument::new( @@ -578,11 +586,45 @@ async fn test_command( global_state .file_fetcher .save_source_file_in_cache(&main_module, source_file); + + let mut maybe_coverage_collector = if coverage { + let inspector = worker + .inspector + .as_mut() + .expect("Inspector is not created."); + + let mut coverage_collector = CoverageCollector::new(&mut **inspector); + coverage_collector.start_collecting().await?; + + Some(coverage_collector) + } else { + None + }; + let execute_result = worker.execute_module(&main_module).await; execute_result?; worker.execute("window.dispatchEvent(new Event('load'))")?; (&mut *worker).await?; - worker.execute("window.dispatchEvent(new Event('unload'))") + worker.execute("window.dispatchEvent(new Event('unload'))")?; + (&mut *worker).await?; + + if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { + let script_coverage = coverage_collector.take_precise_coverage().await?; + coverage_collector.stop_collecting().await?; + + let filtered_coverage = coverage::filter_script_coverages( + script_coverage, + test_file_url, + test_modules, + ); + + let pretty_coverage_reporter = + PrettyCoverageReporter::new(global_state, filtered_coverage); + let report = pretty_coverage_reporter.get_report(); + print!("{}", report) + } + + Ok(()) } pub fn main() { @@ -694,8 +736,11 @@ pub fn main() { include, allow_none, filter, - } => test_command(flags, include, fail_fast, quiet, allow_none, filter) - .boxed_local(), + coverage, + } => test_command( + flags, include, fail_fast, quiet, allow_none, filter, coverage, + ) + .boxed_local(), DenoSubcommand::Completions { buf } => { if let Err(e) = write_to_stdout_ignore_sigpipe(&buf) { eprintln!("{}", e); diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 402268dd12..b584696409 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -2348,6 +2348,12 @@ itest!(proto_exploit { output: "proto_exploit.js.out", }); +itest!(deno_test_coverage { + args: "test --coverage --unstable test_coverage.ts", + output: "test_coverage.out", + exit_code: 0, +}); + itest!(deno_lint { args: "lint --unstable lint/file1.js lint/file2.ts lint/ignored_file.ts", output: "lint/expected.out", diff --git a/cli/tests/test_coverage.out b/cli/tests/test_coverage.out new file mode 100644 index 0000000000..85e229881a --- /dev/null +++ b/cli/tests/test_coverage.out @@ -0,0 +1,10 @@ +[WILDCARD] +running 1 tests +test returnsHiSuccess ... ok ([WILDCARD]) + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD]) + +test coverage: +file://[WILDCARD]/cli/tests/subdir/mod1.ts 57.143% +file://[WILDCARD]/cli/tests/subdir/subdir2/mod2.ts 50.000% +file://[WILDCARD]/cli/tests/subdir/print_hello.ts 50.000% diff --git a/cli/tests/test_coverage.ts b/cli/tests/test_coverage.ts new file mode 100644 index 0000000000..72844422fe --- /dev/null +++ b/cli/tests/test_coverage.ts @@ -0,0 +1,5 @@ +import { returnsHi } from "./subdir/mod1.ts"; + +Deno.test("returnsHiSuccess", function () { + returnsHi(); +});