diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 7c1502ad60..ecbc53db74 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -335,6 +335,20 @@ impl CliOptions { &self.flags.subcommand } + pub fn trace_ops(&self) -> bool { + match self.sub_command() { + DenoSubcommand::Test(flags) => flags.trace_ops, + _ => false, + } + } + + pub fn shuffle_tests(&self) -> Option { + match self.sub_command() { + DenoSubcommand::Test(flags) => flags.shuffle, + _ => None, + } + } + pub fn type_check_mode(&self) -> TypeCheckMode { self.flags.type_check_mode } diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index de74de40ed..dde834221e 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -8,7 +8,6 @@ use crate::args::flags_from_vec; use crate::args::DenoSubcommand; use crate::checksum; use crate::create_main_worker; -use crate::located_script_name; use crate::lsp::client::Client; use crate::lsp::client::TestingNotification; use crate::lsp::config; @@ -166,39 +165,7 @@ async fn test_specifier( stderr: StdioPipe::File(sender.stderr()), }, ); - - worker.js_runtime.execute_script( - &located_script_name!(), - r#"Deno[Deno.internal].enableTestAndBench()"#, - )?; - - worker - .execute_script( - &located_script_name!(), - "Deno.core.enableOpCallTracing();", - ) - .unwrap(); - - if mode != test::TestMode::Documentation { - worker.execute_side_module(&specifier).await?; - } - - worker.dispatch_load_event(&located_script_name!())?; - - let test_result = worker.js_runtime.execute_script( - &located_script_name!(), - r#"Deno[Deno.internal].runTests()"#, - )?; - - worker.js_runtime.resolve_value(test_result).await?; - - loop { - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - worker.run_event_loop(false).await?; - } - worker.dispatch_unload_event(&located_script_name!())?; + worker.run_lsp_test_specifier(mode).await?; } Ok(()) diff --git a/cli/main.rs b/cli/main.rs index 8489a5f2ff..de89c7e12c 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -35,6 +35,7 @@ mod tsc; mod unix_util; mod version; mod windows_util; +mod worker; use crate::args::flags_from_vec; use crate::args::BenchFlags; @@ -67,7 +68,6 @@ use crate::file_watcher::ResolutionResult; use crate::fmt_errors::format_js_error; use crate::graph_util::graph_lock_or_exit; use crate::graph_util::graph_valid; -use crate::module_loader::CliModuleLoader; use crate::proc_state::ProcState; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; @@ -78,26 +78,16 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::error::JsError; use deno_core::futures::future::FutureExt; -use deno_core::futures::future::LocalFutureObj; use deno_core::futures::Future; -use deno_core::located_script_name; use deno_core::parking_lot::RwLock; use deno_core::resolve_url_or_path; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::v8_set_flags; -use deno_core::Extension; use deno_core::ModuleSpecifier; use deno_runtime::colors; -use deno_runtime::ops::worker_host::CreateWebWorkerCb; -use deno_runtime::ops::worker_host::PreloadModuleCb; use deno_runtime::permissions::Permissions; use deno_runtime::tokio_util::run_local; -use deno_runtime::web_worker::WebWorker; -use deno_runtime::web_worker::WebWorkerOptions; -use deno_runtime::worker::MainWorker; -use deno_runtime::worker::WorkerOptions; -use deno_runtime::BootstrapOptions; use log::debug; use log::info; use std::env; @@ -107,167 +97,7 @@ use std::iter::once; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; - -fn create_web_worker_preload_module_callback( - ps: ProcState, -) -> Arc { - let compat = ps.options.compat(); - - Arc::new(move |mut worker| { - let fut = async move { - if compat { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - worker.execute_side_module(&compat::MODULE_URL).await?; - } - - Ok(worker) - }; - LocalFutureObj::new(Box::new(fut)) - }) -} - -fn create_web_worker_callback( - ps: ProcState, - stdio: deno_runtime::ops::io::Stdio, -) -> Arc { - Arc::new(move |args| { - let maybe_inspector_server = ps.maybe_inspector_server.clone(); - - let module_loader = CliModuleLoader::new_for_worker( - ps.clone(), - args.parent_permissions.clone(), - ); - let create_web_worker_cb = - create_web_worker_callback(ps.clone(), stdio.clone()); - let preload_module_cb = - create_web_worker_preload_module_callback(ps.clone()); - - let extensions = ops::cli_exts(ps.clone()); - - let options = WebWorkerOptions { - bootstrap: BootstrapOptions { - args: ps.options.argv().clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - debug_flag: ps - .options - .log_level() - .map_or(false, |l| l == log::Level::Debug), - enable_testing_features: ps.options.enable_testing_features(), - location: Some(args.main_module.clone()), - no_color: !colors::use_color(), - is_tty: colors::is_tty(), - runtime_version: version::deno(), - ts_version: version::TYPESCRIPT.to_string(), - unstable: ps.options.unstable(), - user_agent: version::get_user_agent(), - }, - extensions, - unsafely_ignore_certificate_errors: ps - .options - .unsafely_ignore_certificate_errors() - .map(ToOwned::to_owned), - root_cert_store: Some(ps.root_cert_store.clone()), - seed: ps.options.seed(), - create_web_worker_cb, - preload_module_cb, - format_js_error_fn: Some(Arc::new(format_js_error)), - source_map_getter: Some(Box::new(module_loader.clone())), - module_loader, - worker_type: args.worker_type, - maybe_inspector_server, - get_error_class_fn: Some(&errors::get_error_class_name), - blob_store: ps.blob_store.clone(), - broadcast_channel: ps.broadcast_channel.clone(), - shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), - stdio: stdio.clone(), - }; - - WebWorker::bootstrap_from_options( - args.name, - args.permissions, - args.main_module, - args.worker_id, - options, - ) - }) -} - -pub fn create_main_worker( - ps: &ProcState, - main_module: ModuleSpecifier, - permissions: Permissions, - mut custom_extensions: Vec, - stdio: deno_runtime::ops::io::Stdio, -) -> MainWorker { - let module_loader = CliModuleLoader::new(ps.clone()); - - let maybe_inspector_server = ps.maybe_inspector_server.clone(); - let should_break_on_first_statement = ps.options.inspect_brk().is_some(); - - let create_web_worker_cb = - create_web_worker_callback(ps.clone(), stdio.clone()); - let web_worker_preload_module_cb = - create_web_worker_preload_module_callback(ps.clone()); - - let maybe_storage_key = ps.options.resolve_storage_key(&main_module); - let origin_storage_dir = maybe_storage_key.map(|key| { - ps.dir - .root - // TODO(@crowlKats): change to origin_data for 2.0 - .join("location_data") - .join(checksum::gen(&[key.as_bytes()])) - }); - - let mut extensions = ops::cli_exts(ps.clone()); - extensions.append(&mut custom_extensions); - - let options = WorkerOptions { - bootstrap: BootstrapOptions { - args: ps.options.argv().clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - debug_flag: ps - .options - .log_level() - .map_or(false, |l| l == log::Level::Debug), - enable_testing_features: ps.options.enable_testing_features(), - location: ps.options.location_flag().map(ToOwned::to_owned), - no_color: !colors::use_color(), - is_tty: colors::is_tty(), - runtime_version: version::deno(), - ts_version: version::TYPESCRIPT.to_string(), - unstable: ps.options.unstable(), - user_agent: version::get_user_agent(), - }, - extensions, - unsafely_ignore_certificate_errors: ps - .options - .unsafely_ignore_certificate_errors() - .map(ToOwned::to_owned), - root_cert_store: Some(ps.root_cert_store.clone()), - seed: ps.options.seed(), - source_map_getter: Some(Box::new(module_loader.clone())), - format_js_error_fn: Some(Arc::new(format_js_error)), - create_web_worker_cb, - web_worker_preload_module_cb, - maybe_inspector_server, - should_break_on_first_statement, - module_loader, - get_error_class_fn: Some(&errors::get_error_class_name), - origin_storage_dir, - blob_store: ps.blob_store.clone(), - broadcast_channel: ps.broadcast_channel.clone(), - shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), - stdio, - }; - - MainWorker::bootstrap_from_options(main_module, permissions, options) -} +use worker::create_main_worker; pub fn write_to_stdout_ignore_sigpipe( bytes: &[u8], @@ -479,13 +309,13 @@ async fn install_command( let main_module = resolve_url_or_path(&install_flags.module_url)?; let mut worker = create_main_worker( &ps, - main_module.clone(), + main_module, permissions, vec![], Default::default(), ); // First, fetch and compile the module; this step ensures that the module exists. - worker.preload_main_module(&main_module).await?; + worker.preload_main_module().await?; tools::installer::install(flags, install_flags)?; Ok(0) } @@ -593,21 +423,8 @@ async fn eval_command( // Save our fake file into file fetcher cache // to allow module access by TS compiler. ps.file_fetcher.insert_cached(file); - debug!("main_module {}", &main_module); - if ps.options.compat() { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - } - worker.execute_main_module(&main_module).await?; - worker.dispatch_load_event(&located_script_name!())?; - loop { - worker.run_event_loop(false).await?; - - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - } - worker.dispatch_unload_event(&located_script_name!())?; - Ok(0) + let exit_code = worker.run().await?; + Ok(exit_code) } async fn create_graph_and_maybe_check( @@ -868,15 +685,14 @@ async fn repl_command( vec![], Default::default(), ); - if ps.options.compat() { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - compat::add_global_require(&mut worker.js_runtime, main_module.as_str())?; - worker.run_event_loop(false).await?; - compat::setup_builtin_modules(&mut worker.js_runtime)?; - } - worker.run_event_loop(false).await?; - - tools::repl::run(&ps, worker, repl_flags.eval_files, repl_flags.eval).await + worker.setup_repl().await?; + tools::repl::run( + &ps, + worker.into_main_worker(), + repl_flags.eval_files, + repl_flags.eval, + ) + .await } async fn run_from_stdin(flags: Flags) -> Result { @@ -905,88 +721,13 @@ async fn run_from_stdin(flags: Flags) -> Result { // to allow module access by TS compiler ps.file_fetcher.insert_cached(source_file); - debug!("main_module {}", main_module); - if ps.options.compat() { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - } - worker.execute_main_module(&main_module).await?; - worker.dispatch_load_event(&located_script_name!())?; - loop { - worker.run_event_loop(false).await?; - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - } - worker.dispatch_unload_event(&located_script_name!())?; - Ok(worker.get_exit_code()) + let exit_code = worker.run().await?; + Ok(exit_code) } // TODO(bartlomieju): this function is not handling `exit_code` set by the runtime // code properly. async fn run_with_watch(flags: Flags, script: String) -> Result { - /// The FileWatcherModuleExecutor provides module execution with safe dispatching of life-cycle events by tracking the - /// state of any pending events and emitting accordingly on drop in the case of a future - /// cancellation. - struct FileWatcherModuleExecutor { - worker: MainWorker, - pending_unload: bool, - compat: bool, - } - - impl FileWatcherModuleExecutor { - pub fn new(worker: MainWorker, compat: bool) -> FileWatcherModuleExecutor { - FileWatcherModuleExecutor { - worker, - pending_unload: false, - compat, - } - } - - /// Execute the given main module emitting load and unload events before and after execution - /// respectively. - pub async fn execute( - &mut self, - main_module: &ModuleSpecifier, - ) -> Result<(), AnyError> { - if self.compat { - self.worker.execute_side_module(&compat::GLOBAL_URL).await?; - } - self.worker.execute_main_module(main_module).await?; - self.worker.dispatch_load_event(&located_script_name!())?; - self.pending_unload = true; - - let result = loop { - let result = self.worker.run_event_loop(false).await; - if !self - .worker - .dispatch_beforeunload_event(&located_script_name!())? - { - break result; - } - }; - self.pending_unload = false; - - if let Err(err) = result { - return Err(err); - } - - self.worker.dispatch_unload_event(&located_script_name!())?; - - Ok(()) - } - } - - impl Drop for FileWatcherModuleExecutor { - fn drop(&mut self) { - if self.pending_unload { - self - .worker - .dispatch_unload_event(&located_script_name!()) - .unwrap(); - } - } - } - let flags = Arc::new(flags); let main_module = resolve_url_or_path(&script)?; let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); @@ -1001,20 +742,14 @@ async fn run_with_watch(flags: Flags, script: String) -> Result { let ps = ProcState::build_for_file_watcher((*flags).clone(), sender.clone()) .await?; - // We make use an module executor guard to ensure that unload is always fired when an - // operation is called. - let mut executor = FileWatcherModuleExecutor::new( - create_main_worker( - &ps, - main_module.clone(), - permissions, - vec![], - Default::default(), - ), - flags.compat, + let worker = create_main_worker( + &ps, + main_module.clone(), + permissions, + vec![], + Default::default(), ); - - executor.execute(&main_module).await?; + worker.run_for_watcher().await?; Ok(()) }) @@ -1064,73 +799,8 @@ async fn run_command( Default::default(), ); - let mut maybe_coverage_collector = - if let Some(ref coverage_dir) = ps.coverage_dir { - let session = worker.create_inspector_session().await; - - let coverage_dir = PathBuf::from(coverage_dir); - let mut coverage_collector = - tools::coverage::CoverageCollector::new(coverage_dir, session); - worker - .with_event_loop(coverage_collector.start_collecting().boxed_local()) - .await?; - Some(coverage_collector) - } else { - None - }; - - debug!("main_module {}", main_module); - - if ps.options.compat() { - // TODO(bartlomieju): fix me - assert_eq!(main_module.scheme(), "file"); - - // Set up Node globals - worker.execute_side_module(&compat::GLOBAL_URL).await?; - // And `module` module that we'll use for checking which - // loader to use and potentially load CJS module with. - // This allows to skip permission check for `--allow-net` - // which would otherwise be requested by dynamically importing - // this file. - worker.execute_side_module(&compat::MODULE_URL).await?; - - let use_esm_loader = compat::check_if_should_use_esm_loader(&main_module)?; - - if use_esm_loader { - // ES module execution in Node compatiblity mode - worker.execute_main_module(&main_module).await?; - } else { - // CJS module execution in Node compatiblity mode - compat::load_cjs_module( - &mut worker.js_runtime, - &main_module.to_file_path().unwrap().display().to_string(), - true, - )?; - } - } else { - // Regular ES module execution - worker.execute_main_module(&main_module).await?; - } - - worker.dispatch_load_event(&located_script_name!())?; - - loop { - worker - .run_event_loop(maybe_coverage_collector.is_none()) - .await?; - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - } - - worker.dispatch_unload_event(&located_script_name!())?; - - if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { - worker - .with_event_loop(coverage_collector.stop_collecting().boxed_local()) - .await?; - } - Ok(worker.get_exit_code()) + let exit_code = worker.run().await?; + Ok(exit_code) } async fn task_command( diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index 60bdd62809..3d52e4226f 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -4,7 +4,6 @@ use crate::args::BenchFlags; use crate::args::Flags; use crate::args::TypeCheckMode; use crate::colors; -use crate::compat; use crate::create_main_worker; use crate::file_watcher; use crate::file_watcher::ResolutionResult; @@ -12,7 +11,6 @@ use crate::fs_util::collect_specifiers; use crate::fs_util::is_supported_bench_path; use crate::graph_util::contains_specifier; use crate::graph_util::graph_valid; -use crate::located_script_name; use crate::ops; use crate::proc_state::ProcState; use crate::tools::test::format_test_error; @@ -40,7 +38,6 @@ use tokio::sync::mpsc::UnboundedSender; #[derive(Debug, Clone, Deserialize)] struct BenchSpecifierOptions { - compat_mode: bool, filter: Option, } @@ -367,50 +364,7 @@ async fn bench_specifier( Default::default(), ); - worker.js_runtime.execute_script( - &located_script_name!(), - r#"Deno[Deno.internal].enableTestAndBench()"#, - )?; - - if options.compat_mode { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - worker.execute_side_module(&compat::MODULE_URL).await?; - - let use_esm_loader = compat::check_if_should_use_esm_loader(&specifier)?; - - if use_esm_loader { - worker.execute_side_module(&specifier).await?; - } else { - compat::load_cjs_module( - &mut worker.js_runtime, - &specifier.to_file_path().unwrap().display().to_string(), - false, - )?; - worker.run_event_loop(false).await?; - } - } else { - // We execute the module module as a side module so that import.meta.main is not set. - worker.execute_side_module(&specifier).await?; - } - - worker.dispatch_load_event(&located_script_name!())?; - - let bench_result = worker.js_runtime.execute_script( - &located_script_name!(), - r#"Deno[Deno.internal].runBenchmarks()"#, - )?; - - worker.js_runtime.resolve_value(bench_result).await?; - - loop { - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - worker.run_event_loop(false).await?; - } - worker.dispatch_unload_event(&located_script_name!())?; - - Ok(()) + worker.run_bench_specifier().await } /// Test a collection of specifiers with test modes concurrently. @@ -537,13 +491,11 @@ pub async fn run_benchmarks( check_specifiers(&ps, permissions.clone(), specifiers.clone()).await?; - let compat = ps.options.compat(); bench_specifiers( ps, permissions, specifiers, BenchSpecifierOptions { - compat_mode: compat, filter: bench_flags.filter, }, ) @@ -703,7 +655,6 @@ pub async fn run_benchmarks_with_watch( check_specifiers(&ps, permissions.clone(), specifiers.clone()).await?; let specifier_options = BenchSpecifierOptions { - compat_mode: ps.options.compat(), filter: filter.clone(), }; bench_specifiers(ps, permissions.clone(), specifiers, specifier_options) diff --git a/cli/tools/test.rs b/cli/tools/test.rs index 7df7e560dc..6d24a7e4ce 100644 --- a/cli/tools/test.rs +++ b/cli/tools/test.rs @@ -5,7 +5,6 @@ use crate::args::TestFlags; use crate::args::TypeCheckMode; use crate::checksum; use crate::colors; -use crate::compat; use crate::create_main_worker; use crate::display; use crate::file_fetcher::File; @@ -18,10 +17,8 @@ use crate::fs_util::is_supported_test_path; use crate::fs_util::specifier_to_file_path; use crate::graph_util::contains_specifier; use crate::graph_util::graph_valid; -use crate::located_script_name; use crate::ops; use crate::proc_state::ProcState; -use crate::tools::coverage::CoverageCollector; use deno_ast::swc::common::comments::CommentKind; use deno_ast::MediaType; @@ -34,7 +31,6 @@ use deno_core::futures::stream; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; -use deno_core::serde_json::json; use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::ModuleKind; @@ -247,12 +243,9 @@ pub struct TestSummary { #[derive(Debug, Clone)] struct TestSpecifierOptions { - compat_mode: bool, concurrent_jobs: NonZeroUsize, fail_fast: Option, filter: TestFilter, - shuffle: Option, - trace_ops: bool, } impl TestSummary { @@ -732,90 +725,7 @@ async fn test_specifier( }, ); - worker.js_runtime.execute_script( - &located_script_name!(), - r#"Deno[Deno.internal].enableTestAndBench()"#, - )?; - - let mut maybe_coverage_collector = if let Some(ref coverage_dir) = - ps.coverage_dir - { - let session = worker.create_inspector_session().await; - let coverage_dir = PathBuf::from(coverage_dir); - let mut coverage_collector = CoverageCollector::new(coverage_dir, session); - worker - .with_event_loop(coverage_collector.start_collecting().boxed_local()) - .await?; - - Some(coverage_collector) - } else { - None - }; - - // Enable op call tracing in core to enable better debugging of op sanitizer - // failures. - if options.trace_ops { - worker - .execute_script( - &located_script_name!(), - "Deno.core.enableOpCallTracing();", - ) - .unwrap(); - } - - // We only execute the specifier as a module if it is tagged with TestMode::Module or - // TestMode::Both. - if mode != TestMode::Documentation { - if options.compat_mode { - worker.execute_side_module(&compat::GLOBAL_URL).await?; - worker.execute_side_module(&compat::MODULE_URL).await?; - - let use_esm_loader = compat::check_if_should_use_esm_loader(&specifier)?; - - if use_esm_loader { - worker.execute_side_module(&specifier).await?; - } else { - compat::load_cjs_module( - &mut worker.js_runtime, - &specifier.to_file_path().unwrap().display().to_string(), - false, - )?; - worker.run_event_loop(false).await?; - } - } else { - // We execute the module module as a side module so that import.meta.main is not set. - worker.execute_side_module(&specifier).await?; - } - } - - worker.dispatch_load_event(&located_script_name!())?; - - let test_result = worker.js_runtime.execute_script( - &located_script_name!(), - &format!( - r#"Deno[Deno.internal].runTests({})"#, - json!({ "shuffle": options.shuffle }), - ), - )?; - - worker.js_runtime.resolve_value(test_result).await?; - - loop { - if !worker.dispatch_beforeunload_event(&located_script_name!())? { - break; - } - worker.run_event_loop(false).await?; - } - - worker.dispatch_unload_event(&located_script_name!())?; - - if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { - worker - .with_event_loop(coverage_collector.stop_collecting().boxed_local()) - .await?; - } - - Ok(()) + worker.run_test_specifier(mode).await } fn extract_files_from_regex_blocks( @@ -1076,7 +986,7 @@ async fn test_specifiers( options: TestSpecifierOptions, ) -> Result<(), AnyError> { let log_level = ps.options.log_level(); - let specifiers_with_mode = if let Some(seed) = options.shuffle { + let specifiers_with_mode = if let Some(seed) = ps.options.shuffle_tests() { let mut rng = SmallRng::seed_from_u64(seed); let mut specifiers_with_mode = specifiers_with_mode.clone(); specifiers_with_mode.sort_by_key(|(specifier, _)| specifier.clone()); @@ -1405,18 +1315,14 @@ pub async fn run_tests( return Ok(()); } - let compat = ps.options.compat(); test_specifiers( ps, permissions, specifiers_with_mode, TestSpecifierOptions { - compat_mode: compat, concurrent_jobs: test_flags.concurrent_jobs, fail_fast: test_flags.fail_fast, filter: TestFilter::from_flag(&test_flags.filter), - shuffle: test_flags.shuffle, - trace_ops: test_flags.trace_ops, }, ) .await?; @@ -1561,7 +1467,6 @@ pub async fn run_tests_with_watch( let cli_options = ps.options.clone(); let operation = |modules_to_reload: Vec<(ModuleSpecifier, ModuleKind)>| { - let cli_options = cli_options.clone(); let filter = test_flags.filter.clone(); let include = include.clone(); let ignore = ignore.clone(); @@ -1595,12 +1500,9 @@ pub async fn run_tests_with_watch( permissions.clone(), specifiers_with_mode, TestSpecifierOptions { - compat_mode: cli_options.compat(), concurrent_jobs: test_flags.concurrent_jobs, fail_fast: test_flags.fail_fast, filter: TestFilter::from_flag(&filter), - shuffle: test_flags.shuffle, - trace_ops: test_flags.trace_ops, }, ) .await?; diff --git a/cli/worker.rs b/cli/worker.rs new file mode 100644 index 0000000000..8ab9b4c676 --- /dev/null +++ b/cli/worker.rs @@ -0,0 +1,578 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; +use deno_core::futures::task::LocalFutureObj; +use deno_core::futures::FutureExt; +use deno_core::located_script_name; +use deno_core::serde_json::json; +use deno_core::Extension; +use deno_core::ModuleId; +use deno_runtime::colors; +use deno_runtime::ops::worker_host::CreateWebWorkerCb; +use deno_runtime::ops::worker_host::PreloadModuleCb; +use deno_runtime::permissions::Permissions; +use deno_runtime::web_worker::WebWorker; +use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::worker::MainWorker; +use deno_runtime::worker::WorkerOptions; +use deno_runtime::BootstrapOptions; + +use crate::checksum; +use crate::compat; +use crate::errors; +use crate::fmt_errors::format_js_error; +use crate::module_loader::CliModuleLoader; +use crate::ops; +use crate::proc_state::ProcState; +use crate::tools; +use crate::tools::coverage::CoverageCollector; +use crate::tools::test::TestMode; +use crate::version; + +pub struct CliMainWorker { + main_module: ModuleSpecifier, + worker: MainWorker, + ps: ProcState, +} + +impl CliMainWorker { + pub fn into_main_worker(self) -> MainWorker { + self.worker + } + + pub async fn preload_main_module(&mut self) -> Result { + self.worker.preload_main_module(&self.main_module).await + } + + pub async fn setup_repl(&mut self) -> Result<(), AnyError> { + if self.ps.options.compat() { + self.worker.execute_side_module(&compat::GLOBAL_URL).await?; + compat::add_global_require( + &mut self.worker.js_runtime, + self.main_module.as_str(), + )?; + self.worker.run_event_loop(false).await?; + compat::setup_builtin_modules(&mut self.worker.js_runtime)?; + } + self.worker.run_event_loop(false).await?; + Ok(()) + } + + pub async fn run(&mut self) -> Result { + let mut maybe_coverage_collector = + self.maybe_setup_coverage_collector().await?; + log::debug!("main_module {}", self.main_module); + + if self.ps.options.compat() { + // TODO(bartlomieju): fix me + assert_eq!(self.main_module.scheme(), "file"); + + // Set up Node globals + self.worker.execute_side_module(&compat::GLOBAL_URL).await?; + // And `module` module that we'll use for checking which + // loader to use and potentially load CJS module with. + // This allows to skip permission check for `--allow-net` + // which would otherwise be requested by dynamically importing + // this file. + self.worker.execute_side_module(&compat::MODULE_URL).await?; + + let use_esm_loader = + compat::check_if_should_use_esm_loader(&self.main_module)?; + + if use_esm_loader { + // ES module execution in Node compatiblity mode + self.worker.execute_main_module(&self.main_module).await?; + } else { + // CJS module execution in Node compatiblity mode + compat::load_cjs_module( + &mut self.worker.js_runtime, + &self + .main_module + .to_file_path() + .unwrap() + .display() + .to_string(), + true, + )?; + } + } else { + // Regular ES module execution + self.worker.execute_main_module(&self.main_module).await?; + } + + self.worker.dispatch_load_event(&located_script_name!())?; + + loop { + self + .worker + .run_event_loop(maybe_coverage_collector.is_none()) + .await?; + if !self + .worker + .dispatch_beforeunload_event(&located_script_name!())? + { + break; + } + } + + self.worker.dispatch_unload_event(&located_script_name!())?; + + if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { + self + .worker + .with_event_loop(coverage_collector.stop_collecting().boxed_local()) + .await?; + } + + Ok(self.worker.get_exit_code()) + } + + pub async fn run_for_watcher(self) -> Result<(), AnyError> { + /// The FileWatcherModuleExecutor provides module execution with safe dispatching of life-cycle events by tracking the + /// state of any pending events and emitting accordingly on drop in the case of a future + /// cancellation. + struct FileWatcherModuleExecutor { + worker: MainWorker, + pending_unload: bool, + ps: ProcState, + } + + impl FileWatcherModuleExecutor { + pub fn new( + worker: MainWorker, + ps: ProcState, + ) -> FileWatcherModuleExecutor { + FileWatcherModuleExecutor { + worker, + pending_unload: false, + ps, + } + } + + /// Execute the given main module emitting load and unload events before and after execution + /// respectively. + pub async fn execute( + &mut self, + main_module: &ModuleSpecifier, + ) -> Result<(), AnyError> { + if self.ps.options.compat() { + self.worker.execute_side_module(&compat::GLOBAL_URL).await?; + } + self.worker.execute_main_module(main_module).await?; + self.worker.dispatch_load_event(&located_script_name!())?; + self.pending_unload = true; + + let result = loop { + let result = self.worker.run_event_loop(false).await; + if !self + .worker + .dispatch_beforeunload_event(&located_script_name!())? + { + break result; + } + }; + self.pending_unload = false; + + if let Err(err) = result { + return Err(err); + } + + self.worker.dispatch_unload_event(&located_script_name!())?; + + Ok(()) + } + } + + impl Drop for FileWatcherModuleExecutor { + fn drop(&mut self) { + if self.pending_unload { + self + .worker + .dispatch_unload_event(&located_script_name!()) + .unwrap(); + } + } + } + + let mut executor = FileWatcherModuleExecutor::new(self.worker, self.ps); + executor.execute(&self.main_module).await + } + + pub async fn run_test_specifier( + &mut self, + mode: TestMode, + ) -> Result<(), AnyError> { + self.worker.js_runtime.execute_script( + &located_script_name!(), + r#"Deno[Deno.internal].enableTestAndBench()"#, + )?; + + // Enable op call tracing in core to enable better debugging of op sanitizer + // failures. + if self.ps.options.trace_ops() { + self + .worker + .js_runtime + .execute_script( + &located_script_name!(), + "Deno.core.enableOpCallTracing();", + ) + .unwrap(); + } + + let mut maybe_coverage_collector = + self.maybe_setup_coverage_collector().await?; + + // We only execute the specifier as a module if it is tagged with TestMode::Module or + // TestMode::Both. + if mode != TestMode::Documentation { + if self.ps.options.compat() { + self.worker.execute_side_module(&compat::GLOBAL_URL).await?; + self.worker.execute_side_module(&compat::MODULE_URL).await?; + + let use_esm_loader = + compat::check_if_should_use_esm_loader(&self.main_module)?; + + if use_esm_loader { + self.worker.execute_side_module(&self.main_module).await?; + } else { + compat::load_cjs_module( + &mut self.worker.js_runtime, + &self + .main_module + .to_file_path() + .unwrap() + .display() + .to_string(), + false, + )?; + self.worker.run_event_loop(false).await?; + } + } else { + // We execute the module module as a side module so that import.meta.main is not set. + self.worker.execute_side_module(&self.main_module).await?; + } + } + + self.worker.dispatch_load_event(&located_script_name!())?; + + let test_result = self.worker.js_runtime.execute_script( + &located_script_name!(), + &format!( + r#"Deno[Deno.internal].runTests({})"#, + json!({ "shuffle": self.ps.options.shuffle_tests() }), + ), + )?; + + self.worker.js_runtime.resolve_value(test_result).await?; + + loop { + if !self + .worker + .dispatch_beforeunload_event(&located_script_name!())? + { + break; + } + self.worker.run_event_loop(false).await?; + } + + self.worker.dispatch_unload_event(&located_script_name!())?; + + if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { + self + .worker + .with_event_loop(coverage_collector.stop_collecting().boxed_local()) + .await?; + } + Ok(()) + } + + pub async fn run_lsp_test_specifier( + &mut self, + mode: TestMode, + ) -> Result<(), AnyError> { + self.worker.js_runtime.execute_script( + &located_script_name!(), + r#"Deno[Deno.internal].enableTestAndBench()"#, + )?; + + self + .worker + .execute_script( + &located_script_name!(), + "Deno.core.enableOpCallTracing();", + ) + .unwrap(); + + if mode != TestMode::Documentation { + self.worker.execute_side_module(&self.main_module).await?; + } + + self.worker.dispatch_load_event(&located_script_name!())?; + + let test_result = self.worker.js_runtime.execute_script( + &located_script_name!(), + r#"Deno[Deno.internal].runTests()"#, + )?; + + self.worker.js_runtime.resolve_value(test_result).await?; + + loop { + if !self + .worker + .dispatch_beforeunload_event(&located_script_name!())? + { + break; + } + self.worker.run_event_loop(false).await?; + } + self.worker.dispatch_unload_event(&located_script_name!())?; + Ok(()) + } + + pub async fn run_bench_specifier(&mut self) -> Result<(), AnyError> { + self.worker.js_runtime.execute_script( + &located_script_name!(), + r#"Deno[Deno.internal].enableTestAndBench()"#, + )?; + + if self.ps.options.compat() { + self.worker.execute_side_module(&compat::GLOBAL_URL).await?; + self.worker.execute_side_module(&compat::MODULE_URL).await?; + + let use_esm_loader = + compat::check_if_should_use_esm_loader(&self.main_module)?; + + if use_esm_loader { + self.worker.execute_side_module(&self.main_module).await?; + } else { + compat::load_cjs_module( + &mut self.worker.js_runtime, + &self + .main_module + .to_file_path() + .unwrap() + .display() + .to_string(), + false, + )?; + self.worker.run_event_loop(false).await?; + } + } else { + // We execute the module module as a side module so that import.meta.main is not set. + self.worker.execute_side_module(&self.main_module).await?; + } + + self.worker.dispatch_load_event(&located_script_name!())?; + + let bench_result = self.worker.js_runtime.execute_script( + &located_script_name!(), + r#"Deno[Deno.internal].runBenchmarks()"#, + )?; + + self.worker.js_runtime.resolve_value(bench_result).await?; + + loop { + if !self + .worker + .dispatch_beforeunload_event(&located_script_name!())? + { + break; + } + self.worker.run_event_loop(false).await?; + } + self.worker.dispatch_unload_event(&located_script_name!())?; + Ok(()) + } + + async fn maybe_setup_coverage_collector( + &mut self, + ) -> Result, AnyError> { + if let Some(ref coverage_dir) = self.ps.coverage_dir { + let session = self.worker.create_inspector_session().await; + + let coverage_dir = PathBuf::from(coverage_dir); + let mut coverage_collector = + tools::coverage::CoverageCollector::new(coverage_dir, session); + self + .worker + .with_event_loop(coverage_collector.start_collecting().boxed_local()) + .await?; + Ok(Some(coverage_collector)) + } else { + Ok(None) + } + } +} + +pub fn create_main_worker( + ps: &ProcState, + main_module: ModuleSpecifier, + permissions: Permissions, + mut custom_extensions: Vec, + stdio: deno_runtime::ops::io::Stdio, +) -> CliMainWorker { + let module_loader = CliModuleLoader::new(ps.clone()); + + let maybe_inspector_server = ps.maybe_inspector_server.clone(); + let should_break_on_first_statement = ps.options.inspect_brk().is_some(); + + let create_web_worker_cb = + create_web_worker_callback(ps.clone(), stdio.clone()); + let web_worker_preload_module_cb = + create_web_worker_preload_module_callback(ps.clone()); + + let maybe_storage_key = ps.options.resolve_storage_key(&main_module); + let origin_storage_dir = maybe_storage_key.map(|key| { + ps.dir + .root + // TODO(@crowlKats): change to origin_data for 2.0 + .join("location_data") + .join(checksum::gen(&[key.as_bytes()])) + }); + + let mut extensions = ops::cli_exts(ps.clone()); + extensions.append(&mut custom_extensions); + + let options = WorkerOptions { + bootstrap: BootstrapOptions { + args: ps.options.argv().clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + debug_flag: ps + .options + .log_level() + .map_or(false, |l| l == log::Level::Debug), + enable_testing_features: ps.options.enable_testing_features(), + location: ps.options.location_flag().map(ToOwned::to_owned), + no_color: !colors::use_color(), + is_tty: colors::is_tty(), + runtime_version: version::deno(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: ps.options.unstable(), + user_agent: version::get_user_agent(), + }, + extensions, + unsafely_ignore_certificate_errors: ps + .options + .unsafely_ignore_certificate_errors() + .map(ToOwned::to_owned), + root_cert_store: Some(ps.root_cert_store.clone()), + seed: ps.options.seed(), + source_map_getter: Some(Box::new(module_loader.clone())), + format_js_error_fn: Some(Arc::new(format_js_error)), + create_web_worker_cb, + web_worker_preload_module_cb, + maybe_inspector_server, + should_break_on_first_statement, + module_loader, + get_error_class_fn: Some(&errors::get_error_class_name), + origin_storage_dir, + blob_store: ps.blob_store.clone(), + broadcast_channel: ps.broadcast_channel.clone(), + shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), + stdio, + }; + + let worker = MainWorker::bootstrap_from_options( + main_module.clone(), + permissions, + options, + ); + CliMainWorker { + main_module, + worker, + ps: ps.clone(), + } +} + +fn create_web_worker_preload_module_callback( + ps: ProcState, +) -> Arc { + let compat = ps.options.compat(); + + Arc::new(move |mut worker| { + let fut = async move { + if compat { + worker.execute_side_module(&compat::GLOBAL_URL).await?; + worker.execute_side_module(&compat::MODULE_URL).await?; + } + + Ok(worker) + }; + LocalFutureObj::new(Box::new(fut)) + }) +} + +fn create_web_worker_callback( + ps: ProcState, + stdio: deno_runtime::ops::io::Stdio, +) -> Arc { + Arc::new(move |args| { + let maybe_inspector_server = ps.maybe_inspector_server.clone(); + + let module_loader = CliModuleLoader::new_for_worker( + ps.clone(), + args.parent_permissions.clone(), + ); + let create_web_worker_cb = + create_web_worker_callback(ps.clone(), stdio.clone()); + let preload_module_cb = + create_web_worker_preload_module_callback(ps.clone()); + + let extensions = ops::cli_exts(ps.clone()); + + let options = WebWorkerOptions { + bootstrap: BootstrapOptions { + args: ps.options.argv().clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + debug_flag: ps + .options + .log_level() + .map_or(false, |l| l == log::Level::Debug), + enable_testing_features: ps.options.enable_testing_features(), + location: Some(args.main_module.clone()), + no_color: !colors::use_color(), + is_tty: colors::is_tty(), + runtime_version: version::deno(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: ps.options.unstable(), + user_agent: version::get_user_agent(), + }, + extensions, + unsafely_ignore_certificate_errors: ps + .options + .unsafely_ignore_certificate_errors() + .map(ToOwned::to_owned), + root_cert_store: Some(ps.root_cert_store.clone()), + seed: ps.options.seed(), + create_web_worker_cb, + preload_module_cb, + format_js_error_fn: Some(Arc::new(format_js_error)), + source_map_getter: Some(Box::new(module_loader.clone())), + module_loader, + worker_type: args.worker_type, + maybe_inspector_server, + get_error_class_fn: Some(&errors::get_error_class_name), + blob_store: ps.blob_store.clone(), + broadcast_channel: ps.broadcast_channel.clone(), + shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), + stdio: stdio.clone(), + }; + + WebWorker::bootstrap_from_options( + args.name, + args.permissions, + args.main_module, + args.worker_id, + options, + ) + }) +}