diff --git a/Cargo.lock b/Cargo.lock index aadd0c5f95..7ec72aa3cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,6 +1270,7 @@ dependencies = [ "deno_doc", "deno_error", "deno_graph", + "deno_lib", "deno_lint", "deno_lockfile", "deno_npm", @@ -1891,6 +1892,31 @@ dependencies = [ "url", ] +[[package]] +name = "deno_lib" +version = "0.1.1" +dependencies = [ + "deno_cache_dir", + "deno_error", + "deno_fs", + "deno_node", + "deno_path_util", + "deno_resolver", + "deno_runtime", + "deno_terminal 0.2.0", + "faster-hex", + "log", + "node_resolver", + "parking_lot", + "ring", + "serde", + "sys_traits", + "test_server", + "thiserror 2.0.3", + "tokio", + "url", +] + [[package]] name = "deno_lint" version = "0.68.2" diff --git a/Cargo.toml b/Cargo.toml index d9f976d863..0c11ff9a69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ resolver = "2" members = [ "bench_util", "cli", + "cli/lib", "ext/broadcast_channel", "ext/cache", "ext/canvas", @@ -94,7 +95,8 @@ deno_webidl = { version = "0.185.0", path = "./ext/webidl" } deno_websocket = { version = "0.190.0", path = "./ext/websocket" } deno_webstorage = { version = "0.180.0", path = "./ext/webstorage" } -# resolvers +# workspace libraries +deno_lib = { version = "=0.1.1", path = "./cli/lib" } deno_npm_cache = { version = "0.4.0", path = "./resolvers/npm_cache" } deno_resolver = { version = "0.16.0", path = "./resolvers/deno" } node_resolver = { version = "0.23.0", path = "./resolvers/node" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8248d40701..4525a1bab4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -76,6 +76,7 @@ deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] deno_doc = { version = "=0.164.0", features = ["rust", "comrak"] } deno_error.workspace = true deno_graph = { version = "=0.87.0" } +deno_lib.workspace = true deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 4ba85f1706..29b493046f 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -58,6 +58,9 @@ use deno_core::serde_json; use deno_core::url::Url; use deno_graph::GraphKind; pub use deno_json::check_warn_tsconfig; +use deno_lib::cache::DenoDirProvider; +use deno_lib::env::has_flag_env_var; +use deno_lib::worker::StorageKeyResolver; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::NpmRc; use deno_npm::npm_rc::ResolvedNpmRc; @@ -89,7 +92,6 @@ use serde::Serialize; use sys_traits::EnvHomeDir; use thiserror::Error; -use crate::cache::DenoDirProvider; use crate::file_fetcher::CliFileFetcher; use crate::sys::CliSys; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -768,7 +770,7 @@ pub struct CliOptions { maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, overrides: CliOptionOverrides, pub start_dir: Arc, - pub deno_dir_provider: Arc, + pub deno_dir_provider: Arc>, } impl CliOptions { @@ -1227,6 +1229,16 @@ impl CliOptions { } } + pub fn resolve_storage_key_resolver(&self) -> StorageKeyResolver { + if let Some(location) = &self.flags.location { + StorageKeyResolver::from_flag(location) + } else if let Some(deno_json) = self.start_dir.maybe_deno_json() { + StorageKeyResolver::from_config_file_url(&deno_json.specifier) + } else { + StorageKeyResolver::new_use_main_module() + } + } + // If the main module should be treated as being in an npm package. // This is triggered via a secret environment variable which is used // for functionality like child_process.fork. Users should NOT depend @@ -1871,7 +1883,7 @@ fn resolve_node_modules_folder( cwd: &Path, flags: &Flags, workspace: &Workspace, - deno_dir_provider: &Arc, + deno_dir_provider: &Arc>, ) -> Result, AnyError> { fn resolve_from_root(root_folder: &FolderConfigs, cwd: &Path) -> PathBuf { root_folder @@ -1975,63 +1987,11 @@ fn resolve_import_map_specifier( } } -pub struct StorageKeyResolver(Option>); - -impl StorageKeyResolver { - pub fn from_options(options: &CliOptions) -> Self { - Self(if let Some(location) = &options.flags.location { - // if a location is set, then the ascii serialization of the location is - // used, unless the origin is opaque, and then no storage origin is set, as - // we can't expect the origin to be reproducible - let storage_origin = location.origin(); - if storage_origin.is_tuple() { - Some(Some(storage_origin.ascii_serialization())) - } else { - Some(None) - } - } else { - // otherwise we will use the path to the config file or None to - // fall back to using the main module's path - options - .start_dir - .maybe_deno_json() - .map(|config_file| Some(config_file.specifier.to_string())) - }) - } - - /// Creates a storage key resolver that will always resolve to being empty. - pub fn empty() -> Self { - Self(Some(None)) - } - - /// Resolves the storage key to use based on the current flags, config, or main module. - pub fn resolve_storage_key( - &self, - main_module: &ModuleSpecifier, - ) -> Option { - // use the stored value or fall back to using the path of the main module. - if let Some(maybe_value) = &self.0 { - maybe_value.clone() - } else { - Some(main_module.to_string()) - } - } -} - /// Resolves the no_prompt value based on the cli flags and environment. pub fn resolve_no_prompt(flags: &PermissionFlags) -> bool { flags.no_prompt || has_flag_env_var("DENO_NO_PROMPT") } -pub fn has_trace_permissions_enabled() -> bool { - has_flag_env_var("DENO_TRACE_PERMISSIONS") -} - -pub fn has_flag_env_var(name: &str) -> bool { - let value = env::var(name); - matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) -} - pub fn npm_pkg_req_ref_to_binary_command( req_ref: &NpmPackageReqReference, ) -> String { @@ -2160,27 +2120,6 @@ mod test { assert_eq!(actual, None); } - #[test] - fn storage_key_resolver_test() { - let resolver = StorageKeyResolver(None); - let specifier = ModuleSpecifier::parse("file:///a.ts").unwrap(); - assert_eq!( - resolver.resolve_storage_key(&specifier), - Some(specifier.to_string()) - ); - let resolver = StorageKeyResolver(Some(None)); - assert_eq!(resolver.resolve_storage_key(&specifier), None); - let resolver = StorageKeyResolver(Some(Some("value".to_string()))); - assert_eq!( - resolver.resolve_storage_key(&specifier), - Some("value".to_string()) - ); - - // test empty - let resolver = StorageKeyResolver::empty(); - assert_eq!(resolver.resolve_storage_key(&specifier), None); - } - #[test] fn jsr_urls() { let reg_url = jsr_url(); diff --git a/cli/cache/caches.rs b/cli/cache/caches.rs index b83364c61b..dd4a974814 100644 --- a/cli/cache/caches.rs +++ b/cli/cache/caches.rs @@ -3,20 +3,21 @@ use std::path::PathBuf; use std::sync::Arc; +use deno_lib::cache::DenoDirProvider; use once_cell::sync::OnceCell; use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::check::TYPE_CHECK_CACHE_DB; use super::code_cache::CODE_CACHE_DB; -use super::deno_dir::DenoDirProvider; use super::fast_check::FAST_CHECK_CACHE_DB; use super::incremental::INCREMENTAL_CACHE_DB; use super::module_info::MODULE_INFO_CACHE_DB; use super::node::NODE_ANALYSIS_CACHE_DB; +use crate::sys::CliSys; pub struct Caches { - dir_provider: Arc, + dir_provider: Arc>, fmt_incremental_cache_db: OnceCell, lint_incremental_cache_db: OnceCell, dep_analysis_db: OnceCell, @@ -27,7 +28,7 @@ pub struct Caches { } impl Caches { - pub fn new(dir: Arc) -> Self { + pub fn new(dir: Arc>) -> Self { Self { dir_provider: dir, fmt_incremental_cache_db: Default::default(), diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 2ba43d58b9..e8a940b3be 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -6,19 +6,20 @@ use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::unsync::sync::AtomicFlag; +use deno_lib::cache::DiskCache; -use super::DiskCache; +use crate::sys::CliSys; /// The cache that stores previously emitted files. #[derive(Debug)] pub struct EmitCache { - disk_cache: DiskCache, + disk_cache: DiskCache, emit_failed_flag: AtomicFlag, file_serializer: EmitFileSerializer, } impl EmitCache { - pub fn new(disk_cache: DiskCache) -> Self { + pub fn new(disk_cache: DiskCache) -> Self { Self { disk_cache, emit_failed_flag: Default::default(), diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 0d7808cba6..e16f95e56f 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -31,8 +31,6 @@ mod caches; mod check; mod code_cache; mod common; -mod deno_dir; -mod disk_cache; mod emit; mod fast_check; mod incremental; @@ -47,9 +45,6 @@ pub use code_cache::CodeCache; pub use common::FastInsecureHasher; /// Permissions used to save a file in the disk caches. pub use deno_cache_dir::CACHE_PERM; -pub use deno_dir::DenoDir; -pub use deno_dir::DenoDirProvider; -pub use disk_cache::DiskCache; pub use emit::EmitCache; pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; diff --git a/cli/factory.rs b/cli/factory.rs index 3280fd379b..bfe6d05570 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,6 +11,12 @@ use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::FeatureChecker; use deno_error::JsErrorBox; +use deno_lib::cache::DenoDir; +use deno_lib::cache::DenoDirProvider; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; +use deno_lib::worker::LibMainWorkerFactory; +use deno_lib::worker::LibMainWorkerOptions; use deno_npm_cache::NpmCacheSetting; use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; @@ -42,12 +48,9 @@ use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::NpmInstallDepsProvider; -use crate::args::StorageKeyResolver; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; -use crate::cache::DenoDir; -use crate::cache::DenoDirProvider; use crate::cache::EmitCache; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; @@ -68,6 +71,7 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; +use crate::npm::create_npm_process_state_provider; use crate::npm::installer::NpmInstaller; use crate::npm::installer::NpmResolutionInstaller; use crate::npm::CliByonmNpmResolverCreateOptions; @@ -79,8 +83,6 @@ use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::CliNpmTarballCache; -use crate::npm::NpmRegistryReadPermissionChecker; -use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; use crate::resolver::CliCjsTracker; use crate::resolver::CliDenoResolver; @@ -281,11 +283,13 @@ impl CliFactory { }) } - pub fn deno_dir_provider(&self) -> Result<&Arc, AnyError> { + pub fn deno_dir_provider( + &self, + ) -> Result<&Arc>, AnyError> { Ok(&self.cli_options()?.deno_dir_provider) } - pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { + pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { Ok(self.deno_dir_provider()?.get_or_create()?) } @@ -1083,7 +1087,34 @@ impl CliFactory { Arc::new(NpmRegistryReadPermissionChecker::new(self.sys(), mode)) }; - Ok(CliMainWorkerFactory::new( + let module_loader_factory = CliModuleLoaderFactory::new( + cli_options, + cjs_tracker, + if cli_options.code_cache_enabled() { + Some(self.code_cache()?.clone()) + } else { + None + }, + self.emitter()?.clone(), + in_npm_pkg_checker.clone(), + self.main_module_graph_container().await?.clone(), + self.module_load_preparer().await?.clone(), + node_code_translator.clone(), + node_resolver.clone(), + NpmModuleLoader::new( + self.cjs_tracker()?.clone(), + fs.clone(), + node_code_translator.clone(), + ), + npm_registry_permission_checker, + npm_req_resolver.clone(), + cli_npm_resolver.clone(), + self.parsed_source_cache().clone(), + self.resolver().await?.clone(), + self.sys(), + ); + + let lib_main_worker_factory = LibMainWorkerFactory::new( self.blob_store().clone(), if cli_options.code_cache_enabled() { Some(self.code_cache()?.clone()) @@ -1092,50 +1123,70 @@ impl CliFactory { }, self.feature_checker()?.clone(), fs.clone(), - maybe_file_watcher_communicator, self.maybe_inspector_server()?.clone(), + Box::new(module_loader_factory), + node_resolver.clone(), + create_npm_process_state_provider(npm_resolver), + pkg_json_resolver, + self.root_cert_store_provider().clone(), + cli_options.resolve_storage_key_resolver(), + self.sys(), + self.create_lib_main_worker_options()?, + ); + + Ok(CliMainWorkerFactory::new( + lib_main_worker_factory, + maybe_file_watcher_communicator, cli_options.maybe_lockfile().cloned(), - Box::new(CliModuleLoaderFactory::new( - cli_options, - cjs_tracker, - if cli_options.code_cache_enabled() { - Some(self.code_cache()?.clone()) - } else { - None - }, - self.emitter()?.clone(), - in_npm_pkg_checker.clone(), - self.main_module_graph_container().await?.clone(), - self.module_load_preparer().await?.clone(), - node_code_translator.clone(), - node_resolver.clone(), - NpmModuleLoader::new( - self.cjs_tracker()?.clone(), - fs.clone(), - node_code_translator.clone(), - ), - npm_registry_permission_checker, - npm_req_resolver.clone(), - cli_npm_resolver.clone(), - self.parsed_source_cache().clone(), - self.resolver().await?.clone(), - self.sys(), - )), node_resolver.clone(), self.npm_installer_if_managed()?.cloned(), npm_resolver.clone(), - pkg_json_resolver, - self.root_cert_store_provider().clone(), - self.root_permissions_container()?.clone(), - StorageKeyResolver::from_options(cli_options), self.sys(), - cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, - self.cli_options()?.otel_config(), - self.cli_options()?.default_npm_caching_strategy(), + self.root_permissions_container()?.clone(), )) } + fn create_lib_main_worker_options( + &self, + ) -> Result { + let cli_options = self.cli_options()?; + Ok(LibMainWorkerOptions { + argv: cli_options.argv().clone(), + // This optimization is only available for "run" subcommand + // because we need to register new ops for testing and jupyter + // integration. + skip_op_registration: cli_options.sub_command().is_run(), + log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(), + enable_op_summary_metrics: cli_options.enable_op_summary_metrics(), + enable_testing_features: cli_options.enable_testing_features(), + has_node_modules_dir: cli_options.has_node_modules_dir(), + inspect_brk: cli_options.inspect_brk().is_some(), + inspect_wait: cli_options.inspect_wait().is_some(), + strace_ops: cli_options.strace_ops().clone(), + is_inspecting: cli_options.is_inspecting(), + location: cli_options.location_flag().clone(), + // if the user ran a binary command, we'll need to set process.argv[0] + // to be the name of the binary command instead of deno + argv0: cli_options + .take_binary_npm_command_name() + .or(std::env::args().next()), + node_debug: std::env::var("NODE_DEBUG").ok(), + origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()), + seed: cli_options.seed(), + unsafely_ignore_certificate_errors: cli_options + .unsafely_ignore_certificate_errors() + .clone(), + node_ipc: cli_options.node_ipc_fd(), + serve_port: cli_options.serve_port(), + serve_host: cli_options.serve_host(), + deno_version: crate::version::DENO_VERSION_INFO.deno, + deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, + otel_config: self.cli_options()?.otel_config(), + startup_snapshot: crate::js::deno_isolate_init(), + }) + } + fn create_cli_main_worker_options( &self, ) -> Result { @@ -1167,37 +1218,10 @@ impl CliFactory { }; Ok(CliMainWorkerOptions { - argv: cli_options.argv().clone(), - // This optimization is only available for "run" subcommand - // because we need to register new ops for testing and jupyter - // integration. - skip_op_registration: cli_options.sub_command().is_run(), - log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(), - enable_op_summary_metrics: cli_options.enable_op_summary_metrics(), - enable_testing_features: cli_options.enable_testing_features(), - has_node_modules_dir: cli_options.has_node_modules_dir(), - hmr: cli_options.has_hmr(), - inspect_brk: cli_options.inspect_brk().is_some(), - inspect_wait: cli_options.inspect_wait().is_some(), - strace_ops: cli_options.strace_ops().clone(), - is_inspecting: cli_options.is_inspecting(), - location: cli_options.location_flag().clone(), - // if the user ran a binary command, we'll need to set process.argv[0] - // to be the name of the binary command instead of deno - argv0: cli_options - .take_binary_npm_command_name() - .or(std::env::args().next()), - node_debug: std::env::var("NODE_DEBUG").ok(), - origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()), - seed: cli_options.seed(), - unsafely_ignore_certificate_errors: cli_options - .unsafely_ignore_certificate_errors() - .clone(), + needs_test_modules: cli_options.sub_command().needs_test(), create_hmr_runner, create_coverage_collector, - node_ipc: cli_options.node_ipc_fd(), - serve_port: cli_options.serve_port(), - serve_host: cli_options.serve_host(), + default_npm_caching_strategy: cli_options.default_npm_caching_strategy(), }) } } diff --git a/cli/lib/Cargo.toml b/cli/lib/Cargo.toml new file mode 100644 index 0000000000..6a74b22c2b --- /dev/null +++ b/cli/lib/Cargo.toml @@ -0,0 +1,37 @@ +# Copyright 2018-2025 the Deno authors. MIT license. + +[package] +name = "deno_lib" +version = "0.1.1" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Shared code between the Deno CLI and denort" + +[lib] +path = "lib.rs" + +[dependencies] +deno_cache_dir.workspace = true +deno_error.workspace = true +deno_fs = { workspace = true, features = ["sync_fs"] } +deno_node = { workspace = true, features = ["sync_fs"] } +deno_path_util.workspace = true +deno_resolver = { workspace = true, features = ["sync"] } +deno_runtime.workspace = true +deno_terminal.workspace = true +faster-hex.workspace = true +log.workspace = true +node_resolver = { workspace = true, features = ["sync"] } +parking_lot.workspace = true +ring.workspace = true +serde = { workspace = true, features = ["derive"] } +sys_traits.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true + +[dev-dependencies] +test_util.workspace = true diff --git a/cli/lib/README.md b/cli/lib/README.md new file mode 100644 index 0000000000..bc6d7b57d0 --- /dev/null +++ b/cli/lib/README.md @@ -0,0 +1,4 @@ +# deno_lib + +This crate contains the shared code between the Deno CLI and denort. It is +highly unstable. diff --git a/cli/cache/deno_dir.rs b/cli/lib/cache/deno_dir.rs similarity index 90% rename from cli/cache/deno_dir.rs rename to cli/lib/cache/deno_dir.rs index 1b35f53071..00bc83ff9b 100644 --- a/cli/cache/deno_dir.rs +++ b/cli/lib/cache/deno_dir.rs @@ -4,21 +4,20 @@ use std::env; use std::path::PathBuf; use deno_cache_dir::DenoDirResolutionError; -use once_cell::sync::OnceCell; use super::DiskCache; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; /// Lazily creates the deno dir which might be useful in scenarios /// where functionality wants to continue if the DENO_DIR can't be created. -pub struct DenoDirProvider { - sys: CliSys, +pub struct DenoDirProvider { + sys: TSys, maybe_custom_root: Option, - deno_dir: OnceCell>, + deno_dir: std::sync::OnceLock, DenoDirResolutionError>>, } -impl DenoDirProvider { - pub fn new(sys: CliSys, maybe_custom_root: Option) -> Self { +impl DenoDirProvider { + pub fn new(sys: TSys, maybe_custom_root: Option) -> Self { Self { sys, maybe_custom_root, @@ -26,7 +25,9 @@ impl DenoDirProvider { } } - pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> { + pub fn get_or_create( + &self, + ) -> Result<&DenoDir, DenoDirResolutionError> { self .deno_dir .get_or_init(|| { @@ -49,16 +50,16 @@ impl DenoDirProvider { /// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them /// in single directory that can be controlled with `$DENO_DIR` env variable. #[derive(Debug, Clone)] -pub struct DenoDir { +pub struct DenoDir { /// Example: /Users/rld/.deno/ pub root: PathBuf, /// Used by TsCompiler to cache compiler output. - pub gen_cache: DiskCache, + pub gen_cache: DiskCache, } -impl DenoDir { +impl DenoDir { pub fn new( - sys: CliSys, + sys: TSys, maybe_custom_root: Option, ) -> Result { let root = deno_cache_dir::resolve_deno_dir( diff --git a/cli/cache/disk_cache.rs b/cli/lib/cache/disk_cache.rs similarity index 92% rename from cli/cache/disk_cache.rs rename to cli/lib/cache/disk_cache.rs index f03b60854f..2c735a34b2 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/lib/cache/disk_cache.rs @@ -9,22 +9,22 @@ use std::path::Prefix; use std::str; use deno_cache_dir::url_to_filename; -use deno_core::url::Host; -use deno_core::url::Url; +use deno_cache_dir::CACHE_PERM; use deno_path_util::fs::atomic_write_file_with_retries; +use url::Host; +use url::Url; -use super::CACHE_PERM; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; #[derive(Debug, Clone)] -pub struct DiskCache { - sys: CliSys, +pub struct DiskCache { + sys: TSys, pub location: PathBuf, } -impl DiskCache { +impl DiskCache { /// `location` must be an absolute path. - pub fn new(sys: CliSys, location: &Path) -> Self { + pub fn new(sys: TSys, location: &Path) -> Self { assert!(location.is_absolute()); Self { sys, @@ -130,6 +130,9 @@ impl DiskCache { #[cfg(test)] mod tests { + // ok, testing + #[allow(clippy::disallowed_types)] + use sys_traits::impls::RealSys; use test_util::TempDir; use super::*; @@ -138,7 +141,7 @@ mod tests { fn test_set_get_cache_file() { let temp_dir = TempDir::new(); let sub_dir = temp_dir.path().join("sub_dir"); - let cache = DiskCache::new(CliSys::default(), &sub_dir.to_path_buf()); + let cache = DiskCache::new(RealSys, &sub_dir.to_path_buf()); let path = PathBuf::from("foo/bar.txt"); cache.set(&path, b"hello").unwrap(); assert_eq!(cache.get(&path).unwrap(), b"hello"); @@ -152,7 +155,7 @@ mod tests { PathBuf::from("/deno_dir/") }; - let cache = DiskCache::new(CliSys::default(), &cache_location); + let cache = DiskCache::new(RealSys, &cache_location); let mut test_cases = vec![ ( @@ -208,7 +211,7 @@ mod tests { } else { "/foo" }; - let cache = DiskCache::new(CliSys::default(), &PathBuf::from(p)); + let cache = DiskCache::new(RealSys, &PathBuf::from(p)); let mut test_cases = vec![ ( @@ -256,7 +259,7 @@ mod tests { PathBuf::from("/deno_dir/") }; - let cache = DiskCache::new(CliSys::default(), &cache_location); + let cache = DiskCache::new(RealSys, &cache_location); let mut test_cases = vec!["unknown://localhost/test.ts"]; diff --git a/cli/lib/cache/mod.rs b/cli/lib/cache/mod.rs new file mode 100644 index 0000000000..c4395df3e1 --- /dev/null +++ b/cli/lib/cache/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub use deno_dir::DenoDir; +pub use deno_dir::DenoDirProvider; +pub use disk_cache::DiskCache; + +mod deno_dir; +mod disk_cache; diff --git a/cli/lib/env.rs b/cli/lib/env.rs new file mode 100644 index 0000000000..9c6001478b --- /dev/null +++ b/cli/lib/env.rs @@ -0,0 +1,10 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub fn has_trace_permissions_enabled() -> bool { + has_flag_env_var("DENO_TRACE_PERMISSIONS") +} + +pub fn has_flag_env_var(name: &str) -> bool { + let value = std::env::var(name); + matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) +} diff --git a/cli/lib/lib.rs b/cli/lib/lib.rs new file mode 100644 index 0000000000..5453bddaee --- /dev/null +++ b/cli/lib/lib.rs @@ -0,0 +1,9 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod cache; +pub mod env; +pub mod npm; +pub mod standalone; +pub mod sys; +pub mod util; +pub mod worker; diff --git a/cli/lib/npm/mod.rs b/cli/lib/npm/mod.rs new file mode 100644 index 0000000000..e7d4d8d9d1 --- /dev/null +++ b/cli/lib/npm/mod.rs @@ -0,0 +1,6 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +mod permission_checker; + +pub use permission_checker::NpmRegistryReadPermissionChecker; +pub use permission_checker::NpmRegistryReadPermissionCheckerMode; diff --git a/cli/npm/permission_checker.rs b/cli/lib/npm/permission_checker.rs similarity index 92% rename from cli/npm/permission_checker.rs rename to cli/lib/npm/permission_checker.rs index 53031b5bd4..ebed1270f3 100644 --- a/cli/npm/permission_checker.rs +++ b/cli/lib/npm/permission_checker.rs @@ -6,12 +6,11 @@ use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; -use deno_core::parking_lot::Mutex; use deno_error::JsErrorBox; use deno_runtime::deno_node::NodePermissions; -use sys_traits::FsCanonicalize; +use parking_lot::Mutex; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; #[derive(Debug)] pub enum NpmRegistryReadPermissionCheckerMode { @@ -21,8 +20,8 @@ pub enum NpmRegistryReadPermissionCheckerMode { } #[derive(Debug)] -pub struct NpmRegistryReadPermissionChecker { - sys: CliSys, +pub struct NpmRegistryReadPermissionChecker { + sys: TSys, cache: Mutex>, mode: NpmRegistryReadPermissionCheckerMode, } @@ -37,8 +36,8 @@ struct EnsureRegistryReadPermissionError { source: std::io::Error, } -impl NpmRegistryReadPermissionChecker { - pub fn new(sys: CliSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self { +impl NpmRegistryReadPermissionChecker { + pub fn new(sys: TSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self { Self { sys, cache: Default::default(), diff --git a/cli/lib/standalone/mod.rs b/cli/lib/standalone/mod.rs new file mode 100644 index 0000000000..6e173a457a --- /dev/null +++ b/cli/lib/standalone/mod.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod virtual_fs; diff --git a/cli/lib/standalone/virtual_fs.rs b/cli/lib/standalone/virtual_fs.rs new file mode 100644 index 0000000000..5fc17f27b7 --- /dev/null +++ b/cli/lib/standalone/virtual_fs.rs @@ -0,0 +1,296 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::cmp::Ordering; +use std::path::Path; +use std::path::PathBuf; + +use serde::Deserialize; +use serde::Serialize; + +#[derive(Debug, Copy, Clone)] +pub enum VfsFileSubDataKind { + /// Raw bytes of the file. + Raw, + /// Bytes to use for module loading. For example, for TypeScript + /// files this will be the transpiled JavaScript source. + ModuleGraph, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!( + !name_component.contains('\\'), + "Invalid component: {}", + name_component + ); + debug_assert!( + !name_component.contains('/'), + "Invalid component: {}", + name_component + ); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +pub enum FileSystemCaseSensitivity { + #[serde(rename = "s")] + Sensitive, + #[serde(rename = "i")] + Insensitive, +} +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct VirtualDirectoryEntries(Vec); + +impl VirtualDirectoryEntries { + pub fn new(mut entries: Vec) -> Self { + // needs to be sorted by name + entries.sort_by(|a, b| a.name().cmp(b.name())); + Self(entries) + } + + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, VfsEntry> { + self.0.iter_mut() + } + + pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> { + self.0.iter() + } + + pub fn take_inner(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get_by_name( + &self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Option<&VfsEntry> { + self + .binary_search(name, case_sensitivity) + .ok() + .map(|index| &self.0[index]) + } + + pub fn get_mut_by_name( + &mut self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Option<&mut VfsEntry> { + self + .binary_search(name, case_sensitivity) + .ok() + .map(|index| &mut self.0[index]) + } + + pub fn get_mut_by_index(&mut self, index: usize) -> Option<&mut VfsEntry> { + self.0.get_mut(index) + } + + pub fn binary_search( + &self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Result { + match case_sensitivity { + FileSystemCaseSensitivity::Sensitive => { + self.0.binary_search_by(|e| e.name().cmp(name)) + } + FileSystemCaseSensitivity::Insensitive => self.0.binary_search_by(|e| { + e.name() + .chars() + .zip(name.chars()) + .map(|(a, b)| a.to_ascii_lowercase().cmp(&b.to_ascii_lowercase())) + .find(|&ord| ord != Ordering::Equal) + .unwrap_or_else(|| e.name().len().cmp(&name.len())) + }), + } + } + + pub fn insert( + &mut self, + entry: VfsEntry, + case_sensitivity: FileSystemCaseSensitivity, + ) -> usize { + match self.binary_search(entry.name(), case_sensitivity) { + Ok(index) => { + self.0[index] = entry; + index + } + Err(insert_index) => { + self.0.insert(insert_index, entry); + insert_index + } + } + } + + pub fn insert_or_modify( + &mut self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + on_insert: impl FnOnce() -> VfsEntry, + on_modify: impl FnOnce(&mut VfsEntry), + ) -> usize { + match self.binary_search(name, case_sensitivity) { + Ok(index) => { + on_modify(&mut self.0[index]); + index + } + Err(insert_index) => { + self.0.insert(insert_index, on_insert()); + insert_index + } + } + } + + pub fn remove(&mut self, index: usize) -> VfsEntry { + self.0.remove(index) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualDirectory { + #[serde(rename = "n")] + pub name: String, + // should be sorted by name + #[serde(rename = "e")] + pub entries: VirtualDirectoryEntries, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct OffsetWithLength { + #[serde(rename = "o")] + pub offset: u64, + #[serde(rename = "l")] + pub len: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VirtualFile { + #[serde(rename = "n")] + pub name: String, + #[serde(rename = "o")] + pub offset: OffsetWithLength, + /// Offset file to use for module loading when it differs from the + /// raw file. Often this will be the same offset as above for data + /// such as JavaScript files, but for TypeScript files the `offset` + /// will be the original raw bytes when included as an asset and this + /// offset will be to the transpiled JavaScript source. + #[serde(rename = "m")] + pub module_graph_offset: OffsetWithLength, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } + + pub fn take_parts(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + + pub fn parts(&self) -> &[String] { + &self.0 + } + + pub fn set_parts(&mut self, parts: Vec) { + self.0 = parts; + } + + pub fn display(&self) -> String { + self.0.join("/") + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlink { + #[serde(rename = "n")] + pub name: String, + #[serde(rename = "p")] + pub dest_parts: VirtualSymlinkParts, +} + +impl VirtualSymlink { + pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { + let mut dest = root.to_path_buf(); + for part in &self.dest_parts.0 { + dest.push(part); + } + dest + } +} + +#[derive(Debug, Copy, Clone)] +pub enum VfsEntryRef<'a> { + Dir(&'a VirtualDirectory), + File(&'a VirtualFile), + Symlink(&'a VirtualSymlink), +} + +impl VfsEntryRef<'_> { + pub fn name(&self) -> &str { + match self { + Self::Dir(dir) => &dir.name, + Self::File(file) => &file.name, + Self::Symlink(symlink) => &symlink.name, + } + } +} + +// todo(dsherret): we should store this more efficiently in the binary +#[derive(Debug, Serialize, Deserialize)] +pub enum VfsEntry { + Dir(VirtualDirectory), + File(VirtualFile), + Symlink(VirtualSymlink), +} + +impl VfsEntry { + pub fn name(&self) -> &str { + match self { + Self::Dir(dir) => &dir.name, + Self::File(file) => &file.name, + Self::Symlink(symlink) => &symlink.name, + } + } + + pub fn as_ref(&self) -> VfsEntryRef { + match self { + VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir), + VfsEntry::File(file) => VfsEntryRef::File(file), + VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink), + } + } +} diff --git a/cli/lib/sys.rs b/cli/lib/sys.rs new file mode 100644 index 0000000000..f5ca48b41c --- /dev/null +++ b/cli/lib/sys.rs @@ -0,0 +1,37 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use deno_node::ExtNodeSys; +use sys_traits::FsCanonicalize; +use sys_traits::FsCreateDirAll; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsRead; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; + +pub trait DenoLibSys: + FsCanonicalize + + FsCreateDirAll + + FsReadDir + + FsMetadata + + FsOpen + + FsRemoveFile + + FsRename + + FsRead + + ThreadSleep + + SystemRandom + + ExtNodeSys + + Clone + + Send + + Sync + + std::fmt::Debug + + 'static +{ +} + +// ok, implementation +#[allow(clippy::disallowed_types)] +impl DenoLibSys for sys_traits::impls::RealSys {} diff --git a/cli/util/checksum.rs b/cli/lib/util/checksum.rs similarity index 100% rename from cli/util/checksum.rs rename to cli/lib/util/checksum.rs diff --git a/cli/lib/util/mod.rs b/cli/lib/util/mod.rs new file mode 100644 index 0000000000..8371440750 --- /dev/null +++ b/cli/lib/util/mod.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod checksum; diff --git a/cli/lib/worker.rs b/cli/lib/worker.rs new file mode 100644 index 0000000000..7c9071d0ba --- /dev/null +++ b/cli/lib/worker.rs @@ -0,0 +1,581 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use deno_core::error::JsError; +use deno_node::NodeRequireLoaderRc; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmResolver; +use deno_runtime::colors; +use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; +use deno_runtime::deno_core; +use deno_runtime::deno_core::error::CoreError; +use deno_runtime::deno_core::v8; +use deno_runtime::deno_core::CompiledWasmModuleStore; +use deno_runtime::deno_core::Extension; +use deno_runtime::deno_core::FeatureChecker; +use deno_runtime::deno_core::JsRuntime; +use deno_runtime::deno_core::LocalInspectorSession; +use deno_runtime::deno_core::ModuleLoader; +use deno_runtime::deno_core::SharedArrayBufferStore; +use deno_runtime::deno_fs; +use deno_runtime::deno_node::NodeExtInitServices; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_permissions::PermissionsContainer; +use deno_runtime::deno_telemetry::OtelConfig; +use deno_runtime::deno_tls::RootCertStoreProvider; +use deno_runtime::deno_web::BlobStore; +use deno_runtime::fmt_errors::format_js_error; +use deno_runtime::inspector_server::InspectorServer; +use deno_runtime::ops::process::NpmProcessStateProviderRc; +use deno_runtime::ops::worker_host::CreateWebWorkerCb; +use deno_runtime::web_worker::WebWorker; +use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::web_worker::WebWorkerServiceOptions; +use deno_runtime::worker::MainWorker; +use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; +use deno_runtime::BootstrapOptions; +use deno_runtime::WorkerExecutionMode; +use deno_runtime::WorkerLogLevel; +use deno_runtime::UNSTABLE_GRANULAR_FLAGS; +use url::Url; + +use crate::env::has_trace_permissions_enabled; +use crate::sys::DenoLibSys; +use crate::util::checksum; + +pub struct CreateModuleLoaderResult { + pub module_loader: Rc, + pub node_require_loader: Rc, +} + +pub trait ModuleLoaderFactory: Send + Sync { + fn create_for_main( + &self, + root_permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult; + + fn create_for_worker( + &self, + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult; +} + +enum StorageKeyResolverStrategy { + Specified(Option), + UseMainModule, +} + +pub struct StorageKeyResolver(StorageKeyResolverStrategy); + +impl StorageKeyResolver { + pub fn from_flag(location: &Url) -> Self { + // if a location is set, then the ascii serialization of the location is + // used, unless the origin is opaque, and then no storage origin is set, as + // we can't expect the origin to be reproducible + let storage_origin = location.origin(); + Self(StorageKeyResolverStrategy::Specified( + if storage_origin.is_tuple() { + Some(storage_origin.ascii_serialization()) + } else { + None + }, + )) + } + + pub fn from_config_file_url(url: &Url) -> Self { + Self(StorageKeyResolverStrategy::Specified(Some(url.to_string()))) + } + + pub fn new_use_main_module() -> Self { + Self(StorageKeyResolverStrategy::UseMainModule) + } + + /// Creates a storage key resolver that will always resolve to being empty. + pub fn empty() -> Self { + Self(StorageKeyResolverStrategy::Specified(None)) + } + + /// Resolves the storage key to use based on the current flags, config, or main module. + pub fn resolve_storage_key(&self, main_module: &Url) -> Option { + // use the stored value or fall back to using the path of the main module. + match &self.0 { + StorageKeyResolverStrategy::Specified(value) => value.clone(), + StorageKeyResolverStrategy::UseMainModule => { + Some(main_module.to_string()) + } + } + } +} + +// TODO(bartlomieju): this should be moved to some other place, added to avoid string +// duplication between worker setups and `deno info` output. +pub fn get_cache_storage_dir() -> PathBuf { + // Note: we currently use temp_dir() to avoid managing storage size. + std::env::temp_dir().join("deno_cache") +} + +/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs. +/// Instead probe for the total memory on the system and use it instead +/// as a default. +pub fn create_isolate_create_params() -> Option { + let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info(); + maybe_mem_info.map(|mem_info| { + v8::CreateParams::default() + .heap_limits_from_system_memory(mem_info.total, 0) + }) +} + +pub struct LibMainWorkerOptions { + pub argv: Vec, + pub deno_version: &'static str, + pub deno_user_agent: &'static str, + pub log_level: WorkerLogLevel, + pub enable_op_summary_metrics: bool, + pub enable_testing_features: bool, + pub has_node_modules_dir: bool, + pub inspect_brk: bool, + pub inspect_wait: bool, + pub strace_ops: Option>, + pub is_inspecting: bool, + pub location: Option, + pub argv0: Option, + pub node_debug: Option, + pub otel_config: OtelConfig, + pub origin_data_folder_path: Option, + pub seed: Option, + pub unsafely_ignore_certificate_errors: Option>, + pub skip_op_registration: bool, + pub node_ipc: Option, + pub startup_snapshot: Option<&'static [u8]>, + pub serve_port: Option, + pub serve_host: Option, +} + +struct LibWorkerFactorySharedState { + blob_store: Arc, + broadcast_channel: InMemoryBroadcastChannel, + code_cache: Option>, + compiled_wasm_module_store: CompiledWasmModuleStore, + feature_checker: Arc, + fs: Arc, + maybe_inspector_server: Option>, + module_loader_factory: Box, + node_resolver: + Arc, TSys>>, + npm_process_state_provider: NpmProcessStateProviderRc, + pkg_json_resolver: Arc>, + root_cert_store_provider: Arc, + shared_array_buffer_store: SharedArrayBufferStore, + storage_key_resolver: StorageKeyResolver, + sys: TSys, + options: LibMainWorkerOptions, +} + +impl LibWorkerFactorySharedState { + fn resolve_unstable_features( + &self, + feature_checker: &FeatureChecker, + ) -> Vec { + let mut unstable_features = + Vec::with_capacity(UNSTABLE_GRANULAR_FLAGS.len()); + for granular_flag in UNSTABLE_GRANULAR_FLAGS { + if feature_checker.check(granular_flag.name) { + unstable_features.push(granular_flag.id); + } + } + unstable_features + } + + fn create_node_init_services( + &self, + node_require_loader: NodeRequireLoaderRc, + ) -> NodeExtInitServices, TSys> { + NodeExtInitServices { + node_require_loader, + node_resolver: self.node_resolver.clone(), + pkg_json_resolver: self.pkg_json_resolver.clone(), + sys: self.sys.clone(), + } + } + + fn create_web_worker_callback( + self: &Arc, + stdio: deno_runtime::deno_io::Stdio, + ) -> Arc { + let shared = self.clone(); + Arc::new(move |args| { + let maybe_inspector_server = shared.maybe_inspector_server.clone(); + + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared.module_loader_factory.create_for_worker( + args.parent_permissions.clone(), + args.permissions.clone(), + ); + let create_web_worker_cb = + shared.create_web_worker_callback(stdio.clone()); + + let maybe_storage_key = shared + .storage_key_resolver + .resolve_storage_key(&args.main_module); + let cache_storage_dir = maybe_storage_key.map(|key| { + // TODO(@satyarohith): storage quota management + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) + }); + + // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out + // list of enabled features. + let feature_checker = shared.feature_checker.clone(); + let unstable_features = + shared.resolve_unstable_features(feature_checker.as_ref()); + + let services = WebWorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + shared_array_buffer_store: Some( + shared.shared_array_buffer_store.clone(), + ), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + maybe_inspector_server, + feature_checker, + npm_process_state_provider: Some( + shared.npm_process_state_provider.clone(), + ), + permissions: args.permissions, + }; + let options = WebWorkerOptions { + name: args.name, + main_module: args.main_module.clone(), + worker_id: args.worker_id, + bootstrap: BootstrapOptions { + deno_version: shared.options.deno_version.to_string(), + args: shared.options.argv.clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + log_level: shared.options.log_level, + enable_op_summary_metrics: shared.options.enable_op_summary_metrics, + enable_testing_features: shared.options.enable_testing_features, + locale: deno_core::v8::icu::get_language_tag(), + location: Some(args.main_module), + no_color: !colors::use_color(), + color_level: colors::get_color_level(), + is_stdout_tty: deno_terminal::is_stdout_tty(), + is_stderr_tty: deno_terminal::is_stderr_tty(), + unstable_features, + user_agent: shared.options.deno_user_agent.to_string(), + inspect: shared.options.is_inspecting, + has_node_modules_dir: shared.options.has_node_modules_dir, + argv0: shared.options.argv0.clone(), + node_debug: shared.options.node_debug.clone(), + node_ipc_fd: None, + mode: WorkerExecutionMode::Worker, + serve_port: shared.options.serve_port, + serve_host: shared.options.serve_host.clone(), + otel_config: shared.options.otel_config.clone(), + close_on_idle: args.close_on_idle, + }, + extensions: vec![], + startup_snapshot: shared.options.startup_snapshot, + create_params: create_isolate_create_params(), + unsafely_ignore_certificate_errors: shared + .options + .unsafely_ignore_certificate_errors + .clone(), + seed: shared.options.seed, + create_web_worker_cb, + format_js_error_fn: Some(Arc::new(format_js_error)), + worker_type: args.worker_type, + stdio: stdio.clone(), + cache_storage_dir, + strace_ops: shared.options.strace_ops.clone(), + close_on_idle: args.close_on_idle, + maybe_worker_metadata: args.maybe_worker_metadata, + enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(), + }; + + WebWorker::bootstrap_from_options(services, options) + }) + } +} + +pub struct LibMainWorkerFactory { + shared: Arc>, +} + +impl LibMainWorkerFactory { + #[allow(clippy::too_many_arguments)] + pub fn new( + blob_store: Arc, + code_cache: Option>, + feature_checker: Arc, + fs: Arc, + maybe_inspector_server: Option>, + module_loader_factory: Box, + node_resolver: Arc< + NodeResolver, TSys>, + >, + npm_process_state_provider: NpmProcessStateProviderRc, + pkg_json_resolver: Arc>, + root_cert_store_provider: Arc, + storage_key_resolver: StorageKeyResolver, + sys: TSys, + options: LibMainWorkerOptions, + ) -> Self { + Self { + shared: Arc::new(LibWorkerFactorySharedState { + blob_store, + broadcast_channel: Default::default(), + code_cache, + compiled_wasm_module_store: Default::default(), + feature_checker, + fs, + maybe_inspector_server, + module_loader_factory, + node_resolver, + npm_process_state_provider, + pkg_json_resolver, + root_cert_store_provider, + shared_array_buffer_store: Default::default(), + storage_key_resolver, + sys, + options, + }), + } + } + + pub fn create_custom_worker( + &self, + mode: WorkerExecutionMode, + main_module: Url, + permissions: PermissionsContainer, + custom_extensions: Vec, + stdio: deno_runtime::deno_io::Stdio, + ) -> Result { + let shared = &self.shared; + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared + .module_loader_factory + .create_for_main(permissions.clone()); + + // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out + // list of enabled features. + let feature_checker = shared.feature_checker.clone(); + let unstable_features = + shared.resolve_unstable_features(feature_checker.as_ref()); + let maybe_storage_key = shared + .storage_key_resolver + .resolve_storage_key(&main_module); + let origin_storage_dir = maybe_storage_key.as_ref().map(|key| { + shared + .options + .origin_data_folder_path + .as_ref() + .unwrap() // must be set if storage key resolver returns a value + .join(checksum::gen(&[key.as_bytes()])) + }); + let cache_storage_dir = maybe_storage_key.map(|key| { + // TODO(@satyarohith): storage quota management + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) + }); + + let services = WorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + npm_process_state_provider: Some( + shared.npm_process_state_provider.clone(), + ), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + fetch_dns_resolver: Default::default(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + feature_checker, + permissions, + v8_code_cache: shared.code_cache.clone(), + }; + + let options = WorkerOptions { + bootstrap: BootstrapOptions { + deno_version: shared.options.deno_version.to_string(), + args: shared.options.argv.clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + log_level: shared.options.log_level, + enable_op_summary_metrics: shared.options.enable_op_summary_metrics, + enable_testing_features: shared.options.enable_testing_features, + locale: deno_core::v8::icu::get_language_tag(), + location: shared.options.location.clone(), + no_color: !colors::use_color(), + is_stdout_tty: deno_terminal::is_stdout_tty(), + is_stderr_tty: deno_terminal::is_stderr_tty(), + color_level: colors::get_color_level(), + unstable_features, + user_agent: shared.options.deno_user_agent.to_string(), + inspect: shared.options.is_inspecting, + has_node_modules_dir: shared.options.has_node_modules_dir, + argv0: shared.options.argv0.clone(), + node_debug: shared.options.node_debug.clone(), + node_ipc_fd: shared.options.node_ipc, + mode, + serve_port: shared.options.serve_port, + serve_host: shared.options.serve_host.clone(), + otel_config: shared.options.otel_config.clone(), + close_on_idle: true, + }, + extensions: custom_extensions, + startup_snapshot: shared.options.startup_snapshot, + create_params: create_isolate_create_params(), + unsafely_ignore_certificate_errors: shared + .options + .unsafely_ignore_certificate_errors + .clone(), + seed: shared.options.seed, + format_js_error_fn: Some(Arc::new(format_js_error)), + create_web_worker_cb: shared.create_web_worker_callback(stdio.clone()), + maybe_inspector_server: shared.maybe_inspector_server.clone(), + should_break_on_first_statement: shared.options.inspect_brk, + should_wait_for_inspector_session: shared.options.inspect_wait, + strace_ops: shared.options.strace_ops.clone(), + cache_storage_dir, + origin_storage_dir, + stdio, + skip_op_registration: shared.options.skip_op_registration, + enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(), + }; + + let worker = + MainWorker::bootstrap_from_options(&main_module, services, options); + + Ok(LibMainWorker { + main_module, + worker, + }) + } +} + +pub struct LibMainWorker { + main_module: Url, + worker: MainWorker, +} + +impl LibMainWorker { + pub fn into_main_worker(self) -> MainWorker { + self.worker + } + + pub fn main_module(&self) -> &Url { + &self.main_module + } + + pub fn js_runtime(&mut self) -> &mut JsRuntime { + &mut self.worker.js_runtime + } + + #[inline] + pub fn create_inspector_session(&mut self) -> LocalInspectorSession { + self.worker.create_inspector_session() + } + + #[inline] + pub fn dispatch_load_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_load_event() + } + + #[inline] + pub fn dispatch_beforeunload_event(&mut self) -> Result { + self.worker.dispatch_beforeunload_event() + } + + #[inline] + pub fn dispatch_process_beforeexit_event(&mut self) -> Result { + self.worker.dispatch_process_beforeexit_event() + } + + #[inline] + pub fn dispatch_unload_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_unload_event() + } + + #[inline] + pub fn dispatch_process_exit_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_process_exit_event() + } + + pub async fn execute_main_module(&mut self) -> Result<(), CoreError> { + let id = self.worker.preload_main_module(&self.main_module).await?; + self.worker.evaluate_module(id).await + } + + pub async fn execute_side_module(&mut self) -> Result<(), CoreError> { + let id = self.worker.preload_side_module(&self.main_module).await?; + self.worker.evaluate_module(id).await + } + + #[inline] + pub async fn run_event_loop( + &mut self, + wait_for_inspector: bool, + ) -> Result<(), CoreError> { + self.worker.run_event_loop(wait_for_inspector).await + } + + #[inline] + pub fn exit_code(&self) -> i32 { + self.worker.exit_code() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn storage_key_resolver_test() { + let resolver = + StorageKeyResolver(StorageKeyResolverStrategy::UseMainModule); + let specifier = Url::parse("file:///a.ts").unwrap(); + assert_eq!( + resolver.resolve_storage_key(&specifier), + Some(specifier.to_string()) + ); + let resolver = + StorageKeyResolver(StorageKeyResolverStrategy::Specified(None)); + assert_eq!(resolver.resolve_storage_key(&specifier), None); + let resolver = StorageKeyResolver(StorageKeyResolverStrategy::Specified( + Some("value".to_string()), + )); + assert_eq!( + resolver.resolve_storage_key(&specifier), + Some("value".to_string()) + ); + + // test empty + let resolver = StorageKeyResolver::empty(); + assert_eq!(resolver.resolve_storage_key(&specifier), None); + } +} diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index 97fbbaff14..a65bbd5efe 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -8,9 +8,9 @@ use std::time::SystemTime; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_lib::cache::DenoDir; use deno_path_util::url_to_file_path; -use crate::cache::DenoDir; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; use crate::cache::LocalLspHttpCache; @@ -70,7 +70,7 @@ fn calculate_fs_version_in_cache( #[derive(Debug, Clone)] pub struct LspCache { - deno_dir: DenoDir, + deno_dir: DenoDir, global: Arc, vendors_by_scope: BTreeMap>>, } @@ -121,7 +121,7 @@ impl LspCache { .collect(); } - pub fn deno_dir(&self) -> &DenoDir { + pub fn deno_dir(&self) -> &DenoDir { &self.deno_dir } diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index ba57502298..98c4498a1a 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -41,6 +41,7 @@ use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_lib::env::has_flag_env_var; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; @@ -55,7 +56,6 @@ use super::logging::lsp_log; use super::lsp_custom; use super::urls::url_to_uri; use crate::args::discover_npmrc_from_workspace; -use crate::args::has_flag_env_var; use crate::args::CliLockfile; use crate::args::CliLockfileReadFromPathOptions; use crate::args::ConfigFile; diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index fe8dc4c8d5..3e3e31de28 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -265,7 +265,7 @@ impl TsDiagnosticsStore { } pub fn should_send_diagnostic_batch_index_notifications() -> bool { - crate::args::has_flag_env_var( + deno_lib::env::has_flag_env_var( "DENO_DONT_USE_INTERNAL_LSP_DIAGNOSTIC_SYNC_FLAG", ) } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 0f3bfcdf59..c2fddc08bd 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -27,6 +27,7 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; +use deno_lib::env::has_flag_env_var; use deno_path_util::url_to_file_path; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; @@ -95,7 +96,6 @@ use super::urls::uri_to_url; use super::urls::url_to_uri; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; -use crate::args::has_flag_env_var; use crate::args::CaData; use crate::args::CliOptions; use crate::args::Flags; diff --git a/cli/lsp/testing/definitions.rs b/cli/lsp/testing/definitions.rs index 8277dcbf00..d6630c1844 100644 --- a/cli/lsp/testing/definitions.rs +++ b/cli/lsp/testing/definitions.rs @@ -5,6 +5,7 @@ use std::collections::HashSet; use deno_core::error::AnyError; use deno_core::ModuleSpecifier; +use deno_lib::util::checksum; use lsp::Range; use tower_lsp::lsp_types as lsp; @@ -15,7 +16,6 @@ use crate::lsp::logging::lsp_warn; use crate::lsp::urls::url_to_uri; use crate::tools::test::TestDescription; use crate::tools::test::TestStepDescription; -use crate::util::checksum; #[derive(Debug, Clone, PartialEq)] pub struct TestDefinition { diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index e2a0fc430d..0b53dc8506 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -39,6 +39,7 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; use deno_core::RuntimeOptions; +use deno_lib::worker::create_isolate_create_params; use deno_path_util::url_to_file_path; use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES; use deno_runtime::inspector_server::InspectorServer; @@ -96,7 +97,6 @@ use crate::util::path::relative_specifier; use crate::util::path::to_percent_decoded_str; use crate::util::result::InfallibleResultExt; use crate::util::v8::convert; -use crate::worker::create_isolate_create_params; static BRACKET_ACCESSOR_RE: Lazy = lazy_regex!(r#"^\[['"](.+)[\['"]\]$"#); diff --git a/cli/lsp/urls.rs b/cli/lsp/urls.rs index 2aadaf5352..068e4ad4d5 100644 --- a/cli/lsp/urls.rs +++ b/cli/lsp/urls.rs @@ -81,7 +81,7 @@ fn hash_data_specifier(specifier: &ModuleSpecifier) -> String { file_name_str.push('?'); file_name_str.push_str(query); } - crate::util::checksum::gen(&[file_name_str.as_bytes()]) + deno_lib::util::checksum::gen(&[file_name_str.as_bytes()]) } fn to_deno_uri(specifier: &Url) -> String { diff --git a/cli/module_loader.rs b/cli/module_loader.rs index daeb4dda37..2b0ebca986 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -39,6 +39,9 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::Resolution; use deno_graph::WasmModule; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::worker::CreateModuleLoaderResult; +use deno_lib::worker::ModuleLoaderFactory; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::code_cache; use deno_runtime::deno_node::create_host_defined_options; @@ -70,7 +73,6 @@ use crate::graph_util::ModuleGraphBuilder; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; -use crate::npm::NpmRegistryReadPermissionChecker; use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; @@ -84,8 +86,6 @@ use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; -use crate::worker::CreateModuleLoaderResult; -use crate::worker::ModuleLoaderFactory; #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum PrepareModuleLoadError { @@ -243,7 +243,8 @@ struct SharedCliModuleLoaderState { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: + Arc>, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, parsed_source_cache: Arc, @@ -304,7 +305,9 @@ impl CliModuleLoaderFactory { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: Arc< + NpmRegistryReadPermissionChecker, + >, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, parsed_source_cache: Arc, @@ -1145,7 +1148,8 @@ struct CliNodeRequireLoader { sys: CliSys, graph_container: TGraphContainer, in_npm_pkg_checker: DenoInNpmPackageChecker, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: + Arc>, } impl NodeRequireLoader diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 1c12ce6c59..fc0916cc18 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -3,7 +3,6 @@ mod byonm; pub mod installer; mod managed; -mod permission_checker; use std::sync::Arc; @@ -24,8 +23,6 @@ pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::NpmResolutionInitializer; pub use self::managed::ResolveSnapshotError; -pub use self::permission_checker::NpmRegistryReadPermissionChecker; -pub use self::permission_checker::NpmRegistryReadPermissionCheckerMode; use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; use crate::sys::CliSys; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index c9b57f3d6b..5334b4719d 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -38,6 +38,13 @@ use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; use deno_graph::ModuleGraph; +use deno_lib::cache::DenoDir; +use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::VfsEntry; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; +use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; use deno_npm::resolution::SerializedNpmResolutionSnapshot; use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; @@ -73,20 +80,14 @@ use super::serialization::SourceMapStore; use super::virtual_fs::output_vfs; use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; -use super::virtual_fs::FileSystemCaseSensitivity; use super::virtual_fs::VfsBuilder; -use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; -use super::virtual_fs::VirtualDirectory; -use super::virtual_fs::VirtualDirectoryEntries; -use super::virtual_fs::WindowsSystemRootablePath; use crate::args::CaData; use crate::args::CliOptions; use crate::args::CompileFlags; use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; -use crate::cache::DenoDir; use crate::cache::FastInsecureHasher; use crate::emit::Emitter; use crate::file_fetcher::CliFileFetcher; @@ -94,7 +95,7 @@ use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::shared::ReleaseChannel; -use crate::standalone::virtual_fs::VfsEntry; +use crate::sys::CliSys; use crate::util::archive; use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -411,7 +412,7 @@ pub struct WriteBinOptions<'a> { pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, @@ -425,7 +426,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub fn new( cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index b04db88c90..c4b3ebe728 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; use deno_runtime::deno_fs::AccessCheckCb; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::FsDirEntry; @@ -30,7 +31,6 @@ use super::virtual_fs::FileBackedVfs; use super::virtual_fs::FileBackedVfsDirEntry; use super::virtual_fs::FileBackedVfsFile; use super::virtual_fs::FileBackedVfsMetadata; -use super::virtual_fs::VfsFileSubDataKind; #[derive(Debug, Clone)] pub struct DenoCompileFileSystem(Arc); diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 876c194ed1..f2a0859e8f 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -36,6 +36,15 @@ use deno_core::RequestedModuleType; use deno_core::ResolutionKind; use deno_core::SourceCodeCacheInfo; use deno_error::JsErrorBox; +use deno_lib::cache::DenoDirProvider; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::worker::CreateModuleLoaderResult; +use deno_lib::worker::LibMainWorkerFactory; +use deno_lib::worker::LibMainWorkerOptions; +use deno_lib::worker::ModuleLoaderFactory; +use deno_lib::worker::StorageKeyResolver; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_package_json::PackageJsonDepValue; @@ -69,16 +78,13 @@ use node_resolver::ResolutionMode; use serialization::DenoCompileModuleSource; use serialization::SourceMapStore; use virtual_fs::FileBackedVfs; -use virtual_fs::VfsFileSubDataKind; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; use crate::args::npm_pkg_req_ref_to_binary_command; use crate::args::CaData; use crate::args::NpmInstallDepsProvider; -use crate::args::StorageKeyResolver; use crate::cache::Caches; -use crate::cache::DenoDirProvider; use crate::cache::FastInsecureHasher; use crate::cache::NodeAnalysisCache; use crate::http_util::HttpClientProvider; @@ -86,13 +92,12 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; +use crate::npm::create_npm_process_state_provider; use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::NpmRegistryReadPermissionChecker; -use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; @@ -105,8 +110,6 @@ use crate::util::v8::construct_v8_flags; use crate::worker::CliCodeCache; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; -use crate::worker::CreateModuleLoaderResult; -use crate::worker::ModuleLoaderFactory; pub mod binary; mod code_cache; @@ -129,7 +132,7 @@ struct SharedModuleLoaderState { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: Arc, - npm_registry_permission_checker: NpmRegistryReadPermissionChecker, + npm_registry_permission_checker: NpmRegistryReadPermissionChecker, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, source_maps: SourceMapStore, @@ -962,54 +965,67 @@ pub async fn run( } checker }); - let worker_factory = CliMainWorkerFactory::new( + let lib_main_worker_options = LibMainWorkerOptions { + argv: metadata.argv, + log_level: WorkerLogLevel::Info, + enable_op_summary_metrics: false, + enable_testing_features: false, + has_node_modules_dir, + inspect_brk: false, + inspect_wait: false, + strace_ops: None, + is_inspecting: false, + skip_op_registration: true, + location: metadata.location, + argv0: NpmPackageReqReference::from_specifier(&main_module) + .ok() + .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) + .or(std::env::args().next()), + node_debug: std::env::var("NODE_DEBUG").ok(), + origin_data_folder_path: None, + seed: metadata.seed, + unsafely_ignore_certificate_errors: metadata + .unsafely_ignore_certificate_errors, + node_ipc: None, + serve_port: None, + serve_host: None, + deno_version: crate::version::DENO_VERSION_INFO.deno, + deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, + otel_config: metadata.otel_config, + startup_snapshot: crate::js::deno_isolate_init(), + }; + let lib_main_worker_factory = LibMainWorkerFactory::new( Arc::new(BlobStore::default()), - code_cache, + code_cache.map(|c| c.as_code_cache()), feature_checker, fs, None, - None, - None, Box::new(module_loader_factory), + node_resolver.clone(), + create_npm_process_state_provider(&npm_resolver), + pkg_json_resolver, + root_cert_store_provider, + StorageKeyResolver::empty(), + sys.clone(), + lib_main_worker_options, + ); + // todo(dsherret): use LibMainWorker directly here and don't use CliMainWorkerFactory + let cli_main_worker_options = CliMainWorkerOptions { + create_hmr_runner: None, + create_coverage_collector: None, + needs_test_modules: false, + default_npm_caching_strategy: crate::args::NpmCachingStrategy::Lazy, + }; + let worker_factory = CliMainWorkerFactory::new( + lib_main_worker_factory, + None, + None, node_resolver, None, npm_resolver, - pkg_json_resolver, - root_cert_store_provider, - permissions, - StorageKeyResolver::empty(), sys, - crate::args::DenoSubcommand::Run(Default::default()), - CliMainWorkerOptions { - argv: metadata.argv, - log_level: WorkerLogLevel::Info, - enable_op_summary_metrics: false, - enable_testing_features: false, - has_node_modules_dir, - hmr: false, - inspect_brk: false, - inspect_wait: false, - strace_ops: None, - is_inspecting: false, - skip_op_registration: true, - location: metadata.location, - argv0: NpmPackageReqReference::from_specifier(&main_module) - .ok() - .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) - .or(std::env::args().next()), - node_debug: std::env::var("NODE_DEBUG").ok(), - origin_data_folder_path: None, - seed: metadata.seed, - unsafely_ignore_certificate_errors: metadata - .unsafely_ignore_certificate_errors, - create_hmr_runner: None, - create_coverage_collector: None, - node_ipc: None, - serve_port: None, - serve_host: None, - }, - metadata.otel_config, - crate::args::NpmCachingStrategy::Lazy, + cli_main_worker_options, + permissions, ); // Initialize v8 once from the main thread. diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index ae2e411b5f..ab345917a3 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -17,6 +17,7 @@ use deno_core::url::Url; use deno_core::FastString; use deno_core::ModuleSourceCode; use deno_core::ModuleType; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; use deno_npm::resolution::SerializedNpmResolutionSnapshot; use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; @@ -27,10 +28,7 @@ use indexmap::IndexMap; use super::binary::Metadata; use super::virtual_fs::BuiltVfs; -use super::virtual_fs::FileSystemCaseSensitivity; use super::virtual_fs::VfsBuilder; -use super::virtual_fs::VirtualDirectoryEntries; -use crate::standalone::virtual_fs::VirtualDirectory; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 0b1f33259d..4f761d0d15 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -23,6 +23,17 @@ use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::OffsetWithLength; +use deno_lib::standalone::virtual_fs::VfsEntry; +use deno_lib::standalone::virtual_fs::VfsEntryRef; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; +use deno_lib::standalone::virtual_fs::VirtualFile; +use deno_lib::standalone::virtual_fs::VirtualSymlink; +use deno_lib::standalone::virtual_fs::VirtualSymlinkParts; +use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; use deno_path_util::normalize_path; use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; @@ -41,37 +52,6 @@ use crate::util::display::human_size; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; -#[derive(Debug, PartialEq, Eq)] -pub enum WindowsSystemRootablePath { - /// The root of the system above any drive letters. - WindowSystemRoot, - Path(PathBuf), -} - -impl WindowsSystemRootablePath { - pub fn join(&self, name_component: &str) -> PathBuf { - // this method doesn't handle multiple components - debug_assert!( - !name_component.contains('\\'), - "Invalid component: {}", - name_component - ); - debug_assert!( - !name_component.contains('/'), - "Invalid component: {}", - name_component - ); - - match self { - WindowsSystemRootablePath::WindowSystemRoot => { - // windows drive letter - PathBuf::from(&format!("{}\\", name_component)) - } - WindowsSystemRootablePath::Path(path) => path.join(name_component), - } - } -} - #[derive(Debug)] pub struct BuiltVfs { pub root_path: WindowsSystemRootablePath, @@ -80,15 +60,6 @@ pub struct BuiltVfs { pub files: Vec>, } -#[derive(Debug, Copy, Clone)] -pub enum VfsFileSubDataKind { - /// Raw bytes of the file. - Raw, - /// Bytes to use for module loading. For example, for TypeScript - /// files this will be the transpiled JavaScript source. - ModuleGraph, -} - #[derive(Debug)] pub struct VfsBuilder { executable_root: VirtualDirectory, @@ -232,24 +203,21 @@ impl VfsBuilder { continue; } let name = component.as_os_str().to_string_lossy(); - let index = match current_dir - .entries - .binary_search(&name, self.case_sensitivity) - { - Ok(index) => index, - Err(insert_index) => { - current_dir.entries.0.insert( - insert_index, - VfsEntry::Dir(VirtualDirectory { - name: name.to_string(), - entries: Default::default(), - }), - ); - insert_index - } - }; - match &mut current_dir.entries.0[index] { - VfsEntry::Dir(dir) => { + let index = current_dir.entries.insert_or_modify( + &name, + self.case_sensitivity, + || { + VfsEntry::Dir(VirtualDirectory { + name: name.to_string(), + entries: Default::default(), + }) + }, + |_| { + // ignore + }, + ); + match current_dir.entries.get_mut_by_index(index) { + Some(VfsEntry::Dir(dir)) => { current_dir = dir; } _ => unreachable!(), @@ -325,7 +293,7 @@ impl VfsBuilder { sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { log::debug!("Adding file '{}'", path.display()); - let checksum = util::checksum::gen(&[&data]); + let checksum = deno_lib::util::checksum::gen(&[&data]); let case_sensitivity = self.case_sensitivity; let offset = if let Some(offset) = self.file_offsets.get(&checksum) { // duplicate file, reuse an old offset @@ -341,32 +309,28 @@ impl VfsBuilder { offset, len: data.len() as u64, }; - match dir.entries.binary_search(&name, case_sensitivity) { - Ok(index) => { - let entry = &mut dir.entries.0[index]; - match entry { - VfsEntry::File(virtual_file) => match sub_data_kind { - VfsFileSubDataKind::Raw => { - virtual_file.offset = offset_and_len; - } - VfsFileSubDataKind::ModuleGraph => { - virtual_file.module_graph_offset = offset_and_len; - } - }, - VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), - } - } - Err(insert_index) => { - dir.entries.0.insert( - insert_index, - VfsEntry::File(VirtualFile { - name: name.to_string(), - offset: offset_and_len, - module_graph_offset: offset_and_len, - }), - ); - } - } + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::File(VirtualFile { + name: name.to_string(), + offset: offset_and_len, + module_graph_offset: offset_and_len, + }) + }, + |entry| match entry { + VfsEntry::File(virtual_file) => match sub_data_kind { + VfsFileSubDataKind::Raw => { + virtual_file.offset = offset_and_len; + } + VfsFileSubDataKind::ModuleGraph => { + virtual_file.module_graph_offset = offset_and_len; + } + }, + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), + }, + ); // new file, update the list of files if self.current_offset == offset { @@ -406,18 +370,19 @@ impl VfsBuilder { let target = normalize_path(path.parent().unwrap().join(&target)); let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); - match dir.entries.binary_search(&name, case_sensitivity) { - Ok(_) => {} // previously inserted - Err(insert_index) => { - dir.entries.0.insert( - insert_index, - VfsEntry::Symlink(VirtualSymlink { - name: name.to_string(), - dest_parts: VirtualSymlinkParts::from_path(&target), - }), - ); - } - } + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::Symlink(VirtualSymlink { + name: name.to_string(), + dest_parts: VirtualSymlinkParts::from_path(&target), + }) + }, + |_| { + // ignore previously inserted + }, + ); let target_metadata = std::fs::symlink_metadata(&target).with_context(|| { format!("Reading symlink target '{}'", target.display()) @@ -448,16 +413,20 @@ impl VfsBuilder { dir: &mut VirtualDirectory, parts: &[String], ) { - for entry in &mut dir.entries.0 { + for entry in dir.entries.iter_mut() { match entry { VfsEntry::Dir(dir) => { strip_prefix_from_symlinks(dir, parts); } VfsEntry::File(_) => {} VfsEntry::Symlink(symlink) => { - let old_parts = std::mem::take(&mut symlink.dest_parts.0); - symlink.dest_parts.0 = - old_parts.into_iter().skip(parts.len()).collect(); + let parts = symlink + .dest_parts + .take_parts() + .into_iter() + .skip(parts.len()) + .collect(); + symlink.dest_parts.set_parts(parts); } } } @@ -476,13 +445,13 @@ impl VfsBuilder { if self.min_root_dir.as_ref() == Some(¤t_path) { break; } - match ¤t_dir.entries.0[0] { + match current_dir.entries.iter().next().unwrap() { VfsEntry::Dir(dir) => { if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { // special directory we want to maintain break; } - match current_dir.entries.0.remove(0) { + match current_dir.entries.remove(0) { VfsEntry::Dir(dir) => { current_path = WindowsSystemRootablePath::Path(current_path.join(&dir.name)); @@ -497,7 +466,7 @@ impl VfsBuilder { if let WindowsSystemRootablePath::Path(path) = ¤t_path { strip_prefix_from_symlinks( &mut current_dir, - &VirtualSymlinkParts::from_path(path).0, + VirtualSymlinkParts::from_path(path).parts(), ); } BuiltVfs { @@ -577,7 +546,7 @@ fn vfs_as_display_tree( All(Size), Subset(Vec>), File(Size), - Symlink(&'a [String]), + Symlink(&'a VirtualSymlinkParts), } impl<'a> EntryOutput<'a> { @@ -626,7 +595,7 @@ fn vfs_as_display_tree( format!("{} ({})", name, format_size(*size)) } EntryOutput::Symlink(parts) => { - format!("{} --> {}", name, parts.join("/")) + format!("{} --> {}", name, parts.display()) } }, children: match self { @@ -769,7 +738,7 @@ fn vfs_as_display_tree( EntryOutput::File(file_size(file, seen_offsets)) } VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + EntryOutput::Symlink(&virtual_symlink.dest_parts) } }, }) @@ -806,7 +775,7 @@ fn vfs_as_display_tree( } VfsEntry::File(file) => EntryOutput::File(file_size(file, seen_offsets)), VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + EntryOutput::Symlink(&virtual_symlink.dest_parts) } } } @@ -872,226 +841,6 @@ fn vfs_as_display_tree( } } -#[derive(Debug)] -enum VfsEntryRef<'a> { - Dir(&'a VirtualDirectory), - File(&'a VirtualFile), - Symlink(&'a VirtualSymlink), -} - -impl VfsEntryRef<'_> { - pub fn as_metadata(&self) -> FileBackedVfsMetadata { - FileBackedVfsMetadata { - file_type: match self { - Self::Dir(_) => sys_traits::FileType::Dir, - Self::File(_) => sys_traits::FileType::File, - Self::Symlink(_) => sys_traits::FileType::Symlink, - }, - name: self.name().to_string(), - len: match self { - Self::Dir(_) => 0, - Self::File(file) => file.offset.len, - Self::Symlink(_) => 0, - }, - } - } - - pub fn name(&self) -> &str { - match self { - Self::Dir(dir) => &dir.name, - Self::File(file) => &file.name, - Self::Symlink(symlink) => &symlink.name, - } - } -} - -// todo(dsherret): we should store this more efficiently in the binary -#[derive(Debug, Serialize, Deserialize)] -pub enum VfsEntry { - Dir(VirtualDirectory), - File(VirtualFile), - Symlink(VirtualSymlink), -} - -impl VfsEntry { - pub fn name(&self) -> &str { - match self { - Self::Dir(dir) => &dir.name, - Self::File(file) => &file.name, - Self::Symlink(symlink) => &symlink.name, - } - } - - fn as_ref(&self) -> VfsEntryRef { - match self { - VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir), - VfsEntry::File(file) => VfsEntryRef::File(file), - VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink), - } - } -} - -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -pub enum FileSystemCaseSensitivity { - #[serde(rename = "s")] - Sensitive, - #[serde(rename = "i")] - Insensitive, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct VirtualDirectoryEntries(Vec); - -impl VirtualDirectoryEntries { - pub fn new(mut entries: Vec) -> Self { - // needs to be sorted by name - entries.sort_by(|a, b| a.name().cmp(b.name())); - Self(entries) - } - - pub fn take_inner(&mut self) -> Vec { - std::mem::take(&mut self.0) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn get_by_name( - &self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Option<&VfsEntry> { - self - .binary_search(name, case_sensitivity) - .ok() - .map(|index| &self.0[index]) - } - - pub fn get_mut_by_name( - &mut self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Option<&mut VfsEntry> { - self - .binary_search(name, case_sensitivity) - .ok() - .map(|index| &mut self.0[index]) - } - - pub fn binary_search( - &self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Result { - match case_sensitivity { - FileSystemCaseSensitivity::Sensitive => { - self.0.binary_search_by(|e| e.name().cmp(name)) - } - FileSystemCaseSensitivity::Insensitive => self.0.binary_search_by(|e| { - e.name() - .chars() - .zip(name.chars()) - .map(|(a, b)| a.to_ascii_lowercase().cmp(&b.to_ascii_lowercase())) - .find(|&ord| ord != Ordering::Equal) - .unwrap_or_else(|| e.name().len().cmp(&name.len())) - }), - } - } - - pub fn insert( - &mut self, - entry: VfsEntry, - case_sensitivity: FileSystemCaseSensitivity, - ) { - match self.binary_search(entry.name(), case_sensitivity) { - Ok(index) => { - self.0[index] = entry; - } - Err(insert_index) => { - self.0.insert(insert_index, entry); - } - } - } - - pub fn remove(&mut self, index: usize) -> VfsEntry { - self.0.remove(index) - } - - pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> { - self.0.iter() - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualDirectory { - #[serde(rename = "n")] - pub name: String, - // should be sorted by name - #[serde(rename = "e")] - pub entries: VirtualDirectoryEntries, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct OffsetWithLength { - #[serde(rename = "o")] - pub offset: u64, - #[serde(rename = "l")] - pub len: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VirtualFile { - #[serde(rename = "n")] - pub name: String, - #[serde(rename = "o")] - pub offset: OffsetWithLength, - /// Offset file to use for module loading when it differs from the - /// raw file. Often this will be the same offset as above for data - /// such as JavaScript files, but for TypeScript files the `offset` - /// will be the original raw bytes when included as an asset and this - /// offset will be to the transpiled JavaScript source. - #[serde(rename = "m")] - pub module_graph_offset: OffsetWithLength, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualSymlinkParts(Vec); - -impl VirtualSymlinkParts { - pub fn from_path(path: &Path) -> Self { - Self( - path - .components() - .filter(|c| !matches!(c, std::path::Component::RootDir)) - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect(), - ) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualSymlink { - #[serde(rename = "n")] - pub name: String, - #[serde(rename = "p")] - pub dest_parts: VirtualSymlinkParts, -} - -impl VirtualSymlink { - pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { - let mut dest = root.to_path_buf(); - for part in &self.dest_parts.0 { - dest.push(part); - } - dest - } -} - #[derive(Debug)] pub struct VfsRoot { pub dir: VirtualDirectory, @@ -1430,6 +1179,21 @@ pub struct FileBackedVfsMetadata { } impl FileBackedVfsMetadata { + pub fn from_vfs_entry_ref(vfs_entry: VfsEntryRef) -> Self { + FileBackedVfsMetadata { + file_type: match vfs_entry { + VfsEntryRef::Dir(_) => sys_traits::FileType::Dir, + VfsEntryRef::File(_) => sys_traits::FileType::File, + VfsEntryRef::Symlink(_) => sys_traits::FileType::Symlink, + }, + name: vfs_entry.name().to_string(), + len: match vfs_entry { + VfsEntryRef::Dir(_) => 0, + VfsEntryRef::File(file) => file.offset.len, + VfsEntryRef::Symlink(_) => 0, + }, + } + } pub fn as_fs_stat(&self) -> FsStat { FsStat { is_directory: self.file_type == sys_traits::FileType::Dir, @@ -1521,7 +1285,7 @@ impl FileBackedVfs { let path = path.to_path_buf(); Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry { parent_path: path.to_path_buf(), - metadata: entry.as_ref().as_metadata(), + metadata: FileBackedVfsMetadata::from_vfs_entry_ref(entry.as_ref()), })) } @@ -1544,12 +1308,12 @@ impl FileBackedVfs { let (_, entry) = self .fs_root .find_entry_no_follow(path, self.case_sensitivity)?; - Ok(entry.as_metadata()) + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) } pub fn stat(&self, path: &Path) -> std::io::Result { let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; - Ok(entry.as_metadata()) + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) } pub fn canonicalize(&self, path: &Path) -> std::io::Result { diff --git a/cli/sys.rs b/cli/sys.rs index 718e9981e2..e551eab2e8 100644 --- a/cli/sys.rs +++ b/cli/sys.rs @@ -29,6 +29,8 @@ pub enum CliSys { DenoCompile(DenoCompileFileSystem), } +impl deno_lib::sys::DenoLibSys for CliSys {} + impl Default for CliSys { fn default() -> Self { Self::Real(sys_traits::impls::RealSys) diff --git a/cli/tools/clean.rs b/cli/tools/clean.rs index e6f8c1e52b..a550d2826a 100644 --- a/cli/tools/clean.rs +++ b/cli/tools/clean.rs @@ -4,8 +4,8 @@ use std::path::Path; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_lib::cache::DenoDir; -use crate::cache::DenoDir; use crate::colors; use crate::display; use crate::sys::CliSys; diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 8c3b2665c5..1b2542d427 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -18,6 +18,7 @@ use deno_graph::Module; use deno_graph::ModuleError; use deno_graph::ModuleGraph; use deno_graph::Resolution; +use deno_lib::util::checksum; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::NpmPackageId; @@ -33,7 +34,6 @@ use crate::display; use crate::factory::CliFactory; use crate::graph_util::graph_exit_integrity_errors; use crate::npm::CliManagedNpmResolver; -use crate::util::checksum; use crate::util::display::DisplayTreeNode; const JSON_SCHEMA_VERSION: u8 = 1; @@ -191,7 +191,7 @@ fn print_cache_info( let registry_cache = dir.registries_folder_path(); let mut origin_dir = dir.origin_data_folder_path(); let deno_dir = dir.root_path_for_display().to_string(); - let web_cache_dir = crate::worker::get_cache_storage_dir(); + let web_cache_dir = deno_lib::worker::get_cache_storage_dir(); if let Some(location) = &location { origin_dir = diff --git a/cli/tools/serve.rs b/cli/tools/serve.rs index c2c53c1b69..2143eb33bb 100644 --- a/cli/tools/serve.rs +++ b/cli/tools/serve.rs @@ -43,7 +43,8 @@ pub async fn serve( maybe_npm_install(&factory).await?; - let worker_factory = factory.create_cli_main_worker_factory().await?; + let worker_factory = + Arc::new(factory.create_cli_main_worker_factory().await?); let hmr = serve_flags .watch .map(|watch_flags| watch_flags.hmr) @@ -58,7 +59,7 @@ pub async fn serve( } async fn do_serve( - worker_factory: CliMainWorkerFactory, + worker_factory: Arc, main_module: ModuleSpecifier, worker_count: Option, hmr: bool, @@ -116,7 +117,7 @@ async fn do_serve( async fn run_worker( worker_count: usize, - worker_factory: CliMainWorkerFactory, + worker_factory: Arc, main_module: ModuleSpecifier, hmr: bool, ) -> Result { @@ -164,7 +165,8 @@ async fn serve_with_watch( maybe_npm_install(&factory).await?; let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); - let worker_factory = factory.create_cli_main_worker_factory().await?; + let worker_factory = + Arc::new(factory.create_cli_main_worker_factory().await?); do_serve(worker_factory, main_module.clone(), worker_count, hmr) .await?; diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 37d52c6cf2..1b76b640d3 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -28,6 +28,8 @@ use deno_graph::GraphKind; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; +use deno_lib::util::checksum; +use deno_lib::worker::create_isolate_create_params; use deno_resolver::npm::managed::ResolvePkgFolderFromDenoModuleError; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; use deno_semver::npm::NpmPackageReqReference; @@ -48,9 +50,7 @@ use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::sys::CliSys; -use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; -use crate::worker::create_isolate_create_params; mod diagnostics; diff --git a/cli/util/file_watcher.rs b/cli/util/file_watcher.rs index 65963214b9..d3ff1bae77 100644 --- a/cli/util/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -25,6 +25,7 @@ use notify::Watcher; use tokio::select; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc; +use tokio::sync::mpsc::error::SendError; use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::sleep; @@ -141,36 +142,60 @@ fn create_print_after_restart_fn(clear_screen: bool) -> impl Fn() { } } +#[derive(Debug)] +pub struct WatcherCommunicatorOptions { + /// Send a list of paths that should be watched for changes. + pub paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender>, + /// Listen for a list of paths that were changed. + pub changed_paths_rx: tokio::sync::broadcast::Receiver>>, + pub changed_paths_tx: tokio::sync::broadcast::Sender>>, + /// Send a message to force a restart. + pub restart_tx: tokio::sync::mpsc::UnboundedSender<()>, + pub restart_mode: WatcherRestartMode, + pub banner: String, +} + /// An interface to interact with Deno's CLI file watcher. #[derive(Debug)] pub struct WatcherCommunicator { /// Send a list of paths that should be watched for changes. paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender>, - /// Listen for a list of paths that were changed. changed_paths_rx: tokio::sync::broadcast::Receiver>>, - + changed_paths_tx: tokio::sync::broadcast::Sender>>, /// Send a message to force a restart. restart_tx: tokio::sync::mpsc::UnboundedSender<()>, - restart_mode: Mutex, - banner: String, } impl WatcherCommunicator { - pub fn watch_paths(&self, paths: Vec) -> Result<(), AnyError> { + pub fn new(options: WatcherCommunicatorOptions) -> Self { + Self { + paths_to_watch_tx: options.paths_to_watch_tx, + changed_paths_rx: options.changed_paths_rx, + changed_paths_tx: options.changed_paths_tx, + restart_tx: options.restart_tx, + restart_mode: Mutex::new(options.restart_mode), + banner: options.banner, + } + } + + pub fn watch_paths( + &self, + paths: Vec, + ) -> Result<(), SendError>> { if paths.is_empty() { return Ok(()); } - self.paths_to_watch_tx.send(paths).map_err(AnyError::from) + self.paths_to_watch_tx.send(paths) } - pub fn force_restart(&self) -> Result<(), AnyError> { + pub fn force_restart(&self) -> Result<(), SendError<()>> { // Change back to automatic mode, so that HMR can set up watching // from scratch. *self.restart_mode.lock() = WatcherRestartMode::Automatic; - self.restart_tx.send(()).map_err(AnyError::from) + self.restart_tx.send(()) } pub async fn watch_for_changed_paths( @@ -184,6 +209,22 @@ impl WatcherCommunicator { *self.restart_mode.lock() = restart_mode; } + pub fn send( + &self, + paths: Option>, + ) -> Result<(), SendError>>> { + match *self.restart_mode.lock() { + WatcherRestartMode::Automatic => { + self.restart_tx.send(()).map_err(|_| SendError(None)) + } + WatcherRestartMode::Manual => self + .changed_paths_tx + .send(paths) + .map(|_| ()) + .map_err(|e| SendError(e.0)), + } + } + pub fn print(&self, msg: String) { log::info!("{} {}", self.banner, colors::gray(msg)); } @@ -272,13 +313,15 @@ where } = print_config; let print_after_restart = create_print_after_restart_fn(clear_screen); - let watcher_communicator = Arc::new(WatcherCommunicator { - paths_to_watch_tx: paths_to_watch_tx.clone(), - changed_paths_rx: changed_paths_rx.resubscribe(), - restart_tx: restart_tx.clone(), - restart_mode: Mutex::new(restart_mode), - banner: colors::intense_blue(banner).to_string(), - }); + let watcher_communicator = + Arc::new(WatcherCommunicator::new(WatcherCommunicatorOptions { + paths_to_watch_tx: paths_to_watch_tx.clone(), + changed_paths_rx: changed_paths_rx.resubscribe(), + changed_paths_tx, + restart_tx: restart_tx.clone(), + restart_mode, + banner: colors::intense_blue(banner).to_string(), + })); info!("{} {} started.", colors::intense_blue(banner), job_name); let changed_paths = Rc::new(RefCell::new(None)); @@ -292,15 +335,8 @@ where .borrow_mut() .clone_from(&received_changed_paths); - match *watcher_.restart_mode.lock() { - WatcherRestartMode::Automatic => { - let _ = restart_tx.send(()); - } - WatcherRestartMode::Manual => { - // TODO(bartlomieju): should we fail on sending changed paths? - let _ = changed_paths_tx.send(received_changed_paths); - } - } + // TODO(bartlomieju): should we fail on sending changed paths? + let _ = watcher_.send(received_changed_paths); } }); diff --git a/cli/util/mod.rs b/cli/util/mod.rs index 0578ecb423..702e5673c9 100644 --- a/cli/util/mod.rs +++ b/cli/util/mod.rs @@ -2,7 +2,6 @@ // Note: Only add code in this folder that has no application specific logic pub mod archive; -pub mod checksum; pub mod console; pub mod diff; pub mod display; diff --git a/cli/worker.rs b/cli/worker.rs index d9cdbd3fb0..cf301de83e 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -1,8 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::path::Path; -use std::path::PathBuf; -use std::rc::Rc; use std::sync::Arc; use deno_ast::ModuleSpecifier; @@ -10,77 +8,31 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::error::CoreError; use deno_core::futures::FutureExt; -use deno_core::url::Url; use deno_core::v8; -use deno_core::CompiledWasmModuleStore; use deno_core::Extension; -use deno_core::FeatureChecker; -use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; -use deno_core::SharedArrayBufferStore; use deno_error::JsErrorBox; -use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_lib::worker::LibMainWorker; +use deno_lib::worker::LibMainWorkerFactory; use deno_runtime::code_cache; -use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; -use deno_runtime::deno_fs; -use deno_runtime::deno_node::NodeExtInitServices; -use deno_runtime::deno_node::NodeRequireLoader; -use deno_runtime::deno_node::NodeRequireLoaderRc; use deno_runtime::deno_permissions::PermissionsContainer; -use deno_runtime::deno_tls::RootCertStoreProvider; -use deno_runtime::deno_web::BlobStore; -use deno_runtime::fmt_errors::format_js_error; -use deno_runtime::inspector_server::InspectorServer; -use deno_runtime::ops::process::NpmProcessStateProviderRc; -use deno_runtime::ops::worker_host::CreateWebWorkerCb; -use deno_runtime::web_worker::WebWorker; -use deno_runtime::web_worker::WebWorkerOptions; -use deno_runtime::web_worker::WebWorkerServiceOptions; use deno_runtime::worker::MainWorker; -use deno_runtime::worker::WorkerOptions; -use deno_runtime::worker::WorkerServiceOptions; -use deno_runtime::BootstrapOptions; use deno_runtime::WorkerExecutionMode; -use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; -use deno_telemetry::OtelConfig; -use deno_terminal::colors; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; +use sys_traits::EnvCurrentDir; use tokio::select; use crate::args::CliLockfile; -use crate::args::DenoSubcommand; use crate::args::NpmCachingStrategy; -use crate::args::StorageKeyResolver; use crate::node::CliNodeResolver; -use crate::node::CliPackageJsonResolver; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; use crate::npm::CliNpmResolver; use crate::sys::CliSys; -use crate::util::checksum; use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; -use crate::version; - -pub struct CreateModuleLoaderResult { - pub module_loader: Rc, - pub node_require_loader: Rc, -} - -pub trait ModuleLoaderFactory: Send + Sync { - fn create_for_main( - &self, - root_permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult; - - fn create_for_worker( - &self, - parent_permissions: PermissionsContainer, - permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult; -} #[async_trait::async_trait(?Send)] pub trait HmrRunner: Send + Sync { @@ -115,83 +67,28 @@ pub type CreateCoverageCollectorCb = Box< >; pub struct CliMainWorkerOptions { - pub argv: Vec, - pub log_level: WorkerLogLevel, - pub enable_op_summary_metrics: bool, - pub enable_testing_features: bool, - pub has_node_modules_dir: bool, - pub hmr: bool, - pub inspect_brk: bool, - pub inspect_wait: bool, - pub strace_ops: Option>, - pub is_inspecting: bool, - pub location: Option, - pub argv0: Option, - pub node_debug: Option, - pub origin_data_folder_path: Option, - pub seed: Option, - pub unsafely_ignore_certificate_errors: Option>, - pub skip_op_registration: bool, pub create_hmr_runner: Option, pub create_coverage_collector: Option, - pub node_ipc: Option, - pub serve_port: Option, - pub serve_host: Option, + pub default_npm_caching_strategy: NpmCachingStrategy, + pub needs_test_modules: bool, } -struct SharedWorkerState { - blob_store: Arc, - broadcast_channel: InMemoryBroadcastChannel, - code_cache: Option>, - compiled_wasm_module_store: CompiledWasmModuleStore, - feature_checker: Arc, - fs: Arc, - maybe_file_watcher_communicator: Option>, - maybe_inspector_server: Option>, - maybe_lockfile: Option>, - module_loader_factory: Box, - node_resolver: Arc, - npm_installer: Option>, - npm_resolver: CliNpmResolver, - pkg_json_resolver: Arc, - root_cert_store_provider: Arc, - root_permissions: PermissionsContainer, - shared_array_buffer_store: SharedArrayBufferStore, - storage_key_resolver: StorageKeyResolver, - sys: CliSys, - options: CliMainWorkerOptions, - subcommand: DenoSubcommand, - otel_config: OtelConfig, - default_npm_caching_strategy: NpmCachingStrategy, -} - -impl SharedWorkerState { - pub fn create_node_init_services( - &self, - node_require_loader: NodeRequireLoaderRc, - ) -> NodeExtInitServices { - NodeExtInitServices { - node_require_loader, - node_resolver: self.node_resolver.clone(), - pkg_json_resolver: self.pkg_json_resolver.clone(), - sys: self.sys.clone(), - } - } - - pub fn npm_process_state_provider(&self) -> NpmProcessStateProviderRc { - crate::npm::create_npm_process_state_provider(&self.npm_resolver) - } +/// Data shared between the factory and workers. +struct SharedState { + pub create_hmr_runner: Option, + pub create_coverage_collector: Option, + pub maybe_file_watcher_communicator: Option>, } pub struct CliMainWorker { - main_module: ModuleSpecifier, - worker: MainWorker, - shared: Arc, + worker: LibMainWorker, + shared: Arc, } impl CliMainWorker { + #[inline] pub fn into_main_worker(self) -> MainWorker { - self.worker + self.worker.into_main_worker() } pub async fn setup_repl(&mut self) -> Result<(), AnyError> { @@ -204,16 +101,13 @@ impl CliMainWorker { self.maybe_setup_coverage_collector().await?; let mut maybe_hmr_runner = self.maybe_setup_hmr_runner().await?; - log::debug!("main_module {}", self.main_module); + log::debug!("main_module {}", self.worker.main_module()); self.execute_main_module().await?; self.worker.dispatch_load_event()?; loop { if let Some(hmr_runner) = maybe_hmr_runner.as_mut() { - let watcher_communicator = - self.shared.maybe_file_watcher_communicator.clone().unwrap(); - let hmr_future = hmr_runner.run().boxed_local(); let event_loop_future = self.worker.run_event_loop(false).boxed_local(); @@ -227,7 +121,11 @@ impl CliMainWorker { } } if let Err(e) = result { - watcher_communicator + self + .shared + .maybe_file_watcher_communicator + .as_ref() + .unwrap() .change_restart_mode(WatcherRestartMode::Automatic); return Err(e); } @@ -253,7 +151,7 @@ impl CliMainWorker { if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( coverage_collector.stop_collecting().boxed_local(), PollEventLoopOptions::default(), @@ -263,7 +161,7 @@ impl CliMainWorker { if let Some(hmr_runner) = maybe_hmr_runner.as_mut() { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( hmr_runner.stop().boxed_local(), PollEventLoopOptions::default(), @@ -335,24 +233,20 @@ impl CliMainWorker { executor.execute().await } + #[inline] pub async fn execute_main_module(&mut self) -> Result<(), CoreError> { - let id = self.worker.preload_main_module(&self.main_module).await?; - self.worker.evaluate_module(id).await + self.worker.execute_main_module().await } + #[inline] pub async fn execute_side_module(&mut self) -> Result<(), CoreError> { - let id = self.worker.preload_side_module(&self.main_module).await?; - self.worker.evaluate_module(id).await + self.worker.execute_side_module().await } pub async fn maybe_setup_hmr_runner( &mut self, ) -> Result>, AnyError> { - if !self.shared.options.hmr { - return Ok(None); - } - let Some(setup_hmr_runner) = self.shared.options.create_hmr_runner.as_ref() - else { + let Some(setup_hmr_runner) = self.shared.create_hmr_runner.as_ref() else { return Ok(None); }; @@ -362,7 +256,7 @@ impl CliMainWorker { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( hmr_runner.start().boxed_local(), PollEventLoopOptions::default(), @@ -375,7 +269,7 @@ impl CliMainWorker { &mut self, ) -> Result>, AnyError> { let Some(create_coverage_collector) = - self.shared.options.create_coverage_collector.as_ref() + self.shared.create_coverage_collector.as_ref() else { return Ok(None); }; @@ -384,7 +278,7 @@ impl CliMainWorker { let mut coverage_collector = create_coverage_collector(session); self .worker - .js_runtime + .js_runtime() .with_event_loop_future( coverage_collector.start_collecting().boxed_local(), PollEventLoopOptions::default(), @@ -398,72 +292,51 @@ impl CliMainWorker { name: &'static str, source_code: &'static str, ) -> Result, CoreError> { - self.worker.js_runtime.execute_script(name, source_code) + self.worker.js_runtime().execute_script(name, source_code) } } -// TODO(bartlomieju): this should be moved to some other place, added to avoid string -// duplication between worker setups and `deno info` output. -pub fn get_cache_storage_dir() -> PathBuf { - // Note: we currently use temp_dir() to avoid managing storage size. - std::env::temp_dir().join("deno_cache") -} - -#[derive(Clone)] pub struct CliMainWorkerFactory { - shared: Arc, + lib_main_worker_factory: LibMainWorkerFactory, + maybe_lockfile: Option>, + node_resolver: Arc, + npm_installer: Option>, + npm_resolver: CliNpmResolver, + root_permissions: PermissionsContainer, + shared: Arc, + sys: CliSys, + default_npm_caching_strategy: NpmCachingStrategy, + needs_test_modules: bool, } impl CliMainWorkerFactory { #[allow(clippy::too_many_arguments)] pub fn new( - blob_store: Arc, - code_cache: Option>, - feature_checker: Arc, - fs: Arc, + lib_main_worker_factory: LibMainWorkerFactory, maybe_file_watcher_communicator: Option>, - maybe_inspector_server: Option>, maybe_lockfile: Option>, - module_loader_factory: Box, node_resolver: Arc, npm_installer: Option>, npm_resolver: CliNpmResolver, - pkg_json_resolver: Arc, - root_cert_store_provider: Arc, - root_permissions: PermissionsContainer, - storage_key_resolver: StorageKeyResolver, sys: CliSys, - subcommand: DenoSubcommand, options: CliMainWorkerOptions, - otel_config: OtelConfig, - default_npm_caching_strategy: NpmCachingStrategy, + root_permissions: PermissionsContainer, ) -> Self { Self { - shared: Arc::new(SharedWorkerState { - blob_store, - broadcast_channel: Default::default(), - code_cache, - compiled_wasm_module_store: Default::default(), - feature_checker, - fs, + lib_main_worker_factory, + maybe_lockfile, + node_resolver, + npm_installer, + npm_resolver, + root_permissions, + sys, + shared: Arc::new(SharedState { + create_hmr_runner: options.create_hmr_runner, + create_coverage_collector: options.create_coverage_collector, maybe_file_watcher_communicator, - maybe_inspector_server, - maybe_lockfile, - module_loader_factory, - node_resolver, - npm_installer, - npm_resolver, - pkg_json_resolver, - root_cert_store_provider, - root_permissions, - shared_array_buffer_store: Default::default(), - storage_key_resolver, - sys, - options, - subcommand, - otel_config, - default_npm_caching_strategy, }), + default_npm_caching_strategy: options.default_npm_caching_strategy, + needs_test_modules: options.needs_test_modules, } } @@ -476,7 +349,7 @@ impl CliMainWorkerFactory { .create_custom_worker( mode, main_module, - self.shared.root_permissions.clone(), + self.root_permissions.clone(), vec![], Default::default(), ) @@ -491,23 +364,16 @@ impl CliMainWorkerFactory { custom_extensions: Vec, stdio: deno_runtime::deno_io::Stdio, ) -> Result { - let shared = &self.shared; - let CreateModuleLoaderResult { - module_loader, - node_require_loader, - } = shared - .module_loader_factory - .create_for_main(permissions.clone()); let main_module = if let Ok(package_ref) = NpmPackageReqReference::from_specifier(&main_module) { - if let Some(npm_installer) = &shared.npm_installer { + if let Some(npm_installer) = &self.npm_installer { let reqs = &[package_ref.req().clone()]; npm_installer .add_package_reqs( reqs, if matches!( - shared.default_npm_caching_strategy, + self.default_npm_caching_strategy, NpmCachingStrategy::Lazy ) { PackageCaching::Only(reqs.into()) @@ -520,18 +386,18 @@ impl CliMainWorkerFactory { // use a fake referrer that can be used to discover the package.json if necessary let referrer = ModuleSpecifier::from_directory_path( - self.shared.fs.cwd().map_err(JsErrorBox::from_err)?, + self.sys.env_current_dir().map_err(JsErrorBox::from_err)?, ) .unwrap() .join("package.json")?; - let package_folder = shared + let package_folder = self .npm_resolver .resolve_pkg_folder_from_deno_module_req(package_ref.req(), &referrer) .map_err(JsErrorBox::from_err)?; let main_module = self .resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?; - if let Some(lockfile) = &shared.maybe_lockfile { + if let Some(lockfile) = &self.maybe_lockfile { // For npm binary commands, ensure that the lockfile gets updated // so that we can re-use the npm resolution the next time it runs // for better performance @@ -543,119 +409,18 @@ impl CliMainWorkerFactory { main_module }; - let maybe_inspector_server = shared.maybe_inspector_server.clone(); - - let create_web_worker_cb = - create_web_worker_callback(shared.clone(), stdio.clone()); - - let maybe_storage_key = shared - .storage_key_resolver - .resolve_storage_key(&main_module); - let origin_storage_dir = maybe_storage_key.as_ref().map(|key| { - shared - .options - .origin_data_folder_path - .as_ref() - .unwrap() // must be set if storage key resolver returns a value - .join(checksum::gen(&[key.as_bytes()])) - }); - let cache_storage_dir = maybe_storage_key.map(|key| { - // TODO(@satyarohith): storage quota management - get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) - }); - - // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out - // list of enabled features. - let feature_checker = shared.feature_checker.clone(); - let mut unstable_features = - Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len()); - for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { - if feature_checker.check(granular_flag.name) { - unstable_features.push(granular_flag.id); - } - } - - let services = WorkerServiceOptions { - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), - module_loader, - fs: shared.fs.clone(), - node_services: Some( - shared.create_node_init_services(node_require_loader), - ), - npm_process_state_provider: Some(shared.npm_process_state_provider()), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - fetch_dns_resolver: Default::default(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), - feature_checker, + let mut worker = self.lib_main_worker_factory.create_custom_worker( + mode, + main_module, permissions, - v8_code_cache: shared.code_cache.clone().map(|c| c.as_code_cache()), - }; - - let options = WorkerOptions { - bootstrap: BootstrapOptions { - deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), - args: shared.options.argv.clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - log_level: shared.options.log_level, - enable_op_summary_metrics: shared.options.enable_op_summary_metrics, - enable_testing_features: shared.options.enable_testing_features, - locale: deno_core::v8::icu::get_language_tag(), - location: shared.options.location.clone(), - no_color: !colors::use_color(), - is_stdout_tty: deno_terminal::is_stdout_tty(), - is_stderr_tty: deno_terminal::is_stderr_tty(), - color_level: colors::get_color_level(), - unstable_features, - user_agent: version::DENO_VERSION_INFO.user_agent.to_string(), - inspect: shared.options.is_inspecting, - has_node_modules_dir: shared.options.has_node_modules_dir, - argv0: shared.options.argv0.clone(), - node_debug: shared.options.node_debug.clone(), - node_ipc_fd: shared.options.node_ipc, - mode, - serve_port: shared.options.serve_port, - serve_host: shared.options.serve_host.clone(), - otel_config: shared.otel_config.clone(), - close_on_idle: true, - }, - extensions: custom_extensions, - startup_snapshot: crate::js::deno_isolate_init(), - create_params: create_isolate_create_params(), - unsafely_ignore_certificate_errors: shared - .options - .unsafely_ignore_certificate_errors - .clone(), - seed: shared.options.seed, - format_js_error_fn: Some(Arc::new(format_js_error)), - create_web_worker_cb, - maybe_inspector_server, - should_break_on_first_statement: shared.options.inspect_brk, - should_wait_for_inspector_session: shared.options.inspect_wait, - strace_ops: shared.options.strace_ops.clone(), - cache_storage_dir, - origin_storage_dir, + custom_extensions, stdio, - skip_op_registration: shared.options.skip_op_registration, - enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( - ), - }; + )?; - let mut worker = MainWorker::bootstrap_from_options( - main_module.clone(), - services, - options, - ); - - if self.shared.subcommand.needs_test() { + if self.needs_test_modules { macro_rules! test_file { ($($file:literal),*) => { - $(worker.js_runtime.lazy_load_es_module_with_code( + $(worker.js_runtime().lazy_load_es_module_with_code( concat!("ext:cli/", $file), deno_core::ascii_str_include!(concat!("js/", $file)), )?;)* @@ -673,9 +438,8 @@ impl CliMainWorkerFactory { } Ok(CliMainWorker { - main_module, worker, - shared: shared.clone(), + shared: self.shared.clone(), }) } @@ -685,7 +449,6 @@ impl CliMainWorkerFactory { sub_path: Option<&str>, ) -> Result { match self - .shared .node_resolver .resolve_binary_export(package_folder, sub_path) { @@ -720,7 +483,6 @@ impl CliMainWorkerFactory { } let specifier = self - .shared .node_resolver .resolve_package_subpath_from_deno_module( package_folder, @@ -741,136 +503,20 @@ impl CliMainWorkerFactory { } } -fn create_web_worker_callback( - shared: Arc, - stdio: deno_runtime::deno_io::Stdio, -) -> Arc { - Arc::new(move |args| { - let maybe_inspector_server = shared.maybe_inspector_server.clone(); - - let CreateModuleLoaderResult { - module_loader, - node_require_loader, - } = shared.module_loader_factory.create_for_worker( - args.parent_permissions.clone(), - args.permissions.clone(), - ); - let create_web_worker_cb = - create_web_worker_callback(shared.clone(), stdio.clone()); - - let maybe_storage_key = shared - .storage_key_resolver - .resolve_storage_key(&args.main_module); - let cache_storage_dir = maybe_storage_key.map(|key| { - // TODO(@satyarohith): storage quota management - get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) - }); - - // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out - // list of enabled features. - let feature_checker = shared.feature_checker.clone(); - let mut unstable_features = - Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len()); - for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { - if feature_checker.check(granular_flag.name) { - unstable_features.push(granular_flag.id); - } - } - - let services = WebWorkerServiceOptions { - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), - module_loader, - fs: shared.fs.clone(), - node_services: Some( - shared.create_node_init_services(node_require_loader), - ), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), - maybe_inspector_server, - feature_checker, - npm_process_state_provider: Some(shared.npm_process_state_provider()), - permissions: args.permissions, - }; - let options = WebWorkerOptions { - name: args.name, - main_module: args.main_module.clone(), - worker_id: args.worker_id, - bootstrap: BootstrapOptions { - deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), - args: shared.options.argv.clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - log_level: shared.options.log_level, - enable_op_summary_metrics: shared.options.enable_op_summary_metrics, - enable_testing_features: shared.options.enable_testing_features, - locale: deno_core::v8::icu::get_language_tag(), - location: Some(args.main_module), - no_color: !colors::use_color(), - color_level: colors::get_color_level(), - is_stdout_tty: deno_terminal::is_stdout_tty(), - is_stderr_tty: deno_terminal::is_stderr_tty(), - unstable_features, - user_agent: version::DENO_VERSION_INFO.user_agent.to_string(), - inspect: shared.options.is_inspecting, - has_node_modules_dir: shared.options.has_node_modules_dir, - argv0: shared.options.argv0.clone(), - node_debug: shared.options.node_debug.clone(), - node_ipc_fd: None, - mode: WorkerExecutionMode::Worker, - serve_port: shared.options.serve_port, - serve_host: shared.options.serve_host.clone(), - otel_config: shared.otel_config.clone(), - close_on_idle: args.close_on_idle, - }, - extensions: vec![], - startup_snapshot: crate::js::deno_isolate_init(), - create_params: create_isolate_create_params(), - unsafely_ignore_certificate_errors: shared - .options - .unsafely_ignore_certificate_errors - .clone(), - seed: shared.options.seed, - create_web_worker_cb, - format_js_error_fn: Some(Arc::new(format_js_error)), - worker_type: args.worker_type, - stdio: stdio.clone(), - cache_storage_dir, - strace_ops: shared.options.strace_ops.clone(), - close_on_idle: args.close_on_idle, - maybe_worker_metadata: args.maybe_worker_metadata, - enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( - ), - }; - - WebWorker::bootstrap_from_options(services, options) - }) -} - -/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs. -/// Instead probe for the total memory on the system and use it instead -/// as a default. -pub fn create_isolate_create_params() -> Option { - let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info(); - maybe_mem_info.map(|mem_info| { - v8::CreateParams::default() - .heap_limits_from_system_memory(mem_info.total, 0) - }) -} - #[allow(clippy::print_stdout)] #[allow(clippy::print_stderr)] #[cfg(test)] mod tests { + use std::rc::Rc; + use deno_core::resolve_path; use deno_core::FsModuleLoader; - use deno_fs::RealFs; + use deno_resolver::npm::DenoInNpmPackageChecker; + use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_permissions::Permissions; use deno_runtime::permissions::RuntimePermissionDescriptorParser; + use deno_runtime::worker::WorkerOptions; + use deno_runtime::worker::WorkerServiceOptions; use super::*; @@ -891,7 +537,7 @@ mod tests { CliNpmResolver, CliSys, >( - main_module, + &main_module, WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), permissions: PermissionsContainer::new( diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index e1538b8b75..a1c24f30e4 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -43,7 +43,7 @@ async fn main() -> Result<(), AnyError> { RuntimePermissionDescriptorParser::new(sys_traits::impls::RealSys), ); let mut worker = MainWorker::bootstrap_from_options( - main_module.clone(), + &main_module, WorkerServiceOptions::< DenoInNpmPackageChecker, NpmResolver, diff --git a/runtime/lib.rs b/runtime/lib.rs index c104f5cd61..65d3e88bae 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -18,6 +18,7 @@ pub use deno_net; pub use deno_node; pub use deno_os; pub use deno_permissions; +pub use deno_telemetry; pub use deno_terminal::colors; pub use deno_tls; pub use deno_url; diff --git a/runtime/worker.rs b/runtime/worker.rs index cfcadccc46..426383a19e 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -303,7 +303,7 @@ impl MainWorker { TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TExtNodeSys: ExtNodeSys + 'static, >( - main_module: ModuleSpecifier, + main_module: &ModuleSpecifier, services: WorkerServiceOptions< TInNpmPackageChecker, TNpmPackageFolderResolver, @@ -322,7 +322,7 @@ impl MainWorker { TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TExtNodeSys: ExtNodeSys + 'static, >( - main_module: ModuleSpecifier, + main_module: &ModuleSpecifier, services: WorkerServiceOptions< TInNpmPackageChecker, TNpmPackageFolderResolver,