From 0b033140c07b5abee231711b0fbc3fa24f5f9eec Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 14 Jan 2025 10:01:05 -0500 Subject: [PATCH 1/8] refactor: move `CliNpmResolver` to `deno_resolver::npm::NpmResolver` (#27659) As title. After this PR all npm resolution will be out of the CLI crate. --- Cargo.lock | 1 + cli/cache/mod.rs | 5 +- cli/emit.rs | 6 +- cli/factory.rs | 41 ++-- cli/graph_util.rs | 21 +- cli/lsp/analysis.rs | 26 ++- cli/lsp/resolver.rs | 135 +++++++---- cli/module_loader.rs | 19 +- cli/node.rs | 16 +- cli/npm/byonm.rs | 64 +---- cli/npm/managed.rs | 285 +---------------------- cli/npm/mod.rs | 91 ++------ cli/resolver.rs | 22 +- cli/standalone/binary.rs | 45 ++-- cli/standalone/mod.rs | 30 +-- cli/task_runner.rs | 18 +- cli/tools/check.rs | 29 ++- cli/tools/coverage/mod.rs | 5 +- cli/tools/info.rs | 50 ++-- cli/tools/registry/pm/deps.rs | 18 +- cli/tools/task.rs | 4 +- cli/tsc/mod.rs | 8 +- cli/worker.rs | 16 +- ext/node/lib.rs | 42 ++-- ext/node/ops/require.rs | 51 +++- resolvers/deno/cjs.rs | 25 +- resolvers/deno/lib.rs | 60 ++++- resolvers/deno/npm/byonm.rs | 17 +- resolvers/deno/npm/managed/common.rs | 73 ++++-- resolvers/deno/npm/managed/global.rs | 48 ++-- resolvers/deno/npm/managed/local.rs | 119 +++++----- resolvers/deno/npm/managed/mod.rs | 97 +++++--- resolvers/deno/npm/managed/resolution.rs | 21 +- resolvers/deno/npm/mod.rs | 197 +++++++++++++--- resolvers/node/analyze.rs | 39 +++- resolvers/node/lib.rs | 2 - resolvers/node/npm.rs | 15 +- resolvers/node/resolution.rs | 50 +++- resolvers/node/sync.rs | 7 - runtime/Cargo.toml | 1 + runtime/examples/extension/main.rs | 8 +- runtime/snapshot.rs | 4 + runtime/web_worker.rs | 50 +++- runtime/worker.rs | 50 +++- 44 files changed, 1030 insertions(+), 901 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa13ae0ccd..95633e3ce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2242,6 +2242,7 @@ dependencies = [ "deno_node", "deno_path_util", "deno_permissions", + "deno_resolver", "deno_telemetry", "deno_terminal 0.2.0", "deno_tls", diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index ff9f07fc4e..0d7808cba6 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -15,6 +15,7 @@ use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::deno_permissions::PermissionsContainer; use node_resolver::InNpmPackageChecker; @@ -76,7 +77,7 @@ pub struct FetchCacher { pub file_header_overrides: HashMap>, file_fetcher: Arc, global_http_cache: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, module_info_cache: Arc, permissions: PermissionsContainer, sys: CliSys, @@ -88,7 +89,7 @@ impl FetchCacher { pub fn new( file_fetcher: Arc, global_http_cache: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, module_info_cache: Arc, sys: CliSys, options: FetchCacherOptions, diff --git a/cli/emit.rs b/cli/emit.rs index e9b5a4e250..69ac8323bb 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -24,11 +24,11 @@ use deno_graph::ModuleGraph; use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; #[derive(Debug)] pub struct Emitter { - cjs_tracker: Arc, + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_and_emit_options: @@ -39,7 +39,7 @@ pub struct Emitter { impl Emitter { pub fn new( - cjs_tracker: Arc, + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_options: deno_ast::TranspileOptions, diff --git a/cli/factory.rs b/cli/factory.rs index e3873b5164..3280fd379b 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -16,6 +16,7 @@ use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; use deno_resolver::npm::managed::NpmResolutionCell; use deno_resolver::npm::CreateInNpmPkgCheckerOptions; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::npm::NpmReqResolverOptions; use deno_resolver::sloppy_imports::SloppyImportsCachedFs; use deno_resolver::DenoResolverOptions; @@ -32,7 +33,6 @@ use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use log::warn; use node_resolver::analyze::NodeCodeTranslator; -use node_resolver::InNpmPackageChecker; use once_cell::sync::OnceCell; use crate::args::check_warn_tsconfig; @@ -68,7 +68,6 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; -use crate::npm::create_cli_npm_resolver; use crate::npm::installer::NpmInstaller; use crate::npm::installer::NpmResolutionInstaller; use crate::npm::CliByonmNpmResolverCreateOptions; @@ -83,7 +82,7 @@ use crate::npm::CliNpmTarballCache; use crate::npm::NpmRegistryReadPermissionChecker; use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::resolver::CliDenoResolver; use crate::resolver::CliNpmGraphResolver; use crate::resolver::CliNpmReqResolver; @@ -190,7 +189,7 @@ impl Deferred { struct CliFactoryServices { blob_store: Deferred>, caches: Deferred>, - cjs_tracker: Deferred>, + cjs_tracker: Deferred>, cli_options: Deferred>, code_cache: Deferred>, deno_resolver: Deferred>, @@ -203,7 +202,7 @@ struct CliFactoryServices { global_http_cache: Deferred>, http_cache: Deferred>, http_client_provider: Deferred>, - in_npm_pkg_checker: Deferred>, + in_npm_pkg_checker: Deferred, main_graph_container: Deferred>, maybe_file_watcher_reporter: Deferred>, maybe_inspector_server: Deferred>>, @@ -223,7 +222,7 @@ struct CliFactoryServices { npm_resolution: Arc, npm_resolution_initializer: Deferred>, npm_resolution_installer: Deferred>, - npm_resolver: Deferred>, + npm_resolver: Deferred, npm_tarball_cache: Deferred>, parsed_source_cache: Deferred>, permission_desc_parser: @@ -399,7 +398,7 @@ impl CliFactory { pub fn in_npm_pkg_checker( &self, - ) -> Result<&Arc, AnyError> { + ) -> Result<&DenoInNpmPackageChecker, AnyError> { self.services.in_npm_pkg_checker.get_or_try_init(|| { let cli_options = self.cli_options()?; let options = if cli_options.use_byonm() { @@ -414,7 +413,7 @@ impl CliFactory { }, ) }; - Ok(deno_resolver::npm::create_in_npm_pkg_checker(options)) + Ok(DenoInNpmPackageChecker::new(options)) }) } @@ -559,16 +558,14 @@ impl CliFactory { }) } - pub async fn npm_resolver( - &self, - ) -> Result<&Arc, AnyError> { + pub async fn npm_resolver(&self) -> Result<&CliNpmResolver, AnyError> { self .services .npm_resolver .get_or_try_init_async( async { let cli_options = self.cli_options()?; - Ok(create_cli_npm_resolver(if cli_options.use_byonm() { + Ok(CliNpmResolver::new(if cli_options.use_byonm() { CliNpmResolverCreateOptions::Byonm( CliByonmNpmResolverCreateOptions { sys: self.sys(), @@ -796,11 +793,7 @@ impl CliFactory { Ok(Arc::new(CliNodeResolver::new( self.in_npm_pkg_checker()?.clone(), RealIsBuiltInNodeModuleChecker, - self - .npm_resolver() - .await? - .clone() - .into_npm_pkg_folder_resolver(), + self.npm_resolver().await?.clone(), self.pkg_json_resolver().clone(), self.sys(), node_resolver::ConditionsFromResolutionMode::default(), @@ -833,11 +826,7 @@ impl CliFactory { cjs_esm_analyzer, self.in_npm_pkg_checker()?.clone(), node_resolver, - self - .npm_resolver() - .await? - .clone() - .into_npm_pkg_folder_resolver(), + self.npm_resolver().await?.clone(), self.pkg_json_resolver().clone(), self.sys(), ))) @@ -857,7 +846,7 @@ impl CliFactory { sys: self.sys(), in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(), node_resolver: self.node_resolver().await?.clone(), - npm_resolver: npm_resolver.clone().into_byonm_or_managed(), + npm_resolver: npm_resolver.clone(), }))) }) .await @@ -988,10 +977,10 @@ impl CliFactory { .await } - pub fn cjs_tracker(&self) -> Result<&Arc, AnyError> { + pub fn cjs_tracker(&self) -> Result<&Arc, AnyError> { self.services.cjs_tracker.get_or_try_init(|| { let options = self.cli_options()?; - Ok(Arc::new(CjsTracker::new( + Ok(Arc::new(CliCjsTracker::new( self.in_npm_pkg_checker()?.clone(), self.pkg_json_resolver().clone(), if options.is_node_main() || options.unstable_detect_cjs() { @@ -1040,7 +1029,7 @@ impl CliFactory { self.emitter()?, self.file_fetcher()?, self.http_client_provider(), - self.npm_resolver().await?.as_ref(), + self.npm_resolver().await?, self.workspace_resolver().await?.as_ref(), cli_options.npm_system_info(), )) diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 17d4fef553..e57fcf8a94 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -30,6 +30,7 @@ use deno_graph::ResolutionError; use deno_graph::SpecifierError; use deno_graph::WorkspaceFastCheckOption; use deno_path_util::url_to_file_path; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::sloppy_imports::SloppyImportsCachedFs; use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; use deno_runtime::deno_node; @@ -37,7 +38,6 @@ use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; use deno_semver::SmallStackString; -use node_resolver::InNpmPackageChecker; use crate::args::config_to_deno_graph_workspace_member; use crate::args::jsr_url; @@ -55,7 +55,7 @@ use crate::file_fetcher::CliFileFetcher; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; use crate::npm::CliNpmResolver; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmGraphResolver; use crate::resolver::CliResolver; use crate::resolver::CliSloppyImportsResolver; @@ -493,17 +493,17 @@ pub enum BuildGraphWithNpmResolutionError { pub struct ModuleGraphBuilder { caches: Arc, - cjs_tracker: Arc, + cjs_tracker: Arc, cli_options: Arc, file_fetcher: Arc, global_http_cache: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, lockfile: Option>, maybe_file_watcher_reporter: Option, module_info_cache: Arc, npm_graph_resolver: Arc, npm_installer: Option>, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, parsed_source_cache: Arc, resolver: Arc, root_permissions_container: PermissionsContainer, @@ -514,17 +514,17 @@ impl ModuleGraphBuilder { #[allow(clippy::too_many_arguments)] pub fn new( caches: Arc, - cjs_tracker: Arc, + cjs_tracker: Arc, cli_options: Arc, file_fetcher: Arc, global_http_cache: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, lockfile: Option>, maybe_file_watcher_reporter: Option, module_info_cache: Arc, npm_graph_resolver: Arc, npm_installer: Option>, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, parsed_source_cache: Arc, resolver: Arc, root_permissions_container: PermissionsContainer, @@ -712,8 +712,7 @@ impl ModuleGraphBuilder { let initial_package_deps_len = graph.packages.package_deps_sum(); let initial_package_mappings_len = graph.packages.mappings().len(); - if roots.iter().any(|r| r.scheme() == "npm") - && self.npm_resolver.as_byonm().is_some() + if roots.iter().any(|r| r.scheme() == "npm") && self.npm_resolver.is_byonm() { return Err(BuildGraphWithNpmResolutionError::UnsupportedNpmSpecifierEntrypointResolutionWay); } @@ -1226,7 +1225,7 @@ fn format_deno_graph_error(err: &dyn Error) -> String { #[derive(Debug)] struct CliGraphResolver<'a> { - cjs_tracker: &'a CjsTracker, + cjs_tracker: &'a CliCjsTracker, resolver: &'a CliResolver, jsx_import_source_config: Option, } diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index f7b487f055..f8f382f594 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -19,6 +19,7 @@ use deno_core::ModuleSpecifier; use deno_error::JsErrorBox; use deno_lint::diagnostic::LintDiagnosticRange; use deno_path_util::url_to_file_path; +use deno_resolver::npm::managed::NpmResolutionCell; use deno_runtime::deno_node::PathClean; use deno_semver::jsr::JsrPackageNvReference; use deno_semver::jsr::JsrPackageReqReference; @@ -31,6 +32,7 @@ use deno_semver::SmallStackString; use deno_semver::StackString; use deno_semver::Version; use import_map::ImportMap; +use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; use once_cell::sync::Lazy; @@ -365,7 +367,9 @@ impl<'a> TsResponseImportMapper<'a> { if let Ok(Some(pkg_id)) = npm_resolver.resolve_pkg_id_from_specifier(specifier) { - let pkg_reqs = npm_resolver.resolve_pkg_reqs_from_pkg_id(&pkg_id); + let pkg_reqs = npm_resolver + .resolution() + .resolve_pkg_reqs_from_pkg_id(&pkg_id); // check if any pkg reqs match what is found in an import map if !pkg_reqs.is_empty() { let sub_path = npm_resolver @@ -1295,6 +1299,19 @@ impl CodeActionCollection { range: &lsp::Range, language_server: &language_server::Inner, ) -> Option { + fn top_package_req_for_name( + resolution: &NpmResolutionCell, + name: &str, + ) -> Option { + let package_reqs = resolution.package_reqs(); + let mut entries = package_reqs + .into_iter() + .filter(|(_, nv)| nv.name == name) + .collect::>(); + entries.sort_by(|a, b| a.1.version.cmp(&b.1.version)); + Some(entries.pop()?.0) + } + let (dep_key, dependency, _) = document.get_maybe_dependency(&range.end)?; if dependency.maybe_deno_types_specifier.is_some() { @@ -1382,9 +1399,10 @@ impl CodeActionCollection { .and_then(|versions| versions.first().cloned())?; let types_specifier_text = if let Some(npm_resolver) = managed_npm_resolver { - let mut specifier_text = if let Some(req) = - npm_resolver.top_package_req_for_name(&types_package_name) - { + let mut specifier_text = if let Some(req) = top_package_req_for_name( + npm_resolver.resolution(), + &types_package_name, + ) { format!("npm:{req}") } else { format!("npm:{}@^{}", &types_package_name, types_package_version) diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index af4ab6571f..1b393ad22b 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -26,6 +26,7 @@ use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; use deno_resolver::npm::managed::NpmResolutionCell; use deno_resolver::npm::CreateInNpmPkgCheckerOptions; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::npm::NpmReqResolverOptions; use deno_resolver::DenoResolverOptions; use deno_resolver::NodeAndNpmReqResolver; @@ -35,7 +36,6 @@ use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use indexmap::IndexMap; -use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; @@ -56,10 +56,10 @@ use crate::lsp::config::ConfigData; use crate::lsp::logging::lsp_warn; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; -use crate::npm::create_cli_npm_resolver; use crate::npm::installer::NpmInstaller; use crate::npm::installer::NpmResolutionInstaller; use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedNpmResolver; use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmCache; use crate::npm::CliNpmCacheHttpClient; @@ -67,14 +67,13 @@ use crate::npm::CliNpmRegistryInfoProvider; use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::ManagedCliNpmResolver; use crate::npm::NpmResolutionInitializer; use crate::resolver::CliDenoResolver; +use crate::resolver::CliIsCjsResolver; use crate::resolver::CliNpmGraphResolver; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; use crate::resolver::FoundPackageJsonDepFlag; -use crate::resolver::IsCjsResolver; use crate::sys::CliSys; use crate::tsc::into_specifier_and_media_type; use crate::util::progress_bar::ProgressBar; @@ -83,12 +82,13 @@ use crate::util::progress_bar::ProgressBarStyle; #[derive(Debug, Clone)] struct LspScopeResolver { resolver: Arc, - in_npm_pkg_checker: Arc, - is_cjs_resolver: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, + is_cjs_resolver: Arc, jsr_resolver: Option>, npm_graph_resolver: Arc, npm_installer: Option>, - npm_resolver: Option>, + npm_resolution: Arc, + npm_resolver: Option, node_resolver: Option>, npm_pkg_req_resolver: Option>, pkg_json_resolver: Arc, @@ -111,6 +111,7 @@ impl Default for LspScopeResolver { npm_installer: None, npm_resolver: None, node_resolver: None, + npm_resolution: factory.services.npm_resolution.clone(), npm_pkg_req_resolver: None, pkg_json_resolver: factory.pkg_json_resolver().clone(), redirect_resolver: None, @@ -224,6 +225,7 @@ impl LspScopeResolver { npm_pkg_req_resolver, npm_resolver, npm_installer, + npm_resolution: factory.services.npm_resolution.clone(), node_resolver, pkg_json_resolver, redirect_resolver, @@ -235,12 +237,58 @@ impl LspScopeResolver { } fn snapshot(&self) -> Arc { + // create a copy of the resolution and then re-initialize the npm resolver from that + // todo(dsherret): this is pretty terrible... we should improve this. It should + // be possible to just change the npm_resolution on the new factory then access + // another method to create a new npm resolver let mut factory = ResolverFactory::new(self.config_data.as_ref()); - let npm_resolver = - self.npm_resolver.as_ref().map(|r| r.clone_snapshotted()); + factory + .services + .npm_resolution + .set_snapshot(self.npm_resolution.snapshot()); + let npm_resolver = self.npm_resolver.as_ref(); if let Some(npm_resolver) = &npm_resolver { - factory.set_npm_resolver(npm_resolver.clone()); + factory.set_npm_resolver(CliNpmResolver::new::( + match npm_resolver { + CliNpmResolver::Byonm(byonm_npm_resolver) => { + CliNpmResolverCreateOptions::Byonm( + CliByonmNpmResolverCreateOptions { + root_node_modules_dir: byonm_npm_resolver + .root_node_modules_path() + .map(|p| p.to_path_buf()), + sys: CliSys::default(), + pkg_json_resolver: self.pkg_json_resolver.clone(), + }, + ) + } + CliNpmResolver::Managed(managed_npm_resolver) => { + CliNpmResolverCreateOptions::Managed({ + let npmrc = self + .config_data + .as_ref() + .and_then(|d| d.npmrc.clone()) + .unwrap_or_else(create_default_npmrc); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &CliSys::default(), + managed_npm_resolver.global_cache_root_path().to_path_buf(), + npmrc.get_all_known_registries_urls(), + )); + CliManagedNpmResolverCreateOptions { + sys: CliSys::default(), + npm_cache_dir, + maybe_node_modules_path: managed_npm_resolver + .root_node_modules_path() + .map(|p| p.to_path_buf()), + npmrc, + npm_resolution: factory.services.npm_resolution.clone(), + npm_system_info: NpmSystemInfo::default(), + } + }) + } + }, + )); } + Arc::new(Self { resolver: factory.cli_resolver().clone(), in_npm_pkg_checker: factory.in_npm_pkg_checker().clone(), @@ -250,6 +298,7 @@ impl LspScopeResolver { // npm installer isn't necessary for a snapshot npm_installer: None, npm_pkg_req_resolver: factory.npm_pkg_req_resolver().cloned(), + npm_resolution: factory.services.npm_resolution.clone(), npm_resolver: factory.npm_resolver().cloned(), node_resolver: factory.node_resolver().cloned(), redirect_resolver: self.redirect_resolver.clone(), @@ -366,7 +415,7 @@ impl LspResolver { pub fn as_is_cjs_resolver( &self, file_referrer: Option<&ModuleSpecifier>, - ) -> &IsCjsResolver { + ) -> &CliIsCjsResolver { let resolver = self.get_scope_resolver(file_referrer); resolver.is_cjs_resolver.as_ref() } @@ -382,7 +431,7 @@ impl LspResolver { pub fn in_npm_pkg_checker( &self, file_referrer: Option<&ModuleSpecifier>, - ) -> &Arc { + ) -> &DenoInNpmPackageChecker { let resolver = self.get_scope_resolver(file_referrer); &resolver.in_npm_pkg_checker } @@ -390,7 +439,7 @@ impl LspResolver { pub fn maybe_managed_npm_resolver( &self, file_referrer: Option<&ModuleSpecifier>, - ) -> Option<&ManagedCliNpmResolver> { + ) -> Option<&CliManagedNpmResolver> { let resolver = self.get_scope_resolver(file_referrer); resolver.npm_resolver.as_ref().and_then(|r| r.as_managed()) } @@ -605,13 +654,14 @@ pub struct ScopeDepInfo { struct ResolverFactoryServices { cli_resolver: Deferred>, found_pkg_json_dep_flag: Arc, - in_npm_pkg_checker: Deferred>, - is_cjs_resolver: Deferred>, + in_npm_pkg_checker: Deferred, + is_cjs_resolver: Deferred>, node_resolver: Deferred>>, npm_graph_resolver: Deferred>, npm_installer: Option>, npm_pkg_req_resolver: Deferred>>, - npm_resolver: Option>, + npm_resolver: Option, + npm_resolution: Arc, } struct ResolverFactory<'a> { @@ -686,10 +736,9 @@ impl<'a> ResolverFactory<'a> { npm_client.clone(), npmrc.clone(), )); - let npm_resolution = Arc::new(NpmResolutionCell::default()); let npm_resolution_initializer = Arc::new(NpmResolutionInitializer::new( registry_info_provider.clone(), - npm_resolution.clone(), + self.services.npm_resolution.clone(), match self.config_data.and_then(|d| d.lockfile.as_ref()) { Some(lockfile) => { CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( @@ -712,13 +761,13 @@ impl<'a> ResolverFactory<'a> { )); let npm_resolution_installer = Arc::new(NpmResolutionInstaller::new( registry_info_provider, - npm_resolution.clone(), + self.services.npm_resolution.clone(), maybe_lockfile.clone(), )); let npm_installer = Arc::new(NpmInstaller::new( npm_cache.clone(), Arc::new(NpmInstallDepsProvider::empty()), - npm_resolution.clone(), + self.services.npm_resolution.clone(), npm_resolution_initializer.clone(), npm_resolution_installer, &pb, @@ -745,22 +794,22 @@ impl<'a> ResolverFactory<'a> { npm_cache_dir, maybe_node_modules_path, npmrc, - npm_resolution, + npm_resolution: self.services.npm_resolution.clone(), npm_system_info: NpmSystemInfo::default(), }) }; - self.set_npm_resolver(create_cli_npm_resolver(options)); + self.set_npm_resolver(CliNpmResolver::new(options)); } pub fn set_npm_installer(&mut self, npm_installer: Arc) { self.services.npm_installer = Some(npm_installer); } - pub fn set_npm_resolver(&mut self, npm_resolver: Arc) { + pub fn set_npm_resolver(&mut self, npm_resolver: CliNpmResolver) { self.services.npm_resolver = Some(npm_resolver); } - pub fn npm_resolver(&self) -> Option<&Arc> { + pub fn npm_resolver(&self) -> Option<&CliNpmResolver> { self.services.npm_resolver.as_ref() } @@ -825,29 +874,27 @@ impl<'a> ResolverFactory<'a> { &self.pkg_json_resolver } - pub fn in_npm_pkg_checker(&self) -> &Arc { + pub fn in_npm_pkg_checker(&self) -> &DenoInNpmPackageChecker { self.services.in_npm_pkg_checker.get_or_init(|| { - deno_resolver::npm::create_in_npm_pkg_checker( - match self.services.npm_resolver.as_ref().map(|r| r.as_inner()) { - Some(crate::npm::InnerCliNpmResolverRef::Byonm(_)) | None => { - CreateInNpmPkgCheckerOptions::Byonm - } - Some(crate::npm::InnerCliNpmResolverRef::Managed(m)) => { - CreateInNpmPkgCheckerOptions::Managed( - ManagedInNpmPkgCheckerCreateOptions { - root_cache_dir_url: m.global_cache_root_url(), - maybe_node_modules_path: m.maybe_node_modules_path(), - }, - ) - } - }, - ) + DenoInNpmPackageChecker::new(match &self.services.npm_resolver { + Some(CliNpmResolver::Byonm(_)) | None => { + CreateInNpmPkgCheckerOptions::Byonm + } + Some(CliNpmResolver::Managed(m)) => { + CreateInNpmPkgCheckerOptions::Managed( + ManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: m.global_cache_root_url(), + maybe_node_modules_path: m.root_node_modules_path(), + }, + ) + } + }) }) } - pub fn is_cjs_resolver(&self) -> &Arc { + pub fn is_cjs_resolver(&self) -> &Arc { self.services.is_cjs_resolver.get_or_init(|| { - Arc::new(IsCjsResolver::new( + Arc::new(CliIsCjsResolver::new( self.in_npm_pkg_checker().clone(), self.pkg_json_resolver().clone(), if self @@ -871,7 +918,7 @@ impl<'a> ResolverFactory<'a> { Some(Arc::new(CliNodeResolver::new( self.in_npm_pkg_checker().clone(), RealIsBuiltInNodeModuleChecker, - npm_resolver.clone().into_npm_pkg_folder_resolver(), + npm_resolver.clone(), self.pkg_json_resolver.clone(), self.sys.clone(), node_resolver::ConditionsFromResolutionMode::default(), @@ -890,7 +937,7 @@ impl<'a> ResolverFactory<'a> { Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { in_npm_pkg_checker: self.in_npm_pkg_checker().clone(), node_resolver: node_resolver.clone(), - npm_resolver: npm_resolver.clone().into_byonm_or_managed(), + npm_resolver: npm_resolver.clone(), sys: self.sys.clone(), }))) }) diff --git a/cli/module_loader.rs b/cli/module_loader.rs index ba53077a3c..daeb4dda37 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -39,6 +39,7 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::Resolution; use deno_graph::WasmModule; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::code_cache; use deno_runtime::deno_node::create_host_defined_options; use deno_runtime::deno_node::NodeRequireLoader; @@ -70,7 +71,7 @@ use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; use crate::npm::NpmRegistryReadPermissionChecker; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; use crate::resolver::ModuleCodeStringSource; @@ -233,10 +234,10 @@ struct SharedCliModuleLoaderState { initial_cwd: PathBuf, is_inspecting: bool, is_repl: bool, - cjs_tracker: Arc, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, module_load_preparer: Arc, node_code_translator: Arc, @@ -244,7 +245,7 @@ struct SharedCliModuleLoaderState { npm_module_loader: NpmModuleLoader, npm_registry_permission_checker: Arc, npm_req_resolver: Arc, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, parsed_source_cache: Arc, resolver: Arc, sys: CliSys, @@ -294,10 +295,10 @@ impl CliModuleLoaderFactory { #[allow(clippy::too_many_arguments)] pub fn new( options: &CliOptions, - cjs_tracker: Arc, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, module_load_preparer: Arc, node_code_translator: Arc, @@ -305,7 +306,7 @@ impl CliModuleLoaderFactory { npm_module_loader: NpmModuleLoader, npm_registry_permission_checker: Arc, npm_req_resolver: Arc, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, parsed_source_cache: Arc, resolver: Arc, sys: CliSys, @@ -1139,11 +1140,11 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { #[derive(Debug)] struct CliNodeRequireLoader { - cjs_tracker: Arc, + cjs_tracker: Arc, emitter: Arc, sys: CliSys, graph_container: TGraphContainer, - in_npm_pkg_checker: Arc, + in_npm_pkg_checker: DenoInNpmPackageChecker, npm_registry_permission_checker: Arc, } diff --git a/cli/node.rs b/cli/node.rs index aa44dcab18..892e25914a 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -7,6 +7,7 @@ use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_graph::ParsedSourceStore; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::deno_fs; use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -19,15 +20,22 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; -use crate::resolver::CjsTracker; +use crate::npm::CliNpmResolver; +use crate::resolver::CliCjsTracker; use crate::sys::CliSys; pub type CliNodeCodeTranslator = NodeCodeTranslator< CliCjsCodeAnalyzer, + DenoInNpmPackageChecker, RealIsBuiltInNodeModuleChecker, + CliNpmResolver, + CliSys, +>; +pub type CliNodeResolver = deno_runtime::deno_node::NodeResolver< + DenoInNpmPackageChecker, + CliNpmResolver, CliSys, >; -pub type CliNodeResolver = deno_runtime::deno_node::NodeResolver; pub type CliPackageJsonResolver = node_resolver::PackageJsonResolver; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -43,7 +51,7 @@ pub enum CliCjsAnalysis { pub struct CliCjsCodeAnalyzer { cache: NodeAnalysisCache, - cjs_tracker: Arc, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, parsed_source_cache: Option>, } @@ -51,7 +59,7 @@ pub struct CliCjsCodeAnalyzer { impl CliCjsCodeAnalyzer { pub fn new( cache: NodeAnalysisCache, - cjs_tracker: Arc, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, parsed_source_cache: Option>, ) -> Self { diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index bd29a6ec72..8dc498bb04 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -1,21 +1,12 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::path::Path; -use std::path::PathBuf; use std::sync::Arc; use deno_core::serde_json; -use deno_core::url::Url; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::ByonmNpmResolverCreateOptions; -use deno_resolver::npm::ByonmOrManagedNpmResolver; -use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; use deno_runtime::ops::process::NpmProcessStateProvider; -use deno_semver::package::PackageReq; -use node_resolver::NpmPackageFolderResolver; -use super::CliNpmResolver; -use super::InnerCliNpmResolverRef; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; use crate::sys::CliSys; @@ -24,67 +15,18 @@ pub type CliByonmNpmResolverCreateOptions = ByonmNpmResolverCreateOptions; pub type CliByonmNpmResolver = ByonmNpmResolver; -// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] -struct CliByonmWrapper(Arc); +pub struct CliByonmNpmProcessStateProvider(pub Arc); -impl NpmProcessStateProvider for CliByonmWrapper { +impl NpmProcessStateProvider for CliByonmNpmProcessStateProvider { fn get_npm_process_state(&self) -> String { serde_json::to_string(&NpmProcessState { kind: NpmProcessStateKind::Byonm, local_node_modules_path: self .0 - .root_node_modules_dir() + .root_node_modules_path() .map(|p| p.to_string_lossy().to_string()), }) .unwrap() } } - -impl CliNpmResolver for CliByonmNpmResolver { - fn into_npm_pkg_folder_resolver( - self: Arc, - ) -> Arc { - self - } - - fn into_process_state_provider( - self: Arc, - ) -> Arc { - Arc::new(CliByonmWrapper(self)) - } - - fn into_byonm_or_managed( - self: Arc, - ) -> ByonmOrManagedNpmResolver { - ByonmOrManagedNpmResolver::Byonm(self) - } - - fn clone_snapshotted(&self) -> Arc { - Arc::new(self.clone()) - } - - fn as_inner(&self) -> InnerCliNpmResolverRef { - InnerCliNpmResolverRef::Byonm(self) - } - - fn root_node_modules_path(&self) -> Option<&Path> { - self.root_node_modules_dir() - } - - fn check_state_hash(&self) -> Option { - // it is very difficult to determine the check state hash for byonm - // so we just return None to signify check caching is not supported - None - } - - fn resolve_pkg_folder_from_deno_module_req( - &self, - req: &PackageReq, - referrer: &Url, - ) -> Result { - self - .resolve_pkg_folder_from_deno_module_req(req, referrer) - .map_err(ResolvePkgFolderFromDenoReqError::Byonm) - } -} diff --git a/cli/npm/managed.rs b/cli/npm/managed.rs index 0d4c209acc..4122c881f1 100644 --- a/cli/npm/managed.rs +++ b/cli/npm/managed.rs @@ -4,44 +4,28 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_cache_dir::npm::NpmCacheDir; use deno_core::parking_lot::Mutex; use deno_core::serde_json; -use deno_core::url::Url; use deno_error::JsError; use deno_error::JsErrorBox; -use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmRegistryApi; use deno_npm::resolution::NpmResolutionSnapshot; -use deno_npm::resolution::PackageReqNotFoundError; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; -use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_npm::NpmSystemInfo; +use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions; use deno_resolver::npm::managed::NpmResolutionCell; -use deno_resolver::npm::managed::ResolvePkgFolderFromDenoModuleError; -use deno_resolver::npm::managed::ResolvePkgFolderFromPkgIdError; -use deno_resolver::npm::managed::ResolvePkgIdFromSpecifierError; -use deno_resolver::npm::ByonmOrManagedNpmResolver; -use deno_resolver::npm::ManagedNpmResolver; -use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; +use deno_resolver::npm::ManagedNpmResolverRc; use deno_runtime::ops::process::NpmProcessStateProvider; -use deno_semver::package::PackageNv; -use deno_semver::package::PackageReq; -use node_resolver::NpmPackageFolderResolver; -use sys_traits::FsMetadata; use thiserror::Error; use super::CliNpmRegistryInfoProvider; -use super::CliNpmResolver; -use super::InnerCliNpmResolverRef; use crate::args::CliLockfile; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; -use crate::cache::FastInsecureHasher; use crate::sys::CliSys; +pub type CliManagedNpmResolverCreateOptions = + ManagedNpmResolverCreateOptions; + #[derive(Debug, Clone)] pub enum CliNpmResolverManagedSnapshotOption { ResolveFromLockfile(Arc), @@ -139,36 +123,6 @@ impl NpmResolutionInitializer { } } -pub struct CliManagedNpmResolverCreateOptions { - pub npm_cache_dir: Arc, - pub sys: CliSys, - pub maybe_node_modules_path: Option, - pub npm_system_info: NpmSystemInfo, - pub npmrc: Arc, - pub npm_resolution: Arc, -} - -pub fn create_managed_npm_resolver( - options: CliManagedNpmResolverCreateOptions, -) -> Arc { - let managed_npm_resolver = - Arc::new(ManagedNpmResolver::::new::( - &options.npm_cache_dir, - &options.npmrc, - options.npm_resolution.clone(), - options.sys.clone(), - options.maybe_node_modules_path, - )); - Arc::new(ManagedCliNpmResolver::new( - managed_npm_resolver, - options.npm_cache_dir, - options.npmrc, - options.npm_resolution, - options.sys, - options.npm_system_info, - )) -} - #[derive(Debug, Error, Clone, JsError)] #[error("failed reading lockfile '{}'", lockfile_path.display())] #[class(inherit)] @@ -254,145 +208,6 @@ async fn snapshot_from_lockfile( Ok(snapshot) } -/// An npm resolver where the resolution is managed by Deno rather than -/// the user bringing their own node_modules (BYONM) on the file system. -pub struct ManagedCliNpmResolver { - managed_npm_resolver: Arc>, - npm_cache_dir: Arc, - npm_rc: Arc, - sys: CliSys, - resolution: Arc, - system_info: NpmSystemInfo, -} - -impl std::fmt::Debug for ManagedCliNpmResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ManagedCliNpmResolver") - .field("", &"") - .finish() - } -} - -impl ManagedCliNpmResolver { - #[allow(clippy::too_many_arguments)] - pub fn new( - managed_npm_resolver: Arc>, - npm_cache_dir: Arc, - npm_rc: Arc, - resolution: Arc, - sys: CliSys, - system_info: NpmSystemInfo, - ) -> Self { - Self { - managed_npm_resolver, - npm_cache_dir, - npm_rc, - resolution, - sys, - system_info, - } - } - - pub fn resolve_pkg_folder_from_pkg_id( - &self, - pkg_id: &NpmPackageId, - ) -> Result { - self - .managed_npm_resolver - .resolve_pkg_folder_from_pkg_id(pkg_id) - } - - /// Resolves the package id from the provided specifier. - pub fn resolve_pkg_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result, ResolvePkgIdFromSpecifierError> { - self - .managed_npm_resolver - .resolve_pkg_id_from_specifier(specifier) - } - - pub fn resolve_pkg_reqs_from_pkg_id( - &self, - id: &NpmPackageId, - ) -> Vec { - self.resolution.resolve_pkg_reqs_from_pkg_id(id) - } - - pub fn all_system_packages( - &self, - system_info: &NpmSystemInfo, - ) -> Vec { - self.resolution.all_system_packages(system_info) - } - - /// Checks if the provided package req's folder is cached. - pub fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool { - self - .resolve_pkg_id_from_pkg_req(req) - .ok() - .and_then(|id| { - self - .managed_npm_resolver - .resolve_pkg_folder_from_pkg_id(&id) - .ok() - }) - .map(|folder| self.sys.fs_exists_no_err(folder)) - .unwrap_or(false) - } - - pub fn snapshot(&self) -> NpmResolutionSnapshot { - self.resolution.snapshot() - } - - pub fn top_package_req_for_name(&self, name: &str) -> Option { - let package_reqs = self.resolution.package_reqs(); - let mut entries = package_reqs - .iter() - .filter(|(_, nv)| nv.name == name) - .collect::>(); - entries.sort_by_key(|(_, nv)| &nv.version); - Some(entries.last()?.0.clone()) - } - - pub fn serialized_valid_snapshot_for_system( - &self, - system_info: &NpmSystemInfo, - ) -> ValidSerializedNpmResolutionSnapshot { - self - .resolution - .serialized_valid_snapshot_for_system(system_info) - } - - pub fn resolve_pkg_folder_from_deno_module( - &self, - nv: &PackageNv, - ) -> Result { - self - .managed_npm_resolver - .resolve_pkg_folder_from_deno_module(nv) - } - - pub fn resolve_pkg_id_from_pkg_req( - &self, - req: &PackageReq, - ) -> Result { - self.resolution.resolve_pkg_id_from_pkg_req(req) - } - - pub fn maybe_node_modules_path(&self) -> Option<&Path> { - self.managed_npm_resolver.node_modules_path() - } - - pub fn global_cache_root_path(&self) -> &Path { - self.npm_cache_dir.root_dir() - } - - pub fn global_cache_root_url(&self) -> &Url { - self.npm_cache_dir.root_dir_url() - } -} - pub fn npm_process_state( snapshot: ValidSerializedNpmResolutionSnapshot, node_modules_path: Option<&Path>, @@ -405,92 +220,14 @@ pub fn npm_process_state( .unwrap() } -impl NpmProcessStateProvider for ManagedCliNpmResolver { +#[derive(Debug)] +pub struct CliManagedNpmProcessStateProvider(pub ManagedNpmResolverRc); + +impl NpmProcessStateProvider for CliManagedNpmProcessStateProvider { fn get_npm_process_state(&self) -> String { npm_process_state( - self.resolution.serialized_valid_snapshot(), - self.managed_npm_resolver.node_modules_path(), + self.0.resolution().serialized_valid_snapshot(), + self.0.root_node_modules_path(), ) } } - -impl CliNpmResolver for ManagedCliNpmResolver { - fn into_npm_pkg_folder_resolver( - self: Arc, - ) -> Arc { - self.managed_npm_resolver.clone() - } - - fn into_process_state_provider( - self: Arc, - ) -> Arc { - self - } - - fn into_byonm_or_managed( - self: Arc, - ) -> ByonmOrManagedNpmResolver { - ByonmOrManagedNpmResolver::Managed(self.managed_npm_resolver.clone()) - } - - fn clone_snapshotted(&self) -> Arc { - // create a new snapshotted npm resolution and resolver - let npm_resolution = - Arc::new(NpmResolutionCell::new(self.resolution.snapshot())); - - Arc::new(ManagedCliNpmResolver::new( - Arc::new(ManagedNpmResolver::::new::( - &self.npm_cache_dir, - &self.npm_rc, - npm_resolution.clone(), - self.sys.clone(), - self.root_node_modules_path().map(ToOwned::to_owned), - )), - self.npm_cache_dir.clone(), - self.npm_rc.clone(), - npm_resolution, - self.sys.clone(), - self.system_info.clone(), - )) - } - - fn as_inner(&self) -> InnerCliNpmResolverRef { - InnerCliNpmResolverRef::Managed(self) - } - - fn root_node_modules_path(&self) -> Option<&Path> { - self.managed_npm_resolver.node_modules_path() - } - - fn check_state_hash(&self) -> Option { - // We could go further and check all the individual - // npm packages, but that's probably overkill. - let mut package_reqs = self - .resolution - .package_reqs() - .into_iter() - .collect::>(); - package_reqs.sort_by(|a, b| a.0.cmp(&b.0)); // determinism - let mut hasher = FastInsecureHasher::new_without_deno_version(); - // ensure the cache gets busted when turning nodeModulesDir on or off - // as this could cause changes in resolution - hasher - .write_hashable(self.managed_npm_resolver.node_modules_path().is_some()); - for (pkg_req, pkg_nv) in package_reqs { - hasher.write_hashable(&pkg_req); - hasher.write_hashable(&pkg_nv); - } - Some(hasher.finish()) - } - - fn resolve_pkg_folder_from_deno_module_req( - &self, - req: &PackageReq, - referrer: &Url, - ) -> Result { - self - .managed_npm_resolver - .resolve_pkg_folder_from_deno_module_req(req, referrer) - .map_err(ResolvePkgFolderFromDenoReqError::Managed) - } -} diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 6ad7ad610e..1c12ce6c59 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -5,8 +5,6 @@ pub mod installer; mod managed; mod permission_checker; -use std::path::Path; -use std::path::PathBuf; use std::sync::Arc; use dashmap::DashMap; @@ -15,21 +13,15 @@ use deno_core::url::Url; use deno_error::JsErrorBox; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; -use deno_resolver::npm::ByonmNpmResolver; -use deno_resolver::npm::ByonmOrManagedNpmResolver; -use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; -use deno_runtime::ops::process::NpmProcessStateProvider; +use deno_runtime::ops::process::NpmProcessStateProviderRc; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use http::HeaderName; use http::HeaderValue; -use node_resolver::NpmPackageFolderResolver; -pub use self::byonm::CliByonmNpmResolver; pub use self::byonm::CliByonmNpmResolverCreateOptions; pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; -pub use self::managed::ManagedCliNpmResolver; pub use self::managed::NpmResolutionInitializer; pub use self::managed::ResolveSnapshotError; pub use self::permission_checker::NpmRegistryReadPermissionChecker; @@ -44,6 +36,10 @@ pub type CliNpmTarballCache = pub type CliNpmCache = deno_npm_cache::NpmCache; pub type CliNpmRegistryInfoProvider = deno_npm_cache::RegistryInfoProvider; +pub type CliNpmResolver = deno_resolver::npm::NpmResolver; +pub type CliManagedNpmResolver = deno_resolver::npm::ManagedNpmResolver; +pub type CliNpmResolverCreateOptions = + deno_resolver::npm::NpmResolverCreateOptions; #[derive(Debug)] pub struct CliNpmCacheHttpClient { @@ -63,6 +59,19 @@ impl CliNpmCacheHttpClient { } } +pub fn create_npm_process_state_provider( + npm_resolver: &CliNpmResolver, +) -> NpmProcessStateProviderRc { + match npm_resolver { + CliNpmResolver::Byonm(byonm_npm_resolver) => Arc::new( + byonm::CliByonmNpmProcessStateProvider(byonm_npm_resolver.clone()), + ), + CliNpmResolver::Managed(managed_npm_resolver) => Arc::new( + managed::CliManagedNpmProcessStateProvider(managed_npm_resolver.clone()), + ), + } +} + #[async_trait::async_trait(?Send)] impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient { async fn download_with_retries_on_any_tokio_runtime( @@ -104,70 +113,6 @@ impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient { } } -pub enum CliNpmResolverCreateOptions { - Managed(CliManagedNpmResolverCreateOptions), - Byonm(CliByonmNpmResolverCreateOptions), -} - -pub fn create_cli_npm_resolver( - options: CliNpmResolverCreateOptions, -) -> Arc { - use CliNpmResolverCreateOptions::*; - match options { - Managed(options) => managed::create_managed_npm_resolver(options), - Byonm(options) => Arc::new(ByonmNpmResolver::new(options)), - } -} - -pub enum InnerCliNpmResolverRef<'a> { - Managed(&'a ManagedCliNpmResolver), - #[allow(dead_code)] - Byonm(&'a CliByonmNpmResolver), -} - -// todo(dsherret): replace with an enum -pub trait CliNpmResolver: Send + Sync + std::fmt::Debug { - fn into_npm_pkg_folder_resolver( - self: Arc, - ) -> Arc; - fn into_process_state_provider( - self: Arc, - ) -> Arc; - fn into_byonm_or_managed( - self: Arc, - ) -> ByonmOrManagedNpmResolver; - - fn clone_snapshotted(&self) -> Arc; - - fn as_inner(&self) -> InnerCliNpmResolverRef; - - fn as_managed(&self) -> Option<&ManagedCliNpmResolver> { - match self.as_inner() { - InnerCliNpmResolverRef::Managed(inner) => Some(inner), - InnerCliNpmResolverRef::Byonm(_) => None, - } - } - - fn as_byonm(&self) -> Option<&CliByonmNpmResolver> { - match self.as_inner() { - InnerCliNpmResolverRef::Managed(_) => None, - InnerCliNpmResolverRef::Byonm(inner) => Some(inner), - } - } - - fn resolve_pkg_folder_from_deno_module_req( - &self, - req: &PackageReq, - referrer: &Url, - ) -> Result; - - fn root_node_modules_path(&self) -> Option<&Path>; - - /// Returns a hash returning the state of the npm resolver - /// or `None` if the state currently can't be determined. - fn check_state_hash(&self) -> Option; -} - #[derive(Debug)] pub struct NpmFetchResolver { nv_by_req: DashMap>, diff --git a/cli/resolver.rs b/cli/resolver.rs index 2f3d42e9e1..5677767fdd 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -19,6 +19,7 @@ use deno_graph::source::UnknownBuiltInNodeModuleError; use deno_graph::NpmLoadError; use deno_graph::NpmResolvePkgReqsResult; use deno_npm::resolution::NpmResolutionError; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::sloppy_imports::SloppyImportsCachedFs; use deno_resolver::sloppy_imports::SloppyImportsResolver; use deno_runtime::colors; @@ -35,22 +36,31 @@ use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::node::CliNodeCodeTranslator; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; +use crate::npm::CliNpmResolver; use crate::sys::CliSys; use crate::util::sync::AtomicFlag; use crate::util::text_encoding::from_utf8_lossy_cow; -pub type CjsTracker = deno_resolver::cjs::CjsTracker; -pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver; +pub type CliCjsTracker = + deno_resolver::cjs::CjsTracker; +pub type CliIsCjsResolver = + deno_resolver::cjs::IsCjsResolver; pub type CliSloppyImportsCachedFs = SloppyImportsCachedFs; pub type CliSloppyImportsResolver = SloppyImportsResolver; pub type CliDenoResolver = deno_resolver::DenoResolver< + DenoInNpmPackageChecker, RealIsBuiltInNodeModuleChecker, + CliNpmResolver, CliSloppyImportsCachedFs, CliSys, >; -pub type CliNpmReqResolver = - deno_resolver::npm::NpmReqResolver; +pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver< + DenoInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + CliNpmResolver, + CliSys, +>; pub struct ModuleCodeStringSource { pub code: ModuleSourceCode, @@ -69,14 +79,14 @@ pub struct NotSupportedKindInNpmError { // todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone) #[derive(Clone)] pub struct NpmModuleLoader { - cjs_tracker: Arc, + cjs_tracker: Arc, fs: Arc, node_code_translator: Arc, } impl NpmModuleLoader { pub fn new( - cjs_tracker: Arc, + cjs_tracker: Arc, fs: Arc, node_code_translator: Arc, ) -> Self { diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index ff0343e27f..c9b57f3d6b 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -92,8 +92,7 @@ use crate::emit::Emitter; use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; -use crate::npm::InnerCliNpmResolverRef; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; @@ -410,13 +409,13 @@ pub struct WriteBinOptions<'a> { } pub struct DenoCompileBinaryWriter<'a> { - cjs_tracker: &'a CjsTracker, + cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, - npm_resolver: &'a dyn CliNpmResolver, + npm_resolver: &'a CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, npm_system_info: NpmSystemInfo, } @@ -424,13 +423,13 @@ pub struct DenoCompileBinaryWriter<'a> { impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( - cjs_tracker: &'a CjsTracker, + cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, - npm_resolver: &'a dyn CliNpmResolver, + npm_resolver: &'a CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, npm_system_info: NpmSystemInfo, ) -> Self { @@ -599,10 +598,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => None, }; let mut vfs = VfsBuilder::new(); - let npm_snapshot = match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + let npm_snapshot = match &self.npm_resolver { + CliNpmResolver::Managed(managed) => { + let snapshot = managed + .resolution() + .serialized_valid_snapshot_for_system(&self.npm_system_info); if !snapshot.as_serialized().packages.is_empty() { self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; Some(snapshot) @@ -610,7 +610,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { None } } - InnerCliNpmResolverRef::Byonm(_) => { + CliNpmResolver::Byonm(_) => { self.fill_npm_vfs(&mut vfs)?; None } @@ -751,8 +751,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { ); } - let node_modules = match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(_) => { + let node_modules = match &self.npm_resolver { + CliNpmResolver::Managed(_) => { npm_snapshot.as_ref().map(|_| NodeModules::Managed { node_modules_dir: self.npm_resolver.root_node_modules_path().map( |path| { @@ -765,7 +765,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ), }) } - InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm { + CliNpmResolver::Byonm(resolver) => Some(NodeModules::Byonm { root_node_modules_dir: resolver.root_node_modules_path().map( |node_modules_dir| { root_dir_url @@ -880,16 +880,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(npm_resolver) => { + match &self.npm_resolver { + CliNpmResolver::Managed(npm_resolver) => { if let Some(node_modules_path) = npm_resolver.root_node_modules_path() { maybe_warn_different_system(&self.npm_system_info); builder.add_dir_recursive(node_modules_path)?; Ok(()) } else { // we'll flatten to remove any custom registries later - let mut packages = - npm_resolver.all_system_packages(&self.npm_system_info); + let mut packages = npm_resolver + .resolution() + .all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism for package in packages { let folder = @@ -899,7 +900,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { Ok(()) } } - InnerCliNpmResolverRef::Byonm(_) => { + CliNpmResolver::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; @@ -942,8 +943,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { &self, mut vfs: VfsBuilder, ) -> BuiltVfs { - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(npm_resolver) => { + match &self.npm_resolver { + CliNpmResolver::Managed(npm_resolver) => { if npm_resolver.root_node_modules_path().is_some() { return vfs.build(); } @@ -1035,7 +1036,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { .insert(npm_global_cache_dir_entry, case_sensitivity); built_vfs } - InnerCliNpmResolverRef::Byonm(_) => vfs.build(), + CliNpmResolver::Byonm(_) => vfs.build(), } } } diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 4ca82f6b7d..876c194ed1 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -40,10 +40,11 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_package_json::PackageJsonDepValue; use deno_resolver::cjs::IsCjsResolutionMode; -use deno_resolver::npm::create_in_npm_pkg_checker; use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; use deno_resolver::npm::managed::NpmResolutionCell; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_resolver::npm::CreateInNpmPkgCheckerOptions; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::npm::NpmReqResolverOptions; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; @@ -85,7 +86,6 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; -use crate::npm::create_cli_npm_resolver; use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; @@ -94,7 +94,7 @@ use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::NpmRegistryReadPermissionChecker; use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; use crate::resolver::NpmModuleLoader; use crate::sys::CliSys; @@ -122,7 +122,7 @@ use self::binary::Metadata; pub use self::file_system::DenoCompileFileSystem; struct SharedModuleLoaderState { - cjs_tracker: Arc, + cjs_tracker: Arc, code_cache: Option>, fs: Arc, modules: StandaloneModules, @@ -131,7 +131,7 @@ struct SharedModuleLoaderState { npm_module_loader: Arc, npm_registry_permission_checker: NpmRegistryReadPermissionChecker, npm_req_resolver: Arc, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, source_maps: SourceMapStore, vfs: Arc, workspace_resolver: WorkspaceResolver, @@ -734,7 +734,7 @@ pub async fn run( let maybe_node_modules_path = node_modules_dir .map(|node_modules_dir| root_path.join(node_modules_dir)); let in_npm_pkg_checker = - create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( ManagedInNpmPkgCheckerCreateOptions { root_cache_dir_url: npm_cache_dir.root_dir_url(), maybe_node_modules_path: maybe_node_modules_path.as_deref(), @@ -743,7 +743,7 @@ pub async fn run( let npm_resolution = Arc::new(NpmResolutionCell::new(NpmResolutionSnapshot::new(snapshot))); let npm_resolver = - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliNpmResolver::new(CliNpmResolverCreateOptions::Managed( CliManagedNpmResolverCreateOptions { npm_resolution, npm_cache_dir, @@ -761,8 +761,8 @@ pub async fn run( let root_node_modules_dir = root_node_modules_dir.map(|p| vfs.root().join(p)); let in_npm_pkg_checker = - create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm); - let npm_resolver = create_cli_npm_resolver( + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Byonm); + let npm_resolver = CliNpmResolver::new( CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { sys: sys.clone(), pkg_json_resolver: pkg_json_resolver.clone(), @@ -781,7 +781,7 @@ pub async fn run( npmrc.get_all_known_registries_urls(), )); let in_npm_pkg_checker = - create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( ManagedInNpmPkgCheckerCreateOptions { root_cache_dir_url: npm_cache_dir.root_dir_url(), maybe_node_modules_path: None, @@ -789,7 +789,7 @@ pub async fn run( )); let npm_resolution = Arc::new(NpmResolutionCell::default()); let npm_resolver = - create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( + CliNpmResolver::new(CliNpmResolverCreateOptions::Managed( CliManagedNpmResolverCreateOptions { npm_resolution, sys: sys.clone(), @@ -807,12 +807,12 @@ pub async fn run( let node_resolver = Arc::new(NodeResolver::new( in_npm_pkg_checker.clone(), RealIsBuiltInNodeModuleChecker, - npm_resolver.clone().into_npm_pkg_folder_resolver(), + npm_resolver.clone(), pkg_json_resolver.clone(), sys.clone(), node_resolver::ConditionsFromResolutionMode::default(), )); - let cjs_tracker = Arc::new(CjsTracker::new( + let cjs_tracker = Arc::new(CliCjsTracker::new( in_npm_pkg_checker.clone(), pkg_json_resolver.clone(), if metadata.unstable_config.detect_cjs { @@ -830,7 +830,7 @@ pub async fn run( sys: sys.clone(), in_npm_pkg_checker: in_npm_pkg_checker.clone(), node_resolver: node_resolver.clone(), - npm_resolver: npm_resolver.clone().into_byonm_or_managed(), + npm_resolver: npm_resolver.clone(), })); let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new( node_analysis_cache, @@ -842,7 +842,7 @@ pub async fn run( cjs_esm_code_analyzer, in_npm_pkg_checker, node_resolver.clone(), - npm_resolver.clone().into_npm_pkg_folder_resolver(), + npm_resolver.clone(), pkg_json_resolver.clone(), sys.clone(), )); diff --git a/cli/task_runner.rs b/cli/task_runner.rs index 8510a650e7..50dcd64dc7 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -25,9 +25,8 @@ use tokio::task::LocalSet; use tokio_util::sync::CancellationToken; use crate::node::CliNodeResolver; +use crate::npm::CliManagedNpmResolver; use crate::npm::CliNpmResolver; -use crate::npm::InnerCliNpmResolverRef; -use crate::npm::ManagedCliNpmResolver; pub fn get_script_with_args(script: &str, argv: &[String]) -> String { let additional_args = argv @@ -414,15 +413,15 @@ impl ShellCommand for NodeModulesFileRunCommand { } pub fn resolve_custom_commands( - npm_resolver: &dyn CliNpmResolver, + npm_resolver: &CliNpmResolver, node_resolver: &CliNodeResolver, ) -> Result>, AnyError> { - let mut commands = match npm_resolver.as_inner() { - InnerCliNpmResolverRef::Byonm(npm_resolver) => { + let mut commands = match npm_resolver { + CliNpmResolver::Byonm(npm_resolver) => { let node_modules_dir = npm_resolver.root_node_modules_path().unwrap(); resolve_npm_commands_from_bin_dir(node_modules_dir) } - InnerCliNpmResolverRef::Managed(npm_resolver) => { + CliNpmResolver::Managed(npm_resolver) => { resolve_managed_npm_commands(npm_resolver, node_resolver)? } }; @@ -521,13 +520,12 @@ fn resolve_execution_path_from_npx_shim( } fn resolve_managed_npm_commands( - npm_resolver: &ManagedCliNpmResolver, + npm_resolver: &CliManagedNpmResolver, node_resolver: &CliNodeResolver, ) -> Result>, AnyError> { let mut result = HashMap::new(); - let snapshot = npm_resolver.snapshot(); - for id in snapshot.top_level_packages() { - let package_folder = npm_resolver.resolve_pkg_folder_from_pkg_id(id)?; + for id in npm_resolver.resolution().top_level_packages() { + let package_folder = npm_resolver.resolve_pkg_folder_from_pkg_id(&id)?; let bin_commands = node_resolver.resolve_binary_commands(&package_folder)?; for bin_command in bin_commands { diff --git a/cli/tools/check.rs b/cli/tools/check.rs index c3a285a9b2..8c584113cb 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -112,7 +112,7 @@ pub struct TypeChecker { module_graph_builder: Arc, npm_installer: Option>, node_resolver: Arc, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, sys: CliSys, } @@ -146,7 +146,7 @@ impl TypeChecker { module_graph_builder: Arc, node_resolver: Arc, npm_installer: Option>, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, sys: CliSys, ) -> Self { Self { @@ -189,6 +189,29 @@ impl TypeChecker { mut graph: ModuleGraph, options: CheckOptions, ) -> Result<(Arc, Diagnostics), CheckError> { + fn check_state_hash(resolver: &CliNpmResolver) -> Option { + match resolver { + CliNpmResolver::Byonm(_) => { + // not feasible and probably slower to compute + None + } + CliNpmResolver::Managed(resolver) => { + // we should probably go further and check all the individual npm packages + let mut package_reqs = resolver.resolution().package_reqs(); + package_reqs.sort_by(|a, b| a.0.cmp(&b.0)); // determinism + let mut hasher = FastInsecureHasher::new_without_deno_version(); + // ensure the cache gets busted when turning nodeModulesDir on or off + // as this could cause changes in resolution + hasher.write_hashable(resolver.root_node_modules_path().is_some()); + for (pkg_req, pkg_nv) in package_reqs { + hasher.write_hashable(&pkg_req); + hasher.write_hashable(&pkg_nv); + } + Some(hasher.finish()) + } + } + } + if !options.type_check_mode.is_true() || graph.roots.is_empty() { return Ok((graph.into(), Default::default())); } @@ -240,7 +263,7 @@ impl TypeChecker { &self.sys, &graph, check_js, - self.npm_resolver.check_state_hash(), + check_state_hash(&self.npm_resolver), type_check_mode, &ts_config, ); diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 06afa5fac2..9b6ef81ea3 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -22,6 +22,7 @@ use deno_core::serde_json; use deno_core::sourcemap::SourceMap; use deno_core::url::Url; use deno_core::LocalInspectorSession; +use deno_resolver::npm::DenoInNpmPackageChecker; use node_resolver::InNpmPackageChecker; use regex::Regex; use text_lines::TextLines; @@ -464,7 +465,7 @@ fn filter_coverages( coverages: Vec, include: Vec, exclude: Vec, - in_npm_pkg_checker: &dyn InNpmPackageChecker, + in_npm_pkg_checker: &DenoInNpmPackageChecker, ) -> Vec { let include: Vec = include.iter().map(|e| Regex::new(e).unwrap()).collect(); @@ -532,7 +533,7 @@ pub fn cover_files( script_coverages, coverage_flags.include, coverage_flags.exclude, - in_npm_pkg_checker.as_ref(), + in_npm_pkg_checker, ); if script_coverages.is_empty() { return Err(anyhow!("No covered files included in the report")); diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 50faeda7d3..8c3b2665c5 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -32,8 +32,7 @@ use crate::args::InfoFlags; use crate::display; use crate::factory::CliFactory; use crate::graph_util::graph_exit_integrity_errors; -use crate::npm::CliNpmResolver; -use crate::npm::ManagedCliNpmResolver; +use crate::npm::CliManagedNpmResolver; use crate::util::checksum; use crate::util::display::DisplayTreeNode; @@ -138,6 +137,10 @@ pub async fn info( lockfile.write_if_changed()?; } + let maybe_npm_info = npm_resolver + .as_managed() + .map(|r| (r, r.resolution().snapshot())); + if info_flags.json { let mut json_graph = serde_json::json!(graph); if let Some(output) = json_graph.as_object_mut() { @@ -148,11 +151,19 @@ pub async fn info( ); } - add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref(), npmrc); + add_npm_packages_to_json( + &mut json_graph, + maybe_npm_info.as_ref().map(|(_, s)| s), + npmrc, + ); display::write_json_to_stdout(&json_graph)?; } else { let mut output = String::new(); - GraphDisplayContext::write(&graph, npm_resolver.as_ref(), &mut output)?; + GraphDisplayContext::write( + &graph, + maybe_npm_info.as_ref().map(|(r, s)| (*r, s)), + &mut output, + )?; display::write_to_stdout_ignore_sigpipe(output.as_bytes())?; } } else { @@ -251,15 +262,14 @@ fn print_cache_info( fn add_npm_packages_to_json( json: &mut serde_json::Value, - npm_resolver: &dyn CliNpmResolver, + npm_snapshot: Option<&NpmResolutionSnapshot>, npmrc: &ResolvedNpmRc, ) { - let Some(npm_resolver) = npm_resolver.as_managed() else { + let Some(npm_snapshot) = npm_snapshot else { return; // does not include byonm to deno info's output }; // ideally deno_graph could handle this, but for now we just modify the json here - let snapshot = npm_resolver.snapshot(); let json = json.as_object_mut().unwrap(); let modules = json.get_mut("modules").and_then(|m| m.as_array_mut()); if let Some(modules) = modules { @@ -273,7 +283,7 @@ fn add_npm_packages_to_json( .and_then(|k| k.as_str()) .and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok()) .and_then(|package_ref| { - snapshot + npm_snapshot .resolve_package_from_deno_module(package_ref.nv()) .ok() }); @@ -295,7 +305,8 @@ fn add_npm_packages_to_json( if let Some(specifier) = dep.get("specifier").and_then(|s| s.as_str()) { if let Ok(npm_ref) = NpmPackageReqReference::from_str(specifier) { - if let Ok(pkg) = snapshot.resolve_pkg_from_pkg_req(npm_ref.req()) + if let Ok(pkg) = + npm_snapshot.resolve_pkg_from_pkg_req(npm_ref.req()) { dep.insert( "npmPackage".to_string(), @@ -321,8 +332,9 @@ fn add_npm_packages_to_json( } } - let mut sorted_packages = - snapshot.all_packages_for_every_system().collect::>(); + let mut sorted_packages = npm_snapshot + .all_packages_for_every_system() + .collect::>(); sorted_packages.sort_by(|a, b| a.id.cmp(&b.id)); let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len()); for pkg in sorted_packages { @@ -356,7 +368,7 @@ struct NpmInfo { impl NpmInfo { pub fn build<'a>( graph: &'a ModuleGraph, - npm_resolver: &'a ManagedCliNpmResolver, + npm_resolver: &'a CliManagedNpmResolver, npm_snapshot: &'a NpmResolutionSnapshot, ) -> Self { let mut info = NpmInfo::default(); @@ -382,7 +394,7 @@ impl NpmInfo { fn fill_package_info<'a>( &mut self, package: &NpmResolutionPackage, - npm_resolver: &'a ManagedCliNpmResolver, + npm_resolver: &'a CliManagedNpmResolver, npm_snapshot: &'a NpmResolutionSnapshot, ) { self.packages.insert(package.id.clone(), package.clone()); @@ -419,13 +431,15 @@ struct GraphDisplayContext<'a> { impl<'a> GraphDisplayContext<'a> { pub fn write( graph: &'a ModuleGraph, - npm_resolver: &'a dyn CliNpmResolver, + managed_npm_info: Option<( + &'a CliManagedNpmResolver, + &'a NpmResolutionSnapshot, + )>, writer: &mut TWrite, ) -> Result<(), AnyError> { - let npm_info = match npm_resolver.as_managed() { - Some(npm_resolver) => { - let npm_snapshot = npm_resolver.snapshot(); - NpmInfo::build(graph, npm_resolver, &npm_snapshot) + let npm_info = match managed_npm_info { + Some((npm_resolver, npm_snapshot)) => { + NpmInfo::build(graph, npm_resolver, npm_snapshot) } None => NpmInfo::default(), }; diff --git a/cli/tools/registry/pm/deps.rs b/cli/tools/registry/pm/deps.rs index 1dbb464dbe..3d152490e8 100644 --- a/cli/tools/registry/pm/deps.rs +++ b/cli/tools/registry/pm/deps.rs @@ -451,7 +451,7 @@ pub struct DepManager { // TODO(nathanwhit): probably shouldn't be pub pub(crate) jsr_fetch_resolver: Arc, pub(crate) npm_fetch_resolver: Arc, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, npm_installer: Arc, permissions_container: PermissionsContainer, main_module_graph_container: Arc, @@ -463,7 +463,7 @@ pub struct DepManagerArgs { pub jsr_fetch_resolver: Arc, pub npm_fetch_resolver: Arc, pub npm_installer: Arc, - pub npm_resolver: Arc, + pub npm_resolver: CliNpmResolver, pub permissions_container: PermissionsContainer, pub main_module_graph_container: Arc, pub lockfile: Option>, @@ -551,9 +551,10 @@ impl DepManager { let npm_resolver = self.npm_resolver.as_managed().unwrap(); if self.deps.iter().all(|dep| match dep.kind { - DepKind::Npm => { - npm_resolver.resolve_pkg_id_from_pkg_req(&dep.req).is_ok() - } + DepKind::Npm => npm_resolver + .resolution() + .resolve_pkg_id_from_pkg_req(&dep.req) + .is_ok(), DepKind::Jsr => graph.packages.mappings().contains_key(&dep.req), }) { self.dependencies_resolved.raise(); @@ -630,7 +631,12 @@ impl DepManager { let graph = self.main_module_graph_container.graph(); let mut resolved = Vec::with_capacity(self.deps.len()); - let snapshot = self.npm_resolver.as_managed().unwrap().snapshot(); + let snapshot = self + .npm_resolver + .as_managed() + .unwrap() + .resolution() + .snapshot(); let resolved_npm = snapshot.package_reqs(); let resolved_jsr = graph.packages.mappings(); for dep in &self.deps { diff --git a/cli/tools/task.rs b/cli/tools/task.rs index d6d87dd45e..329195ab46 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -220,7 +220,7 @@ pub async fn execute_script( let task_runner = TaskRunner { task_flags: &task_flags, npm_installer: npm_installer.map(|n| n.as_ref()), - npm_resolver: npm_resolver.as_ref(), + npm_resolver, node_resolver: node_resolver.as_ref(), env_vars, cli_options, @@ -271,7 +271,7 @@ struct RunSingleOptions<'a> { struct TaskRunner<'a> { task_flags: &'a TaskFlags, npm_installer: Option<&'a NpmInstaller>, - npm_resolver: &'a dyn CliNpmResolver, + npm_resolver: &'a CliNpmResolver, node_resolver: &'a CliNodeResolver, env_vars: HashMap, cli_options: &'a CliOptions, diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index b6cf691c38..37d52c6cf2 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -46,7 +46,7 @@ use crate::cache::FastInsecureHasher; use crate::cache::ModuleInfoCache; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; -use crate::resolver::CjsTracker; +use crate::resolver::CliCjsTracker; use crate::sys::CliSys; use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; @@ -300,13 +300,13 @@ pub fn into_specifier_and_media_type( #[derive(Debug)] pub struct TypeCheckingCjsTracker { - cjs_tracker: Arc, + cjs_tracker: Arc, module_info_cache: Arc, } impl TypeCheckingCjsTracker { pub fn new( - cjs_tracker: Arc, + cjs_tracker: Arc, module_info_cache: Arc, ) -> Self { Self { @@ -358,7 +358,7 @@ impl TypeCheckingCjsTracker { pub struct RequestNpmState { pub cjs_tracker: Arc, pub node_resolver: Arc, - pub npm_resolver: Arc, + pub npm_resolver: CliNpmResolver, } /// A structure representing a request to be sent to the tsc runtime. diff --git a/cli/worker.rs b/cli/worker.rs index d93c6c5f40..45d4b0af70 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -19,6 +19,7 @@ use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; use deno_core::SharedArrayBufferStore; use deno_error::JsErrorBox; +use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::code_cache; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; use deno_runtime::deno_fs; @@ -151,7 +152,7 @@ struct SharedWorkerState { module_loader_factory: Box, node_resolver: Arc, npm_installer: Option>, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, pkg_json_resolver: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, @@ -168,18 +169,17 @@ impl SharedWorkerState { pub fn create_node_init_services( &self, node_require_loader: NodeRequireLoaderRc, - ) -> NodeExtInitServices { + ) -> NodeExtInitServices { NodeExtInitServices { node_require_loader, node_resolver: self.node_resolver.clone(), - npm_resolver: self.npm_resolver.clone().into_npm_pkg_folder_resolver(), pkg_json_resolver: self.pkg_json_resolver.clone(), sys: self.sys.clone(), } } pub fn npm_process_state_provider(&self) -> NpmProcessStateProviderRc { - self.npm_resolver.clone().into_process_state_provider() + crate::npm::create_npm_process_state_provider(&self.npm_resolver) } } @@ -427,7 +427,7 @@ impl CliMainWorkerFactory { module_loader_factory: Box, node_resolver: Arc, npm_installer: Option>, - npm_resolver: Arc, + npm_resolver: CliNpmResolver, pkg_json_resolver: Arc, root_cert_store_provider: Arc, root_permissions: PermissionsContainer, @@ -886,7 +886,11 @@ mod tests { ..Default::default() }; - MainWorker::bootstrap_from_options::( + MainWorker::bootstrap_from_options::< + DenoInNpmPackageChecker, + CliNpmResolver, + CliSys, + >( main_module, WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 731cd3a9c7..325cec6f5b 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -15,8 +15,9 @@ use deno_core::v8; use deno_core::v8::ExternalReference; use deno_error::JsErrorBox; use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::InNpmPackageChecker; use node_resolver::IsBuiltInNodeModuleChecker; -use node_resolver::NpmPackageFolderResolverRc; +use node_resolver::NpmPackageFolderResolver; use node_resolver::PackageJsonResolverRc; use once_cell::sync::Lazy; @@ -185,17 +186,21 @@ fn op_node_build_os() -> String { } #[derive(Clone)] -pub struct NodeExtInitServices { +pub struct NodeExtInitServices< + TInNpmPackageChecker: InNpmPackageChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TSys: ExtNodeSys, +> { pub node_require_loader: NodeRequireLoaderRc, - pub node_resolver: NodeResolverRc, - pub npm_resolver: NpmPackageFolderResolverRc, + pub node_resolver: + NodeResolverRc, pub pkg_json_resolver: PackageJsonResolverRc, pub sys: TSys, } deno_core::extension!(deno_node, deps = [ deno_io, deno_fs ], - parameters = [P: NodePermissions, TSys: ExtNodeSys], + parameters = [P: NodePermissions, TInNpmPackageChecker: InNpmPackageChecker, TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: ExtNodeSys], ops = [ ops::blocklist::op_socket_address_parse, ops::blocklist::op_socket_address_get_serialization, @@ -395,13 +400,13 @@ deno_core::extension!(deno_node, ops::require::op_require_init_paths, ops::require::op_require_node_module_paths, ops::require::op_require_proxy_path, - ops::require::op_require_is_deno_dir_package, - ops::require::op_require_resolve_deno_dir, + ops::require::op_require_is_deno_dir_package, + ops::require::op_require_resolve_deno_dir, ops::require::op_require_is_maybe_cjs, ops::require::op_require_is_request_relative, ops::require::op_require_resolve_lookup_paths, ops::require::op_require_try_self_parent_path, - ops::require::op_require_try_self, + ops::require::op_require_try_self, ops::require::op_require_real_path, ops::require::op_require_path_is_absolute, ops::require::op_require_path_dirname, @@ -410,9 +415,9 @@ deno_core::extension!(deno_node, ops::require::op_require_path_basename, ops::require::op_require_read_file

, ops::require::op_require_as_file_path, - ops::require::op_require_resolve_exports, + ops::require::op_require_resolve_exports, ops::require::op_require_read_package_scope, - ops::require::op_require_package_imports_resolve, + ops::require::op_require_package_imports_resolve, ops::require::op_require_break_on_next_statement, ops::util::op_node_guess_handle_type, ops::worker_threads::op_worker_threads_filename, @@ -681,7 +686,7 @@ deno_core::extension!(deno_node, "node:zlib" = "zlib.ts", ], options = { - maybe_init: Option>, + maybe_init: Option>, fs: deno_fs::FileSystemRc, }, state = |state, options| { @@ -691,7 +696,6 @@ deno_core::extension!(deno_node, state.put(init.sys.clone()); state.put(init.node_require_loader.clone()); state.put(init.node_resolver.clone()); - state.put(init.npm_resolver.clone()); state.put(init.pkg_json_resolver.clone()); } }, @@ -833,10 +837,18 @@ pub trait ExtNodeSys: impl ExtNodeSys for sys_traits::impls::RealSys {} -pub type NodeResolver = - node_resolver::NodeResolver; +pub type NodeResolver = + node_resolver::NodeResolver< + TInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >; #[allow(clippy::disallowed_types)] -pub type NodeResolverRc = deno_fs::sync::MaybeArc>; +pub type NodeResolverRc = + deno_fs::sync::MaybeArc< + NodeResolver, + >; #[allow(clippy::disallowed_types)] pub fn create_host_defined_options<'s>( diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 3135ba1e86..bff0cd79ed 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -19,7 +19,9 @@ use deno_path_util::normalize_path; use deno_path_util::url_from_file_path; use deno_path_util::url_to_file_path; use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionKind; +use node_resolver::NpmPackageFolderResolver; use node_resolver::ResolutionMode; use node_resolver::REQUIRE_CONDITIONS; use sys_traits::FsCanonicalize; @@ -30,7 +32,6 @@ use crate::ExtNodeSys; use crate::NodePermissions; use crate::NodeRequireLoaderRc; use crate::NodeResolverRc; -use crate::NpmPackageFolderResolverRc; use crate::PackageJsonResolverRc; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] @@ -256,12 +257,20 @@ pub fn op_require_is_request_relative(#[string] request: String) -> bool { #[op2] #[string] -pub fn op_require_resolve_deno_dir( +pub fn op_require_resolve_deno_dir< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TSys: ExtNodeSys + 'static, +>( state: &mut OpState, #[string] request: String, #[string] parent_filename: String, ) -> Result, deno_path_util::PathToUrlError> { - let resolver = state.borrow::(); + let resolver = state.borrow::>(); Ok( resolver @@ -275,11 +284,19 @@ pub fn op_require_resolve_deno_dir( } #[op2(fast)] -pub fn op_require_is_deno_dir_package( +pub fn op_require_is_deno_dir_package< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TSys: ExtNodeSys + 'static, +>( state: &mut OpState, #[string] path: String, ) -> bool { - let resolver = state.borrow::>(); + let resolver = state.borrow::>(); match deno_path_util::url_from_file_path(&PathBuf::from(path)) { Ok(specifier) => resolver.in_npm_package(&specifier), Err(_) => false, @@ -451,6 +468,8 @@ pub fn op_require_try_self_parent_path< #[string] pub fn op_require_try_self< P: NodePermissions + 'static, + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TSys: ExtNodeSys + 'static, >( state: &mut OpState, @@ -493,7 +512,11 @@ pub fn op_require_try_self< let referrer = deno_core::url::Url::from_file_path(&pkg.path).unwrap(); if let Some(exports) = &pkg.exports { - let node_resolver = state.borrow::>(); + let node_resolver = state.borrow::>(); let r = node_resolver.package_exports_resolve( &pkg.path, &expansion, @@ -552,6 +575,8 @@ pub fn op_require_as_file_path(#[string] file_or_url: String) -> String { #[string] pub fn op_require_resolve_exports< P: NodePermissions + 'static, + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TSys: ExtNodeSys + 'static, >( state: &mut OpState, @@ -563,7 +588,11 @@ pub fn op_require_resolve_exports< #[string] parent_path: String, ) -> Result, RequireError> { let sys = state.borrow::(); - let node_resolver = state.borrow::>(); + let node_resolver = state.borrow::>(); let pkg_json_resolver = state.borrow::>(); let modules_path = PathBuf::from(&modules_path_str); @@ -655,6 +684,8 @@ pub fn op_require_read_package_scope< #[string] pub fn op_require_package_imports_resolve< P: NodePermissions + 'static, + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TSys: ExtNodeSys + 'static, >( state: &mut OpState, @@ -672,7 +703,11 @@ pub fn op_require_package_imports_resolve< }; if pkg.imports.is_some() { - let node_resolver = state.borrow::>(); + let node_resolver = state.borrow::>(); let referrer_url = Url::from_file_path(&referrer_filename).unwrap(); let url = node_resolver.package_imports_resolve( &request, diff --git a/resolvers/deno/cjs.rs b/resolvers/deno/cjs.rs index bae645b481..4358b0ced2 100644 --- a/resolvers/deno/cjs.rs +++ b/resolvers/deno/cjs.rs @@ -2,7 +2,7 @@ use deno_media_type::MediaType; use node_resolver::errors::ClosestPkgJsonError; -use node_resolver::InNpmPackageCheckerRc; +use node_resolver::InNpmPackageChecker; use node_resolver::PackageJsonResolverRc; use node_resolver::ResolutionMode; use sys_traits::FsRead; @@ -16,14 +16,16 @@ use crate::sync::MaybeDashMap; /// be CJS or ESM after they're loaded based on their contents. So these /// files will be "maybe CJS" until they're loaded. #[derive(Debug)] -pub struct CjsTracker { - is_cjs_resolver: IsCjsResolver, +pub struct CjsTracker { + is_cjs_resolver: IsCjsResolver, known: MaybeDashMap, } -impl CjsTracker { +impl + CjsTracker +{ pub fn new( - in_npm_pkg_checker: InNpmPackageCheckerRc, + in_npm_pkg_checker: TInNpmPackageChecker, pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, ) -> Self { @@ -125,15 +127,20 @@ pub enum IsCjsResolutionMode { /// Resolves whether a module is CJS or ESM. #[derive(Debug)] -pub struct IsCjsResolver { - in_npm_pkg_checker: InNpmPackageCheckerRc, +pub struct IsCjsResolver< + TInNpmPackageChecker: InNpmPackageChecker, + TSys: FsRead, +> { + in_npm_pkg_checker: TInNpmPackageChecker, pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, } -impl IsCjsResolver { +impl + IsCjsResolver +{ pub fn new( - in_npm_pkg_checker: InNpmPackageCheckerRc, + in_npm_pkg_checker: TInNpmPackageChecker, pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, ) -> Self { diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 12d8effc46..454c7f12e2 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -19,11 +19,12 @@ use deno_package_json::PackageJsonDepValueParseError; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::NodeResolveError; use node_resolver::errors::PackageSubpathResolveError; -use node_resolver::InNpmPackageCheckerRc; +use node_resolver::InNpmPackageChecker; use node_resolver::IsBuiltInNodeModuleChecker; use node_resolver::NodeResolution; use node_resolver::NodeResolutionKind; use node_resolver::NodeResolverRc; +use node_resolver::NpmPackageFolderResolver; use node_resolver::ResolutionMode; use npm::MissingPackageNodeModulesFolderError; use npm::NodeModulesOutOfDateError; @@ -101,22 +102,42 @@ pub enum DenoResolveErrorKind { #[derive(Debug)] pub struct NodeAndNpmReqResolver< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { - pub node_resolver: NodeResolverRc, - pub npm_req_resolver: NpmReqResolverRc, + pub node_resolver: NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + pub npm_req_resolver: NpmReqResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, } pub struct DenoResolverOptions< 'a, + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSloppyImportResolverFs: SloppyImportResolverFs, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { - pub in_npm_pkg_checker: InNpmPackageCheckerRc, - pub node_and_req_resolver: - Option>, + pub in_npm_pkg_checker: TInNpmPackageChecker, + pub node_and_req_resolver: Option< + NodeAndNpmReqResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + >, pub sloppy_imports_resolver: Option>, pub workspace_resolver: WorkspaceResolverRc, @@ -131,13 +152,21 @@ pub struct DenoResolverOptions< /// import map, JSX settings. #[derive(Debug)] pub struct DenoResolver< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSloppyImportResolverFs: SloppyImportResolverFs, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { - in_npm_pkg_checker: InNpmPackageCheckerRc, - node_and_npm_resolver: - Option>, + in_npm_pkg_checker: TInNpmPackageChecker, + node_and_npm_resolver: Option< + NodeAndNpmReqResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + >, sloppy_imports_resolver: Option>, workspace_resolver: WorkspaceResolverRc, @@ -146,14 +175,25 @@ pub struct DenoResolver< } impl< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSloppyImportResolverFs: SloppyImportResolverFs, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, - > DenoResolver + > + DenoResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSloppyImportResolverFs, + TSys, + > { pub fn new( options: DenoResolverOptions< + TInNpmPackageChecker, TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, TSloppyImportResolverFs, TSys, >, diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index bf25ba5646..4f72692f8b 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -27,8 +27,6 @@ use thiserror::Error; use url::Url; use super::local::normalize_pkg_name_for_node_modules_deno_folder; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; #[derive(Debug, Error, deno_error::JsError)] pub enum ByonmResolvePkgFolderFromDenoReqError { @@ -89,7 +87,7 @@ impl } } - pub fn root_node_modules_dir(&self) -> Option<&Path> { + pub fn root_node_modules_path(&self) -> Option<&Path> { self.root_node_modules_dir.as_deref() } @@ -377,15 +375,8 @@ impl } } -impl< - TSys: FsCanonicalize - + FsMetadata - + FsRead - + FsReadDir - + MaybeSend - + MaybeSync - + std::fmt::Debug, - > NpmPackageFolderResolver for ByonmNpmResolver +impl + NpmPackageFolderResolver for ByonmNpmResolver { fn resolve_package_folder_from_package( &self, @@ -438,7 +429,7 @@ impl< } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ByonmInNpmPackageChecker; impl InNpmPackageChecker for ByonmInNpmPackageChecker { diff --git a/resolvers/deno/npm/managed/common.rs b/resolvers/deno/npm/managed/common.rs index 6569a4594f..a92fe226dd 100644 --- a/resolvers/deno/npm/managed/common.rs +++ b/resolvers/deno/npm/managed/common.rs @@ -6,26 +6,69 @@ use std::path::PathBuf; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use node_resolver::NpmPackageFolderResolver; +use sys_traits::FsCanonicalize; +use sys_traits::FsMetadata; use url::Url; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; +#[derive(Debug)] +pub enum NpmPackageFsResolver { + Local(super::local::LocalNpmPackageResolver), + Global(super::global::GlobalNpmPackageResolver), +} -#[allow(clippy::disallowed_types)] -pub type NpmPackageFsResolverRc = - crate::sync::MaybeArc; +impl NpmPackageFsResolver { + /// The local node_modules folder (only for the local resolver). + pub fn node_modules_path(&self) -> Option<&Path> { + match self { + NpmPackageFsResolver::Local(resolver) => resolver.node_modules_path(), + NpmPackageFsResolver::Global(_) => None, + } + } -/// Part of the resolution that interacts with the file system. -pub trait NpmPackageFsResolver: - NpmPackageFolderResolver + MaybeSend + MaybeSync -{ - /// The local node_modules folder if it is applicable to the implementation. - fn node_modules_path(&self) -> Option<&Path>; + pub fn maybe_package_folder( + &self, + package_id: &NpmPackageId, + ) -> Option { + match self { + NpmPackageFsResolver::Local(resolver) => { + resolver.maybe_package_folder(package_id) + } + NpmPackageFsResolver::Global(resolver) => { + resolver.maybe_package_folder(package_id) + } + } + } - fn maybe_package_folder(&self, package_id: &NpmPackageId) -> Option; - - fn resolve_package_cache_folder_id_from_specifier( + pub fn resolve_package_cache_folder_id_from_specifier( &self, specifier: &Url, - ) -> Result, std::io::Error>; + ) -> Result, std::io::Error> { + match self { + NpmPackageFsResolver::Local(resolver) => { + resolver.resolve_package_cache_folder_id_from_specifier(specifier) + } + NpmPackageFsResolver::Global(resolver) => { + resolver.resolve_package_cache_folder_id_from_specifier(specifier) + } + } + } +} + +impl NpmPackageFolderResolver + for NpmPackageFsResolver +{ + fn resolve_package_folder_from_package( + &self, + specifier: &str, + referrer: &Url, + ) -> Result { + match self { + NpmPackageFsResolver::Local(r) => { + r.resolve_package_folder_from_package(specifier, referrer) + } + NpmPackageFsResolver::Global(r) => { + r.resolve_package_folder_from_package(specifier, referrer) + } + } + } } diff --git a/resolvers/deno/npm/managed/global.rs b/resolvers/deno/npm/managed/global.rs index d16234d8f4..271851c79c 100644 --- a/resolvers/deno/npm/managed/global.rs +++ b/resolvers/deno/npm/managed/global.rs @@ -2,7 +2,6 @@ //! Code for global npm cache resolution. -use std::path::Path; use std::path::PathBuf; use deno_npm::NpmPackageCacheFolderId; @@ -18,7 +17,6 @@ use url::Url; use super::resolution::NpmResolutionCellRc; use super::NpmCacheDirRc; -use super::NpmPackageFsResolver; use crate::ResolvedNpmRcRc; /// Resolves packages from the global npm cache. @@ -42,6 +40,26 @@ impl GlobalNpmPackageResolver { } } + pub fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { + let folder_copy_index = self + .resolution + .resolve_pkg_cache_folder_copy_index_from_pkg_id(id)?; + let registry_url = self.npm_rc.get_registry_url(&id.nv.name); + Some(self.cache.package_folder_for_id( + &id.nv.name, + &id.nv.version.to_string(), + folder_copy_index, + registry_url, + )) + } + + pub fn resolve_package_cache_folder_id_from_specifier( + &self, + specifier: &Url, + ) -> Result, std::io::Error> { + Ok(self.resolve_package_cache_folder_id_from_specifier_inner(specifier)) + } + fn resolve_package_cache_folder_id_from_specifier_inner( &self, specifier: &Url, @@ -121,29 +139,3 @@ impl NpmPackageFolderResolver for GlobalNpmPackageResolver { } } } - -impl NpmPackageFsResolver for GlobalNpmPackageResolver { - fn node_modules_path(&self) -> Option<&Path> { - None - } - - fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { - let folder_copy_index = self - .resolution - .resolve_pkg_cache_folder_copy_index_from_pkg_id(id)?; - let registry_url = self.npm_rc.get_registry_url(&id.nv.name); - Some(self.cache.package_folder_for_id( - &id.nv.name, - &id.nv.version.to_string(), - folder_copy_index, - registry_url, - )) - } - - fn resolve_package_cache_folder_id_from_specifier( - &self, - specifier: &Url, - ) -> Result, std::io::Error> { - Ok(self.resolve_package_cache_folder_id_from_specifier_inner(specifier)) - } -} diff --git a/resolvers/deno/npm/managed/local.rs b/resolvers/deno/npm/managed/local.rs index 72b0b0d451..e453101df4 100644 --- a/resolvers/deno/npm/managed/local.rs +++ b/resolvers/deno/npm/managed/local.rs @@ -20,27 +20,20 @@ use sys_traits::FsMetadata; use url::Url; use super::resolution::NpmResolutionCellRc; -use super::NpmPackageFsResolver; use crate::npm::local::get_package_folder_id_folder_name_from_parts; use crate::npm::local::get_package_folder_id_from_folder_name; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; /// Resolver that creates a local node_modules directory /// and resolves packages from it. #[derive(Debug)] -pub struct LocalNpmPackageResolver< - TSys: FsCanonicalize + FsMetadata + MaybeSend + MaybeSync, -> { +pub struct LocalNpmPackageResolver { resolution: NpmResolutionCellRc, sys: TSys, root_node_modules_path: PathBuf, root_node_modules_url: Url, } -impl - LocalNpmPackageResolver -{ +impl LocalNpmPackageResolver { #[allow(clippy::too_many_arguments)] pub fn new( resolution: NpmResolutionCellRc, @@ -56,6 +49,55 @@ impl } } + pub fn node_modules_path(&self) -> Option<&Path> { + Some(self.root_node_modules_path.as_ref()) + } + + pub fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { + let folder_copy_index = self + .resolution + .resolve_pkg_cache_folder_copy_index_from_pkg_id(id)?; + // package is stored at: + // node_modules/.deno//node_modules/ + Some( + self + .root_node_modules_path + .join(".deno") + .join(get_package_folder_id_folder_name_from_parts( + &id.nv, + folder_copy_index, + )) + .join("node_modules") + .join(&id.nv.name), + ) + } + + pub fn resolve_package_cache_folder_id_from_specifier( + &self, + specifier: &Url, + ) -> Result, std::io::Error> { + let Some(folder_path) = + self.resolve_package_folder_from_specifier(specifier)? + else { + return Ok(None); + }; + // ex. project/node_modules/.deno/preact@10.24.3/node_modules/preact/ + let Some(node_modules_ancestor) = folder_path + .ancestors() + .find(|ancestor| ancestor.ends_with("node_modules")) + else { + return Ok(None); + }; + let Some(folder_name) = + node_modules_ancestor.parent().and_then(|p| p.file_name()) + else { + return Ok(None); + }; + Ok(get_package_folder_id_from_folder_name( + &folder_name.to_string_lossy(), + )) + } + fn resolve_package_root(&self, path: &Path) -> PathBuf { let mut last_found = path; loop { @@ -101,9 +143,8 @@ impl } } -impl< - TSys: FsCanonicalize + FsMetadata + MaybeSend + MaybeSync + std::fmt::Debug, - > NpmPackageFolderResolver for LocalNpmPackageResolver +impl NpmPackageFolderResolver + for LocalNpmPackageResolver { fn resolve_package_folder_from_package( &self, @@ -163,60 +204,6 @@ impl< } } -impl< - TSys: FsCanonicalize + FsMetadata + MaybeSend + MaybeSync + std::fmt::Debug, - > NpmPackageFsResolver for LocalNpmPackageResolver -{ - fn node_modules_path(&self) -> Option<&Path> { - Some(self.root_node_modules_path.as_ref()) - } - - fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { - let folder_copy_index = self - .resolution - .resolve_pkg_cache_folder_copy_index_from_pkg_id(id)?; - // package is stored at: - // node_modules/.deno//node_modules/ - Some( - self - .root_node_modules_path - .join(".deno") - .join(get_package_folder_id_folder_name_from_parts( - &id.nv, - folder_copy_index, - )) - .join("node_modules") - .join(&id.nv.name), - ) - } - - fn resolve_package_cache_folder_id_from_specifier( - &self, - specifier: &Url, - ) -> Result, std::io::Error> { - let Some(folder_path) = - self.resolve_package_folder_from_specifier(specifier)? - else { - return Ok(None); - }; - // ex. project/node_modules/.deno/preact@10.24.3/node_modules/preact/ - let Some(node_modules_ancestor) = folder_path - .ancestors() - .find(|ancestor| ancestor.ends_with("node_modules")) - else { - return Ok(None); - }; - let Some(folder_name) = - node_modules_ancestor.parent().and_then(|p| p.file_name()) - else { - return Ok(None); - }; - Ok(get_package_folder_id_from_folder_name( - &folder_name.to_string_lossy(), - )) - } -} - fn join_package_name(path: &Path, package_name: &str) -> PathBuf { let mut path = path.to_path_buf(); // ensure backslashes are used on windows diff --git a/resolvers/deno/npm/managed/mod.rs b/resolvers/deno/npm/managed/mod.rs index 62147b2ac3..291e299bc8 100644 --- a/resolvers/deno/npm/managed/mod.rs +++ b/resolvers/deno/npm/managed/mod.rs @@ -13,6 +13,7 @@ use deno_npm::resolution::PackageNvNotFoundError; use deno_npm::resolution::PackageReqNotFoundError; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; +use deno_npm::NpmSystemInfo; use deno_path_util::fs::canonicalize_path_maybe_not_exists; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; @@ -23,14 +24,10 @@ use sys_traits::FsMetadata; use url::Url; use self::common::NpmPackageFsResolver; -use self::common::NpmPackageFsResolverRc; use self::global::GlobalNpmPackageResolver; use self::local::LocalNpmPackageResolver; pub use self::resolution::NpmResolutionCell; pub use self::resolution::NpmResolutionCellRc; -use crate::sync::new_rc; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; use crate::NpmCacheDirRc; use crate::ResolvedNpmRcRc; @@ -89,58 +86,84 @@ pub enum ResolvePkgIdFromSpecifierError { NotFound(#[from] PackageCacheFolderIdNotFoundError), } +pub struct ManagedNpmResolverCreateOptions< + TSys: FsCanonicalize + FsMetadata + Clone, +> { + pub npm_cache_dir: NpmCacheDirRc, + pub sys: TSys, + pub maybe_node_modules_path: Option, + pub npm_system_info: NpmSystemInfo, + pub npmrc: ResolvedNpmRcRc, + pub npm_resolution: NpmResolutionCellRc, +} + #[allow(clippy::disallowed_types)] pub type ManagedNpmResolverRc = crate::sync::MaybeArc>; #[derive(Debug)] -pub struct ManagedNpmResolver { - fs_resolver: NpmPackageFsResolverRc, +pub struct ManagedNpmResolver { + fs_resolver: NpmPackageFsResolver, + npm_cache_dir: NpmCacheDirRc, resolution: NpmResolutionCellRc, sys: TSys, } -impl ManagedNpmResolver { - pub fn new< - TCreateSys: FsCanonicalize - + FsMetadata - + std::fmt::Debug - + MaybeSend - + MaybeSync - + Clone - + 'static, - >( - npm_cache_dir: &NpmCacheDirRc, - npm_rc: &ResolvedNpmRcRc, - resolution: NpmResolutionCellRc, - sys: TCreateSys, - maybe_node_modules_path: Option, +impl ManagedNpmResolver { + pub fn new( + options: ManagedNpmResolverCreateOptions, ) -> ManagedNpmResolver { - let fs_resolver: NpmPackageFsResolverRc = match maybe_node_modules_path { - Some(node_modules_folder) => new_rc(LocalNpmPackageResolver::new( - resolution.clone(), - sys.clone(), - node_modules_folder, - )), - None => new_rc(GlobalNpmPackageResolver::new( - npm_cache_dir.clone(), - npm_rc.clone(), - resolution.clone(), + let fs_resolver = match options.maybe_node_modules_path { + Some(node_modules_folder) => { + NpmPackageFsResolver::Local(LocalNpmPackageResolver::new( + options.npm_resolution.clone(), + options.sys.clone(), + node_modules_folder, + )) + } + None => NpmPackageFsResolver::Global(GlobalNpmPackageResolver::new( + options.npm_cache_dir.clone(), + options.npmrc.clone(), + options.npm_resolution.clone(), )), }; ManagedNpmResolver { fs_resolver, - sys, - resolution, + npm_cache_dir: options.npm_cache_dir, + sys: options.sys, + resolution: options.npm_resolution, } } #[inline] - pub fn node_modules_path(&self) -> Option<&Path> { + pub fn root_node_modules_path(&self) -> Option<&Path> { self.fs_resolver.node_modules_path() } + pub fn global_cache_root_path(&self) -> &Path { + self.npm_cache_dir.root_dir() + } + + pub fn global_cache_root_url(&self) -> &Url { + self.npm_cache_dir.root_dir_url() + } + + pub fn resolution(&self) -> &NpmResolutionCell { + self.resolution.as_ref() + } + + /// Checks if the provided package req's folder is cached. + pub fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool { + self + .resolution + .resolve_pkg_id_from_pkg_req(req) + .ok() + .and_then(|id| self.resolve_pkg_folder_from_pkg_id(&id).ok()) + .map(|folder| self.sys.fs_exists_no_err(folder)) + .unwrap_or(false) + } + pub fn resolve_pkg_folder_from_pkg_id( &self, package_id: &NpmPackageId, @@ -213,8 +236,8 @@ impl ManagedNpmResolver { } } -impl - NpmPackageFolderResolver for ManagedNpmResolver +impl NpmPackageFolderResolver + for ManagedNpmResolver { fn resolve_package_folder_from_package( &self, @@ -234,7 +257,7 @@ impl } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ManagedInNpmPackageChecker { root_dir: Url, } diff --git a/resolvers/deno/npm/managed/resolution.rs b/resolvers/deno/npm/managed/resolution.rs index 08dabf5a76..faa0d43664 100644 --- a/resolvers/deno/npm/managed/resolution.rs +++ b/resolvers/deno/npm/managed/resolution.rs @@ -1,7 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::collections::HashMap; - use deno_npm::resolution::NpmPackagesPartitioned; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::resolution::PackageCacheFolderIdNotFoundError; @@ -124,8 +122,23 @@ impl NpmResolutionCell { .map(|pkg| pkg.id.clone()) } - pub fn package_reqs(&self) -> HashMap { - self.snapshot.read().package_reqs().clone() + pub fn package_reqs(&self) -> Vec<(PackageReq, PackageNv)> { + self + .snapshot + .read() + .package_reqs() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + } + + pub fn top_level_packages(&self) -> Vec { + self + .snapshot + .read() + .top_level_packages() + .cloned() + .collect::>() } pub fn all_system_packages( diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs index 26b156d29d..fed3d388bf 100644 --- a/resolvers/deno/npm/mod.rs +++ b/resolvers/deno/npm/mod.rs @@ -1,6 +1,7 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::fmt::Debug; +use std::path::Path; use std::path::PathBuf; use boxed_error::Boxed; @@ -14,11 +15,12 @@ use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::PackageResolveErrorKind; use node_resolver::errors::PackageSubpathResolveError; -use node_resolver::InNpmPackageCheckerRc; +use node_resolver::InNpmPackageChecker; use node_resolver::IsBuiltInNodeModuleChecker; use node_resolver::NodeResolution; use node_resolver::NodeResolutionKind; use node_resolver::NodeResolverRc; +use node_resolver::NpmPackageFolderResolver; use node_resolver::ResolutionMode; use sys_traits::FsCanonicalize; use sys_traits::FsMetadata; @@ -35,10 +37,14 @@ pub use self::byonm::ByonmResolvePkgFolderFromDenoReqError; pub use self::local::get_package_folder_id_folder_name; pub use self::local::normalize_pkg_name_for_node_modules_deno_folder; use self::managed::create_managed_in_npm_pkg_checker; +use self::managed::ManagedInNpmPackageChecker; use self::managed::ManagedInNpmPkgCheckerCreateOptions; pub use self::managed::ManagedNpmResolver; +use self::managed::ManagedNpmResolverCreateOptions; pub use self::managed::ManagedNpmResolverRc; use crate::sync::new_rc; +use crate::sync::MaybeSend; +use crate::sync::MaybeSync; mod byonm; mod local; @@ -49,14 +55,33 @@ pub enum CreateInNpmPkgCheckerOptions<'a> { Byonm, } -pub fn create_in_npm_pkg_checker( - options: CreateInNpmPkgCheckerOptions, -) -> InNpmPackageCheckerRc { - match options { - CreateInNpmPkgCheckerOptions::Managed(options) => { - new_rc(create_managed_in_npm_pkg_checker(options)) +#[derive(Debug, Clone)] +pub enum DenoInNpmPackageChecker { + Managed(ManagedInNpmPackageChecker), + Byonm(ByonmInNpmPackageChecker), +} + +impl DenoInNpmPackageChecker { + pub fn new(options: CreateInNpmPkgCheckerOptions) -> Self { + match options { + CreateInNpmPkgCheckerOptions::Managed(options) => { + DenoInNpmPackageChecker::Managed(create_managed_in_npm_pkg_checker( + options, + )) + } + CreateInNpmPkgCheckerOptions::Byonm => { + DenoInNpmPackageChecker::Byonm(ByonmInNpmPackageChecker) + } + } + } +} + +impl InNpmPackageChecker for DenoInNpmPackageChecker { + fn in_npm_package(&self, specifier: &Url) -> bool { + match self { + DenoInNpmPackageChecker::Managed(c) => c.in_npm_package(specifier), + DenoInNpmPackageChecker::Byonm(c) => c.in_npm_package(specifier), } - CreateInNpmPkgCheckerOptions::Byonm => new_rc(ByonmInNpmPackageChecker), } } @@ -115,10 +140,22 @@ pub enum ResolvePkgFolderFromDenoReqError { Byonm(byonm::ByonmResolvePkgFolderFromDenoReqError), } -#[derive(Debug, Clone)] -pub enum ByonmOrManagedNpmResolver< - TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, +pub enum NpmResolverCreateOptions< + TSys: FsRead + + FsCanonicalize + + FsMetadata + + std::fmt::Debug + + MaybeSend + + MaybeSync + + Clone + + 'static, > { + Managed(ManagedNpmResolverCreateOptions), + Byonm(ByonmNpmResolverCreateOptions), +} + +#[derive(Debug, Clone)] +pub enum NpmResolver { /// The resolver when "bring your own node_modules" is enabled where Deno /// does not setup the node_modules directories automatically, but instead /// uses what already exists on the file system. @@ -126,57 +163,158 @@ pub enum ByonmOrManagedNpmResolver< Managed(ManagedNpmResolverRc), } -impl - ByonmOrManagedNpmResolver -{ +impl NpmResolver { + pub fn new< + TCreateSys: FsCanonicalize + + FsMetadata + + FsRead + + FsReadDir + + std::fmt::Debug + + MaybeSend + + MaybeSync + + Clone + + 'static, + >( + options: NpmResolverCreateOptions, + ) -> NpmResolver { + match options { + NpmResolverCreateOptions::Managed(options) => { + NpmResolver::Managed(new_rc(ManagedNpmResolver::::new::< + TCreateSys, + >(options))) + } + NpmResolverCreateOptions::Byonm(options) => { + NpmResolver::Byonm(new_rc(ByonmNpmResolver::new(options))) + } + } + } + + pub fn is_byonm(&self) -> bool { + matches!(self, NpmResolver::Byonm(_)) + } + + pub fn is_managed(&self) -> bool { + matches!(self, NpmResolver::Managed(_)) + } + + pub fn as_managed(&self) -> Option<&ManagedNpmResolver> { + match self { + NpmResolver::Managed(resolver) => Some(resolver), + NpmResolver::Byonm(_) => None, + } + } + + pub fn root_node_modules_path(&self) -> Option<&Path> { + match self { + NpmResolver::Byonm(resolver) => resolver.root_node_modules_path(), + NpmResolver::Managed(resolver) => resolver.root_node_modules_path(), + } + } + pub fn resolve_pkg_folder_from_deno_module_req( &self, req: &PackageReq, referrer: &Url, ) -> Result { match self { - ByonmOrManagedNpmResolver::Byonm(byonm_resolver) => byonm_resolver + NpmResolver::Byonm(byonm_resolver) => byonm_resolver .resolve_pkg_folder_from_deno_module_req(req, referrer) .map_err(ResolvePkgFolderFromDenoReqError::Byonm), - ByonmOrManagedNpmResolver::Managed(managed_resolver) => managed_resolver + NpmResolver::Managed(managed_resolver) => managed_resolver .resolve_pkg_folder_from_deno_module_req(req, referrer) .map_err(ResolvePkgFolderFromDenoReqError::Managed), } } } +impl + NpmPackageFolderResolver for NpmResolver +{ + fn resolve_package_folder_from_package( + &self, + specifier: &str, + referrer: &Url, + ) -> Result { + match self { + NpmResolver::Byonm(byonm_resolver) => { + byonm_resolver.resolve_package_folder_from_package(specifier, referrer) + } + NpmResolver::Managed(managed_resolver) => managed_resolver + .resolve_package_folder_from_package(specifier, referrer), + } + } +} + pub struct NpmReqResolverOptions< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { - pub in_npm_pkg_checker: InNpmPackageCheckerRc, - pub node_resolver: NodeResolverRc, - pub npm_resolver: ByonmOrManagedNpmResolver, + pub in_npm_pkg_checker: TInNpmPackageChecker, + pub node_resolver: NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + pub npm_resolver: NpmResolver, pub sys: TSys, } #[allow(clippy::disallowed_types)] -pub type NpmReqResolverRc = - crate::sync::MaybeArc>; +pub type NpmReqResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, +> = crate::sync::MaybeArc< + NpmReqResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, +>; #[derive(Debug)] pub struct NpmReqResolver< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { sys: TSys, - in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, - npm_resolver: ByonmOrManagedNpmResolver, + in_npm_pkg_checker: TInNpmPackageChecker, + node_resolver: NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + npm_resolver: NpmResolver, } impl< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, - > NpmReqResolver + > + NpmReqResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + > { pub fn new( - options: NpmReqResolverOptions, + options: NpmReqResolverOptions< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, ) -> Self { Self { sys: options.sys, @@ -224,7 +362,7 @@ impl< match resolution_result { Ok(url) => Ok(url), Err(err) => { - if matches!(self.npm_resolver, ByonmOrManagedNpmResolver::Byonm(_)) { + if matches!(self.npm_resolver, NpmResolver::Byonm(_)) { let package_json_path = package_folder.join("package.json"); if !self.sys.fs_exists_no_err(&package_json_path) { return Err( @@ -292,9 +430,8 @@ impl< .into_box(), ); } - if let ByonmOrManagedNpmResolver::Byonm( - byonm_npm_resolver, - ) = &self.npm_resolver + if let NpmResolver::Byonm(byonm_npm_resolver) = + &self.npm_resolver { if byonm_npm_resolver .find_ancestor_package_json_with_dep( diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index a6ba3927aa..e144e2b8fb 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -20,11 +20,11 @@ use sys_traits::FsMetadata; use sys_traits::FsRead; use url::Url; -use crate::npm::InNpmPackageCheckerRc; use crate::resolution::NodeResolverRc; +use crate::InNpmPackageChecker; use crate::IsBuiltInNodeModuleChecker; use crate::NodeResolutionKind; -use crate::NpmPackageFolderResolverRc; +use crate::NpmPackageFolderResolver; use crate::PackageJsonResolverRc; use crate::PathClean; use crate::ResolutionMode; @@ -62,28 +62,49 @@ pub trait CjsCodeAnalyzer { pub struct NodeCodeTranslator< TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead, > { cjs_code_analyzer: TCjsCodeAnalyzer, - in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, - npm_resolver: NpmPackageFolderResolverRc, + in_npm_pkg_checker: TInNpmPackageChecker, + node_resolver: NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + npm_resolver: TNpmPackageFolderResolver, pkg_json_resolver: PackageJsonResolverRc, sys: TSys, } impl< TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead, - > NodeCodeTranslator + > + NodeCodeTranslator< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + > { pub fn new( cjs_code_analyzer: TCjsCodeAnalyzer, - in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, - npm_resolver: NpmPackageFolderResolverRc, + in_npm_pkg_checker: TInNpmPackageChecker, + node_resolver: NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + npm_resolver: TNpmPackageFolderResolver, pkg_json_resolver: PackageJsonResolverRc, sys: TSys, ) -> Self { diff --git a/resolvers/node/lib.rs b/resolvers/node/lib.rs index faee4f1645..c0e6383237 100644 --- a/resolvers/node/lib.rs +++ b/resolvers/node/lib.rs @@ -14,9 +14,7 @@ mod sync; pub use deno_package_json::PackageJson; pub use npm::InNpmPackageChecker; -pub use npm::InNpmPackageCheckerRc; pub use npm::NpmPackageFolderResolver; -pub use npm::NpmPackageFolderResolverRc; pub use package_json::PackageJsonResolver; pub use package_json::PackageJsonResolverRc; pub use package_json::PackageJsonThreadLocalCache; diff --git a/resolvers/node/npm.rs b/resolvers/node/npm.rs index f799d6ddee..bb3de2a7f5 100644 --- a/resolvers/node/npm.rs +++ b/resolvers/node/npm.rs @@ -9,16 +9,8 @@ use url::Url; use crate::errors; use crate::path::PathClean; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; -#[allow(clippy::disallowed_types)] -pub type NpmPackageFolderResolverRc = - crate::sync::MaybeArc; - -pub trait NpmPackageFolderResolver: - std::fmt::Debug + MaybeSend + MaybeSync -{ +pub trait NpmPackageFolderResolver { /// Resolves an npm package folder path from the specified referrer. fn resolve_package_folder_from_package( &self, @@ -27,11 +19,8 @@ pub trait NpmPackageFolderResolver: ) -> Result; } -#[allow(clippy::disallowed_types)] -pub type InNpmPackageCheckerRc = crate::sync::MaybeArc; - /// Checks if a provided specifier is in an npm package. -pub trait InNpmPackageChecker: std::fmt::Debug + MaybeSend + MaybeSync { +pub trait InNpmPackageChecker { fn in_npm_package(&self, specifier: &Url) -> bool; fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index a60a6bec9e..9ea5e17e41 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -46,8 +46,8 @@ use crate::errors::TypesNotFoundError; use crate::errors::TypesNotFoundErrorData; use crate::errors::UnsupportedDirImportError; use crate::errors::UnsupportedEsmUrlSchemeError; -use crate::npm::InNpmPackageCheckerRc; -use crate::NpmPackageFolderResolverRc; +use crate::InNpmPackageChecker; +use crate::NpmPackageFolderResolver; use crate::PackageJsonResolverRc; use crate::PathClean; @@ -137,31 +137,52 @@ pub trait IsBuiltInNodeModuleChecker: std::fmt::Debug { } #[allow(clippy::disallowed_types)] -pub type NodeResolverRc = - crate::sync::MaybeArc>; +pub type NodeResolverRc< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, +> = crate::sync::MaybeArc< + NodeResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, +>; #[derive(Debug)] pub struct NodeResolver< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead, > { - in_npm_pkg_checker: InNpmPackageCheckerRc, + in_npm_pkg_checker: TInNpmPackageChecker, is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker, - npm_pkg_folder_resolver: NpmPackageFolderResolverRc, + npm_pkg_folder_resolver: TNpmPackageFolderResolver, pkg_json_resolver: PackageJsonResolverRc, sys: TSys, conditions_from_resolution_mode: ConditionsFromResolutionMode, } impl< + TInNpmPackageChecker: InNpmPackageChecker, TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, TSys: FsCanonicalize + FsMetadata + FsRead, - > NodeResolver + > + NodeResolver< + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + > { pub fn new( - in_npm_pkg_checker: InNpmPackageCheckerRc, + in_npm_pkg_checker: TInNpmPackageChecker, is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker, - npm_pkg_folder_resolver: NpmPackageFolderResolverRc, + npm_pkg_folder_resolver: TNpmPackageFolderResolver, pkg_json_resolver: PackageJsonResolverRc, sys: TSys, conditions_from_resolution_mode: ConditionsFromResolutionMode, @@ -444,6 +465,17 @@ impl< Ok(url) } + /// Resolves an npm package folder path from the specified referrer. + pub fn resolve_package_folder_from_package( + &self, + specifier: &str, + referrer: &Url, + ) -> Result { + self + .npm_pkg_folder_resolver + .resolve_package_folder_from_package(specifier, referrer) + } + /// Checks if the resolved file has a corresponding declaration file. fn path_to_declaration_url( &self, diff --git a/resolvers/node/sync.rs b/resolvers/node/sync.rs index 8cf06932ac..218253b453 100644 --- a/resolvers/node/sync.rs +++ b/resolvers/node/sync.rs @@ -6,17 +6,10 @@ pub use inner::*; mod inner { #![allow(clippy::disallowed_types)] - pub use core::marker::Send as MaybeSend; - pub use core::marker::Sync as MaybeSync; pub use std::sync::Arc as MaybeArc; } #[cfg(not(feature = "sync"))] mod inner { pub use std::rc::Rc as MaybeArc; - - pub trait MaybeSync {} - impl MaybeSync for T where T: ?Sized {} - pub trait MaybeSend {} - impl MaybeSend for T where T: ?Sized {} } diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 5ecd911e8a..f59325962f 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -91,6 +91,7 @@ deno_net.workspace = true deno_node.workspace = true deno_path_util.workspace = true deno_permissions.workspace = true +deno_resolver.workspace = true deno_telemetry.workspace = true deno_terminal.workspace = true deno_tls.workspace = true diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index 9f0eb63b01..e1538b8b75 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -12,6 +12,8 @@ use deno_core::op2; use deno_core::FsModuleLoader; use deno_core::ModuleSpecifier; use deno_fs::RealFs; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmResolver; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::worker::MainWorker; @@ -42,7 +44,11 @@ async fn main() -> Result<(), AnyError> { ); let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), - WorkerServiceOptions:: { + WorkerServiceOptions::< + DenoInNpmPackageChecker, + NpmResolver, + sys_traits::impls::RealSys, + > { module_loader: Rc::new(FsModuleLoader), permissions: PermissionsContainer::allow_all(permission_desc_parser), blob_store: Default::default(), diff --git a/runtime/snapshot.rs b/runtime/snapshot.rs index 8d008eeab0..331369551c 100644 --- a/runtime/snapshot.rs +++ b/runtime/snapshot.rs @@ -14,6 +14,8 @@ use deno_core::Extension; use deno_http::DefaultHttpPropertyExtractor; use deno_io::fs::FsError; use deno_permissions::PermissionCheckError; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmResolver; use crate::ops; use crate::ops::bootstrap::SnapshotOptions; @@ -310,6 +312,8 @@ pub fn create_runtime_snapshot( deno_fs::deno_fs::init_ops_and_esm::(fs.clone()), deno_node::deno_node::init_ops_and_esm::< Permissions, + DenoInNpmPackageChecker, + NpmResolver, sys_traits::impls::RealSys, >(None, fs.clone()), runtime::init_ops_and_esm(), diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 64393bb64c..4389c7752c 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -54,6 +54,8 @@ use deno_web::JsMessageData; use deno_web::MessagePort; use deno_web::Transferable; use log::debug; +use node_resolver::InNpmPackageChecker; +use node_resolver::NpmPackageFolderResolver; use crate::inspector_server::InspectorServer; use crate::ops; @@ -334,7 +336,11 @@ fn create_handles( (internal_handle, external_handle) } -pub struct WebWorkerServiceOptions { +pub struct WebWorkerServiceOptions< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TExtNodeSys: ExtNodeSys + 'static, +> { pub blob_store: Arc, pub broadcast_channel: InMemoryBroadcastChannel, pub compiled_wasm_module_store: Option, @@ -342,7 +348,13 @@ pub struct WebWorkerServiceOptions { pub fs: Arc, pub maybe_inspector_server: Option>, pub module_loader: Rc, - pub node_services: Option>, + pub node_services: Option< + NodeExtInitServices< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, + >, pub npm_process_state_provider: Option, pub permissions: PermissionsContainer, pub root_cert_store_provider: Option>, @@ -398,8 +410,16 @@ impl Drop for WebWorker { } impl WebWorker { - pub fn bootstrap_from_options( - services: WebWorkerServiceOptions, + pub fn bootstrap_from_options< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TExtNodeSys: ExtNodeSys + 'static, + >( + services: WebWorkerServiceOptions< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, options: WebWorkerOptions, ) -> (Self, SendableWebWorkerHandle) { let (mut worker, handle, bootstrap_options) = @@ -408,8 +428,16 @@ impl WebWorker { (worker, handle) } - fn from_options( - services: WebWorkerServiceOptions, + fn from_options< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TExtNodeSys: ExtNodeSys + 'static, + >( + services: WebWorkerServiceOptions< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, mut options: WebWorkerOptions, ) -> (Self, SendableWebWorkerHandle, BootstrapOptions) { deno_core::extension!(deno_permissions_web_worker, @@ -500,10 +528,12 @@ impl WebWorker { deno_fs::deno_fs::init_ops_and_esm::( services.fs.clone(), ), - deno_node::deno_node::init_ops_and_esm::( - services.node_services, - services.fs, - ), + deno_node::deno_node::init_ops_and_esm::< + PermissionsContainer, + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >(services.node_services, services.fs), // Runtime ops that are always initialized for WebWorkers ops::runtime::deno_runtime::init_ops_and_esm(options.main_module.clone()), ops::worker_host::deno_worker_host::init_ops_and_esm( diff --git a/runtime/worker.rs b/runtime/worker.rs index a649c83d47..7cd6468fc5 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -45,6 +45,8 @@ use deno_tls::RootCertStoreProvider; use deno_tls::TlsKeys; use deno_web::BlobStore; use log::debug; +use node_resolver::InNpmPackageChecker; +use node_resolver::NpmPackageFolderResolver; use crate::code_cache::CodeCache; use crate::code_cache::CodeCacheType; @@ -128,7 +130,11 @@ pub struct MainWorker { dispatch_process_exit_event_fn_global: v8::Global, } -pub struct WorkerServiceOptions { +pub struct WorkerServiceOptions< + TInNpmPackageChecker: InNpmPackageChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TExtNodeSys: ExtNodeSys, +> { pub blob_store: Arc, pub broadcast_channel: InMemoryBroadcastChannel, pub feature_checker: Arc, @@ -139,7 +145,13 @@ pub struct WorkerServiceOptions { /// If not provided runtime will error if code being /// executed tries to load modules. pub module_loader: Rc, - pub node_services: Option>, + pub node_services: Option< + NodeExtInitServices< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, + >, pub npm_process_state_provider: Option, pub permissions: PermissionsContainer, pub root_cert_store_provider: Option>, @@ -300,9 +312,17 @@ pub fn create_op_metrics( } impl MainWorker { - pub fn bootstrap_from_options( + pub fn bootstrap_from_options< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TExtNodeSys: ExtNodeSys + 'static, + >( main_module: ModuleSpecifier, - services: WorkerServiceOptions, + services: WorkerServiceOptions< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, options: WorkerOptions, ) -> Self { let (mut worker, bootstrap_options) = @@ -311,9 +331,17 @@ impl MainWorker { worker } - fn from_options( + fn from_options< + TInNpmPackageChecker: InNpmPackageChecker + 'static, + TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, + TExtNodeSys: ExtNodeSys + 'static, + >( main_module: ModuleSpecifier, - services: WorkerServiceOptions, + services: WorkerServiceOptions< + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >, mut options: WorkerOptions, ) -> (Self, BootstrapOptions) { deno_core::extension!(deno_permissions_worker, @@ -413,10 +441,12 @@ impl MainWorker { deno_fs::deno_fs::init_ops_and_esm::( services.fs.clone(), ), - deno_node::deno_node::init_ops_and_esm::( - services.node_services, - services.fs, - ), + deno_node::deno_node::init_ops_and_esm::< + PermissionsContainer, + TInNpmPackageChecker, + TNpmPackageFolderResolver, + TExtNodeSys, + >(services.node_services, services.fs), // Ops from this crate ops::runtime::deno_runtime::init_ops_and_esm(main_module.clone()), ops::worker_host::deno_worker_host::init_ops_and_esm( From c943f56949d723ba891b26a0cc4aaaaf7b358d95 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Wed, 15 Jan 2025 01:00:55 +0900 Subject: [PATCH 2/8] fix(ext/node): fix playwright http client (#27662) --- ext/node/polyfills/_tls_wrap.ts | 2 +- ext/node/polyfills/net.ts | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index 0edea1c053..dc7dac77ac 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -154,7 +154,7 @@ export class TLSSocket extends net.Socket { const afterConnect = handle.afterConnect; handle.afterConnect = async (req: any, status: number) => { options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined - if (tlssock._isNpmAgent) { + if (tlssock._needsSockInitWorkaround) { // skips the TLS handshake for @npmcli/agent as it's handled by // onSocket handler of ClientRequest object. tlssock.emit("secure"); diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index bebdd8dade..2d57507c1b 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -1157,6 +1157,13 @@ function _emitCloseNT(s: Socket | Server) { s.emit("close"); } +// The packages that need socket initialization workaround +const pkgsNeedsSockInitWorkaround = [ + "@npmcli/agent", + "npm-check-updates", + "playwright-core", +]; + /** * This class is an abstraction of a TCP socket or a streaming `IPC` endpoint * (uses named pipes on Windows, and Unix domain sockets otherwise). It is also @@ -1201,9 +1208,11 @@ export class Socket extends Duplex { _host: string | null = null; // deno-lint-ignore no-explicit-any _parent: any = null; - // The flag for detecting if it's called in @npmcli/agent + // Skip some initialization (initial read and tls handshake if it's tls socket). + // If this flag is true, it's used as connection for http(s) request, and + // the reading and TLS handshake is done by the http client. // See discussions in https://github.com/denoland/deno/pull/25470 for more details. - _isNpmAgent = false; + _needsSockInitWorkaround = false; autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined; constructor(options: SocketOptions | number) { @@ -1224,21 +1233,20 @@ export class Socket extends Duplex { super(options); - // Note: If the socket is created from one of: - // - @npmcli/agent - // - npm-check-updates (bundles @npmcli/agent as a dependency) + // Note: If the socket is created from one of `pkgNeedsSockInitWorkaround`, // the 'socket' event on ClientRequest object happens after 'connect' event on Socket object. // That swaps the sequence of op_node_http_request_with_conn() call and // initial socket read. That causes op_node_http_request_with_conn() not // working. // To avoid the above situation, we detect the socket created from - // @npmcli/agent and pause the socket (and also skips the startTls call - // if it's TLSSocket) + // one of those packages using stack trace and pause the socket + // (and also skips the startTls call if it's TLSSocket) // TODO(kt3k): Remove this workaround const errorStack = new Error().stack; - this._isNpmAgent = errorStack?.includes("@npmcli/agent") || - errorStack?.includes("npm-check-updates") || false; - if (this._isNpmAgent) { + this._needsSockInitWorkaround = pkgsNeedsSockInitWorkaround.some((pkg) => + errorStack?.includes(pkg) + ); + if (this._needsSockInitWorkaround) { this.pause(); } From 974e2f44b2660a1cd13f8c901ff28a2f57ed391f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 14 Jan 2025 16:29:36 +0000 Subject: [PATCH 3/8] refactor: add 'deno_os' crate (#27655) This commit creates "deno_os" extension crate and moves numerous ops from "runtime/" crate to the new crate. --- Cargo.lock | 25 +++++++++-- Cargo.toml | 1 + cli/js/40_bench.js | 2 +- cli/js/40_test.js | 2 +- cli/task_runner.rs | 2 +- cli/worker.rs | 2 +- ext/node/polyfills/os.ts | 2 +- ext/node/polyfills/process.ts | 2 +- {runtime/js => ext/os}/30_os.js | 0 {runtime/js => ext/os}/40_signals.js | 0 ext/os/Cargo.toml | 33 ++++++++++++++ {runtime/ops => ext}/os/README.md | 4 +- runtime/ops/os/mod.rs => ext/os/lib.rs | 61 +++++++++++++++++++++++--- ext/os/ops/mod.rs | 3 ++ {runtime => ext/os}/ops/signal.rs | 19 ++------ {runtime => ext/os}/signal.rs | 0 {runtime => ext/os}/sys_info.rs | 0 runtime/Cargo.toml | 5 +-- runtime/js/90_deno_ns.js | 4 +- runtime/js/99_main.js | 2 +- runtime/lib.rs | 10 +---- runtime/ops/mod.rs | 2 - runtime/ops/process.rs | 9 ++-- runtime/shared.rs | 2 - runtime/snapshot.rs | 3 +- runtime/web_worker.rs | 3 +- runtime/worker.rs | 21 ++------- tests/integration/run_tests.rs | 2 +- tools/core_import_map.json | 4 +- 29 files changed, 145 insertions(+), 80 deletions(-) rename {runtime/js => ext/os}/30_os.js (100%) rename {runtime/js => ext/os}/40_signals.js (100%) create mode 100644 ext/os/Cargo.toml rename {runtime/ops => ext}/os/README.md (97%) rename runtime/ops/os/mod.rs => ext/os/lib.rs (90%) create mode 100644 ext/os/ops/mod.rs rename {runtime => ext/os}/ops/signal.rs (95%) rename {runtime => ext/os}/signal.rs (100%) rename {runtime => ext/os}/sys_info.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 95633e3ce9..aadd0c5f95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2140,6 +2140,27 @@ dependencies = [ "thiserror 2.0.3", ] +[[package]] +name = "deno_os" +version = "0.1.0" +dependencies = [ + "deno_core", + "deno_error", + "deno_path_util", + "deno_permissions", + "deno_telemetry", + "libc", + "netif", + "ntapi", + "once_cell", + "serde", + "signal-hook", + "signal-hook-registry", + "thiserror 2.0.3", + "tokio", + "winapi", +] + [[package]] name = "deno_package_json" version = "0.4.0" @@ -2240,6 +2261,7 @@ dependencies = [ "deno_napi", "deno_net", "deno_node", + "deno_os", "deno_path_util", "deno_permissions", "deno_resolver", @@ -2263,7 +2285,6 @@ dependencies = [ "hyper-util", "libc", "log", - "netif", "nix", "node_resolver", "notify", @@ -2274,8 +2295,6 @@ dependencies = [ "rustyline", "same-file", "serde", - "signal-hook", - "signal-hook-registry", "sys_traits", "tempfile", "test_server", diff --git a/Cargo.toml b/Cargo.toml index 48aeccdaef..d9f976d863 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ deno_kv = { version = "0.93.0", path = "./ext/kv" } deno_napi = { version = "0.116.0", path = "./ext/napi" } deno_net = { version = "0.177.0", path = "./ext/net" } deno_node = { version = "0.123.0", path = "./ext/node" } +deno_os = { version = "0.1.0", path = "./ext/os" } deno_telemetry = { version = "0.7.0", path = "./ext/telemetry" } deno_tls = { version = "0.172.0", path = "./ext/tls" } deno_url = { version = "0.185.0", path = "./ext/url" } diff --git a/cli/js/40_bench.js b/cli/js/40_bench.js index 0ad05c5197..fb0e86463d 100644 --- a/cli/js/40_bench.js +++ b/cli/js/40_bench.js @@ -8,7 +8,7 @@ import { restorePermissions, } from "ext:cli/40_test_common.js"; import { Console } from "ext:deno_console/01_console.js"; -import { setExitHandler } from "ext:runtime/30_os.js"; +import { setExitHandler } from "ext:deno_os/30_os.js"; const { op_register_bench, op_bench_get_origin, diff --git a/cli/js/40_test.js b/cli/js/40_test.js index 34d9ec6550..3dbb7ec340 100644 --- a/cli/js/40_test.js +++ b/cli/js/40_test.js @@ -26,7 +26,7 @@ const { TypeError, } = primordials; -import { setExitHandler } from "ext:runtime/30_os.js"; +import { setExitHandler } from "ext:deno_os/30_os.js"; // Capture `Deno` global so that users deleting or mangling it, won't // have impact on our sanitizers. diff --git a/cli/task_runner.rs b/cli/task_runner.rs index 50dcd64dc7..14e850ee76 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -596,7 +596,7 @@ async fn listen_ctrl_c(kill_signal: KillSignal) { #[cfg(unix)] async fn listen_and_forward_all_signals(kill_signal: KillSignal) { use deno_core::futures::FutureExt; - use deno_runtime::signal::SIGNAL_NUMS; + use deno_runtime::deno_os::signal::SIGNAL_NUMS; // listen and forward every signal we support let mut futures = Vec::with_capacity(SIGNAL_NUMS.len()); diff --git a/cli/worker.rs b/cli/worker.rs index 45d4b0af70..d9cdbd3fb0 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -855,7 +855,7 @@ fn create_web_worker_callback( /// Instead probe for the total memory on the system and use it instead /// as a default. pub fn create_isolate_create_params() -> Option { - let maybe_mem_info = deno_runtime::sys_info::mem_info(); + let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info(); maybe_mem_info.map(|mem_info| { v8::CreateParams::default() .heap_limits_from_system_memory(mem_info.total, 0) diff --git a/ext/node/polyfills/os.ts b/ext/node/polyfills/os.ts index bd4b285ba3..4901b20fa8 100644 --- a/ext/node/polyfills/os.ts +++ b/ext/node/polyfills/os.ts @@ -35,7 +35,7 @@ import { validateIntegerRange } from "ext:deno_node/_utils.ts"; import process from "node:process"; import { isWindows } from "ext:deno_node/_util/os.ts"; import { os } from "ext:deno_node/internal_binding/constants.ts"; -import { osUptime } from "ext:runtime/30_os.js"; +import { osUptime } from "ext:deno_os/30_os.js"; import { Buffer } from "ext:deno_node/internal/buffer.mjs"; import { primordials } from "ext:core/mod.js"; const { StringPrototypeEndsWith, StringPrototypeSlice } = primordials; diff --git a/ext/node/polyfills/process.ts b/ext/node/polyfills/process.ts index e91a2ee005..b3fe0883dc 100644 --- a/ext/node/polyfills/process.ts +++ b/ext/node/polyfills/process.ts @@ -58,7 +58,7 @@ import { } from "ext:deno_node/_next_tick.ts"; import { isWindows } from "ext:deno_node/_util/os.ts"; import * as io from "ext:deno_io/12_io.js"; -import * as denoOs from "ext:runtime/30_os.js"; +import * as denoOs from "ext:deno_os/30_os.js"; export let argv0 = ""; diff --git a/runtime/js/30_os.js b/ext/os/30_os.js similarity index 100% rename from runtime/js/30_os.js rename to ext/os/30_os.js diff --git a/runtime/js/40_signals.js b/ext/os/40_signals.js similarity index 100% rename from runtime/js/40_signals.js rename to ext/os/40_signals.js diff --git a/ext/os/Cargo.toml b/ext/os/Cargo.toml new file mode 100644 index 0000000000..fb553d1532 --- /dev/null +++ b/ext/os/Cargo.toml @@ -0,0 +1,33 @@ +# Copyright 2018-2025 the Deno authors. MIT license. + +[package] +name = "deno_os" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "OS specific APIs for Deno" + +[lib] +path = "lib.rs" + +[dependencies] +deno_core.workspace = true +deno_error.workspace = true +deno_path_util.workspace = true +deno_permissions.workspace = true +deno_telemetry.workspace = true +libc.workspace = true +netif = "0.1.6" +once_cell.workspace = true +serde.workspace = true +signal-hook = "0.3.17" +signal-hook-registry = "1.4.0" +thiserror.workspace = true +tokio.workspace = true + +[target.'cfg(windows)'.dependencies] +winapi = { workspace = true, features = ["commapi", "knownfolders", "mswsock", "objbase", "psapi", "shlobj", "tlhelp32", "winbase", "winerror", "winuser", "winsock2"] } +ntapi = "0.4.0" diff --git a/runtime/ops/os/README.md b/ext/os/README.md similarity index 97% rename from runtime/ops/os/README.md rename to ext/os/README.md index ae1a5958e4..f308ed3d2b 100644 --- a/runtime/ops/os/README.md +++ b/ext/os/README.md @@ -1,4 +1,6 @@ -## `os` ops +# deno_os + +This crate implements OS specific APIs for Deno `loadavg` diff --git a/runtime/ops/os/mod.rs b/ext/os/lib.rs similarity index 90% rename from runtime/ops/os/mod.rs rename to ext/os/lib.rs index ad3a292c3a..f084601678 100644 --- a/runtime/ops/os/mod.rs +++ b/ext/os/lib.rs @@ -1,19 +1,54 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::collections::HashMap; +use std::collections::HashSet; use std::env; +use std::sync::atomic::AtomicI32; +use std::sync::atomic::Ordering; +use std::sync::Arc; use deno_core::op2; use deno_core::v8; use deno_core::OpState; -use deno_node::NODE_ENV_VAR_ALLOWLIST; use deno_path_util::normalize_path; use deno_permissions::PermissionCheckError; use deno_permissions::PermissionsContainer; +use once_cell::sync::Lazy; use serde::Serialize; -use crate::sys_info; -use crate::worker::ExitCode; +mod ops; +pub mod signal; +pub mod sys_info; + +pub use ops::signal::SignalError; + +pub static NODE_ENV_VAR_ALLOWLIST: Lazy> = Lazy::new(|| { + // The full list of environment variables supported by Node.js is available + // at https://nodejs.org/api/cli.html#environment-variables + let mut set = HashSet::new(); + set.insert("NODE_DEBUG".to_string()); + set.insert("NODE_OPTIONS".to_string()); + set +}); + +#[derive(Clone, Default)] +pub struct ExitCode(Arc); + +impl ExitCode { + pub fn get(&self) -> i32 { + self.0.load(Ordering::Relaxed) + } + + pub fn set(&mut self, code: i32) { + self.0.store(code, Ordering::Relaxed); + } +} + +pub fn exit(code: i32) -> ! { + deno_telemetry::flush(); + #[allow(clippy::disallowed_methods)] + std::process::exit(code); +} deno_core::extension!( deno_os, @@ -35,13 +70,21 @@ deno_core::extension!( op_system_memory_info, op_uid, op_runtime_memory_usage, + ops::signal::op_signal_bind, + ops::signal::op_signal_unbind, + ops::signal::op_signal_poll, ], + esm = ["30_os.js", "40_signals.js"], options = { exit_code: ExitCode, }, state = |state, options| { state.put::(options.exit_code); - }, + #[cfg(unix)] + { + state.put(ops::signal::SignalState::default()); + } + } ); deno_core::extension!( @@ -64,12 +107,16 @@ deno_core::extension!( op_system_memory_info, op_uid, op_runtime_memory_usage, + ops::signal::op_signal_bind, + ops::signal::op_signal_unbind, + ops::signal::op_signal_poll, ], + esm = ["30_os.js", "40_signals.js"], middleware = |op| match op.name { "op_exit" | "op_set_exit_code" | "op_get_exit_code" => op.with_implementation_from(&deno_core::op_void_sync()), _ => op, - }, + } ); #[derive(Debug, thiserror::Error, deno_error::JsError)] @@ -196,7 +243,7 @@ fn op_get_exit_code(state: &mut OpState) -> i32 { #[op2(fast)] fn op_exit(state: &mut OpState) { let code = state.borrow::().get(); - crate::exit(code) + exit(code) } #[op2(stack_trace)] @@ -239,7 +286,7 @@ fn op_network_interfaces( Ok(netif::up()?.map(NetworkInterface::from).collect()) } -#[derive(serde::Serialize)] +#[derive(Serialize)] struct NetworkInterface { family: &'static str, name: String, diff --git a/ext/os/ops/mod.rs b/ext/os/ops/mod.rs new file mode 100644 index 0000000000..857d4688c7 --- /dev/null +++ b/ext/os/ops/mod.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod signal; diff --git a/runtime/ops/signal.rs b/ext/os/ops/signal.rs similarity index 95% rename from runtime/ops/signal.rs rename to ext/os/ops/signal.rs index beefa1cd78..5b556e3a2c 100644 --- a/runtime/ops/signal.rs +++ b/ext/os/ops/signal.rs @@ -33,17 +33,6 @@ use tokio::signal::windows::CtrlBreak; #[cfg(windows)] use tokio::signal::windows::CtrlC; -deno_core::extension!( - deno_signal, - ops = [op_signal_bind, op_signal_unbind, op_signal_poll], - state = |state| { - #[cfg(unix)] - { - state.put(SignalState::default()); - } - } -); - #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum SignalError { #[class(type)] @@ -62,7 +51,7 @@ pub enum SignalError { #[cfg(unix)] #[derive(Default)] -struct SignalState { +pub struct SignalState { enable_default_handlers: BTreeMap>, } @@ -164,7 +153,7 @@ impl Resource for SignalStreamResource { #[cfg(unix)] #[op2(fast)] #[smi] -fn op_signal_bind( +pub fn op_signal_bind( state: &mut OpState, #[string] sig: &str, ) -> Result { @@ -201,7 +190,7 @@ fn op_signal_bind( #[cfg(windows)] #[op2(fast)] #[smi] -fn op_signal_bind( +pub fn op_signal_bind( state: &mut OpState, #[string] sig: &str, ) -> Result { @@ -225,7 +214,7 @@ fn op_signal_bind( } #[op2(async)] -async fn op_signal_poll( +pub async fn op_signal_poll( state: Rc>, #[smi] rid: ResourceId, ) -> Result { diff --git a/runtime/signal.rs b/ext/os/signal.rs similarity index 100% rename from runtime/signal.rs rename to ext/os/signal.rs diff --git a/runtime/sys_info.rs b/ext/os/sys_info.rs similarity index 100% rename from runtime/sys_info.rs rename to ext/os/sys_info.rs diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index f59325962f..808f79ab7b 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -49,6 +49,7 @@ deno_core.workspace = true deno_cron.workspace = true deno_crypto.workspace = true deno_fetch.workspace = true +deno_os.workspace = true deno_ffi.workspace = true deno_fs = { workspace = true, features = ["sync_fs"] } deno_http.workspace = true @@ -89,6 +90,7 @@ deno_kv.workspace = true deno_napi.workspace = true deno_net.workspace = true deno_node.workspace = true +deno_os.workspace = true deno_path_util.workspace = true deno_permissions.workspace = true deno_resolver.workspace = true @@ -114,7 +116,6 @@ hyper-util.workspace = true hyper_v014 = { workspace = true, features = ["server", "stream", "http1", "http2", "runtime"] } libc.workspace = true log.workspace = true -netif = "0.1.6" notify.workspace = true once_cell.workspace = true percent-encoding.workspace = true @@ -122,8 +123,6 @@ regex.workspace = true rustyline = { workspace = true, features = ["custom-bindings"] } same-file = "1.0.6" serde.workspace = true -signal-hook = "0.3.17" -signal-hook-registry = "1.4.0" sys_traits.workspace = true tempfile.workspace = true thiserror.workspace = true diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index 8127ac9618..b241028077 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -21,10 +21,10 @@ import * as version from "ext:runtime/01_version.ts"; import * as permissions from "ext:runtime/10_permissions.js"; import * as io from "ext:deno_io/12_io.js"; import * as fs from "ext:deno_fs/30_fs.js"; -import * as os from "ext:runtime/30_os.js"; +import * as os from "ext:deno_os/30_os.js"; import * as fsEvents from "ext:runtime/40_fs_events.js"; import * as process from "ext:runtime/40_process.js"; -import * as signals from "ext:runtime/40_signals.js"; +import * as signals from "ext:deno_os/40_signals.js"; import * as tty from "ext:runtime/40_tty.js"; import * as kv from "ext:deno_kv/01_db.ts"; import * as cron from "ext:deno_cron/01_cron.ts"; diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 03c662bcab..2971fd2c00 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -55,7 +55,7 @@ import { registerDeclarativeServer } from "ext:deno_http/00_serve.ts"; import * as event from "ext:deno_web/02_event.js"; import * as location from "ext:deno_web/12_location.js"; import * as version from "ext:runtime/01_version.ts"; -import * as os from "ext:runtime/30_os.js"; +import * as os from "ext:deno_os/30_os.js"; import * as timers from "ext:deno_web/02_timers.js"; import { getDefaultInspectOptions, diff --git a/runtime/lib.rs b/runtime/lib.rs index 9afe91cd25..c104f5cd61 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -16,6 +16,7 @@ pub use deno_kv; pub use deno_napi; pub use deno_net; pub use deno_node; +pub use deno_os; pub use deno_permissions; pub use deno_terminal::colors; pub use deno_tls; @@ -33,9 +34,7 @@ pub mod inspector_server; pub mod js; pub mod ops; pub mod permissions; -pub mod signal; pub mod snapshot; -pub mod sys_info; pub mod tokio_util; pub mod web_worker; pub mod worker; @@ -46,6 +45,7 @@ pub use worker_bootstrap::WorkerExecutionMode; pub use worker_bootstrap::WorkerLogLevel; pub mod shared; +pub use deno_os::exit; pub use shared::runtime; pub struct UnstableGranularFlag { @@ -147,12 +147,6 @@ pub static UNSTABLE_GRANULAR_FLAGS: &[UnstableGranularFlag] = &[ }, ]; -pub fn exit(code: i32) -> ! { - deno_telemetry::flush(); - #[allow(clippy::disallowed_methods)] - std::process::exit(code); -} - #[cfg(test)] mod test { use super::*; diff --git a/runtime/ops/mod.rs b/runtime/ops/mod.rs index 438c3896e6..d131e9aab5 100644 --- a/runtime/ops/mod.rs +++ b/runtime/ops/mod.rs @@ -3,11 +3,9 @@ pub mod bootstrap; pub mod fs_events; pub mod http; -pub mod os; pub mod permissions; pub mod process; pub mod runtime; -pub mod signal; pub mod tty; pub mod web_worker; pub mod worker_host; diff --git a/runtime/ops/process.rs b/runtime/ops/process.rs index 4682085aea..fc32f7e066 100644 --- a/runtime/ops/process.rs +++ b/runtime/ops/process.rs @@ -31,14 +31,13 @@ use deno_io::ChildStderrResource; use deno_io::ChildStdinResource; use deno_io::ChildStdoutResource; use deno_io::IntoRawIoHandle; +use deno_os::SignalError; use deno_permissions::PermissionsContainer; use deno_permissions::RunQueryDescriptor; use serde::Deserialize; use serde::Serialize; use tokio::process::Command; -use crate::ops::signal::SignalError; - pub const UNSTABLE_FEATURE_NAME: &str = "process"; #[derive(Copy, Clone, Eq, PartialEq, Deserialize)] @@ -296,7 +295,7 @@ impl TryFrom for ChildStatus { success: false, code: 128 + signal, #[cfg(unix)] - signal: Some(crate::signal::signal_int_to_str(signal)?.to_string()), + signal: Some(deno_os::signal::signal_int_to_str(signal)?.to_string()), #[cfg(not(unix))] signal: None, } @@ -1116,7 +1115,7 @@ mod deprecated { #[cfg(unix)] pub fn kill(pid: i32, signal: &str) -> Result<(), ProcessError> { - let signo = crate::signal::signal_str_to_int(signal) + let signo = deno_os::signal::signal_str_to_int(signal) .map_err(SignalError::InvalidSignalStr)?; use nix::sys::signal::kill as unix_kill; use nix::sys::signal::Signal; @@ -1144,7 +1143,7 @@ mod deprecated { if !matches!(signal, "SIGKILL" | "SIGTERM") { Err( - SignalError::InvalidSignalStr(crate::signal::InvalidSignalStrError( + SignalError::InvalidSignalStr(deno_os::signal::InvalidSignalStrError( signal.to_string(), )) .into(), diff --git a/runtime/shared.rs b/runtime/shared.rs index f8588dd72c..0e747b0565 100644 --- a/runtime/shared.rs +++ b/runtime/shared.rs @@ -42,10 +42,8 @@ extension!(runtime, "06_util.js", "10_permissions.js", "11_workers.js", - "30_os.js", "40_fs_events.js", "40_process.js", - "40_signals.js", "40_tty.js", "41_prompt.js", "90_deno_ns.js", diff --git a/runtime/snapshot.rs b/runtime/snapshot.rs index 331369551c..08ea17a986 100644 --- a/runtime/snapshot.rs +++ b/runtime/snapshot.rs @@ -310,6 +310,7 @@ pub fn create_runtime_snapshot( ), deno_io::deno_io::init_ops_and_esm(Default::default()), deno_fs::deno_fs::init_ops_and_esm::(fs.clone()), + deno_os::deno_os::init_ops_and_esm(Default::default()), deno_node::deno_node::init_ops_and_esm::< Permissions, DenoInNpmPackageChecker, @@ -323,10 +324,8 @@ pub fn create_runtime_snapshot( None, ), ops::fs_events::deno_fs_events::init_ops(), - ops::os::deno_os::init_ops(Default::default()), ops::permissions::deno_permissions::init_ops(), ops::process::deno_process::init_ops(None), - ops::signal::deno_signal::init_ops(), ops::tty::deno_tty::init_ops(), ops::http::deno_http_runtime::init_ops(), ops::bootstrap::deno_bootstrap::init_ops(Some(snapshot_options)), diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 4389c7752c..e4ea42a2f7 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -528,6 +528,7 @@ impl WebWorker { deno_fs::deno_fs::init_ops_and_esm::( services.fs.clone(), ), + deno_os::deno_os_worker::init_ops_and_esm(), deno_node::deno_node::init_ops_and_esm::< PermissionsContainer, TInNpmPackageChecker, @@ -541,12 +542,10 @@ impl WebWorker { options.format_js_error_fn, ), ops::fs_events::deno_fs_events::init_ops_and_esm(), - ops::os::deno_os_worker::init_ops_and_esm(), ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, ), - ops::signal::deno_signal::init_ops_and_esm(), ops::tty::deno_tty::init_ops_and_esm(), ops::http::deno_http_runtime::init_ops_and_esm(), ops::bootstrap::deno_bootstrap::init_ops_and_esm( diff --git a/runtime/worker.rs b/runtime/worker.rs index 7cd6468fc5..cfcadccc46 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -3,8 +3,6 @@ use std::borrow::Cow; use std::collections::HashMap; use std::rc::Rc; use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicI32; -use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -40,6 +38,7 @@ use deno_io::Stdio; use deno_kv::dynamic::MultiBackendDbHandler; use deno_node::ExtNodeSys; use deno_node::NodeExtInitServices; +use deno_os::ExitCode; use deno_permissions::PermissionsContainer; use deno_tls::RootCertStoreProvider; use deno_tls::TlsKeys; @@ -97,19 +96,6 @@ pub fn validate_import_attributes_callback( } } -#[derive(Clone, Default)] -pub struct ExitCode(Arc); - -impl ExitCode { - pub fn get(&self) -> i32 { - self.0.load(Relaxed) - } - - pub fn set(&mut self, code: i32) { - self.0.store(code, Relaxed); - } -} - /// This worker is created and used by almost all /// subcommands in Deno executable. /// @@ -363,7 +349,7 @@ impl MainWorker { // Permissions: many ops depend on this let enable_testing_features = options.bootstrap.enable_testing_features; - let exit_code = ExitCode(Arc::new(AtomicI32::new(0))); + let exit_code = ExitCode::default(); let create_cache = options.cache_storage_dir.map(|storage_dir| { let create_cache_fn = move || SqliteBackedCache::new(storage_dir.clone()); CreateCache(Arc::new(create_cache_fn)) @@ -441,6 +427,7 @@ impl MainWorker { deno_fs::deno_fs::init_ops_and_esm::( services.fs.clone(), ), + deno_os::deno_os::init_ops_and_esm(exit_code.clone()), deno_node::deno_node::init_ops_and_esm::< PermissionsContainer, TInNpmPackageChecker, @@ -454,12 +441,10 @@ impl MainWorker { options.format_js_error_fn.clone(), ), ops::fs_events::deno_fs_events::init_ops_and_esm(), - ops::os::deno_os::init_ops_and_esm(exit_code.clone()), ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, ), - ops::signal::deno_signal::init_ops_and_esm(), ops::tty::deno_tty::init_ops_and_esm(), ops::http::deno_http_runtime::init_ops_and_esm(), ops::bootstrap::deno_bootstrap::init_ops_and_esm( diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index 5e4db3d3e4..abf1c200d9 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -445,7 +445,7 @@ fn permissions_trace() { test_util::assertions::assert_wildcard_match(&text, concat!( "┏ ⚠️ Deno requests sys access to \"hostname\".\r\n", "┠─ Requested by `Deno.hostname()` API.\r\n", - "┃ ├─ Object.hostname (ext:runtime/30_os.js:43:10)\r\n", + "┃ ├─ Object.hostname (ext:deno_os/30_os.js:43:10)\r\n", "┃ ├─ foo (file://[WILDCARD]/run/permissions_trace.ts:2:8)\r\n", "┃ ├─ bar (file://[WILDCARD]/run/permissions_trace.ts:6:3)\r\n", "┃ └─ file://[WILDCARD]/run/permissions_trace.ts:9:1\r\n", diff --git a/tools/core_import_map.json b/tools/core_import_map.json index feeae1ac70..4843884015 100644 --- a/tools/core_import_map.json +++ b/tools/core_import_map.json @@ -239,10 +239,10 @@ "ext:runtime/06_util.js": "../runtime/js/06_util.js", "ext:runtime/10_permissions.js": "../runtime/js/10_permissions.js", "ext:runtime/11_workers.js": "../runtime/js/11_workers.js", - "ext:runtime/30_os.js": "../runtime/js/30_os.js", + "ext:deno_os/30_os.js": "../ext/os/30_os.js", "ext:runtime/40_fs_events.js": "../runtime/js/40_fs_events.js", "ext:runtime/40_process.js": "../runtime/js/40_process.js", - "ext:runtime/40_signals.js": "../runtime/js/40_signals.js", + "ext:deno_os/40_signals.js": "../ext/os/40_signals.js", "ext:runtime/40_tty.js": "../runtime/js/40_tty.js", "ext:runtime/41_prompt.js": "../runtime/js/41_prompt.js", "ext:runtime/90_deno_ns.js": "../runtime/js/90_deno_ns.js", From afc23fb2e007b521b9f4fcae632db2c4e41f8595 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 14 Jan 2025 23:06:57 -0500 Subject: [PATCH 4/8] chore: fix ci by removing remote server dependent test (#27674) This was using the lockfile and esm.sh changed breaking the lockfile. We could pin to a specific esm.sh version, but ideally we shouldn't have the test suite dependent on remote servers. --- tests/specs/info/import_map/__test__.jsonc | 2 +- tests/specs/info/import_map/deno.json | 3 +-- tests/specs/info/import_map/deno.lock | 8 ++----- .../specs/info/import_map/with_import_map.out | 21 ++++++------------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/tests/specs/info/import_map/__test__.jsonc b/tests/specs/info/import_map/__test__.jsonc index 725276925e..275f8000dc 100644 --- a/tests/specs/info/import_map/__test__.jsonc +++ b/tests/specs/info/import_map/__test__.jsonc @@ -1,5 +1,5 @@ { - "args": "info preact/debug", + "args": "info --allow-import myentry/welcome.ts", "output": "with_import_map.out", "exitCode": 0 } diff --git a/tests/specs/info/import_map/deno.json b/tests/specs/info/import_map/deno.json index aaf7260c64..9741f29622 100644 --- a/tests/specs/info/import_map/deno.json +++ b/tests/specs/info/import_map/deno.json @@ -1,6 +1,5 @@ { "imports": { - "preact": "https://esm.sh/preact@10.15.1", - "preact/": "https://esm.sh/preact@10.15.1/" + "myentry/": "http://localhost:4545/" } } diff --git a/tests/specs/info/import_map/deno.lock b/tests/specs/info/import_map/deno.lock index cb5c6ca45d..b7b217cd6a 100644 --- a/tests/specs/info/import_map/deno.lock +++ b/tests/specs/info/import_map/deno.lock @@ -1,10 +1,6 @@ { - "version": "3", + "version": "4", "remote": { - "https://esm.sh/preact@10.15.1": "4bfd0b2c5a2d432e0c8cda295d6b7304152ae08c85f7d0a22f91289c97085b89", - "https://esm.sh/preact@10.15.1/debug": "4bfd0b2c5a2d432e0c8cda295d6b7304152ae08c85f7d0a22f91289c97085b89", - "https://esm.sh/stable/preact@10.15.1/denonext/debug.js": "e8e5e198bd48c93d484c91c4c78af1900bd81d9bfcfd543e8ac75216f5404c10", - "https://esm.sh/stable/preact@10.15.1/denonext/devtools.js": "f61430e179a84483f8ea8dc098d7d0d46b2f0546de4027518bfcef197cd665c9", - "https://esm.sh/stable/preact@10.15.1/denonext/preact.mjs": "30710ac1d5ff3711ae0c04eddbeb706f34f82d97489f61aaf09897bc75d2a628" + "http://localhost:4545/welcome.ts": "7353d5fcbc36c45d26bcbca478cf973092523b07c45999f41319820092b4de31" } } diff --git a/tests/specs/info/import_map/with_import_map.out b/tests/specs/info/import_map/with_import_map.out index 29dc17737a..ef2f53d623 100644 --- a/tests/specs/info/import_map/with_import_map.out +++ b/tests/specs/info/import_map/with_import_map.out @@ -1,16 +1,7 @@ -Download https://esm.sh/preact@10.15.1/debug -Download https://esm.sh/stable/preact@10.15.1/denonext/preact.mjs -Download https://esm.sh/stable/preact@10.15.1/denonext/devtools.js -Download https://esm.sh/stable/preact@10.15.1/denonext/debug.js -local: [WILDCARD] -type: JavaScript -dependencies: 3 unique -size: [WILDCARD] +Download http://localhost:4545/welcome.ts +local: [WILDLINE] +type: TypeScript +dependencies: 0 unique +size: [WILDLINE] -https://esm.sh/preact@10.15.1/debug [WILDCARD] -├── https://esm.sh/stable/preact@10.15.1/denonext/preact.mjs [WILDCARD] -├─┬ https://esm.sh/stable/preact@10.15.1/denonext/devtools.js [WILDCARD] -│ └── https://esm.sh/stable/preact@10.15.1/denonext/preact.mjs [WILDCARD] -└─┬ https://esm.sh/stable/preact@10.15.1/denonext/debug.js [WILDCARD] - ├── https://esm.sh/stable/preact@10.15.1/denonext/preact.mjs [WILDCARD] - └── https://esm.sh/stable/preact@10.15.1/denonext/devtools.js [WILDCARD] +http://localhost:4545/welcome.ts ([WILDLINE]) From b22a50cb0cc124e3d41041abc9d08f2c289e9963 Mon Sep 17 00:00:00 2001 From: Masato Yoshioka Date: Wed, 15 Jan 2025 17:15:07 +0900 Subject: [PATCH 5/8] fix(ext/node): add chown method to FileHandle class (#27638) --- ext/node/polyfills/internal/fs/handle.ts | 5 ++++ tests/unit_node/_fs/_fs_handle_test.ts | 29 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ext/node/polyfills/internal/fs/handle.ts b/ext/node/polyfills/internal/fs/handle.ts index 774b36dfc2..e22c3e0ae6 100644 --- a/ext/node/polyfills/internal/fs/handle.ts +++ b/ext/node/polyfills/internal/fs/handle.ts @@ -165,6 +165,11 @@ export class FileHandle extends EventEmitter { assertNotClosed(this, promises.utimes.name); return promises.utimes(this.#path, atime, mtime); } + + chown(uid: number, gid: number): Promise { + assertNotClosed(this, promises.chown.name); + return promises.chown(this.#path, uid, gid); + } } function assertNotClosed(handle: FileHandle, syscall: string) { diff --git a/tests/unit_node/_fs/_fs_handle_test.ts b/tests/unit_node/_fs/_fs_handle_test.ts index 9134c2cb0a..d7ae32b5ea 100644 --- a/tests/unit_node/_fs/_fs_handle_test.ts +++ b/tests/unit_node/_fs/_fs_handle_test.ts @@ -2,7 +2,7 @@ import * as path from "@std/path"; import { Buffer } from "node:buffer"; import * as fs from "node:fs/promises"; -import { assert, assertEquals } from "@std/assert"; +import { assert, assertEquals, assertRejects } from "@std/assert"; const moduleDir = path.dirname(path.fromFileUrl(import.meta.url)); const testData = path.resolve(moduleDir, "testdata", "hello.txt"); @@ -273,3 +273,30 @@ Deno.test({ await fileHandle.close(); }, }); + +Deno.test({ + name: "[node/fs filehandle.chown] Change owner of the file", + ignore: Deno.build.os === "windows", + async fn() { + const fileHandle = await fs.open(testData); + + const nobodyUid = 65534; + const nobodyGid = 65534; + + await assertRejects( + async () => await fileHandle.chown(nobodyUid, nobodyGid), + Deno.errors.PermissionDenied, + "Operation not permitted", + ); + + const realUid = Deno.uid() || 1000; + const realGid = Deno.gid() || 1000; + + await fileHandle.chown(realUid, realGid); + + assertEquals(Deno.statSync(testData).uid, realUid); + assertEquals(Deno.statSync(testData).gid, realGid); + + await fileHandle.close(); + }, +}); From 836a623d998ea8f3b674a29bf7590ada49dbfa72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 15 Jan 2025 11:03:05 +0000 Subject: [PATCH 6/8] ci: use self-hosted mac arm runner (#27568) --- .github/workflows/ci.generate.ts | 8 ++++---- .github/workflows/ci.yml | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 3214fad509..55c88fbe85 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 33; +const cacheVersion = 34; const ubuntuX86Runner = "ubuntu-24.04"; const ubuntuX86XlRunner = "ubuntu-24.04-xl"; @@ -14,7 +14,7 @@ const windowsX86Runner = "windows-2022"; const windowsX86XlRunner = "windows-2022-xl"; const macosX86Runner = "macos-13"; const macosArmRunner = "macos-14"; -const selfHostedMacosArmRunner = "self-hosted"; +const selfHostedMacosArmRunner = "ghcr.io/cirruslabs/macos-runner:sonoma"; const Runners = { linuxX86: { @@ -42,7 +42,7 @@ const Runners = { os: "macos", arch: "aarch64", runner: - `\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, + `\${{ github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, windowsX86: { os: "windows", @@ -486,7 +486,7 @@ const ci = { }, { name: "Cache Cargo home", - uses: "actions/cache@v4", + uses: "cirruslabs/cache@v4", with: { // See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci // Note that with the new sparse registry format, we no longer have to cache a `.git` dir diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e560840d2e..f217d85032 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,12 +68,12 @@ jobs: skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''ghcr.io/cirruslabs/macos-runner:sonoma'' || ''macos-14'' }}' job: test profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''ghcr.io/cirruslabs/macos-runner:sonoma'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' @@ -175,7 +175,7 @@ jobs: tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \ -czvf target/release/deno_src.tar.gz -C .. deno - name: Cache Cargo home - uses: actions/cache@v4 + uses: cirruslabs/cache@v4 with: path: |- ~/.cargo/.crates.toml @@ -184,8 +184,8 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - key: '33-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '33-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' + key: '34-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '34-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - uses: dsherret/rust-toolchain-file@v1 if: '!(matrix.skip)' @@ -379,7 +379,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '33-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '34-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -689,7 +689,7 @@ jobs: !./target/*/gn_root !./target/*/*.zip !./target/*/*.tar.gz - key: '33-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '34-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-24.04 From 05dc69932d45a656943898d216eb2edef95bd808 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 15 Jan 2025 09:35:46 -0500 Subject: [PATCH 7/8] refactor: create deno_lib crate (#27673) Shifts just some code down for now. I'll do the rest of the refactor in the next pr, but didn't want to drop a huge refactor. --- Cargo.lock | 26 ++ Cargo.toml | 4 +- cli/Cargo.toml | 1 + cli/args/mod.rs | 91 +--- cli/cache/caches.rs | 7 +- cli/cache/emit.rs | 7 +- cli/cache/mod.rs | 5 - cli/factory.rs | 166 ++++--- cli/lib/Cargo.toml | 37 ++ cli/lib/README.md | 4 + cli/{ => lib}/cache/deno_dir.rs | 25 +- cli/{ => lib}/cache/disk_cache.rs | 27 +- cli/lib/cache/mod.rs | 8 + cli/lib/env.rs | 10 + cli/lib/lib.rs | 9 + cli/lib/npm/mod.rs | 6 + cli/{ => lib}/npm/permission_checker.rs | 13 +- cli/lib/standalone/mod.rs | 3 + cli/lib/standalone/virtual_fs.rs | 296 ++++++++++++ cli/lib/sys.rs | 37 ++ cli/{ => lib}/util/checksum.rs | 0 cli/lib/util/mod.rs | 3 + cli/lib/worker.rs | 581 ++++++++++++++++++++++++ cli/lsp/cache.rs | 6 +- cli/lsp/config.rs | 2 +- cli/lsp/diagnostics.rs | 2 +- cli/lsp/language_server.rs | 2 +- cli/lsp/testing/definitions.rs | 2 +- cli/lsp/tsc.rs | 2 +- cli/lsp/urls.rs | 2 +- cli/module_loader.rs | 16 +- cli/npm/mod.rs | 3 - cli/standalone/binary.rs | 19 +- cli/standalone/file_system.rs | 2 +- cli/standalone/mod.rs | 110 +++-- cli/standalone/serialization.rs | 4 +- cli/standalone/virtual_fs.rs | 426 ++++------------- cli/sys.rs | 2 + cli/tools/clean.rs | 2 +- cli/tools/info.rs | 4 +- cli/tools/serve.rs | 10 +- cli/tsc/mod.rs | 4 +- cli/util/file_watcher.rs | 84 +++- cli/util/mod.rs | 1 - cli/worker.rs | 506 ++++----------------- runtime/examples/extension/main.rs | 2 +- runtime/lib.rs | 1 + runtime/worker.rs | 4 +- 48 files changed, 1518 insertions(+), 1066 deletions(-) create mode 100644 cli/lib/Cargo.toml create mode 100644 cli/lib/README.md rename cli/{ => lib}/cache/deno_dir.rs (90%) rename cli/{ => lib}/cache/disk_cache.rs (92%) create mode 100644 cli/lib/cache/mod.rs create mode 100644 cli/lib/env.rs create mode 100644 cli/lib/lib.rs create mode 100644 cli/lib/npm/mod.rs rename cli/{ => lib}/npm/permission_checker.rs (92%) create mode 100644 cli/lib/standalone/mod.rs create mode 100644 cli/lib/standalone/virtual_fs.rs create mode 100644 cli/lib/sys.rs rename cli/{ => lib}/util/checksum.rs (100%) create mode 100644 cli/lib/util/mod.rs create mode 100644 cli/lib/worker.rs diff --git a/Cargo.lock b/Cargo.lock index aadd0c5f95..7ec72aa3cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,6 +1270,7 @@ dependencies = [ "deno_doc", "deno_error", "deno_graph", + "deno_lib", "deno_lint", "deno_lockfile", "deno_npm", @@ -1891,6 +1892,31 @@ dependencies = [ "url", ] +[[package]] +name = "deno_lib" +version = "0.1.1" +dependencies = [ + "deno_cache_dir", + "deno_error", + "deno_fs", + "deno_node", + "deno_path_util", + "deno_resolver", + "deno_runtime", + "deno_terminal 0.2.0", + "faster-hex", + "log", + "node_resolver", + "parking_lot", + "ring", + "serde", + "sys_traits", + "test_server", + "thiserror 2.0.3", + "tokio", + "url", +] + [[package]] name = "deno_lint" version = "0.68.2" diff --git a/Cargo.toml b/Cargo.toml index d9f976d863..0c11ff9a69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ resolver = "2" members = [ "bench_util", "cli", + "cli/lib", "ext/broadcast_channel", "ext/cache", "ext/canvas", @@ -94,7 +95,8 @@ deno_webidl = { version = "0.185.0", path = "./ext/webidl" } deno_websocket = { version = "0.190.0", path = "./ext/websocket" } deno_webstorage = { version = "0.180.0", path = "./ext/webstorage" } -# resolvers +# workspace libraries +deno_lib = { version = "=0.1.1", path = "./cli/lib" } deno_npm_cache = { version = "0.4.0", path = "./resolvers/npm_cache" } deno_resolver = { version = "0.16.0", path = "./resolvers/deno" } node_resolver = { version = "0.23.0", path = "./resolvers/node" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8248d40701..4525a1bab4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -76,6 +76,7 @@ deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] deno_doc = { version = "=0.164.0", features = ["rust", "comrak"] } deno_error.workspace = true deno_graph = { version = "=0.87.0" } +deno_lib.workspace = true deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 4ba85f1706..29b493046f 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -58,6 +58,9 @@ use deno_core::serde_json; use deno_core::url::Url; use deno_graph::GraphKind; pub use deno_json::check_warn_tsconfig; +use deno_lib::cache::DenoDirProvider; +use deno_lib::env::has_flag_env_var; +use deno_lib::worker::StorageKeyResolver; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::NpmRc; use deno_npm::npm_rc::ResolvedNpmRc; @@ -89,7 +92,6 @@ use serde::Serialize; use sys_traits::EnvHomeDir; use thiserror::Error; -use crate::cache::DenoDirProvider; use crate::file_fetcher::CliFileFetcher; use crate::sys::CliSys; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -768,7 +770,7 @@ pub struct CliOptions { maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, overrides: CliOptionOverrides, pub start_dir: Arc, - pub deno_dir_provider: Arc, + pub deno_dir_provider: Arc>, } impl CliOptions { @@ -1227,6 +1229,16 @@ impl CliOptions { } } + pub fn resolve_storage_key_resolver(&self) -> StorageKeyResolver { + if let Some(location) = &self.flags.location { + StorageKeyResolver::from_flag(location) + } else if let Some(deno_json) = self.start_dir.maybe_deno_json() { + StorageKeyResolver::from_config_file_url(&deno_json.specifier) + } else { + StorageKeyResolver::new_use_main_module() + } + } + // If the main module should be treated as being in an npm package. // This is triggered via a secret environment variable which is used // for functionality like child_process.fork. Users should NOT depend @@ -1871,7 +1883,7 @@ fn resolve_node_modules_folder( cwd: &Path, flags: &Flags, workspace: &Workspace, - deno_dir_provider: &Arc, + deno_dir_provider: &Arc>, ) -> Result, AnyError> { fn resolve_from_root(root_folder: &FolderConfigs, cwd: &Path) -> PathBuf { root_folder @@ -1975,63 +1987,11 @@ fn resolve_import_map_specifier( } } -pub struct StorageKeyResolver(Option>); - -impl StorageKeyResolver { - pub fn from_options(options: &CliOptions) -> Self { - Self(if let Some(location) = &options.flags.location { - // if a location is set, then the ascii serialization of the location is - // used, unless the origin is opaque, and then no storage origin is set, as - // we can't expect the origin to be reproducible - let storage_origin = location.origin(); - if storage_origin.is_tuple() { - Some(Some(storage_origin.ascii_serialization())) - } else { - Some(None) - } - } else { - // otherwise we will use the path to the config file or None to - // fall back to using the main module's path - options - .start_dir - .maybe_deno_json() - .map(|config_file| Some(config_file.specifier.to_string())) - }) - } - - /// Creates a storage key resolver that will always resolve to being empty. - pub fn empty() -> Self { - Self(Some(None)) - } - - /// Resolves the storage key to use based on the current flags, config, or main module. - pub fn resolve_storage_key( - &self, - main_module: &ModuleSpecifier, - ) -> Option { - // use the stored value or fall back to using the path of the main module. - if let Some(maybe_value) = &self.0 { - maybe_value.clone() - } else { - Some(main_module.to_string()) - } - } -} - /// Resolves the no_prompt value based on the cli flags and environment. pub fn resolve_no_prompt(flags: &PermissionFlags) -> bool { flags.no_prompt || has_flag_env_var("DENO_NO_PROMPT") } -pub fn has_trace_permissions_enabled() -> bool { - has_flag_env_var("DENO_TRACE_PERMISSIONS") -} - -pub fn has_flag_env_var(name: &str) -> bool { - let value = env::var(name); - matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) -} - pub fn npm_pkg_req_ref_to_binary_command( req_ref: &NpmPackageReqReference, ) -> String { @@ -2160,27 +2120,6 @@ mod test { assert_eq!(actual, None); } - #[test] - fn storage_key_resolver_test() { - let resolver = StorageKeyResolver(None); - let specifier = ModuleSpecifier::parse("file:///a.ts").unwrap(); - assert_eq!( - resolver.resolve_storage_key(&specifier), - Some(specifier.to_string()) - ); - let resolver = StorageKeyResolver(Some(None)); - assert_eq!(resolver.resolve_storage_key(&specifier), None); - let resolver = StorageKeyResolver(Some(Some("value".to_string()))); - assert_eq!( - resolver.resolve_storage_key(&specifier), - Some("value".to_string()) - ); - - // test empty - let resolver = StorageKeyResolver::empty(); - assert_eq!(resolver.resolve_storage_key(&specifier), None); - } - #[test] fn jsr_urls() { let reg_url = jsr_url(); diff --git a/cli/cache/caches.rs b/cli/cache/caches.rs index b83364c61b..dd4a974814 100644 --- a/cli/cache/caches.rs +++ b/cli/cache/caches.rs @@ -3,20 +3,21 @@ use std::path::PathBuf; use std::sync::Arc; +use deno_lib::cache::DenoDirProvider; use once_cell::sync::OnceCell; use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::check::TYPE_CHECK_CACHE_DB; use super::code_cache::CODE_CACHE_DB; -use super::deno_dir::DenoDirProvider; use super::fast_check::FAST_CHECK_CACHE_DB; use super::incremental::INCREMENTAL_CACHE_DB; use super::module_info::MODULE_INFO_CACHE_DB; use super::node::NODE_ANALYSIS_CACHE_DB; +use crate::sys::CliSys; pub struct Caches { - dir_provider: Arc, + dir_provider: Arc>, fmt_incremental_cache_db: OnceCell, lint_incremental_cache_db: OnceCell, dep_analysis_db: OnceCell, @@ -27,7 +28,7 @@ pub struct Caches { } impl Caches { - pub fn new(dir: Arc) -> Self { + pub fn new(dir: Arc>) -> Self { Self { dir_provider: dir, fmt_incremental_cache_db: Default::default(), diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 2ba43d58b9..e8a940b3be 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -6,19 +6,20 @@ use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::unsync::sync::AtomicFlag; +use deno_lib::cache::DiskCache; -use super::DiskCache; +use crate::sys::CliSys; /// The cache that stores previously emitted files. #[derive(Debug)] pub struct EmitCache { - disk_cache: DiskCache, + disk_cache: DiskCache, emit_failed_flag: AtomicFlag, file_serializer: EmitFileSerializer, } impl EmitCache { - pub fn new(disk_cache: DiskCache) -> Self { + pub fn new(disk_cache: DiskCache) -> Self { Self { disk_cache, emit_failed_flag: Default::default(), diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 0d7808cba6..e16f95e56f 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -31,8 +31,6 @@ mod caches; mod check; mod code_cache; mod common; -mod deno_dir; -mod disk_cache; mod emit; mod fast_check; mod incremental; @@ -47,9 +45,6 @@ pub use code_cache::CodeCache; pub use common::FastInsecureHasher; /// Permissions used to save a file in the disk caches. pub use deno_cache_dir::CACHE_PERM; -pub use deno_dir::DenoDir; -pub use deno_dir::DenoDirProvider; -pub use disk_cache::DiskCache; pub use emit::EmitCache; pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; diff --git a/cli/factory.rs b/cli/factory.rs index 3280fd379b..bfe6d05570 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,6 +11,12 @@ use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::FeatureChecker; use deno_error::JsErrorBox; +use deno_lib::cache::DenoDir; +use deno_lib::cache::DenoDirProvider; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; +use deno_lib::worker::LibMainWorkerFactory; +use deno_lib::worker::LibMainWorkerOptions; use deno_npm_cache::NpmCacheSetting; use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; @@ -42,12 +48,9 @@ use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::NpmInstallDepsProvider; -use crate::args::StorageKeyResolver; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; -use crate::cache::DenoDir; -use crate::cache::DenoDirProvider; use crate::cache::EmitCache; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; @@ -68,6 +71,7 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; +use crate::npm::create_npm_process_state_provider; use crate::npm::installer::NpmInstaller; use crate::npm::installer::NpmResolutionInstaller; use crate::npm::CliByonmNpmResolverCreateOptions; @@ -79,8 +83,6 @@ use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::CliNpmTarballCache; -use crate::npm::NpmRegistryReadPermissionChecker; -use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; use crate::resolver::CliCjsTracker; use crate::resolver::CliDenoResolver; @@ -281,11 +283,13 @@ impl CliFactory { }) } - pub fn deno_dir_provider(&self) -> Result<&Arc, AnyError> { + pub fn deno_dir_provider( + &self, + ) -> Result<&Arc>, AnyError> { Ok(&self.cli_options()?.deno_dir_provider) } - pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { + pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { Ok(self.deno_dir_provider()?.get_or_create()?) } @@ -1083,7 +1087,34 @@ impl CliFactory { Arc::new(NpmRegistryReadPermissionChecker::new(self.sys(), mode)) }; - Ok(CliMainWorkerFactory::new( + let module_loader_factory = CliModuleLoaderFactory::new( + cli_options, + cjs_tracker, + if cli_options.code_cache_enabled() { + Some(self.code_cache()?.clone()) + } else { + None + }, + self.emitter()?.clone(), + in_npm_pkg_checker.clone(), + self.main_module_graph_container().await?.clone(), + self.module_load_preparer().await?.clone(), + node_code_translator.clone(), + node_resolver.clone(), + NpmModuleLoader::new( + self.cjs_tracker()?.clone(), + fs.clone(), + node_code_translator.clone(), + ), + npm_registry_permission_checker, + npm_req_resolver.clone(), + cli_npm_resolver.clone(), + self.parsed_source_cache().clone(), + self.resolver().await?.clone(), + self.sys(), + ); + + let lib_main_worker_factory = LibMainWorkerFactory::new( self.blob_store().clone(), if cli_options.code_cache_enabled() { Some(self.code_cache()?.clone()) @@ -1092,50 +1123,70 @@ impl CliFactory { }, self.feature_checker()?.clone(), fs.clone(), - maybe_file_watcher_communicator, self.maybe_inspector_server()?.clone(), + Box::new(module_loader_factory), + node_resolver.clone(), + create_npm_process_state_provider(npm_resolver), + pkg_json_resolver, + self.root_cert_store_provider().clone(), + cli_options.resolve_storage_key_resolver(), + self.sys(), + self.create_lib_main_worker_options()?, + ); + + Ok(CliMainWorkerFactory::new( + lib_main_worker_factory, + maybe_file_watcher_communicator, cli_options.maybe_lockfile().cloned(), - Box::new(CliModuleLoaderFactory::new( - cli_options, - cjs_tracker, - if cli_options.code_cache_enabled() { - Some(self.code_cache()?.clone()) - } else { - None - }, - self.emitter()?.clone(), - in_npm_pkg_checker.clone(), - self.main_module_graph_container().await?.clone(), - self.module_load_preparer().await?.clone(), - node_code_translator.clone(), - node_resolver.clone(), - NpmModuleLoader::new( - self.cjs_tracker()?.clone(), - fs.clone(), - node_code_translator.clone(), - ), - npm_registry_permission_checker, - npm_req_resolver.clone(), - cli_npm_resolver.clone(), - self.parsed_source_cache().clone(), - self.resolver().await?.clone(), - self.sys(), - )), node_resolver.clone(), self.npm_installer_if_managed()?.cloned(), npm_resolver.clone(), - pkg_json_resolver, - self.root_cert_store_provider().clone(), - self.root_permissions_container()?.clone(), - StorageKeyResolver::from_options(cli_options), self.sys(), - cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, - self.cli_options()?.otel_config(), - self.cli_options()?.default_npm_caching_strategy(), + self.root_permissions_container()?.clone(), )) } + fn create_lib_main_worker_options( + &self, + ) -> Result { + let cli_options = self.cli_options()?; + Ok(LibMainWorkerOptions { + argv: cli_options.argv().clone(), + // This optimization is only available for "run" subcommand + // because we need to register new ops for testing and jupyter + // integration. + skip_op_registration: cli_options.sub_command().is_run(), + log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(), + enable_op_summary_metrics: cli_options.enable_op_summary_metrics(), + enable_testing_features: cli_options.enable_testing_features(), + has_node_modules_dir: cli_options.has_node_modules_dir(), + inspect_brk: cli_options.inspect_brk().is_some(), + inspect_wait: cli_options.inspect_wait().is_some(), + strace_ops: cli_options.strace_ops().clone(), + is_inspecting: cli_options.is_inspecting(), + location: cli_options.location_flag().clone(), + // if the user ran a binary command, we'll need to set process.argv[0] + // to be the name of the binary command instead of deno + argv0: cli_options + .take_binary_npm_command_name() + .or(std::env::args().next()), + node_debug: std::env::var("NODE_DEBUG").ok(), + origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()), + seed: cli_options.seed(), + unsafely_ignore_certificate_errors: cli_options + .unsafely_ignore_certificate_errors() + .clone(), + node_ipc: cli_options.node_ipc_fd(), + serve_port: cli_options.serve_port(), + serve_host: cli_options.serve_host(), + deno_version: crate::version::DENO_VERSION_INFO.deno, + deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, + otel_config: self.cli_options()?.otel_config(), + startup_snapshot: crate::js::deno_isolate_init(), + }) + } + fn create_cli_main_worker_options( &self, ) -> Result { @@ -1167,37 +1218,10 @@ impl CliFactory { }; Ok(CliMainWorkerOptions { - argv: cli_options.argv().clone(), - // This optimization is only available for "run" subcommand - // because we need to register new ops for testing and jupyter - // integration. - skip_op_registration: cli_options.sub_command().is_run(), - log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(), - enable_op_summary_metrics: cli_options.enable_op_summary_metrics(), - enable_testing_features: cli_options.enable_testing_features(), - has_node_modules_dir: cli_options.has_node_modules_dir(), - hmr: cli_options.has_hmr(), - inspect_brk: cli_options.inspect_brk().is_some(), - inspect_wait: cli_options.inspect_wait().is_some(), - strace_ops: cli_options.strace_ops().clone(), - is_inspecting: cli_options.is_inspecting(), - location: cli_options.location_flag().clone(), - // if the user ran a binary command, we'll need to set process.argv[0] - // to be the name of the binary command instead of deno - argv0: cli_options - .take_binary_npm_command_name() - .or(std::env::args().next()), - node_debug: std::env::var("NODE_DEBUG").ok(), - origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()), - seed: cli_options.seed(), - unsafely_ignore_certificate_errors: cli_options - .unsafely_ignore_certificate_errors() - .clone(), + needs_test_modules: cli_options.sub_command().needs_test(), create_hmr_runner, create_coverage_collector, - node_ipc: cli_options.node_ipc_fd(), - serve_port: cli_options.serve_port(), - serve_host: cli_options.serve_host(), + default_npm_caching_strategy: cli_options.default_npm_caching_strategy(), }) } } diff --git a/cli/lib/Cargo.toml b/cli/lib/Cargo.toml new file mode 100644 index 0000000000..6a74b22c2b --- /dev/null +++ b/cli/lib/Cargo.toml @@ -0,0 +1,37 @@ +# Copyright 2018-2025 the Deno authors. MIT license. + +[package] +name = "deno_lib" +version = "0.1.1" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Shared code between the Deno CLI and denort" + +[lib] +path = "lib.rs" + +[dependencies] +deno_cache_dir.workspace = true +deno_error.workspace = true +deno_fs = { workspace = true, features = ["sync_fs"] } +deno_node = { workspace = true, features = ["sync_fs"] } +deno_path_util.workspace = true +deno_resolver = { workspace = true, features = ["sync"] } +deno_runtime.workspace = true +deno_terminal.workspace = true +faster-hex.workspace = true +log.workspace = true +node_resolver = { workspace = true, features = ["sync"] } +parking_lot.workspace = true +ring.workspace = true +serde = { workspace = true, features = ["derive"] } +sys_traits.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true + +[dev-dependencies] +test_util.workspace = true diff --git a/cli/lib/README.md b/cli/lib/README.md new file mode 100644 index 0000000000..bc6d7b57d0 --- /dev/null +++ b/cli/lib/README.md @@ -0,0 +1,4 @@ +# deno_lib + +This crate contains the shared code between the Deno CLI and denort. It is +highly unstable. diff --git a/cli/cache/deno_dir.rs b/cli/lib/cache/deno_dir.rs similarity index 90% rename from cli/cache/deno_dir.rs rename to cli/lib/cache/deno_dir.rs index 1b35f53071..00bc83ff9b 100644 --- a/cli/cache/deno_dir.rs +++ b/cli/lib/cache/deno_dir.rs @@ -4,21 +4,20 @@ use std::env; use std::path::PathBuf; use deno_cache_dir::DenoDirResolutionError; -use once_cell::sync::OnceCell; use super::DiskCache; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; /// Lazily creates the deno dir which might be useful in scenarios /// where functionality wants to continue if the DENO_DIR can't be created. -pub struct DenoDirProvider { - sys: CliSys, +pub struct DenoDirProvider { + sys: TSys, maybe_custom_root: Option, - deno_dir: OnceCell>, + deno_dir: std::sync::OnceLock, DenoDirResolutionError>>, } -impl DenoDirProvider { - pub fn new(sys: CliSys, maybe_custom_root: Option) -> Self { +impl DenoDirProvider { + pub fn new(sys: TSys, maybe_custom_root: Option) -> Self { Self { sys, maybe_custom_root, @@ -26,7 +25,9 @@ impl DenoDirProvider { } } - pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> { + pub fn get_or_create( + &self, + ) -> Result<&DenoDir, DenoDirResolutionError> { self .deno_dir .get_or_init(|| { @@ -49,16 +50,16 @@ impl DenoDirProvider { /// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them /// in single directory that can be controlled with `$DENO_DIR` env variable. #[derive(Debug, Clone)] -pub struct DenoDir { +pub struct DenoDir { /// Example: /Users/rld/.deno/ pub root: PathBuf, /// Used by TsCompiler to cache compiler output. - pub gen_cache: DiskCache, + pub gen_cache: DiskCache, } -impl DenoDir { +impl DenoDir { pub fn new( - sys: CliSys, + sys: TSys, maybe_custom_root: Option, ) -> Result { let root = deno_cache_dir::resolve_deno_dir( diff --git a/cli/cache/disk_cache.rs b/cli/lib/cache/disk_cache.rs similarity index 92% rename from cli/cache/disk_cache.rs rename to cli/lib/cache/disk_cache.rs index f03b60854f..2c735a34b2 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/lib/cache/disk_cache.rs @@ -9,22 +9,22 @@ use std::path::Prefix; use std::str; use deno_cache_dir::url_to_filename; -use deno_core::url::Host; -use deno_core::url::Url; +use deno_cache_dir::CACHE_PERM; use deno_path_util::fs::atomic_write_file_with_retries; +use url::Host; +use url::Url; -use super::CACHE_PERM; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; #[derive(Debug, Clone)] -pub struct DiskCache { - sys: CliSys, +pub struct DiskCache { + sys: TSys, pub location: PathBuf, } -impl DiskCache { +impl DiskCache { /// `location` must be an absolute path. - pub fn new(sys: CliSys, location: &Path) -> Self { + pub fn new(sys: TSys, location: &Path) -> Self { assert!(location.is_absolute()); Self { sys, @@ -130,6 +130,9 @@ impl DiskCache { #[cfg(test)] mod tests { + // ok, testing + #[allow(clippy::disallowed_types)] + use sys_traits::impls::RealSys; use test_util::TempDir; use super::*; @@ -138,7 +141,7 @@ mod tests { fn test_set_get_cache_file() { let temp_dir = TempDir::new(); let sub_dir = temp_dir.path().join("sub_dir"); - let cache = DiskCache::new(CliSys::default(), &sub_dir.to_path_buf()); + let cache = DiskCache::new(RealSys, &sub_dir.to_path_buf()); let path = PathBuf::from("foo/bar.txt"); cache.set(&path, b"hello").unwrap(); assert_eq!(cache.get(&path).unwrap(), b"hello"); @@ -152,7 +155,7 @@ mod tests { PathBuf::from("/deno_dir/") }; - let cache = DiskCache::new(CliSys::default(), &cache_location); + let cache = DiskCache::new(RealSys, &cache_location); let mut test_cases = vec![ ( @@ -208,7 +211,7 @@ mod tests { } else { "/foo" }; - let cache = DiskCache::new(CliSys::default(), &PathBuf::from(p)); + let cache = DiskCache::new(RealSys, &PathBuf::from(p)); let mut test_cases = vec![ ( @@ -256,7 +259,7 @@ mod tests { PathBuf::from("/deno_dir/") }; - let cache = DiskCache::new(CliSys::default(), &cache_location); + let cache = DiskCache::new(RealSys, &cache_location); let mut test_cases = vec!["unknown://localhost/test.ts"]; diff --git a/cli/lib/cache/mod.rs b/cli/lib/cache/mod.rs new file mode 100644 index 0000000000..c4395df3e1 --- /dev/null +++ b/cli/lib/cache/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub use deno_dir::DenoDir; +pub use deno_dir::DenoDirProvider; +pub use disk_cache::DiskCache; + +mod deno_dir; +mod disk_cache; diff --git a/cli/lib/env.rs b/cli/lib/env.rs new file mode 100644 index 0000000000..9c6001478b --- /dev/null +++ b/cli/lib/env.rs @@ -0,0 +1,10 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub fn has_trace_permissions_enabled() -> bool { + has_flag_env_var("DENO_TRACE_PERMISSIONS") +} + +pub fn has_flag_env_var(name: &str) -> bool { + let value = std::env::var(name); + matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) +} diff --git a/cli/lib/lib.rs b/cli/lib/lib.rs new file mode 100644 index 0000000000..5453bddaee --- /dev/null +++ b/cli/lib/lib.rs @@ -0,0 +1,9 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod cache; +pub mod env; +pub mod npm; +pub mod standalone; +pub mod sys; +pub mod util; +pub mod worker; diff --git a/cli/lib/npm/mod.rs b/cli/lib/npm/mod.rs new file mode 100644 index 0000000000..e7d4d8d9d1 --- /dev/null +++ b/cli/lib/npm/mod.rs @@ -0,0 +1,6 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +mod permission_checker; + +pub use permission_checker::NpmRegistryReadPermissionChecker; +pub use permission_checker::NpmRegistryReadPermissionCheckerMode; diff --git a/cli/npm/permission_checker.rs b/cli/lib/npm/permission_checker.rs similarity index 92% rename from cli/npm/permission_checker.rs rename to cli/lib/npm/permission_checker.rs index 53031b5bd4..ebed1270f3 100644 --- a/cli/npm/permission_checker.rs +++ b/cli/lib/npm/permission_checker.rs @@ -6,12 +6,11 @@ use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; -use deno_core::parking_lot::Mutex; use deno_error::JsErrorBox; use deno_runtime::deno_node::NodePermissions; -use sys_traits::FsCanonicalize; +use parking_lot::Mutex; -use crate::sys::CliSys; +use crate::sys::DenoLibSys; #[derive(Debug)] pub enum NpmRegistryReadPermissionCheckerMode { @@ -21,8 +20,8 @@ pub enum NpmRegistryReadPermissionCheckerMode { } #[derive(Debug)] -pub struct NpmRegistryReadPermissionChecker { - sys: CliSys, +pub struct NpmRegistryReadPermissionChecker { + sys: TSys, cache: Mutex>, mode: NpmRegistryReadPermissionCheckerMode, } @@ -37,8 +36,8 @@ struct EnsureRegistryReadPermissionError { source: std::io::Error, } -impl NpmRegistryReadPermissionChecker { - pub fn new(sys: CliSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self { +impl NpmRegistryReadPermissionChecker { + pub fn new(sys: TSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self { Self { sys, cache: Default::default(), diff --git a/cli/lib/standalone/mod.rs b/cli/lib/standalone/mod.rs new file mode 100644 index 0000000000..6e173a457a --- /dev/null +++ b/cli/lib/standalone/mod.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod virtual_fs; diff --git a/cli/lib/standalone/virtual_fs.rs b/cli/lib/standalone/virtual_fs.rs new file mode 100644 index 0000000000..5fc17f27b7 --- /dev/null +++ b/cli/lib/standalone/virtual_fs.rs @@ -0,0 +1,296 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::cmp::Ordering; +use std::path::Path; +use std::path::PathBuf; + +use serde::Deserialize; +use serde::Serialize; + +#[derive(Debug, Copy, Clone)] +pub enum VfsFileSubDataKind { + /// Raw bytes of the file. + Raw, + /// Bytes to use for module loading. For example, for TypeScript + /// files this will be the transpiled JavaScript source. + ModuleGraph, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!( + !name_component.contains('\\'), + "Invalid component: {}", + name_component + ); + debug_assert!( + !name_component.contains('/'), + "Invalid component: {}", + name_component + ); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +pub enum FileSystemCaseSensitivity { + #[serde(rename = "s")] + Sensitive, + #[serde(rename = "i")] + Insensitive, +} +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct VirtualDirectoryEntries(Vec); + +impl VirtualDirectoryEntries { + pub fn new(mut entries: Vec) -> Self { + // needs to be sorted by name + entries.sort_by(|a, b| a.name().cmp(b.name())); + Self(entries) + } + + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, VfsEntry> { + self.0.iter_mut() + } + + pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> { + self.0.iter() + } + + pub fn take_inner(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get_by_name( + &self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Option<&VfsEntry> { + self + .binary_search(name, case_sensitivity) + .ok() + .map(|index| &self.0[index]) + } + + pub fn get_mut_by_name( + &mut self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Option<&mut VfsEntry> { + self + .binary_search(name, case_sensitivity) + .ok() + .map(|index| &mut self.0[index]) + } + + pub fn get_mut_by_index(&mut self, index: usize) -> Option<&mut VfsEntry> { + self.0.get_mut(index) + } + + pub fn binary_search( + &self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Result { + match case_sensitivity { + FileSystemCaseSensitivity::Sensitive => { + self.0.binary_search_by(|e| e.name().cmp(name)) + } + FileSystemCaseSensitivity::Insensitive => self.0.binary_search_by(|e| { + e.name() + .chars() + .zip(name.chars()) + .map(|(a, b)| a.to_ascii_lowercase().cmp(&b.to_ascii_lowercase())) + .find(|&ord| ord != Ordering::Equal) + .unwrap_or_else(|| e.name().len().cmp(&name.len())) + }), + } + } + + pub fn insert( + &mut self, + entry: VfsEntry, + case_sensitivity: FileSystemCaseSensitivity, + ) -> usize { + match self.binary_search(entry.name(), case_sensitivity) { + Ok(index) => { + self.0[index] = entry; + index + } + Err(insert_index) => { + self.0.insert(insert_index, entry); + insert_index + } + } + } + + pub fn insert_or_modify( + &mut self, + name: &str, + case_sensitivity: FileSystemCaseSensitivity, + on_insert: impl FnOnce() -> VfsEntry, + on_modify: impl FnOnce(&mut VfsEntry), + ) -> usize { + match self.binary_search(name, case_sensitivity) { + Ok(index) => { + on_modify(&mut self.0[index]); + index + } + Err(insert_index) => { + self.0.insert(insert_index, on_insert()); + insert_index + } + } + } + + pub fn remove(&mut self, index: usize) -> VfsEntry { + self.0.remove(index) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualDirectory { + #[serde(rename = "n")] + pub name: String, + // should be sorted by name + #[serde(rename = "e")] + pub entries: VirtualDirectoryEntries, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct OffsetWithLength { + #[serde(rename = "o")] + pub offset: u64, + #[serde(rename = "l")] + pub len: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VirtualFile { + #[serde(rename = "n")] + pub name: String, + #[serde(rename = "o")] + pub offset: OffsetWithLength, + /// Offset file to use for module loading when it differs from the + /// raw file. Often this will be the same offset as above for data + /// such as JavaScript files, but for TypeScript files the `offset` + /// will be the original raw bytes when included as an asset and this + /// offset will be to the transpiled JavaScript source. + #[serde(rename = "m")] + pub module_graph_offset: OffsetWithLength, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } + + pub fn take_parts(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + + pub fn parts(&self) -> &[String] { + &self.0 + } + + pub fn set_parts(&mut self, parts: Vec) { + self.0 = parts; + } + + pub fn display(&self) -> String { + self.0.join("/") + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlink { + #[serde(rename = "n")] + pub name: String, + #[serde(rename = "p")] + pub dest_parts: VirtualSymlinkParts, +} + +impl VirtualSymlink { + pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { + let mut dest = root.to_path_buf(); + for part in &self.dest_parts.0 { + dest.push(part); + } + dest + } +} + +#[derive(Debug, Copy, Clone)] +pub enum VfsEntryRef<'a> { + Dir(&'a VirtualDirectory), + File(&'a VirtualFile), + Symlink(&'a VirtualSymlink), +} + +impl VfsEntryRef<'_> { + pub fn name(&self) -> &str { + match self { + Self::Dir(dir) => &dir.name, + Self::File(file) => &file.name, + Self::Symlink(symlink) => &symlink.name, + } + } +} + +// todo(dsherret): we should store this more efficiently in the binary +#[derive(Debug, Serialize, Deserialize)] +pub enum VfsEntry { + Dir(VirtualDirectory), + File(VirtualFile), + Symlink(VirtualSymlink), +} + +impl VfsEntry { + pub fn name(&self) -> &str { + match self { + Self::Dir(dir) => &dir.name, + Self::File(file) => &file.name, + Self::Symlink(symlink) => &symlink.name, + } + } + + pub fn as_ref(&self) -> VfsEntryRef { + match self { + VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir), + VfsEntry::File(file) => VfsEntryRef::File(file), + VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink), + } + } +} diff --git a/cli/lib/sys.rs b/cli/lib/sys.rs new file mode 100644 index 0000000000..f5ca48b41c --- /dev/null +++ b/cli/lib/sys.rs @@ -0,0 +1,37 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use deno_node::ExtNodeSys; +use sys_traits::FsCanonicalize; +use sys_traits::FsCreateDirAll; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsRead; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; + +pub trait DenoLibSys: + FsCanonicalize + + FsCreateDirAll + + FsReadDir + + FsMetadata + + FsOpen + + FsRemoveFile + + FsRename + + FsRead + + ThreadSleep + + SystemRandom + + ExtNodeSys + + Clone + + Send + + Sync + + std::fmt::Debug + + 'static +{ +} + +// ok, implementation +#[allow(clippy::disallowed_types)] +impl DenoLibSys for sys_traits::impls::RealSys {} diff --git a/cli/util/checksum.rs b/cli/lib/util/checksum.rs similarity index 100% rename from cli/util/checksum.rs rename to cli/lib/util/checksum.rs diff --git a/cli/lib/util/mod.rs b/cli/lib/util/mod.rs new file mode 100644 index 0000000000..8371440750 --- /dev/null +++ b/cli/lib/util/mod.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub mod checksum; diff --git a/cli/lib/worker.rs b/cli/lib/worker.rs new file mode 100644 index 0000000000..7c9071d0ba --- /dev/null +++ b/cli/lib/worker.rs @@ -0,0 +1,581 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use deno_core::error::JsError; +use deno_node::NodeRequireLoaderRc; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmResolver; +use deno_runtime::colors; +use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; +use deno_runtime::deno_core; +use deno_runtime::deno_core::error::CoreError; +use deno_runtime::deno_core::v8; +use deno_runtime::deno_core::CompiledWasmModuleStore; +use deno_runtime::deno_core::Extension; +use deno_runtime::deno_core::FeatureChecker; +use deno_runtime::deno_core::JsRuntime; +use deno_runtime::deno_core::LocalInspectorSession; +use deno_runtime::deno_core::ModuleLoader; +use deno_runtime::deno_core::SharedArrayBufferStore; +use deno_runtime::deno_fs; +use deno_runtime::deno_node::NodeExtInitServices; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_permissions::PermissionsContainer; +use deno_runtime::deno_telemetry::OtelConfig; +use deno_runtime::deno_tls::RootCertStoreProvider; +use deno_runtime::deno_web::BlobStore; +use deno_runtime::fmt_errors::format_js_error; +use deno_runtime::inspector_server::InspectorServer; +use deno_runtime::ops::process::NpmProcessStateProviderRc; +use deno_runtime::ops::worker_host::CreateWebWorkerCb; +use deno_runtime::web_worker::WebWorker; +use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::web_worker::WebWorkerServiceOptions; +use deno_runtime::worker::MainWorker; +use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; +use deno_runtime::BootstrapOptions; +use deno_runtime::WorkerExecutionMode; +use deno_runtime::WorkerLogLevel; +use deno_runtime::UNSTABLE_GRANULAR_FLAGS; +use url::Url; + +use crate::env::has_trace_permissions_enabled; +use crate::sys::DenoLibSys; +use crate::util::checksum; + +pub struct CreateModuleLoaderResult { + pub module_loader: Rc, + pub node_require_loader: Rc, +} + +pub trait ModuleLoaderFactory: Send + Sync { + fn create_for_main( + &self, + root_permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult; + + fn create_for_worker( + &self, + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult; +} + +enum StorageKeyResolverStrategy { + Specified(Option), + UseMainModule, +} + +pub struct StorageKeyResolver(StorageKeyResolverStrategy); + +impl StorageKeyResolver { + pub fn from_flag(location: &Url) -> Self { + // if a location is set, then the ascii serialization of the location is + // used, unless the origin is opaque, and then no storage origin is set, as + // we can't expect the origin to be reproducible + let storage_origin = location.origin(); + Self(StorageKeyResolverStrategy::Specified( + if storage_origin.is_tuple() { + Some(storage_origin.ascii_serialization()) + } else { + None + }, + )) + } + + pub fn from_config_file_url(url: &Url) -> Self { + Self(StorageKeyResolverStrategy::Specified(Some(url.to_string()))) + } + + pub fn new_use_main_module() -> Self { + Self(StorageKeyResolverStrategy::UseMainModule) + } + + /// Creates a storage key resolver that will always resolve to being empty. + pub fn empty() -> Self { + Self(StorageKeyResolverStrategy::Specified(None)) + } + + /// Resolves the storage key to use based on the current flags, config, or main module. + pub fn resolve_storage_key(&self, main_module: &Url) -> Option { + // use the stored value or fall back to using the path of the main module. + match &self.0 { + StorageKeyResolverStrategy::Specified(value) => value.clone(), + StorageKeyResolverStrategy::UseMainModule => { + Some(main_module.to_string()) + } + } + } +} + +// TODO(bartlomieju): this should be moved to some other place, added to avoid string +// duplication between worker setups and `deno info` output. +pub fn get_cache_storage_dir() -> PathBuf { + // Note: we currently use temp_dir() to avoid managing storage size. + std::env::temp_dir().join("deno_cache") +} + +/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs. +/// Instead probe for the total memory on the system and use it instead +/// as a default. +pub fn create_isolate_create_params() -> Option { + let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info(); + maybe_mem_info.map(|mem_info| { + v8::CreateParams::default() + .heap_limits_from_system_memory(mem_info.total, 0) + }) +} + +pub struct LibMainWorkerOptions { + pub argv: Vec, + pub deno_version: &'static str, + pub deno_user_agent: &'static str, + pub log_level: WorkerLogLevel, + pub enable_op_summary_metrics: bool, + pub enable_testing_features: bool, + pub has_node_modules_dir: bool, + pub inspect_brk: bool, + pub inspect_wait: bool, + pub strace_ops: Option>, + pub is_inspecting: bool, + pub location: Option, + pub argv0: Option, + pub node_debug: Option, + pub otel_config: OtelConfig, + pub origin_data_folder_path: Option, + pub seed: Option, + pub unsafely_ignore_certificate_errors: Option>, + pub skip_op_registration: bool, + pub node_ipc: Option, + pub startup_snapshot: Option<&'static [u8]>, + pub serve_port: Option, + pub serve_host: Option, +} + +struct LibWorkerFactorySharedState { + blob_store: Arc, + broadcast_channel: InMemoryBroadcastChannel, + code_cache: Option>, + compiled_wasm_module_store: CompiledWasmModuleStore, + feature_checker: Arc, + fs: Arc, + maybe_inspector_server: Option>, + module_loader_factory: Box, + node_resolver: + Arc, TSys>>, + npm_process_state_provider: NpmProcessStateProviderRc, + pkg_json_resolver: Arc>, + root_cert_store_provider: Arc, + shared_array_buffer_store: SharedArrayBufferStore, + storage_key_resolver: StorageKeyResolver, + sys: TSys, + options: LibMainWorkerOptions, +} + +impl LibWorkerFactorySharedState { + fn resolve_unstable_features( + &self, + feature_checker: &FeatureChecker, + ) -> Vec { + let mut unstable_features = + Vec::with_capacity(UNSTABLE_GRANULAR_FLAGS.len()); + for granular_flag in UNSTABLE_GRANULAR_FLAGS { + if feature_checker.check(granular_flag.name) { + unstable_features.push(granular_flag.id); + } + } + unstable_features + } + + fn create_node_init_services( + &self, + node_require_loader: NodeRequireLoaderRc, + ) -> NodeExtInitServices, TSys> { + NodeExtInitServices { + node_require_loader, + node_resolver: self.node_resolver.clone(), + pkg_json_resolver: self.pkg_json_resolver.clone(), + sys: self.sys.clone(), + } + } + + fn create_web_worker_callback( + self: &Arc, + stdio: deno_runtime::deno_io::Stdio, + ) -> Arc { + let shared = self.clone(); + Arc::new(move |args| { + let maybe_inspector_server = shared.maybe_inspector_server.clone(); + + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared.module_loader_factory.create_for_worker( + args.parent_permissions.clone(), + args.permissions.clone(), + ); + let create_web_worker_cb = + shared.create_web_worker_callback(stdio.clone()); + + let maybe_storage_key = shared + .storage_key_resolver + .resolve_storage_key(&args.main_module); + let cache_storage_dir = maybe_storage_key.map(|key| { + // TODO(@satyarohith): storage quota management + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) + }); + + // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out + // list of enabled features. + let feature_checker = shared.feature_checker.clone(); + let unstable_features = + shared.resolve_unstable_features(feature_checker.as_ref()); + + let services = WebWorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + shared_array_buffer_store: Some( + shared.shared_array_buffer_store.clone(), + ), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + maybe_inspector_server, + feature_checker, + npm_process_state_provider: Some( + shared.npm_process_state_provider.clone(), + ), + permissions: args.permissions, + }; + let options = WebWorkerOptions { + name: args.name, + main_module: args.main_module.clone(), + worker_id: args.worker_id, + bootstrap: BootstrapOptions { + deno_version: shared.options.deno_version.to_string(), + args: shared.options.argv.clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + log_level: shared.options.log_level, + enable_op_summary_metrics: shared.options.enable_op_summary_metrics, + enable_testing_features: shared.options.enable_testing_features, + locale: deno_core::v8::icu::get_language_tag(), + location: Some(args.main_module), + no_color: !colors::use_color(), + color_level: colors::get_color_level(), + is_stdout_tty: deno_terminal::is_stdout_tty(), + is_stderr_tty: deno_terminal::is_stderr_tty(), + unstable_features, + user_agent: shared.options.deno_user_agent.to_string(), + inspect: shared.options.is_inspecting, + has_node_modules_dir: shared.options.has_node_modules_dir, + argv0: shared.options.argv0.clone(), + node_debug: shared.options.node_debug.clone(), + node_ipc_fd: None, + mode: WorkerExecutionMode::Worker, + serve_port: shared.options.serve_port, + serve_host: shared.options.serve_host.clone(), + otel_config: shared.options.otel_config.clone(), + close_on_idle: args.close_on_idle, + }, + extensions: vec![], + startup_snapshot: shared.options.startup_snapshot, + create_params: create_isolate_create_params(), + unsafely_ignore_certificate_errors: shared + .options + .unsafely_ignore_certificate_errors + .clone(), + seed: shared.options.seed, + create_web_worker_cb, + format_js_error_fn: Some(Arc::new(format_js_error)), + worker_type: args.worker_type, + stdio: stdio.clone(), + cache_storage_dir, + strace_ops: shared.options.strace_ops.clone(), + close_on_idle: args.close_on_idle, + maybe_worker_metadata: args.maybe_worker_metadata, + enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(), + }; + + WebWorker::bootstrap_from_options(services, options) + }) + } +} + +pub struct LibMainWorkerFactory { + shared: Arc>, +} + +impl LibMainWorkerFactory { + #[allow(clippy::too_many_arguments)] + pub fn new( + blob_store: Arc, + code_cache: Option>, + feature_checker: Arc, + fs: Arc, + maybe_inspector_server: Option>, + module_loader_factory: Box, + node_resolver: Arc< + NodeResolver, TSys>, + >, + npm_process_state_provider: NpmProcessStateProviderRc, + pkg_json_resolver: Arc>, + root_cert_store_provider: Arc, + storage_key_resolver: StorageKeyResolver, + sys: TSys, + options: LibMainWorkerOptions, + ) -> Self { + Self { + shared: Arc::new(LibWorkerFactorySharedState { + blob_store, + broadcast_channel: Default::default(), + code_cache, + compiled_wasm_module_store: Default::default(), + feature_checker, + fs, + maybe_inspector_server, + module_loader_factory, + node_resolver, + npm_process_state_provider, + pkg_json_resolver, + root_cert_store_provider, + shared_array_buffer_store: Default::default(), + storage_key_resolver, + sys, + options, + }), + } + } + + pub fn create_custom_worker( + &self, + mode: WorkerExecutionMode, + main_module: Url, + permissions: PermissionsContainer, + custom_extensions: Vec, + stdio: deno_runtime::deno_io::Stdio, + ) -> Result { + let shared = &self.shared; + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared + .module_loader_factory + .create_for_main(permissions.clone()); + + // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out + // list of enabled features. + let feature_checker = shared.feature_checker.clone(); + let unstable_features = + shared.resolve_unstable_features(feature_checker.as_ref()); + let maybe_storage_key = shared + .storage_key_resolver + .resolve_storage_key(&main_module); + let origin_storage_dir = maybe_storage_key.as_ref().map(|key| { + shared + .options + .origin_data_folder_path + .as_ref() + .unwrap() // must be set if storage key resolver returns a value + .join(checksum::gen(&[key.as_bytes()])) + }); + let cache_storage_dir = maybe_storage_key.map(|key| { + // TODO(@satyarohith): storage quota management + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) + }); + + let services = WorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + npm_process_state_provider: Some( + shared.npm_process_state_provider.clone(), + ), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + fetch_dns_resolver: Default::default(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + feature_checker, + permissions, + v8_code_cache: shared.code_cache.clone(), + }; + + let options = WorkerOptions { + bootstrap: BootstrapOptions { + deno_version: shared.options.deno_version.to_string(), + args: shared.options.argv.clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + log_level: shared.options.log_level, + enable_op_summary_metrics: shared.options.enable_op_summary_metrics, + enable_testing_features: shared.options.enable_testing_features, + locale: deno_core::v8::icu::get_language_tag(), + location: shared.options.location.clone(), + no_color: !colors::use_color(), + is_stdout_tty: deno_terminal::is_stdout_tty(), + is_stderr_tty: deno_terminal::is_stderr_tty(), + color_level: colors::get_color_level(), + unstable_features, + user_agent: shared.options.deno_user_agent.to_string(), + inspect: shared.options.is_inspecting, + has_node_modules_dir: shared.options.has_node_modules_dir, + argv0: shared.options.argv0.clone(), + node_debug: shared.options.node_debug.clone(), + node_ipc_fd: shared.options.node_ipc, + mode, + serve_port: shared.options.serve_port, + serve_host: shared.options.serve_host.clone(), + otel_config: shared.options.otel_config.clone(), + close_on_idle: true, + }, + extensions: custom_extensions, + startup_snapshot: shared.options.startup_snapshot, + create_params: create_isolate_create_params(), + unsafely_ignore_certificate_errors: shared + .options + .unsafely_ignore_certificate_errors + .clone(), + seed: shared.options.seed, + format_js_error_fn: Some(Arc::new(format_js_error)), + create_web_worker_cb: shared.create_web_worker_callback(stdio.clone()), + maybe_inspector_server: shared.maybe_inspector_server.clone(), + should_break_on_first_statement: shared.options.inspect_brk, + should_wait_for_inspector_session: shared.options.inspect_wait, + strace_ops: shared.options.strace_ops.clone(), + cache_storage_dir, + origin_storage_dir, + stdio, + skip_op_registration: shared.options.skip_op_registration, + enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(), + }; + + let worker = + MainWorker::bootstrap_from_options(&main_module, services, options); + + Ok(LibMainWorker { + main_module, + worker, + }) + } +} + +pub struct LibMainWorker { + main_module: Url, + worker: MainWorker, +} + +impl LibMainWorker { + pub fn into_main_worker(self) -> MainWorker { + self.worker + } + + pub fn main_module(&self) -> &Url { + &self.main_module + } + + pub fn js_runtime(&mut self) -> &mut JsRuntime { + &mut self.worker.js_runtime + } + + #[inline] + pub fn create_inspector_session(&mut self) -> LocalInspectorSession { + self.worker.create_inspector_session() + } + + #[inline] + pub fn dispatch_load_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_load_event() + } + + #[inline] + pub fn dispatch_beforeunload_event(&mut self) -> Result { + self.worker.dispatch_beforeunload_event() + } + + #[inline] + pub fn dispatch_process_beforeexit_event(&mut self) -> Result { + self.worker.dispatch_process_beforeexit_event() + } + + #[inline] + pub fn dispatch_unload_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_unload_event() + } + + #[inline] + pub fn dispatch_process_exit_event(&mut self) -> Result<(), JsError> { + self.worker.dispatch_process_exit_event() + } + + pub async fn execute_main_module(&mut self) -> Result<(), CoreError> { + let id = self.worker.preload_main_module(&self.main_module).await?; + self.worker.evaluate_module(id).await + } + + pub async fn execute_side_module(&mut self) -> Result<(), CoreError> { + let id = self.worker.preload_side_module(&self.main_module).await?; + self.worker.evaluate_module(id).await + } + + #[inline] + pub async fn run_event_loop( + &mut self, + wait_for_inspector: bool, + ) -> Result<(), CoreError> { + self.worker.run_event_loop(wait_for_inspector).await + } + + #[inline] + pub fn exit_code(&self) -> i32 { + self.worker.exit_code() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn storage_key_resolver_test() { + let resolver = + StorageKeyResolver(StorageKeyResolverStrategy::UseMainModule); + let specifier = Url::parse("file:///a.ts").unwrap(); + assert_eq!( + resolver.resolve_storage_key(&specifier), + Some(specifier.to_string()) + ); + let resolver = + StorageKeyResolver(StorageKeyResolverStrategy::Specified(None)); + assert_eq!(resolver.resolve_storage_key(&specifier), None); + let resolver = StorageKeyResolver(StorageKeyResolverStrategy::Specified( + Some("value".to_string()), + )); + assert_eq!( + resolver.resolve_storage_key(&specifier), + Some("value".to_string()) + ); + + // test empty + let resolver = StorageKeyResolver::empty(); + assert_eq!(resolver.resolve_storage_key(&specifier), None); + } +} diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index 97fbbaff14..a65bbd5efe 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -8,9 +8,9 @@ use std::time::SystemTime; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_lib::cache::DenoDir; use deno_path_util::url_to_file_path; -use crate::cache::DenoDir; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; use crate::cache::LocalLspHttpCache; @@ -70,7 +70,7 @@ fn calculate_fs_version_in_cache( #[derive(Debug, Clone)] pub struct LspCache { - deno_dir: DenoDir, + deno_dir: DenoDir, global: Arc, vendors_by_scope: BTreeMap>>, } @@ -121,7 +121,7 @@ impl LspCache { .collect(); } - pub fn deno_dir(&self) -> &DenoDir { + pub fn deno_dir(&self) -> &DenoDir { &self.deno_dir } diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index ba57502298..98c4498a1a 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -41,6 +41,7 @@ use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_lib::env::has_flag_env_var; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; @@ -55,7 +56,6 @@ use super::logging::lsp_log; use super::lsp_custom; use super::urls::url_to_uri; use crate::args::discover_npmrc_from_workspace; -use crate::args::has_flag_env_var; use crate::args::CliLockfile; use crate::args::CliLockfileReadFromPathOptions; use crate::args::ConfigFile; diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index fe8dc4c8d5..3e3e31de28 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -265,7 +265,7 @@ impl TsDiagnosticsStore { } pub fn should_send_diagnostic_batch_index_notifications() -> bool { - crate::args::has_flag_env_var( + deno_lib::env::has_flag_env_var( "DENO_DONT_USE_INTERNAL_LSP_DIAGNOSTIC_SYNC_FLAG", ) } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 0f3bfcdf59..c2fddc08bd 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -27,6 +27,7 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; +use deno_lib::env::has_flag_env_var; use deno_path_util::url_to_file_path; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; @@ -95,7 +96,6 @@ use super::urls::uri_to_url; use super::urls::url_to_uri; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; -use crate::args::has_flag_env_var; use crate::args::CaData; use crate::args::CliOptions; use crate::args::Flags; diff --git a/cli/lsp/testing/definitions.rs b/cli/lsp/testing/definitions.rs index 8277dcbf00..d6630c1844 100644 --- a/cli/lsp/testing/definitions.rs +++ b/cli/lsp/testing/definitions.rs @@ -5,6 +5,7 @@ use std::collections::HashSet; use deno_core::error::AnyError; use deno_core::ModuleSpecifier; +use deno_lib::util::checksum; use lsp::Range; use tower_lsp::lsp_types as lsp; @@ -15,7 +16,6 @@ use crate::lsp::logging::lsp_warn; use crate::lsp::urls::url_to_uri; use crate::tools::test::TestDescription; use crate::tools::test::TestStepDescription; -use crate::util::checksum; #[derive(Debug, Clone, PartialEq)] pub struct TestDefinition { diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index e2a0fc430d..0b53dc8506 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -39,6 +39,7 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; use deno_core::RuntimeOptions; +use deno_lib::worker::create_isolate_create_params; use deno_path_util::url_to_file_path; use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES; use deno_runtime::inspector_server::InspectorServer; @@ -96,7 +97,6 @@ use crate::util::path::relative_specifier; use crate::util::path::to_percent_decoded_str; use crate::util::result::InfallibleResultExt; use crate::util::v8::convert; -use crate::worker::create_isolate_create_params; static BRACKET_ACCESSOR_RE: Lazy = lazy_regex!(r#"^\[['"](.+)[\['"]\]$"#); diff --git a/cli/lsp/urls.rs b/cli/lsp/urls.rs index 2aadaf5352..068e4ad4d5 100644 --- a/cli/lsp/urls.rs +++ b/cli/lsp/urls.rs @@ -81,7 +81,7 @@ fn hash_data_specifier(specifier: &ModuleSpecifier) -> String { file_name_str.push('?'); file_name_str.push_str(query); } - crate::util::checksum::gen(&[file_name_str.as_bytes()]) + deno_lib::util::checksum::gen(&[file_name_str.as_bytes()]) } fn to_deno_uri(specifier: &Url) -> String { diff --git a/cli/module_loader.rs b/cli/module_loader.rs index daeb4dda37..2b0ebca986 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -39,6 +39,9 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::Resolution; use deno_graph::WasmModule; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::worker::CreateModuleLoaderResult; +use deno_lib::worker::ModuleLoaderFactory; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::code_cache; use deno_runtime::deno_node::create_host_defined_options; @@ -70,7 +73,6 @@ use crate::graph_util::ModuleGraphBuilder; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; -use crate::npm::NpmRegistryReadPermissionChecker; use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; @@ -84,8 +86,6 @@ use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; -use crate::worker::CreateModuleLoaderResult; -use crate::worker::ModuleLoaderFactory; #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum PrepareModuleLoadError { @@ -243,7 +243,8 @@ struct SharedCliModuleLoaderState { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: + Arc>, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, parsed_source_cache: Arc, @@ -304,7 +305,9 @@ impl CliModuleLoaderFactory { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: Arc< + NpmRegistryReadPermissionChecker, + >, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, parsed_source_cache: Arc, @@ -1145,7 +1148,8 @@ struct CliNodeRequireLoader { sys: CliSys, graph_container: TGraphContainer, in_npm_pkg_checker: DenoInNpmPackageChecker, - npm_registry_permission_checker: Arc, + npm_registry_permission_checker: + Arc>, } impl NodeRequireLoader diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 1c12ce6c59..fc0916cc18 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -3,7 +3,6 @@ mod byonm; pub mod installer; mod managed; -mod permission_checker; use std::sync::Arc; @@ -24,8 +23,6 @@ pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::NpmResolutionInitializer; pub use self::managed::ResolveSnapshotError; -pub use self::permission_checker::NpmRegistryReadPermissionChecker; -pub use self::permission_checker::NpmRegistryReadPermissionCheckerMode; use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; use crate::sys::CliSys; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index c9b57f3d6b..5334b4719d 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -38,6 +38,13 @@ use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; use deno_graph::ModuleGraph; +use deno_lib::cache::DenoDir; +use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::VfsEntry; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; +use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; use deno_npm::resolution::SerializedNpmResolutionSnapshot; use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; @@ -73,20 +80,14 @@ use super::serialization::SourceMapStore; use super::virtual_fs::output_vfs; use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; -use super::virtual_fs::FileSystemCaseSensitivity; use super::virtual_fs::VfsBuilder; -use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; -use super::virtual_fs::VirtualDirectory; -use super::virtual_fs::VirtualDirectoryEntries; -use super::virtual_fs::WindowsSystemRootablePath; use crate::args::CaData; use crate::args::CliOptions; use crate::args::CompileFlags; use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; -use crate::cache::DenoDir; use crate::cache::FastInsecureHasher; use crate::emit::Emitter; use crate::file_fetcher::CliFileFetcher; @@ -94,7 +95,7 @@ use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::shared::ReleaseChannel; -use crate::standalone::virtual_fs::VfsEntry; +use crate::sys::CliSys; use crate::util::archive; use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -411,7 +412,7 @@ pub struct WriteBinOptions<'a> { pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, @@ -425,7 +426,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub fn new( cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index b04db88c90..c4b3ebe728 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; use deno_runtime::deno_fs::AccessCheckCb; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::FsDirEntry; @@ -30,7 +31,6 @@ use super::virtual_fs::FileBackedVfs; use super::virtual_fs::FileBackedVfsDirEntry; use super::virtual_fs::FileBackedVfsFile; use super::virtual_fs::FileBackedVfsMetadata; -use super::virtual_fs::VfsFileSubDataKind; #[derive(Debug, Clone)] pub struct DenoCompileFileSystem(Arc); diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 876c194ed1..f2a0859e8f 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -36,6 +36,15 @@ use deno_core::RequestedModuleType; use deno_core::ResolutionKind; use deno_core::SourceCodeCacheInfo; use deno_error::JsErrorBox; +use deno_lib::cache::DenoDirProvider; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::worker::CreateModuleLoaderResult; +use deno_lib::worker::LibMainWorkerFactory; +use deno_lib::worker::LibMainWorkerOptions; +use deno_lib::worker::ModuleLoaderFactory; +use deno_lib::worker::StorageKeyResolver; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_package_json::PackageJsonDepValue; @@ -69,16 +78,13 @@ use node_resolver::ResolutionMode; use serialization::DenoCompileModuleSource; use serialization::SourceMapStore; use virtual_fs::FileBackedVfs; -use virtual_fs::VfsFileSubDataKind; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; use crate::args::npm_pkg_req_ref_to_binary_command; use crate::args::CaData; use crate::args::NpmInstallDepsProvider; -use crate::args::StorageKeyResolver; use crate::cache::Caches; -use crate::cache::DenoDirProvider; use crate::cache::FastInsecureHasher; use crate::cache::NodeAnalysisCache; use crate::http_util::HttpClientProvider; @@ -86,13 +92,12 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; +use crate::npm::create_npm_process_state_provider; use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::NpmRegistryReadPermissionChecker; -use crate::npm::NpmRegistryReadPermissionCheckerMode; use crate::npm::NpmResolutionInitializer; use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; @@ -105,8 +110,6 @@ use crate::util::v8::construct_v8_flags; use crate::worker::CliCodeCache; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; -use crate::worker::CreateModuleLoaderResult; -use crate::worker::ModuleLoaderFactory; pub mod binary; mod code_cache; @@ -129,7 +132,7 @@ struct SharedModuleLoaderState { node_code_translator: Arc, node_resolver: Arc, npm_module_loader: Arc, - npm_registry_permission_checker: NpmRegistryReadPermissionChecker, + npm_registry_permission_checker: NpmRegistryReadPermissionChecker, npm_req_resolver: Arc, npm_resolver: CliNpmResolver, source_maps: SourceMapStore, @@ -962,54 +965,67 @@ pub async fn run( } checker }); - let worker_factory = CliMainWorkerFactory::new( + let lib_main_worker_options = LibMainWorkerOptions { + argv: metadata.argv, + log_level: WorkerLogLevel::Info, + enable_op_summary_metrics: false, + enable_testing_features: false, + has_node_modules_dir, + inspect_brk: false, + inspect_wait: false, + strace_ops: None, + is_inspecting: false, + skip_op_registration: true, + location: metadata.location, + argv0: NpmPackageReqReference::from_specifier(&main_module) + .ok() + .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) + .or(std::env::args().next()), + node_debug: std::env::var("NODE_DEBUG").ok(), + origin_data_folder_path: None, + seed: metadata.seed, + unsafely_ignore_certificate_errors: metadata + .unsafely_ignore_certificate_errors, + node_ipc: None, + serve_port: None, + serve_host: None, + deno_version: crate::version::DENO_VERSION_INFO.deno, + deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, + otel_config: metadata.otel_config, + startup_snapshot: crate::js::deno_isolate_init(), + }; + let lib_main_worker_factory = LibMainWorkerFactory::new( Arc::new(BlobStore::default()), - code_cache, + code_cache.map(|c| c.as_code_cache()), feature_checker, fs, None, - None, - None, Box::new(module_loader_factory), + node_resolver.clone(), + create_npm_process_state_provider(&npm_resolver), + pkg_json_resolver, + root_cert_store_provider, + StorageKeyResolver::empty(), + sys.clone(), + lib_main_worker_options, + ); + // todo(dsherret): use LibMainWorker directly here and don't use CliMainWorkerFactory + let cli_main_worker_options = CliMainWorkerOptions { + create_hmr_runner: None, + create_coverage_collector: None, + needs_test_modules: false, + default_npm_caching_strategy: crate::args::NpmCachingStrategy::Lazy, + }; + let worker_factory = CliMainWorkerFactory::new( + lib_main_worker_factory, + None, + None, node_resolver, None, npm_resolver, - pkg_json_resolver, - root_cert_store_provider, - permissions, - StorageKeyResolver::empty(), sys, - crate::args::DenoSubcommand::Run(Default::default()), - CliMainWorkerOptions { - argv: metadata.argv, - log_level: WorkerLogLevel::Info, - enable_op_summary_metrics: false, - enable_testing_features: false, - has_node_modules_dir, - hmr: false, - inspect_brk: false, - inspect_wait: false, - strace_ops: None, - is_inspecting: false, - skip_op_registration: true, - location: metadata.location, - argv0: NpmPackageReqReference::from_specifier(&main_module) - .ok() - .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) - .or(std::env::args().next()), - node_debug: std::env::var("NODE_DEBUG").ok(), - origin_data_folder_path: None, - seed: metadata.seed, - unsafely_ignore_certificate_errors: metadata - .unsafely_ignore_certificate_errors, - create_hmr_runner: None, - create_coverage_collector: None, - node_ipc: None, - serve_port: None, - serve_host: None, - }, - metadata.otel_config, - crate::args::NpmCachingStrategy::Lazy, + cli_main_worker_options, + permissions, ); // Initialize v8 once from the main thread. diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index ae2e411b5f..ab345917a3 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -17,6 +17,7 @@ use deno_core::url::Url; use deno_core::FastString; use deno_core::ModuleSourceCode; use deno_core::ModuleType; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; use deno_npm::resolution::SerializedNpmResolutionSnapshot; use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; @@ -27,10 +28,7 @@ use indexmap::IndexMap; use super::binary::Metadata; use super::virtual_fs::BuiltVfs; -use super::virtual_fs::FileSystemCaseSensitivity; use super::virtual_fs::VfsBuilder; -use super::virtual_fs::VirtualDirectoryEntries; -use crate::standalone::virtual_fs::VirtualDirectory; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 0b1f33259d..4f761d0d15 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -23,6 +23,17 @@ use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::OffsetWithLength; +use deno_lib::standalone::virtual_fs::VfsEntry; +use deno_lib::standalone::virtual_fs::VfsEntryRef; +use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; +use deno_lib::standalone::virtual_fs::VirtualFile; +use deno_lib::standalone::virtual_fs::VirtualSymlink; +use deno_lib::standalone::virtual_fs::VirtualSymlinkParts; +use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; use deno_path_util::normalize_path; use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; @@ -41,37 +52,6 @@ use crate::util::display::human_size; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; -#[derive(Debug, PartialEq, Eq)] -pub enum WindowsSystemRootablePath { - /// The root of the system above any drive letters. - WindowSystemRoot, - Path(PathBuf), -} - -impl WindowsSystemRootablePath { - pub fn join(&self, name_component: &str) -> PathBuf { - // this method doesn't handle multiple components - debug_assert!( - !name_component.contains('\\'), - "Invalid component: {}", - name_component - ); - debug_assert!( - !name_component.contains('/'), - "Invalid component: {}", - name_component - ); - - match self { - WindowsSystemRootablePath::WindowSystemRoot => { - // windows drive letter - PathBuf::from(&format!("{}\\", name_component)) - } - WindowsSystemRootablePath::Path(path) => path.join(name_component), - } - } -} - #[derive(Debug)] pub struct BuiltVfs { pub root_path: WindowsSystemRootablePath, @@ -80,15 +60,6 @@ pub struct BuiltVfs { pub files: Vec>, } -#[derive(Debug, Copy, Clone)] -pub enum VfsFileSubDataKind { - /// Raw bytes of the file. - Raw, - /// Bytes to use for module loading. For example, for TypeScript - /// files this will be the transpiled JavaScript source. - ModuleGraph, -} - #[derive(Debug)] pub struct VfsBuilder { executable_root: VirtualDirectory, @@ -232,24 +203,21 @@ impl VfsBuilder { continue; } let name = component.as_os_str().to_string_lossy(); - let index = match current_dir - .entries - .binary_search(&name, self.case_sensitivity) - { - Ok(index) => index, - Err(insert_index) => { - current_dir.entries.0.insert( - insert_index, - VfsEntry::Dir(VirtualDirectory { - name: name.to_string(), - entries: Default::default(), - }), - ); - insert_index - } - }; - match &mut current_dir.entries.0[index] { - VfsEntry::Dir(dir) => { + let index = current_dir.entries.insert_or_modify( + &name, + self.case_sensitivity, + || { + VfsEntry::Dir(VirtualDirectory { + name: name.to_string(), + entries: Default::default(), + }) + }, + |_| { + // ignore + }, + ); + match current_dir.entries.get_mut_by_index(index) { + Some(VfsEntry::Dir(dir)) => { current_dir = dir; } _ => unreachable!(), @@ -325,7 +293,7 @@ impl VfsBuilder { sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { log::debug!("Adding file '{}'", path.display()); - let checksum = util::checksum::gen(&[&data]); + let checksum = deno_lib::util::checksum::gen(&[&data]); let case_sensitivity = self.case_sensitivity; let offset = if let Some(offset) = self.file_offsets.get(&checksum) { // duplicate file, reuse an old offset @@ -341,32 +309,28 @@ impl VfsBuilder { offset, len: data.len() as u64, }; - match dir.entries.binary_search(&name, case_sensitivity) { - Ok(index) => { - let entry = &mut dir.entries.0[index]; - match entry { - VfsEntry::File(virtual_file) => match sub_data_kind { - VfsFileSubDataKind::Raw => { - virtual_file.offset = offset_and_len; - } - VfsFileSubDataKind::ModuleGraph => { - virtual_file.module_graph_offset = offset_and_len; - } - }, - VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), - } - } - Err(insert_index) => { - dir.entries.0.insert( - insert_index, - VfsEntry::File(VirtualFile { - name: name.to_string(), - offset: offset_and_len, - module_graph_offset: offset_and_len, - }), - ); - } - } + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::File(VirtualFile { + name: name.to_string(), + offset: offset_and_len, + module_graph_offset: offset_and_len, + }) + }, + |entry| match entry { + VfsEntry::File(virtual_file) => match sub_data_kind { + VfsFileSubDataKind::Raw => { + virtual_file.offset = offset_and_len; + } + VfsFileSubDataKind::ModuleGraph => { + virtual_file.module_graph_offset = offset_and_len; + } + }, + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), + }, + ); // new file, update the list of files if self.current_offset == offset { @@ -406,18 +370,19 @@ impl VfsBuilder { let target = normalize_path(path.parent().unwrap().join(&target)); let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); - match dir.entries.binary_search(&name, case_sensitivity) { - Ok(_) => {} // previously inserted - Err(insert_index) => { - dir.entries.0.insert( - insert_index, - VfsEntry::Symlink(VirtualSymlink { - name: name.to_string(), - dest_parts: VirtualSymlinkParts::from_path(&target), - }), - ); - } - } + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::Symlink(VirtualSymlink { + name: name.to_string(), + dest_parts: VirtualSymlinkParts::from_path(&target), + }) + }, + |_| { + // ignore previously inserted + }, + ); let target_metadata = std::fs::symlink_metadata(&target).with_context(|| { format!("Reading symlink target '{}'", target.display()) @@ -448,16 +413,20 @@ impl VfsBuilder { dir: &mut VirtualDirectory, parts: &[String], ) { - for entry in &mut dir.entries.0 { + for entry in dir.entries.iter_mut() { match entry { VfsEntry::Dir(dir) => { strip_prefix_from_symlinks(dir, parts); } VfsEntry::File(_) => {} VfsEntry::Symlink(symlink) => { - let old_parts = std::mem::take(&mut symlink.dest_parts.0); - symlink.dest_parts.0 = - old_parts.into_iter().skip(parts.len()).collect(); + let parts = symlink + .dest_parts + .take_parts() + .into_iter() + .skip(parts.len()) + .collect(); + symlink.dest_parts.set_parts(parts); } } } @@ -476,13 +445,13 @@ impl VfsBuilder { if self.min_root_dir.as_ref() == Some(¤t_path) { break; } - match ¤t_dir.entries.0[0] { + match current_dir.entries.iter().next().unwrap() { VfsEntry::Dir(dir) => { if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { // special directory we want to maintain break; } - match current_dir.entries.0.remove(0) { + match current_dir.entries.remove(0) { VfsEntry::Dir(dir) => { current_path = WindowsSystemRootablePath::Path(current_path.join(&dir.name)); @@ -497,7 +466,7 @@ impl VfsBuilder { if let WindowsSystemRootablePath::Path(path) = ¤t_path { strip_prefix_from_symlinks( &mut current_dir, - &VirtualSymlinkParts::from_path(path).0, + VirtualSymlinkParts::from_path(path).parts(), ); } BuiltVfs { @@ -577,7 +546,7 @@ fn vfs_as_display_tree( All(Size), Subset(Vec>), File(Size), - Symlink(&'a [String]), + Symlink(&'a VirtualSymlinkParts), } impl<'a> EntryOutput<'a> { @@ -626,7 +595,7 @@ fn vfs_as_display_tree( format!("{} ({})", name, format_size(*size)) } EntryOutput::Symlink(parts) => { - format!("{} --> {}", name, parts.join("/")) + format!("{} --> {}", name, parts.display()) } }, children: match self { @@ -769,7 +738,7 @@ fn vfs_as_display_tree( EntryOutput::File(file_size(file, seen_offsets)) } VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + EntryOutput::Symlink(&virtual_symlink.dest_parts) } }, }) @@ -806,7 +775,7 @@ fn vfs_as_display_tree( } VfsEntry::File(file) => EntryOutput::File(file_size(file, seen_offsets)), VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + EntryOutput::Symlink(&virtual_symlink.dest_parts) } } } @@ -872,226 +841,6 @@ fn vfs_as_display_tree( } } -#[derive(Debug)] -enum VfsEntryRef<'a> { - Dir(&'a VirtualDirectory), - File(&'a VirtualFile), - Symlink(&'a VirtualSymlink), -} - -impl VfsEntryRef<'_> { - pub fn as_metadata(&self) -> FileBackedVfsMetadata { - FileBackedVfsMetadata { - file_type: match self { - Self::Dir(_) => sys_traits::FileType::Dir, - Self::File(_) => sys_traits::FileType::File, - Self::Symlink(_) => sys_traits::FileType::Symlink, - }, - name: self.name().to_string(), - len: match self { - Self::Dir(_) => 0, - Self::File(file) => file.offset.len, - Self::Symlink(_) => 0, - }, - } - } - - pub fn name(&self) -> &str { - match self { - Self::Dir(dir) => &dir.name, - Self::File(file) => &file.name, - Self::Symlink(symlink) => &symlink.name, - } - } -} - -// todo(dsherret): we should store this more efficiently in the binary -#[derive(Debug, Serialize, Deserialize)] -pub enum VfsEntry { - Dir(VirtualDirectory), - File(VirtualFile), - Symlink(VirtualSymlink), -} - -impl VfsEntry { - pub fn name(&self) -> &str { - match self { - Self::Dir(dir) => &dir.name, - Self::File(file) => &file.name, - Self::Symlink(symlink) => &symlink.name, - } - } - - fn as_ref(&self) -> VfsEntryRef { - match self { - VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir), - VfsEntry::File(file) => VfsEntryRef::File(file), - VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink), - } - } -} - -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -pub enum FileSystemCaseSensitivity { - #[serde(rename = "s")] - Sensitive, - #[serde(rename = "i")] - Insensitive, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct VirtualDirectoryEntries(Vec); - -impl VirtualDirectoryEntries { - pub fn new(mut entries: Vec) -> Self { - // needs to be sorted by name - entries.sort_by(|a, b| a.name().cmp(b.name())); - Self(entries) - } - - pub fn take_inner(&mut self) -> Vec { - std::mem::take(&mut self.0) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn get_by_name( - &self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Option<&VfsEntry> { - self - .binary_search(name, case_sensitivity) - .ok() - .map(|index| &self.0[index]) - } - - pub fn get_mut_by_name( - &mut self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Option<&mut VfsEntry> { - self - .binary_search(name, case_sensitivity) - .ok() - .map(|index| &mut self.0[index]) - } - - pub fn binary_search( - &self, - name: &str, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Result { - match case_sensitivity { - FileSystemCaseSensitivity::Sensitive => { - self.0.binary_search_by(|e| e.name().cmp(name)) - } - FileSystemCaseSensitivity::Insensitive => self.0.binary_search_by(|e| { - e.name() - .chars() - .zip(name.chars()) - .map(|(a, b)| a.to_ascii_lowercase().cmp(&b.to_ascii_lowercase())) - .find(|&ord| ord != Ordering::Equal) - .unwrap_or_else(|| e.name().len().cmp(&name.len())) - }), - } - } - - pub fn insert( - &mut self, - entry: VfsEntry, - case_sensitivity: FileSystemCaseSensitivity, - ) { - match self.binary_search(entry.name(), case_sensitivity) { - Ok(index) => { - self.0[index] = entry; - } - Err(insert_index) => { - self.0.insert(insert_index, entry); - } - } - } - - pub fn remove(&mut self, index: usize) -> VfsEntry { - self.0.remove(index) - } - - pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> { - self.0.iter() - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualDirectory { - #[serde(rename = "n")] - pub name: String, - // should be sorted by name - #[serde(rename = "e")] - pub entries: VirtualDirectoryEntries, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct OffsetWithLength { - #[serde(rename = "o")] - pub offset: u64, - #[serde(rename = "l")] - pub len: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VirtualFile { - #[serde(rename = "n")] - pub name: String, - #[serde(rename = "o")] - pub offset: OffsetWithLength, - /// Offset file to use for module loading when it differs from the - /// raw file. Often this will be the same offset as above for data - /// such as JavaScript files, but for TypeScript files the `offset` - /// will be the original raw bytes when included as an asset and this - /// offset will be to the transpiled JavaScript source. - #[serde(rename = "m")] - pub module_graph_offset: OffsetWithLength, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualSymlinkParts(Vec); - -impl VirtualSymlinkParts { - pub fn from_path(path: &Path) -> Self { - Self( - path - .components() - .filter(|c| !matches!(c, std::path::Component::RootDir)) - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect(), - ) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VirtualSymlink { - #[serde(rename = "n")] - pub name: String, - #[serde(rename = "p")] - pub dest_parts: VirtualSymlinkParts, -} - -impl VirtualSymlink { - pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { - let mut dest = root.to_path_buf(); - for part in &self.dest_parts.0 { - dest.push(part); - } - dest - } -} - #[derive(Debug)] pub struct VfsRoot { pub dir: VirtualDirectory, @@ -1430,6 +1179,21 @@ pub struct FileBackedVfsMetadata { } impl FileBackedVfsMetadata { + pub fn from_vfs_entry_ref(vfs_entry: VfsEntryRef) -> Self { + FileBackedVfsMetadata { + file_type: match vfs_entry { + VfsEntryRef::Dir(_) => sys_traits::FileType::Dir, + VfsEntryRef::File(_) => sys_traits::FileType::File, + VfsEntryRef::Symlink(_) => sys_traits::FileType::Symlink, + }, + name: vfs_entry.name().to_string(), + len: match vfs_entry { + VfsEntryRef::Dir(_) => 0, + VfsEntryRef::File(file) => file.offset.len, + VfsEntryRef::Symlink(_) => 0, + }, + } + } pub fn as_fs_stat(&self) -> FsStat { FsStat { is_directory: self.file_type == sys_traits::FileType::Dir, @@ -1521,7 +1285,7 @@ impl FileBackedVfs { let path = path.to_path_buf(); Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry { parent_path: path.to_path_buf(), - metadata: entry.as_ref().as_metadata(), + metadata: FileBackedVfsMetadata::from_vfs_entry_ref(entry.as_ref()), })) } @@ -1544,12 +1308,12 @@ impl FileBackedVfs { let (_, entry) = self .fs_root .find_entry_no_follow(path, self.case_sensitivity)?; - Ok(entry.as_metadata()) + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) } pub fn stat(&self, path: &Path) -> std::io::Result { let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; - Ok(entry.as_metadata()) + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) } pub fn canonicalize(&self, path: &Path) -> std::io::Result { diff --git a/cli/sys.rs b/cli/sys.rs index 718e9981e2..e551eab2e8 100644 --- a/cli/sys.rs +++ b/cli/sys.rs @@ -29,6 +29,8 @@ pub enum CliSys { DenoCompile(DenoCompileFileSystem), } +impl deno_lib::sys::DenoLibSys for CliSys {} + impl Default for CliSys { fn default() -> Self { Self::Real(sys_traits::impls::RealSys) diff --git a/cli/tools/clean.rs b/cli/tools/clean.rs index e6f8c1e52b..a550d2826a 100644 --- a/cli/tools/clean.rs +++ b/cli/tools/clean.rs @@ -4,8 +4,8 @@ use std::path::Path; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_lib::cache::DenoDir; -use crate::cache::DenoDir; use crate::colors; use crate::display; use crate::sys::CliSys; diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 8c3b2665c5..1b2542d427 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -18,6 +18,7 @@ use deno_graph::Module; use deno_graph::ModuleError; use deno_graph::ModuleGraph; use deno_graph::Resolution; +use deno_lib::util::checksum; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::NpmPackageId; @@ -33,7 +34,6 @@ use crate::display; use crate::factory::CliFactory; use crate::graph_util::graph_exit_integrity_errors; use crate::npm::CliManagedNpmResolver; -use crate::util::checksum; use crate::util::display::DisplayTreeNode; const JSON_SCHEMA_VERSION: u8 = 1; @@ -191,7 +191,7 @@ fn print_cache_info( let registry_cache = dir.registries_folder_path(); let mut origin_dir = dir.origin_data_folder_path(); let deno_dir = dir.root_path_for_display().to_string(); - let web_cache_dir = crate::worker::get_cache_storage_dir(); + let web_cache_dir = deno_lib::worker::get_cache_storage_dir(); if let Some(location) = &location { origin_dir = diff --git a/cli/tools/serve.rs b/cli/tools/serve.rs index c2c53c1b69..2143eb33bb 100644 --- a/cli/tools/serve.rs +++ b/cli/tools/serve.rs @@ -43,7 +43,8 @@ pub async fn serve( maybe_npm_install(&factory).await?; - let worker_factory = factory.create_cli_main_worker_factory().await?; + let worker_factory = + Arc::new(factory.create_cli_main_worker_factory().await?); let hmr = serve_flags .watch .map(|watch_flags| watch_flags.hmr) @@ -58,7 +59,7 @@ pub async fn serve( } async fn do_serve( - worker_factory: CliMainWorkerFactory, + worker_factory: Arc, main_module: ModuleSpecifier, worker_count: Option, hmr: bool, @@ -116,7 +117,7 @@ async fn do_serve( async fn run_worker( worker_count: usize, - worker_factory: CliMainWorkerFactory, + worker_factory: Arc, main_module: ModuleSpecifier, hmr: bool, ) -> Result { @@ -164,7 +165,8 @@ async fn serve_with_watch( maybe_npm_install(&factory).await?; let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); - let worker_factory = factory.create_cli_main_worker_factory().await?; + let worker_factory = + Arc::new(factory.create_cli_main_worker_factory().await?); do_serve(worker_factory, main_module.clone(), worker_count, hmr) .await?; diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 37d52c6cf2..1b76b640d3 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -28,6 +28,8 @@ use deno_graph::GraphKind; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; +use deno_lib::util::checksum; +use deno_lib::worker::create_isolate_create_params; use deno_resolver::npm::managed::ResolvePkgFolderFromDenoModuleError; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; use deno_semver::npm::NpmPackageReqReference; @@ -48,9 +50,7 @@ use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::sys::CliSys; -use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; -use crate::worker::create_isolate_create_params; mod diagnostics; diff --git a/cli/util/file_watcher.rs b/cli/util/file_watcher.rs index 65963214b9..d3ff1bae77 100644 --- a/cli/util/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -25,6 +25,7 @@ use notify::Watcher; use tokio::select; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc; +use tokio::sync::mpsc::error::SendError; use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::sleep; @@ -141,36 +142,60 @@ fn create_print_after_restart_fn(clear_screen: bool) -> impl Fn() { } } +#[derive(Debug)] +pub struct WatcherCommunicatorOptions { + /// Send a list of paths that should be watched for changes. + pub paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender>, + /// Listen for a list of paths that were changed. + pub changed_paths_rx: tokio::sync::broadcast::Receiver>>, + pub changed_paths_tx: tokio::sync::broadcast::Sender>>, + /// Send a message to force a restart. + pub restart_tx: tokio::sync::mpsc::UnboundedSender<()>, + pub restart_mode: WatcherRestartMode, + pub banner: String, +} + /// An interface to interact with Deno's CLI file watcher. #[derive(Debug)] pub struct WatcherCommunicator { /// Send a list of paths that should be watched for changes. paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender>, - /// Listen for a list of paths that were changed. changed_paths_rx: tokio::sync::broadcast::Receiver>>, - + changed_paths_tx: tokio::sync::broadcast::Sender>>, /// Send a message to force a restart. restart_tx: tokio::sync::mpsc::UnboundedSender<()>, - restart_mode: Mutex, - banner: String, } impl WatcherCommunicator { - pub fn watch_paths(&self, paths: Vec) -> Result<(), AnyError> { + pub fn new(options: WatcherCommunicatorOptions) -> Self { + Self { + paths_to_watch_tx: options.paths_to_watch_tx, + changed_paths_rx: options.changed_paths_rx, + changed_paths_tx: options.changed_paths_tx, + restart_tx: options.restart_tx, + restart_mode: Mutex::new(options.restart_mode), + banner: options.banner, + } + } + + pub fn watch_paths( + &self, + paths: Vec, + ) -> Result<(), SendError>> { if paths.is_empty() { return Ok(()); } - self.paths_to_watch_tx.send(paths).map_err(AnyError::from) + self.paths_to_watch_tx.send(paths) } - pub fn force_restart(&self) -> Result<(), AnyError> { + pub fn force_restart(&self) -> Result<(), SendError<()>> { // Change back to automatic mode, so that HMR can set up watching // from scratch. *self.restart_mode.lock() = WatcherRestartMode::Automatic; - self.restart_tx.send(()).map_err(AnyError::from) + self.restart_tx.send(()) } pub async fn watch_for_changed_paths( @@ -184,6 +209,22 @@ impl WatcherCommunicator { *self.restart_mode.lock() = restart_mode; } + pub fn send( + &self, + paths: Option>, + ) -> Result<(), SendError>>> { + match *self.restart_mode.lock() { + WatcherRestartMode::Automatic => { + self.restart_tx.send(()).map_err(|_| SendError(None)) + } + WatcherRestartMode::Manual => self + .changed_paths_tx + .send(paths) + .map(|_| ()) + .map_err(|e| SendError(e.0)), + } + } + pub fn print(&self, msg: String) { log::info!("{} {}", self.banner, colors::gray(msg)); } @@ -272,13 +313,15 @@ where } = print_config; let print_after_restart = create_print_after_restart_fn(clear_screen); - let watcher_communicator = Arc::new(WatcherCommunicator { - paths_to_watch_tx: paths_to_watch_tx.clone(), - changed_paths_rx: changed_paths_rx.resubscribe(), - restart_tx: restart_tx.clone(), - restart_mode: Mutex::new(restart_mode), - banner: colors::intense_blue(banner).to_string(), - }); + let watcher_communicator = + Arc::new(WatcherCommunicator::new(WatcherCommunicatorOptions { + paths_to_watch_tx: paths_to_watch_tx.clone(), + changed_paths_rx: changed_paths_rx.resubscribe(), + changed_paths_tx, + restart_tx: restart_tx.clone(), + restart_mode, + banner: colors::intense_blue(banner).to_string(), + })); info!("{} {} started.", colors::intense_blue(banner), job_name); let changed_paths = Rc::new(RefCell::new(None)); @@ -292,15 +335,8 @@ where .borrow_mut() .clone_from(&received_changed_paths); - match *watcher_.restart_mode.lock() { - WatcherRestartMode::Automatic => { - let _ = restart_tx.send(()); - } - WatcherRestartMode::Manual => { - // TODO(bartlomieju): should we fail on sending changed paths? - let _ = changed_paths_tx.send(received_changed_paths); - } - } + // TODO(bartlomieju): should we fail on sending changed paths? + let _ = watcher_.send(received_changed_paths); } }); diff --git a/cli/util/mod.rs b/cli/util/mod.rs index 0578ecb423..702e5673c9 100644 --- a/cli/util/mod.rs +++ b/cli/util/mod.rs @@ -2,7 +2,6 @@ // Note: Only add code in this folder that has no application specific logic pub mod archive; -pub mod checksum; pub mod console; pub mod diff; pub mod display; diff --git a/cli/worker.rs b/cli/worker.rs index d9cdbd3fb0..cf301de83e 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -1,8 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::path::Path; -use std::path::PathBuf; -use std::rc::Rc; use std::sync::Arc; use deno_ast::ModuleSpecifier; @@ -10,77 +8,31 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::error::CoreError; use deno_core::futures::FutureExt; -use deno_core::url::Url; use deno_core::v8; -use deno_core::CompiledWasmModuleStore; use deno_core::Extension; -use deno_core::FeatureChecker; -use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; -use deno_core::SharedArrayBufferStore; use deno_error::JsErrorBox; -use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_lib::worker::LibMainWorker; +use deno_lib::worker::LibMainWorkerFactory; use deno_runtime::code_cache; -use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; -use deno_runtime::deno_fs; -use deno_runtime::deno_node::NodeExtInitServices; -use deno_runtime::deno_node::NodeRequireLoader; -use deno_runtime::deno_node::NodeRequireLoaderRc; use deno_runtime::deno_permissions::PermissionsContainer; -use deno_runtime::deno_tls::RootCertStoreProvider; -use deno_runtime::deno_web::BlobStore; -use deno_runtime::fmt_errors::format_js_error; -use deno_runtime::inspector_server::InspectorServer; -use deno_runtime::ops::process::NpmProcessStateProviderRc; -use deno_runtime::ops::worker_host::CreateWebWorkerCb; -use deno_runtime::web_worker::WebWorker; -use deno_runtime::web_worker::WebWorkerOptions; -use deno_runtime::web_worker::WebWorkerServiceOptions; use deno_runtime::worker::MainWorker; -use deno_runtime::worker::WorkerOptions; -use deno_runtime::worker::WorkerServiceOptions; -use deno_runtime::BootstrapOptions; use deno_runtime::WorkerExecutionMode; -use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; -use deno_telemetry::OtelConfig; -use deno_terminal::colors; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; +use sys_traits::EnvCurrentDir; use tokio::select; use crate::args::CliLockfile; -use crate::args::DenoSubcommand; use crate::args::NpmCachingStrategy; -use crate::args::StorageKeyResolver; use crate::node::CliNodeResolver; -use crate::node::CliPackageJsonResolver; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; use crate::npm::CliNpmResolver; use crate::sys::CliSys; -use crate::util::checksum; use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; -use crate::version; - -pub struct CreateModuleLoaderResult { - pub module_loader: Rc, - pub node_require_loader: Rc, -} - -pub trait ModuleLoaderFactory: Send + Sync { - fn create_for_main( - &self, - root_permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult; - - fn create_for_worker( - &self, - parent_permissions: PermissionsContainer, - permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult; -} #[async_trait::async_trait(?Send)] pub trait HmrRunner: Send + Sync { @@ -115,83 +67,28 @@ pub type CreateCoverageCollectorCb = Box< >; pub struct CliMainWorkerOptions { - pub argv: Vec, - pub log_level: WorkerLogLevel, - pub enable_op_summary_metrics: bool, - pub enable_testing_features: bool, - pub has_node_modules_dir: bool, - pub hmr: bool, - pub inspect_brk: bool, - pub inspect_wait: bool, - pub strace_ops: Option>, - pub is_inspecting: bool, - pub location: Option, - pub argv0: Option, - pub node_debug: Option, - pub origin_data_folder_path: Option, - pub seed: Option, - pub unsafely_ignore_certificate_errors: Option>, - pub skip_op_registration: bool, pub create_hmr_runner: Option, pub create_coverage_collector: Option, - pub node_ipc: Option, - pub serve_port: Option, - pub serve_host: Option, + pub default_npm_caching_strategy: NpmCachingStrategy, + pub needs_test_modules: bool, } -struct SharedWorkerState { - blob_store: Arc, - broadcast_channel: InMemoryBroadcastChannel, - code_cache: Option>, - compiled_wasm_module_store: CompiledWasmModuleStore, - feature_checker: Arc, - fs: Arc, - maybe_file_watcher_communicator: Option>, - maybe_inspector_server: Option>, - maybe_lockfile: Option>, - module_loader_factory: Box, - node_resolver: Arc, - npm_installer: Option>, - npm_resolver: CliNpmResolver, - pkg_json_resolver: Arc, - root_cert_store_provider: Arc, - root_permissions: PermissionsContainer, - shared_array_buffer_store: SharedArrayBufferStore, - storage_key_resolver: StorageKeyResolver, - sys: CliSys, - options: CliMainWorkerOptions, - subcommand: DenoSubcommand, - otel_config: OtelConfig, - default_npm_caching_strategy: NpmCachingStrategy, -} - -impl SharedWorkerState { - pub fn create_node_init_services( - &self, - node_require_loader: NodeRequireLoaderRc, - ) -> NodeExtInitServices { - NodeExtInitServices { - node_require_loader, - node_resolver: self.node_resolver.clone(), - pkg_json_resolver: self.pkg_json_resolver.clone(), - sys: self.sys.clone(), - } - } - - pub fn npm_process_state_provider(&self) -> NpmProcessStateProviderRc { - crate::npm::create_npm_process_state_provider(&self.npm_resolver) - } +/// Data shared between the factory and workers. +struct SharedState { + pub create_hmr_runner: Option, + pub create_coverage_collector: Option, + pub maybe_file_watcher_communicator: Option>, } pub struct CliMainWorker { - main_module: ModuleSpecifier, - worker: MainWorker, - shared: Arc, + worker: LibMainWorker, + shared: Arc, } impl CliMainWorker { + #[inline] pub fn into_main_worker(self) -> MainWorker { - self.worker + self.worker.into_main_worker() } pub async fn setup_repl(&mut self) -> Result<(), AnyError> { @@ -204,16 +101,13 @@ impl CliMainWorker { self.maybe_setup_coverage_collector().await?; let mut maybe_hmr_runner = self.maybe_setup_hmr_runner().await?; - log::debug!("main_module {}", self.main_module); + log::debug!("main_module {}", self.worker.main_module()); self.execute_main_module().await?; self.worker.dispatch_load_event()?; loop { if let Some(hmr_runner) = maybe_hmr_runner.as_mut() { - let watcher_communicator = - self.shared.maybe_file_watcher_communicator.clone().unwrap(); - let hmr_future = hmr_runner.run().boxed_local(); let event_loop_future = self.worker.run_event_loop(false).boxed_local(); @@ -227,7 +121,11 @@ impl CliMainWorker { } } if let Err(e) = result { - watcher_communicator + self + .shared + .maybe_file_watcher_communicator + .as_ref() + .unwrap() .change_restart_mode(WatcherRestartMode::Automatic); return Err(e); } @@ -253,7 +151,7 @@ impl CliMainWorker { if let Some(coverage_collector) = maybe_coverage_collector.as_mut() { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( coverage_collector.stop_collecting().boxed_local(), PollEventLoopOptions::default(), @@ -263,7 +161,7 @@ impl CliMainWorker { if let Some(hmr_runner) = maybe_hmr_runner.as_mut() { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( hmr_runner.stop().boxed_local(), PollEventLoopOptions::default(), @@ -335,24 +233,20 @@ impl CliMainWorker { executor.execute().await } + #[inline] pub async fn execute_main_module(&mut self) -> Result<(), CoreError> { - let id = self.worker.preload_main_module(&self.main_module).await?; - self.worker.evaluate_module(id).await + self.worker.execute_main_module().await } + #[inline] pub async fn execute_side_module(&mut self) -> Result<(), CoreError> { - let id = self.worker.preload_side_module(&self.main_module).await?; - self.worker.evaluate_module(id).await + self.worker.execute_side_module().await } pub async fn maybe_setup_hmr_runner( &mut self, ) -> Result>, AnyError> { - if !self.shared.options.hmr { - return Ok(None); - } - let Some(setup_hmr_runner) = self.shared.options.create_hmr_runner.as_ref() - else { + let Some(setup_hmr_runner) = self.shared.create_hmr_runner.as_ref() else { return Ok(None); }; @@ -362,7 +256,7 @@ impl CliMainWorker { self .worker - .js_runtime + .js_runtime() .with_event_loop_future( hmr_runner.start().boxed_local(), PollEventLoopOptions::default(), @@ -375,7 +269,7 @@ impl CliMainWorker { &mut self, ) -> Result>, AnyError> { let Some(create_coverage_collector) = - self.shared.options.create_coverage_collector.as_ref() + self.shared.create_coverage_collector.as_ref() else { return Ok(None); }; @@ -384,7 +278,7 @@ impl CliMainWorker { let mut coverage_collector = create_coverage_collector(session); self .worker - .js_runtime + .js_runtime() .with_event_loop_future( coverage_collector.start_collecting().boxed_local(), PollEventLoopOptions::default(), @@ -398,72 +292,51 @@ impl CliMainWorker { name: &'static str, source_code: &'static str, ) -> Result, CoreError> { - self.worker.js_runtime.execute_script(name, source_code) + self.worker.js_runtime().execute_script(name, source_code) } } -// TODO(bartlomieju): this should be moved to some other place, added to avoid string -// duplication between worker setups and `deno info` output. -pub fn get_cache_storage_dir() -> PathBuf { - // Note: we currently use temp_dir() to avoid managing storage size. - std::env::temp_dir().join("deno_cache") -} - -#[derive(Clone)] pub struct CliMainWorkerFactory { - shared: Arc, + lib_main_worker_factory: LibMainWorkerFactory, + maybe_lockfile: Option>, + node_resolver: Arc, + npm_installer: Option>, + npm_resolver: CliNpmResolver, + root_permissions: PermissionsContainer, + shared: Arc, + sys: CliSys, + default_npm_caching_strategy: NpmCachingStrategy, + needs_test_modules: bool, } impl CliMainWorkerFactory { #[allow(clippy::too_many_arguments)] pub fn new( - blob_store: Arc, - code_cache: Option>, - feature_checker: Arc, - fs: Arc, + lib_main_worker_factory: LibMainWorkerFactory, maybe_file_watcher_communicator: Option>, - maybe_inspector_server: Option>, maybe_lockfile: Option>, - module_loader_factory: Box, node_resolver: Arc, npm_installer: Option>, npm_resolver: CliNpmResolver, - pkg_json_resolver: Arc, - root_cert_store_provider: Arc, - root_permissions: PermissionsContainer, - storage_key_resolver: StorageKeyResolver, sys: CliSys, - subcommand: DenoSubcommand, options: CliMainWorkerOptions, - otel_config: OtelConfig, - default_npm_caching_strategy: NpmCachingStrategy, + root_permissions: PermissionsContainer, ) -> Self { Self { - shared: Arc::new(SharedWorkerState { - blob_store, - broadcast_channel: Default::default(), - code_cache, - compiled_wasm_module_store: Default::default(), - feature_checker, - fs, + lib_main_worker_factory, + maybe_lockfile, + node_resolver, + npm_installer, + npm_resolver, + root_permissions, + sys, + shared: Arc::new(SharedState { + create_hmr_runner: options.create_hmr_runner, + create_coverage_collector: options.create_coverage_collector, maybe_file_watcher_communicator, - maybe_inspector_server, - maybe_lockfile, - module_loader_factory, - node_resolver, - npm_installer, - npm_resolver, - pkg_json_resolver, - root_cert_store_provider, - root_permissions, - shared_array_buffer_store: Default::default(), - storage_key_resolver, - sys, - options, - subcommand, - otel_config, - default_npm_caching_strategy, }), + default_npm_caching_strategy: options.default_npm_caching_strategy, + needs_test_modules: options.needs_test_modules, } } @@ -476,7 +349,7 @@ impl CliMainWorkerFactory { .create_custom_worker( mode, main_module, - self.shared.root_permissions.clone(), + self.root_permissions.clone(), vec![], Default::default(), ) @@ -491,23 +364,16 @@ impl CliMainWorkerFactory { custom_extensions: Vec, stdio: deno_runtime::deno_io::Stdio, ) -> Result { - let shared = &self.shared; - let CreateModuleLoaderResult { - module_loader, - node_require_loader, - } = shared - .module_loader_factory - .create_for_main(permissions.clone()); let main_module = if let Ok(package_ref) = NpmPackageReqReference::from_specifier(&main_module) { - if let Some(npm_installer) = &shared.npm_installer { + if let Some(npm_installer) = &self.npm_installer { let reqs = &[package_ref.req().clone()]; npm_installer .add_package_reqs( reqs, if matches!( - shared.default_npm_caching_strategy, + self.default_npm_caching_strategy, NpmCachingStrategy::Lazy ) { PackageCaching::Only(reqs.into()) @@ -520,18 +386,18 @@ impl CliMainWorkerFactory { // use a fake referrer that can be used to discover the package.json if necessary let referrer = ModuleSpecifier::from_directory_path( - self.shared.fs.cwd().map_err(JsErrorBox::from_err)?, + self.sys.env_current_dir().map_err(JsErrorBox::from_err)?, ) .unwrap() .join("package.json")?; - let package_folder = shared + let package_folder = self .npm_resolver .resolve_pkg_folder_from_deno_module_req(package_ref.req(), &referrer) .map_err(JsErrorBox::from_err)?; let main_module = self .resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?; - if let Some(lockfile) = &shared.maybe_lockfile { + if let Some(lockfile) = &self.maybe_lockfile { // For npm binary commands, ensure that the lockfile gets updated // so that we can re-use the npm resolution the next time it runs // for better performance @@ -543,119 +409,18 @@ impl CliMainWorkerFactory { main_module }; - let maybe_inspector_server = shared.maybe_inspector_server.clone(); - - let create_web_worker_cb = - create_web_worker_callback(shared.clone(), stdio.clone()); - - let maybe_storage_key = shared - .storage_key_resolver - .resolve_storage_key(&main_module); - let origin_storage_dir = maybe_storage_key.as_ref().map(|key| { - shared - .options - .origin_data_folder_path - .as_ref() - .unwrap() // must be set if storage key resolver returns a value - .join(checksum::gen(&[key.as_bytes()])) - }); - let cache_storage_dir = maybe_storage_key.map(|key| { - // TODO(@satyarohith): storage quota management - get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) - }); - - // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out - // list of enabled features. - let feature_checker = shared.feature_checker.clone(); - let mut unstable_features = - Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len()); - for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { - if feature_checker.check(granular_flag.name) { - unstable_features.push(granular_flag.id); - } - } - - let services = WorkerServiceOptions { - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), - module_loader, - fs: shared.fs.clone(), - node_services: Some( - shared.create_node_init_services(node_require_loader), - ), - npm_process_state_provider: Some(shared.npm_process_state_provider()), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - fetch_dns_resolver: Default::default(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), - feature_checker, + let mut worker = self.lib_main_worker_factory.create_custom_worker( + mode, + main_module, permissions, - v8_code_cache: shared.code_cache.clone().map(|c| c.as_code_cache()), - }; - - let options = WorkerOptions { - bootstrap: BootstrapOptions { - deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), - args: shared.options.argv.clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - log_level: shared.options.log_level, - enable_op_summary_metrics: shared.options.enable_op_summary_metrics, - enable_testing_features: shared.options.enable_testing_features, - locale: deno_core::v8::icu::get_language_tag(), - location: shared.options.location.clone(), - no_color: !colors::use_color(), - is_stdout_tty: deno_terminal::is_stdout_tty(), - is_stderr_tty: deno_terminal::is_stderr_tty(), - color_level: colors::get_color_level(), - unstable_features, - user_agent: version::DENO_VERSION_INFO.user_agent.to_string(), - inspect: shared.options.is_inspecting, - has_node_modules_dir: shared.options.has_node_modules_dir, - argv0: shared.options.argv0.clone(), - node_debug: shared.options.node_debug.clone(), - node_ipc_fd: shared.options.node_ipc, - mode, - serve_port: shared.options.serve_port, - serve_host: shared.options.serve_host.clone(), - otel_config: shared.otel_config.clone(), - close_on_idle: true, - }, - extensions: custom_extensions, - startup_snapshot: crate::js::deno_isolate_init(), - create_params: create_isolate_create_params(), - unsafely_ignore_certificate_errors: shared - .options - .unsafely_ignore_certificate_errors - .clone(), - seed: shared.options.seed, - format_js_error_fn: Some(Arc::new(format_js_error)), - create_web_worker_cb, - maybe_inspector_server, - should_break_on_first_statement: shared.options.inspect_brk, - should_wait_for_inspector_session: shared.options.inspect_wait, - strace_ops: shared.options.strace_ops.clone(), - cache_storage_dir, - origin_storage_dir, + custom_extensions, stdio, - skip_op_registration: shared.options.skip_op_registration, - enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( - ), - }; + )?; - let mut worker = MainWorker::bootstrap_from_options( - main_module.clone(), - services, - options, - ); - - if self.shared.subcommand.needs_test() { + if self.needs_test_modules { macro_rules! test_file { ($($file:literal),*) => { - $(worker.js_runtime.lazy_load_es_module_with_code( + $(worker.js_runtime().lazy_load_es_module_with_code( concat!("ext:cli/", $file), deno_core::ascii_str_include!(concat!("js/", $file)), )?;)* @@ -673,9 +438,8 @@ impl CliMainWorkerFactory { } Ok(CliMainWorker { - main_module, worker, - shared: shared.clone(), + shared: self.shared.clone(), }) } @@ -685,7 +449,6 @@ impl CliMainWorkerFactory { sub_path: Option<&str>, ) -> Result { match self - .shared .node_resolver .resolve_binary_export(package_folder, sub_path) { @@ -720,7 +483,6 @@ impl CliMainWorkerFactory { } let specifier = self - .shared .node_resolver .resolve_package_subpath_from_deno_module( package_folder, @@ -741,136 +503,20 @@ impl CliMainWorkerFactory { } } -fn create_web_worker_callback( - shared: Arc, - stdio: deno_runtime::deno_io::Stdio, -) -> Arc { - Arc::new(move |args| { - let maybe_inspector_server = shared.maybe_inspector_server.clone(); - - let CreateModuleLoaderResult { - module_loader, - node_require_loader, - } = shared.module_loader_factory.create_for_worker( - args.parent_permissions.clone(), - args.permissions.clone(), - ); - let create_web_worker_cb = - create_web_worker_callback(shared.clone(), stdio.clone()); - - let maybe_storage_key = shared - .storage_key_resolver - .resolve_storage_key(&args.main_module); - let cache_storage_dir = maybe_storage_key.map(|key| { - // TODO(@satyarohith): storage quota management - get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) - }); - - // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out - // list of enabled features. - let feature_checker = shared.feature_checker.clone(); - let mut unstable_features = - Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len()); - for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { - if feature_checker.check(granular_flag.name) { - unstable_features.push(granular_flag.id); - } - } - - let services = WebWorkerServiceOptions { - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), - module_loader, - fs: shared.fs.clone(), - node_services: Some( - shared.create_node_init_services(node_require_loader), - ), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), - maybe_inspector_server, - feature_checker, - npm_process_state_provider: Some(shared.npm_process_state_provider()), - permissions: args.permissions, - }; - let options = WebWorkerOptions { - name: args.name, - main_module: args.main_module.clone(), - worker_id: args.worker_id, - bootstrap: BootstrapOptions { - deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), - args: shared.options.argv.clone(), - cpu_count: std::thread::available_parallelism() - .map(|p| p.get()) - .unwrap_or(1), - log_level: shared.options.log_level, - enable_op_summary_metrics: shared.options.enable_op_summary_metrics, - enable_testing_features: shared.options.enable_testing_features, - locale: deno_core::v8::icu::get_language_tag(), - location: Some(args.main_module), - no_color: !colors::use_color(), - color_level: colors::get_color_level(), - is_stdout_tty: deno_terminal::is_stdout_tty(), - is_stderr_tty: deno_terminal::is_stderr_tty(), - unstable_features, - user_agent: version::DENO_VERSION_INFO.user_agent.to_string(), - inspect: shared.options.is_inspecting, - has_node_modules_dir: shared.options.has_node_modules_dir, - argv0: shared.options.argv0.clone(), - node_debug: shared.options.node_debug.clone(), - node_ipc_fd: None, - mode: WorkerExecutionMode::Worker, - serve_port: shared.options.serve_port, - serve_host: shared.options.serve_host.clone(), - otel_config: shared.otel_config.clone(), - close_on_idle: args.close_on_idle, - }, - extensions: vec![], - startup_snapshot: crate::js::deno_isolate_init(), - create_params: create_isolate_create_params(), - unsafely_ignore_certificate_errors: shared - .options - .unsafely_ignore_certificate_errors - .clone(), - seed: shared.options.seed, - create_web_worker_cb, - format_js_error_fn: Some(Arc::new(format_js_error)), - worker_type: args.worker_type, - stdio: stdio.clone(), - cache_storage_dir, - strace_ops: shared.options.strace_ops.clone(), - close_on_idle: args.close_on_idle, - maybe_worker_metadata: args.maybe_worker_metadata, - enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( - ), - }; - - WebWorker::bootstrap_from_options(services, options) - }) -} - -/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs. -/// Instead probe for the total memory on the system and use it instead -/// as a default. -pub fn create_isolate_create_params() -> Option { - let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info(); - maybe_mem_info.map(|mem_info| { - v8::CreateParams::default() - .heap_limits_from_system_memory(mem_info.total, 0) - }) -} - #[allow(clippy::print_stdout)] #[allow(clippy::print_stderr)] #[cfg(test)] mod tests { + use std::rc::Rc; + use deno_core::resolve_path; use deno_core::FsModuleLoader; - use deno_fs::RealFs; + use deno_resolver::npm::DenoInNpmPackageChecker; + use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_permissions::Permissions; use deno_runtime::permissions::RuntimePermissionDescriptorParser; + use deno_runtime::worker::WorkerOptions; + use deno_runtime::worker::WorkerServiceOptions; use super::*; @@ -891,7 +537,7 @@ mod tests { CliNpmResolver, CliSys, >( - main_module, + &main_module, WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), permissions: PermissionsContainer::new( diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index e1538b8b75..a1c24f30e4 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -43,7 +43,7 @@ async fn main() -> Result<(), AnyError> { RuntimePermissionDescriptorParser::new(sys_traits::impls::RealSys), ); let mut worker = MainWorker::bootstrap_from_options( - main_module.clone(), + &main_module, WorkerServiceOptions::< DenoInNpmPackageChecker, NpmResolver, diff --git a/runtime/lib.rs b/runtime/lib.rs index c104f5cd61..65d3e88bae 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -18,6 +18,7 @@ pub use deno_net; pub use deno_node; pub use deno_os; pub use deno_permissions; +pub use deno_telemetry; pub use deno_terminal::colors; pub use deno_tls; pub use deno_url; diff --git a/runtime/worker.rs b/runtime/worker.rs index cfcadccc46..426383a19e 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -303,7 +303,7 @@ impl MainWorker { TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TExtNodeSys: ExtNodeSys + 'static, >( - main_module: ModuleSpecifier, + main_module: &ModuleSpecifier, services: WorkerServiceOptions< TInNpmPackageChecker, TNpmPackageFolderResolver, @@ -322,7 +322,7 @@ impl MainWorker { TNpmPackageFolderResolver: NpmPackageFolderResolver + 'static, TExtNodeSys: ExtNodeSys + 'static, >( - main_module: ModuleSpecifier, + main_module: &ModuleSpecifier, services: WorkerServiceOptions< TInNpmPackageChecker, TNpmPackageFolderResolver, From a02ee7adf9097ffeaf3b34f8d00946fdcd0a76b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 15 Jan 2025 18:09:08 +0000 Subject: [PATCH 8/8] ci: try to fix caching on Mac ARM (#27685) --- .github/workflows/ci.generate.ts | 10 ++++++++-- .github/workflows/ci.yml | 10 +++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 55c88fbe85..d0924f79ac 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 34; +const cacheVersion = 35; const ubuntuX86Runner = "ubuntu-24.04"; const ubuntuX86XlRunner = "ubuntu-24.04-xl"; @@ -41,6 +41,12 @@ const Runners = { macosArm: { os: "macos", arch: "aarch64", + runner: macosArmRunner, + }, + macosArmSelfHosted: { + os: "macos", + arch: "aarch64", + // Actually use self-hosted runner only in denoland/deno on `main` branch. runner: `\${{ github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, @@ -384,7 +390,7 @@ const ci = { job: "test", profile: "debug", }, { - ...Runners.macosArm, + ...Runners.macosArmSelfHosted, job: "test", profile: "release", skip_pr: true, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f217d85032..75ded6fe53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: '${{ github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''ghcr.io/cirruslabs/macos-runner:sonoma'' || ''macos-14'' }}' + runner: macos-14 job: test profile: debug - os: macos @@ -184,8 +184,8 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - key: '34-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '34-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' + key: '35-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '35-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - uses: dsherret/rust-toolchain-file@v1 if: '!(matrix.skip)' @@ -379,7 +379,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '34-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '35-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -689,7 +689,7 @@ jobs: !./target/*/gn_root !./target/*/*.zip !./target/*/*.tar.gz - key: '34-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '35-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-24.04