From ce0968ef3ad32fd007fe592495788fa453f2bb0b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 8 Jan 2025 18:46:37 -0500 Subject: [PATCH] refactor(npm): split some resolution from installation (#27595) This splits away some npm resolution code from installation. It will allow for more easily extracting out resolution code in the future. --- cli/npm/managed/installer.rs | 226 ++++ .../common/bin_entries.rs | 0 .../common/lifecycle_scripts.rs | 0 cli/npm/managed/installers/common/mod.rs | 18 + cli/npm/managed/installers/global.rs | 190 +++ cli/npm/managed/installers/local.rs | 1032 +++++++++++++++++ cli/npm/managed/installers/mod.rs | 55 + cli/npm/managed/mod.rs | 55 +- cli/npm/managed/resolution.rs | 214 +--- cli/npm/managed/resolvers/common.rs | 11 - cli/npm/managed/resolvers/global.rs | 167 +-- cli/npm/managed/resolvers/local.rs | 981 +--------------- cli/npm/managed/resolvers/mod.rs | 27 +- cli/util/sync/mod.rs | 2 - cli/util/sync/sync_read_async_write_lock.rs | 62 - 15 files changed, 1571 insertions(+), 1469 deletions(-) create mode 100644 cli/npm/managed/installer.rs rename cli/npm/managed/{resolvers => installers}/common/bin_entries.rs (100%) rename cli/npm/managed/{resolvers => installers}/common/lifecycle_scripts.rs (100%) create mode 100644 cli/npm/managed/installers/common/mod.rs create mode 100644 cli/npm/managed/installers/global.rs create mode 100644 cli/npm/managed/installers/local.rs create mode 100644 cli/npm/managed/installers/mod.rs delete mode 100644 cli/util/sync/sync_read_async_write_lock.rs diff --git a/cli/npm/managed/installer.rs b/cli/npm/managed/installer.rs new file mode 100644 index 0000000000..30ac807023 --- /dev/null +++ b/cli/npm/managed/installer.rs @@ -0,0 +1,226 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::collections::HashSet; +use std::sync::Arc; + +use capacity_builder::StringBuilder; +use deno_core::error::AnyError; +use deno_error::JsErrorBox; +use deno_lockfile::NpmPackageDependencyLockfileInfo; +use deno_lockfile::NpmPackageLockfileInfo; +use deno_npm::registry::NpmRegistryApi; +use deno_npm::resolution::AddPkgReqsOptions; +use deno_npm::resolution::NpmResolutionError; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::NpmResolutionPackage; +use deno_semver::jsr::JsrDepPackageReq; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageReq; +use deno_semver::SmallStackString; +use deno_semver::VersionReq; + +use super::resolution::NpmResolution; +use crate::args::CliLockfile; +use crate::npm::CliNpmRegistryInfoProvider; +use crate::util::sync::TaskQueue; + +pub struct AddPkgReqsResult { + /// Results from adding the individual packages. + /// + /// The indexes of the results correspond to the indexes of the provided + /// package requirements. + pub results: Vec>, + /// The final result of resolving and caching all the package requirements. + pub dependencies_result: Result<(), JsErrorBox>, +} + +/// Updates the npm resolution with the provided package requirements. +pub struct NpmResolutionInstaller { + registry_info_provider: Arc, + resolution: Arc, + maybe_lockfile: Option>, + update_queue: TaskQueue, +} + +impl NpmResolutionInstaller { + pub fn new( + registry_info_provider: Arc, + resolution: Arc, + maybe_lockfile: Option>, + ) -> Self { + Self { + registry_info_provider, + resolution, + maybe_lockfile, + update_queue: Default::default(), + } + } + + pub async fn add_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> AddPkgReqsResult { + // only allow one thread in here at a time + let _snapshot_lock = self.update_queue.acquire().await; + let result = add_package_reqs_to_snapshot( + &self.registry_info_provider, + package_reqs, + self.maybe_lockfile.clone(), + || self.resolution.snapshot(), + ) + .await; + + AddPkgReqsResult { + results: result.results, + dependencies_result: match result.dep_graph_result { + Ok(snapshot) => { + self.resolution.set_snapshot(snapshot); + Ok(()) + } + Err(err) => Err(JsErrorBox::from_err(err)), + }, + } + } + + pub async fn set_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _snapshot_lock = self.update_queue.acquire().await; + + let reqs_set = package_reqs.iter().collect::>(); + let snapshot = add_package_reqs_to_snapshot( + &self.registry_info_provider, + package_reqs, + self.maybe_lockfile.clone(), + || { + let snapshot = self.resolution.snapshot(); + let has_removed_package = !snapshot + .package_reqs() + .keys() + .all(|req| reqs_set.contains(req)); + // if any packages were removed, we need to completely recreate the npm resolution snapshot + if has_removed_package { + snapshot.into_empty() + } else { + snapshot + } + }, + ) + .await + .into_result()?; + + self.resolution.set_snapshot(snapshot); + + Ok(()) + } +} + +async fn add_package_reqs_to_snapshot( + registry_info_provider: &Arc, + package_reqs: &[PackageReq], + maybe_lockfile: Option>, + get_new_snapshot: impl Fn() -> NpmResolutionSnapshot, +) -> deno_npm::resolution::AddPkgReqsResult { + let snapshot = get_new_snapshot(); + if package_reqs + .iter() + .all(|req| snapshot.package_reqs().contains_key(req)) + { + log::debug!("Snapshot already up to date. Skipping npm resolution."); + return deno_npm::resolution::AddPkgReqsResult { + results: package_reqs + .iter() + .map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone())) + .collect(), + dep_graph_result: Ok(snapshot), + }; + } + log::debug!( + /* this string is used in tests */ + "Running npm resolution." + ); + let npm_registry_api = registry_info_provider.as_npm_registry_api(); + let result = snapshot + .add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs)) + .await; + let result = match &result.dep_graph_result { + Err(NpmResolutionError::Resolution(err)) + if npm_registry_api.mark_force_reload() => + { + log::debug!("{err:#}"); + log::debug!("npm resolution failed. Trying again..."); + + // try again with forced reloading + let snapshot = get_new_snapshot(); + snapshot + .add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs)) + .await + } + _ => result, + }; + + registry_info_provider.clear_memory_cache(); + + if let Ok(snapshot) = &result.dep_graph_result { + if let Some(lockfile) = maybe_lockfile { + populate_lockfile_from_snapshot(&lockfile, snapshot); + } + } + + result +} + +fn get_add_pkg_reqs_options(package_reqs: &[PackageReq]) -> AddPkgReqsOptions { + AddPkgReqsOptions { + package_reqs, + // WARNING: When bumping this version, check if anything needs to be + // updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js + types_node_version_req: Some( + VersionReq::parse_from_npm("22.0.0 - 22.5.4").unwrap(), + ), + } +} + +fn populate_lockfile_from_snapshot( + lockfile: &CliLockfile, + snapshot: &NpmResolutionSnapshot, +) { + fn npm_package_to_lockfile_info( + pkg: &NpmResolutionPackage, + ) -> NpmPackageLockfileInfo { + let dependencies = pkg + .dependencies + .iter() + .map(|(name, id)| NpmPackageDependencyLockfileInfo { + name: name.clone(), + id: id.as_serialized(), + }) + .collect(); + + NpmPackageLockfileInfo { + serialized_id: pkg.id.as_serialized(), + integrity: pkg.dist.integrity().for_lockfile(), + dependencies, + } + } + + let mut lockfile = lockfile.lock(); + for (package_req, nv) in snapshot.package_reqs() { + let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id; + lockfile.insert_package_specifier( + JsrDepPackageReq::npm(package_req.clone()), + { + StringBuilder::::build(|builder| { + builder.append(&id.nv.version); + builder.append(&id.peer_dependencies); + }) + .unwrap() + }, + ); + } + for package in snapshot.all_packages_for_every_system() { + lockfile.insert_npm_package(npm_package_to_lockfile_info(package)); + } +} diff --git a/cli/npm/managed/resolvers/common/bin_entries.rs b/cli/npm/managed/installers/common/bin_entries.rs similarity index 100% rename from cli/npm/managed/resolvers/common/bin_entries.rs rename to cli/npm/managed/installers/common/bin_entries.rs diff --git a/cli/npm/managed/resolvers/common/lifecycle_scripts.rs b/cli/npm/managed/installers/common/lifecycle_scripts.rs similarity index 100% rename from cli/npm/managed/resolvers/common/lifecycle_scripts.rs rename to cli/npm/managed/installers/common/lifecycle_scripts.rs diff --git a/cli/npm/managed/installers/common/mod.rs b/cli/npm/managed/installers/common/mod.rs new file mode 100644 index 0000000000..9659649a2e --- /dev/null +++ b/cli/npm/managed/installers/common/mod.rs @@ -0,0 +1,18 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use async_trait::async_trait; +use deno_error::JsErrorBox; + +use crate::npm::PackageCaching; + +pub mod bin_entries; +pub mod lifecycle_scripts; + +/// Part of the resolution that interacts with the file system. +#[async_trait(?Send)] +pub trait NpmPackageFsInstaller: Send + Sync { + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), JsErrorBox>; +} diff --git a/cli/npm/managed/installers/global.rs b/cli/npm/managed/installers/global.rs new file mode 100644 index 0000000000..d637c96122 --- /dev/null +++ b/cli/npm/managed/installers/global.rs @@ -0,0 +1,190 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use deno_core::futures::stream::FuturesUnordered; +use deno_core::futures::StreamExt; +use deno_error::JsErrorBox; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; + +use super::super::resolution::NpmResolution; +use super::common::lifecycle_scripts::LifecycleScriptsStrategy; +use super::common::NpmPackageFsInstaller; +use crate::args::LifecycleScriptsConfig; +use crate::cache::FastInsecureHasher; +use crate::colors; +use crate::npm::managed::PackageCaching; +use crate::npm::CliNpmCache; +use crate::npm::CliNpmTarballCache; + +/// Resolves packages from the global npm cache. +#[derive(Debug)] +pub struct GlobalNpmPackageInstaller { + cache: Arc, + tarball_cache: Arc, + resolution: Arc, + system_info: NpmSystemInfo, + lifecycle_scripts: LifecycleScriptsConfig, +} + +impl GlobalNpmPackageInstaller { + pub fn new( + cache: Arc, + tarball_cache: Arc, + resolution: Arc, + system_info: NpmSystemInfo, + lifecycle_scripts: LifecycleScriptsConfig, + ) -> Self { + Self { + cache, + tarball_cache, + resolution, + system_info, + lifecycle_scripts, + } + } +} + +#[async_trait(?Send)] +impl NpmPackageFsInstaller for GlobalNpmPackageInstaller { + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), JsErrorBox> { + let package_partitions = match caching { + PackageCaching::All => self + .resolution + .all_system_packages_partitioned(&self.system_info), + PackageCaching::Only(reqs) => self + .resolution + .subset(&reqs) + .all_system_packages_partitioned(&self.system_info), + }; + cache_packages(&package_partitions.packages, &self.tarball_cache) + .await + .map_err(JsErrorBox::from_err)?; + + // create the copy package folders + for copy in package_partitions.copy_packages { + self + .cache + .ensure_copy_package(©.get_package_cache_folder_id()) + .map_err(JsErrorBox::from_err)?; + } + + let mut lifecycle_scripts = + super::common::lifecycle_scripts::LifecycleScripts::new( + &self.lifecycle_scripts, + GlobalLifecycleScripts::new(self, &self.lifecycle_scripts.root_dir), + ); + for package in &package_partitions.packages { + let package_folder = self.cache.package_folder_for_nv(&package.id.nv); + lifecycle_scripts.add(package, Cow::Borrowed(&package_folder)); + } + + lifecycle_scripts + .warn_not_run_scripts() + .map_err(JsErrorBox::from_err)?; + + Ok(()) + } +} + +async fn cache_packages( + packages: &[NpmResolutionPackage], + tarball_cache: &Arc, +) -> Result<(), deno_npm_cache::EnsurePackageError> { + let mut futures_unordered = FuturesUnordered::new(); + for package in packages { + futures_unordered.push(async move { + tarball_cache + .ensure_package(&package.id.nv, &package.dist) + .await + }); + } + while let Some(result) = futures_unordered.next().await { + // surface the first error + result?; + } + Ok(()) +} + +struct GlobalLifecycleScripts<'a> { + installer: &'a GlobalNpmPackageInstaller, + path_hash: u64, +} + +impl<'a> GlobalLifecycleScripts<'a> { + fn new(installer: &'a GlobalNpmPackageInstaller, root_dir: &Path) -> Self { + let mut hasher = FastInsecureHasher::new_without_deno_version(); + hasher.write(root_dir.to_string_lossy().as_bytes()); + let path_hash = hasher.finish(); + Self { + installer, + path_hash, + } + } + + fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { + self + .package_path(package) + .join(format!(".scripts-warned-{}", self.path_hash)) + } +} + +impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy + for GlobalLifecycleScripts<'a> +{ + fn can_run_scripts(&self) -> bool { + false + } + fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf { + self.installer.cache.package_folder_for_nv(&package.id.nv) + } + + fn warn_on_scripts_not_run( + &self, + packages: &[(&NpmResolutionPackage, PathBuf)], + ) -> std::result::Result<(), std::io::Error> { + log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall")); + for (package, _) in packages { + log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv))); + } + log::warn!("┃"); + log::warn!( + "┠─ {}", + colors::italic("This may cause the packages to not work correctly.") + ); + log::warn!("┠─ {}", colors::italic("Lifecycle scripts are only supported when using a `node_modules` directory.")); + log::warn!( + "┠─ {}", + colors::italic("Enable it in your deno config file:") + ); + log::warn!("┖─ {}", colors::bold("\"nodeModulesDir\": \"auto\"")); + + for (package, _) in packages { + std::fs::write(self.warned_scripts_file(package), "")?; + } + Ok(()) + } + + fn did_run_scripts( + &self, + _package: &NpmResolutionPackage, + ) -> Result<(), std::io::Error> { + Ok(()) + } + + fn has_warned(&self, package: &NpmResolutionPackage) -> bool { + self.warned_scripts_file(package).exists() + } + + fn has_run(&self, _package: &NpmResolutionPackage) -> bool { + false + } +} diff --git a/cli/npm/managed/installers/local.rs b/cli/npm/managed/installers/local.rs new file mode 100644 index 0000000000..e2c5653801 --- /dev/null +++ b/cli/npm/managed/installers/local.rs @@ -0,0 +1,1032 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +//! Code for local node_modules resolution. + +use std::cell::RefCell; +use std::cmp::Ordering; +use std::collections::hash_map::Entry; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use async_trait::async_trait; +use deno_core::futures::stream::FuturesUnordered; +use deno_core::futures::StreamExt; +use deno_core::parking_lot::Mutex; +use deno_error::JsErrorBox; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_path_util::fs::atomic_write_file_with_retries; +use deno_semver::package::PackageNv; +use deno_semver::StackString; +use serde::Deserialize; +use serde::Serialize; + +use super::super::resolution::NpmResolution; +use super::common::bin_entries; +use super::common::NpmPackageFsInstaller; +use crate::args::LifecycleScriptsConfig; +use crate::args::NpmInstallDepsProvider; +use crate::cache::CACHE_PERM; +use crate::colors; +use crate::npm::managed::resolvers::get_package_folder_id_folder_name; +use crate::npm::managed::PackageCaching; +use crate::npm::CliNpmCache; +use crate::npm::CliNpmTarballCache; +use crate::sys::CliSys; +use crate::util::fs::clone_dir_recursive; +use crate::util::fs::symlink_dir; +use crate::util::fs::LaxSingleProcessFsFlag; +use crate::util::progress_bar::ProgressBar; +use crate::util::progress_bar::ProgressMessagePrompt; + +/// Resolver that creates a local node_modules directory +/// and resolves packages from it. +#[derive(Debug)] +pub struct LocalNpmPackageInstaller { + cache: Arc, + npm_install_deps_provider: Arc, + progress_bar: ProgressBar, + resolution: Arc, + sys: CliSys, + tarball_cache: Arc, + root_node_modules_path: PathBuf, + system_info: NpmSystemInfo, + lifecycle_scripts: LifecycleScriptsConfig, +} + +impl LocalNpmPackageInstaller { + #[allow(clippy::too_many_arguments)] + pub fn new( + cache: Arc, + npm_install_deps_provider: Arc, + progress_bar: ProgressBar, + resolution: Arc, + sys: CliSys, + tarball_cache: Arc, + node_modules_folder: PathBuf, + system_info: NpmSystemInfo, + lifecycle_scripts: LifecycleScriptsConfig, + ) -> Self { + Self { + cache, + npm_install_deps_provider, + progress_bar, + resolution, + tarball_cache, + sys, + root_node_modules_path: node_modules_folder, + system_info, + lifecycle_scripts, + } + } +} + +#[async_trait(?Send)] +impl NpmPackageFsInstaller for LocalNpmPackageInstaller { + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), JsErrorBox> { + let snapshot = match caching { + PackageCaching::All => self.resolution.snapshot(), + PackageCaching::Only(reqs) => self.resolution.subset(&reqs), + }; + sync_resolution_with_fs( + &snapshot, + &self.cache, + &self.npm_install_deps_provider, + &self.progress_bar, + &self.tarball_cache, + &self.root_node_modules_path, + &self.sys, + &self.system_info, + &self.lifecycle_scripts, + ) + .await + .map_err(JsErrorBox::from_err) + } +} + +/// `node_modules/.deno//node_modules/` +/// +/// Where the actual package is stored. +fn local_node_modules_package_contents_path( + local_registry_dir: &Path, + package: &NpmResolutionPackage, +) -> PathBuf { + local_registry_dir + .join(get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) + .join("node_modules") + .join(&package.id.nv.name) +} + +#[derive(Debug, thiserror::Error, deno_error::JsError)] +pub enum SyncResolutionWithFsError { + #[class(inherit)] + #[error("Creating '{path}'")] + Creating { + path: PathBuf, + #[source] + #[inherit] + source: std::io::Error, + }, + #[class(inherit)] + #[error(transparent)] + CopyDirRecursive(#[from] crate::util::fs::CopyDirRecursiveError), + #[class(inherit)] + #[error(transparent)] + SymlinkPackageDir(#[from] SymlinkPackageDirError), + #[class(inherit)] + #[error(transparent)] + BinEntries(#[from] bin_entries::BinEntriesError), + #[class(inherit)] + #[error(transparent)] + LifecycleScripts( + #[from] super::common::lifecycle_scripts::LifecycleScriptsError, + ), + #[class(inherit)] + #[error(transparent)] + Io(#[from] std::io::Error), + #[class(inherit)] + #[error(transparent)] + Other(#[from] JsErrorBox), +} + +/// Creates a pnpm style folder structure. +#[allow(clippy::too_many_arguments)] +async fn sync_resolution_with_fs( + snapshot: &NpmResolutionSnapshot, + cache: &Arc, + npm_install_deps_provider: &NpmInstallDepsProvider, + progress_bar: &ProgressBar, + tarball_cache: &Arc, + root_node_modules_dir_path: &Path, + sys: &CliSys, + system_info: &NpmSystemInfo, + lifecycle_scripts: &LifecycleScriptsConfig, +) -> Result<(), SyncResolutionWithFsError> { + if snapshot.is_empty() + && npm_install_deps_provider.workspace_pkgs().is_empty() + { + return Ok(()); // don't create the directory + } + + // don't set up node_modules (and more importantly try to acquire the file lock) + // if we're running as part of a lifecycle script + if super::common::lifecycle_scripts::is_running_lifecycle_script() { + return Ok(()); + } + + let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); + let deno_node_modules_dir = deno_local_registry_dir.join("node_modules"); + fs::create_dir_all(&deno_node_modules_dir).map_err(|source| { + SyncResolutionWithFsError::Creating { + path: deno_local_registry_dir.to_path_buf(), + source, + } + })?; + let bin_node_modules_dir_path = root_node_modules_dir_path.join(".bin"); + fs::create_dir_all(&bin_node_modules_dir_path).map_err(|source| { + SyncResolutionWithFsError::Creating { + path: deno_local_registry_dir.to_path_buf(), + source, + } + })?; + + let single_process_lock = LaxSingleProcessFsFlag::lock( + deno_local_registry_dir.join(".deno.lock"), + // similar message used by cargo build + "waiting for file lock on node_modules directory", + ) + .await; + + // load this after we get the directory lock + let mut setup_cache = + SetupCache::load(deno_local_registry_dir.join(".setup-cache.bin")); + + let pb_clear_guard = progress_bar.clear_guard(); // prevent flickering + + // 1. Write all the packages out the .deno directory. + // + // Copy (hardlink in future) // to + // node_modules/.deno//node_modules/ + let package_partitions = + snapshot.all_system_packages_partitioned(system_info); + let mut cache_futures = FuturesUnordered::new(); + let mut newest_packages_by_name: HashMap< + &StackString, + &NpmResolutionPackage, + > = HashMap::with_capacity(package_partitions.packages.len()); + let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new())); + let mut lifecycle_scripts = + super::common::lifecycle_scripts::LifecycleScripts::new( + lifecycle_scripts, + LocalLifecycleScripts { + deno_local_registry_dir: &deno_local_registry_dir, + }, + ); + let packages_with_deprecation_warnings = Arc::new(Mutex::new(Vec::new())); + + let mut package_tags: HashMap<&PackageNv, BTreeSet<&str>> = HashMap::new(); + for (package_req, package_nv) in snapshot.package_reqs() { + if let Some(tag) = package_req.version_req.tag() { + package_tags.entry(package_nv).or_default().insert(tag); + } + } + + for package in &package_partitions.packages { + if let Some(current_pkg) = + newest_packages_by_name.get_mut(&package.id.nv.name) + { + if current_pkg.id.nv.cmp(&package.id.nv) == Ordering::Less { + *current_pkg = package; + } + } else { + newest_packages_by_name.insert(&package.id.nv.name, package); + }; + + let package_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + let folder_path = deno_local_registry_dir.join(&package_folder_name); + let tags = package_tags + .get(&package.id.nv) + .map(|tags| { + capacity_builder::StringBuilder::::build(|builder| { + for (i, tag) in tags.iter().enumerate() { + if i > 0 { + builder.append(',') + } + builder.append(*tag); + } + }) + .unwrap() + }) + .unwrap_or_default(); + enum PackageFolderState { + UpToDate, + Uninitialized, + TagsOutdated, + } + let initialized_file = folder_path.join(".initialized"); + let package_state = std::fs::read_to_string(&initialized_file) + .map(|s| { + if s != tags { + PackageFolderState::TagsOutdated + } else { + PackageFolderState::UpToDate + } + }) + .unwrap_or(PackageFolderState::Uninitialized); + if !cache + .cache_setting() + .should_use_for_npm_package(&package.id.nv.name) + || matches!(package_state, PackageFolderState::Uninitialized) + { + // cache bust the dep from the dep setup cache so the symlinks + // are forced to be recreated + setup_cache.remove_dep(&package_folder_name); + + let folder_path = folder_path.clone(); + let bin_entries_to_setup = bin_entries.clone(); + let packages_with_deprecation_warnings = + packages_with_deprecation_warnings.clone(); + + cache_futures.push(async move { + tarball_cache + .ensure_package(&package.id.nv, &package.dist) + .await + .map_err(JsErrorBox::from_err)?; + let pb_guard = progress_bar.update_with_prompt( + ProgressMessagePrompt::Initialize, + &package.id.nv.to_string(), + ); + let sub_node_modules = folder_path.join("node_modules"); + let package_path = + join_package_name(&sub_node_modules, &package.id.nv.name); + let cache_folder = cache.package_folder_for_nv(&package.id.nv); + + deno_core::unsync::spawn_blocking({ + let package_path = package_path.clone(); + let sys = sys.clone(); + move || { + clone_dir_recursive(&sys, &cache_folder, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, tags)?; + + Ok::<_, SyncResolutionWithFsError>(()) + } + }) + .await + .map_err(JsErrorBox::from_err)? + .map_err(JsErrorBox::from_err)?; + + if package.bin.is_some() { + bin_entries_to_setup.borrow_mut().add(package, package_path); + } + + if let Some(deprecated) = &package.deprecated { + packages_with_deprecation_warnings + .lock() + .push((package.id.clone(), deprecated.clone())); + } + + // finally stop showing the progress bar + drop(pb_guard); // explicit for clarity + Ok::<_, JsErrorBox>(()) + }); + } else if matches!(package_state, PackageFolderState::TagsOutdated) { + fs::write(initialized_file, tags)?; + } + + let sub_node_modules = folder_path.join("node_modules"); + let package_path = + join_package_name(&sub_node_modules, &package.id.nv.name); + lifecycle_scripts.add(package, package_path.into()); + } + + while let Some(result) = cache_futures.next().await { + result?; // surface the first error + } + + // 2. Create any "copy" packages, which are used for peer dependencies + for package in &package_partitions.copy_packages { + let package_cache_folder_id = package.get_package_cache_folder_id(); + let destination_path = deno_local_registry_dir + .join(get_package_folder_id_folder_name(&package_cache_folder_id)); + let initialized_file = destination_path.join(".initialized"); + if !initialized_file.exists() { + let sub_node_modules = destination_path.join("node_modules"); + let package_path = + join_package_name(&sub_node_modules, &package.id.nv.name); + + let source_path = join_package_name( + &deno_local_registry_dir + .join(get_package_folder_id_folder_name( + &package_cache_folder_id.with_no_count(), + )) + .join("node_modules"), + &package.id.nv.name, + ); + + clone_dir_recursive(sys, &source_path, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + // 3. Symlink all the dependencies into the .deno directory. + // + // Symlink node_modules/.deno//node_modules/ to + // node_modules/.deno//node_modules/ + for package in package_partitions.iter_all() { + let package_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + let sub_node_modules = deno_local_registry_dir + .join(&package_folder_name) + .join("node_modules"); + let mut dep_setup_cache = setup_cache.with_dep(&package_folder_name); + for (name, dep_id) in &package.dependencies { + let dep = snapshot.package_from_id(dep_id).unwrap(); + if package.optional_dependencies.contains(name) + && !dep.system.matches_system(system_info) + { + continue; // this isn't a dependency for the current system + } + let dep_cache_folder_id = dep.get_package_cache_folder_id(); + let dep_folder_name = + get_package_folder_id_folder_name(&dep_cache_folder_id); + if dep_setup_cache.insert(name, &dep_folder_name) { + let dep_folder_path = join_package_name( + &deno_local_registry_dir + .join(dep_folder_name) + .join("node_modules"), + &dep_id.nv.name, + ); + symlink_package_dir( + &dep_folder_path, + &join_package_name(&sub_node_modules, name), + )?; + } + } + } + + let mut found_names: HashMap<&StackString, &PackageNv> = HashMap::new(); + + // set of node_modules in workspace packages that we've already ensured exist + let mut existing_child_node_modules_dirs: HashSet = HashSet::new(); + + // 4. Create symlinks for package json dependencies + { + for remote in npm_install_deps_provider.remote_pkgs() { + let remote_pkg = if let Ok(remote_pkg) = + snapshot.resolve_pkg_from_pkg_req(&remote.req) + { + remote_pkg + } else if remote.req.version_req.tag().is_some() { + // couldn't find a match, and `resolve_best_package_id` + // panics if you give it a tag + continue; + } else if let Some(remote_id) = snapshot + .resolve_best_package_id(&remote.req.name, &remote.req.version_req) + { + snapshot.package_from_id(&remote_id).unwrap() + } else { + continue; // skip, package not found + }; + let Some(remote_alias) = &remote.alias else { + continue; + }; + let alias_clashes = remote.req.name != *remote_alias + && newest_packages_by_name.contains_key(remote_alias); + let install_in_child = { + // we'll install in the child if the alias is taken by another package, or + // if there's already a package with the same name but different version + // linked into the root + match found_names.entry(remote_alias) { + Entry::Occupied(nv) => { + // alias to a different package (in case of duplicate aliases) + // or the version doesn't match the version in the root node_modules + alias_clashes || &remote_pkg.id.nv != *nv.get() + } + Entry::Vacant(entry) => { + entry.insert(&remote_pkg.id.nv); + alias_clashes + } + } + }; + let target_folder_name = get_package_folder_id_folder_name( + &remote_pkg.get_package_cache_folder_id(), + ); + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(&target_folder_name) + .join("node_modules"), + &remote_pkg.id.nv.name, + ); + if install_in_child { + // symlink the dep into the package's child node_modules folder + let dest_node_modules = remote.base_dir.join("node_modules"); + if !existing_child_node_modules_dirs.contains(&dest_node_modules) { + fs::create_dir_all(&dest_node_modules).map_err(|source| { + SyncResolutionWithFsError::Creating { + path: dest_node_modules.clone(), + source, + } + })?; + existing_child_node_modules_dirs.insert(dest_node_modules.clone()); + } + let mut dest_path = dest_node_modules; + dest_path.push(remote_alias); + + symlink_package_dir(&local_registry_package_path, &dest_path)?; + } else { + // symlink the package into `node_modules/` + if setup_cache + .insert_root_symlink(&remote_pkg.id.nv.name, &target_folder_name) + { + symlink_package_dir( + &local_registry_package_path, + &join_package_name(root_node_modules_dir_path, remote_alias), + )?; + } + } + } + } + + // 5. Create symlinks for the remaining top level packages in the node_modules folder. + // (These may be present if they are not in the package.json dependencies) + // Symlink node_modules/.deno//node_modules/ to + // node_modules/ + let mut ids = snapshot + .top_level_packages() + .filter(|f| !found_names.contains_key(&f.nv.name)) + .collect::>(); + ids.sort_by(|a, b| b.cmp(a)); // create determinism and only include the latest version + for id in ids { + match found_names.entry(&id.nv.name) { + Entry::Occupied(_) => { + continue; // skip, already handled + } + Entry::Vacant(entry) => { + entry.insert(&id.nv); + } + } + let package = snapshot.package_from_id(id).unwrap(); + let target_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + if setup_cache.insert_root_symlink(&id.nv.name, &target_folder_name) { + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(target_folder_name) + .join("node_modules"), + &id.nv.name, + ); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(root_node_modules_dir_path, &id.nv.name), + )?; + } + } + + // 6. Create a node_modules/.deno/node_modules/ directory with + // the remaining packages + for package in newest_packages_by_name.values() { + match found_names.entry(&package.id.nv.name) { + Entry::Occupied(_) => { + continue; // skip, already handled + } + Entry::Vacant(entry) => { + entry.insert(&package.id.nv); + } + } + + let target_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + if setup_cache.insert_deno_symlink(&package.id.nv.name, &target_folder_name) + { + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(target_folder_name) + .join("node_modules"), + &package.id.nv.name, + ); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(&deno_node_modules_dir, &package.id.nv.name), + )?; + } + } + + // 7. Set up `node_modules/.bin` entries for packages that need it. + { + let bin_entries = std::mem::take(&mut *bin_entries.borrow_mut()); + bin_entries.finish( + snapshot, + &bin_node_modules_dir_path, + |setup_outcome| { + match setup_outcome { + bin_entries::EntrySetupOutcome::MissingEntrypoint { + package, + package_path, + .. + } if super::common::lifecycle_scripts::has_lifecycle_scripts( + package, + package_path, + ) && lifecycle_scripts.can_run_scripts(&package.id.nv) + && !lifecycle_scripts.has_run_scripts(package) => + { + // ignore, it might get fixed when the lifecycle scripts run. + // if not, we'll warn then + } + outcome => outcome.warn_if_failed(), + } + }, + )?; + } + + // 8. Create symlinks for the workspace packages + { + // todo(dsherret): this is not exactly correct because it should + // install correctly for a workspace (potentially in sub directories), + // but this is good enough for a first pass + for workspace in npm_install_deps_provider.workspace_pkgs() { + let Some(workspace_alias) = &workspace.alias else { + continue; + }; + symlink_package_dir( + &workspace.target_dir, + &root_node_modules_dir_path.join(workspace_alias), + )?; + } + } + + { + let packages_with_deprecation_warnings = + packages_with_deprecation_warnings.lock(); + if !packages_with_deprecation_warnings.is_empty() { + log::warn!( + "{} The following packages are deprecated:", + colors::yellow("Warning") + ); + let len = packages_with_deprecation_warnings.len(); + for (idx, (package_id, msg)) in + packages_with_deprecation_warnings.iter().enumerate() + { + if idx != len - 1 { + log::warn!( + "┠─ {}", + colors::gray(format!("npm:{:?} ({})", package_id, msg)) + ); + } else { + log::warn!( + "┖─ {}", + colors::gray(format!("npm:{:?} ({})", package_id, msg)) + ); + } + } + } + } + + lifecycle_scripts + .finish( + snapshot, + &package_partitions.packages, + root_node_modules_dir_path, + progress_bar, + ) + .await?; + + setup_cache.save(); + drop(single_process_lock); + drop(pb_clear_guard); + + Ok(()) +} + +/// `node_modules/.deno//` +fn local_node_modules_package_folder( + local_registry_dir: &Path, + package: &NpmResolutionPackage, +) -> PathBuf { + local_registry_dir.join(get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) +} + +struct LocalLifecycleScripts<'a> { + deno_local_registry_dir: &'a Path, +} + +impl<'a> LocalLifecycleScripts<'a> { + /// `node_modules/.deno//.scripts-run` + fn ran_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { + local_node_modules_package_folder(self.deno_local_registry_dir, package) + .join(".scripts-run") + } + + /// `node_modules/.deno//.scripts-warned` + fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { + local_node_modules_package_folder(self.deno_local_registry_dir, package) + .join(".scripts-warned") + } +} + +impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy + for LocalLifecycleScripts<'a> +{ + fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf { + local_node_modules_package_contents_path( + self.deno_local_registry_dir, + package, + ) + } + + fn did_run_scripts( + &self, + package: &NpmResolutionPackage, + ) -> std::result::Result<(), std::io::Error> { + std::fs::write(self.ran_scripts_file(package), "")?; + Ok(()) + } + + fn warn_on_scripts_not_run( + &self, + packages: &[(&NpmResolutionPackage, std::path::PathBuf)], + ) -> Result<(), std::io::Error> { + if !packages.is_empty() { + log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall")); + + for (package, _) in packages { + log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv))); + } + + log::warn!("┃"); + log::warn!( + "┠─ {}", + colors::italic("This may cause the packages to not work correctly.") + ); + log::warn!("┖─ {}", colors::italic("To run lifecycle scripts, use the `--allow-scripts` flag with `deno install`:")); + let packages_comma_separated = packages + .iter() + .map(|(p, _)| format!("npm:{}", p.id.nv)) + .collect::>() + .join(","); + log::warn!( + " {}", + colors::bold(format!( + "deno install --allow-scripts={}", + packages_comma_separated + )) + ); + + for (package, _) in packages { + let _ignore_err = fs::write(self.warned_scripts_file(package), ""); + } + } + Ok(()) + } + + fn has_warned(&self, package: &NpmResolutionPackage) -> bool { + self.warned_scripts_file(package).exists() + } + + fn has_run(&self, package: &NpmResolutionPackage) -> bool { + self.ran_scripts_file(package).exists() + } +} + +// Uses BTreeMap to preserve the ordering of the elements in memory, to ensure +// the file generated from this datastructure is deterministic. +// See: https://github.com/denoland/deno/issues/24479 +/// Represents a dependency at `node_modules/.deno//` +struct SetupCacheDep<'a> { + previous: Option<&'a BTreeMap>, + current: &'a mut BTreeMap, +} + +impl<'a> SetupCacheDep<'a> { + pub fn insert(&mut self, name: &str, target_folder_name: &str) -> bool { + self + .current + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self.previous.and_then(|p| p.get(name)) { + previous_target != target_folder_name + } else { + true + } + } +} + +// Uses BTreeMap to preserve the ordering of the elements in memory, to ensure +// the file generated from this datastructure is deterministic. +// See: https://github.com/denoland/deno/issues/24479 +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct SetupCacheData { + root_symlinks: BTreeMap, + deno_symlinks: BTreeMap, + dep_symlinks: BTreeMap>, +} + +/// It is very slow to try to re-setup the symlinks each time, so this will +/// cache what we've setup on the last run and only update what is necessary. +/// Obviously this could lead to issues if the cache gets out of date with the +/// file system, such as if the user manually deletes a symlink. +struct SetupCache { + file_path: PathBuf, + previous: Option, + current: SetupCacheData, +} + +impl SetupCache { + pub fn load(file_path: PathBuf) -> Self { + let previous = std::fs::read(&file_path) + .ok() + .and_then(|data| bincode::deserialize(&data).ok()); + Self { + file_path, + previous, + current: Default::default(), + } + } + + pub fn save(&self) -> bool { + if let Some(previous) = &self.previous { + if previous == &self.current { + return false; // nothing to save + } + } + + bincode::serialize(&self.current).ok().and_then(|data| { + atomic_write_file_with_retries( + &CliSys::default(), + &self.file_path, + &data, + CACHE_PERM, + ) + .ok() + }); + true + } + + /// Inserts and checks for the existence of a root symlink + /// at `node_modules/` pointing to + /// `node_modules/.deno//` + pub fn insert_root_symlink( + &mut self, + name: &str, + target_folder_name: &str, + ) -> bool { + self + .current + .root_symlinks + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self + .previous + .as_ref() + .and_then(|p| p.root_symlinks.get(name)) + { + previous_target != target_folder_name + } else { + true + } + } + + /// Inserts and checks for the existence of a symlink at + /// `node_modules/.deno/node_modules/` pointing to + /// `node_modules/.deno//` + pub fn insert_deno_symlink( + &mut self, + name: &str, + target_folder_name: &str, + ) -> bool { + self + .current + .deno_symlinks + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self + .previous + .as_ref() + .and_then(|p| p.deno_symlinks.get(name)) + { + previous_target != target_folder_name + } else { + true + } + } + + pub fn remove_dep(&mut self, parent_name: &str) { + if let Some(previous) = &mut self.previous { + previous.dep_symlinks.remove(parent_name); + } + } + + pub fn with_dep(&mut self, parent_name: &str) -> SetupCacheDep<'_> { + SetupCacheDep { + previous: self + .previous + .as_ref() + .and_then(|p| p.dep_symlinks.get(parent_name)), + current: self + .current + .dep_symlinks + .entry(parent_name.to_string()) + .or_default(), + } + } +} + +#[derive(Debug, thiserror::Error, deno_error::JsError)] +pub enum SymlinkPackageDirError { + #[class(inherit)] + #[error("Creating '{parent}'")] + Creating { + parent: PathBuf, + #[source] + #[inherit] + source: std::io::Error, + }, + #[class(inherit)] + #[error(transparent)] + Other(#[from] std::io::Error), + #[cfg(windows)] + #[class(inherit)] + #[error("Creating junction in node_modules folder")] + FailedCreatingJunction { + #[source] + #[inherit] + source: std::io::Error, + }, +} + +fn symlink_package_dir( + old_path: &Path, + new_path: &Path, +) -> Result<(), SymlinkPackageDirError> { + let new_parent = new_path.parent().unwrap(); + if new_parent.file_name().unwrap() != "node_modules" { + // create the parent folder that will contain the symlink + fs::create_dir_all(new_parent).map_err(|source| { + SymlinkPackageDirError::Creating { + parent: new_parent.to_path_buf(), + source, + } + })?; + } + + // need to delete the previous symlink before creating a new one + let _ignore = fs::remove_dir_all(new_path); + + let old_path_relative = + crate::util::path::relative_path(new_parent, old_path) + .unwrap_or_else(|| old_path.to_path_buf()); + + #[cfg(windows)] + { + junction_or_symlink_dir(&old_path_relative, old_path, new_path) + } + #[cfg(not(windows))] + { + symlink_dir(&crate::sys::CliSys::default(), &old_path_relative, new_path) + .map_err(Into::into) + } +} + +#[cfg(windows)] +fn junction_or_symlink_dir( + old_path_relative: &Path, + old_path: &Path, + new_path: &Path, +) -> Result<(), SymlinkPackageDirError> { + static USE_JUNCTIONS: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + + if USE_JUNCTIONS.load(std::sync::atomic::Ordering::Relaxed) { + // Use junctions because they're supported on ntfs file systems without + // needing to elevate privileges on Windows. + // Note: junctions don't support relative paths, so we need to use the + // absolute path here. + return junction::create(old_path, new_path).map_err(|source| { + SymlinkPackageDirError::FailedCreatingJunction { source } + }); + } + + match symlink_dir(&crate::sys::CliSys::default(), old_path_relative, new_path) + { + Ok(()) => Ok(()), + Err(symlink_err) + if symlink_err.kind() == std::io::ErrorKind::PermissionDenied => + { + USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); + junction::create(old_path, new_path).map_err(|source| { + SymlinkPackageDirError::FailedCreatingJunction { source } + }) + } + Err(symlink_err) => { + log::warn!( + "{} Unexpected error symlinking node_modules: {symlink_err}", + colors::yellow("Warning") + ); + USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); + junction::create(old_path, new_path).map_err(|source| { + SymlinkPackageDirError::FailedCreatingJunction { source } + }) + } + } +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} + +#[cfg(test)] +mod test { + use test_util::TempDir; + + use super::*; + + #[test] + fn test_setup_cache() { + let temp_dir = TempDir::new(); + let cache_bin_path = temp_dir.path().join("cache.bin").to_path_buf(); + let mut cache = SetupCache::load(cache_bin_path.clone()); + assert!(cache.insert_deno_symlink("package-a", "package-a@1.0.0")); + assert!(cache.insert_root_symlink("package-a", "package-a@1.0.0")); + assert!(cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + assert!(cache.save()); + + let mut cache = SetupCache::load(cache_bin_path.clone()); + assert!(!cache.insert_deno_symlink("package-a", "package-a@1.0.0")); + assert!(!cache.insert_root_symlink("package-a", "package-a@1.0.0")); + assert!(!cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + assert!(!cache.save()); + assert!(cache.insert_root_symlink("package-b", "package-b@0.2.0")); + assert!(cache.save()); + + let mut cache = SetupCache::load(cache_bin_path); + cache.remove_dep("package-a"); + assert!(cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + } +} diff --git a/cli/npm/managed/installers/mod.rs b/cli/npm/managed/installers/mod.rs new file mode 100644 index 0000000000..4514fa1ec3 --- /dev/null +++ b/cli/npm/managed/installers/mod.rs @@ -0,0 +1,55 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::path::PathBuf; +use std::sync::Arc; + +use deno_npm::NpmSystemInfo; + +pub use self::common::NpmPackageFsInstaller; +use self::global::GlobalNpmPackageInstaller; +use self::local::LocalNpmPackageInstaller; +use super::resolution::NpmResolution; +use crate::args::LifecycleScriptsConfig; +use crate::args::NpmInstallDepsProvider; +use crate::npm::CliNpmCache; +use crate::npm::CliNpmTarballCache; +use crate::sys::CliSys; +use crate::util::progress_bar::ProgressBar; + +mod common; +mod global; +mod local; + +#[allow(clippy::too_many_arguments)] +pub fn create_npm_fs_installer( + npm_cache: Arc, + npm_install_deps_provider: &Arc, + progress_bar: &ProgressBar, + resolution: Arc, + sys: CliSys, + tarball_cache: Arc, + maybe_node_modules_path: Option, + system_info: NpmSystemInfo, + lifecycle_scripts: LifecycleScriptsConfig, +) -> Arc { + match maybe_node_modules_path { + Some(node_modules_folder) => Arc::new(LocalNpmPackageInstaller::new( + npm_cache, + npm_install_deps_provider.clone(), + progress_bar.clone(), + resolution, + sys, + tarball_cache, + node_modules_folder, + system_info, + lifecycle_scripts, + )), + None => Arc::new(GlobalNpmPackageInstaller::new( + npm_cache, + tarball_cache, + resolution, + system_info, + lifecycle_scripts, + )), + } +} diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 831b0b0ba8..6729a56b1e 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -28,11 +28,14 @@ use deno_runtime::colors; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use installer::AddPkgReqsResult; +use installer::NpmResolutionInstaller; +use installers::create_npm_fs_installer; +use installers::NpmPackageFsInstaller; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::InNpmPackageChecker; use node_resolver::NpmPackageFolderResolver; -use resolution::AddPkgReqsResult; use self::resolution::NpmResolution; use self::resolvers::create_npm_fs_resolver; @@ -55,6 +58,8 @@ use crate::sys::CliSys; use crate::util::progress_bar::ProgressBar; use crate::util::sync::AtomicFlag; +mod installer; +mod installers; mod resolution; mod resolvers; @@ -156,11 +161,7 @@ fn create_inner( snapshot: Option, lifecycle_scripts: LifecycleScriptsConfig, ) -> Arc { - let resolution = Arc::new(NpmResolution::from_serialized( - registry_info_provider.clone(), - snapshot, - maybe_lockfile.clone(), - )); + let resolution = Arc::new(NpmResolution::from_serialized(snapshot)); let tarball_cache = Arc::new(CliNpmTarballCache::new( npm_cache.clone(), http_client, @@ -168,18 +169,25 @@ fn create_inner( npm_rc.clone(), )); - let fs_resolver = create_npm_fs_resolver( + let fs_installer = create_npm_fs_installer( npm_cache.clone(), &npm_install_deps_provider, &text_only_progress_bar, resolution.clone(), sys.clone(), tarball_cache.clone(), - node_modules_dir_path, + node_modules_dir_path.clone(), npm_system_info.clone(), lifecycle_scripts.clone(), ); + let fs_resolver = create_npm_fs_resolver( + npm_cache.clone(), + resolution.clone(), + sys.clone(), + node_modules_dir_path, + ); Arc::new(ManagedCliNpmResolver::new( + fs_installer, fs_resolver, maybe_lockfile, registry_info_provider, @@ -301,6 +309,7 @@ pub enum PackageCaching<'a> { /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { + fs_installer: Arc, fs_resolver: Arc, maybe_lockfile: Option>, registry_info_provider: Arc, @@ -308,6 +317,7 @@ pub struct ManagedCliNpmResolver { npm_install_deps_provider: Arc, sys: CliSys, resolution: Arc, + resolution_installer: NpmResolutionInstaller, tarball_cache: Arc, text_only_progress_bar: ProgressBar, npm_system_info: NpmSystemInfo, @@ -348,6 +358,7 @@ pub enum ResolvePkgFolderFromDenoModuleError { impl ManagedCliNpmResolver { #[allow(clippy::too_many_arguments)] pub fn new( + fs_installer: Arc, fs_resolver: Arc, maybe_lockfile: Option>, registry_info_provider: Arc, @@ -360,7 +371,13 @@ impl ManagedCliNpmResolver { npm_system_info: NpmSystemInfo, lifecycle_scripts: LifecycleScriptsConfig, ) -> Self { + let resolution_installer = NpmResolutionInstaller::new( + registry_info_provider.clone(), + resolution.clone(), + maybe_lockfile.clone(), + ); Self { + fs_installer, fs_resolver, maybe_lockfile, registry_info_provider, @@ -368,6 +385,7 @@ impl ManagedCliNpmResolver { npm_install_deps_provider, text_only_progress_bar, resolution, + resolution_installer, sys, tarball_cache, npm_system_info, @@ -489,7 +507,7 @@ impl ManagedCliNpmResolver { }; } - let mut result = self.resolution.add_package_reqs(packages).await; + let mut result = self.resolution_installer.add_package_reqs(packages).await; if result.dependencies_result.is_ok() { if let Some(lockfile) = self.maybe_lockfile.as_ref() { @@ -512,7 +530,7 @@ impl ManagedCliNpmResolver { &self, packages: &[PackageReq], ) -> Result<(), AnyError> { - self.resolution.set_package_reqs(packages).await + self.resolution_installer.set_package_reqs(packages).await } pub fn snapshot(&self) -> NpmResolutionSnapshot { @@ -554,7 +572,7 @@ impl ManagedCliNpmResolver { &self, caching: PackageCaching<'_>, ) -> Result<(), JsErrorBox> { - self.fs_resolver.cache_packages(caching).await + self.fs_installer.cache_packages(caching).await } pub fn resolve_pkg_folder_from_deno_module( @@ -738,14 +756,11 @@ impl CliNpmResolver for ManagedCliNpmResolver { fn clone_snapshotted(&self) -> Arc { // create a new snapshotted npm resolution and resolver - let npm_resolution = Arc::new(NpmResolution::new( - self.registry_info_provider.clone(), - self.resolution.snapshot(), - self.maybe_lockfile.clone(), - )); + let npm_resolution = + Arc::new(NpmResolution::new(self.resolution.snapshot())); Arc::new(ManagedCliNpmResolver::new( - create_npm_fs_resolver( + create_npm_fs_installer( self.npm_cache.clone(), &self.npm_install_deps_provider, &self.text_only_progress_bar, @@ -756,6 +771,12 @@ impl CliNpmResolver for ManagedCliNpmResolver { self.npm_system_info.clone(), self.lifecycle_scripts.clone(), ), + create_npm_fs_resolver( + self.npm_cache.clone(), + npm_resolution.clone(), + self.sys.clone(), + self.root_node_modules_path().map(ToOwned::to_owned), + ), self.maybe_lockfile.clone(), self.registry_info_provider.clone(), self.npm_cache.clone(), diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 8259062c05..5178c20608 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -1,18 +1,9 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::collections::HashMap; -use std::collections::HashSet; -use std::sync::Arc; -use capacity_builder::StringBuilder; -use deno_core::error::AnyError; -use deno_error::JsErrorBox; -use deno_lockfile::NpmPackageDependencyLockfileInfo; -use deno_lockfile::NpmPackageLockfileInfo; -use deno_npm::registry::NpmRegistryApi; -use deno_npm::resolution::AddPkgReqsOptions; +use deno_core::parking_lot::RwLock; use deno_npm::resolution::NpmPackagesPartitioned; -use deno_npm::resolution::NpmResolutionError; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::resolution::PackageCacheFolderIdNotFoundError; use deno_npm::resolution::PackageNotFoundFromReferrerError; @@ -23,25 +14,8 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; -use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; -use deno_semver::SmallStackString; -use deno_semver::VersionReq; - -use crate::args::CliLockfile; -use crate::npm::CliNpmRegistryInfoProvider; -use crate::util::sync::SyncReadAsyncWriteLock; - -pub struct AddPkgReqsResult { - /// Results from adding the individual packages. - /// - /// The indexes of the results correspond to the indexes of the provided - /// package requirements. - pub results: Vec>, - /// The final result of resolving and caching all the package requirements. - pub dependencies_result: Result<(), JsErrorBox>, -} /// Handles updating and storing npm resolution in memory where the underlying /// snapshot can be updated concurrently. Additionally handles updating the lockfile @@ -49,9 +23,7 @@ pub struct AddPkgReqsResult { /// /// This does not interact with the file system. pub struct NpmResolution { - registry_info_provider: Arc, - snapshot: SyncReadAsyncWriteLock, - maybe_lockfile: Option>, + snapshot: RwLock, } impl std::fmt::Debug for NpmResolution { @@ -65,87 +37,19 @@ impl std::fmt::Debug for NpmResolution { impl NpmResolution { pub fn from_serialized( - registry_info_provider: Arc, initial_snapshot: Option, - maybe_lockfile: Option>, ) -> Self { let snapshot = NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default()); - Self::new(registry_info_provider, snapshot, maybe_lockfile) + Self::new(snapshot) } - pub fn new( - registry_info_provider: Arc, - initial_snapshot: NpmResolutionSnapshot, - maybe_lockfile: Option>, - ) -> Self { + pub fn new(initial_snapshot: NpmResolutionSnapshot) -> Self { Self { - registry_info_provider, - snapshot: SyncReadAsyncWriteLock::new(initial_snapshot), - maybe_lockfile, + snapshot: RwLock::new(initial_snapshot), } } - pub async fn add_package_reqs( - &self, - package_reqs: &[PackageReq], - ) -> AddPkgReqsResult { - // only allow one thread in here at a time - let snapshot_lock = self.snapshot.acquire().await; - let result = add_package_reqs_to_snapshot( - &self.registry_info_provider, - package_reqs, - self.maybe_lockfile.clone(), - || snapshot_lock.read().clone(), - ) - .await; - - AddPkgReqsResult { - results: result.results, - dependencies_result: match result.dep_graph_result { - Ok(snapshot) => { - *snapshot_lock.write() = snapshot; - Ok(()) - } - Err(err) => Err(JsErrorBox::from_err(err)), - }, - } - } - - pub async fn set_package_reqs( - &self, - package_reqs: &[PackageReq], - ) -> Result<(), AnyError> { - // only allow one thread in here at a time - let snapshot_lock = self.snapshot.acquire().await; - - let reqs_set = package_reqs.iter().collect::>(); - let snapshot = add_package_reqs_to_snapshot( - &self.registry_info_provider, - package_reqs, - self.maybe_lockfile.clone(), - || { - let snapshot = snapshot_lock.read().clone(); - let has_removed_package = !snapshot - .package_reqs() - .keys() - .all(|req| reqs_set.contains(req)); - // if any packages were removed, we need to completely recreate the npm resolution snapshot - if has_removed_package { - snapshot.into_empty() - } else { - snapshot - } - }, - ) - .await - .into_result()?; - - *snapshot_lock.write() = snapshot; - - Ok(()) - } - pub fn resolve_pkg_cache_folder_id_from_pkg_id( &self, id: &NpmPackageId, @@ -262,112 +166,8 @@ impl NpmResolution { pub fn subset(&self, package_reqs: &[PackageReq]) -> NpmResolutionSnapshot { self.snapshot.read().subset(package_reqs) } -} -async fn add_package_reqs_to_snapshot( - registry_info_provider: &Arc, - package_reqs: &[PackageReq], - maybe_lockfile: Option>, - get_new_snapshot: impl Fn() -> NpmResolutionSnapshot, -) -> deno_npm::resolution::AddPkgReqsResult { - let snapshot = get_new_snapshot(); - if package_reqs - .iter() - .all(|req| snapshot.package_reqs().contains_key(req)) - { - log::debug!("Snapshot already up to date. Skipping npm resolution."); - return deno_npm::resolution::AddPkgReqsResult { - results: package_reqs - .iter() - .map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone())) - .collect(), - dep_graph_result: Ok(snapshot), - }; - } - log::debug!( - /* this string is used in tests */ - "Running npm resolution." - ); - let npm_registry_api = registry_info_provider.as_npm_registry_api(); - let result = snapshot - .add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs)) - .await; - let result = match &result.dep_graph_result { - Err(NpmResolutionError::Resolution(err)) - if npm_registry_api.mark_force_reload() => - { - log::debug!("{err:#}"); - log::debug!("npm resolution failed. Trying again..."); - - // try again with forced reloading - let snapshot = get_new_snapshot(); - snapshot - .add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs)) - .await - } - _ => result, - }; - - registry_info_provider.clear_memory_cache(); - - if let Ok(snapshot) = &result.dep_graph_result { - if let Some(lockfile) = maybe_lockfile { - populate_lockfile_from_snapshot(&lockfile, snapshot); - } - } - - result -} - -fn get_add_pkg_reqs_options(package_reqs: &[PackageReq]) -> AddPkgReqsOptions { - AddPkgReqsOptions { - package_reqs, - // WARNING: When bumping this version, check if anything needs to be - // updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js - types_node_version_req: Some( - VersionReq::parse_from_npm("22.0.0 - 22.5.4").unwrap(), - ), - } -} - -fn populate_lockfile_from_snapshot( - lockfile: &CliLockfile, - snapshot: &NpmResolutionSnapshot, -) { - let mut lockfile = lockfile.lock(); - for (package_req, nv) in snapshot.package_reqs() { - let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id; - lockfile.insert_package_specifier( - JsrDepPackageReq::npm(package_req.clone()), - { - StringBuilder::::build(|builder| { - builder.append(&id.nv.version); - builder.append(&id.peer_dependencies); - }) - .unwrap() - }, - ); - } - for package in snapshot.all_packages_for_every_system() { - lockfile.insert_npm_package(npm_package_to_lockfile_info(package)); - } -} - -fn npm_package_to_lockfile_info( - pkg: &NpmResolutionPackage, -) -> NpmPackageLockfileInfo { - let dependencies = pkg - .dependencies - .iter() - .map(|(name, id)| NpmPackageDependencyLockfileInfo { - name: name.clone(), - id: id.as_serialized(), - }) - .collect(); - - NpmPackageLockfileInfo { - serialized_id: pkg.id.as_serialized(), - integrity: pkg.dist.integrity().for_lockfile(), - dependencies, + pub fn set_snapshot(&self, snapshot: NpmResolutionSnapshot) { + *self.snapshot.write() = snapshot; } } diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 0d5fab10d3..be0b40fa2a 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -1,20 +1,14 @@ // Copyright 2018-2025 the Deno authors. MIT license. -pub mod bin_entries; -pub mod lifecycle_scripts; - use std::path::Path; use std::path::PathBuf; use async_trait::async_trait; use deno_ast::ModuleSpecifier; -use deno_error::JsErrorBox; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use node_resolver::errors::PackageFolderResolveError; -use super::super::PackageCaching; - #[derive(Debug, thiserror::Error, deno_error::JsError)] #[class(generic)] #[error("Package folder not found for '{0}'")] @@ -47,9 +41,4 @@ pub trait NpmPackageFsResolver: Send + Sync { &self, specifier: &ModuleSpecifier, ) -> Result, std::io::Error>; - - async fn cache_packages<'a>( - &self, - caching: PackageCaching<'a>, - ) -> Result<(), JsErrorBox>; } diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 9af35b169d..d63d3a7e45 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -2,59 +2,32 @@ //! Code for global npm cache resolution. -use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; use deno_ast::ModuleSpecifier; -use deno_core::futures::stream::FuturesUnordered; -use deno_core::futures::StreamExt; -use deno_error::JsErrorBox; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_npm::NpmSystemInfo; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::ReferrerNotFoundError; use super::super::resolution::NpmResolution; -use super::common::lifecycle_scripts::LifecycleScriptsStrategy; use super::common::NpmPackageFsResolver; -use crate::args::LifecycleScriptsConfig; -use crate::cache::FastInsecureHasher; -use crate::colors; -use crate::npm::managed::PackageCaching; use crate::npm::CliNpmCache; -use crate::npm::CliNpmTarballCache; /// Resolves packages from the global npm cache. #[derive(Debug)] pub struct GlobalNpmPackageResolver { cache: Arc, - tarball_cache: Arc, resolution: Arc, - system_info: NpmSystemInfo, - lifecycle_scripts: LifecycleScriptsConfig, } impl GlobalNpmPackageResolver { - pub fn new( - cache: Arc, - tarball_cache: Arc, - resolution: Arc, - system_info: NpmSystemInfo, - lifecycle_scripts: LifecycleScriptsConfig, - ) -> Self { - Self { - cache, - tarball_cache, - resolution, - system_info, - lifecycle_scripts, - } + pub fn new(cache: Arc, resolution: Arc) -> Self { + Self { cache, resolution } } } @@ -141,140 +114,4 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { .resolve_package_folder_id_from_specifier(specifier), ) } - - async fn cache_packages<'a>( - &self, - caching: PackageCaching<'a>, - ) -> Result<(), JsErrorBox> { - let package_partitions = match caching { - PackageCaching::All => self - .resolution - .all_system_packages_partitioned(&self.system_info), - PackageCaching::Only(reqs) => self - .resolution - .subset(&reqs) - .all_system_packages_partitioned(&self.system_info), - }; - cache_packages(&package_partitions.packages, &self.tarball_cache) - .await - .map_err(JsErrorBox::from_err)?; - - // create the copy package folders - for copy in package_partitions.copy_packages { - self - .cache - .ensure_copy_package(©.get_package_cache_folder_id()) - .map_err(JsErrorBox::from_err)?; - } - - let mut lifecycle_scripts = - super::common::lifecycle_scripts::LifecycleScripts::new( - &self.lifecycle_scripts, - GlobalLifecycleScripts::new(self, &self.lifecycle_scripts.root_dir), - ); - for package in &package_partitions.packages { - let package_folder = self.cache.package_folder_for_nv(&package.id.nv); - lifecycle_scripts.add(package, Cow::Borrowed(&package_folder)); - } - - lifecycle_scripts - .warn_not_run_scripts() - .map_err(JsErrorBox::from_err)?; - - Ok(()) - } -} - -async fn cache_packages( - packages: &[NpmResolutionPackage], - tarball_cache: &Arc, -) -> Result<(), deno_npm_cache::EnsurePackageError> { - let mut futures_unordered = FuturesUnordered::new(); - for package in packages { - futures_unordered.push(async move { - tarball_cache - .ensure_package(&package.id.nv, &package.dist) - .await - }); - } - while let Some(result) = futures_unordered.next().await { - // surface the first error - result?; - } - Ok(()) -} - -struct GlobalLifecycleScripts<'a> { - resolver: &'a GlobalNpmPackageResolver, - path_hash: u64, -} - -impl<'a> GlobalLifecycleScripts<'a> { - fn new(resolver: &'a GlobalNpmPackageResolver, root_dir: &Path) -> Self { - let mut hasher = FastInsecureHasher::new_without_deno_version(); - hasher.write(root_dir.to_string_lossy().as_bytes()); - let path_hash = hasher.finish(); - Self { - resolver, - path_hash, - } - } - - fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { - self - .package_path(package) - .join(format!(".scripts-warned-{}", self.path_hash)) - } -} - -impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy - for GlobalLifecycleScripts<'a> -{ - fn can_run_scripts(&self) -> bool { - false - } - fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf { - self.resolver.cache.package_folder_for_nv(&package.id.nv) - } - - fn warn_on_scripts_not_run( - &self, - packages: &[(&NpmResolutionPackage, PathBuf)], - ) -> std::result::Result<(), std::io::Error> { - log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall")); - for (package, _) in packages { - log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv))); - } - log::warn!("┃"); - log::warn!( - "┠─ {}", - colors::italic("This may cause the packages to not work correctly.") - ); - log::warn!("┠─ {}", colors::italic("Lifecycle scripts are only supported when using a `node_modules` directory.")); - log::warn!( - "┠─ {}", - colors::italic("Enable it in your deno config file:") - ); - log::warn!("┖─ {}", colors::bold("\"nodeModulesDir\": \"auto\"")); - - for (package, _) in packages { - std::fs::write(self.warned_scripts_file(package), "")?; - } - Ok(()) - } - - fn did_run_scripts( - &self, - _package: &NpmResolutionPackage, - ) -> Result<(), std::io::Error> { - Ok(()) - } - - fn has_warned(&self, package: &NpmResolutionPackage) -> bool { - self.warned_scripts_file(package).exists() - } - - fn has_run(&self, _package: &NpmResolutionPackage) -> bool { - false - } } diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index e1ac3df43b..52d5ac5a9e 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -3,33 +3,16 @@ //! Code for local node_modules resolution. use std::borrow::Cow; -use std::cell::RefCell; -use std::cmp::Ordering; -use std::collections::hash_map::Entry; -use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::collections::HashMap; -use std::collections::HashSet; -use std::fs; use std::path::Path; use std::path::PathBuf; -use std::rc::Rc; use std::sync::Arc; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_cache_dir::npm::mixed_case_package_name_decode; -use deno_core::futures::stream::FuturesUnordered; -use deno_core::futures::StreamExt; -use deno_core::parking_lot::Mutex; use deno_core::url::Url; -use deno_error::JsErrorBox; -use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_npm::NpmSystemInfo; -use deno_path_util::fs::atomic_write_file_with_retries; use deno_path_util::fs::canonicalize_path_maybe_not_exists; use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; use deno_semver::package::PackageNv; @@ -38,68 +21,35 @@ use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::ReferrerNotFoundError; -use serde::Deserialize; -use serde::Serialize; use sys_traits::FsMetadata; use super::super::resolution::NpmResolution; -use super::common::bin_entries; use super::common::NpmPackageFsResolver; -use crate::args::LifecycleScriptsConfig; -use crate::args::NpmInstallDepsProvider; -use crate::cache::CACHE_PERM; -use crate::colors; -use crate::npm::managed::PackageCaching; -use crate::npm::CliNpmCache; -use crate::npm::CliNpmTarballCache; use crate::sys::CliSys; -use crate::util::fs::clone_dir_recursive; -use crate::util::fs::symlink_dir; -use crate::util::fs::LaxSingleProcessFsFlag; -use crate::util::progress_bar::ProgressBar; -use crate::util::progress_bar::ProgressMessagePrompt; /// Resolver that creates a local node_modules directory /// and resolves packages from it. #[derive(Debug)] pub struct LocalNpmPackageResolver { - cache: Arc, - npm_install_deps_provider: Arc, - progress_bar: ProgressBar, resolution: Arc, sys: CliSys, - tarball_cache: Arc, root_node_modules_path: PathBuf, root_node_modules_url: Url, - system_info: NpmSystemInfo, - lifecycle_scripts: LifecycleScriptsConfig, } impl LocalNpmPackageResolver { #[allow(clippy::too_many_arguments)] pub fn new( - cache: Arc, - npm_install_deps_provider: Arc, - progress_bar: ProgressBar, resolution: Arc, sys: CliSys, - tarball_cache: Arc, node_modules_folder: PathBuf, - system_info: NpmSystemInfo, - lifecycle_scripts: LifecycleScriptsConfig, ) -> Self { Self { - cache, - npm_install_deps_provider, - progress_bar, resolution, - tarball_cache, sys, root_node_modules_url: Url::from_directory_path(&node_modules_folder) .unwrap(), root_node_modules_path: node_modules_folder, - system_info, - lifecycle_scripts, } } @@ -246,809 +196,9 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { &folder_name.to_string_lossy(), )) } - - async fn cache_packages<'a>( - &self, - caching: PackageCaching<'a>, - ) -> Result<(), JsErrorBox> { - let snapshot = match caching { - PackageCaching::All => self.resolution.snapshot(), - PackageCaching::Only(reqs) => self.resolution.subset(&reqs), - }; - sync_resolution_with_fs( - &snapshot, - &self.cache, - &self.npm_install_deps_provider, - &self.progress_bar, - &self.tarball_cache, - &self.root_node_modules_path, - &self.system_info, - &self.lifecycle_scripts, - ) - .await - .map_err(JsErrorBox::from_err) - } } -/// `node_modules/.deno//node_modules/` -/// -/// Where the actual package is stored. -fn local_node_modules_package_contents_path( - local_registry_dir: &Path, - package: &NpmResolutionPackage, -) -> PathBuf { - local_registry_dir - .join(get_package_folder_id_folder_name( - &package.get_package_cache_folder_id(), - )) - .join("node_modules") - .join(&package.id.nv.name) -} - -#[derive(Debug, thiserror::Error, deno_error::JsError)] -pub enum SyncResolutionWithFsError { - #[class(inherit)] - #[error("Creating '{path}'")] - Creating { - path: PathBuf, - #[source] - #[inherit] - source: std::io::Error, - }, - #[class(inherit)] - #[error(transparent)] - CopyDirRecursive(#[from] crate::util::fs::CopyDirRecursiveError), - #[class(inherit)] - #[error(transparent)] - SymlinkPackageDir(#[from] SymlinkPackageDirError), - #[class(inherit)] - #[error(transparent)] - BinEntries(#[from] bin_entries::BinEntriesError), - #[class(inherit)] - #[error(transparent)] - LifecycleScripts( - #[from] super::common::lifecycle_scripts::LifecycleScriptsError, - ), - #[class(inherit)] - #[error(transparent)] - Io(#[from] std::io::Error), - #[class(inherit)] - #[error(transparent)] - Other(#[from] JsErrorBox), -} - -/// Creates a pnpm style folder structure. -#[allow(clippy::too_many_arguments)] -async fn sync_resolution_with_fs( - snapshot: &NpmResolutionSnapshot, - cache: &Arc, - npm_install_deps_provider: &NpmInstallDepsProvider, - progress_bar: &ProgressBar, - tarball_cache: &Arc, - root_node_modules_dir_path: &Path, - system_info: &NpmSystemInfo, - lifecycle_scripts: &LifecycleScriptsConfig, -) -> Result<(), SyncResolutionWithFsError> { - if snapshot.is_empty() - && npm_install_deps_provider.workspace_pkgs().is_empty() - { - return Ok(()); // don't create the directory - } - - // don't set up node_modules (and more importantly try to acquire the file lock) - // if we're running as part of a lifecycle script - if super::common::lifecycle_scripts::is_running_lifecycle_script() { - return Ok(()); - } - - let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); - let deno_node_modules_dir = deno_local_registry_dir.join("node_modules"); - fs::create_dir_all(&deno_node_modules_dir).map_err(|source| { - SyncResolutionWithFsError::Creating { - path: deno_local_registry_dir.to_path_buf(), - source, - } - })?; - let bin_node_modules_dir_path = root_node_modules_dir_path.join(".bin"); - fs::create_dir_all(&bin_node_modules_dir_path).map_err(|source| { - SyncResolutionWithFsError::Creating { - path: deno_local_registry_dir.to_path_buf(), - source, - } - })?; - - let single_process_lock = LaxSingleProcessFsFlag::lock( - deno_local_registry_dir.join(".deno.lock"), - // similar message used by cargo build - "waiting for file lock on node_modules directory", - ) - .await; - - // load this after we get the directory lock - let mut setup_cache = - SetupCache::load(deno_local_registry_dir.join(".setup-cache.bin")); - - let pb_clear_guard = progress_bar.clear_guard(); // prevent flickering - - // 1. Write all the packages out the .deno directory. - // - // Copy (hardlink in future) // to - // node_modules/.deno//node_modules/ - let package_partitions = - snapshot.all_system_packages_partitioned(system_info); - let mut cache_futures = FuturesUnordered::new(); - let mut newest_packages_by_name: HashMap< - &StackString, - &NpmResolutionPackage, - > = HashMap::with_capacity(package_partitions.packages.len()); - let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new())); - let mut lifecycle_scripts = - super::common::lifecycle_scripts::LifecycleScripts::new( - lifecycle_scripts, - LocalLifecycleScripts { - deno_local_registry_dir: &deno_local_registry_dir, - }, - ); - let packages_with_deprecation_warnings = Arc::new(Mutex::new(Vec::new())); - - let mut package_tags: HashMap<&PackageNv, BTreeSet<&str>> = HashMap::new(); - for (package_req, package_nv) in snapshot.package_reqs() { - if let Some(tag) = package_req.version_req.tag() { - package_tags.entry(package_nv).or_default().insert(tag); - } - } - - for package in &package_partitions.packages { - if let Some(current_pkg) = - newest_packages_by_name.get_mut(&package.id.nv.name) - { - if current_pkg.id.nv.cmp(&package.id.nv) == Ordering::Less { - *current_pkg = package; - } - } else { - newest_packages_by_name.insert(&package.id.nv.name, package); - }; - - let package_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - let folder_path = deno_local_registry_dir.join(&package_folder_name); - let tags = package_tags - .get(&package.id.nv) - .map(|tags| { - capacity_builder::StringBuilder::::build(|builder| { - for (i, tag) in tags.iter().enumerate() { - if i > 0 { - builder.append(',') - } - builder.append(*tag); - } - }) - .unwrap() - }) - .unwrap_or_default(); - enum PackageFolderState { - UpToDate, - Uninitialized, - TagsOutdated, - } - let initialized_file = folder_path.join(".initialized"); - let package_state = std::fs::read_to_string(&initialized_file) - .map(|s| { - if s != tags { - PackageFolderState::TagsOutdated - } else { - PackageFolderState::UpToDate - } - }) - .unwrap_or(PackageFolderState::Uninitialized); - if !cache - .cache_setting() - .should_use_for_npm_package(&package.id.nv.name) - || matches!(package_state, PackageFolderState::Uninitialized) - { - // cache bust the dep from the dep setup cache so the symlinks - // are forced to be recreated - setup_cache.remove_dep(&package_folder_name); - - let folder_path = folder_path.clone(); - let bin_entries_to_setup = bin_entries.clone(); - let packages_with_deprecation_warnings = - packages_with_deprecation_warnings.clone(); - - cache_futures.push(async move { - tarball_cache - .ensure_package(&package.id.nv, &package.dist) - .await - .map_err(JsErrorBox::from_err)?; - let pb_guard = progress_bar.update_with_prompt( - ProgressMessagePrompt::Initialize, - &package.id.nv.to_string(), - ); - let sub_node_modules = folder_path.join("node_modules"); - let package_path = - join_package_name(&sub_node_modules, &package.id.nv.name); - let cache_folder = cache.package_folder_for_nv(&package.id.nv); - - deno_core::unsync::spawn_blocking({ - let package_path = package_path.clone(); - move || { - clone_dir_recursive( - &crate::sys::CliSys::default(), - &cache_folder, - &package_path, - )?; - // write out a file that indicates this folder has been initialized - fs::write(initialized_file, tags)?; - - Ok::<_, SyncResolutionWithFsError>(()) - } - }) - .await - .map_err(JsErrorBox::from_err)? - .map_err(JsErrorBox::from_err)?; - - if package.bin.is_some() { - bin_entries_to_setup.borrow_mut().add(package, package_path); - } - - if let Some(deprecated) = &package.deprecated { - packages_with_deprecation_warnings - .lock() - .push((package.id.clone(), deprecated.clone())); - } - - // finally stop showing the progress bar - drop(pb_guard); // explicit for clarity - Ok::<_, JsErrorBox>(()) - }); - } else if matches!(package_state, PackageFolderState::TagsOutdated) { - fs::write(initialized_file, tags)?; - } - - let sub_node_modules = folder_path.join("node_modules"); - let package_path = - join_package_name(&sub_node_modules, &package.id.nv.name); - lifecycle_scripts.add(package, package_path.into()); - } - - while let Some(result) = cache_futures.next().await { - result?; // surface the first error - } - - // 2. Create any "copy" packages, which are used for peer dependencies - for package in &package_partitions.copy_packages { - let package_cache_folder_id = package.get_package_cache_folder_id(); - let destination_path = deno_local_registry_dir - .join(get_package_folder_id_folder_name(&package_cache_folder_id)); - let initialized_file = destination_path.join(".initialized"); - if !initialized_file.exists() { - let sub_node_modules = destination_path.join("node_modules"); - let package_path = - join_package_name(&sub_node_modules, &package.id.nv.name); - - let source_path = join_package_name( - &deno_local_registry_dir - .join(get_package_folder_id_folder_name( - &package_cache_folder_id.with_no_count(), - )) - .join("node_modules"), - &package.id.nv.name, - ); - - clone_dir_recursive( - &crate::sys::CliSys::default(), - &source_path, - &package_path, - )?; - // write out a file that indicates this folder has been initialized - fs::write(initialized_file, "")?; - } - } - - // 3. Symlink all the dependencies into the .deno directory. - // - // Symlink node_modules/.deno//node_modules/ to - // node_modules/.deno//node_modules/ - for package in package_partitions.iter_all() { - let package_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - let sub_node_modules = deno_local_registry_dir - .join(&package_folder_name) - .join("node_modules"); - let mut dep_setup_cache = setup_cache.with_dep(&package_folder_name); - for (name, dep_id) in &package.dependencies { - let dep = snapshot.package_from_id(dep_id).unwrap(); - if package.optional_dependencies.contains(name) - && !dep.system.matches_system(system_info) - { - continue; // this isn't a dependency for the current system - } - let dep_cache_folder_id = dep.get_package_cache_folder_id(); - let dep_folder_name = - get_package_folder_id_folder_name(&dep_cache_folder_id); - if dep_setup_cache.insert(name, &dep_folder_name) { - let dep_folder_path = join_package_name( - &deno_local_registry_dir - .join(dep_folder_name) - .join("node_modules"), - &dep_id.nv.name, - ); - symlink_package_dir( - &dep_folder_path, - &join_package_name(&sub_node_modules, name), - )?; - } - } - } - - let mut found_names: HashMap<&StackString, &PackageNv> = HashMap::new(); - - // set of node_modules in workspace packages that we've already ensured exist - let mut existing_child_node_modules_dirs: HashSet = HashSet::new(); - - // 4. Create symlinks for package json dependencies - { - for remote in npm_install_deps_provider.remote_pkgs() { - let remote_pkg = if let Ok(remote_pkg) = - snapshot.resolve_pkg_from_pkg_req(&remote.req) - { - remote_pkg - } else if remote.req.version_req.tag().is_some() { - // couldn't find a match, and `resolve_best_package_id` - // panics if you give it a tag - continue; - } else if let Some(remote_id) = snapshot - .resolve_best_package_id(&remote.req.name, &remote.req.version_req) - { - snapshot.package_from_id(&remote_id).unwrap() - } else { - continue; // skip, package not found - }; - let Some(remote_alias) = &remote.alias else { - continue; - }; - let alias_clashes = remote.req.name != *remote_alias - && newest_packages_by_name.contains_key(remote_alias); - let install_in_child = { - // we'll install in the child if the alias is taken by another package, or - // if there's already a package with the same name but different version - // linked into the root - match found_names.entry(remote_alias) { - Entry::Occupied(nv) => { - // alias to a different package (in case of duplicate aliases) - // or the version doesn't match the version in the root node_modules - alias_clashes || &remote_pkg.id.nv != *nv.get() - } - Entry::Vacant(entry) => { - entry.insert(&remote_pkg.id.nv); - alias_clashes - } - } - }; - let target_folder_name = get_package_folder_id_folder_name( - &remote_pkg.get_package_cache_folder_id(), - ); - let local_registry_package_path = join_package_name( - &deno_local_registry_dir - .join(&target_folder_name) - .join("node_modules"), - &remote_pkg.id.nv.name, - ); - if install_in_child { - // symlink the dep into the package's child node_modules folder - let dest_node_modules = remote.base_dir.join("node_modules"); - if !existing_child_node_modules_dirs.contains(&dest_node_modules) { - fs::create_dir_all(&dest_node_modules).map_err(|source| { - SyncResolutionWithFsError::Creating { - path: dest_node_modules.clone(), - source, - } - })?; - existing_child_node_modules_dirs.insert(dest_node_modules.clone()); - } - let mut dest_path = dest_node_modules; - dest_path.push(remote_alias); - - symlink_package_dir(&local_registry_package_path, &dest_path)?; - } else { - // symlink the package into `node_modules/` - if setup_cache - .insert_root_symlink(&remote_pkg.id.nv.name, &target_folder_name) - { - symlink_package_dir( - &local_registry_package_path, - &join_package_name(root_node_modules_dir_path, remote_alias), - )?; - } - } - } - } - - // 5. Create symlinks for the remaining top level packages in the node_modules folder. - // (These may be present if they are not in the package.json dependencies) - // Symlink node_modules/.deno//node_modules/ to - // node_modules/ - let mut ids = snapshot - .top_level_packages() - .filter(|f| !found_names.contains_key(&f.nv.name)) - .collect::>(); - ids.sort_by(|a, b| b.cmp(a)); // create determinism and only include the latest version - for id in ids { - match found_names.entry(&id.nv.name) { - Entry::Occupied(_) => { - continue; // skip, already handled - } - Entry::Vacant(entry) => { - entry.insert(&id.nv); - } - } - let package = snapshot.package_from_id(id).unwrap(); - let target_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - if setup_cache.insert_root_symlink(&id.nv.name, &target_folder_name) { - let local_registry_package_path = join_package_name( - &deno_local_registry_dir - .join(target_folder_name) - .join("node_modules"), - &id.nv.name, - ); - - symlink_package_dir( - &local_registry_package_path, - &join_package_name(root_node_modules_dir_path, &id.nv.name), - )?; - } - } - - // 6. Create a node_modules/.deno/node_modules/ directory with - // the remaining packages - for package in newest_packages_by_name.values() { - match found_names.entry(&package.id.nv.name) { - Entry::Occupied(_) => { - continue; // skip, already handled - } - Entry::Vacant(entry) => { - entry.insert(&package.id.nv); - } - } - - let target_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - if setup_cache.insert_deno_symlink(&package.id.nv.name, &target_folder_name) - { - let local_registry_package_path = join_package_name( - &deno_local_registry_dir - .join(target_folder_name) - .join("node_modules"), - &package.id.nv.name, - ); - - symlink_package_dir( - &local_registry_package_path, - &join_package_name(&deno_node_modules_dir, &package.id.nv.name), - )?; - } - } - - // 7. Set up `node_modules/.bin` entries for packages that need it. - { - let bin_entries = std::mem::take(&mut *bin_entries.borrow_mut()); - bin_entries.finish( - snapshot, - &bin_node_modules_dir_path, - |setup_outcome| { - match setup_outcome { - bin_entries::EntrySetupOutcome::MissingEntrypoint { - package, - package_path, - .. - } if super::common::lifecycle_scripts::has_lifecycle_scripts( - package, - package_path, - ) && lifecycle_scripts.can_run_scripts(&package.id.nv) - && !lifecycle_scripts.has_run_scripts(package) => - { - // ignore, it might get fixed when the lifecycle scripts run. - // if not, we'll warn then - } - outcome => outcome.warn_if_failed(), - } - }, - )?; - } - - // 8. Create symlinks for the workspace packages - { - // todo(dsherret): this is not exactly correct because it should - // install correctly for a workspace (potentially in sub directories), - // but this is good enough for a first pass - for workspace in npm_install_deps_provider.workspace_pkgs() { - let Some(workspace_alias) = &workspace.alias else { - continue; - }; - symlink_package_dir( - &workspace.target_dir, - &root_node_modules_dir_path.join(workspace_alias), - )?; - } - } - - { - let packages_with_deprecation_warnings = - packages_with_deprecation_warnings.lock(); - if !packages_with_deprecation_warnings.is_empty() { - log::warn!( - "{} The following packages are deprecated:", - colors::yellow("Warning") - ); - let len = packages_with_deprecation_warnings.len(); - for (idx, (package_id, msg)) in - packages_with_deprecation_warnings.iter().enumerate() - { - if idx != len - 1 { - log::warn!( - "┠─ {}", - colors::gray(format!("npm:{:?} ({})", package_id, msg)) - ); - } else { - log::warn!( - "┖─ {}", - colors::gray(format!("npm:{:?} ({})", package_id, msg)) - ); - } - } - } - } - - lifecycle_scripts - .finish( - snapshot, - &package_partitions.packages, - root_node_modules_dir_path, - progress_bar, - ) - .await?; - - setup_cache.save(); - drop(single_process_lock); - drop(pb_clear_guard); - - Ok(()) -} - -/// `node_modules/.deno//` -fn local_node_modules_package_folder( - local_registry_dir: &Path, - package: &NpmResolutionPackage, -) -> PathBuf { - local_registry_dir.join(get_package_folder_id_folder_name( - &package.get_package_cache_folder_id(), - )) -} - -struct LocalLifecycleScripts<'a> { - deno_local_registry_dir: &'a Path, -} - -impl<'a> LocalLifecycleScripts<'a> { - /// `node_modules/.deno//.scripts-run` - fn ran_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { - local_node_modules_package_folder(self.deno_local_registry_dir, package) - .join(".scripts-run") - } - - /// `node_modules/.deno//.scripts-warned` - fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf { - local_node_modules_package_folder(self.deno_local_registry_dir, package) - .join(".scripts-warned") - } -} - -impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy - for LocalLifecycleScripts<'a> -{ - fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf { - local_node_modules_package_contents_path( - self.deno_local_registry_dir, - package, - ) - } - - fn did_run_scripts( - &self, - package: &NpmResolutionPackage, - ) -> std::result::Result<(), std::io::Error> { - std::fs::write(self.ran_scripts_file(package), "")?; - Ok(()) - } - - fn warn_on_scripts_not_run( - &self, - packages: &[(&NpmResolutionPackage, std::path::PathBuf)], - ) -> Result<(), std::io::Error> { - if !packages.is_empty() { - log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall")); - - for (package, _) in packages { - log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv))); - } - - log::warn!("┃"); - log::warn!( - "┠─ {}", - colors::italic("This may cause the packages to not work correctly.") - ); - log::warn!("┖─ {}", colors::italic("To run lifecycle scripts, use the `--allow-scripts` flag with `deno install`:")); - let packages_comma_separated = packages - .iter() - .map(|(p, _)| format!("npm:{}", p.id.nv)) - .collect::>() - .join(","); - log::warn!( - " {}", - colors::bold(format!( - "deno install --allow-scripts={}", - packages_comma_separated - )) - ); - - for (package, _) in packages { - let _ignore_err = fs::write(self.warned_scripts_file(package), ""); - } - } - Ok(()) - } - - fn has_warned(&self, package: &NpmResolutionPackage) -> bool { - self.warned_scripts_file(package).exists() - } - - fn has_run(&self, package: &NpmResolutionPackage) -> bool { - self.ran_scripts_file(package).exists() - } -} - -// Uses BTreeMap to preserve the ordering of the elements in memory, to ensure -// the file generated from this datastructure is deterministic. -// See: https://github.com/denoland/deno/issues/24479 -/// Represents a dependency at `node_modules/.deno//` -struct SetupCacheDep<'a> { - previous: Option<&'a BTreeMap>, - current: &'a mut BTreeMap, -} - -impl<'a> SetupCacheDep<'a> { - pub fn insert(&mut self, name: &str, target_folder_name: &str) -> bool { - self - .current - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self.previous.and_then(|p| p.get(name)) { - previous_target != target_folder_name - } else { - true - } - } -} - -// Uses BTreeMap to preserve the ordering of the elements in memory, to ensure -// the file generated from this datastructure is deterministic. -// See: https://github.com/denoland/deno/issues/24479 -#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] -struct SetupCacheData { - root_symlinks: BTreeMap, - deno_symlinks: BTreeMap, - dep_symlinks: BTreeMap>, -} - -/// It is very slow to try to re-setup the symlinks each time, so this will -/// cache what we've setup on the last run and only update what is necessary. -/// Obviously this could lead to issues if the cache gets out of date with the -/// file system, such as if the user manually deletes a symlink. -struct SetupCache { - file_path: PathBuf, - previous: Option, - current: SetupCacheData, -} - -impl SetupCache { - pub fn load(file_path: PathBuf) -> Self { - let previous = std::fs::read(&file_path) - .ok() - .and_then(|data| bincode::deserialize(&data).ok()); - Self { - file_path, - previous, - current: Default::default(), - } - } - - pub fn save(&self) -> bool { - if let Some(previous) = &self.previous { - if previous == &self.current { - return false; // nothing to save - } - } - - bincode::serialize(&self.current).ok().and_then(|data| { - atomic_write_file_with_retries( - &CliSys::default(), - &self.file_path, - &data, - CACHE_PERM, - ) - .ok() - }); - true - } - - /// Inserts and checks for the existence of a root symlink - /// at `node_modules/` pointing to - /// `node_modules/.deno//` - pub fn insert_root_symlink( - &mut self, - name: &str, - target_folder_name: &str, - ) -> bool { - self - .current - .root_symlinks - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self - .previous - .as_ref() - .and_then(|p| p.root_symlinks.get(name)) - { - previous_target != target_folder_name - } else { - true - } - } - - /// Inserts and checks for the existence of a symlink at - /// `node_modules/.deno/node_modules/` pointing to - /// `node_modules/.deno//` - pub fn insert_deno_symlink( - &mut self, - name: &str, - target_folder_name: &str, - ) -> bool { - self - .current - .deno_symlinks - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self - .previous - .as_ref() - .and_then(|p| p.deno_symlinks.get(name)) - { - previous_target != target_folder_name - } else { - true - } - } - - pub fn remove_dep(&mut self, parent_name: &str) { - if let Some(previous) = &mut self.previous { - previous.dep_symlinks.remove(parent_name); - } - } - - pub fn with_dep(&mut self, parent_name: &str) -> SetupCacheDep<'_> { - SetupCacheDep { - previous: self - .previous - .as_ref() - .and_then(|p| p.dep_symlinks.get(parent_name)), - current: self - .current - .dep_symlinks - .entry(parent_name.to_string()) - .or_default(), - } - } -} - -fn get_package_folder_id_folder_name( +pub fn get_package_folder_id_folder_name( folder_id: &NpmPackageCacheFolderId, ) -> String { let copy_str = if folder_id.copy_index == 0 { @@ -1085,105 +235,6 @@ fn get_package_folder_id_from_folder_name( }) } -#[derive(Debug, thiserror::Error, deno_error::JsError)] -pub enum SymlinkPackageDirError { - #[class(inherit)] - #[error("Creating '{parent}'")] - Creating { - parent: PathBuf, - #[source] - #[inherit] - source: std::io::Error, - }, - #[class(inherit)] - #[error(transparent)] - Other(#[from] std::io::Error), - #[cfg(windows)] - #[class(inherit)] - #[error("Creating junction in node_modules folder")] - FailedCreatingJunction { - #[source] - #[inherit] - source: std::io::Error, - }, -} - -fn symlink_package_dir( - old_path: &Path, - new_path: &Path, -) -> Result<(), SymlinkPackageDirError> { - let new_parent = new_path.parent().unwrap(); - if new_parent.file_name().unwrap() != "node_modules" { - // create the parent folder that will contain the symlink - fs::create_dir_all(new_parent).map_err(|source| { - SymlinkPackageDirError::Creating { - parent: new_parent.to_path_buf(), - source, - } - })?; - } - - // need to delete the previous symlink before creating a new one - let _ignore = fs::remove_dir_all(new_path); - - let old_path_relative = - crate::util::path::relative_path(new_parent, old_path) - .unwrap_or_else(|| old_path.to_path_buf()); - - #[cfg(windows)] - { - junction_or_symlink_dir(&old_path_relative, old_path, new_path) - } - #[cfg(not(windows))] - { - symlink_dir(&crate::sys::CliSys::default(), &old_path_relative, new_path) - .map_err(Into::into) - } -} - -#[cfg(windows)] -fn junction_or_symlink_dir( - old_path_relative: &Path, - old_path: &Path, - new_path: &Path, -) -> Result<(), SymlinkPackageDirError> { - static USE_JUNCTIONS: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - - if USE_JUNCTIONS.load(std::sync::atomic::Ordering::Relaxed) { - // Use junctions because they're supported on ntfs file systems without - // needing to elevate privileges on Windows. - // Note: junctions don't support relative paths, so we need to use the - // absolute path here. - return junction::create(old_path, new_path).map_err(|source| { - SymlinkPackageDirError::FailedCreatingJunction { source } - }); - } - - match symlink_dir(&crate::sys::CliSys::default(), old_path_relative, new_path) - { - Ok(()) => Ok(()), - Err(symlink_err) - if symlink_err.kind() == std::io::ErrorKind::PermissionDenied => - { - USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); - junction::create(old_path, new_path).map_err(|source| { - SymlinkPackageDirError::FailedCreatingJunction { source } - }) - } - Err(symlink_err) => { - log::warn!( - "{} Unexpected error symlinking node_modules: {symlink_err}", - colors::yellow("Warning") - ); - USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); - junction::create(old_path, new_path).map_err(|source| { - SymlinkPackageDirError::FailedCreatingJunction { source } - }) - } - } -} - fn join_package_name(path: &Path, package_name: &str) -> PathBuf { let mut path = path.to_path_buf(); // ensure backslashes are used on windows @@ -1197,7 +248,6 @@ fn join_package_name(path: &Path, package_name: &str) -> PathBuf { mod test { use deno_npm::NpmPackageCacheFolderId; use deno_semver::package::PackageNv; - use test_util::TempDir; use super::*; @@ -1225,33 +275,4 @@ mod test { assert_eq!(folder_id, input); } } - - #[test] - fn test_setup_cache() { - let temp_dir = TempDir::new(); - let cache_bin_path = temp_dir.path().join("cache.bin").to_path_buf(); - let mut cache = SetupCache::load(cache_bin_path.clone()); - assert!(cache.insert_deno_symlink("package-a", "package-a@1.0.0")); - assert!(cache.insert_root_symlink("package-a", "package-a@1.0.0")); - assert!(cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - assert!(cache.save()); - - let mut cache = SetupCache::load(cache_bin_path.clone()); - assert!(!cache.insert_deno_symlink("package-a", "package-a@1.0.0")); - assert!(!cache.insert_root_symlink("package-a", "package-a@1.0.0")); - assert!(!cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - assert!(!cache.save()); - assert!(cache.insert_root_symlink("package-b", "package-b@0.2.0")); - assert!(cache.save()); - - let mut cache = SetupCache::load(cache_bin_path); - cache.remove_dep("package-a"); - assert!(cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - } } diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index 77d00a896e..8a89208c77 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -7,50 +7,27 @@ mod local; use std::path::PathBuf; use std::sync::Arc; -use deno_npm::NpmSystemInfo; - pub use self::common::NpmPackageFsResolver; pub use self::common::NpmPackageFsResolverPackageFolderError; use self::global::GlobalNpmPackageResolver; +pub use self::local::get_package_folder_id_folder_name; use self::local::LocalNpmPackageResolver; use super::resolution::NpmResolution; -use crate::args::LifecycleScriptsConfig; -use crate::args::NpmInstallDepsProvider; use crate::npm::CliNpmCache; -use crate::npm::CliNpmTarballCache; use crate::sys::CliSys; -use crate::util::progress_bar::ProgressBar; -#[allow(clippy::too_many_arguments)] pub fn create_npm_fs_resolver( npm_cache: Arc, - npm_install_deps_provider: &Arc, - progress_bar: &ProgressBar, resolution: Arc, sys: CliSys, - tarball_cache: Arc, maybe_node_modules_path: Option, - system_info: NpmSystemInfo, - lifecycle_scripts: LifecycleScriptsConfig, ) -> Arc { match maybe_node_modules_path { Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( - npm_cache, - npm_install_deps_provider.clone(), - progress_bar.clone(), resolution, sys, - tarball_cache, node_modules_folder, - system_info, - lifecycle_scripts, - )), - None => Arc::new(GlobalNpmPackageResolver::new( - npm_cache, - tarball_cache, - resolution, - system_info, - lifecycle_scripts, )), + None => Arc::new(GlobalNpmPackageResolver::new(npm_cache, resolution)), } } diff --git a/cli/util/sync/mod.rs b/cli/util/sync/mod.rs index 74ec469533..717dcd768a 100644 --- a/cli/util/sync/mod.rs +++ b/cli/util/sync/mod.rs @@ -1,11 +1,9 @@ // Copyright 2018-2025 the Deno authors. MIT license. mod async_flag; -mod sync_read_async_write_lock; mod task_queue; pub use async_flag::AsyncFlag; pub use deno_core::unsync::sync::AtomicFlag; -pub use sync_read_async_write_lock::SyncReadAsyncWriteLock; pub use task_queue::TaskQueue; pub use task_queue::TaskQueuePermit; diff --git a/cli/util/sync/sync_read_async_write_lock.rs b/cli/util/sync/sync_read_async_write_lock.rs deleted file mode 100644 index 48ad6bb863..0000000000 --- a/cli/util/sync/sync_read_async_write_lock.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -use deno_core::parking_lot::RwLock; -use deno_core::parking_lot::RwLockReadGuard; -use deno_core::parking_lot::RwLockWriteGuard; - -use super::TaskQueue; -use super::TaskQueuePermit; - -/// A lock that can be read synchronously at any time (including when -/// being written to), but must write asynchronously. -pub struct SyncReadAsyncWriteLockWriteGuard<'a, T: Send + Sync> { - _update_permit: TaskQueuePermit<'a>, - data: &'a RwLock, -} - -impl<'a, T: Send + Sync> SyncReadAsyncWriteLockWriteGuard<'a, T> { - pub fn read(&self) -> RwLockReadGuard<'_, T> { - self.data.read() - } - - /// Warning: Only `write()` with data you created within this - /// write this `SyncReadAsyncWriteLockWriteGuard`. - /// - /// ```rs - /// let mut data = lock.write().await; - /// - /// let mut data = data.read().clone(); - /// data.value = 2; - /// *data.write() = data; - /// ``` - pub fn write(&self) -> RwLockWriteGuard<'_, T> { - self.data.write() - } -} - -/// A lock that can only be -pub struct SyncReadAsyncWriteLock { - data: RwLock, - update_queue: TaskQueue, -} - -impl SyncReadAsyncWriteLock { - pub fn new(data: T) -> Self { - Self { - data: RwLock::new(data), - update_queue: TaskQueue::default(), - } - } - - pub fn read(&self) -> RwLockReadGuard<'_, T> { - self.data.read() - } - - pub async fn acquire(&self) -> SyncReadAsyncWriteLockWriteGuard<'_, T> { - let update_permit = self.update_queue.acquire().await; - SyncReadAsyncWriteLockWriteGuard { - _update_permit: update_permit, - data: &self.data, - } - } -}