1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-21 04:52:26 -05:00

refactor(npm): split some resolution from installation (#27595)

This splits away some npm resolution code from installation. It will
allow for more easily extracting out resolution code in the future.
This commit is contained in:
David Sherret 2025-01-08 18:46:37 -05:00 committed by GitHub
parent ea30e188a8
commit ce0968ef3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1571 additions and 1469 deletions

View file

@ -0,0 +1,226 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::collections::HashSet;
use std::sync::Arc;
use capacity_builder::StringBuilder;
use deno_core::error::AnyError;
use deno_error::JsErrorBox;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::resolution::AddPkgReqsOptions;
use deno_npm::resolution::NpmResolutionError;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmResolutionPackage;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::SmallStackString;
use deno_semver::VersionReq;
use super::resolution::NpmResolution;
use crate::args::CliLockfile;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::util::sync::TaskQueue;
pub struct AddPkgReqsResult {
/// Results from adding the individual packages.
///
/// The indexes of the results correspond to the indexes of the provided
/// package requirements.
pub results: Vec<Result<PackageNv, NpmResolutionError>>,
/// The final result of resolving and caching all the package requirements.
pub dependencies_result: Result<(), JsErrorBox>,
}
/// Updates the npm resolution with the provided package requirements.
pub struct NpmResolutionInstaller {
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
resolution: Arc<NpmResolution>,
maybe_lockfile: Option<Arc<CliLockfile>>,
update_queue: TaskQueue,
}
impl NpmResolutionInstaller {
pub fn new(
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
resolution: Arc<NpmResolution>,
maybe_lockfile: Option<Arc<CliLockfile>>,
) -> Self {
Self {
registry_info_provider,
resolution,
maybe_lockfile,
update_queue: Default::default(),
}
}
pub async fn add_package_reqs(
&self,
package_reqs: &[PackageReq],
) -> AddPkgReqsResult {
// only allow one thread in here at a time
let _snapshot_lock = self.update_queue.acquire().await;
let result = add_package_reqs_to_snapshot(
&self.registry_info_provider,
package_reqs,
self.maybe_lockfile.clone(),
|| self.resolution.snapshot(),
)
.await;
AddPkgReqsResult {
results: result.results,
dependencies_result: match result.dep_graph_result {
Ok(snapshot) => {
self.resolution.set_snapshot(snapshot);
Ok(())
}
Err(err) => Err(JsErrorBox::from_err(err)),
},
}
}
pub async fn set_package_reqs(
&self,
package_reqs: &[PackageReq],
) -> Result<(), AnyError> {
// only allow one thread in here at a time
let _snapshot_lock = self.update_queue.acquire().await;
let reqs_set = package_reqs.iter().collect::<HashSet<_>>();
let snapshot = add_package_reqs_to_snapshot(
&self.registry_info_provider,
package_reqs,
self.maybe_lockfile.clone(),
|| {
let snapshot = self.resolution.snapshot();
let has_removed_package = !snapshot
.package_reqs()
.keys()
.all(|req| reqs_set.contains(req));
// if any packages were removed, we need to completely recreate the npm resolution snapshot
if has_removed_package {
snapshot.into_empty()
} else {
snapshot
}
},
)
.await
.into_result()?;
self.resolution.set_snapshot(snapshot);
Ok(())
}
}
async fn add_package_reqs_to_snapshot(
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
package_reqs: &[PackageReq],
maybe_lockfile: Option<Arc<CliLockfile>>,
get_new_snapshot: impl Fn() -> NpmResolutionSnapshot,
) -> deno_npm::resolution::AddPkgReqsResult {
let snapshot = get_new_snapshot();
if package_reqs
.iter()
.all(|req| snapshot.package_reqs().contains_key(req))
{
log::debug!("Snapshot already up to date. Skipping npm resolution.");
return deno_npm::resolution::AddPkgReqsResult {
results: package_reqs
.iter()
.map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone()))
.collect(),
dep_graph_result: Ok(snapshot),
};
}
log::debug!(
/* this string is used in tests */
"Running npm resolution."
);
let npm_registry_api = registry_info_provider.as_npm_registry_api();
let result = snapshot
.add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs))
.await;
let result = match &result.dep_graph_result {
Err(NpmResolutionError::Resolution(err))
if npm_registry_api.mark_force_reload() =>
{
log::debug!("{err:#}");
log::debug!("npm resolution failed. Trying again...");
// try again with forced reloading
let snapshot = get_new_snapshot();
snapshot
.add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs))
.await
}
_ => result,
};
registry_info_provider.clear_memory_cache();
if let Ok(snapshot) = &result.dep_graph_result {
if let Some(lockfile) = maybe_lockfile {
populate_lockfile_from_snapshot(&lockfile, snapshot);
}
}
result
}
fn get_add_pkg_reqs_options(package_reqs: &[PackageReq]) -> AddPkgReqsOptions {
AddPkgReqsOptions {
package_reqs,
// WARNING: When bumping this version, check if anything needs to be
// updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js
types_node_version_req: Some(
VersionReq::parse_from_npm("22.0.0 - 22.5.4").unwrap(),
),
}
}
fn populate_lockfile_from_snapshot(
lockfile: &CliLockfile,
snapshot: &NpmResolutionSnapshot,
) {
fn npm_package_to_lockfile_info(
pkg: &NpmResolutionPackage,
) -> NpmPackageLockfileInfo {
let dependencies = pkg
.dependencies
.iter()
.map(|(name, id)| NpmPackageDependencyLockfileInfo {
name: name.clone(),
id: id.as_serialized(),
})
.collect();
NpmPackageLockfileInfo {
serialized_id: pkg.id.as_serialized(),
integrity: pkg.dist.integrity().for_lockfile(),
dependencies,
}
}
let mut lockfile = lockfile.lock();
for (package_req, nv) in snapshot.package_reqs() {
let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id;
lockfile.insert_package_specifier(
JsrDepPackageReq::npm(package_req.clone()),
{
StringBuilder::<SmallStackString>::build(|builder| {
builder.append(&id.nv.version);
builder.append(&id.peer_dependencies);
})
.unwrap()
},
);
}
for package in snapshot.all_packages_for_every_system() {
lockfile.insert_npm_package(npm_package_to_lockfile_info(package));
}
}

View file

@ -0,0 +1,18 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use async_trait::async_trait;
use deno_error::JsErrorBox;
use crate::npm::PackageCaching;
pub mod bin_entries;
pub mod lifecycle_scripts;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
pub trait NpmPackageFsInstaller: Send + Sync {
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,
) -> Result<(), JsErrorBox>;
}

View file

@ -0,0 +1,190 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::StreamExt;
use deno_error::JsErrorBox;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use super::super::resolution::NpmResolution;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
use super::common::NpmPackageFsInstaller;
use crate::args::LifecycleScriptsConfig;
use crate::cache::FastInsecureHasher;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageInstaller {
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
}
impl GlobalNpmPackageInstaller {
pub fn new(
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
}
}
}
#[async_trait(?Send)]
impl NpmPackageFsInstaller for GlobalNpmPackageInstaller {
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,
) -> Result<(), JsErrorBox> {
let package_partitions = match caching {
PackageCaching::All => self
.resolution
.all_system_packages_partitioned(&self.system_info),
PackageCaching::Only(reqs) => self
.resolution
.subset(&reqs)
.all_system_packages_partitioned(&self.system_info),
};
cache_packages(&package_partitions.packages, &self.tarball_cache)
.await
.map_err(JsErrorBox::from_err)?;
// create the copy package folders
for copy in package_partitions.copy_packages {
self
.cache
.ensure_copy_package(&copy.get_package_cache_folder_id())
.map_err(JsErrorBox::from_err)?;
}
let mut lifecycle_scripts =
super::common::lifecycle_scripts::LifecycleScripts::new(
&self.lifecycle_scripts,
GlobalLifecycleScripts::new(self, &self.lifecycle_scripts.root_dir),
);
for package in &package_partitions.packages {
let package_folder = self.cache.package_folder_for_nv(&package.id.nv);
lifecycle_scripts.add(package, Cow::Borrowed(&package_folder));
}
lifecycle_scripts
.warn_not_run_scripts()
.map_err(JsErrorBox::from_err)?;
Ok(())
}
}
async fn cache_packages(
packages: &[NpmResolutionPackage],
tarball_cache: &Arc<CliNpmTarballCache>,
) -> Result<(), deno_npm_cache::EnsurePackageError> {
let mut futures_unordered = FuturesUnordered::new();
for package in packages {
futures_unordered.push(async move {
tarball_cache
.ensure_package(&package.id.nv, &package.dist)
.await
});
}
while let Some(result) = futures_unordered.next().await {
// surface the first error
result?;
}
Ok(())
}
struct GlobalLifecycleScripts<'a> {
installer: &'a GlobalNpmPackageInstaller,
path_hash: u64,
}
impl<'a> GlobalLifecycleScripts<'a> {
fn new(installer: &'a GlobalNpmPackageInstaller, root_dir: &Path) -> Self {
let mut hasher = FastInsecureHasher::new_without_deno_version();
hasher.write(root_dir.to_string_lossy().as_bytes());
let path_hash = hasher.finish();
Self {
installer,
path_hash,
}
}
fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf {
self
.package_path(package)
.join(format!(".scripts-warned-{}", self.path_hash))
}
}
impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy
for GlobalLifecycleScripts<'a>
{
fn can_run_scripts(&self) -> bool {
false
}
fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf {
self.installer.cache.package_folder_for_nv(&package.id.nv)
}
fn warn_on_scripts_not_run(
&self,
packages: &[(&NpmResolutionPackage, PathBuf)],
) -> std::result::Result<(), std::io::Error> {
log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall"));
for (package, _) in packages {
log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv)));
}
log::warn!("");
log::warn!(
"┠─ {}",
colors::italic("This may cause the packages to not work correctly.")
);
log::warn!("┠─ {}", colors::italic("Lifecycle scripts are only supported when using a `node_modules` directory."));
log::warn!(
"┠─ {}",
colors::italic("Enable it in your deno config file:")
);
log::warn!("┖─ {}", colors::bold("\"nodeModulesDir\": \"auto\""));
for (package, _) in packages {
std::fs::write(self.warned_scripts_file(package), "")?;
}
Ok(())
}
fn did_run_scripts(
&self,
_package: &NpmResolutionPackage,
) -> Result<(), std::io::Error> {
Ok(())
}
fn has_warned(&self, package: &NpmResolutionPackage) -> bool {
self.warned_scripts_file(package).exists()
}
fn has_run(&self, _package: &NpmResolutionPackage) -> bool {
false
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,55 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::PathBuf;
use std::sync::Arc;
use deno_npm::NpmSystemInfo;
pub use self::common::NpmPackageFsInstaller;
use self::global::GlobalNpmPackageInstaller;
use self::local::LocalNpmPackageInstaller;
use super::resolution::NpmResolution;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
mod common;
mod global;
mod local;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_installer(
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Arc<dyn NpmPackageFsInstaller> {
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageInstaller::new(
npm_cache,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
lifecycle_scripts,
)),
None => Arc::new(GlobalNpmPackageInstaller::new(
npm_cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
)),
}
}

View file

@ -28,11 +28,14 @@ use deno_runtime::colors;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use installer::AddPkgReqsResult;
use installer::NpmResolutionInstaller;
use installers::create_npm_fs_installer;
use installers::NpmPackageFsInstaller;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmPackageFolderResolver;
use resolution::AddPkgReqsResult;
use self::resolution::NpmResolution;
use self::resolvers::create_npm_fs_resolver;
@ -55,6 +58,8 @@ use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
mod installer;
mod installers;
mod resolution;
mod resolvers;
@ -156,11 +161,7 @@ fn create_inner(
snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Arc<dyn CliNpmResolver> {
let resolution = Arc::new(NpmResolution::from_serialized(
registry_info_provider.clone(),
snapshot,
maybe_lockfile.clone(),
));
let resolution = Arc::new(NpmResolution::from_serialized(snapshot));
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
http_client,
@ -168,18 +169,25 @@ fn create_inner(
npm_rc.clone(),
));
let fs_resolver = create_npm_fs_resolver(
let fs_installer = create_npm_fs_installer(
npm_cache.clone(),
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
sys.clone(),
tarball_cache.clone(),
node_modules_dir_path,
node_modules_dir_path.clone(),
npm_system_info.clone(),
lifecycle_scripts.clone(),
);
let fs_resolver = create_npm_fs_resolver(
npm_cache.clone(),
resolution.clone(),
sys.clone(),
node_modules_dir_path,
);
Arc::new(ManagedCliNpmResolver::new(
fs_installer,
fs_resolver,
maybe_lockfile,
registry_info_provider,
@ -301,6 +309,7 @@ pub enum PackageCaching<'a> {
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
fs_installer: Arc<dyn NpmPackageFsInstaller>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
@ -308,6 +317,7 @@ pub struct ManagedCliNpmResolver {
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
sys: CliSys,
resolution: Arc<NpmResolution>,
resolution_installer: NpmResolutionInstaller,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
@ -348,6 +358,7 @@ pub enum ResolvePkgFolderFromDenoModuleError {
impl ManagedCliNpmResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
fs_installer: Arc<dyn NpmPackageFsInstaller>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
@ -360,7 +371,13 @@ impl ManagedCliNpmResolver {
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
let resolution_installer = NpmResolutionInstaller::new(
registry_info_provider.clone(),
resolution.clone(),
maybe_lockfile.clone(),
);
Self {
fs_installer,
fs_resolver,
maybe_lockfile,
registry_info_provider,
@ -368,6 +385,7 @@ impl ManagedCliNpmResolver {
npm_install_deps_provider,
text_only_progress_bar,
resolution,
resolution_installer,
sys,
tarball_cache,
npm_system_info,
@ -489,7 +507,7 @@ impl ManagedCliNpmResolver {
};
}
let mut result = self.resolution.add_package_reqs(packages).await;
let mut result = self.resolution_installer.add_package_reqs(packages).await;
if result.dependencies_result.is_ok() {
if let Some(lockfile) = self.maybe_lockfile.as_ref() {
@ -512,7 +530,7 @@ impl ManagedCliNpmResolver {
&self,
packages: &[PackageReq],
) -> Result<(), AnyError> {
self.resolution.set_package_reqs(packages).await
self.resolution_installer.set_package_reqs(packages).await
}
pub fn snapshot(&self) -> NpmResolutionSnapshot {
@ -554,7 +572,7 @@ impl ManagedCliNpmResolver {
&self,
caching: PackageCaching<'_>,
) -> Result<(), JsErrorBox> {
self.fs_resolver.cache_packages(caching).await
self.fs_installer.cache_packages(caching).await
}
pub fn resolve_pkg_folder_from_deno_module(
@ -738,14 +756,11 @@ impl CliNpmResolver for ManagedCliNpmResolver {
fn clone_snapshotted(&self) -> Arc<dyn CliNpmResolver> {
// create a new snapshotted npm resolution and resolver
let npm_resolution = Arc::new(NpmResolution::new(
self.registry_info_provider.clone(),
self.resolution.snapshot(),
self.maybe_lockfile.clone(),
));
let npm_resolution =
Arc::new(NpmResolution::new(self.resolution.snapshot()));
Arc::new(ManagedCliNpmResolver::new(
create_npm_fs_resolver(
create_npm_fs_installer(
self.npm_cache.clone(),
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
@ -756,6 +771,12 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self.npm_system_info.clone(),
self.lifecycle_scripts.clone(),
),
create_npm_fs_resolver(
self.npm_cache.clone(),
npm_resolution.clone(),
self.sys.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
),
self.maybe_lockfile.clone(),
self.registry_info_provider.clone(),
self.npm_cache.clone(),

View file

@ -1,18 +1,9 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use capacity_builder::StringBuilder;
use deno_core::error::AnyError;
use deno_error::JsErrorBox;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::resolution::AddPkgReqsOptions;
use deno_core::parking_lot::RwLock;
use deno_npm::resolution::NpmPackagesPartitioned;
use deno_npm::resolution::NpmResolutionError;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::PackageCacheFolderIdNotFoundError;
use deno_npm::resolution::PackageNotFoundFromReferrerError;
@ -23,25 +14,8 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::SmallStackString;
use deno_semver::VersionReq;
use crate::args::CliLockfile;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::util::sync::SyncReadAsyncWriteLock;
pub struct AddPkgReqsResult {
/// Results from adding the individual packages.
///
/// The indexes of the results correspond to the indexes of the provided
/// package requirements.
pub results: Vec<Result<PackageNv, NpmResolutionError>>,
/// The final result of resolving and caching all the package requirements.
pub dependencies_result: Result<(), JsErrorBox>,
}
/// Handles updating and storing npm resolution in memory where the underlying
/// snapshot can be updated concurrently. Additionally handles updating the lockfile
@ -49,9 +23,7 @@ pub struct AddPkgReqsResult {
///
/// This does not interact with the file system.
pub struct NpmResolution {
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
snapshot: SyncReadAsyncWriteLock<NpmResolutionSnapshot>,
maybe_lockfile: Option<Arc<CliLockfile>>,
snapshot: RwLock<NpmResolutionSnapshot>,
}
impl std::fmt::Debug for NpmResolution {
@ -65,87 +37,19 @@ impl std::fmt::Debug for NpmResolution {
impl NpmResolution {
pub fn from_serialized(
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
initial_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
maybe_lockfile: Option<Arc<CliLockfile>>,
) -> Self {
let snapshot =
NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default());
Self::new(registry_info_provider, snapshot, maybe_lockfile)
Self::new(snapshot)
}
pub fn new(
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
initial_snapshot: NpmResolutionSnapshot,
maybe_lockfile: Option<Arc<CliLockfile>>,
) -> Self {
pub fn new(initial_snapshot: NpmResolutionSnapshot) -> Self {
Self {
registry_info_provider,
snapshot: SyncReadAsyncWriteLock::new(initial_snapshot),
maybe_lockfile,
snapshot: RwLock::new(initial_snapshot),
}
}
pub async fn add_package_reqs(
&self,
package_reqs: &[PackageReq],
) -> AddPkgReqsResult {
// only allow one thread in here at a time
let snapshot_lock = self.snapshot.acquire().await;
let result = add_package_reqs_to_snapshot(
&self.registry_info_provider,
package_reqs,
self.maybe_lockfile.clone(),
|| snapshot_lock.read().clone(),
)
.await;
AddPkgReqsResult {
results: result.results,
dependencies_result: match result.dep_graph_result {
Ok(snapshot) => {
*snapshot_lock.write() = snapshot;
Ok(())
}
Err(err) => Err(JsErrorBox::from_err(err)),
},
}
}
pub async fn set_package_reqs(
&self,
package_reqs: &[PackageReq],
) -> Result<(), AnyError> {
// only allow one thread in here at a time
let snapshot_lock = self.snapshot.acquire().await;
let reqs_set = package_reqs.iter().collect::<HashSet<_>>();
let snapshot = add_package_reqs_to_snapshot(
&self.registry_info_provider,
package_reqs,
self.maybe_lockfile.clone(),
|| {
let snapshot = snapshot_lock.read().clone();
let has_removed_package = !snapshot
.package_reqs()
.keys()
.all(|req| reqs_set.contains(req));
// if any packages were removed, we need to completely recreate the npm resolution snapshot
if has_removed_package {
snapshot.into_empty()
} else {
snapshot
}
},
)
.await
.into_result()?;
*snapshot_lock.write() = snapshot;
Ok(())
}
pub fn resolve_pkg_cache_folder_id_from_pkg_id(
&self,
id: &NpmPackageId,
@ -262,112 +166,8 @@ impl NpmResolution {
pub fn subset(&self, package_reqs: &[PackageReq]) -> NpmResolutionSnapshot {
self.snapshot.read().subset(package_reqs)
}
}
async fn add_package_reqs_to_snapshot(
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
package_reqs: &[PackageReq],
maybe_lockfile: Option<Arc<CliLockfile>>,
get_new_snapshot: impl Fn() -> NpmResolutionSnapshot,
) -> deno_npm::resolution::AddPkgReqsResult {
let snapshot = get_new_snapshot();
if package_reqs
.iter()
.all(|req| snapshot.package_reqs().contains_key(req))
{
log::debug!("Snapshot already up to date. Skipping npm resolution.");
return deno_npm::resolution::AddPkgReqsResult {
results: package_reqs
.iter()
.map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone()))
.collect(),
dep_graph_result: Ok(snapshot),
};
}
log::debug!(
/* this string is used in tests */
"Running npm resolution."
);
let npm_registry_api = registry_info_provider.as_npm_registry_api();
let result = snapshot
.add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs))
.await;
let result = match &result.dep_graph_result {
Err(NpmResolutionError::Resolution(err))
if npm_registry_api.mark_force_reload() =>
{
log::debug!("{err:#}");
log::debug!("npm resolution failed. Trying again...");
// try again with forced reloading
let snapshot = get_new_snapshot();
snapshot
.add_pkg_reqs(&npm_registry_api, get_add_pkg_reqs_options(package_reqs))
.await
}
_ => result,
};
registry_info_provider.clear_memory_cache();
if let Ok(snapshot) = &result.dep_graph_result {
if let Some(lockfile) = maybe_lockfile {
populate_lockfile_from_snapshot(&lockfile, snapshot);
}
}
result
}
fn get_add_pkg_reqs_options(package_reqs: &[PackageReq]) -> AddPkgReqsOptions {
AddPkgReqsOptions {
package_reqs,
// WARNING: When bumping this version, check if anything needs to be
// updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js
types_node_version_req: Some(
VersionReq::parse_from_npm("22.0.0 - 22.5.4").unwrap(),
),
}
}
fn populate_lockfile_from_snapshot(
lockfile: &CliLockfile,
snapshot: &NpmResolutionSnapshot,
) {
let mut lockfile = lockfile.lock();
for (package_req, nv) in snapshot.package_reqs() {
let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id;
lockfile.insert_package_specifier(
JsrDepPackageReq::npm(package_req.clone()),
{
StringBuilder::<SmallStackString>::build(|builder| {
builder.append(&id.nv.version);
builder.append(&id.peer_dependencies);
})
.unwrap()
},
);
}
for package in snapshot.all_packages_for_every_system() {
lockfile.insert_npm_package(npm_package_to_lockfile_info(package));
}
}
fn npm_package_to_lockfile_info(
pkg: &NpmResolutionPackage,
) -> NpmPackageLockfileInfo {
let dependencies = pkg
.dependencies
.iter()
.map(|(name, id)| NpmPackageDependencyLockfileInfo {
name: name.clone(),
id: id.as_serialized(),
})
.collect();
NpmPackageLockfileInfo {
serialized_id: pkg.id.as_serialized(),
integrity: pkg.dist.integrity().for_lockfile(),
dependencies,
pub fn set_snapshot(&self, snapshot: NpmResolutionSnapshot) {
*self.snapshot.write() = snapshot;
}
}

View file

@ -1,20 +1,14 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub mod bin_entries;
pub mod lifecycle_scripts;
use std::path::Path;
use std::path::PathBuf;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_error::JsErrorBox;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use node_resolver::errors::PackageFolderResolveError;
use super::super::PackageCaching;
#[derive(Debug, thiserror::Error, deno_error::JsError)]
#[class(generic)]
#[error("Package folder not found for '{0}'")]
@ -47,9 +41,4 @@ pub trait NpmPackageFsResolver: Send + Sync {
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<NpmPackageCacheFolderId>, std::io::Error>;
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,
) -> Result<(), JsErrorBox>;
}

View file

@ -2,59 +2,32 @@
//! Code for global npm cache resolution.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::StreamExt;
use deno_error::JsErrorBox;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::ReferrerNotFoundError;
use super::super::resolution::NpmResolution;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
use super::common::NpmPackageFsResolver;
use crate::args::LifecycleScriptsConfig;
use crate::cache::FastInsecureHasher;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageResolver {
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
}
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
}
pub fn new(cache: Arc<CliNpmCache>, resolution: Arc<NpmResolution>) -> Self {
Self { cache, resolution }
}
}
@ -141,140 +114,4 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver {
.resolve_package_folder_id_from_specifier(specifier),
)
}
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,
) -> Result<(), JsErrorBox> {
let package_partitions = match caching {
PackageCaching::All => self
.resolution
.all_system_packages_partitioned(&self.system_info),
PackageCaching::Only(reqs) => self
.resolution
.subset(&reqs)
.all_system_packages_partitioned(&self.system_info),
};
cache_packages(&package_partitions.packages, &self.tarball_cache)
.await
.map_err(JsErrorBox::from_err)?;
// create the copy package folders
for copy in package_partitions.copy_packages {
self
.cache
.ensure_copy_package(&copy.get_package_cache_folder_id())
.map_err(JsErrorBox::from_err)?;
}
let mut lifecycle_scripts =
super::common::lifecycle_scripts::LifecycleScripts::new(
&self.lifecycle_scripts,
GlobalLifecycleScripts::new(self, &self.lifecycle_scripts.root_dir),
);
for package in &package_partitions.packages {
let package_folder = self.cache.package_folder_for_nv(&package.id.nv);
lifecycle_scripts.add(package, Cow::Borrowed(&package_folder));
}
lifecycle_scripts
.warn_not_run_scripts()
.map_err(JsErrorBox::from_err)?;
Ok(())
}
}
async fn cache_packages(
packages: &[NpmResolutionPackage],
tarball_cache: &Arc<CliNpmTarballCache>,
) -> Result<(), deno_npm_cache::EnsurePackageError> {
let mut futures_unordered = FuturesUnordered::new();
for package in packages {
futures_unordered.push(async move {
tarball_cache
.ensure_package(&package.id.nv, &package.dist)
.await
});
}
while let Some(result) = futures_unordered.next().await {
// surface the first error
result?;
}
Ok(())
}
struct GlobalLifecycleScripts<'a> {
resolver: &'a GlobalNpmPackageResolver,
path_hash: u64,
}
impl<'a> GlobalLifecycleScripts<'a> {
fn new(resolver: &'a GlobalNpmPackageResolver, root_dir: &Path) -> Self {
let mut hasher = FastInsecureHasher::new_without_deno_version();
hasher.write(root_dir.to_string_lossy().as_bytes());
let path_hash = hasher.finish();
Self {
resolver,
path_hash,
}
}
fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf {
self
.package_path(package)
.join(format!(".scripts-warned-{}", self.path_hash))
}
}
impl<'a> super::common::lifecycle_scripts::LifecycleScriptsStrategy
for GlobalLifecycleScripts<'a>
{
fn can_run_scripts(&self) -> bool {
false
}
fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf {
self.resolver.cache.package_folder_for_nv(&package.id.nv)
}
fn warn_on_scripts_not_run(
&self,
packages: &[(&NpmResolutionPackage, PathBuf)],
) -> std::result::Result<(), std::io::Error> {
log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall"));
for (package, _) in packages {
log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv)));
}
log::warn!("");
log::warn!(
"┠─ {}",
colors::italic("This may cause the packages to not work correctly.")
);
log::warn!("┠─ {}", colors::italic("Lifecycle scripts are only supported when using a `node_modules` directory."));
log::warn!(
"┠─ {}",
colors::italic("Enable it in your deno config file:")
);
log::warn!("┖─ {}", colors::bold("\"nodeModulesDir\": \"auto\""));
for (package, _) in packages {
std::fs::write(self.warned_scripts_file(package), "")?;
}
Ok(())
}
fn did_run_scripts(
&self,
_package: &NpmResolutionPackage,
) -> Result<(), std::io::Error> {
Ok(())
}
fn has_warned(&self, package: &NpmResolutionPackage) -> bool {
self.warned_scripts_file(package).exists()
}
fn has_run(&self, _package: &NpmResolutionPackage) -> bool {
false
}
}

File diff suppressed because it is too large Load diff

View file

@ -7,50 +7,27 @@ mod local;
use std::path::PathBuf;
use std::sync::Arc;
use deno_npm::NpmSystemInfo;
pub use self::common::NpmPackageFsResolver;
pub use self::common::NpmPackageFsResolverPackageFolderError;
use self::global::GlobalNpmPackageResolver;
pub use self::local::get_package_folder_id_folder_name;
use self::local::LocalNpmPackageResolver;
use super::resolution::NpmResolution;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Arc<dyn NpmPackageFsResolver> {
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
npm_cache,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
lifecycle_scripts,
)),
None => Arc::new(GlobalNpmPackageResolver::new(
npm_cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
)),
None => Arc::new(GlobalNpmPackageResolver::new(npm_cache, resolution)),
}
}

View file

@ -1,11 +1,9 @@
// Copyright 2018-2025 the Deno authors. MIT license.
mod async_flag;
mod sync_read_async_write_lock;
mod task_queue;
pub use async_flag::AsyncFlag;
pub use deno_core::unsync::sync::AtomicFlag;
pub use sync_read_async_write_lock::SyncReadAsyncWriteLock;
pub use task_queue::TaskQueue;
pub use task_queue::TaskQueuePermit;

View file

@ -1,62 +0,0 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use deno_core::parking_lot::RwLock;
use deno_core::parking_lot::RwLockReadGuard;
use deno_core::parking_lot::RwLockWriteGuard;
use super::TaskQueue;
use super::TaskQueuePermit;
/// A lock that can be read synchronously at any time (including when
/// being written to), but must write asynchronously.
pub struct SyncReadAsyncWriteLockWriteGuard<'a, T: Send + Sync> {
_update_permit: TaskQueuePermit<'a>,
data: &'a RwLock<T>,
}
impl<'a, T: Send + Sync> SyncReadAsyncWriteLockWriteGuard<'a, T> {
pub fn read(&self) -> RwLockReadGuard<'_, T> {
self.data.read()
}
/// Warning: Only `write()` with data you created within this
/// write this `SyncReadAsyncWriteLockWriteGuard`.
///
/// ```rs
/// let mut data = lock.write().await;
///
/// let mut data = data.read().clone();
/// data.value = 2;
/// *data.write() = data;
/// ```
pub fn write(&self) -> RwLockWriteGuard<'_, T> {
self.data.write()
}
}
/// A lock that can only be
pub struct SyncReadAsyncWriteLock<T: Send + Sync> {
data: RwLock<T>,
update_queue: TaskQueue,
}
impl<T: Send + Sync> SyncReadAsyncWriteLock<T> {
pub fn new(data: T) -> Self {
Self {
data: RwLock::new(data),
update_queue: TaskQueue::default(),
}
}
pub fn read(&self) -> RwLockReadGuard<'_, T> {
self.data.read()
}
pub async fn acquire(&self) -> SyncReadAsyncWriteLockWriteGuard<'_, T> {
let update_permit = self.update_queue.acquire().await;
SyncReadAsyncWriteLockWriteGuard {
_update_permit: update_permit,
data: &self.data,
}
}
}