foster/denoland-deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-02-01 12:16:11 -05:00
Code Issues Packages 1 Wiki Activity

Merge branch 'main' into lint_plugins

Browse source
This commit is contained in:
Bartek Iwańczuk 2024-12-04 01:04:37 +01:00
commit c2ac42485f
No known key found for this signature in database
GPG key ID: 0C6BCDDC3B3AD750
44 changed files with 777 additions and 563 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
Cargo.lock generated
View file

@ -1220,6 +1220,7 @@ dependencies = [
"deno_lint",
"deno_lockfile",
"deno_npm",
"deno_npm_cache",
"deno_package_json",
"deno_path_util",
"deno_resolver",
@ -1998,6 +1999,35 @@ dependencies = [
"url",
]
[[package]]
name = "deno_npm_cache"
version = "0.0.1"
dependencies = [
"anyhow",
"async-trait",
"base64 0.21.7",
"boxed_error",
"deno_cache_dir",
"deno_core",
"deno_npm",
"deno_semver",
"deno_unsync",
"faster-hex",
"flate2",
"futures",
"http 1.1.0",
"log",
"parking_lot",
"percent-encoding",
"rand",
"ring",
"serde_json",
"tar",
"tempfile",
"thiserror 1.0.64",
"url",
]
[[package]]
name = "deno_ops"
version = "0.199.0"
@ -2260,10 +2290,11 @@ dependencies = [
[[package]]
name = "deno_unsync"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f36b4ef61a04ce201b925a5dffa90f88437d37fee4836c758470dd15ba7f05e"
checksum = "d774fd83f26b24f0805a6ab8b26834a0d06ceac0db517b769b1e4633c96a2057"
dependencies = [
"futures",
"parking_lot",
"tokio",
]
@ -4682,9 +4713,9 @@ dependencies = [
[[package]]
name = "markup_fmt"
version = "0.16.0"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f303c36143671ac6c54112eb5aa95649b169dae783fdb6ead2c0e88b408c425c"
checksum = "fa7605bb4ad755a9ab5c96f2ce3bfd4eb8acd559b842c041fc8a5f84d63aed3a"
dependencies = [
"aho-corasick",
"css_dataset",

3
Cargo.toml
View file

@ -30,6 +30,7 @@ members = [
"ext/webstorage",
"resolvers/deno",
"resolvers/node",
"resolvers/npm_cache",
"runtime",
"runtime/permissions",
"tests",
@ -93,6 +94,7 @@ deno_websocket = { version = "0.185.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.175.0", path = "./ext/webstorage" }
# resolvers
deno_npm_cache = { version = "0.0.1", path = "./resolvers/npm_cache" }
deno_resolver = { version = "0.12.0", path = "./resolvers/deno" }
node_resolver = { version = "0.19.0", path = "./resolvers/node" }
@ -117,6 +119,7 @@ data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.14.0"
deno_package_json = { version = "0.2.1", default-features = false }
deno_unsync = "0.4.2"
dlopen2 = "0.6.1"
ecb = "=0.1.2"
elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem", "jwk"] }

3
cli/Cargo.toml
View file

@ -77,6 +77,7 @@ deno_graph = { version = "=0.86.3" }
deno_lint = { version = "=0.68.2", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true
deno_npm_cache.workspace = true
deno_package_json.workspace = true
deno_path_util.workspace = true
deno_resolver.workspace = true
@ -130,7 +131,7 @@ libz-sys.workspace = true
log = { workspace = true, features = ["serde"] }
lsp-types.workspace = true
malva = "=0.11.0"
markup_fmt = "=0.16.0"
markup_fmt = "=0.18.0"
memmem.workspace = true
monch.workspace = true
notify.workspace = true

28
cli/args/mod.rs
View file

@ -27,6 +27,7 @@ use deno_npm::npm_rc::NpmRc;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::normalize_path;
use deno_semver::npm::NpmPackageReqReference;
use deno_telemetry::OtelConfig;
@ -238,20 +239,25 @@ pub enum CacheSetting {
}
impl CacheSetting {
pub fn should_use_for_npm_package(&self, package_name: &str) -> bool {
pub fn as_npm_cache_setting(&self) -> NpmCacheSetting {
match self {
CacheSetting::ReloadAll => false,
CacheSetting::ReloadSome(list) => {
if list.iter().any(|i| i == "npm:") {
return false;
CacheSetting::Only => NpmCacheSetting::Only,
CacheSetting::ReloadAll => NpmCacheSetting::ReloadAll,
CacheSetting::ReloadSome(values) => {
if values.iter().any(|v| v == "npm:") {
NpmCacheSetting::ReloadAll
} else {
NpmCacheSetting::ReloadSome {
npm_package_names: values
.iter()
.filter_map(|v| v.strip_prefix("npm:"))
.map(|n| n.to_string())
.collect(),
}
}
let specifier = format!("npm:{package_name}");
if list.contains(&specifier) {
return false;
}
true
}
_ => true,
CacheSetting::RespectHeaders => unreachable!(), // not supported
CacheSetting::Use => NpmCacheSetting::Use,
}
}
}

73
cli/npm/managed/mod.rs
View file

@ -5,8 +5,6 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use cache::RegistryInfoDownloader;
use cache::TarballCache;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
@ -42,22 +40,23 @@ use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
use self::cache::NpmCache;
use self::registry::CliNpmRegistryApi;
use self::resolution::NpmResolution;
use self::resolvers::create_npm_fs_resolver;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmCache;
use super::CliNpmCacheEnv;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
use super::InnerCliNpmResolverRef;
use super::ResolvePkgFolderFromDenoReqError;
pub mod cache;
mod registry;
mod resolution;
mod resolvers;
@ -85,8 +84,9 @@ pub struct CliManagedNpmResolverCreateOptions {
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
let cache_env = create_cache_env(&options);
let npm_cache = create_cache(cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
@ -97,8 +97,8 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
};
create_inner(
cache_env,
options.fs,
options.http_client_provider,
options.maybe_lockfile,
npm_api,
npm_cache,
@ -118,12 +118,13 @@ pub async fn create_managed_npm_resolver_for_lsp(
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
let npm_cache_env = create_cache_env(&options);
let npm_cache = create_cache(npm_cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options);
let snapshot = resolve_snapshot(&npm_api, options.snapshot).await?;
Ok(create_inner(
npm_cache_env,
options.fs,
options.http_client_provider,
options.maybe_lockfile,
npm_api,
npm_cache,
@ -139,11 +140,11 @@ pub async fn create_managed_npm_resolver(
#[allow(clippy::too_many_arguments)]
fn create_inner(
env: Arc<CliNpmCacheEnv>,
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
@ -157,12 +158,10 @@ fn create_inner(
snapshot,
maybe_lockfile.clone(),
));
let tarball_cache = Arc::new(TarballCache::new(
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
fs.clone(),
http_client_provider.clone(),
env,
npm_rc.clone(),
text_only_progress_bar.clone(),
));
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
@ -190,25 +189,39 @@ fn create_inner(
))
}
fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc<NpmCache> {
Arc::new(NpmCache::new(
fn create_cache_env(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCacheEnv> {
Arc::new(CliNpmCacheEnv::new(
options.fs.clone(),
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
))
}
fn create_cache(
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.cache_setting.clone(),
options.cache_setting.as_npm_cache_setting(),
env,
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
npm_cache: Arc<NpmCache>,
) -> Arc<CliNpmRegistryApi> {
Arc::new(CliNpmRegistryApi::new(
npm_cache.clone(),
Arc::new(RegistryInfoDownloader::new(
npm_cache,
options.http_client_provider.clone(),
cache.clone(),
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
env,
options.npmrc.clone(),
options.text_only_progress_bar.clone(),
)),
))
}
@ -292,10 +305,10 @@ pub struct ManagedCliNpmResolver {
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
top_level_install_flag: AtomicFlag,
@ -317,10 +330,10 @@ impl ManagedCliNpmResolver {
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

23
cli/npm/managed/registry.rs
View file

@ -14,27 +14,28 @@ use deno_core::parking_lot::Mutex;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_npm_cache::NpmCacheSetting;
use crate::args::CacheSetting;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::util::sync::AtomicFlag;
use super::cache::NpmCache;
use super::cache::RegistryInfoDownloader;
// todo(#27198): Remove this and move functionality down into
// RegistryInfoProvider, which already does most of this.
#[derive(Debug)]
pub struct CliNpmRegistryApi(Option<Arc<CliNpmRegistryApiInner>>);
impl CliNpmRegistryApi {
pub fn new(
cache: Arc<NpmCache>,
registry_info_downloader: Arc<RegistryInfoDownloader>,
cache: Arc<CliNpmCache>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
) -> Self {
Self(Some(Arc::new(CliNpmRegistryApiInner {
cache,
force_reload_flag: Default::default(),
mem_cache: Default::default(),
previously_reloaded_packages: Default::default(),
registry_info_downloader,
registry_info_provider,
})))
}
@ -83,11 +84,11 @@ enum CacheItem {
#[derive(Debug)]
struct CliNpmRegistryApiInner {
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
force_reload_flag: AtomicFlag,
mem_cache: Mutex<HashMap<String, CacheItem>>,
previously_reloaded_packages: Mutex<HashSet<String>>,
registry_info_downloader: Arc<RegistryInfoDownloader>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
}
impl CliNpmRegistryApiInner {
@ -118,7 +119,7 @@ impl CliNpmRegistryApiInner {
return Ok(result);
}
}
api.registry_info_downloader
api.registry_info_provider
.load_package_info(&name)
.await
.map_err(Arc::new)
@ -159,7 +160,7 @@ impl CliNpmRegistryApiInner {
// is disabled or if we're already reloading
if matches!(
self.cache.cache_setting(),
CacheSetting::Only | CacheSetting::ReloadAll
NpmCacheSetting::Only | NpmCacheSetting::ReloadAll
) {
return false;
}

4
cli/npm/managed/resolvers/common.rs
View file

@ -24,7 +24,7 @@ use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use crate::npm::managed::cache::TarballCache;
use crate::npm::CliNpmTarballCache;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
@ -140,7 +140,7 @@ impl RegistryReadPermissionChecker {
/// Caches all the packages in parallel.
pub async fn cache_packages(
packages: &[NpmResolutionPackage],
tarball_cache: &Arc<TarballCache>,
tarball_cache: &Arc<CliNpmTarballCache>,
) -> Result<(), AnyError> {
let mut futures_unordered = futures::stream::FuturesUnordered::new();
for package in packages {

12
cli/npm/managed/resolvers/global.rs
View file

@ -8,6 +8,8 @@ use std::path::PathBuf;
use std::sync::Arc;
use crate::colors;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
@ -24,8 +26,6 @@ use node_resolver::errors::ReferrerNotFoundError;
use crate::args::LifecycleScriptsConfig;
use crate::cache::FastInsecureHasher;
use super::super::cache::NpmCache;
use super::super::cache::TarballCache;
use super::super::resolution::NpmResolution;
use super::common::cache_packages;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
@ -35,8 +35,8 @@ use super::common::RegistryReadPermissionChecker;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageResolver {
cache: Arc<NpmCache>,
tarball_cache: Arc<TarballCache>,
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
registry_read_permission_checker: RegistryReadPermissionChecker,
@ -45,9 +45,9 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn FileSystem>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

16
cli/npm/managed/resolvers/local.rs
View file

@ -17,6 +17,8 @@ use std::sync::Arc;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
@ -52,8 +54,6 @@ use crate::util::fs::LaxSingleProcessFsFlag;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressMessagePrompt;
use super::super::cache::NpmCache;
use super::super::cache::TarballCache;
use super::super::resolution::NpmResolution;
use super::common::bin_entries;
use super::common::NpmPackageFsResolver;
@ -63,12 +63,12 @@ use super::common::RegistryReadPermissionChecker;
/// and resolves packages from it.
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
system_info: NpmSystemInfo,
@ -79,12 +79,12 @@ pub struct LocalNpmPackageResolver {
impl LocalNpmPackageResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
@ -284,10 +284,10 @@ fn local_node_modules_package_contents_path(
#[allow(clippy::too_many_arguments)]
async fn sync_resolution_with_fs(
snapshot: &NpmResolutionSnapshot,
cache: &Arc<NpmCache>,
cache: &Arc<CliNpmCache>,
npm_install_deps_provider: &NpmInstallDepsProvider,
progress_bar: &ProgressBar,
tarball_cache: &Arc<TarballCache>,
tarball_cache: &Arc<CliNpmTarballCache>,
root_node_modules_dir_path: &Path,
system_info: &NpmSystemInfo,
lifecycle_scripts: &LifecycleScriptsConfig,

8
cli/npm/managed/resolvers/mod.rs
View file

@ -12,6 +12,8 @@ use deno_runtime::deno_fs::FileSystem;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::util::progress_bar::ProgressBar;
pub use self::common::NpmPackageFsResolver;
@ -19,18 +21,16 @@ pub use self::common::NpmPackageFsResolver;
use self::global::GlobalNpmPackageResolver;
use self::local::LocalNpmPackageResolver;
use super::cache::NpmCache;
use super::cache::TarballCache;
use super::resolution::NpmResolution;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

111
cli/npm/mod.rs
View file

@ -1,33 +1,39 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod byonm;
mod common;
mod managed;
use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
use common::maybe_auth_header_for_npm_registry;
use dashmap::DashMap;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use managed::cache::registry_info::get_package_url;
use http::HeaderName;
use http::HeaderValue;
use managed::create_managed_in_npm_pkg_checker;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmPackageFolderResolver;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::hard_link_dir_recursive;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::progress_bar::ProgressBar;
pub use self::byonm::CliByonmNpmResolver;
pub use self::byonm::CliByonmNpmResolverCreateOptions;
@ -36,6 +42,99 @@ pub use self::managed::CliManagedNpmResolverCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub type CliNpmTarballCache = deno_npm_cache::TarballCache<CliNpmCacheEnv>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliNpmCacheEnv>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheEnv>;
#[derive(Debug)]
pub struct CliNpmCacheEnv {
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
}
impl CliNpmCacheEnv {
pub fn new(
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
) -> Self {
Self {
fs,
http_client_provider,
progress_bar,
}
}
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
fn exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
// todo(dsherret): use self.fs here instead
hard_link_dir_recursive(from, to)
}
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.fs.as_ref(),
write_mode: crate::cache::CACHE_PERM,
},
file_path,
data,
)
}
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,
maybe_auth_header: Option<(HeaderName, HeaderValue)>,
) -> Result<Option<Vec<u8>>, deno_npm_cache::DownloadError> {
let guard = self.progress_bar.update(url.as_str());
let client = self.http_client_provider.get_or_create().map_err(|err| {
deno_npm_cache::DownloadError {
status_code: None,
error: err,
}
})?;
client
.download_with_progress_and_retries(url, maybe_auth_header, &guard)
.await
.map_err(|err| {
use crate::http_util::DownloadError::*;
let status_code = match &err {
Fetch { .. }
| UrlParse { .. }
| HttpParse { .. }
| Json { .. }
| ToStr { .. }
| NoRedirectHeader { .. }
| TooManyRedirects => None,
BadResponse(bad_response_error) => {
Some(bad_response_error.status_code)
}
};
deno_npm_cache::DownloadError {
status_code,
error: err.into(),
}
})
}
}
pub enum CliNpmResolverCreateOptions {
Managed(CliManagedNpmResolverCreateOptions),
Byonm(CliByonmNpmResolverCreateOptions),
@ -179,13 +278,15 @@ impl NpmFetchResolver {
if let Some(info) = self.info_by_name.get(name) {
return info.value().clone();
}
// todo(#27198): use RegistryInfoProvider instead
let fetch_package_info = || async {
let info_url = get_package_url(&self.npmrc, name);
let info_url = deno_npm_cache::get_package_url(&self.npmrc, name);
let file_fetcher = self.file_fetcher.clone();
let registry_config = self.npmrc.get_registry_config(name);
// TODO(bartlomieju): this should error out, not use `.ok()`.
let maybe_auth_header =
maybe_auth_header_for_npm_registry(registry_config).ok()?;
deno_npm_cache::maybe_auth_header_for_npm_registry(registry_config)
.ok()?;
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn(async move {
file_fetcher

19
cli/tools/fmt.rs
View file

@ -440,8 +440,10 @@ pub fn format_html(
)
}
_ => {
let mut typescript_config =
get_resolved_typescript_config(fmt_options);
let mut typescript_config_builder =
get_typescript_config_builder(fmt_options);
typescript_config_builder.file_indent_level(hints.indent_level);
let mut typescript_config = typescript_config_builder.build();
typescript_config.line_width = hints.print_width as u32;
dprint_plugin_typescript::format_text(
&path,
@ -919,9 +921,9 @@ fn files_str(len: usize) -> &'static str {
}
}
fn get_resolved_typescript_config(
fn get_typescript_config_builder(
options: &FmtOptionsConfig,
) -> dprint_plugin_typescript::configuration::Configuration {
) -> dprint_plugin_typescript::configuration::ConfigurationBuilder {
let mut builder =
dprint_plugin_typescript::configuration::ConfigurationBuilder::new();
builder.deno();
@ -953,7 +955,13 @@ fn get_resolved_typescript_config(
});
}
builder.build()
builder
}
fn get_resolved_typescript_config(
options: &FmtOptionsConfig,
) -> dprint_plugin_typescript::configuration::Configuration {
get_typescript_config_builder(options).build()
}
fn get_resolved_markdown_config(
@ -1075,6 +1083,7 @@ fn get_resolved_markup_fmt_config(
};
let language_options = LanguageOptions {
script_formatter: Some(markup_fmt::config::ScriptFormatter::Dprint),
quotes: Quotes::Double,
format_comments: false,
script_indent: true,

40
cli/tools/task.rs
View file

@ -241,12 +241,15 @@ pub async fn execute_script(
description: None,
},
kill_signal,
cli_options.argv(),
)
.await;
}
for task_config in &packages_task_configs {
let exit_code = task_runner.run_tasks(task_config, &kill_signal).await?;
let exit_code = task_runner
.run_tasks(task_config, &kill_signal, cli_options.argv())
.await?;
if exit_code > 0 {
return Ok(exit_code);
}
@ -263,6 +266,7 @@ struct RunSingleOptions<'a> {
cwd: &'a Path,
custom_commands: HashMap<String, Rc<dyn ShellCommand>>,
kill_signal: KillSignal,
argv: &'a [String],
}
struct TaskRunner<'a> {
@ -279,9 +283,10 @@ impl<'a> TaskRunner<'a> {
&self,
pkg_tasks_config: &PackageTaskInfo,
kill_signal: &KillSignal,
argv: &[String],
) -> Result<i32, deno_core::anyhow::Error> {
match sort_tasks_topo(pkg_tasks_config) {
Ok(sorted) => self.run_tasks_in_parallel(sorted, kill_signal).await,
Ok(sorted) => self.run_tasks_in_parallel(sorted, kill_signal, argv).await,
Err(err) => match err {
TaskError::NotFound(name) => {
if self.task_flags.is_run {
@ -317,6 +322,7 @@ impl<'a> TaskRunner<'a> {
&self,
tasks: Vec<ResolvedTask<'a>>,
kill_signal: &KillSignal,
args: &[String],
) -> Result<i32, deno_core::anyhow::Error> {
struct PendingTasksContext<'a> {
completed: HashSet<usize>,
@ -338,13 +344,21 @@ impl<'a> TaskRunner<'a> {
&mut self,
runner: &'b TaskRunner<'b>,
kill_signal: &KillSignal,
argv: &'a [String],
) -> Option<
LocalBoxFuture<'b, Result<(i32, &'a ResolvedTask<'a>), AnyError>>,
>
where
'a: 'b,
{
for task in self.tasks.iter() {
let mut tasks_iter = self.tasks.iter().peekable();
while let Some(task) = tasks_iter.next() {
let args = if tasks_iter.peek().is_none() {
argv
} else {
&[]
};
if self.completed.contains(&task.id)
|| self.running.contains(&task.id)
{
@ -366,7 +380,13 @@ impl<'a> TaskRunner<'a> {
match task.task_or_script {
TaskOrScript::Task(_, def) => {
runner
.run_deno_task(task.folder_url, task.name, def, kill_signal)
.run_deno_task(
task.folder_url,
task.name,
def,
kill_signal,
args,
)
.await
}
TaskOrScript::Script(scripts, _) => {
@ -376,6 +396,7 @@ impl<'a> TaskRunner<'a> {
task.name,
scripts,
kill_signal,
args,
)
.await
}
@ -399,7 +420,7 @@ impl<'a> TaskRunner<'a> {
while context.has_remaining_tasks() {
while queue.len() < self.concurrency {
if let Some(task) = context.get_next_task(self, kill_signal) {
if let Some(task) = context.get_next_task(self, kill_signal, args) {
queue.push(task);
} else {
break;
@ -429,6 +450,7 @@ impl<'a> TaskRunner<'a> {
task_name: &str,
definition: &TaskDefinition,
kill_signal: KillSignal,
argv: &'a [String],
) -> Result<i32, deno_core::anyhow::Error> {
let cwd = match &self.task_flags.cwd {
Some(path) => canonicalize_path(&PathBuf::from(path))
@ -447,6 +469,7 @@ impl<'a> TaskRunner<'a> {
cwd: &cwd,
custom_commands,
kill_signal,
argv,
})
.await
}
@ -457,6 +480,7 @@ impl<'a> TaskRunner<'a> {
task_name: &str,
scripts: &IndexMap<String, String>,
kill_signal: KillSignal,
argv: &[String],
) -> Result<i32, deno_core::anyhow::Error> {
// ensure the npm packages are installed if using a managed resolver
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
@ -489,6 +513,7 @@ impl<'a> TaskRunner<'a> {
cwd: &cwd,
custom_commands: custom_commands.clone(),
kill_signal: kill_signal.clone(),
argv,
})
.await?;
if exit_code > 0 {
@ -510,11 +535,12 @@ impl<'a> TaskRunner<'a> {
cwd,
custom_commands,
kill_signal,
argv,
} = opts;
output_task(
opts.task_name,
&task_runner::get_script_with_args(script, self.cli_options.argv()),
&task_runner::get_script_with_args(script, argv),
);
Ok(
@ -525,7 +551,7 @@ impl<'a> TaskRunner<'a> {
env_vars: self.env_vars.clone(),
custom_commands,
init_cwd: self.cli_options.initial_cwd(),
argv: self.cli_options.argv(),
argv,
root_node_modules_dir: self.npm_resolver.root_node_modules_path(),
stdio: None,
kill_signal,

13
cli/util/path.rs
View file

@ -51,19 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option<String> {
.map(|e| e.to_lowercase());
}
pub fn get_atomic_dir_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let new_file_name = format!(
".{}_{}",
file_path
.file_name()
.map(|f| f.to_string_lossy())
.unwrap_or(Cow::Borrowed("")),
rand
);
file_path.with_file_name(new_file_name)
}
pub fn get_atomic_file_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let extension = format!("{rand}.tmp");

2
cli/util/sync/mod.rs
View file

@ -3,11 +3,9 @@
mod async_flag;
mod sync_read_async_write_lock;
mod task_queue;
mod value_creator;
pub use async_flag::AsyncFlag;
pub use deno_core::unsync::sync::AtomicFlag;
pub use sync_read_async_write_lock::SyncReadAsyncWriteLock;
pub use task_queue::TaskQueue;
pub use task_queue::TaskQueuePermit;
pub use value_creator::MultiRuntimeAsyncValueCreator;

213
cli/util/sync/value_creator.rs
View file

@ -1,213 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::sync::Arc;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::future::LocalBoxFuture;
use deno_core::futures::future::Shared;
use deno_core::futures::FutureExt;
use deno_core::parking_lot::Mutex;
use tokio::task::JoinError;
type JoinResult<TResult> = Result<TResult, Arc<JoinError>>;
type CreateFutureFn<TResult> =
Box<dyn Fn() -> LocalBoxFuture<'static, TResult> + Send + Sync>;
#[derive(Debug)]
struct State<TResult> {
retry_index: usize,
future: Option<Shared<BoxFuture<'static, JoinResult<TResult>>>>,
}
/// Attempts to create a shared value asynchronously on one tokio runtime while
/// many runtimes are requesting the value.
///
/// This is only useful when the value needs to get created once across
/// many runtimes.
///
/// This handles the case where the tokio runtime creating the value goes down
/// while another one is waiting on the value.
pub struct MultiRuntimeAsyncValueCreator<TResult: Send + Clone + 'static> {
create_future: CreateFutureFn<TResult>,
state: Mutex<State<TResult>>,
}
impl<TResult: Send + Clone + 'static> std::fmt::Debug
for MultiRuntimeAsyncValueCreator<TResult>
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MultiRuntimeAsyncValueCreator").finish()
}
}
impl<TResult: Send + Clone + 'static> MultiRuntimeAsyncValueCreator<TResult> {
pub fn new(create_future: CreateFutureFn<TResult>) -> Self {
Self {
state: Mutex::new(State {
retry_index: 0,
future: None,
}),
create_future,
}
}
pub async fn get(&self) -> TResult {
let (mut future, mut retry_index) = {
let mut state = self.state.lock();
let future = match &state.future {
Some(future) => future.clone(),
None => {
let future = self.create_shared_future();
state.future = Some(future.clone());
future
}
};
(future, state.retry_index)
};
loop {
let result = future.await;
match result {
Ok(result) => return result,
Err(join_error) => {
if join_error.is_cancelled() {
let mut state = self.state.lock();
if state.retry_index == retry_index {
// we were the first one to retry, so create a new future
// that we'll run from the current runtime
state.retry_index += 1;
state.future = Some(self.create_shared_future());
}
retry_index = state.retry_index;
future = state.future.as_ref().unwrap().clone();
// just in case we're stuck in a loop
if retry_index > 1000 {
panic!("Something went wrong.") // should never happen
}
} else {
panic!("{}", join_error);
}
}
}
}
}
fn create_shared_future(
&self,
) -> Shared<BoxFuture<'static, JoinResult<TResult>>> {
let future = (self.create_future)();
deno_core::unsync::spawn(future)
.map(|result| result.map_err(Arc::new))
.boxed()
.shared()
}
}
#[cfg(test)]
mod test {
use deno_core::unsync::spawn;
use super::*;
#[tokio::test]
async fn single_runtime() {
let value_creator = MultiRuntimeAsyncValueCreator::new(Box::new(|| {
async { 1 }.boxed_local()
}));
let value = value_creator.get().await;
assert_eq!(value, 1);
}
#[test]
fn multi_runtimes() {
let value_creator =
Arc::new(MultiRuntimeAsyncValueCreator::new(Box::new(|| {
async {
tokio::task::yield_now().await;
1
}
.boxed_local()
})));
let handles = (0..3)
.map(|_| {
let value_creator = value_creator.clone();
std::thread::spawn(|| {
create_runtime().block_on(async move { value_creator.get().await })
})
})
.collect::<Vec<_>>();
for handle in handles {
assert_eq!(handle.join().unwrap(), 1);
}
}
#[test]
fn multi_runtimes_first_never_finishes() {
let is_first_run = Arc::new(Mutex::new(true));
let (tx, rx) = std::sync::mpsc::channel::<()>();
let value_creator = Arc::new(MultiRuntimeAsyncValueCreator::new({
let is_first_run = is_first_run.clone();
Box::new(move || {
let is_first_run = is_first_run.clone();
let tx = tx.clone();
async move {
let is_first_run = {
let mut is_first_run = is_first_run.lock();
let initial_value = *is_first_run;
*is_first_run = false;
tx.send(()).unwrap();
initial_value
};
if is_first_run {
tokio::time::sleep(std::time::Duration::from_millis(30_000)).await;
panic!("TIMED OUT"); // should not happen
} else {
tokio::task::yield_now().await;
}
1
}
.boxed_local()
})
}));
std::thread::spawn({
let value_creator = value_creator.clone();
let is_first_run = is_first_run.clone();
move || {
create_runtime().block_on(async {
let value_creator = value_creator.clone();
// spawn a task that will never complete
spawn(async move { value_creator.get().await });
// wait for the task to set is_first_run to false
while *is_first_run.lock() {
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
// now exit the runtime while the value_creator is still pending
})
}
});
let handle = {
let value_creator = value_creator.clone();
std::thread::spawn(|| {
create_runtime().block_on(async move {
let value_creator = value_creator.clone();
rx.recv().unwrap();
// even though the other runtime shutdown, this get() should
// recover and still get the value
value_creator.get().await
})
})
};
assert_eq!(handle.join().unwrap(), 1);
}
fn create_runtime() -> tokio::runtime::Runtime {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
}
}

105
ext/node/polyfills/_fs/_fs_readdir.ts
View file

@ -4,12 +4,13 @@
// deno-lint-ignore-file prefer-primordials
import { TextDecoder, TextEncoder } from "ext:deno_web/08_text_encoding.js";
import { asyncIterableToCallback } from "ext:deno_node/_fs/_fs_watch.ts";
import Dirent from "ext:deno_node/_fs/_fs_dirent.ts";
import { denoErrorToNodeError } from "ext:deno_node/internal/errors.ts";
import { getValidatedPath } from "ext:deno_node/internal/fs/utils.mjs";
import { Buffer } from "node:buffer";
import { promisify } from "ext:deno_node/internal/util.mjs";
import { op_fs_read_dir_async, op_fs_read_dir_sync } from "ext:core/ops";
import { join, relative } from "node:path";
function toDirent(val: Deno.DirEntry & { parentPath: string }): Dirent {
return new Dirent(val);
@ -18,6 +19,7 @@ function toDirent(val: Deno.DirEntry & { parentPath: string }): Dirent {
type readDirOptions = {
encoding?: string;
withFileTypes?: boolean;
recursive?: boolean;
};
type readDirCallback = (err: Error | null, files: string[]) => void;
@ -30,12 +32,12 @@ type readDirBoth = (
export function readdir(
path: string | Buffer | URL,
options: { withFileTypes?: false; encoding?: string },
options: readDirOptions,
callback: readDirCallback,
): void;
export function readdir(
path: string | Buffer | URL,
options: { withFileTypes: true; encoding?: string },
options: readDirOptions,
callback: readDirCallbackDirent,
): void;
export function readdir(path: string | URL, callback: readDirCallback): void;
@ -51,8 +53,7 @@ export function readdir(
const options = typeof optionsOrCallback === "object"
? optionsOrCallback
: null;
const result: Array<string | Dirent> = [];
path = getValidatedPath(path);
path = getValidatedPath(path).toString();
if (!callback) throw new Error("No callback function supplied");
@ -66,24 +67,44 @@ export function readdir(
}
}
try {
path = path.toString();
asyncIterableToCallback(Deno.readDir(path), (val, done) => {
if (typeof path !== "string") return;
if (done) {
callback(null, result);
const result: Array<string | Dirent> = [];
const dirs = [path];
let current: string | undefined;
(async () => {
while ((current = dirs.shift()) !== undefined) {
try {
const entries = await op_fs_read_dir_async(current);
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
if (options?.recursive && entry.isDirectory) {
dirs.push(join(current, entry.name));
}
if (options?.withFileTypes) {
entry.parentPath = current;
result.push(toDirent(entry));
} else {
let name = decode(entry.name, options?.encoding);
if (options?.recursive) {
name = relative(path, join(current, name));
}
result.push(name);
}
}
} catch (err) {
callback(
denoErrorToNodeError(err as Error, {
syscall: "readdir",
path: current,
}),
);
return;
}
if (options?.withFileTypes) {
val.parentPath = path;
result.push(toDirent(val));
} else result.push(decode(val.name));
}, (e) => {
callback(denoErrorToNodeError(e as Error, { syscall: "readdir" }));
});
} catch (e) {
callback(denoErrorToNodeError(e as Error, { syscall: "readdir" }));
}
}
callback(null, result);
})();
}
function decode(str: string, encoding?: string): string {
@ -118,8 +139,7 @@ export function readdirSync(
path: string | Buffer | URL,
options?: readDirOptions,
): Array<string | Dirent> {
const result = [];
path = getValidatedPath(path);
path = getValidatedPath(path).toString();
if (options?.encoding) {
try {
@ -131,16 +151,37 @@ export function readdirSync(
}
}
try {
path = path.toString();
for (const file of Deno.readDirSync(path)) {
if (options?.withFileTypes) {
file.parentPath = path;
result.push(toDirent(file));
} else result.push(decode(file.name));
const result: Array<string | Dirent> = [];
const dirs = [path];
let current: string | undefined;
while ((current = dirs.shift()) !== undefined) {
try {
const entries = op_fs_read_dir_sync(current);
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
if (options?.recursive && entry.isDirectory) {
dirs.push(join(current, entry.name));
}
if (options?.withFileTypes) {
entry.parentPath = current;
result.push(toDirent(entry));
} else {
let name = decode(entry.name, options?.encoding);
if (options?.recursive) {
name = relative(path, join(current, name));
}
result.push(name);
}
}
} catch (e) {
throw denoErrorToNodeError(e as Error, {
syscall: "readdir",
path: current,
});
}
} catch (e) {
throw denoErrorToNodeError(e as Error, { syscall: "readdir" });
}
return result;
}

8
ext/web/00_infra.js
View file

@ -271,7 +271,7 @@ function addPaddingToBase64url(base64url) {
if (base64url.length % 4 === 2) return base64url + "==";
if (base64url.length % 4 === 3) return base64url + "=";
if (base64url.length % 4 === 1) {
throw new TypeError("Illegal base64url string!");
throw new TypeError("Illegal base64url string");
}
return base64url;
}
@ -382,7 +382,7 @@ function assert(cond, msg = "Assertion failed.") {
function serializeJSValueToJSONString(value) {
const result = JSONStringify(value);
if (result === undefined) {
throw new TypeError("Value is not JSON serializable.");
throw new TypeError("Value is not JSON serializable");
}
return result;
}
@ -429,7 +429,7 @@ function pathFromURLWin32(url) {
*/
function pathFromURLPosix(url) {
if (url.hostname !== "") {
throw new TypeError(`Host must be empty.`);
throw new TypeError("Host must be empty");
}
return decodeURIComponent(
@ -444,7 +444,7 @@ function pathFromURLPosix(url) {
function pathFromURL(pathOrUrl) {
if (ObjectPrototypeIsPrototypeOf(URLPrototype, pathOrUrl)) {
if (pathOrUrl.protocol != "file:") {
throw new TypeError("Must be a file URL.");
throw new TypeError("Must be a file URL");
}
return core.build.os == "windows"

4
ext/web/02_event.js
View file

@ -1031,11 +1031,11 @@ class EventTarget {
}
if (getDispatched(event)) {
throw new DOMException("Invalid event state.", "InvalidStateError");
throw new DOMException("Invalid event state", "InvalidStateError");
}
if (event.eventPhase !== Event.NONE) {
throw new DOMException("Invalid event state.", "InvalidStateError");
throw new DOMException("Invalid event state", "InvalidStateError");
}
return dispatch(self, event);

2
ext/web/03_abort_signal.js
View file

@ -196,7 +196,7 @@ class AbortSignal extends EventTarget {
constructor(key = null) {
if (key !== illegalConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
super();
}

6
ext/web/04_global_interfaces.js
View file

@ -16,7 +16,7 @@ const illegalConstructorKey = Symbol("illegalConstructorKey");
class Window extends EventTarget {
constructor(key = null) {
if (key !== illegalConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
super();
}
@ -29,7 +29,7 @@ class Window extends EventTarget {
class WorkerGlobalScope extends EventTarget {
constructor(key = null) {
if (key != illegalConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
super();
}
@ -42,7 +42,7 @@ class WorkerGlobalScope extends EventTarget {
class DedicatedWorkerGlobalScope extends WorkerGlobalScope {
constructor(key = null) {
if (key != illegalConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
super();
}

2
ext/web/05_base64.js
View file

@ -50,7 +50,7 @@ function btoa(data) {
} catch (e) {
if (ObjectPrototypeIsPrototypeOf(TypeErrorPrototype, e)) {
throw new DOMException(
"The string to be encoded contains characters outside of the Latin1 range.",
"Cannot encode string: string contains characters outside of the Latin1 range",
"InvalidCharacterError",
);
}

58
ext/web/06_streams.js
View file

@ -523,10 +523,14 @@ function dequeueValue(container) {
function enqueueValueWithSize(container, value, size) {
assert(container[_queue] && typeof container[_queueTotalSize] === "number");
if (isNonNegativeNumber(size) === false) {
throw new RangeError("chunk size isn't a positive number");
throw new RangeError(
"Cannot enqueue value with size: chunk size must be a positive number",
);
}
if (size === Infinity) {
throw new RangeError("chunk size is invalid");
throw new RangeError(
"Cannot enqueue value with size: chunk size is invalid",
);
}
container[_queue].enqueue({ value, size });
container[_queueTotalSize] += size;
@ -1097,7 +1101,7 @@ async function readableStreamCollectIntoUint8Array(stream) {
if (TypedArrayPrototypeGetSymbolToStringTag(chunk) !== "Uint8Array") {
throw new TypeError(
"Can't convert value to Uint8Array while consuming the stream",
"Cannot convert value to Uint8Array while consuming the stream",
);
}
@ -1347,7 +1351,7 @@ function readableByteStreamControllerEnqueue(controller, chunk) {
if (isDetachedBuffer(buffer)) {
throw new TypeError(
"chunk's buffer is detached and so cannot be enqueued",
"Chunk's buffer is detached and so cannot be enqueued",
);
}
const transferredBuffer = ArrayBufferPrototypeTransferToFixedLength(buffer);
@ -2095,14 +2099,14 @@ function readableByteStreamControllerRespond(controller, bytesWritten) {
if (state === "closed") {
if (bytesWritten !== 0) {
throw new TypeError(
"bytesWritten must be 0 when calling respond() on a closed stream",
`"bytesWritten" must be 0 when calling respond() on a closed stream: received ${bytesWritten}`,
);
}
} else {
assert(state === "readable");
if (bytesWritten === 0) {
throw new TypeError(
"bytesWritten must be greater than 0 when calling respond() on a readable stream",
'"bytesWritten" must be greater than 0 when calling respond() on a readable stream',
);
}
if (
@ -2110,7 +2114,7 @@ function readableByteStreamControllerRespond(controller, bytesWritten) {
// deno-lint-ignore prefer-primordials
firstDescriptor.byteLength
) {
throw new RangeError("bytesWritten out of range");
throw new RangeError('"bytesWritten" out of range');
}
}
firstDescriptor.buffer = ArrayBufferPrototypeTransferToFixedLength(
@ -2305,7 +2309,7 @@ function readableByteStreamControllerRespondWithNewView(controller, view) {
if (state === "closed") {
if (byteLength !== 0) {
throw new TypeError(
"The view's length must be 0 when calling respondWithNewView() on a closed stream",
`The view's length must be 0 when calling respondWithNewView() on a closed stream: received ${byteLength}`,
);
}
} else {
@ -3577,7 +3581,7 @@ function setUpReadableByteStreamControllerFromUnderlyingSource(
}
const autoAllocateChunkSize = underlyingSourceDict["autoAllocateChunkSize"];
if (autoAllocateChunkSize === 0) {
throw new TypeError("autoAllocateChunkSize must be greater than 0");
throw new TypeError('"autoAllocateChunkSize" must be greater than 0');
}
setUpReadableByteStreamController(
stream,
@ -3706,7 +3710,7 @@ function setUpReadableStreamDefaultControllerFromUnderlyingSource(
*/
function setUpReadableStreamBYOBReader(reader, stream) {
if (isReadableStreamLocked(stream)) {
throw new TypeError("ReadableStream is locked.");
throw new TypeError("ReadableStream is locked");
}
if (
!(ObjectPrototypeIsPrototypeOf(
@ -3727,7 +3731,7 @@ function setUpReadableStreamBYOBReader(reader, stream) {
*/
function setUpReadableStreamDefaultReader(reader, stream) {
if (isReadableStreamLocked(stream)) {
throw new TypeError("ReadableStream is locked.");
throw new TypeError("ReadableStream is locked");
}
readableStreamReaderGenericInitialize(reader, stream);
reader[_readRequests] = new Queue();
@ -3961,7 +3965,7 @@ function setUpWritableStreamDefaultControllerFromUnderlyingSink(
*/
function setUpWritableStreamDefaultWriter(writer, stream) {
if (isWritableStreamLocked(stream) === true) {
throw new TypeError("The stream is already locked.");
throw new TypeError("The stream is already locked");
}
writer[_stream] = stream;
stream[_writer] = writer;
@ -4019,7 +4023,7 @@ function transformStreamDefaultControllerEnqueue(controller, chunk) {
/** @type {ReadableStreamDefaultController<O>} */ readableController,
) === false
) {
throw new TypeError("Readable stream is unavailable.");
throw new TypeError("Readable stream is unavailable");
}
try {
readableStreamDefaultControllerEnqueue(
@ -5143,7 +5147,7 @@ class ReadableStream {
if (underlyingSourceDict.type === "bytes") {
if (strategy.size !== undefined) {
throw new RangeError(
`${prefix}: When underlying source is "bytes", strategy.size must be undefined.`,
`${prefix}: When underlying source is "bytes", strategy.size must be 'undefined'`,
);
}
const highWaterMark = extractHighWaterMark(strategy, 0);
@ -5273,10 +5277,10 @@ class ReadableStream {
const { readable, writable } = transform;
const { preventClose, preventAbort, preventCancel, signal } = options;
if (isReadableStreamLocked(this)) {
throw new TypeError("ReadableStream is already locked.");
throw new TypeError("ReadableStream is already locked");
}
if (isWritableStreamLocked(writable)) {
throw new TypeError("Target WritableStream is already locked.");
throw new TypeError("Target WritableStream is already locked");
}
const promise = readableStreamPipeTo(
this,
@ -5814,7 +5818,7 @@ class ReadableByteStreamController {
}
if (this[_stream][_state] !== "readable") {
throw new TypeError(
"ReadableByteStreamController's stream is not in a readable state.",
"ReadableByteStreamController's stream is not in a readable state",
);
}
readableByteStreamControllerClose(this);
@ -5846,7 +5850,7 @@ class ReadableByteStreamController {
if (byteLength === 0) {
throw webidl.makeException(
TypeError,
"length must be non-zero",
"Length must be non-zero",
prefix,
arg1,
);
@ -5854,19 +5858,19 @@ class ReadableByteStreamController {
if (getArrayBufferByteLength(buffer) === 0) {
throw webidl.makeException(
TypeError,
"buffer length must be non-zero",
"Buffer length must be non-zero",
prefix,
arg1,
);
}
if (this[_closeRequested] === true) {
throw new TypeError(
"Cannot enqueue chunk after a close has been requested.",
"Cannot enqueue chunk after a close has been requested",
);
}
if (this[_stream][_state] !== "readable") {
throw new TypeError(
"Cannot enqueue chunk when underlying stream is not readable.",
"Cannot enqueue chunk when underlying stream is not readable",
);
}
return readableByteStreamControllerEnqueue(this, chunk);
@ -6006,7 +6010,7 @@ class ReadableStreamDefaultController {
close() {
webidl.assertBranded(this, ReadableStreamDefaultControllerPrototype);
if (readableStreamDefaultControllerCanCloseOrEnqueue(this) === false) {
throw new TypeError("The stream controller cannot close or enqueue.");
throw new TypeError("The stream controller cannot close or enqueue");
}
readableStreamDefaultControllerClose(this);
}
@ -6021,7 +6025,7 @@ class ReadableStreamDefaultController {
chunk = webidl.converters.any(chunk);
}
if (readableStreamDefaultControllerCanCloseOrEnqueue(this) === false) {
throw new TypeError("The stream controller cannot close or enqueue.");
throw new TypeError("The stream controller cannot close or enqueue");
}
readableStreamDefaultControllerEnqueue(this, chunk);
}
@ -6146,12 +6150,12 @@ class TransformStream {
);
if (transformerDict.readableType !== undefined) {
throw new RangeError(
`${prefix}: readableType transformers not supported.`,
`${prefix}: readableType transformers not supported`,
);
}
if (transformerDict.writableType !== undefined) {
throw new RangeError(
`${prefix}: writableType transformers not supported.`,
`${prefix}: writableType transformers not supported`,
);
}
const readableHighWaterMark = extractHighWaterMark(readableStrategy, 0);
@ -6356,7 +6360,7 @@ class WritableStream {
);
if (underlyingSinkDict.type != null) {
throw new RangeError(
`${prefix}: WritableStream does not support 'type' in the underlying sink.`,
`${prefix}: WritableStream does not support 'type' in the underlying sink`,
);
}
initializeWritableStream(this);
@ -6483,7 +6487,7 @@ class WritableStreamDefaultWriter {
webidl.assertBranded(this, WritableStreamDefaultWriterPrototype);
if (this[_stream] === undefined) {
throw new TypeError(
"A writable stream is not associated with the writer.",
"A writable stream is not associated with the writer",
);
}
return writableStreamDefaultWriterGetDesiredSize(this);

2
ext/web/10_filereader.js
View file

@ -65,7 +65,7 @@ class FileReader extends EventTarget {
// 1. If fr's state is "loading", throw an InvalidStateError DOMException.
if (this[state] === "loading") {
throw new DOMException(
"Invalid FileReader state.",
"Invalid FileReader state",
"InvalidStateError",
);
}

50
ext/web/12_location.js
View file

@ -28,7 +28,7 @@ const locationConstructorKey = Symbol("locationConstructorKey");
class Location {
constructor(href = null, key = null) {
if (key != locationConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
const url = new URL(href);
url.username = "";
@ -41,7 +41,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.hash".`,
`Cannot set "location.hash"`,
"NotSupportedError",
);
},
@ -54,7 +54,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.host".`,
`Cannot set "location.host"`,
"NotSupportedError",
);
},
@ -67,7 +67,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.hostname".`,
`Cannot set "location.hostname"`,
"NotSupportedError",
);
},
@ -80,7 +80,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.href".`,
`Cannot set "location.href"`,
"NotSupportedError",
);
},
@ -100,7 +100,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.pathname".`,
`Cannot set "location.pathname"`,
"NotSupportedError",
);
},
@ -113,7 +113,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.port".`,
`Cannot set "location.port"`,
"NotSupportedError",
);
},
@ -126,7 +126,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.protocol".`,
`Cannot set "location.protocol"`,
"NotSupportedError",
);
},
@ -139,7 +139,7 @@ class Location {
},
set() {
throw new DOMException(
`Cannot set "location.search".`,
`Cannot set "location.search"`,
"NotSupportedError",
);
},
@ -161,7 +161,7 @@ class Location {
__proto__: null,
value: function assign() {
throw new DOMException(
`Cannot call "location.assign()".`,
`Cannot call "location.assign()"`,
"NotSupportedError",
);
},
@ -171,7 +171,7 @@ class Location {
__proto__: null,
value: function reload() {
throw new DOMException(
`Cannot call "location.reload()".`,
`Cannot call "location.reload()"`,
"NotSupportedError",
);
},
@ -181,7 +181,7 @@ class Location {
__proto__: null,
value: function replace() {
throw new DOMException(
`Cannot call "location.replace()".`,
`Cannot call "location.replace()"`,
"NotSupportedError",
);
},
@ -229,7 +229,7 @@ const workerLocationUrls = new SafeWeakMap();
class WorkerLocation {
constructor(href = null, key = null) {
if (key != locationConstructorKey) {
throw new TypeError("Illegal constructor.");
throw new TypeError("Illegal constructor");
}
const url = new URL(href);
url.username = "";
@ -244,7 +244,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.hash;
},
@ -256,7 +256,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.host;
},
@ -268,7 +268,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.hostname;
},
@ -280,7 +280,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.href;
},
@ -292,7 +292,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.origin;
},
@ -304,7 +304,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.pathname;
},
@ -316,7 +316,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.port;
},
@ -328,7 +328,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.protocol;
},
@ -340,7 +340,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
get() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.search;
},
@ -352,7 +352,7 @@ ObjectDefineProperties(WorkerLocation.prototype, {
value: function toString() {
const url = WeakMapPrototypeGet(workerLocationUrls, this);
if (url == null) {
throw new TypeError("Illegal invocation.");
throw new TypeError("Illegal invocation");
}
return url.href;
},
@ -414,7 +414,7 @@ const locationDescriptor = {
return location;
},
set() {
throw new DOMException(`Cannot set "location".`, "NotSupportedError");
throw new DOMException(`Cannot set "location"`, "NotSupportedError");
},
enumerable: true,
};
@ -422,7 +422,7 @@ const workerLocationDescriptor = {
get() {
if (workerLocation == null) {
throw new Error(
`Assertion: "globalThis.location" must be defined in a worker.`,
`Assertion: "globalThis.location" must be defined in a worker`,
);
}
return workerLocation;

14
ext/web/15_performance.js
View file

@ -123,14 +123,14 @@ function convertMarkToTimestamp(mark) {
const entry = findMostRecent(mark, "mark");
if (!entry) {
throw new DOMException(
`Cannot find mark: "${mark}".`,
`Cannot find mark: "${mark}"`,
"SyntaxError",
);
}
return entry.startTime;
}
if (mark < 0) {
throw new TypeError("Mark cannot be negative.");
throw new TypeError(`Mark cannot be negative: received ${mark}`);
}
return mark;
}
@ -261,7 +261,9 @@ class PerformanceMark extends PerformanceEntry {
super(name, "mark", startTime, 0, illegalConstructorKey);
this[webidl.brand] = webidl.brand;
if (startTime < 0) {
throw new TypeError("startTime cannot be negative");
throw new TypeError(
`Cannot construct PerformanceMark: startTime cannot be negative, received ${startTime}`,
);
}
this[_detail] = structuredClone(detail);
}
@ -504,14 +506,14 @@ class Performance extends EventTarget {
ObjectKeys(startOrMeasureOptions).length > 0
) {
if (endMark) {
throw new TypeError("Options cannot be passed with endMark.");
throw new TypeError('Options cannot be passed with "endMark"');
}
if (
!ReflectHas(startOrMeasureOptions, "start") &&
!ReflectHas(startOrMeasureOptions, "end")
) {
throw new TypeError(
"A start or end mark must be supplied in options.",
'A "start" or "end" mark must be supplied in options',
);
}
if (
@ -520,7 +522,7 @@ class Performance extends EventTarget {
ReflectHas(startOrMeasureOptions, "end")
) {
throw new TypeError(
"Cannot specify start, end, and duration together in options.",
'Cannot specify "start", "end", and "duration" together in options',
);
}
}

16
ext/web/16_image_data.js
View file

@ -84,35 +84,35 @@ class ImageData {
if (dataLength === 0) {
throw new DOMException(
"Failed to construct 'ImageData': The input data has zero elements.",
"Failed to construct 'ImageData': the input data has zero elements",
"InvalidStateError",
);
}
if (dataLength % 4 !== 0) {
throw new DOMException(
"Failed to construct 'ImageData': The input data length is not a multiple of 4.",
`Failed to construct 'ImageData': the input data length is not a multiple of 4, received ${dataLength}`,
"InvalidStateError",
);
}
if (sourceWidth < 1) {
throw new DOMException(
"Failed to construct 'ImageData': The source width is zero or not a number.",
"Failed to construct 'ImageData': the source width is zero or not a number",
"IndexSizeError",
);
}
if (webidl.type(sourceHeight) !== "Undefined" && sourceHeight < 1) {
throw new DOMException(
"Failed to construct 'ImageData': The source height is zero or not a number.",
"Failed to construct 'ImageData': the source height is zero or not a number",
"IndexSizeError",
);
}
if (dataLength / 4 % sourceWidth !== 0) {
throw new DOMException(
"Failed to construct 'ImageData': The input data length is not a multiple of (4 * width).",
"Failed to construct 'ImageData': the input data length is not a multiple of (4 * width)",
"IndexSizeError",
);
}
@ -122,7 +122,7 @@ class ImageData {
(sourceWidth * sourceHeight * 4 !== dataLength)
) {
throw new DOMException(
"Failed to construct 'ImageData': The input data length is not equal to (4 * width * height).",
"Failed to construct 'ImageData': the input data length is not equal to (4 * width * height)",
"IndexSizeError",
);
}
@ -159,14 +159,14 @@ class ImageData {
if (sourceWidth < 1) {
throw new DOMException(
"Failed to construct 'ImageData': The source width is zero or not a number.",
"Failed to construct 'ImageData': the source width is zero or not a number",
"IndexSizeError",
);
}
if (sourceHeight < 1) {
throw new DOMException(
"Failed to construct 'ImageData': The source height is zero or not a number.",
"Failed to construct 'ImageData': the source height is zero or not a number",
"IndexSizeError",
);
}

42
resolvers/npm_cache/Cargo.toml Normal file
View file

@ -0,0 +1,42 @@
# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
[package]
name = "deno_npm_cache"
version = "0.0.1"
authors.workspace = true
edition.workspace = true
license.workspace = true
readme = "README.md"
repository.workspace = true
description = "Helpers for downloading and caching npm dependencies for Deno"
[lib]
path = "lib.rs"
[dependencies]
# todo(dsherret): remove this dependency
anyhow.workspace = true
# todo(dsherret): remove this dependency
deno_core.workspace = true
async-trait.workspace = true
base64.workspace = true
boxed_error.workspace = true
deno_cache_dir.workspace = true
deno_npm.workspace = true
deno_semver.workspace = true
deno_unsync = { workspace = true, features = ["tokio"] }
faster-hex.workspace = true
flate2 = { workspace = true, features = ["zlib-ng-compat"] }
futures.workspace = true
http.workspace = true
log.workspace = true
parking_lot.workspace = true
percent-encoding.workspace = true
rand.workspace = true
ring.workspace = true
serde_json.workspace = true
tar.workspace = true
tempfile = "3.4.0"
thiserror.workspace = true
url.workspace = true

6
resolvers/npm_cache/README.md Normal file
View file

@ -0,0 +1,6 @@
# deno_npm_cache
[![crates](https://img.shields.io/crates/v/deno_npm_cache.svg)](https://crates.io/crates/deno_npm_cache)
[![docs](https://docs.rs/deno_npm_cache/badge.svg)](https://docs.rs/deno_npm_cache)
Helpers for downloading and caching npm dependencies for Deno.

130
cli/npm/managed/cache/mod.rs → resolvers/npm_cache/lib.rs
View file

@ -1,63 +1,133 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use anyhow::bail;
use anyhow::Context;
use anyhow::Error as AnyError;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use http::HeaderName;
use http::HeaderValue;
use http::StatusCode;
use parking_lot::Mutex;
use url::Url;
use crate::args::CacheSetting;
use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::hard_link_dir_recursive;
pub mod registry_info;
mod registry_info;
mod remote;
mod tarball;
mod tarball_extract;
pub use registry_info::RegistryInfoDownloader;
pub use registry_info::RegistryInfoProvider;
pub use tarball::TarballCache;
// todo(#27198): make both of these private and get the rest of the code
// using RegistryInfoProvider.
pub use registry_info::get_package_url;
pub use remote::maybe_auth_header_for_npm_registry;
#[derive(Debug)]
pub struct DownloadError {
pub status_code: Option<StatusCode>,
pub error: AnyError,
}
impl std::error::Error for DownloadError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self.error.as_ref())
}
}
impl std::fmt::Display for DownloadError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
self.error.fmt(f)
}
}
#[async_trait::async_trait(?Send)]
pub trait NpmCacheEnv: Send + Sync + 'static {
fn exists(&self, path: &Path) -> bool;
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError>;
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()>;
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,
maybe_auth_header: Option<(HeaderName, HeaderValue)>,
) -> Result<Option<Vec<u8>>, DownloadError>;
}
/// Indicates how cached source files should be handled.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum NpmCacheSetting {
/// Only the cached files should be used. Any files not in the cache will
/// error. This is the equivalent of `--cached-only` in the CLI.
Only,
/// No cached source files should be used, and all files should be reloaded.
/// This is the equivalent of `--reload` in the CLI.
ReloadAll,
/// Only some cached resources should be used. This is the equivalent of
/// `--reload=npm:chalk`
ReloadSome { npm_package_names: Vec<String> },
/// The cached source files should be used for local modules. This is the
/// default behavior of the CLI.
Use,
}
impl NpmCacheSetting {
pub fn should_use_for_npm_package(&self, package_name: &str) -> bool {
match self {
NpmCacheSetting::ReloadAll => false,
NpmCacheSetting::ReloadSome { npm_package_names } => {
!npm_package_names.iter().any(|n| n == package_name)
}
_ => true,
}
}
}
/// Stores a single copy of npm packages in a cache.
#[derive(Debug)]
pub struct NpmCache {
pub struct NpmCache<TEnv: NpmCacheEnv> {
env: Arc<TEnv>,
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
cache_setting: NpmCacheSetting,
npmrc: Arc<ResolvedNpmRc>,
/// ensures a package is only downloaded once per run
previously_reloaded_packages: Mutex<HashSet<PackageNv>>,
}
impl NpmCache {
impl<TEnv: NpmCacheEnv> NpmCache<TEnv> {
pub fn new(
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
cache_setting: NpmCacheSetting,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
cache_dir,
cache_setting,
env,
previously_reloaded_packages: Default::default(),
npmrc,
}
}
pub fn cache_setting(&self) -> &CacheSetting {
pub fn cache_setting(&self) -> &NpmCacheSetting {
&self.cache_setting
}
@ -118,7 +188,9 @@ impl NpmCache {
// it seems Windows does an "AccessDenied" error when moving a
// directory with hard links, so that's why this solution is done
with_folder_sync_lock(&folder_id.nv, &package_folder, || {
hard_link_dir_recursive(&original_package_folder, &package_folder)
self
.env
.hard_link_dir_recursive(&original_package_folder, &package_folder)
})?;
Ok(())
}
@ -158,7 +230,7 @@ impl NpmCache {
pub fn resolve_package_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
specifier: &Url,
) -> Option<NpmPackageCacheFolderId> {
self
.cache_dir
@ -180,7 +252,7 @@ impl NpmCache {
) -> Result<Option<NpmPackageInfo>, AnyError> {
let file_cache_path = self.get_registry_package_info_file_cache_path(name);
let file_text = match fs::read_to_string(file_cache_path) {
let file_text = match std::fs::read_to_string(file_cache_path) {
Ok(file_text) => file_text,
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err.into()),
@ -195,7 +267,9 @@ impl NpmCache {
) -> Result<(), AnyError> {
let file_cache_path = self.get_registry_package_info_file_cache_path(name);
let file_text = serde_json::to_string(&package_info)?;
atomic_write_file_with_retries(&file_cache_path, file_text, CACHE_PERM)?;
self
.env
.atomic_write_file_with_retries(&file_cache_path, file_text.as_bytes())?;
Ok(())
}
@ -216,7 +290,7 @@ fn with_folder_sync_lock(
output_folder: &Path,
action: impl FnOnce() -> Result<(), AnyError>,
) -> Result<(), AnyError> {
fs::create_dir_all(output_folder).with_context(|| {
std::fs::create_dir_all(output_folder).with_context(|| {
format!("Error creating '{}'.", output_folder.display())
})?;
@ -229,7 +303,7 @@ fn with_folder_sync_lock(
// then wait until the other process finishes with a timeout), but
// for now this is good enough.
let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME);
match fs::OpenOptions::new()
match std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
@ -257,7 +331,7 @@ fn with_folder_sync_lock(
match inner(output_folder, action) {
Ok(()) => Ok(()),
Err(err) => {
if let Err(remove_err) = fs::remove_dir_all(output_folder) {
if let Err(remove_err) = std::fs::remove_dir_all(output_folder) {
if remove_err.kind() != std::io::ErrorKind::NotFound {
bail!(
concat!(

71
cli/npm/managed/cache/registry_info.rs → resolvers/npm_cache/registry_info.rs
View file

@ -3,28 +3,22 @@
use std::collections::HashMap;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures::future::LocalBoxFuture;
use deno_core::futures::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_core::url::Url;
use anyhow::anyhow;
use anyhow::bail;
use anyhow::Context;
use anyhow::Error as AnyError;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_unsync::sync::MultiRuntimeAsyncValueCreator;
use futures::future::LocalBoxFuture;
use futures::FutureExt;
use parking_lot::Mutex;
use url::Url;
use crate::args::CacheSetting;
use crate::http_util::HttpClientProvider;
use crate::npm::common::maybe_auth_header_for_npm_registry;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::MultiRuntimeAsyncValueCreator;
use super::NpmCache;
// todo(dsherret): create seams and unit test this
use crate::remote::maybe_auth_header_for_npm_registry;
use crate::NpmCache;
use crate::NpmCacheEnv;
use crate::NpmCacheSetting;
type LoadResult = Result<FutureResult, Arc<AnyError>>;
type LoadFuture = LocalBoxFuture<'static, LoadResult>;
@ -49,30 +43,31 @@ enum MemoryCacheItem {
MemoryCached(Result<Option<Arc<NpmPackageInfo>>, Arc<AnyError>>),
}
// todo(#27198): refactor to store this only in the http cache and also
// consolidate with CliNpmRegistryApi.
/// Downloads packuments from the npm registry.
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct RegistryInfoDownloader {
cache: Arc<NpmCache>,
http_client_provider: Arc<HttpClientProvider>,
pub struct RegistryInfoProvider<TEnv: NpmCacheEnv> {
// todo(#27198): remove this
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
memory_cache: Mutex<HashMap<String, MemoryCacheItem>>,
}
impl RegistryInfoDownloader {
impl<TEnv: NpmCacheEnv> RegistryInfoProvider<TEnv> {
pub fn new(
cache: Arc<NpmCache>,
http_client_provider: Arc<HttpClientProvider>,
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
) -> Self {
Self {
cache,
http_client_provider,
env,
npmrc,
progress_bar,
memory_cache: Default::default(),
}
}
@ -94,8 +89,8 @@ impl RegistryInfoDownloader {
self: &Arc<Self>,
name: &str,
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
if *self.cache.cache_setting() == CacheSetting::Only {
return Err(custom_error(
if *self.cache.cache_setting() == NpmCacheSetting::Only {
return Err(deno_core::error::custom_error(
"NotCached",
format!(
"An npm specifier not found in cache: \"{name}\", --cached-only is specified."
@ -167,7 +162,7 @@ impl RegistryInfoDownloader {
) -> Result<NpmPackageInfo, AnyError> {
// this scenario failing should be exceptionally rare so let's
// deal with improving it only when anyone runs into an issue
let maybe_package_info = deno_core::unsync::spawn_blocking({
let maybe_package_info = deno_unsync::spawn_blocking({
let cache = self.cache.clone();
let name = name.to_string();
move || cache.load_package_info(&name)
@ -199,20 +194,18 @@ impl RegistryInfoDownloader {
return std::future::ready(Err(Arc::new(err))).boxed_local()
}
};
let guard = self.progress_bar.update(package_url.as_str());
let name = name.to_string();
async move {
let client = downloader.http_client_provider.get_or_create()?;
let maybe_bytes = client
.download_with_progress_and_retries(
let maybe_bytes = downloader
.env
.download_with_retries_on_any_tokio_runtime(
package_url,
maybe_auth_header,
&guard,
)
.await?;
match maybe_bytes {
Some(bytes) => {
let future_result = deno_core::unsync::spawn_blocking(
let future_result = deno_unsync::spawn_blocking(
move || -> Result<FutureResult, AnyError> {
let package_info = serde_json::from_slice(&bytes)?;
match downloader.cache.save_package_info(&name, &package_info) {
@ -241,6 +234,8 @@ impl RegistryInfoDownloader {
}
}
// todo(#27198): make this private and only use RegistryInfoProvider in the rest of
// the code
pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url {
let registry_url = npmrc.get_registry_url(name);
// The '/' character in scoped package names "@scope/name" must be

6
cli/npm/common.rs → resolvers/npm_cache/remote.rs
View file

@ -1,10 +1,10 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use anyhow::bail;
use anyhow::Context;
use anyhow::Error as AnyError;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_npm::npm_rc::RegistryConfig;
use http::header;

76
cli/npm/managed/cache/tarball.rs → resolvers/npm_cache/tarball.rs
View file

@ -3,33 +3,26 @@
use std::collections::HashMap;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures::future::LocalBoxFuture;
use deno_core::futures::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::url::Url;
use anyhow::anyhow;
use anyhow::bail;
use anyhow::Context;
use anyhow::Error as AnyError;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageVersionDistInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_semver::package::PackageNv;
use deno_unsync::sync::MultiRuntimeAsyncValueCreator;
use futures::future::LocalBoxFuture;
use futures::FutureExt;
use http::StatusCode;
use parking_lot::Mutex;
use url::Url;
use crate::args::CacheSetting;
use crate::http_util::DownloadError;
use crate::http_util::HttpClientProvider;
use crate::npm::common::maybe_auth_header_for_npm_registry;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::MultiRuntimeAsyncValueCreator;
use super::tarball_extract::verify_and_extract_tarball;
use super::tarball_extract::TarballExtractionMode;
use super::NpmCache;
// todo(dsherret): create seams and unit test this
use crate::remote::maybe_auth_header_for_npm_registry;
use crate::tarball_extract::verify_and_extract_tarball;
use crate::tarball_extract::TarballExtractionMode;
use crate::NpmCache;
use crate::NpmCacheEnv;
use crate::NpmCacheSetting;
type LoadResult = Result<(), Arc<AnyError>>;
type LoadFuture = LocalBoxFuture<'static, LoadResult>;
@ -49,29 +42,23 @@ enum MemoryCacheItem {
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct TarballCache {
cache: Arc<NpmCache>,
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
pub struct TarballCache<TEnv: NpmCacheEnv> {
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
memory_cache: Mutex<HashMap<PackageNv, MemoryCacheItem>>,
}
impl TarballCache {
impl<TEnv: NpmCacheEnv> TarballCache<TEnv> {
pub fn new(
cache: Arc<NpmCache>,
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
) -> Self {
Self {
cache,
fs,
http_client_provider,
env,
npmrc,
progress_bar,
memory_cache: Default::default(),
}
}
@ -144,11 +131,11 @@ impl TarballCache {
let package_folder =
tarball_cache.cache.package_folder_for_nv_and_url(&package_nv, registry_url);
let should_use_cache = tarball_cache.cache.should_use_cache_for_package(&package_nv);
let package_folder_exists = tarball_cache.fs.exists_sync(&package_folder);
let package_folder_exists = tarball_cache.env.exists(&package_folder);
if should_use_cache && package_folder_exists {
return Ok(());
} else if tarball_cache.cache.cache_setting() == &CacheSetting::Only {
return Err(custom_error(
} else if tarball_cache.cache.cache_setting() == &NpmCacheSetting::Only {
return Err(deno_core::error::custom_error(
"NotCached",
format!(
"An npm specifier not found in cache: \"{}\", --cached-only is specified.",
@ -169,15 +156,13 @@ impl TarballCache {
tarball_cache.npmrc.tarball_config(&tarball_uri);
let maybe_auth_header = maybe_registry_config.and_then(|c| maybe_auth_header_for_npm_registry(c).ok()?);
let guard = tarball_cache.progress_bar.update(&dist.tarball);
let result = tarball_cache.http_client_provider
.get_or_create()?
.download_with_progress_and_retries(tarball_uri, maybe_auth_header, &guard)
let result = tarball_cache.env
.download_with_retries_on_any_tokio_runtime(tarball_uri, maybe_auth_header)
.await;
let maybe_bytes = match result {
Ok(maybe_bytes) => maybe_bytes,
Err(DownloadError::BadResponse(err)) => {
if err.status_code == StatusCode::UNAUTHORIZED
Err(err) => {
if err.status_code == Some(StatusCode::UNAUTHORIZED)
&& maybe_registry_config.is_none()
&& tarball_cache.npmrc.get_registry_config(&package_nv.name).auth_token.is_some()
{
@ -194,7 +179,6 @@ impl TarballCache {
}
return Err(err.into())
},
Err(err) => return Err(err.into()),
};
match maybe_bytes {
Some(bytes) => {
@ -213,7 +197,7 @@ impl TarballCache {
};
let dist = dist.clone();
let package_nv = package_nv.clone();
deno_core::unsync::spawn_blocking(move || {
deno_unsync::spawn_blocking(move || {
verify_and_extract_tarball(
&package_nv,
&bytes,

45
cli/npm/managed/cache/tarball_extract.rs → resolvers/npm_cache/tarball_extract.rs
View file

@ -1,16 +1,17 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::collections::HashSet;
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use anyhow::bail;
use anyhow::Context;
use anyhow::Error as AnyError;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_npm::registry::NpmPackageVersionDistInfo;
use deno_npm::registry::NpmPackageVersionDistInfoIntegrity;
use deno_semver::package::PackageNv;
@ -18,8 +19,6 @@ use flate2::read::GzDecoder;
use tar::Archive;
use tar::EntryType;
use crate::util::path::get_atomic_dir_path;
#[derive(Debug, Copy, Clone)]
pub enum TarballExtractionMode {
/// Overwrites the destination directory without deleting any files.
@ -206,10 +205,30 @@ fn extract_tarball(data: &[u8], output_folder: &Path) -> Result<(), AnyError> {
Ok(())
}
fn get_atomic_dir_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let new_file_name = format!(
".{}_{}",
file_path
.file_name()
.map(|f| f.to_string_lossy())
.unwrap_or(Cow::Borrowed("")),
rand
);
file_path.with_file_name(new_file_name)
}
fn gen_rand_path_component() -> String {
(0..4).fold(String::new(), |mut output, _| {
output.push_str(&format!("{:02x}", rand::random::<u8>()));
output
})
}
#[cfg(test)]
mod test {
use deno_semver::Version;
use test_util::TempDir;
use tempfile::TempDir;
use super::*;
@ -303,21 +322,21 @@ mod test {
#[test]
fn rename_with_retries_succeeds_exists() {
let temp_dir = TempDir::new();
let temp_dir = TempDir::new().unwrap();
let folder_1 = temp_dir.path().join("folder_1");
let folder_2 = temp_dir.path().join("folder_2");
folder_1.create_dir_all();
folder_1.join("a.txt").write("test");
folder_2.create_dir_all();
std::fs::create_dir_all(&folder_1).unwrap();
std::fs::write(folder_1.join("a.txt"), "test").unwrap();
std::fs::create_dir_all(&folder_2).unwrap();
// this will not end up in the output as rename_with_retries assumes
// the folders ending up at the destination are the same
folder_2.join("b.txt").write("test2");
std::fs::write(folder_2.join("b.txt"), "test2").unwrap();
let dest_folder = temp_dir.path().join("dest_folder");
rename_with_retries(folder_1.as_path(), dest_folder.as_path()).unwrap();
rename_with_retries(folder_2.as_path(), dest_folder.as_path()).unwrap();
rename_with_retries(folder_1.as_path(), &dest_folder).unwrap();
rename_with_retries(folder_2.as_path(), &dest_folder).unwrap();
assert!(dest_folder.join("a.txt").exists());
assert!(!dest_folder.join("b.txt").exists());
}

9
resolvers/npm_cache/todo.md Normal file
View file

@ -0,0 +1,9 @@
This crate is a work in progress:
1. Remove `deno_core` dependency.
1. Remove `anyhow` dependency.
1. Add a clippy.toml file that bans accessing the file system directory and
instead does it through a trait.
1. Make this crate work in Wasm.
1. Refactor to store npm packument in a single place:
https://github.com/denoland/deno/issues/27198

4
tests/specs/fmt/html/__test__.jsonc
View file

@ -12,6 +12,10 @@
"broken": {
"args": "fmt broken.html",
"output": "broken.out"
},
"with_js": {
"args": "fmt --check with_js.html",
"output": "Checked 1 file\n"
}
}
}

9
tests/specs/fmt/html/with_js.html Normal file
View file

@ -0,0 +1,9 @@
<html>
<body>
<script>
/* some multi-line comment
with function below it */
someFunc();
</script>
</body>
</html>

1
tests/specs/npm/cjs_with_deps/__test__.jsonc
View file

@ -1,4 +1,5 @@
{
"tempDir": true,
"tests": {
"cjs_with_deps": {
"args": "run --allow-read --allow-env main.js",

2
tests/specs/npm/cjs_with_deps/main_info.out
View file

@ -3,7 +3,7 @@ type: JavaScript
dependencies: 14 unique
size: [WILDCARD]
file:///[WILDCARD]/cjs_with_deps/main.js ([WILDCARD])
file:///[WILDCARD]/main.js ([WILDCARD])
├─┬ npm:/chalk@4.1.2 ([WILDCARD])
│ ├─┬ npm:/ansi-styles@4.3.0 ([WILDCARD])
│ │ └─┬ npm:/color-convert@2.0.1 ([WILDCARD])

4
tests/specs/run/_070_location/070_location.ts.out
View file

@ -11,5 +11,5 @@ Location {
protocol: "https:",
search: "?baz"
}
NotSupportedError: Cannot set "location".
NotSupportedError: Cannot set "location.hostname".
NotSupportedError: Cannot set "location"
NotSupportedError: Cannot set "location.hostname"

5
tests/specs/task/dependencies/__test__.jsonc
View file

@ -56,6 +56,11 @@
"args": "task a",
"output": "./cycle_2.out",
"exitCode": 1
},
"arg_task_with_deps": {
"cwd": "arg_task_with_deps",
"args": "task a a",
"output": "./arg_task_with_deps.out"
}
}
}

4
tests/specs/task/dependencies/arg_task_with_deps.out Normal file
View file

@ -0,0 +1,4 @@
Task b echo 'b'
b
Task a echo "a"
a

9
tests/specs/task/dependencies/arg_task_with_deps/deno.json Normal file
View file

@ -0,0 +1,9 @@
{
"tasks": {
"a": {
"command": "echo",
"dependencies": ["b"]
},
"b": "echo 'b'"
}
}

43
tests/unit_node/_fs/_fs_readdir_test.ts
View file

@ -53,6 +53,29 @@ Deno.test({
},
});
Deno.test("ASYNC: read dirs recursively", async () => {
const dir = Deno.makeTempDirSync();
Deno.writeTextFileSync(join(dir, "file1.txt"), "hi");
Deno.mkdirSync(join(dir, "sub"));
Deno.writeTextFileSync(join(dir, "sub", "file2.txt"), "hi");
try {
const files = await new Promise<string[]>((resolve, reject) => {
readdir(dir, { recursive: true }, (err, files) => {
if (err) reject(err);
resolve(files.map((f) => f.toString()));
});
});
assertEqualsArrayAnyOrder(
files,
["file1.txt", "sub", join("sub", "file2.txt")],
);
} finally {
Deno.removeSync(dir, { recursive: true });
}
});
Deno.test({
name: "SYNC: reading empty the directory",
fn() {
@ -75,6 +98,26 @@ Deno.test({
},
});
Deno.test("SYNC: read dirs recursively", () => {
const dir = Deno.makeTempDirSync();
Deno.writeTextFileSync(join(dir, "file1.txt"), "hi");
Deno.mkdirSync(join(dir, "sub"));
Deno.writeTextFileSync(join(dir, "sub", "file2.txt"), "hi");
try {
const files = readdirSync(dir, { recursive: true }).map((f) =>
f.toString()
);
assertEqualsArrayAnyOrder(
files,
["file1.txt", "sub", join("sub", "file2.txt")],
);
} finally {
Deno.removeSync(dir, { recursive: true });
}
});
Deno.test("[std/node/fs] readdir callback isn't called twice if error is thrown", async () => {
// The correct behaviour is not to catch any errors thrown,
// but that means there'll be an uncaught error and the test will fail.