foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-20 20:42:19 -05:00
Code Issues Packages 1 Wiki Activity

Merge branch 'main' into webgpu-wrap

Browse source 
# Conflicts:
#	Cargo.lock
This commit is contained in:
crowlkats 2025-01-17 18:50:19 +01:00
commit 679a46d6f0
No known key found for this signature in database
GPG key ID: A82C9D461FC483E8
244 changed files with 20882 additions and 7468 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.github/workflows/ci.generate.ts vendored
View file

@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify";
// Bump this number when you want to purge the cache.
// Note: the tools/release/01_bump_crate_versions.ts script will update this version
// automatically via regex, so ensure that this line maintains this format.
const cacheVersion = 33;
const cacheVersion = 36;
const ubuntuX86Runner = "ubuntu-24.04";
const ubuntuX86XlRunner = "ubuntu-24.04-xl";
@ -14,7 +14,7 @@ const windowsX86Runner = "windows-2022";
const windowsX86XlRunner = "windows-2022-xl";
const macosX86Runner = "macos-13";
const macosArmRunner = "macos-14";
const selfHostedMacosArmRunner = "self-hosted";
const selfHostedMacosArmRunner = "ghcr.io/cirruslabs/macos-runner:sonoma";
const Runners = {
linuxX86: {
@ -41,8 +41,14 @@ const Runners = {
macosArm: {
os: "macos",
arch: "aarch64",
runner: macosArmRunner,
},
macosArmSelfHosted: {
os: "macos",
arch: "aarch64",
// Actually use self-hosted runner only in denoland/deno on `main` branch and for tags (release) builds.
runner:
`\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`,
`\${{ github.repository == 'denoland/deno' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`,
},
windowsX86: {
os: "windows",
@ -384,7 +390,7 @@ const ci = {
job: "test",
profile: "debug",
}, {
...Runners.macosArm,
...Runners.macosArmSelfHosted,
job: "test",
profile: "release",
skip_pr: true,
@ -486,7 +492,7 @@ const ci = {
},
{
name: "Cache Cargo home",
uses: "actions/cache@v4",
uses: "cirruslabs/cache@v4",
with: {
// See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
// Note that with the new sparse registry format, we no longer have to cache a `.git` dir

14
.github/workflows/ci.yml vendored
View file

@ -68,12 +68,12 @@ jobs:
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'
- os: macos
arch: aarch64
runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}'
runner: macos-14
job: test
profile: debug
- os: macos
arch: aarch64
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}'
runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && (github.ref == ''refs/heads/main'' || startsWith(github.ref, ''refs/tags/'')) && ''ghcr.io/cirruslabs/macos-runner:sonoma'' || ''macos-14'' }}'
job: test
profile: release
skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'
@ -175,7 +175,7 @@ jobs:
tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
-czvf target/release/deno_src.tar.gz -C .. deno
- name: Cache Cargo home
uses: actions/cache@v4
uses: cirruslabs/cache@v4
with:
path: |-
~/.cargo/.crates.toml
@ -184,8 +184,8 @@ jobs:
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: '33-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '33-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
key: '36-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '36-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
if: '!(matrix.skip)'
- uses: dsherret/rust-toolchain-file@v1
if: '!(matrix.skip)'
@ -379,7 +379,7 @@ jobs:
!./target/*/*.zip
!./target/*/*.tar.gz
key: never_saved
restore-keys: '33-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
restore-keys: '36-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
- name: Apply and update mtime cache
if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))'
uses: ./.github/mtime_cache
@ -689,7 +689,7 @@ jobs:
!./target/*/gn_root
!./target/*/*.zip
!./target/*/*.tar.gz
key: '33-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
key: '36-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
publish-canary:
name: publish canary
runs-on: ubuntu-24.04

389
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

74
Cargo.toml
View file

@ -5,6 +5,7 @@ resolver = "2"
members = [
"bench_util",
"cli",
"cli/lib",
"ext/broadcast_channel",
"ext/cache",
"ext/canvas",
@ -48,19 +49,19 @@ repository = "https://github.com/denoland/deno"
[workspace.dependencies]
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_core = { version = "0.330.0" }
deno_core = { version = "0.331.0" }
deno_bench_util = { version = "0.179.0", path = "./bench_util" }
deno_config = { version = "=0.43.0", features = ["workspace", "sync"] }
deno_bench_util = { version = "0.180.0", path = "./bench_util" }
deno_config = { version = "=0.45.0", features = ["workspace", "sync"] }
deno_lockfile = "=0.24.0"
deno_media_type = { version = "0.2.3", features = ["module_specifier"] }
deno_npm = "=0.27.0"
deno_npm = "=0.27.2"
deno_path_util = "=0.3.0"
deno_permissions = { version = "0.44.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.193.0", path = "./runtime" }
deno_permissions = { version = "0.45.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.194.0", path = "./runtime" }
deno_semver = "=0.7.1"
deno_terminal = "0.2.0"
napi_sym = { version = "0.115.0", path = "./ext/napi/sym" }
napi_sym = { version = "0.116.0", path = "./ext/napi/sym" }
test_util = { package = "test_server", path = "./tests/util/server" }
denokv_proto = "0.9.0"
@ -69,34 +70,37 @@ denokv_remote = "0.9.0"
denokv_sqlite = { default-features = false, version = "0.9.0" }
# exts
deno_broadcast_channel = { version = "0.179.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.117.0", path = "./ext/cache" }
deno_canvas = { version = "0.54.0", path = "./ext/canvas" }
deno_console = { version = "0.185.0", path = "./ext/console" }
deno_cron = { version = "0.65.0", path = "./ext/cron" }
deno_crypto = { version = "0.199.0", path = "./ext/crypto" }
deno_fetch = { version = "0.209.0", path = "./ext/fetch" }
deno_ffi = { version = "0.172.0", path = "./ext/ffi" }
deno_fs = { version = "0.95.0", path = "./ext/fs" }
deno_http = { version = "0.183.0", path = "./ext/http" }
deno_io = { version = "0.95.0", path = "./ext/io" }
deno_kv = { version = "0.93.0", path = "./ext/kv" }
deno_napi = { version = "0.116.0", path = "./ext/napi" }
deno_net = { version = "0.177.0", path = "./ext/net" }
deno_node = { version = "0.123.0", path = "./ext/node" }
deno_telemetry = { version = "0.7.0", path = "./ext/telemetry" }
deno_tls = { version = "0.172.0", path = "./ext/tls" }
deno_url = { version = "0.185.0", path = "./ext/url" }
deno_web = { version = "0.216.0", path = "./ext/web" }
deno_webgpu = { version = "0.152.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.185.0", path = "./ext/webidl" }
deno_websocket = { version = "0.190.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.180.0", path = "./ext/webstorage" }
deno_broadcast_channel = { version = "0.180.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.118.0", path = "./ext/cache" }
deno_canvas = { version = "0.55.0", path = "./ext/canvas" }
deno_console = { version = "0.186.0", path = "./ext/console" }
deno_cron = { version = "0.66.0", path = "./ext/cron" }
deno_crypto = { version = "0.200.0", path = "./ext/crypto" }
deno_fetch = { version = "0.210.0", path = "./ext/fetch" }
deno_ffi = { version = "0.173.0", path = "./ext/ffi" }
deno_fs = { version = "0.96.0", path = "./ext/fs" }
deno_http = { version = "0.184.0", path = "./ext/http" }
deno_io = { version = "0.96.0", path = "./ext/io" }
deno_kv = { version = "0.94.0", path = "./ext/kv" }
deno_napi = { version = "0.117.0", path = "./ext/napi" }
deno_net = { version = "0.178.0", path = "./ext/net" }
deno_node = { version = "0.124.0", path = "./ext/node" }
deno_os = { version = "0.3.0", path = "./ext/os" }
deno_process = { version = "0.1.0", path = "./ext/process" }
deno_telemetry = { version = "0.8.0", path = "./ext/telemetry" }
deno_tls = { version = "0.173.0", path = "./ext/tls" }
deno_url = { version = "0.186.0", path = "./ext/url" }
deno_web = { version = "0.217.0", path = "./ext/web" }
deno_webgpu = { version = "0.153.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.186.0", path = "./ext/webidl" }
deno_websocket = { version = "0.191.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.181.0", path = "./ext/webstorage" }
# resolvers
deno_npm_cache = { version = "0.4.0", path = "./resolvers/npm_cache" }
deno_resolver = { version = "0.16.0", path = "./resolvers/deno" }
node_resolver = { version = "0.23.0", path = "./resolvers/node" }
# workspace libraries
deno_lib = { version = "0.2.0", path = "./cli/lib" }
deno_npm_cache = { version = "0.5.0", path = "./resolvers/npm_cache" }
deno_resolver = { version = "0.17.0", path = "./resolvers/deno" }
node_resolver = { version = "0.24.0", path = "./resolvers/node" }
aes = "=0.8.3"
anyhow = "1.0.57"
@ -119,7 +123,7 @@ dashmap = "5.5.3"
data-encoding = "2.3.3"
data-url = "=0.3.1"
deno_cache_dir = "=0.16.0"
deno_error = "=0.5.3"
deno_error = "=0.5.5"
deno_package_json = { version = "0.4.0", default-features = false }
deno_unsync = "0.4.2"
dlopen2 = "0.6.1"

26
Releases.md
View file

@ -6,6 +6,32 @@ https://github.com/denoland/deno/releases
We also have one-line install commands at:
https://github.com/denoland/deno_install
### 2.1.6 / 2025.01.16
- fix(check/lsp): correctly resolve compilerOptions.types (#27686)
- fix(check/lsp): fix bugs with tsc type resolution, allow npm packages to
augment `ImportMeta` (#27690)
- fix(compile): store embedded fs case sensitivity (#27653)
- fix(compile/windows): better handling of deno_dir on different drive letter
than code (#27654)
- fix(ext/console): change Temporal color (#27684)
- fix(ext/node): add `writev` method to `FileHandle` (#27563)
- fix(ext/node): add chown method to FileHandle class (#27638)
- fix(ext/node): apply `@npmcli/agent` workaround to `npm-check-updates`
(#27639)
- fix(ext/node): fix playwright http client (#27662)
- fix(ext/node): show bare-node-builtin hint when using an import map (#27632)
- fix(ext/node): use primordials in `ext/node/polyfills/_fs_common.ts` (#27589)
- fix(lsp): handle pathless untitled URIs (#27637)
- fix(lsp/check): don't resolve unknown media types to a `.js` extension
(#27631)
- fix(node): Prevent node:child_process from always inheriting the parent
environment (#27343) (#27340)
- fix(node/fs): add utimes method to the FileHandle class (#27582)
- fix(outdated): Use `latest` tag even when it's the same as the current version
(#27699)
- fix(outdated): retain strict semver specifier when updating (#27701)
### 2.1.5 / 2025.01.09
- feat(unstable): implement QUIC (#21942)

2
bench_util/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_bench_util"
version = "0.179.0"
version = "0.180.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

3
cli/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno"
version = "2.1.5"
version = "2.1.6"
authors.workspace = true
default-run = "deno"
edition.workspace = true
@ -76,6 +76,7 @@ deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"]
deno_doc = { version = "=0.164.0", features = ["rust", "comrak"] }
deno_error.workspace = true
deno_graph = { version = "=0.87.0" }
deno_lib.workspace = true
deno_lint = { version = "=0.68.2", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true

24
cli/args/lockfile.rs
View file

@ -61,11 +61,13 @@ impl<'a, T> std::ops::DerefMut for Guard<'a, T> {
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
#[error("Failed writing lockfile")]
#[class(inherit)]
struct AtomicWriteFileWithRetriesError {
#[source]
source: std::io::Error,
pub enum AtomicWriteFileWithRetriesError {
#[class(inherit)]
#[error(transparent)]
Changed(JsErrorBox),
#[class(inherit)]
#[error("Failed writing lockfile")]
Io(#[source] std::io::Error),
}
impl CliLockfile {
@ -87,12 +89,16 @@ impl CliLockfile {
self.lockfile.lock().overwrite
}
pub fn write_if_changed(&self) -> Result<(), JsErrorBox> {
pub fn write_if_changed(
&self,
) -> Result<(), AtomicWriteFileWithRetriesError> {
if self.skip_write {
return Ok(());
}
self.error_if_changed()?;
self
.error_if_changed()
.map_err(AtomicWriteFileWithRetriesError::Changed)?;
let mut lockfile = self.lockfile.lock();
let Some(bytes) = lockfile.resolve_write_bytes() else {
return Ok(()); // nothing to do
@ -105,9 +111,7 @@ impl CliLockfile {
&bytes,
cache::CACHE_PERM,
)
.map_err(|source| {
JsErrorBox::from_err(AtomicWriteFileWithRetriesError { source })
})?;
.map_err(AtomicWriteFileWithRetriesError::Io)?;
lockfile.has_content_changed = false;
Ok(())
}

114
cli/args/mod.rs
View file

@ -56,9 +56,11 @@ use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsErrorBox;
use deno_graph::GraphKind;
pub use deno_json::check_warn_tsconfig;
use deno_lib::cache::DenoDirProvider;
use deno_lib::env::has_flag_env_var;
use deno_lib::worker::StorageKeyResolver;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_npm::npm_rc::NpmRc;
use deno_npm::npm_rc::ResolvedNpmRc;
@ -80,6 +82,7 @@ use deno_terminal::colors;
use dotenvy::from_filename;
pub use flags::*;
use import_map::resolve_import_map_value_from_specifier;
pub use lockfile::AtomicWriteFileWithRetriesError;
pub use lockfile::CliLockfile;
pub use lockfile::CliLockfileReadFromPathOptions;
use once_cell::sync::Lazy;
@ -90,7 +93,6 @@ use serde::Serialize;
use sys_traits::EnvHomeDir;
use thiserror::Error;
use crate::cache::DenoDirProvider;
use crate::file_fetcher::CliFileFetcher;
use crate::sys::CliSys;
use crate::util::fs::canonicalize_path_maybe_not_exists;
@ -718,7 +720,7 @@ pub enum NpmProcessStateKind {
}
static NPM_PROCESS_STATE: Lazy<Option<NpmProcessState>> = Lazy::new(|| {
use deno_runtime::ops::process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME;
use deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME;
let fd = std::env::var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME).ok()?;
std::env::remove_var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME);
let fd = fd.parse::<usize>().ok()?;
@ -769,7 +771,7 @@ pub struct CliOptions {
maybe_external_import_map: Option<(PathBuf, serde_json::Value)>,
overrides: CliOptionOverrides,
pub start_dir: Arc<WorkspaceDirectory>,
pub deno_dir_provider: Arc<DenoDirProvider>,
pub deno_dir_provider: Arc<DenoDirProvider<CliSys>>,
}
impl CliOptions {
@ -842,8 +844,6 @@ impl CliOptions {
} else {
&[]
};
let config_parse_options =
deno_config::deno_json::ConfigParseOptions::default();
let discover_pkg_json = flags.config_flag != ConfigFlag::Disabled
&& !flags.no_npm
&& !has_flag_env_var("DENO_NO_PACKAGE_JSON");
@ -854,7 +854,6 @@ impl CliOptions {
deno_json_cache: None,
pkg_json_cache: Some(&node_resolver::PackageJsonThreadLocalCache),
workspace_cache: None,
config_parse_options,
additional_config_file_names,
discover_pkg_json,
maybe_vendor_override,
@ -1103,11 +1102,11 @@ impl CliOptions {
}
};
Ok(self.workspace().create_resolver(
&CliSys::default(),
CreateResolverOptions {
pkg_json_dep_resolution,
specified_import_map: cli_arg_specified_import_map,
},
|path| std::fs::read_to_string(path).map_err(JsErrorBox::from_err),
)?)
}
@ -1231,6 +1230,16 @@ impl CliOptions {
}
}
pub fn resolve_storage_key_resolver(&self) -> StorageKeyResolver {
if let Some(location) = &self.flags.location {
StorageKeyResolver::from_flag(location)
} else if let Some(deno_json) = self.start_dir.maybe_deno_json() {
StorageKeyResolver::from_config_file_url(&deno_json.specifier)
} else {
StorageKeyResolver::new_use_main_module()
}
}
// If the main module should be treated as being in an npm package.
// This is triggered via a secret environment variable which is used
// for functionality like child_process.fork. Users should NOT depend
@ -1875,7 +1884,7 @@ fn resolve_node_modules_folder(
cwd: &Path,
flags: &Flags,
workspace: &Workspace,
deno_dir_provider: &Arc<DenoDirProvider>,
deno_dir_provider: &Arc<DenoDirProvider<CliSys>>,
) -> Result<Option<PathBuf>, AnyError> {
fn resolve_from_root(root_folder: &FolderConfigs, cwd: &Path) -> PathBuf {
root_folder
@ -1979,63 +1988,11 @@ fn resolve_import_map_specifier(
}
}
pub struct StorageKeyResolver(Option<Option<String>>);
impl StorageKeyResolver {
pub fn from_options(options: &CliOptions) -> Self {
Self(if let Some(location) = &options.flags.location {
// if a location is set, then the ascii serialization of the location is
// used, unless the origin is opaque, and then no storage origin is set, as
// we can't expect the origin to be reproducible
let storage_origin = location.origin();
if storage_origin.is_tuple() {
Some(Some(storage_origin.ascii_serialization()))
} else {
Some(None)
}
} else {
// otherwise we will use the path to the config file or None to
// fall back to using the main module's path
options
.start_dir
.maybe_deno_json()
.map(|config_file| Some(config_file.specifier.to_string()))
})
}
/// Creates a storage key resolver that will always resolve to being empty.
pub fn empty() -> Self {
Self(Some(None))
}
/// Resolves the storage key to use based on the current flags, config, or main module.
pub fn resolve_storage_key(
&self,
main_module: &ModuleSpecifier,
) -> Option<String> {
// use the stored value or fall back to using the path of the main module.
if let Some(maybe_value) = &self.0 {
maybe_value.clone()
} else {
Some(main_module.to_string())
}
}
}
/// Resolves the no_prompt value based on the cli flags and environment.
pub fn resolve_no_prompt(flags: &PermissionFlags) -> bool {
flags.no_prompt || has_flag_env_var("DENO_NO_PROMPT")
}
pub fn has_trace_permissions_enabled() -> bool {
has_flag_env_var("DENO_TRACE_PERMISSIONS")
}
pub fn has_flag_env_var(name: &str) -> bool {
let value = env::var(name);
matches!(value.as_ref().map(|s| s.as_str()), Ok("1"))
}
pub fn npm_pkg_req_ref_to_binary_command(
req_ref: &NpmPackageReqReference,
) -> String {
@ -2126,12 +2083,7 @@ mod test {
let cwd = &std::env::current_dir().unwrap();
let config_specifier =
ModuleSpecifier::parse("file:///deno/deno.jsonc").unwrap();
let config_file = ConfigFile::new(
config_text,
config_specifier,
&deno_config::deno_json::ConfigParseOptions::default(),
)
.unwrap();
let config_file = ConfigFile::new(config_text, config_specifier).unwrap();
let actual = resolve_import_map_specifier(
Some("import-map.json"),
Some(&config_file),
@ -2150,12 +2102,7 @@ mod test {
let config_text = r#"{}"#;
let config_specifier =
ModuleSpecifier::parse("file:///deno/deno.jsonc").unwrap();
let config_file = ConfigFile::new(
config_text,
config_specifier,
&deno_config::deno_json::ConfigParseOptions::default(),
)
.unwrap();
let config_file = ConfigFile::new(config_text, config_specifier).unwrap();
let actual = resolve_import_map_specifier(
None,
Some(&config_file),
@ -2174,27 +2121,6 @@ mod test {
assert_eq!(actual, None);
}
#[test]
fn storage_key_resolver_test() {
let resolver = StorageKeyResolver(None);
let specifier = ModuleSpecifier::parse("file:///a.ts").unwrap();
assert_eq!(
resolver.resolve_storage_key(&specifier),
Some(specifier.to_string())
);
let resolver = StorageKeyResolver(Some(None));
assert_eq!(resolver.resolve_storage_key(&specifier), None);
let resolver = StorageKeyResolver(Some(Some("value".to_string())));
assert_eq!(
resolver.resolve_storage_key(&specifier),
Some("value".to_string())
);
// test empty
let resolver = StorageKeyResolver::empty();
assert_eq!(resolver.resolve_storage_key(&specifier), None);
}
#[test]
fn jsr_urls() {
let reg_url = jsr_url();

7
cli/cache/caches.rs vendored
View file

@ -3,20 +3,21 @@
use std::path::PathBuf;
use std::sync::Arc;
use deno_lib::cache::DenoDirProvider;
use once_cell::sync::OnceCell;
use super::cache_db::CacheDB;
use super::cache_db::CacheDBConfiguration;
use super::check::TYPE_CHECK_CACHE_DB;
use super::code_cache::CODE_CACHE_DB;
use super::deno_dir::DenoDirProvider;
use super::fast_check::FAST_CHECK_CACHE_DB;
use super::incremental::INCREMENTAL_CACHE_DB;
use super::module_info::MODULE_INFO_CACHE_DB;
use super::node::NODE_ANALYSIS_CACHE_DB;
use crate::sys::CliSys;
pub struct Caches {
dir_provider: Arc<DenoDirProvider>,
dir_provider: Arc<DenoDirProvider<CliSys>>,
fmt_incremental_cache_db: OnceCell<CacheDB>,
lint_incremental_cache_db: OnceCell<CacheDB>,
dep_analysis_db: OnceCell<CacheDB>,
@ -27,7 +28,7 @@ pub struct Caches {
}
impl Caches {
pub fn new(dir: Arc<DenoDirProvider>) -> Self {
pub fn new(dir: Arc<DenoDirProvider<CliSys>>) -> Self {
Self {
dir_provider: dir,
fmt_incremental_cache_db: Default::default(),

7
cli/cache/emit.rs vendored
View file

@ -6,19 +6,20 @@ use deno_ast::ModuleSpecifier;
use deno_core::anyhow::anyhow;
use deno_core::error::AnyError;
use deno_core::unsync::sync::AtomicFlag;
use deno_lib::cache::DiskCache;
use super::DiskCache;
use crate::sys::CliSys;
/// The cache that stores previously emitted files.
#[derive(Debug)]
pub struct EmitCache {
disk_cache: DiskCache,
disk_cache: DiskCache<CliSys>,
emit_failed_flag: AtomicFlag,
file_serializer: EmitFileSerializer,
}
impl EmitCache {
pub fn new(disk_cache: DiskCache) -> Self {
pub fn new(disk_cache: DiskCache<CliSys>) -> Self {
Self {
disk_cache,
emit_failed_flag: Default::default(),

10
cli/cache/mod.rs vendored
View file

@ -15,6 +15,7 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_runtime::deno_permissions::PermissionsContainer;
use node_resolver::InNpmPackageChecker;
@ -30,8 +31,6 @@ mod caches;
mod check;
mod code_cache;
mod common;
mod deno_dir;
mod disk_cache;
mod emit;
mod fast_check;
mod incremental;
@ -46,9 +45,6 @@ pub use code_cache::CodeCache;
pub use common::FastInsecureHasher;
/// Permissions used to save a file in the disk caches.
pub use deno_cache_dir::CACHE_PERM;
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
pub use emit::EmitCache;
pub use fast_check::FastCheckCache;
pub use incremental::IncrementalCache;
@ -76,7 +72,7 @@ pub struct FetchCacher {
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
sys: CliSys,
@ -88,7 +84,7 @@ impl FetchCacher {
pub fn new(
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
module_info_cache: Arc<ModuleInfoCache>,
sys: CliSys,
options: FetchCacherOptions,

10
cli/emit.rs
View file

@ -24,11 +24,11 @@ use deno_graph::ModuleGraph;
use crate::cache::EmitCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use crate::resolver::CliCjsTracker;
#[derive(Debug)]
pub struct Emitter {
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
emit_cache: Arc<EmitCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
transpile_and_emit_options:
@ -39,7 +39,7 @@ pub struct Emitter {
impl Emitter {
pub fn new(
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
emit_cache: Arc<EmitCache>,
parsed_source_cache: Arc<ParsedSourceCache>,
transpile_options: deno_ast::TranspileOptions,
@ -112,9 +112,9 @@ impl Emitter {
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
module_kind: ModuleKind,
source: &Arc<str>,
) -> Result<String, AnyError> {
) -> Result<String, EmitParsedSourceHelperError> {
// Note: keep this in sync with the sync version below
let helper = EmitParsedSourceHelper(self);
match helper.pre_emit_parsed_source(specifier, module_kind, source) {

428
cli/factory.rs
View file

@ -11,9 +11,18 @@ use deno_core::error::AnyError;
use deno_core::futures::FutureExt;
use deno_core::FeatureChecker;
use deno_error::JsErrorBox;
use deno_lib::cache::DenoDir;
use deno_lib::cache::DenoDirProvider;
use deno_lib::npm::NpmRegistryReadPermissionChecker;
use deno_lib::npm::NpmRegistryReadPermissionCheckerMode;
use deno_lib::worker::LibMainWorkerFactory;
use deno_lib::worker::LibMainWorkerOptions;
use deno_npm_cache::NpmCacheSetting;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_resolver::npm::CreateInNpmPkgCheckerOptions;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::sloppy_imports::SloppyImportsCachedFs;
use deno_resolver::DenoResolverOptions;
@ -30,7 +39,6 @@ use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::permissions::RuntimePermissionDescriptorParser;
use log::warn;
use node_resolver::analyze::NodeCodeTranslator;
use node_resolver::InNpmPackageChecker;
use once_cell::sync::OnceCell;
use crate::args::check_warn_tsconfig;
@ -40,12 +48,9 @@ use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
use crate::cache::CodeCache;
use crate::cache::DenoDir;
use crate::cache::DenoDirProvider;
use crate::cache::EmitCache;
use crate::cache::GlobalHttpCache;
use crate::cache::HttpCache;
@ -66,20 +71,26 @@ use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_npm_process_state_provider;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::NpmResolutionInstaller;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmCacheHttpClient;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::NpmRegistryReadPermissionChecker;
use crate::npm::NpmRegistryReadPermissionCheckerMode;
use crate::resolver::CjsTracker;
use crate::npm::CliNpmTarballCache;
use crate::npm::NpmResolutionInitializer;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliNpmGraphResolver;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
use crate::resolver::CliSloppyImportsResolver;
use crate::resolver::FoundPackageJsonDepFlag;
use crate::resolver::NpmModuleLoader;
use crate::standalone::binary::DenoCompileBinaryWriter;
use crate::sys::CliSys;
@ -180,7 +191,7 @@ impl<T> Deferred<T> {
struct CliFactoryServices {
blob_store: Deferred<Arc<BlobStore>>,
caches: Deferred<Arc<Caches>>,
cjs_tracker: Deferred<Arc<CjsTracker>>,
cjs_tracker: Deferred<Arc<CliCjsTracker>>,
cli_options: Deferred<Arc<CliOptions>>,
code_cache: Deferred<Arc<CodeCache>>,
deno_resolver: Deferred<Arc<CliDenoResolver>>,
@ -188,11 +199,12 @@ struct CliFactoryServices {
emitter: Deferred<Arc<Emitter>>,
feature_checker: Deferred<Arc<FeatureChecker>>,
file_fetcher: Deferred<Arc<CliFileFetcher>>,
found_pkg_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
fs: Deferred<Arc<dyn deno_fs::FileSystem>>,
global_http_cache: Deferred<Arc<GlobalHttpCache>>,
http_cache: Deferred<Arc<dyn HttpCache>>,
http_client_provider: Deferred<Arc<HttpClientProvider>>,
in_npm_pkg_checker: Deferred<Arc<dyn InNpmPackageChecker>>,
in_npm_pkg_checker: Deferred<DenoInNpmPackageChecker>,
main_graph_container: Deferred<Arc<MainModuleGraphContainer>>,
maybe_file_watcher_reporter: Deferred<Option<FileWatcherReporter>>,
maybe_inspector_server: Deferred<Option<Arc<InspectorServer>>>,
@ -202,9 +214,18 @@ struct CliFactoryServices {
module_load_preparer: Deferred<Arc<ModuleLoadPreparer>>,
node_code_translator: Deferred<Arc<CliNodeCodeTranslator>>,
node_resolver: Deferred<Arc<CliNodeResolver>>,
npm_cache: Deferred<Arc<CliNpmCache>>,
npm_cache_dir: Deferred<Arc<NpmCacheDir>>,
npm_cache_http_client: Deferred<Arc<CliNpmCacheHttpClient>>,
npm_graph_resolver: Deferred<Arc<CliNpmGraphResolver>>,
npm_installer: Deferred<Arc<NpmInstaller>>,
npm_registry_info_provider: Deferred<Arc<CliNpmRegistryInfoProvider>>,
npm_req_resolver: Deferred<Arc<CliNpmReqResolver>>,
npm_resolver: Deferred<Arc<dyn CliNpmResolver>>,
npm_resolution: Arc<NpmResolutionCell>,
npm_resolution_initializer: Deferred<Arc<NpmResolutionInitializer>>,
npm_resolution_installer: Deferred<Arc<NpmResolutionInstaller>>,
npm_resolver: Deferred<CliNpmResolver>,
npm_tarball_cache: Deferred<Arc<CliNpmTarballCache>>,
parsed_source_cache: Deferred<Arc<ParsedSourceCache>>,
permission_desc_parser:
Deferred<Arc<RuntimePermissionDescriptorParser<CliSys>>>,
@ -262,11 +283,13 @@ impl CliFactory {
})
}
pub fn deno_dir_provider(&self) -> Result<&Arc<DenoDirProvider>, AnyError> {
pub fn deno_dir_provider(
&self,
) -> Result<&Arc<DenoDirProvider<CliSys>>, AnyError> {
Ok(&self.cli_options()?.deno_dir_provider)
}
pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> {
pub fn deno_dir(&self) -> Result<&DenoDir<CliSys>, AnyError> {
Ok(self.deno_dir_provider()?.get_or_create()?)
}
@ -379,7 +402,7 @@ impl CliFactory {
pub fn in_npm_pkg_checker(
&self,
) -> Result<&Arc<dyn InNpmPackageChecker>, AnyError> {
) -> Result<&DenoInNpmPackageChecker, AnyError> {
self.services.in_npm_pkg_checker.get_or_try_init(|| {
let cli_options = self.cli_options()?;
let options = if cli_options.use_byonm() {
@ -394,7 +417,19 @@ impl CliFactory {
},
)
};
Ok(deno_resolver::npm::create_in_npm_pkg_checker(options))
Ok(DenoInNpmPackageChecker::new(options))
})
}
pub fn npm_cache(&self) -> Result<&Arc<CliNpmCache>, AnyError> {
self.services.npm_cache.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(CliNpmCache::new(
self.npm_cache_dir()?.clone(),
self.sys(),
NpmCacheSetting::from_cache_setting(&cli_options.cache_setting()),
cli_options.npmrc().clone(),
)))
})
}
@ -410,16 +445,131 @@ impl CliFactory {
})
}
pub async fn npm_resolver(
pub fn npm_cache_http_client(&self) -> &Arc<CliNpmCacheHttpClient> {
self.services.npm_cache_http_client.get_or_init(|| {
Arc::new(CliNpmCacheHttpClient::new(
self.http_client_provider().clone(),
self.text_only_progress_bar().clone(),
))
})
}
pub fn npm_graph_resolver(
&self,
) -> Result<&Arc<dyn CliNpmResolver>, AnyError> {
) -> Result<&Arc<CliNpmGraphResolver>, AnyError> {
self.services.npm_graph_resolver.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(CliNpmGraphResolver::new(
self.npm_installer_if_managed()?.cloned(),
self.services.found_pkg_json_dep_flag.clone(),
cli_options.unstable_bare_node_builtins(),
cli_options.default_npm_caching_strategy(),
)))
})
}
pub fn npm_installer_if_managed(
&self,
) -> Result<Option<&Arc<NpmInstaller>>, AnyError> {
let options = self.cli_options()?;
if options.use_byonm() || options.no_npm() {
Ok(None)
} else {
Ok(Some(self.npm_installer()?))
}
}
pub fn npm_installer(&self) -> Result<&Arc<NpmInstaller>, AnyError> {
self.services.npm_installer.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmInstaller::new(
self.npm_cache()?.clone(),
Arc::new(NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
)),
self.npm_resolution().clone(),
self.npm_resolution_initializer()?.clone(),
self.npm_resolution_installer()?.clone(),
self.text_only_progress_bar(),
self.sys(),
self.npm_tarball_cache()?.clone(),
cli_options.maybe_lockfile().cloned(),
cli_options.node_modules_dir_path().cloned(),
cli_options.lifecycle_scripts_config(),
cli_options.npm_system_info(),
)))
})
}
pub fn npm_registry_info_provider(
&self,
) -> Result<&Arc<CliNpmRegistryInfoProvider>, AnyError> {
self
.services
.npm_registry_info_provider
.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(CliNpmRegistryInfoProvider::new(
self.npm_cache()?.clone(),
self.npm_cache_http_client().clone(),
cli_options.npmrc().clone(),
)))
})
}
pub fn npm_resolution(&self) -> &Arc<NpmResolutionCell> {
&self.services.npm_resolution
}
pub fn npm_resolution_initializer(
&self,
) -> Result<&Arc<NpmResolutionInitializer>, AnyError> {
self
.services
.npm_resolution_initializer
.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmResolutionInitializer::new(
self.npm_registry_info_provider()?.clone(),
self.npm_resolution().clone(),
match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot))
}
None => match cli_options.maybe_lockfile() {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
lockfile.clone(),
)
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
},
},
)))
})
}
pub fn npm_resolution_installer(
&self,
) -> Result<&Arc<NpmResolutionInstaller>, AnyError> {
self.services.npm_resolution_installer.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmResolutionInstaller::new(
self.npm_registry_info_provider()?.clone(),
self.npm_resolution().clone(),
cli_options.maybe_lockfile().cloned(),
)))
})
}
pub async fn npm_resolver(&self) -> Result<&CliNpmResolver, AnyError> {
self
.services
.npm_resolver
.get_or_try_init_async(
async {
let cli_options = self.cli_options()?;
create_cli_npm_resolver(if cli_options.use_byonm() {
Ok(CliNpmResolver::new(if cli_options.use_byonm() {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
sys: self.sys(),
@ -438,52 +588,43 @@ impl CliFactory {
},
)
} else {
self
.npm_resolution_initializer()?
.ensure_initialized()
.await?;
CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
http_client_provider: self.http_client_provider().clone(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
sys: self.sys(),
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(
snapshot,
))
}
None => match cli_options.maybe_lockfile() {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
lockfile.clone(),
)
}
None => {
CliNpmResolverManagedSnapshotOption::Specified(None)
}
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
npm_resolution: self.npm_resolution().clone(),
npm_cache_dir: self.npm_cache_dir()?.clone(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.cloned(),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
},
)
})
.await
}))
}
.boxed_local(),
)
.await
}
pub fn npm_tarball_cache(
&self,
) -> Result<&Arc<CliNpmTarballCache>, AnyError> {
self.services.npm_tarball_cache.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(CliNpmTarballCache::new(
self.npm_cache()?.clone(),
self.npm_cache_http_client().clone(),
self.sys(),
cli_options.npmrc().clone(),
)))
})
}
pub fn sloppy_imports_resolver(
&self,
) -> Result<Option<&Arc<CliSloppyImportsResolver>>, AnyError> {
@ -571,17 +712,10 @@ impl CliFactory {
.resolver
.get_or_try_init_async(
async {
let cli_options = self.cli_options()?;
Ok(Arc::new(CliResolver::new(CliResolverOptions {
npm_resolver: if cli_options.no_npm() {
None
} else {
Some(self.npm_resolver().await?.clone())
},
bare_node_builtins_enabled: cli_options
.unstable_bare_node_builtins(),
deno_resolver: self.deno_resolver().await?.clone(),
})))
Ok(Arc::new(CliResolver::new(
self.deno_resolver().await?.clone(),
self.services.found_pkg_json_dep_flag.clone(),
)))
}
.boxed_local(),
)
@ -663,13 +797,10 @@ impl CliFactory {
Ok(Arc::new(CliNodeResolver::new(
self.in_npm_pkg_checker()?.clone(),
RealIsBuiltInNodeModuleChecker,
self
.npm_resolver()
.await?
.clone()
.into_npm_pkg_folder_resolver(),
self.npm_resolver().await?.clone(),
self.pkg_json_resolver().clone(),
self.sys(),
node_resolver::ConditionsFromResolutionMode::default(),
)))
}
.boxed_local(),
@ -699,11 +830,7 @@ impl CliFactory {
cjs_esm_analyzer,
self.in_npm_pkg_checker()?.clone(),
node_resolver,
self
.npm_resolver()
.await?
.clone()
.into_npm_pkg_folder_resolver(),
self.npm_resolver().await?.clone(),
self.pkg_json_resolver().clone(),
self.sys(),
)))
@ -720,11 +847,10 @@ impl CliFactory {
.get_or_try_init_async(async {
let npm_resolver = self.npm_resolver().await?;
Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
sys: self.sys(),
in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(),
node_resolver: self.node_resolver().await?.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
npm_resolver: npm_resolver.clone(),
})))
})
.await
@ -752,6 +878,7 @@ impl CliFactory {
cli_options.clone(),
self.module_graph_builder().await?.clone(),
self.node_resolver().await?.clone(),
self.npm_installer_if_managed()?.cloned(),
self.npm_resolver().await?.clone(),
self.sys(),
)))
@ -777,6 +904,8 @@ impl CliFactory {
cli_options.maybe_lockfile().cloned(),
self.maybe_file_watcher_reporter().clone(),
self.module_info_cache()?.clone(),
self.npm_graph_resolver()?.clone(),
self.npm_installer_if_managed()?.cloned(),
self.npm_resolver().await?.clone(),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
@ -797,7 +926,7 @@ impl CliFactory {
let cli_options = self.cli_options()?;
Ok(Arc::new(ModuleGraphCreator::new(
cli_options.clone(),
self.npm_resolver().await?.clone(),
self.npm_installer_if_managed()?.cloned(),
self.module_graph_builder().await?.clone(),
self.type_checker().await?.clone(),
)))
@ -852,10 +981,10 @@ impl CliFactory {
.await
}
pub fn cjs_tracker(&self) -> Result<&Arc<CjsTracker>, AnyError> {
pub fn cjs_tracker(&self) -> Result<&Arc<CliCjsTracker>, AnyError> {
self.services.cjs_tracker.get_or_try_init(|| {
let options = self.cli_options()?;
Ok(Arc::new(CjsTracker::new(
Ok(Arc::new(CliCjsTracker::new(
self.in_npm_pkg_checker()?.clone(),
self.pkg_json_resolver().clone(),
if options.is_node_main() || options.unstable_detect_cjs() {
@ -904,7 +1033,7 @@ impl CliFactory {
self.emitter()?,
self.file_fetcher()?,
self.http_client_provider(),
self.npm_resolver().await?.as_ref(),
self.npm_resolver().await?,
self.workspace_resolver().await?.as_ref(),
cli_options.npm_system_info(),
))
@ -958,7 +1087,34 @@ impl CliFactory {
Arc::new(NpmRegistryReadPermissionChecker::new(self.sys(), mode))
};
Ok(CliMainWorkerFactory::new(
let module_loader_factory = CliModuleLoaderFactory::new(
cli_options,
cjs_tracker,
if cli_options.code_cache_enabled() {
Some(self.code_cache()?.clone())
} else {
None
},
self.emitter()?.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
node_code_translator.clone(),
node_resolver.clone(),
NpmModuleLoader::new(
self.cjs_tracker()?.clone(),
fs.clone(),
node_code_translator.clone(),
),
npm_registry_permission_checker,
npm_req_resolver.clone(),
cli_npm_resolver.clone(),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.sys(),
);
let lib_main_worker_factory = LibMainWorkerFactory::new(
self.blob_store().clone(),
if cli_options.code_cache_enabled() {
Some(self.code_cache()?.clone())
@ -967,49 +1123,70 @@ impl CliFactory {
},
self.feature_checker()?.clone(),
fs.clone(),
maybe_file_watcher_communicator,
self.maybe_inspector_server()?.clone(),
cli_options.maybe_lockfile().cloned(),
Box::new(CliModuleLoaderFactory::new(
cli_options,
cjs_tracker,
if cli_options.code_cache_enabled() {
Some(self.code_cache()?.clone())
} else {
None
},
self.emitter()?.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
node_code_translator.clone(),
node_resolver.clone(),
NpmModuleLoader::new(
self.cjs_tracker()?.clone(),
fs.clone(),
node_code_translator.clone(),
),
npm_registry_permission_checker,
npm_req_resolver.clone(),
cli_npm_resolver.clone(),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.sys(),
)),
Box::new(module_loader_factory),
node_resolver.clone(),
npm_resolver.clone(),
create_npm_process_state_provider(npm_resolver),
pkg_json_resolver,
self.root_cert_store_provider().clone(),
self.root_permissions_container()?.clone(),
StorageKeyResolver::from_options(cli_options),
cli_options.resolve_storage_key_resolver(),
self.sys(),
self.create_lib_main_worker_options()?,
);
Ok(CliMainWorkerFactory::new(
lib_main_worker_factory,
maybe_file_watcher_communicator,
cli_options.maybe_lockfile().cloned(),
node_resolver.clone(),
self.npm_installer_if_managed()?.cloned(),
npm_resolver.clone(),
self.sys(),
cli_options.sub_command().clone(),
self.create_cli_main_worker_options()?,
self.cli_options()?.otel_config(),
self.cli_options()?.default_npm_caching_strategy(),
self.root_permissions_container()?.clone(),
))
}
fn create_lib_main_worker_options(
&self,
) -> Result<LibMainWorkerOptions, AnyError> {
let cli_options = self.cli_options()?;
Ok(LibMainWorkerOptions {
argv: cli_options.argv().clone(),
// This optimization is only available for "run" subcommand
// because we need to register new ops for testing and jupyter
// integration.
skip_op_registration: cli_options.sub_command().is_run(),
log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(),
enable_op_summary_metrics: cli_options.enable_op_summary_metrics(),
enable_testing_features: cli_options.enable_testing_features(),
has_node_modules_dir: cli_options.has_node_modules_dir(),
inspect_brk: cli_options.inspect_brk().is_some(),
inspect_wait: cli_options.inspect_wait().is_some(),
strace_ops: cli_options.strace_ops().clone(),
is_inspecting: cli_options.is_inspecting(),
location: cli_options.location_flag().clone(),
// if the user ran a binary command, we'll need to set process.argv[0]
// to be the name of the binary command instead of deno
argv0: cli_options
.take_binary_npm_command_name()
.or(std::env::args().next()),
node_debug: std::env::var("NODE_DEBUG").ok(),
origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()),
seed: cli_options.seed(),
unsafely_ignore_certificate_errors: cli_options
.unsafely_ignore_certificate_errors()
.clone(),
node_ipc: cli_options.node_ipc_fd(),
serve_port: cli_options.serve_port(),
serve_host: cli_options.serve_host(),
deno_version: crate::version::DENO_VERSION_INFO.deno,
deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent,
otel_config: self.cli_options()?.otel_config(),
startup_snapshot: crate::js::deno_isolate_init(),
})
}
fn create_cli_main_worker_options(
&self,
) -> Result<CliMainWorkerOptions, AnyError> {
@ -1041,37 +1218,10 @@ impl CliFactory {
};
Ok(CliMainWorkerOptions {
argv: cli_options.argv().clone(),
// This optimization is only available for "run" subcommand
// because we need to register new ops for testing and jupyter
// integration.
skip_op_registration: cli_options.sub_command().is_run(),
log_level: cli_options.log_level().unwrap_or(log::Level::Info).into(),
enable_op_summary_metrics: cli_options.enable_op_summary_metrics(),
enable_testing_features: cli_options.enable_testing_features(),
has_node_modules_dir: cli_options.has_node_modules_dir(),
hmr: cli_options.has_hmr(),
inspect_brk: cli_options.inspect_brk().is_some(),
inspect_wait: cli_options.inspect_wait().is_some(),
strace_ops: cli_options.strace_ops().clone(),
is_inspecting: cli_options.is_inspecting(),
location: cli_options.location_flag().clone(),
// if the user ran a binary command, we'll need to set process.argv[0]
// to be the name of the binary command instead of deno
argv0: cli_options
.take_binary_npm_command_name()
.or(std::env::args().next()),
node_debug: std::env::var("NODE_DEBUG").ok(),
origin_data_folder_path: Some(self.deno_dir()?.origin_data_folder_path()),
seed: cli_options.seed(),
unsafely_ignore_certificate_errors: cli_options
.unsafely_ignore_certificate_errors()
.clone(),
needs_test_modules: cli_options.sub_command().needs_test(),
create_hmr_runner,
create_coverage_collector,
node_ipc: cli_options.node_ipc_fd(),
serve_port: cli_options.serve_port(),
serve_host: cli_options.serve_host(),
default_npm_caching_strategy: cli_options.default_npm_caching_strategy(),
})
}
}

77
cli/graph_util.rs
View file

@ -7,6 +7,7 @@ use std::sync::Arc;
use deno_config::deno_json;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::deno_json::NodeModulesDirMode;
use deno_config::workspace::JsrPackageConfig;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
@ -29,6 +30,7 @@ use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_graph::WorkspaceFastCheckOption;
use deno_path_util::url_to_file_path;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::sloppy_imports::SloppyImportsCachedFs;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_node;
@ -36,8 +38,6 @@ use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::SmallStackString;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use crate::args::config_to_deno_graph_workspace_member;
use crate::args::jsr_url;
@ -52,8 +52,11 @@ use crate::cache::ModuleInfoCache;
use crate::cache::ParsedSourceCache;
use crate::colors;
use crate::file_fetcher::CliFileFetcher;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::PackageCaching;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliNpmGraphResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliSloppyImportsResolver;
use crate::sys::CliSys;
@ -264,7 +267,7 @@ pub struct CreateGraphOptions<'a> {
pub struct ModuleGraphCreator {
options: Arc<CliOptions>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
module_graph_builder: Arc<ModuleGraphBuilder>,
type_checker: Arc<TypeChecker>,
}
@ -272,13 +275,13 @@ pub struct ModuleGraphCreator {
impl ModuleGraphCreator {
pub fn new(
options: Arc<CliOptions>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
module_graph_builder: Arc<ModuleGraphBuilder>,
type_checker: Arc<TypeChecker>,
) -> Self {
Self {
options,
npm_resolver,
npm_installer,
module_graph_builder,
type_checker,
}
@ -401,9 +404,9 @@ impl ModuleGraphCreator {
.build_graph_with_npm_resolution(&mut graph, options)
.await?;
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
if let Some(npm_installer) = &self.npm_installer {
if graph.has_node_specifier && self.options.type_check_mode().is_true() {
npm_resolver.inject_synthetic_types_node_package().await?;
npm_installer.inject_synthetic_types_node_package().await?;
}
}
@ -490,15 +493,17 @@ pub enum BuildGraphWithNpmResolutionError {
pub struct ModuleGraphBuilder {
caches: Arc<cache::Caches>,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
lockfile: Option<Arc<CliLockfile>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
module_info_cache: Arc<ModuleInfoCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_graph_resolver: Arc<CliNpmGraphResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
@ -509,15 +514,17 @@ impl ModuleGraphBuilder {
#[allow(clippy::too_many_arguments)]
pub fn new(
caches: Arc<cache::Caches>,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
lockfile: Option<Arc<CliLockfile>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
module_info_cache: Arc<ModuleInfoCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_graph_resolver: Arc<CliNpmGraphResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
@ -533,6 +540,8 @@ impl ModuleGraphBuilder {
lockfile,
maybe_file_watcher_reporter,
module_info_cache,
npm_graph_resolver,
npm_installer,
npm_resolver,
parsed_source_cache,
resolver,
@ -631,10 +640,7 @@ impl ModuleGraphBuilder {
Some(loader) => MutLoaderRef::Borrowed(loader),
None => MutLoaderRef::Owned(self.create_graph_loader()),
};
let cli_resolver = &self.resolver;
let graph_resolver = self.create_graph_resolver()?;
let graph_npm_resolver =
cli_resolver.create_graph_npm_resolver(options.npm_caching);
let maybe_file_watcher_reporter = self
.maybe_file_watcher_reporter
.as_ref()
@ -655,7 +661,7 @@ impl ModuleGraphBuilder {
executor: Default::default(),
file_system: &self.sys,
jsr_url_provider: &CliJsrUrlProvider,
npm_resolver: Some(&graph_npm_resolver),
npm_resolver: Some(self.npm_graph_resolver.as_ref()),
module_analyzer: &analyzer,
reporter: maybe_file_watcher_reporter,
resolver: Some(&graph_resolver),
@ -679,16 +685,15 @@ impl ModuleGraphBuilder {
if self
.cli_options
.node_modules_dir()?
.map(|m| m.uses_node_modules_dir())
.map(|m| m == NodeModulesDirMode::Auto)
.unwrap_or(false)
{
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
let already_done =
npm_resolver.ensure_top_level_package_json_install().await?;
if let Some(npm_installer) = &self.npm_installer {
let already_done = npm_installer
.ensure_top_level_package_json_install()
.await?;
if !already_done && matches!(npm_caching, NpmCachingStrategy::Eager) {
npm_resolver
.cache_packages(crate::npm::PackageCaching::All)
.await?;
npm_installer.cache_packages(PackageCaching::All).await?;
}
}
}
@ -707,8 +712,7 @@ impl ModuleGraphBuilder {
let initial_package_deps_len = graph.packages.package_deps_sum();
let initial_package_mappings_len = graph.packages.mappings().len();
if roots.iter().any(|r| r.scheme() == "npm")
&& self.npm_resolver.as_byonm().is_some()
if roots.iter().any(|r| r.scheme() == "npm") && self.npm_resolver.is_byonm()
{
return Err(BuildGraphWithNpmResolutionError::UnsupportedNpmSpecifierEntrypointResolutionWay);
}
@ -776,11 +780,7 @@ impl ModuleGraphBuilder {
None
};
let parser = self.parsed_source_cache.as_capturing_parser();
let cli_resolver = &self.resolver;
let graph_resolver = self.create_graph_resolver()?;
let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(
self.cli_options.default_npm_caching_strategy(),
);
graph.build_fast_check_type_graph(
deno_graph::BuildFastCheckTypeGraphOptions {
@ -789,7 +789,7 @@ impl ModuleGraphBuilder {
fast_check_dts: false,
jsr_url_provider: &CliJsrUrlProvider,
resolver: Some(&graph_resolver),
npm_resolver: Some(&graph_npm_resolver),
npm_resolver: Some(self.npm_graph_resolver.as_ref()),
workspace_fast_check: options.workspace_fast_check,
},
);
@ -1024,14 +1024,11 @@ fn get_resolution_error_bare_specifier(
{
Some(specifier.as_str())
} else if let ResolutionError::ResolverError { error, .. } = error {
if let ResolveError::Other(error) = (*error).as_ref() {
if let Some(import_map::ImportMapErrorKind::UnmappedBareSpecifier(
if let ResolveError::ImportMap(error) = (*error).as_ref() {
if let import_map::ImportMapErrorKind::UnmappedBareSpecifier(
specifier,
_,
)) = error
.as_any()
.downcast_ref::<ImportMapError>()
.map(|e| &**e)
) = error.as_kind()
{
Some(specifier.as_str())
} else {
@ -1228,7 +1225,7 @@ fn format_deno_graph_error(err: &dyn Error) -> String {
#[derive(Debug)]
struct CliGraphResolver<'a> {
cjs_tracker: &'a CjsTracker,
cjs_tracker: &'a CliCjsTracker,
resolver: &'a CliResolver,
jsx_import_source_config: Option<JsxImportSourceConfig>,
}
@ -1324,7 +1321,7 @@ mod test {
let specifier = ModuleSpecifier::parse("file:///file.ts").unwrap();
let err = import_map.resolve(input, &specifier).err().unwrap();
let err = ResolutionError::ResolverError {
error: Arc::new(ResolveError::Other(JsErrorBox::from_err(err))),
error: Arc::new(ResolveError::ImportMap(err)),
specifier: input.to_string(),
range: Range {
specifier,

2
cli/js/40_bench.js
View file

@ -8,7 +8,7 @@ import {
restorePermissions,
} from "ext:cli/40_test_common.js";
import { Console } from "ext:deno_console/01_console.js";
import { setExitHandler } from "ext:runtime/30_os.js";
import { setExitHandler } from "ext:deno_os/30_os.js";
const {
op_register_bench,
op_bench_get_origin,

873
cli/js/40_lint.js
View file

File diff suppressed because it is too large Load diff

36
cli/js/40_lint_selector.js
View file

@ -744,8 +744,7 @@ export function compileSelector(selector) {
fn = matchNthChild(node, fn);
break;
case PSEUDO_HAS:
// FIXME
// fn = matchIs(part, fn);
// TODO(@marvinhagemeister)
throw new Error("TODO: :has");
case PSEUDO_NOT:
fn = matchNot(node.selectors, fn);
@ -767,8 +766,7 @@ export function compileSelector(selector) {
*/
function matchFirstChild(next) {
return (ctx, id) => {
const parent = ctx.getParent(id);
const first = ctx.getFirstChild(parent);
const first = ctx.getFirstChild(id);
return first === id && next(ctx, first);
};
}
@ -779,8 +777,7 @@ function matchFirstChild(next) {
*/
function matchLastChild(next) {
return (ctx, id) => {
const parent = ctx.getParent(id);
const last = ctx.getLastChild(parent);
const last = ctx.getLastChild(id);
return last === id && next(ctx, id);
};
}
@ -955,7 +952,9 @@ function matchElem(part, next) {
else if (part.elem === 0) return false;
const type = ctx.getType(id);
if (type > 0 && type === part.elem) return next(ctx, id);
if (type > 0 && type === part.elem) {
return next(ctx, id);
}
return false;
};
@ -968,7 +967,16 @@ function matchElem(part, next) {
*/
function matchAttrExists(attr, next) {
return (ctx, id) => {
return ctx.hasAttrPath(id, attr.prop, 0) ? next(ctx, id) : false;
try {
ctx.getAttrPathValue(id, attr.prop, 0);
return next(ctx, id);
} catch (err) {
if (err === -1) {
return false;
}
throw err;
}
};
}
@ -979,9 +987,15 @@ function matchAttrExists(attr, next) {
*/
function matchAttrBin(attr, next) {
return (ctx, id) => {
if (!ctx.hasAttrPath(id, attr.prop, 0)) return false;
const value = ctx.getAttrPathValue(id, attr.prop, 0);
if (!matchAttrValue(attr, value)) return false;
try {
const value = ctx.getAttrPathValue(id, attr.prop, 0);
if (!matchAttrValue(attr, value)) return false;
} catch (err) {
if (err === -1) {
return false;
}
throw err;
}
return next(ctx, id);
};
}

9
cli/js/40_lint_types.d.ts vendored
View file

@ -12,6 +12,8 @@ export interface AstContext {
strTableOffset: number;
rootOffset: number;
nodes: Map<number, NodeFacade>;
spansOffset: number;
propsOffset: number;
strByType: number[];
strByProp: number[];
typeByStr: Map<string, number>;
@ -19,6 +21,12 @@ export interface AstContext {
matcher: MatchContext;
}
export interface Node {
range: Range;
}
export type Range = [number, number];
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface RuleContext {
id: string;
@ -121,7 +129,6 @@ export interface MatchContext {
getSiblings(id: number): number[];
getParent(id: number): number;
getType(id: number): number;
hasAttrPath(id: number, propIds: number[], idx: number): boolean;
getAttrPathValue(id: number, propIds: number[], idx: number): unknown;
}

2
cli/js/40_test.js
View file

@ -26,7 +26,7 @@ const {
TypeError,
} = primordials;
import { setExitHandler } from "ext:runtime/30_os.js";
import { setExitHandler } from "ext:deno_os/30_os.js";
// Capture `Deno` global so that users deleting or mangling it, won't
// have impact on our sanitizers.

37
cli/lib/Cargo.toml Normal file
View file

@ -0,0 +1,37 @@
# Copyright 2018-2025 the Deno authors. MIT license.
[package]
name = "deno_lib"
version = "0.2.0"
authors.workspace = true
edition.workspace = true
license.workspace = true
readme = "README.md"
repository.workspace = true
description = "Shared code between the Deno CLI and denort"
[lib]
path = "lib.rs"
[dependencies]
deno_cache_dir.workspace = true
deno_error.workspace = true
deno_fs = { workspace = true, features = ["sync_fs"] }
deno_node = { workspace = true, features = ["sync_fs"] }
deno_path_util.workspace = true
deno_resolver = { workspace = true, features = ["sync"] }
deno_runtime.workspace = true
deno_terminal.workspace = true
faster-hex.workspace = true
log.workspace = true
node_resolver = { workspace = true, features = ["sync"] }
parking_lot.workspace = true
ring.workspace = true
serde = { workspace = true, features = ["derive"] }
sys_traits = { workspace = true, features = ["getrandom"] }
thiserror.workspace = true
tokio.workspace = true
url.workspace = true
[dev-dependencies]
test_util.workspace = true

4
cli/lib/README.md Normal file
View file

@ -0,0 +1,4 @@
# deno_lib
This crate contains the shared code between the Deno CLI and denort. It is
highly unstable.

25
cli/cache/deno_dir.rs → cli/lib/cache/deno_dir.rs vendored
View file

@ -4,21 +4,20 @@ use std::env;
use std::path::PathBuf;
use deno_cache_dir::DenoDirResolutionError;
use once_cell::sync::OnceCell;
use super::DiskCache;
use crate::sys::CliSys;
use crate::sys::DenoLibSys;
/// Lazily creates the deno dir which might be useful in scenarios
/// where functionality wants to continue if the DENO_DIR can't be created.
pub struct DenoDirProvider {
sys: CliSys,
pub struct DenoDirProvider<TSys: DenoLibSys> {
sys: TSys,
maybe_custom_root: Option<PathBuf>,
deno_dir: OnceCell<Result<DenoDir, DenoDirResolutionError>>,
deno_dir: std::sync::OnceLock<Result<DenoDir<TSys>, DenoDirResolutionError>>,
}
impl DenoDirProvider {
pub fn new(sys: CliSys, maybe_custom_root: Option<PathBuf>) -> Self {
impl<TSys: DenoLibSys> DenoDirProvider<TSys> {
pub fn new(sys: TSys, maybe_custom_root: Option<PathBuf>) -> Self {
Self {
sys,
maybe_custom_root,
@ -26,7 +25,9 @@ impl DenoDirProvider {
}
}
pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> {
pub fn get_or_create(
&self,
) -> Result<&DenoDir<TSys>, DenoDirResolutionError> {
self
.deno_dir
.get_or_init(|| {
@ -49,16 +50,16 @@ impl DenoDirProvider {
/// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them
/// in single directory that can be controlled with `$DENO_DIR` env variable.
#[derive(Debug, Clone)]
pub struct DenoDir {
pub struct DenoDir<TSys: DenoLibSys> {
/// Example: /Users/rld/.deno/
pub root: PathBuf,
/// Used by TsCompiler to cache compiler output.
pub gen_cache: DiskCache,
pub gen_cache: DiskCache<TSys>,
}
impl DenoDir {
impl<TSys: DenoLibSys> DenoDir<TSys> {
pub fn new(
sys: CliSys,
sys: TSys,
maybe_custom_root: Option<PathBuf>,
) -> Result<Self, deno_cache_dir::DenoDirResolutionError> {
let root = deno_cache_dir::resolve_deno_dir(

27
cli/cache/disk_cache.rs → cli/lib/cache/disk_cache.rs vendored
View file

@ -9,22 +9,22 @@ use std::path::Prefix;
use std::str;
use deno_cache_dir::url_to_filename;
use deno_core::url::Host;
use deno_core::url::Url;
use deno_cache_dir::CACHE_PERM;
use deno_path_util::fs::atomic_write_file_with_retries;
use url::Host;
use url::Url;
use super::CACHE_PERM;
use crate::sys::CliSys;
use crate::sys::DenoLibSys;
#[derive(Debug, Clone)]
pub struct DiskCache {
sys: CliSys,
pub struct DiskCache<TSys: DenoLibSys> {
sys: TSys,
pub location: PathBuf,
}
impl DiskCache {
impl<TSys: DenoLibSys> DiskCache<TSys> {
/// `location` must be an absolute path.
pub fn new(sys: CliSys, location: &Path) -> Self {
pub fn new(sys: TSys, location: &Path) -> Self {
assert!(location.is_absolute());
Self {
sys,
@ -130,6 +130,9 @@ impl DiskCache {
#[cfg(test)]
mod tests {
// ok, testing
#[allow(clippy::disallowed_types)]
use sys_traits::impls::RealSys;
use test_util::TempDir;
use super::*;
@ -138,7 +141,7 @@ mod tests {
fn test_set_get_cache_file() {
let temp_dir = TempDir::new();
let sub_dir = temp_dir.path().join("sub_dir");
let cache = DiskCache::new(CliSys::default(), &sub_dir.to_path_buf());
let cache = DiskCache::new(RealSys, &sub_dir.to_path_buf());
let path = PathBuf::from("foo/bar.txt");
cache.set(&path, b"hello").unwrap();
assert_eq!(cache.get(&path).unwrap(), b"hello");
@ -152,7 +155,7 @@ mod tests {
PathBuf::from("/deno_dir/")
};
let cache = DiskCache::new(CliSys::default(), &cache_location);
let cache = DiskCache::new(RealSys, &cache_location);
let mut test_cases = vec![
(
@ -208,7 +211,7 @@ mod tests {
} else {
"/foo"
};
let cache = DiskCache::new(CliSys::default(), &PathBuf::from(p));
let cache = DiskCache::new(RealSys, &PathBuf::from(p));
let mut test_cases = vec![
(
@ -256,7 +259,7 @@ mod tests {
PathBuf::from("/deno_dir/")
};
let cache = DiskCache::new(CliSys::default(), &cache_location);
let cache = DiskCache::new(RealSys, &cache_location);
let mut test_cases = vec!["unknown://localhost/test.ts"];

8
cli/lib/cache/mod.rs vendored Normal file
View file

@ -0,0 +1,8 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
mod deno_dir;
mod disk_cache;

10
cli/lib/env.rs Normal file
View file

@ -0,0 +1,10 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub fn has_trace_permissions_enabled() -> bool {
has_flag_env_var("DENO_TRACE_PERMISSIONS")
}
pub fn has_flag_env_var(name: &str) -> bool {
let value = std::env::var(name);
matches!(value.as_ref().map(|s| s.as_str()), Ok("1"))
}

9
cli/lib/lib.rs Normal file
View file

@ -0,0 +1,9 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub mod cache;
pub mod env;
pub mod npm;
pub mod standalone;
pub mod sys;
pub mod util;
pub mod worker;

6
cli/lib/npm/mod.rs Normal file
View file

@ -0,0 +1,6 @@
// Copyright 2018-2025 the Deno authors. MIT license.
mod permission_checker;
pub use permission_checker::NpmRegistryReadPermissionChecker;
pub use permission_checker::NpmRegistryReadPermissionCheckerMode;

13
cli/npm/permission_checker.rs → cli/lib/npm/permission_checker.rs
View file

@ -6,12 +6,11 @@ use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use deno_core::parking_lot::Mutex;
use deno_error::JsErrorBox;
use deno_runtime::deno_node::NodePermissions;
use sys_traits::FsCanonicalize;
use parking_lot::Mutex;
use crate::sys::CliSys;
use crate::sys::DenoLibSys;
#[derive(Debug)]
pub enum NpmRegistryReadPermissionCheckerMode {
@ -21,8 +20,8 @@ pub enum NpmRegistryReadPermissionCheckerMode {
}
#[derive(Debug)]
pub struct NpmRegistryReadPermissionChecker {
sys: CliSys,
pub struct NpmRegistryReadPermissionChecker<TSys: DenoLibSys> {
sys: TSys,
cache: Mutex<HashMap<PathBuf, PathBuf>>,
mode: NpmRegistryReadPermissionCheckerMode,
}
@ -37,8 +36,8 @@ struct EnsureRegistryReadPermissionError {
source: std::io::Error,
}
impl NpmRegistryReadPermissionChecker {
pub fn new(sys: CliSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self {
impl<TSys: DenoLibSys> NpmRegistryReadPermissionChecker<TSys> {
pub fn new(sys: TSys, mode: NpmRegistryReadPermissionCheckerMode) -> Self {
Self {
sys,
cache: Default::default(),

3
cli/lib/standalone/mod.rs Normal file
View file

@ -0,0 +1,3 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub mod virtual_fs;

296
cli/lib/standalone/virtual_fs.rs Normal file
View file

@ -0,0 +1,296 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::cmp::Ordering;
use std::path::Path;
use std::path::PathBuf;
use serde::Deserialize;
use serde::Serialize;
#[derive(Debug, Copy, Clone)]
pub enum VfsFileSubDataKind {
/// Raw bytes of the file.
Raw,
/// Bytes to use for module loading. For example, for TypeScript
/// files this will be the transpiled JavaScript source.
ModuleGraph,
}
#[derive(Debug, PartialEq, Eq)]
pub enum WindowsSystemRootablePath {
/// The root of the system above any drive letters.
WindowSystemRoot,
Path(PathBuf),
}
impl WindowsSystemRootablePath {
pub fn join(&self, name_component: &str) -> PathBuf {
// this method doesn't handle multiple components
debug_assert!(
!name_component.contains('\\'),
"Invalid component: {}",
name_component
);
debug_assert!(
!name_component.contains('/'),
"Invalid component: {}",
name_component
);
match self {
WindowsSystemRootablePath::WindowSystemRoot => {
// windows drive letter
PathBuf::from(&format!("{}\\", name_component))
}
WindowsSystemRootablePath::Path(path) => path.join(name_component),
}
}
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum FileSystemCaseSensitivity {
#[serde(rename = "s")]
Sensitive,
#[serde(rename = "i")]
Insensitive,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct VirtualDirectoryEntries(Vec<VfsEntry>);
impl VirtualDirectoryEntries {
pub fn new(mut entries: Vec<VfsEntry>) -> Self {
// needs to be sorted by name
entries.sort_by(|a, b| a.name().cmp(b.name()));
Self(entries)
}
pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, VfsEntry> {
self.0.iter_mut()
}
pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> {
self.0.iter()
}
pub fn take_inner(&mut self) -> Vec<VfsEntry> {
std::mem::take(&mut self.0)
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn get_by_name(
&self,
name: &str,
case_sensitivity: FileSystemCaseSensitivity,
) -> Option<&VfsEntry> {
self
.binary_search(name, case_sensitivity)
.ok()
.map(|index| &self.0[index])
}
pub fn get_mut_by_name(
&mut self,
name: &str,
case_sensitivity: FileSystemCaseSensitivity,
) -> Option<&mut VfsEntry> {
self
.binary_search(name, case_sensitivity)
.ok()
.map(|index| &mut self.0[index])
}
pub fn get_mut_by_index(&mut self, index: usize) -> Option<&mut VfsEntry> {
self.0.get_mut(index)
}
pub fn binary_search(
&self,
name: &str,
case_sensitivity: FileSystemCaseSensitivity,
) -> Result<usize, usize> {
match case_sensitivity {
FileSystemCaseSensitivity::Sensitive => {
self.0.binary_search_by(|e| e.name().cmp(name))
}
FileSystemCaseSensitivity::Insensitive => self.0.binary_search_by(|e| {
e.name()
.chars()
.zip(name.chars())
.map(|(a, b)| a.to_ascii_lowercase().cmp(&b.to_ascii_lowercase()))
.find(|&ord| ord != Ordering::Equal)
.unwrap_or_else(|| e.name().len().cmp(&name.len()))
}),
}
}
pub fn insert(
&mut self,
entry: VfsEntry,
case_sensitivity: FileSystemCaseSensitivity,
) -> usize {
match self.binary_search(entry.name(), case_sensitivity) {
Ok(index) => {
self.0[index] = entry;
index
}
Err(insert_index) => {
self.0.insert(insert_index, entry);
insert_index
}
}
}
pub fn insert_or_modify(
&mut self,
name: &str,
case_sensitivity: FileSystemCaseSensitivity,
on_insert: impl FnOnce() -> VfsEntry,
on_modify: impl FnOnce(&mut VfsEntry),
) -> usize {
match self.binary_search(name, case_sensitivity) {
Ok(index) => {
on_modify(&mut self.0[index]);
index
}
Err(insert_index) => {
self.0.insert(insert_index, on_insert());
insert_index
}
}
}
pub fn remove(&mut self, index: usize) -> VfsEntry {
self.0.remove(index)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
#[serde(rename = "n")]
pub name: String,
// should be sorted by name
#[serde(rename = "e")]
pub entries: VirtualDirectoryEntries,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct OffsetWithLength {
#[serde(rename = "o")]
pub offset: u64,
#[serde(rename = "l")]
pub len: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VirtualFile {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "o")]
pub offset: OffsetWithLength,
/// Offset file to use for module loading when it differs from the
/// raw file. Often this will be the same offset as above for data
/// such as JavaScript files, but for TypeScript files the `offset`
/// will be the original raw bytes when included as an asset and this
/// offset will be to the transpiled JavaScript source.
#[serde(rename = "m")]
pub module_graph_offset: OffsetWithLength,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlinkParts(Vec<String>);
impl VirtualSymlinkParts {
pub fn from_path(path: &Path) -> Self {
Self(
path
.components()
.filter(|c| !matches!(c, std::path::Component::RootDir))
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect(),
)
}
pub fn take_parts(&mut self) -> Vec<String> {
std::mem::take(&mut self.0)
}
pub fn parts(&self) -> &[String] {
&self.0
}
pub fn set_parts(&mut self, parts: Vec<String>) {
self.0 = parts;
}
pub fn display(&self) -> String {
self.0.join("/")
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "p")]
pub dest_parts: VirtualSymlinkParts,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts.0 {
dest.push(part);
}
dest
}
}
#[derive(Debug, Copy, Clone)]
pub enum VfsEntryRef<'a> {
Dir(&'a VirtualDirectory),
File(&'a VirtualFile),
Symlink(&'a VirtualSymlink),
}
impl VfsEntryRef<'_> {
pub fn name(&self) -> &str {
match self {
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
}
// todo(dsherret): we should store this more efficiently in the binary
#[derive(Debug, Serialize, Deserialize)]
pub enum VfsEntry {
Dir(VirtualDirectory),
File(VirtualFile),
Symlink(VirtualSymlink),
}
impl VfsEntry {
pub fn name(&self) -> &str {
match self {
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
pub fn as_ref(&self) -> VfsEntryRef {
match self {
VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir),
VfsEntry::File(file) => VfsEntryRef::File(file),
VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink),
}
}
}

37
cli/lib/sys.rs Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use deno_node::ExtNodeSys;
use sys_traits::FsCanonicalize;
use sys_traits::FsCreateDirAll;
use sys_traits::FsMetadata;
use sys_traits::FsOpen;
use sys_traits::FsRead;
use sys_traits::FsReadDir;
use sys_traits::FsRemoveFile;
use sys_traits::FsRename;
use sys_traits::SystemRandom;
use sys_traits::ThreadSleep;
pub trait DenoLibSys:
FsCanonicalize
+ FsCreateDirAll
+ FsReadDir
+ FsMetadata
+ FsOpen
+ FsRemoveFile
+ FsRename
+ FsRead
+ ThreadSleep
+ SystemRandom
+ ExtNodeSys
+ Clone
+ Send
+ Sync
+ std::fmt::Debug
+ 'static
{
}
// ok, implementation
#[allow(clippy::disallowed_types)]
impl DenoLibSys for sys_traits::impls::RealSys {}

0
cli/util/checksum.rs → cli/lib/util/checksum.rs
View file

3
cli/lib/util/mod.rs Normal file
View file

@ -0,0 +1,3 @@
// Copyright 2018-2025 the Deno authors. MIT license.
pub mod checksum;

581
cli/lib/worker.rs Normal file
View file

@ -0,0 +1,581 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_core::error::JsError;
use deno_node::NodeRequireLoaderRc;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::npm::NpmResolver;
use deno_runtime::colors;
use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
use deno_runtime::deno_core;
use deno_runtime::deno_core::error::CoreError;
use deno_runtime::deno_core::v8;
use deno_runtime::deno_core::CompiledWasmModuleStore;
use deno_runtime::deno_core::Extension;
use deno_runtime::deno_core::FeatureChecker;
use deno_runtime::deno_core::JsRuntime;
use deno_runtime::deno_core::LocalInspectorSession;
use deno_runtime::deno_core::ModuleLoader;
use deno_runtime::deno_core::SharedArrayBufferStore;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeExtInitServices;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_process::NpmProcessStateProviderRc;
use deno_runtime::deno_telemetry::OtelConfig;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::fmt_errors::format_js_error;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::ops::worker_host::CreateWebWorkerCb;
use deno_runtime::web_worker::WebWorker;
use deno_runtime::web_worker::WebWorkerOptions;
use deno_runtime::web_worker::WebWorkerServiceOptions;
use deno_runtime::worker::MainWorker;
use deno_runtime::worker::WorkerOptions;
use deno_runtime::worker::WorkerServiceOptions;
use deno_runtime::BootstrapOptions;
use deno_runtime::WorkerExecutionMode;
use deno_runtime::WorkerLogLevel;
use deno_runtime::UNSTABLE_GRANULAR_FLAGS;
use url::Url;
use crate::env::has_trace_permissions_enabled;
use crate::sys::DenoLibSys;
use crate::util::checksum;
pub struct CreateModuleLoaderResult {
pub module_loader: Rc<dyn ModuleLoader>,
pub node_require_loader: Rc<dyn NodeRequireLoader>,
}
pub trait ModuleLoaderFactory: Send + Sync {
fn create_for_main(
&self,
root_permissions: PermissionsContainer,
) -> CreateModuleLoaderResult;
fn create_for_worker(
&self,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
) -> CreateModuleLoaderResult;
}
enum StorageKeyResolverStrategy {
Specified(Option<String>),
UseMainModule,
}
pub struct StorageKeyResolver(StorageKeyResolverStrategy);
impl StorageKeyResolver {
pub fn from_flag(location: &Url) -> Self {
// if a location is set, then the ascii serialization of the location is
// used, unless the origin is opaque, and then no storage origin is set, as
// we can't expect the origin to be reproducible
let storage_origin = location.origin();
Self(StorageKeyResolverStrategy::Specified(
if storage_origin.is_tuple() {
Some(storage_origin.ascii_serialization())
} else {
None
},
))
}
pub fn from_config_file_url(url: &Url) -> Self {
Self(StorageKeyResolverStrategy::Specified(Some(url.to_string())))
}
pub fn new_use_main_module() -> Self {
Self(StorageKeyResolverStrategy::UseMainModule)
}
/// Creates a storage key resolver that will always resolve to being empty.
pub fn empty() -> Self {
Self(StorageKeyResolverStrategy::Specified(None))
}
/// Resolves the storage key to use based on the current flags, config, or main module.
pub fn resolve_storage_key(&self, main_module: &Url) -> Option<String> {
// use the stored value or fall back to using the path of the main module.
match &self.0 {
StorageKeyResolverStrategy::Specified(value) => value.clone(),
StorageKeyResolverStrategy::UseMainModule => {
Some(main_module.to_string())
}
}
}
}
// TODO(bartlomieju): this should be moved to some other place, added to avoid string
// duplication between worker setups and `deno info` output.
pub fn get_cache_storage_dir() -> PathBuf {
// Note: we currently use temp_dir() to avoid managing storage size.
std::env::temp_dir().join("deno_cache")
}
/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs.
/// Instead probe for the total memory on the system and use it instead
/// as a default.
pub fn create_isolate_create_params() -> Option<v8::CreateParams> {
let maybe_mem_info = deno_runtime::deno_os::sys_info::mem_info();
maybe_mem_info.map(|mem_info| {
v8::CreateParams::default()
.heap_limits_from_system_memory(mem_info.total, 0)
})
}
pub struct LibMainWorkerOptions {
pub argv: Vec<String>,
pub deno_version: &'static str,
pub deno_user_agent: &'static str,
pub log_level: WorkerLogLevel,
pub enable_op_summary_metrics: bool,
pub enable_testing_features: bool,
pub has_node_modules_dir: bool,
pub inspect_brk: bool,
pub inspect_wait: bool,
pub strace_ops: Option<Vec<String>>,
pub is_inspecting: bool,
pub location: Option<Url>,
pub argv0: Option<String>,
pub node_debug: Option<String>,
pub otel_config: OtelConfig,
pub origin_data_folder_path: Option<PathBuf>,
pub seed: Option<u64>,
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub skip_op_registration: bool,
pub node_ipc: Option<i64>,
pub startup_snapshot: Option<&'static [u8]>,
pub serve_port: Option<u16>,
pub serve_host: Option<String>,
}
struct LibWorkerFactorySharedState<TSys: DenoLibSys> {
blob_store: Arc<BlobStore>,
broadcast_channel: InMemoryBroadcastChannel,
code_cache: Option<Arc<dyn deno_runtime::code_cache::CodeCache>>,
compiled_wasm_module_store: CompiledWasmModuleStore,
feature_checker: Arc<FeatureChecker>,
fs: Arc<dyn deno_fs::FileSystem>,
maybe_inspector_server: Option<Arc<InspectorServer>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver:
Arc<NodeResolver<DenoInNpmPackageChecker, NpmResolver<TSys>, TSys>>,
npm_process_state_provider: NpmProcessStateProviderRc,
pkg_json_resolver: Arc<node_resolver::PackageJsonResolver<TSys>>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
shared_array_buffer_store: SharedArrayBufferStore,
storage_key_resolver: StorageKeyResolver,
sys: TSys,
options: LibMainWorkerOptions,
}
impl<TSys: DenoLibSys> LibWorkerFactorySharedState<TSys> {
fn resolve_unstable_features(
&self,
feature_checker: &FeatureChecker,
) -> Vec<i32> {
let mut unstable_features =
Vec::with_capacity(UNSTABLE_GRANULAR_FLAGS.len());
for granular_flag in UNSTABLE_GRANULAR_FLAGS {
if feature_checker.check(granular_flag.name) {
unstable_features.push(granular_flag.id);
}
}
unstable_features
}
fn create_node_init_services(
&self,
node_require_loader: NodeRequireLoaderRc,
) -> NodeExtInitServices<DenoInNpmPackageChecker, NpmResolver<TSys>, TSys> {
NodeExtInitServices {
node_require_loader,
node_resolver: self.node_resolver.clone(),
pkg_json_resolver: self.pkg_json_resolver.clone(),
sys: self.sys.clone(),
}
}
fn create_web_worker_callback(
self: &Arc<Self>,
stdio: deno_runtime::deno_io::Stdio,
) -> Arc<CreateWebWorkerCb> {
let shared = self.clone();
Arc::new(move |args| {
let maybe_inspector_server = shared.maybe_inspector_server.clone();
let CreateModuleLoaderResult {
module_loader,
node_require_loader,
} = shared.module_loader_factory.create_for_worker(
args.parent_permissions.clone(),
args.permissions.clone(),
);
let create_web_worker_cb =
shared.create_web_worker_callback(stdio.clone());
let maybe_storage_key = shared
.storage_key_resolver
.resolve_storage_key(&args.main_module);
let cache_storage_dir = maybe_storage_key.map(|key| {
// TODO(@satyarohith): storage quota management
get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()]))
});
// TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
// list of enabled features.
let feature_checker = shared.feature_checker.clone();
let unstable_features =
shared.resolve_unstable_features(feature_checker.as_ref());
let services = WebWorkerServiceOptions {
root_cert_store_provider: Some(shared.root_cert_store_provider.clone()),
module_loader,
fs: shared.fs.clone(),
node_services: Some(
shared.create_node_init_services(node_require_loader),
),
blob_store: shared.blob_store.clone(),
broadcast_channel: shared.broadcast_channel.clone(),
shared_array_buffer_store: Some(
shared.shared_array_buffer_store.clone(),
),
compiled_wasm_module_store: Some(
shared.compiled_wasm_module_store.clone(),
),
maybe_inspector_server,
feature_checker,
npm_process_state_provider: Some(
shared.npm_process_state_provider.clone(),
),
permissions: args.permissions,
};
let options = WebWorkerOptions {
name: args.name,
main_module: args.main_module.clone(),
worker_id: args.worker_id,
bootstrap: BootstrapOptions {
deno_version: shared.options.deno_version.to_string(),
args: shared.options.argv.clone(),
cpu_count: std::thread::available_parallelism()
.map(|p| p.get())
.unwrap_or(1),
log_level: shared.options.log_level,
enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
enable_testing_features: shared.options.enable_testing_features,
locale: deno_core::v8::icu::get_language_tag(),
location: Some(args.main_module),
no_color: !colors::use_color(),
color_level: colors::get_color_level(),
is_stdout_tty: deno_terminal::is_stdout_tty(),
is_stderr_tty: deno_terminal::is_stderr_tty(),
unstable_features,
user_agent: shared.options.deno_user_agent.to_string(),
inspect: shared.options.is_inspecting,
has_node_modules_dir: shared.options.has_node_modules_dir,
argv0: shared.options.argv0.clone(),
node_debug: shared.options.node_debug.clone(),
node_ipc_fd: None,
mode: WorkerExecutionMode::Worker,
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.options.otel_config.clone(),
close_on_idle: args.close_on_idle,
},
extensions: vec![],
startup_snapshot: shared.options.startup_snapshot,
create_params: create_isolate_create_params(),
unsafely_ignore_certificate_errors: shared
.options
.unsafely_ignore_certificate_errors
.clone(),
seed: shared.options.seed,
create_web_worker_cb,
format_js_error_fn: Some(Arc::new(format_js_error)),
worker_type: args.worker_type,
stdio: stdio.clone(),
cache_storage_dir,
strace_ops: shared.options.strace_ops.clone(),
close_on_idle: args.close_on_idle,
maybe_worker_metadata: args.maybe_worker_metadata,
enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(),
};
WebWorker::bootstrap_from_options(services, options)
})
}
}
pub struct LibMainWorkerFactory<TSys: DenoLibSys> {
shared: Arc<LibWorkerFactorySharedState<TSys>>,
}
impl<TSys: DenoLibSys> LibMainWorkerFactory<TSys> {
#[allow(clippy::too_many_arguments)]
pub fn new(
blob_store: Arc<BlobStore>,
code_cache: Option<Arc<dyn deno_runtime::code_cache::CodeCache>>,
feature_checker: Arc<FeatureChecker>,
fs: Arc<dyn deno_fs::FileSystem>,
maybe_inspector_server: Option<Arc<InspectorServer>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver: Arc<
NodeResolver<DenoInNpmPackageChecker, NpmResolver<TSys>, TSys>,
>,
npm_process_state_provider: NpmProcessStateProviderRc,
pkg_json_resolver: Arc<node_resolver::PackageJsonResolver<TSys>>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
storage_key_resolver: StorageKeyResolver,
sys: TSys,
options: LibMainWorkerOptions,
) -> Self {
Self {
shared: Arc::new(LibWorkerFactorySharedState {
blob_store,
broadcast_channel: Default::default(),
code_cache,
compiled_wasm_module_store: Default::default(),
feature_checker,
fs,
maybe_inspector_server,
module_loader_factory,
node_resolver,
npm_process_state_provider,
pkg_json_resolver,
root_cert_store_provider,
shared_array_buffer_store: Default::default(),
storage_key_resolver,
sys,
options,
}),
}
}
pub fn create_custom_worker(
&self,
mode: WorkerExecutionMode,
main_module: Url,
permissions: PermissionsContainer,
custom_extensions: Vec<Extension>,
stdio: deno_runtime::deno_io::Stdio,
) -> Result<LibMainWorker, CoreError> {
let shared = &self.shared;
let CreateModuleLoaderResult {
module_loader,
node_require_loader,
} = shared
.module_loader_factory
.create_for_main(permissions.clone());
// TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
// list of enabled features.
let feature_checker = shared.feature_checker.clone();
let unstable_features =
shared.resolve_unstable_features(feature_checker.as_ref());
let maybe_storage_key = shared
.storage_key_resolver
.resolve_storage_key(&main_module);
let origin_storage_dir = maybe_storage_key.as_ref().map(|key| {
shared
.options
.origin_data_folder_path
.as_ref()
.unwrap() // must be set if storage key resolver returns a value
.join(checksum::gen(&[key.as_bytes()]))
});
let cache_storage_dir = maybe_storage_key.map(|key| {
// TODO(@satyarohith): storage quota management
get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()]))
});
let services = WorkerServiceOptions {
root_cert_store_provider: Some(shared.root_cert_store_provider.clone()),
module_loader,
fs: shared.fs.clone(),
node_services: Some(
shared.create_node_init_services(node_require_loader),
),
npm_process_state_provider: Some(
shared.npm_process_state_provider.clone(),
),
blob_store: shared.blob_store.clone(),
broadcast_channel: shared.broadcast_channel.clone(),
fetch_dns_resolver: Default::default(),
shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(
shared.compiled_wasm_module_store.clone(),
),
feature_checker,
permissions,
v8_code_cache: shared.code_cache.clone(),
};
let options = WorkerOptions {
bootstrap: BootstrapOptions {
deno_version: shared.options.deno_version.to_string(),
args: shared.options.argv.clone(),
cpu_count: std::thread::available_parallelism()
.map(|p| p.get())
.unwrap_or(1),
log_level: shared.options.log_level,
enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
enable_testing_features: shared.options.enable_testing_features,
locale: deno_core::v8::icu::get_language_tag(),
location: shared.options.location.clone(),
no_color: !colors::use_color(),
is_stdout_tty: deno_terminal::is_stdout_tty(),
is_stderr_tty: deno_terminal::is_stderr_tty(),
color_level: colors::get_color_level(),
unstable_features,
user_agent: shared.options.deno_user_agent.to_string(),
inspect: shared.options.is_inspecting,
has_node_modules_dir: shared.options.has_node_modules_dir,
argv0: shared.options.argv0.clone(),
node_debug: shared.options.node_debug.clone(),
node_ipc_fd: shared.options.node_ipc,
mode,
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.options.otel_config.clone(),
close_on_idle: true,
},
extensions: custom_extensions,
startup_snapshot: shared.options.startup_snapshot,
create_params: create_isolate_create_params(),
unsafely_ignore_certificate_errors: shared
.options
.unsafely_ignore_certificate_errors
.clone(),
seed: shared.options.seed,
format_js_error_fn: Some(Arc::new(format_js_error)),
create_web_worker_cb: shared.create_web_worker_callback(stdio.clone()),
maybe_inspector_server: shared.maybe_inspector_server.clone(),
should_break_on_first_statement: shared.options.inspect_brk,
should_wait_for_inspector_session: shared.options.inspect_wait,
strace_ops: shared.options.strace_ops.clone(),
cache_storage_dir,
origin_storage_dir,
stdio,
skip_op_registration: shared.options.skip_op_registration,
enable_stack_trace_arg_in_ops: has_trace_permissions_enabled(),
};
let worker =
MainWorker::bootstrap_from_options(&main_module, services, options);
Ok(LibMainWorker {
main_module,
worker,
})
}
}
pub struct LibMainWorker {
main_module: Url,
worker: MainWorker,
}
impl LibMainWorker {
pub fn into_main_worker(self) -> MainWorker {
self.worker
}
pub fn main_module(&self) -> &Url {
&self.main_module
}
pub fn js_runtime(&mut self) -> &mut JsRuntime {
&mut self.worker.js_runtime
}
#[inline]
pub fn create_inspector_session(&mut self) -> LocalInspectorSession {
self.worker.create_inspector_session()
}
#[inline]
pub fn dispatch_load_event(&mut self) -> Result<(), JsError> {
self.worker.dispatch_load_event()
}
#[inline]
pub fn dispatch_beforeunload_event(&mut self) -> Result<bool, JsError> {
self.worker.dispatch_beforeunload_event()
}
#[inline]
pub fn dispatch_process_beforeexit_event(&mut self) -> Result<bool, JsError> {
self.worker.dispatch_process_beforeexit_event()
}
#[inline]
pub fn dispatch_unload_event(&mut self) -> Result<(), JsError> {
self.worker.dispatch_unload_event()
}
#[inline]
pub fn dispatch_process_exit_event(&mut self) -> Result<(), JsError> {
self.worker.dispatch_process_exit_event()
}
pub async fn execute_main_module(&mut self) -> Result<(), CoreError> {
let id = self.worker.preload_main_module(&self.main_module).await?;
self.worker.evaluate_module(id).await
}
pub async fn execute_side_module(&mut self) -> Result<(), CoreError> {
let id = self.worker.preload_side_module(&self.main_module).await?;
self.worker.evaluate_module(id).await
}
#[inline]
pub async fn run_event_loop(
&mut self,
wait_for_inspector: bool,
) -> Result<(), CoreError> {
self.worker.run_event_loop(wait_for_inspector).await
}
#[inline]
pub fn exit_code(&self) -> i32 {
self.worker.exit_code()
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn storage_key_resolver_test() {
let resolver =
StorageKeyResolver(StorageKeyResolverStrategy::UseMainModule);
let specifier = Url::parse("file:///a.ts").unwrap();
assert_eq!(
resolver.resolve_storage_key(&specifier),
Some(specifier.to_string())
);
let resolver =
StorageKeyResolver(StorageKeyResolverStrategy::Specified(None));
assert_eq!(resolver.resolve_storage_key(&specifier), None);
let resolver = StorageKeyResolver(StorageKeyResolverStrategy::Specified(
Some("value".to_string()),
));
assert_eq!(
resolver.resolve_storage_key(&specifier),
Some("value".to_string())
);
// test empty
let resolver = StorageKeyResolver::empty();
assert_eq!(resolver.resolve_storage_key(&specifier), None);
}
}

26
cli/lsp/analysis.rs
View file

@ -19,6 +19,7 @@ use deno_core::ModuleSpecifier;
use deno_error::JsErrorBox;
use deno_lint::diagnostic::LintDiagnosticRange;
use deno_path_util::url_to_file_path;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_runtime::deno_node::PathClean;
use deno_semver::jsr::JsrPackageNvReference;
use deno_semver::jsr::JsrPackageReqReference;
@ -31,6 +32,7 @@ use deno_semver::SmallStackString;
use deno_semver::StackString;
use deno_semver::Version;
use import_map::ImportMap;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use once_cell::sync::Lazy;
@ -365,7 +367,9 @@ impl<'a> TsResponseImportMapper<'a> {
if let Ok(Some(pkg_id)) =
npm_resolver.resolve_pkg_id_from_specifier(specifier)
{
let pkg_reqs = npm_resolver.resolve_pkg_reqs_from_pkg_id(&pkg_id);
let pkg_reqs = npm_resolver
.resolution()
.resolve_pkg_reqs_from_pkg_id(&pkg_id);
// check if any pkg reqs match what is found in an import map
if !pkg_reqs.is_empty() {
let sub_path = npm_resolver
@ -1295,6 +1299,19 @@ impl CodeActionCollection {
range: &lsp::Range,
language_server: &language_server::Inner,
) -> Option<lsp::CodeAction> {
fn top_package_req_for_name(
resolution: &NpmResolutionCell,
name: &str,
) -> Option<PackageReq> {
let package_reqs = resolution.package_reqs();
let mut entries = package_reqs
.into_iter()
.filter(|(_, nv)| nv.name == name)
.collect::<Vec<_>>();
entries.sort_by(|a, b| a.1.version.cmp(&b.1.version));
Some(entries.pop()?.0)
}
let (dep_key, dependency, _) =
document.get_maybe_dependency(&range.end)?;
if dependency.maybe_deno_types_specifier.is_some() {
@ -1382,9 +1399,10 @@ impl CodeActionCollection {
.and_then(|versions| versions.first().cloned())?;
let types_specifier_text =
if let Some(npm_resolver) = managed_npm_resolver {
let mut specifier_text = if let Some(req) =
npm_resolver.top_package_req_for_name(&types_package_name)
{
let mut specifier_text = if let Some(req) = top_package_req_for_name(
npm_resolver.resolution(),
&types_package_name,
) {
format!("npm:{req}")
} else {
format!("npm:{}@^{}", &types_package_name, types_package_version)

6
cli/lsp/cache.rs
View file

@ -8,9 +8,9 @@ use std::time::SystemTime;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_lib::cache::DenoDir;
use deno_path_util::url_to_file_path;
use crate::cache::DenoDir;
use crate::cache::GlobalHttpCache;
use crate::cache::HttpCache;
use crate::cache::LocalLspHttpCache;
@ -70,7 +70,7 @@ fn calculate_fs_version_in_cache(
#[derive(Debug, Clone)]
pub struct LspCache {
deno_dir: DenoDir,
deno_dir: DenoDir<CliSys>,
global: Arc<GlobalHttpCache>,
vendors_by_scope: BTreeMap<ModuleSpecifier, Option<Arc<LocalLspHttpCache>>>,
}
@ -121,7 +121,7 @@ impl LspCache {
.collect();
}
pub fn deno_dir(&self) -> &DenoDir {
pub fn deno_dir(&self) -> &DenoDir<CliSys> {
&self.deno_dir
}

17
cli/lsp/config.rs
View file

@ -41,7 +41,7 @@ use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_error::JsErrorBox;
use deno_lib::env::has_flag_env_var;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_package_json::PackageJsonCache;
@ -56,7 +56,6 @@ use super::logging::lsp_log;
use super::lsp_custom;
use super::urls::url_to_uri;
use crate::args::discover_npmrc_from_workspace;
use crate::args::has_flag_env_var;
use crate::args::CliLockfile;
use crate::args::CliLockfileReadFromPathOptions;
use crate::args::ConfigFile;
@ -1247,7 +1246,6 @@ impl ConfigData {
pkg_json_cache: Some(pkg_json_cache),
workspace_cache: Some(workspace_cache),
discover_pkg_json: !has_flag_env_var("DENO_NO_PACKAGE_JSON"),
config_parse_options: Default::default(),
maybe_vendor_override: None,
},
)
@ -1572,11 +1570,11 @@ impl ConfigData {
let resolver = member_dir
.workspace
.create_resolver(
&CliSys::default(),
CreateResolverOptions {
pkg_json_dep_resolution,
specified_import_map,
},
|path| std::fs::read_to_string(path).map_err(JsErrorBox::from_err),
)
.inspect_err(|err| {
lsp_warn!(
@ -2078,7 +2076,6 @@ impl deno_config::workspace::WorkspaceCache for WorkspaceMemCache {
#[cfg(test)]
mod tests {
use deno_config::deno_json::ConfigParseOptions;
use deno_core::resolve_url;
use deno_core::serde_json;
use deno_core::serde_json::json;
@ -2352,12 +2349,7 @@ mod tests {
config
.tree
.inject_config_file(
ConfigFile::new(
"{}",
root_uri.join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
ConfigFile::new("{}", root_uri.join("deno.json").unwrap()).unwrap(),
)
.await;
assert!(config.specifier_enabled(&root_uri));
@ -2413,7 +2405,6 @@ mod tests {
})
.to_string(),
root_uri.join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
)
@ -2439,7 +2430,6 @@ mod tests {
})
.to_string(),
root_uri.join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
)
@ -2457,7 +2447,6 @@ mod tests {
})
.to_string(),
root_uri.join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
)

9
cli/lsp/diagnostics.rs
View file

@ -265,7 +265,7 @@ impl TsDiagnosticsStore {
}
pub fn should_send_diagnostic_batch_index_notifications() -> bool {
crate::args::has_flag_env_var(
deno_lib::env::has_flag_env_var(
"DENO_DONT_USE_INTERNAL_LSP_DIAGNOSTIC_SYNC_FLAG",
)
}
@ -1695,12 +1695,7 @@ mod tests {
let mut config = Config::new_with_roots([root_uri.clone()]);
if let Some((relative_path, json_string)) = maybe_import_map {
let base_url = root_uri.join(relative_path).unwrap();
let config_file = ConfigFile::new(
json_string,
base_url,
&deno_config::deno_json::ConfigParseOptions::default(),
)
.unwrap();
let config_file = ConfigFile::new(json_string, base_url).unwrap();
config.tree.inject_config_file(config_file).await;
}
let resolver =

13
cli/lsp/documents.rs
View file

@ -480,7 +480,7 @@ impl Document {
let is_cjs_resolver =
resolver.as_is_cjs_resolver(self.file_referrer.as_ref());
let npm_resolver =
resolver.create_graph_npm_resolver(self.file_referrer.as_ref());
resolver.as_graph_npm_resolver(self.file_referrer.as_ref());
let config_data = resolver.as_config_data(self.file_referrer.as_ref());
let jsx_import_source_config =
config_data.and_then(|d| d.maybe_jsx_import_source_config());
@ -503,7 +503,7 @@ impl Document {
s,
&CliJsrUrlProvider,
Some(&resolver),
Some(&npm_resolver),
Some(npm_resolver.as_ref()),
),
)
})
@ -513,7 +513,7 @@ impl Document {
Arc::new(d.with_new_resolver(
&CliJsrUrlProvider,
Some(&resolver),
Some(&npm_resolver),
Some(npm_resolver.as_ref()),
))
});
is_script = self.is_script;
@ -1702,7 +1702,7 @@ fn analyze_module(
) -> (ModuleResult, ResolutionMode) {
match parsed_source_result {
Ok(parsed_source) => {
let npm_resolver = resolver.create_graph_npm_resolver(file_referrer);
let npm_resolver = resolver.as_graph_npm_resolver(file_referrer);
let cli_resolver = resolver.as_cli_resolver(file_referrer);
let is_cjs_resolver = resolver.as_is_cjs_resolver(file_referrer);
let config_data = resolver.as_config_data(file_referrer);
@ -1731,7 +1731,7 @@ fn analyze_module(
file_system: &deno_graph::source::NullFileSystem,
jsr_url_provider: &CliJsrUrlProvider,
maybe_resolver: Some(&resolver),
maybe_npm_resolver: Some(&npm_resolver),
maybe_npm_resolver: Some(npm_resolver.as_ref()),
},
)),
module_resolution_mode,
@ -1767,7 +1767,6 @@ fn bytes_to_content(
#[cfg(test)]
mod tests {
use deno_config::deno_json::ConfigFile;
use deno_config::deno_json::ConfigParseOptions;
use deno_core::serde_json;
use deno_core::serde_json::json;
use pretty_assertions::assert_eq;
@ -1924,7 +1923,6 @@ console.log(b, "hello deno");
})
.to_string(),
config.root_uri().unwrap().join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
)
@ -1968,7 +1966,6 @@ console.log(b, "hello deno");
})
.to_string(),
config.root_uri().unwrap().join("deno.json").unwrap(),
&ConfigParseOptions::default(),
)
.unwrap(),
)

12
cli/lsp/language_server.rs
View file

@ -27,6 +27,7 @@ use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_graph::GraphKind;
use deno_graph::Resolution;
use deno_lib::env::has_flag_env_var;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::RootCertStoreProvider;
@ -95,7 +96,6 @@ use super::urls::uri_to_url;
use super::urls::url_to_uri;
use crate::args::create_default_npmrc;
use crate::args::get_root_cert_store;
use crate::args::has_flag_env_var;
use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::Flags;
@ -1419,18 +1419,16 @@ impl Inner {
// the file path is only used to determine what formatter should
// be used to format the file, so give the filepath an extension
// that matches what the user selected as the language
let file_path = document
let ext = document
.maybe_language_id()
.and_then(|id| id.as_extension())
.map(|ext| file_path.with_extension(ext))
.unwrap_or(file_path);
.and_then(|id| id.as_extension().map(|s| s.to_string()));
// it's not a js/ts file, so attempt to format its contents
format_file(
&file_path,
document.content(),
&fmt_options,
&unstable_options,
None,
ext,
)
}
};
@ -3623,8 +3621,6 @@ impl Inner {
deno_json_cache: None,
pkg_json_cache: None,
workspace_cache: None,
config_parse_options:
deno_config::deno_json::ConfigParseOptions::default(),
additional_config_file_names: &[],
discover_pkg_json: !has_flag_env_var("DENO_NO_PACKAGE_JSON"),
maybe_vendor_override: if force_global_cache {

300
cli/lsp/resolver.rs
View file

@ -9,7 +9,6 @@ use std::sync::Arc;
use dashmap::DashMap;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_cache_dir::npm::NpmCacheDir;
use deno_cache_dir::HttpCache;
use deno_config::deno_json::JsxImportSourceConfig;
@ -21,10 +20,13 @@ use deno_graph::GraphImport;
use deno_graph::ModuleSpecifier;
use deno_graph::Range;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::TarballCache;
use deno_path_util::url_to_file_path;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_resolver::npm::CreateInNpmPkgCheckerOptions;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
@ -34,7 +36,6 @@ use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use indexmap::IndexMap;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
@ -42,6 +43,8 @@ use super::cache::LspCache;
use super::jsr::JsrCacheResolver;
use crate::args::create_default_npmrc;
use crate::args::CliLockfile;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmCachingStrategy;
use crate::args::NpmInstallDepsProvider;
use crate::factory::Deferred;
use crate::graph_util::to_node_resolution_kind;
@ -53,19 +56,24 @@ use crate::lsp::config::ConfigData;
use crate::lsp::logging::lsp_warn;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver_for_lsp;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::NpmResolutionInstaller;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedNpmResolver;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmCacheHttpClient;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::ManagedCliNpmResolver;
use crate::npm::NpmResolutionInitializer;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliIsCjsResolver;
use crate::resolver::CliNpmGraphResolver;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
use crate::resolver::IsCjsResolver;
use crate::resolver::WorkerCliNpmGraphResolver;
use crate::resolver::FoundPackageJsonDepFlag;
use crate::sys::CliSys;
use crate::tsc::into_specifier_and_media_type;
use crate::util::progress_bar::ProgressBar;
@ -74,10 +82,13 @@ use crate::util::progress_bar::ProgressBarStyle;
#[derive(Debug, Clone)]
struct LspScopeResolver {
resolver: Arc<CliResolver>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
is_cjs_resolver: Arc<IsCjsResolver>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
is_cjs_resolver: Arc<CliIsCjsResolver>,
jsr_resolver: Option<Arc<JsrCacheResolver>>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
npm_graph_resolver: Arc<CliNpmGraphResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolution: Arc<NpmResolutionCell>,
npm_resolver: Option<CliNpmResolver>,
node_resolver: Option<Arc<CliNodeResolver>>,
npm_pkg_req_resolver: Option<Arc<CliNpmReqResolver>>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
@ -96,8 +107,11 @@ impl Default for LspScopeResolver {
in_npm_pkg_checker: factory.in_npm_pkg_checker().clone(),
is_cjs_resolver: factory.is_cjs_resolver().clone(),
jsr_resolver: None,
npm_graph_resolver: factory.npm_graph_resolver().clone(),
npm_installer: None,
npm_resolver: None,
node_resolver: None,
npm_resolution: factory.services.npm_resolution.clone(),
npm_pkg_req_resolver: None,
pkg_json_resolver: factory.pkg_json_resolver().clone(),
redirect_resolver: None,
@ -121,6 +135,7 @@ impl LspScopeResolver {
}
let in_npm_pkg_checker = factory.in_npm_pkg_checker().clone();
let npm_resolver = factory.npm_resolver().cloned();
let npm_installer = factory.npm_installer().cloned();
let node_resolver = factory.node_resolver().cloned();
let npm_pkg_req_resolver = factory.npm_pkg_req_resolver().cloned();
let cli_resolver = factory.cli_resolver().clone();
@ -133,8 +148,7 @@ impl LspScopeResolver {
cache.for_specifier(config_data.map(|d| d.scope.as_ref())),
config_data.and_then(|d| d.lockfile.clone()),
)));
let npm_graph_resolver = cli_resolver
.create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager);
let npm_graph_resolver = factory.npm_graph_resolver();
let maybe_jsx_import_source_config =
config_data.and_then(|d| d.maybe_jsx_import_source_config());
let graph_imports = config_data
@ -156,7 +170,7 @@ impl LspScopeResolver {
imports,
&CliJsrUrlProvider,
Some(&resolver),
Some(&npm_graph_resolver),
Some(npm_graph_resolver.as_ref()),
);
(referrer, graph_import)
})
@ -207,8 +221,11 @@ impl LspScopeResolver {
in_npm_pkg_checker,
is_cjs_resolver: factory.is_cjs_resolver().clone(),
jsr_resolver,
npm_graph_resolver: factory.npm_graph_resolver().clone(),
npm_pkg_req_resolver,
npm_resolver,
npm_installer,
npm_resolution: factory.services.npm_resolution.clone(),
node_resolver,
pkg_json_resolver,
redirect_resolver,
@ -220,18 +237,68 @@ impl LspScopeResolver {
}
fn snapshot(&self) -> Arc<Self> {
// create a copy of the resolution and then re-initialize the npm resolver from that
// todo(dsherret): this is pretty terrible... we should improve this. It should
// be possible to just change the npm_resolution on the new factory then access
// another method to create a new npm resolver
let mut factory = ResolverFactory::new(self.config_data.as_ref());
let npm_resolver =
self.npm_resolver.as_ref().map(|r| r.clone_snapshotted());
factory
.services
.npm_resolution
.set_snapshot(self.npm_resolution.snapshot());
let npm_resolver = self.npm_resolver.as_ref();
if let Some(npm_resolver) = &npm_resolver {
factory.set_npm_resolver(npm_resolver.clone());
factory.set_npm_resolver(CliNpmResolver::new::<CliSys>(
match npm_resolver {
CliNpmResolver::Byonm(byonm_npm_resolver) => {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
root_node_modules_dir: byonm_npm_resolver
.root_node_modules_path()
.map(|p| p.to_path_buf()),
sys: CliSys::default(),
pkg_json_resolver: self.pkg_json_resolver.clone(),
},
)
}
CliNpmResolver::Managed(managed_npm_resolver) => {
CliNpmResolverCreateOptions::Managed({
let npmrc = self
.config_data
.as_ref()
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc);
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&CliSys::default(),
managed_npm_resolver.global_cache_root_path().to_path_buf(),
npmrc.get_all_known_registries_urls(),
));
CliManagedNpmResolverCreateOptions {
sys: CliSys::default(),
npm_cache_dir,
maybe_node_modules_path: managed_npm_resolver
.root_node_modules_path()
.map(|p| p.to_path_buf()),
npmrc,
npm_resolution: factory.services.npm_resolution.clone(),
npm_system_info: NpmSystemInfo::default(),
}
})
}
},
));
}
Arc::new(Self {
resolver: factory.cli_resolver().clone(),
in_npm_pkg_checker: factory.in_npm_pkg_checker().clone(),
is_cjs_resolver: factory.is_cjs_resolver().clone(),
jsr_resolver: self.jsr_resolver.clone(),
npm_graph_resolver: factory.npm_graph_resolver().clone(),
// npm installer isn't necessary for a snapshot
npm_installer: None,
npm_pkg_req_resolver: factory.npm_pkg_req_resolver().cloned(),
npm_resolution: factory.services.npm_resolution.clone(),
npm_resolver: factory.npm_resolver().cloned(),
node_resolver: factory.node_resolver().cloned(),
redirect_resolver: self.redirect_resolver.clone(),
@ -318,14 +385,12 @@ impl LspResolver {
if let Some(dep_info) = dep_info {
*resolver.dep_info.lock() = dep_info.clone();
}
if let Some(npm_resolver) = resolver.npm_resolver.as_ref() {
if let Some(npm_resolver) = npm_resolver.as_managed() {
let reqs = dep_info
.map(|i| i.npm_reqs.iter().cloned().collect::<Vec<_>>())
.unwrap_or_default();
if let Err(err) = npm_resolver.set_package_reqs(&reqs).await {
lsp_warn!("Could not set npm package requirements: {:#}", err);
}
if let Some(npm_installer) = resolver.npm_installer.as_ref() {
let reqs = dep_info
.map(|i| i.npm_reqs.iter().cloned().collect::<Vec<_>>())
.unwrap_or_default();
if let Err(err) = npm_installer.set_package_reqs(&reqs).await {
lsp_warn!("Could not set npm package requirements: {:#}", err);
}
}
}
@ -339,20 +404,18 @@ impl LspResolver {
resolver.resolver.as_ref()
}
pub fn create_graph_npm_resolver(
pub fn as_graph_npm_resolver(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> WorkerCliNpmGraphResolver {
) -> &Arc<CliNpmGraphResolver> {
let resolver = self.get_scope_resolver(file_referrer);
resolver
.resolver
.create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager)
&resolver.npm_graph_resolver
}
pub fn as_is_cjs_resolver(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> &IsCjsResolver {
) -> &CliIsCjsResolver {
let resolver = self.get_scope_resolver(file_referrer);
resolver.is_cjs_resolver.as_ref()
}
@ -368,7 +431,7 @@ impl LspResolver {
pub fn in_npm_pkg_checker(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> &Arc<dyn InNpmPackageChecker> {
) -> &DenoInNpmPackageChecker {
let resolver = self.get_scope_resolver(file_referrer);
&resolver.in_npm_pkg_checker
}
@ -376,7 +439,7 @@ impl LspResolver {
pub fn maybe_managed_npm_resolver(
&self,
file_referrer: Option<&ModuleSpecifier>,
) -> Option<&ManagedCliNpmResolver> {
) -> Option<&CliManagedNpmResolver> {
let resolver = self.get_scope_resolver(file_referrer);
resolver.npm_resolver.as_ref().and_then(|r| r.as_managed())
}
@ -590,11 +653,15 @@ pub struct ScopeDepInfo {
#[derive(Default)]
struct ResolverFactoryServices {
cli_resolver: Deferred<Arc<CliResolver>>,
in_npm_pkg_checker: Deferred<Arc<dyn InNpmPackageChecker>>,
is_cjs_resolver: Deferred<Arc<IsCjsResolver>>,
found_pkg_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
in_npm_pkg_checker: Deferred<DenoInNpmPackageChecker>,
is_cjs_resolver: Deferred<Arc<CliIsCjsResolver>>,
node_resolver: Deferred<Option<Arc<CliNodeResolver>>>,
npm_graph_resolver: Deferred<Arc<CliNpmGraphResolver>>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_pkg_req_resolver: Deferred<Option<Arc<CliNpmReqResolver>>>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
npm_resolver: Option<CliNpmResolver>,
npm_resolution: Arc<NpmResolutionCell>,
}
struct ResolverFactory<'a> {
@ -616,6 +683,10 @@ impl<'a> ResolverFactory<'a> {
}
}
// todo(dsherret): probably this method could be removed in the future
// and instead just `npm_resolution_initializer.ensure_initialized()` could
// be called. The reason this exists is because creating the npm resolvers
// used to be async.
async fn init_npm_resolver(
&mut self,
http_client_provider: &Arc<HttpClientProvider>,
@ -645,11 +716,30 @@ impl<'a> ResolverFactory<'a> {
cache.deno_dir().npm_folder_path(),
npmrc.get_all_known_registries_urls(),
));
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
http_client_provider: http_client_provider.clone(),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) {
let npm_cache = Arc::new(CliNpmCache::new(
npm_cache_dir.clone(),
sys.clone(),
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
// the cache from being filled with lots of packages while
// the user is typing.
deno_npm_cache::NpmCacheSetting::Only,
npmrc.clone(),
));
let pb = ProgressBar::new(ProgressBarStyle::TextOnly);
let npm_client = Arc::new(CliNpmCacheHttpClient::new(
http_client_provider.clone(),
pb.clone(),
));
let registry_info_provider = Arc::new(CliNpmRegistryInfoProvider::new(
npm_cache.clone(),
npm_client.clone(),
npmrc.clone(),
));
let npm_resolution_initializer = Arc::new(NpmResolutionInitializer::new(
registry_info_provider.clone(),
self.services.npm_resolution.clone(),
match self.config_data.and_then(|d| d.lockfile.as_ref()) {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
lockfile.clone(),
@ -657,33 +747,69 @@ impl<'a> ResolverFactory<'a> {
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
},
));
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
let maybe_lockfile: Option<Arc<CliLockfile>> = None;
let maybe_node_modules_path =
self.config_data.and_then(|d| d.node_modules_dir.clone());
let tarball_cache = Arc::new(TarballCache::new(
npm_cache.clone(),
npm_client.clone(),
sys.clone(),
npmrc.clone(),
));
let npm_resolution_installer = Arc::new(NpmResolutionInstaller::new(
registry_info_provider,
self.services.npm_resolution.clone(),
maybe_lockfile.clone(),
));
let npm_installer = Arc::new(NpmInstaller::new(
npm_cache.clone(),
Arc::new(NpmInstallDepsProvider::empty()),
self.services.npm_resolution.clone(),
npm_resolution_initializer.clone(),
npm_resolution_installer,
&pb,
sys.clone(),
tarball_cache.clone(),
maybe_lockfile,
maybe_node_modules_path.clone(),
LifecycleScriptsConfig::default(),
NpmSystemInfo::default(),
));
self.set_npm_installer(npm_installer);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
if let Err(err) = npm_resolution_initializer.ensure_initialized().await
{
log::warn!("failed to initialize npm resolution: {}", err);
}
})
.await
.unwrap();
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
sys: CliSys::default(),
npm_cache_dir,
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
// the cache from being filled with lots of packages while
// the user is typing.
cache_setting: CacheSetting::Only,
text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly),
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
maybe_node_modules_path: self
.config_data
.and_then(|d| d.node_modules_dir.clone()),
maybe_node_modules_path,
npmrc,
npm_resolution: self.services.npm_resolution.clone(),
npm_system_info: NpmSystemInfo::default(),
lifecycle_scripts: Default::default(),
})
};
self.set_npm_resolver(create_cli_npm_resolver_for_lsp(options).await);
self.set_npm_resolver(CliNpmResolver::new(options));
}
pub fn set_npm_resolver(&mut self, npm_resolver: Arc<dyn CliNpmResolver>) {
pub fn set_npm_installer(&mut self, npm_installer: Arc<NpmInstaller>) {
self.services.npm_installer = Some(npm_installer);
}
pub fn set_npm_resolver(&mut self, npm_resolver: CliNpmResolver) {
self.services.npm_resolver = Some(npm_resolver);
}
pub fn npm_resolver(&self) -> Option<&Arc<dyn CliNpmResolver>> {
pub fn npm_resolver(&self) -> Option<&CliNpmResolver> {
self.services.npm_resolver.as_ref()
}
@ -720,13 +846,27 @@ impl<'a> ResolverFactory<'a> {
is_byonm: self.config_data.map(|d| d.byonm).unwrap_or(false),
maybe_vendor_dir: self.config_data.and_then(|d| d.vendor_dir.as_ref()),
}));
Arc::new(CliResolver::new(CliResolverOptions {
Arc::new(CliResolver::new(
deno_resolver,
npm_resolver: self.npm_resolver().cloned(),
bare_node_builtins_enabled: self
self.services.found_pkg_json_dep_flag.clone(),
))
})
}
pub fn npm_installer(&self) -> Option<&Arc<NpmInstaller>> {
self.services.npm_installer.as_ref()
}
pub fn npm_graph_resolver(&self) -> &Arc<CliNpmGraphResolver> {
self.services.npm_graph_resolver.get_or_init(|| {
Arc::new(CliNpmGraphResolver::new(
None,
self.services.found_pkg_json_dep_flag.clone(),
self
.config_data
.is_some_and(|d| d.unstable.contains("bare-node-builtins")),
}))
NpmCachingStrategy::Eager,
))
})
}
@ -734,29 +874,27 @@ impl<'a> ResolverFactory<'a> {
&self.pkg_json_resolver
}
pub fn in_npm_pkg_checker(&self) -> &Arc<dyn InNpmPackageChecker> {
pub fn in_npm_pkg_checker(&self) -> &DenoInNpmPackageChecker {
self.services.in_npm_pkg_checker.get_or_init(|| {
deno_resolver::npm::create_in_npm_pkg_checker(
match self.services.npm_resolver.as_ref().map(|r| r.as_inner()) {
Some(crate::npm::InnerCliNpmResolverRef::Byonm(_)) | None => {
CreateInNpmPkgCheckerOptions::Byonm
}
Some(crate::npm::InnerCliNpmResolverRef::Managed(m)) => {
CreateInNpmPkgCheckerOptions::Managed(
ManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: m.global_cache_root_url(),
maybe_node_modules_path: m.maybe_node_modules_path(),
},
)
}
},
)
DenoInNpmPackageChecker::new(match &self.services.npm_resolver {
Some(CliNpmResolver::Byonm(_)) | None => {
CreateInNpmPkgCheckerOptions::Byonm
}
Some(CliNpmResolver::Managed(m)) => {
CreateInNpmPkgCheckerOptions::Managed(
ManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: m.global_cache_root_url(),
maybe_node_modules_path: m.root_node_modules_path(),
},
)
}
})
})
}
pub fn is_cjs_resolver(&self) -> &Arc<IsCjsResolver> {
pub fn is_cjs_resolver(&self) -> &Arc<CliIsCjsResolver> {
self.services.is_cjs_resolver.get_or_init(|| {
Arc::new(IsCjsResolver::new(
Arc::new(CliIsCjsResolver::new(
self.in_npm_pkg_checker().clone(),
self.pkg_json_resolver().clone(),
if self
@ -780,9 +918,10 @@ impl<'a> ResolverFactory<'a> {
Some(Arc::new(CliNodeResolver::new(
self.in_npm_pkg_checker().clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
npm_resolver.clone(),
self.pkg_json_resolver.clone(),
self.sys.clone(),
node_resolver::ConditionsFromResolutionMode::default(),
)))
})
.as_ref()
@ -796,10 +935,9 @@ impl<'a> ResolverFactory<'a> {
let node_resolver = self.node_resolver()?;
let npm_resolver = self.npm_resolver()?;
Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
in_npm_pkg_checker: self.in_npm_pkg_checker().clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
npm_resolver: npm_resolver.clone(),
sys: self.sys.clone(),
})))
})

2
cli/lsp/testing/definitions.rs
View file

@ -5,6 +5,7 @@ use std::collections::HashSet;
use deno_core::error::AnyError;
use deno_core::ModuleSpecifier;
use deno_lib::util::checksum;
use lsp::Range;
use tower_lsp::lsp_types as lsp;
@ -15,7 +16,6 @@ use crate::lsp::logging::lsp_warn;
use crate::lsp::urls::url_to_uri;
use crate::tools::test::TestDescription;
use crate::tools::test::TestStepDescription;
use crate::util::checksum;
#[derive(Debug, Clone, PartialEq)]
pub struct TestDefinition {

76
cli/lsp/tsc.rs
View file

@ -39,6 +39,7 @@ use deno_core::ModuleSpecifier;
use deno_core::OpState;
use deno_core::PollEventLoopOptions;
use deno_core::RuntimeOptions;
use deno_lib::worker::create_isolate_create_params;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES;
use deno_runtime::inspector_server::InspectorServer;
@ -72,6 +73,7 @@ use super::documents::Document;
use super::documents::DocumentsFilter;
use super::language_server;
use super::language_server::StateSnapshot;
use super::logging::lsp_log;
use super::performance::Performance;
use super::performance::PerformanceMark;
use super::refactor::RefactorCodeActionData;
@ -96,7 +98,6 @@ use crate::util::path::relative_specifier;
use crate::util::path::to_percent_decoded_str;
use crate::util::result::InfallibleResultExt;
use crate::util::v8::convert;
use crate::worker::create_isolate_create_params;
static BRACKET_ACCESSOR_RE: Lazy<Regex> =
lazy_regex!(r#"^\[['"](.+)[\['"]\]$"#);
@ -4340,7 +4341,9 @@ impl TscSpecifierMap {
if let Some(specifier) = self.normalized_specifiers.get(original) {
return Ok(specifier.clone());
}
let specifier_str = original.replace(".d.ts.d.ts", ".d.ts");
let specifier_str = original
.replace(".d.ts.d.ts", ".d.ts")
.replace("$node_modules", "node_modules");
let specifier = match ModuleSpecifier::parse(&specifier_str) {
Ok(s) => s,
Err(err) => return Err(err),
@ -4509,11 +4512,12 @@ fn op_release(
#[op2]
#[serde]
#[allow(clippy::type_complexity)]
fn op_resolve(
state: &mut OpState,
#[string] base: String,
#[serde] specifiers: Vec<(bool, String)>,
) -> Result<Vec<Option<(String, String)>>, deno_core::url::ParseError> {
) -> Result<Vec<Option<(String, Option<String>)>>, deno_core::url::ParseError> {
op_resolve_inner(state, ResolveArgs { base, specifiers })
}
@ -4595,10 +4599,11 @@ async fn op_poll_requests(
}
#[inline]
#[allow(clippy::type_complexity)]
fn op_resolve_inner(
state: &mut OpState,
args: ResolveArgs,
) -> Result<Vec<Option<(String, String)>>, deno_core::url::ParseError> {
) -> Result<Vec<Option<(String, Option<String>)>>, deno_core::url::ParseError> {
let state = state.borrow_mut::<State>();
let mark = state.performance.mark_with_args("tsc.op.op_resolve", &args);
let referrer = state.specifier_map.normalize(&args.base)?;
@ -4611,7 +4616,11 @@ fn op_resolve_inner(
o.map(|(s, mt)| {
(
state.specifier_map.denormalize(&s),
mt.as_ts_extension().to_string(),
if matches!(mt, MediaType::Unknown) {
None
} else {
Some(mt.as_ts_extension().to_string())
},
)
})
})
@ -4689,7 +4698,24 @@ fn op_script_names(state: &mut OpState) -> ScriptNames {
.graph_imports_by_referrer(scope)
{
for specifier in specifiers {
script_names.insert(specifier.to_string());
if let Ok(req_ref) =
deno_semver::npm::NpmPackageReqReference::from_specifier(specifier)
{
let Some((resolved, _)) =
state.state_snapshot.resolver.npm_to_file_url(
&req_ref,
scope,
ResolutionMode::Import,
Some(scope),
)
else {
lsp_log!("failed to resolve {req_ref} to file URL");
continue;
};
script_names.insert(resolved.to_string());
} else {
script_names.insert(specifier.to_string());
}
}
}
}
@ -5583,7 +5609,6 @@ mod tests {
})
.to_string(),
temp_dir.url().join("deno.json").unwrap(),
&Default::default(),
)
.unwrap(),
)
@ -6240,7 +6265,40 @@ mod tests {
"kind": "keyword"
}
],
"documentation": []
"documentation": [
{
"text": "Outputs a message to the console",
"kind": "text",
},
],
"tags": [
{
"name": "param",
"text": [
{
"text": "data",
"kind": "parameterName",
},
{
"text": " ",
"kind": "space",
},
{
"text": "Values to be printed to the console",
"kind": "text",
},
],
},
{
"name": "example",
"text": [
{
"text": "```ts\nconsole.log('Hello', 'World', 123);\n```",
"kind": "text",
},
],
},
]
})
);
}
@ -6461,7 +6519,7 @@ mod tests {
resolved,
vec![Some((
temp_dir.url().join("b.ts").unwrap().to_string(),
MediaType::TypeScript.as_ts_extension().to_string()
Some(MediaType::TypeScript.as_ts_extension().to_string())
))]
);
}

25
cli/lsp/urls.rs
View file

@ -81,7 +81,7 @@ fn hash_data_specifier(specifier: &ModuleSpecifier) -> String {
file_name_str.push('?');
file_name_str.push_str(query);
}
crate::util::checksum::gen(&[file_name_str.as_bytes()])
deno_lib::util::checksum::gen(&[file_name_str.as_bytes()])
}
fn to_deno_uri(specifier: &Url) -> String {
@ -282,24 +282,26 @@ impl LspUrlMap {
}
}
/// Convert a e.g. `deno-notebook-cell:` specifier to a `file:` specifier.
/// Convert a e.g. `vscode-notebook-cell:` specifier to a `file:` specifier.
/// ```rust
/// assert_eq!(
/// file_like_to_file_specifier(
/// &Url::parse("deno-notebook-cell:/path/to/file.ipynb#abc").unwrap(),
/// &Url::parse("vscode-notebook-cell:/path/to/file.ipynb#abc").unwrap(),
/// ),
/// Some(Url::parse("file:///path/to/file.ipynb.ts?scheme=deno-notebook-cell#abc").unwrap()),
/// Some(Url::parse("file:///path/to/file.ipynb?scheme=untitled#abc").unwrap()),
/// );
fn file_like_to_file_specifier(specifier: &Url) -> Option<Url> {
if matches!(specifier.scheme(), "untitled" | "deno-notebook-cell") {
if matches!(
specifier.scheme(),
"untitled" | "vscode-notebook-cell" | "deno-notebook-cell"
) {
if let Ok(mut s) = ModuleSpecifier::parse(&format!(
"file://{}",
"file:///{}",
&specifier.as_str()[deno_core::url::quirks::internal_components(specifier)
.host_end as usize..],
.host_end as usize..].trim_start_matches('/'),
)) {
s.query_pairs_mut()
.append_pair("scheme", specifier.scheme());
s.set_path(&format!("{}.ts", s.path()));
return Some(s);
}
}
@ -432,11 +434,11 @@ mod tests {
fn test_file_like_to_file_specifier() {
assert_eq!(
file_like_to_file_specifier(
&Url::parse("deno-notebook-cell:/path/to/file.ipynb#abc").unwrap(),
&Url::parse("vscode-notebook-cell:/path/to/file.ipynb#abc").unwrap(),
),
Some(
Url::parse(
"file:///path/to/file.ipynb.ts?scheme=deno-notebook-cell#abc"
"file:///path/to/file.ipynb?scheme=vscode-notebook-cell#abc"
)
.unwrap()
),
@ -446,8 +448,7 @@ mod tests {
&Url::parse("untitled:/path/to/file.ipynb#123").unwrap(),
),
Some(
Url::parse("file:///path/to/file.ipynb.ts?scheme=untitled#123")
.unwrap()
Url::parse("file:///path/to/file.ipynb?scheme=untitled#123").unwrap()
),
);
}

14
cli/main.rs
View file

@ -40,7 +40,6 @@ use deno_core::error::AnyError;
use deno_core::error::CoreError;
use deno_core::futures::FutureExt;
use deno_core::unsync::JoinHandle;
use deno_npm::resolution::SnapshotFromLockfileError;
use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::fmt_errors::format_js_error;
@ -52,6 +51,7 @@ use factory::CliFactory;
use standalone::MODULE_NOT_FOUND;
use standalone::UNSUPPORTED_SCHEME;
use self::npm::ResolveSnapshotError;
use crate::args::flags_from_vec;
use crate::args::DenoSubcommand;
use crate::args::Flags;
@ -201,7 +201,7 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
match result {
Ok(v) => Ok(v),
Err(script_err) => {
if let Some(ResolvePkgFolderFromDenoReqError::Byonm(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq(_))) = util::result::any_and_jserrorbox_downcast_ref::<ResolvePkgFolderFromDenoReqError>(&script_err) {
if let Some(worker::CreateCustomWorkerError::ResolvePkgFolderFromDenoReq(ResolvePkgFolderFromDenoReqError::Byonm(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq(_)))) = util::result::any_and_jserrorbox_downcast_ref::<worker::CreateCustomWorkerError>(&script_err) {
if flags.node_modules_dir.is_none() {
let mut flags = flags.deref().clone();
let watch = match &flags.subcommand {
@ -376,13 +376,15 @@ fn exit_for_error(error: AnyError) -> ! {
util::result::any_and_jserrorbox_downcast_ref::<CoreError>(&error)
{
error_string = format_js_error(e);
} else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) =
util::result::any_and_jserrorbox_downcast_ref::<SnapshotFromLockfileError>(
} else if let Some(e @ ResolveSnapshotError { .. }) =
util::result::any_and_jserrorbox_downcast_ref::<ResolveSnapshotError>(
&error,
)
{
error_string = e.to_string();
error_code = 10;
if let Some(e) = e.maybe_integrity_check_error() {
error_string = e.to_string();
error_code = 10;
}
}
exit_with_message(&error_string, error_code);

128
cli/module_loader.rs
View file

@ -13,8 +13,6 @@ use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::error::ModuleLoaderError;
use deno_core::futures::future::FutureExt;
@ -39,8 +37,13 @@ use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::Resolution;
use deno_graph::WasmModule;
use deno_lib::npm::NpmRegistryReadPermissionChecker;
use deno_lib::worker::CreateModuleLoaderResult;
use deno_lib::worker::ModuleLoaderFactory;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_runtime::code_cache;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::ops::require::UnableToGetCwdError;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageReqReference;
@ -69,8 +72,7 @@ use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeCodeTranslator;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::npm::NpmRegistryReadPermissionChecker;
use crate::resolver::CjsTracker;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
@ -83,8 +85,6 @@ use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum PrepareModuleLoadError {
@ -98,6 +98,11 @@ pub enum PrepareModuleLoadError {
Check(#[from] CheckError),
#[class(inherit)]
#[error(transparent)]
AtomicWriteFileWithRetries(
#[from] crate::args::AtomicWriteFileWithRetriesError,
),
#[class(inherit)]
#[error(transparent)]
Other(#[from] JsErrorBox),
}
@ -233,18 +238,19 @@ struct SharedCliModuleLoaderState {
initial_cwd: PathBuf,
is_inspecting: bool,
is_repl: bool,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_module_loader: NpmModuleLoader,
npm_registry_permission_checker: Arc<NpmRegistryReadPermissionChecker>,
npm_registry_permission_checker:
Arc<NpmRegistryReadPermissionChecker<CliSys>>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: CliSys,
@ -294,18 +300,20 @@ impl CliModuleLoaderFactory {
#[allow(clippy::too_many_arguments)]
pub fn new(
options: &CliOptions,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_module_loader: NpmModuleLoader,
npm_registry_permission_checker: Arc<NpmRegistryReadPermissionChecker>,
npm_registry_permission_checker: Arc<
NpmRegistryReadPermissionChecker<CliSys>,
>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: CliSys,
@ -415,6 +423,55 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory {
}
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum LoadCodeSourceError {
#[class(inherit)]
#[error(transparent)]
NpmModuleLoad(crate::resolver::NpmModuleLoadError),
#[class(inherit)]
#[error(transparent)]
LoadPreparedModule(#[from] LoadPreparedModuleError),
#[class(generic)]
#[error("Loading unprepared module: {}{}", .specifier, .maybe_referrer.as_ref().map(|r| format!(", imported from: {}", r)).unwrap_or_default())]
LoadUnpreparedModule {
specifier: ModuleSpecifier,
maybe_referrer: Option<ModuleSpecifier>,
},
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum LoadPreparedModuleError {
#[class(inherit)]
#[error(transparent)]
NpmModuleLoad(#[from] crate::emit::EmitParsedSourceHelperError),
#[class(inherit)]
#[error(transparent)]
LoadMaybeCjs(#[from] LoadMaybeCjsError),
#[class(inherit)]
#[error(transparent)]
Other(#[from] JsErrorBox),
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum LoadMaybeCjsError {
#[class(inherit)]
#[error(transparent)]
NpmModuleLoad(#[from] crate::emit::EmitParsedSourceHelperError),
#[class(inherit)]
#[error(transparent)]
TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError),
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
#[class(inherit)]
#[error("Could not resolve '{reference}'")]
pub struct CouldNotResolveError {
reference: deno_semver::npm::NpmPackageNvReference,
#[source]
#[inherit]
source: node_resolver::errors::PackageSubpathResolveError,
}
struct CliModuleLoaderInner<TGraphContainer: ModuleGraphContainer> {
lib: TsTypeLib,
is_worker: bool,
@ -439,7 +496,10 @@ impl<TGraphContainer: ModuleGraphContainer>
maybe_referrer: Option<&ModuleSpecifier>,
requested_module_type: RequestedModuleType,
) -> Result<ModuleSource, ModuleLoaderError> {
let code_source = self.load_code_source(specifier, maybe_referrer).await?;
let code_source = self
.load_code_source(specifier, maybe_referrer)
.await
.map_err(JsErrorBox::from_err)?;
let code = if self.shared.is_inspecting
|| code_source.media_type == MediaType::Wasm
{
@ -500,7 +560,7 @@ impl<TGraphContainer: ModuleGraphContainer>
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<ModuleCodeStringSource, AnyError> {
) -> Result<ModuleCodeStringSource, LoadCodeSourceError> {
if let Some(code_source) = self.load_prepared_module(specifier).await? {
return Ok(code_source);
}
@ -509,14 +569,14 @@ impl<TGraphContainer: ModuleGraphContainer>
.shared
.npm_module_loader
.load(specifier, maybe_referrer)
.await;
.await
.map_err(LoadCodeSourceError::NpmModuleLoad);
}
let mut msg = format!("Loading unprepared module: {specifier}");
if let Some(referrer) = maybe_referrer {
msg = format!("{}, imported from: {}", msg, referrer.as_str());
}
Err(anyhow!(msg))
Err(LoadCodeSourceError::LoadUnpreparedModule {
specifier: specifier.clone(),
maybe_referrer: maybe_referrer.cloned(),
})
}
fn resolve_referrer(
@ -539,7 +599,8 @@ impl<TGraphContainer: ModuleGraphContainer>
.map_err(|e| e.into())
} else {
// this cwd check is slow, so try to avoid it
let cwd = std::env::current_dir().context("Unable to get CWD")?;
let cwd = std::env::current_dir()
.map_err(|e| JsErrorBox::from_err(UnableToGetCwdError(e)))?;
deno_core::resolve_path(referrer, &cwd).map_err(|e| e.into())
}
}
@ -618,8 +679,11 @@ impl<TGraphContainer: ModuleGraphContainer>
ResolutionMode::Import,
NodeResolutionKind::Execution,
)
.with_context(|| {
format!("Could not resolve '{}'.", module.nv_reference)
.map_err(|source| {
JsErrorBox::from_err(CouldNotResolveError {
reference: module.nv_reference.clone(),
source,
})
})?
}
Some(Module::Node(module)) => module.specifier.clone(),
@ -640,7 +704,7 @@ impl<TGraphContainer: ModuleGraphContainer>
async fn load_prepared_module(
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<ModuleCodeStringSource>, AnyError> {
) -> Result<Option<ModuleCodeStringSource>, LoadPreparedModuleError> {
// Note: keep this in sync with the sync version below
let graph = self.graph_container.graph();
match self.load_prepared_module_or_defer_emit(&graph, specifier)? {
@ -672,7 +736,8 @@ impl<TGraphContainer: ModuleGraphContainer>
}) => self
.load_maybe_cjs(specifier, media_type, source)
.await
.map(Some),
.map(Some)
.map_err(LoadPreparedModuleError::LoadMaybeCjs),
None => Ok(None),
}
}
@ -833,7 +898,7 @@ impl<TGraphContainer: ModuleGraphContainer>
specifier: &ModuleSpecifier,
media_type: MediaType,
original_source: &Arc<str>,
) -> Result<ModuleCodeStringSource, AnyError> {
) -> Result<ModuleCodeStringSource, LoadMaybeCjsError> {
let js_source = if media_type.is_emittable() {
Cow::Owned(
self
@ -1139,12 +1204,13 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
#[derive(Debug)]
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
emitter: Arc<Emitter>,
sys: CliSys,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_registry_permission_checker: Arc<NpmRegistryReadPermissionChecker>,
in_npm_pkg_checker: DenoInNpmPackageChecker,
npm_registry_permission_checker:
Arc<NpmRegistryReadPermissionChecker<CliSys>>,
}
impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader

33
cli/node.rs
View file

@ -5,12 +5,14 @@ use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_error::JsErrorBox;
use deno_graph::ParsedSourceStore;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis;
use node_resolver::analyze::CjsAnalysisExports;
use node_resolver::analyze::CjsCodeAnalysisError;
use node_resolver::analyze::CjsCodeAnalyzer;
use node_resolver::analyze::NodeCodeTranslator;
use serde::Deserialize;
@ -19,15 +21,22 @@ use serde::Serialize;
use crate::cache::CacheDBHash;
use crate::cache::NodeAnalysisCache;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use crate::npm::CliNpmResolver;
use crate::resolver::CliCjsTracker;
use crate::sys::CliSys;
pub type CliNodeCodeTranslator = NodeCodeTranslator<
CliCjsCodeAnalyzer,
DenoInNpmPackageChecker,
RealIsBuiltInNodeModuleChecker,
CliNpmResolver,
CliSys,
>;
pub type CliNodeResolver = deno_runtime::deno_node::NodeResolver<
DenoInNpmPackageChecker,
CliNpmResolver,
CliSys,
>;
pub type CliNodeResolver = deno_runtime::deno_node::NodeResolver<CliSys>;
pub type CliPackageJsonResolver = node_resolver::PackageJsonResolver<CliSys>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@ -43,7 +52,7 @@ pub enum CliCjsAnalysis {
pub struct CliCjsCodeAnalyzer {
cache: NodeAnalysisCache,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
fs: deno_fs::FileSystemRc,
parsed_source_cache: Option<Arc<ParsedSourceCache>>,
}
@ -51,7 +60,7 @@ pub struct CliCjsCodeAnalyzer {
impl CliCjsCodeAnalyzer {
pub fn new(
cache: NodeAnalysisCache,
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
fs: deno_fs::FileSystemRc,
parsed_source_cache: Option<Arc<ParsedSourceCache>>,
) -> Self {
@ -67,7 +76,7 @@ impl CliCjsCodeAnalyzer {
&self,
specifier: &ModuleSpecifier,
source: &str,
) -> Result<CliCjsAnalysis, AnyError> {
) -> Result<CliCjsAnalysis, CjsCodeAnalysisError> {
let source_hash = CacheDBHash::from_hashable(source);
if let Some(analysis) =
self.cache.get_cjs_analysis(specifier.as_str(), source_hash)
@ -94,9 +103,10 @@ impl CliCjsCodeAnalyzer {
deno_core::unsync::spawn_blocking({
let specifier = specifier.clone();
let source: Arc<str> = source.into();
move || -> Result<_, AnyError> {
let parsed_source =
maybe_parsed_source.map(Ok).unwrap_or_else(|| {
move || -> Result<_, CjsCodeAnalysisError> {
let parsed_source = maybe_parsed_source
.map(Ok)
.unwrap_or_else(|| {
deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text: source,
@ -105,7 +115,8 @@ impl CliCjsCodeAnalyzer {
scope_analysis: false,
maybe_syntax: None,
})
})?;
})
.map_err(JsErrorBox::from_err)?;
let is_script = parsed_source.compute_is_script();
let is_cjs = cjs_tracker.is_cjs_with_known_is_script(
parsed_source.specifier(),
@ -143,7 +154,7 @@ impl CjsCodeAnalyzer for CliCjsCodeAnalyzer {
&self,
specifier: &ModuleSpecifier,
source: Option<Cow<'a, str>>,
) -> Result<ExtNodeCjsAnalysis<'a>, AnyError> {
) -> Result<ExtNodeCjsAnalysis<'a>, CjsCodeAnalysisError> {
let source = match source {
Some(source) => source,
None => {

54
cli/npm/byonm.rs
View file

@ -1,17 +1,12 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::Path;
use std::sync::Arc;
use deno_core::serde_json;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::ops::process::NpmProcessStateProvider;
use node_resolver::NpmPackageFolderResolver;
use deno_runtime::deno_process::NpmProcessStateProvider;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::sys::CliSys;
@ -20,59 +15,18 @@ pub type CliByonmNpmResolverCreateOptions =
ByonmNpmResolverCreateOptions<CliSys>;
pub type CliByonmNpmResolver = ByonmNpmResolver<CliSys>;
// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
#[derive(Debug)]
struct CliByonmWrapper(Arc<CliByonmNpmResolver>);
pub struct CliByonmNpmProcessStateProvider(pub Arc<CliByonmNpmResolver>);
impl NpmProcessStateProvider for CliByonmWrapper {
impl NpmProcessStateProvider for CliByonmNpmProcessStateProvider {
fn get_npm_process_state(&self) -> String {
serde_json::to_string(&NpmProcessState {
kind: NpmProcessStateKind::Byonm,
local_node_modules_path: self
.0
.root_node_modules_dir()
.root_node_modules_path()
.map(|p| p.to_string_lossy().to_string()),
})
.unwrap()
}
}
impl CliNpmResolver for CliByonmNpmResolver {
fn into_npm_pkg_folder_resolver(
self: Arc<Self>,
) -> Arc<dyn NpmPackageFolderResolver> {
self
}
fn into_npm_req_resolver(self: Arc<Self>) -> Arc<dyn CliNpmReqResolver> {
self
}
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider> {
Arc::new(CliByonmWrapper(self))
}
fn into_maybe_byonm(self: Arc<Self>) -> Option<Arc<CliByonmNpmResolver>> {
Some(self)
}
fn clone_snapshotted(&self) -> Arc<dyn CliNpmResolver> {
Arc::new(self.clone())
}
fn as_inner(&self) -> InnerCliNpmResolverRef {
InnerCliNpmResolverRef::Byonm(self)
}
fn root_node_modules_path(&self) -> Option<&Path> {
self.root_node_modules_dir()
}
fn check_state_hash(&self) -> Option<u64> {
// it is very difficult to determine the check state hash for byonm
// so we just return None to signify check caching is not supported
None
}
}

3
cli/npm/managed/installers/common/bin_entries.rs → cli/npm/installer/common/bin_entries.rs
View file

@ -8,8 +8,7 @@ use std::path::PathBuf;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use crate::npm::managed::NpmResolutionPackage;
use deno_npm::NpmResolutionPackage;
#[derive(Default)]
pub struct BinEntries<'a> {

4
cli/npm/managed/installers/common/lifecycle_scripts.rs → cli/npm/installer/common/lifecycle_scripts.rs
View file

@ -240,7 +240,7 @@ impl<'a> LifecycleScripts<'a> {
// However, if we concurrently run scripts in the future we will
// have to have multiple temp files.
let temp_file_fd =
deno_runtime::ops::process::npm_process_state_tempfile(
deno_runtime::deno_process::npm_process_state_tempfile(
process_state.as_bytes(),
)
.map_err(LifecycleScriptsError::CreateNpmProcessState)?;
@ -248,7 +248,7 @@ impl<'a> LifecycleScripts<'a> {
let _temp_file =
unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) }; // make sure the file gets closed
env_vars.insert(
deno_runtime::ops::process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME
deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME
.to_string(),
(temp_file_fd as usize).to_string(),
);

4
cli/npm/managed/installers/common/mod.rs → cli/npm/installer/common/mod.rs
View file

@ -3,14 +3,14 @@
use async_trait::async_trait;
use deno_error::JsErrorBox;
use crate::npm::PackageCaching;
use super::PackageCaching;
pub mod bin_entries;
pub mod lifecycle_scripts;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
pub trait NpmPackageFsInstaller: Send + Sync {
pub trait NpmPackageFsInstaller: std::fmt::Debug + Send + Sync {
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,

14
cli/npm/managed/installers/global.rs → cli/npm/installer/global.rs
View file

@ -11,14 +11,14 @@ use deno_core::futures::StreamExt;
use deno_error::JsErrorBox;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_resolver::npm::managed::NpmResolution;
use deno_resolver::npm::managed::NpmResolutionCell;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
use super::common::NpmPackageFsInstaller;
use super::PackageCaching;
use crate::args::LifecycleScriptsConfig;
use crate::cache::FastInsecureHasher;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
@ -27,25 +27,25 @@ use crate::npm::CliNpmTarballCache;
pub struct GlobalNpmPackageInstaller {
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
resolution: Arc<NpmResolutionCell>,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
}
impl GlobalNpmPackageInstaller {
pub fn new(
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
resolution: Arc<NpmResolutionCell>,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
) -> Self {
Self {
cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
system_info,
}
}
}

14
cli/npm/managed/installers/local.rs → cli/npm/installer/local.rs
View file

@ -25,7 +25,7 @@ use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_resolver::npm::get_package_folder_id_folder_name;
use deno_resolver::npm::managed::NpmResolution;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_semver::package::PackageNv;
use deno_semver::StackString;
use serde::Deserialize;
@ -33,11 +33,11 @@ use serde::Serialize;
use super::common::bin_entries;
use super::common::NpmPackageFsInstaller;
use super::PackageCaching;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
@ -54,12 +54,12 @@ pub struct LocalNpmPackageInstaller {
cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
resolution: Arc<NpmResolutionCell>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
lifecycle_scripts: LifecycleScriptsConfig,
root_node_modules_path: PathBuf,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
}
impl LocalNpmPackageInstaller {
@ -68,12 +68,12 @@ impl LocalNpmPackageInstaller {
cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
resolution: Arc<NpmResolutionCell>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
) -> Self {
Self {
cache,
@ -82,9 +82,9 @@ impl LocalNpmPackageInstaller {
resolution,
tarball_cache,
sys,
lifecycle_scripts,
root_node_modules_path: node_modules_folder,
system_info,
lifecycle_scripts,
}
}
}

283
cli/npm/installer/mod.rs Normal file
View file

@ -0,0 +1,283 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::path::PathBuf;
use std::sync::Arc;
use deno_core::error::AnyError;
use deno_core::unsync::sync::AtomicFlag;
use deno_error::JsErrorBox;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_npm::NpmSystemInfo;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_runtime::colors;
use deno_semver::package::PackageReq;
pub use self::common::NpmPackageFsInstaller;
use self::global::GlobalNpmPackageInstaller;
use self::local::LocalNpmPackageInstaller;
pub use self::resolution::AddPkgReqsResult;
pub use self::resolution::NpmResolutionInstaller;
use super::NpmResolutionInitializer;
use crate::args::CliLockfile;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
mod common;
mod global;
mod local;
mod resolution;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PackageCaching<'a> {
Only(Cow<'a, [PackageReq]>),
All,
}
#[derive(Debug)]
pub struct NpmInstaller {
fs_installer: Arc<dyn NpmPackageFsInstaller>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
npm_resolution_initializer: Arc<NpmResolutionInitializer>,
npm_resolution_installer: Arc<NpmResolutionInstaller>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_resolution: Arc<NpmResolutionCell>,
top_level_install_flag: AtomicFlag,
}
impl NpmInstaller {
#[allow(clippy::too_many_arguments)]
pub fn new(
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
npm_resolution: Arc<NpmResolutionCell>,
npm_resolution_initializer: Arc<NpmResolutionInitializer>,
npm_resolution_installer: Arc<NpmResolutionInstaller>,
progress_bar: &ProgressBar,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_lockfile: Option<Arc<CliLockfile>>,
maybe_node_modules_path: Option<PathBuf>,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
) -> Self {
let fs_installer: Arc<dyn NpmPackageFsInstaller> =
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageInstaller::new(
npm_cache,
npm_install_deps_provider.clone(),
progress_bar.clone(),
npm_resolution.clone(),
sys,
tarball_cache,
node_modules_folder,
lifecycle_scripts,
system_info,
)),
None => Arc::new(GlobalNpmPackageInstaller::new(
npm_cache,
tarball_cache,
npm_resolution.clone(),
lifecycle_scripts,
system_info,
)),
};
Self {
fs_installer,
npm_install_deps_provider,
npm_resolution,
npm_resolution_initializer,
npm_resolution_installer,
maybe_lockfile,
top_level_install_flag: Default::default(),
}
}
/// Adds package requirements to the resolver and ensures everything is setup.
/// This includes setting up the `node_modules` directory, if applicable.
pub async fn add_and_cache_package_reqs(
&self,
packages: &[PackageReq],
) -> Result<(), JsErrorBox> {
self.npm_resolution_initializer.ensure_initialized().await?;
self
.add_package_reqs_raw(
packages,
Some(PackageCaching::Only(packages.into())),
)
.await
.dependencies_result
}
pub async fn add_package_reqs_no_cache(
&self,
packages: &[PackageReq],
) -> Result<(), JsErrorBox> {
self.npm_resolution_initializer.ensure_initialized().await?;
self
.add_package_reqs_raw(packages, None)
.await
.dependencies_result
}
pub async fn add_package_reqs(
&self,
packages: &[PackageReq],
caching: PackageCaching<'_>,
) -> Result<(), JsErrorBox> {
self
.add_package_reqs_raw(packages, Some(caching))
.await
.dependencies_result
}
pub async fn add_package_reqs_raw<'a>(
&self,
packages: &[PackageReq],
caching: Option<PackageCaching<'a>>,
) -> AddPkgReqsResult {
if packages.is_empty() {
return AddPkgReqsResult {
dependencies_result: Ok(()),
results: vec![],
};
}
#[cfg(debug_assertions)]
self.npm_resolution_initializer.debug_assert_initialized();
let mut result = self
.npm_resolution_installer
.add_package_reqs(packages)
.await;
if result.dependencies_result.is_ok() {
if let Some(lockfile) = self.maybe_lockfile.as_ref() {
result.dependencies_result = lockfile.error_if_changed();
}
}
if result.dependencies_result.is_ok() {
if let Some(caching) = caching {
result.dependencies_result = self.cache_packages(caching).await;
}
}
result
}
/// Sets package requirements to the resolver, removing old requirements and adding new ones.
///
/// This will retrieve and resolve package information, but not cache any package files.
pub async fn set_package_reqs(
&self,
packages: &[PackageReq],
) -> Result<(), AnyError> {
self
.npm_resolution_installer
.set_package_reqs(packages)
.await
}
pub async fn inject_synthetic_types_node_package(
&self,
) -> Result<(), JsErrorBox> {
self.npm_resolution_initializer.ensure_initialized().await?;
let reqs = &[PackageReq::from_str("@types/node").unwrap()];
// add and ensure this isn't added to the lockfile
self
.add_package_reqs(reqs, PackageCaching::Only(reqs.into()))
.await?;
Ok(())
}
pub async fn cache_package_info(
&self,
package_name: &str,
) -> Result<Arc<NpmPackageInfo>, NpmRegistryPackageInfoLoadError> {
self
.npm_resolution_installer
.cache_package_info(package_name)
.await
}
pub async fn cache_packages(
&self,
caching: PackageCaching<'_>,
) -> Result<(), JsErrorBox> {
self.npm_resolution_initializer.ensure_initialized().await?;
self.fs_installer.cache_packages(caching).await
}
pub fn ensure_no_pkg_json_dep_errors(
&self,
) -> Result<(), Box<PackageJsonDepValueParseWithLocationError>> {
for err in self.npm_install_deps_provider.pkg_json_dep_errors() {
match err.source.as_kind() {
deno_package_json::PackageJsonDepValueParseErrorKind::VersionReq(_) => {
return Err(Box::new(err.clone()));
}
deno_package_json::PackageJsonDepValueParseErrorKind::Unsupported {
..
} => {
// only warn for this one
log::warn!(
"{} {}\n at {}",
colors::yellow("Warning"),
err.source,
err.location,
)
}
}
}
Ok(())
}
/// Ensures that the top level `package.json` dependencies are installed.
/// This may set up the `node_modules` directory.
///
/// Returns `true` if the top level packages are already installed. A
/// return value of `false` means that new packages were added to the NPM resolution.
pub async fn ensure_top_level_package_json_install(
&self,
) -> Result<bool, JsErrorBox> {
if !self.top_level_install_flag.raise() {
return Ok(true); // already did this
}
self.npm_resolution_initializer.ensure_initialized().await?;
let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs();
if pkg_json_remote_pkgs.is_empty() {
return Ok(true);
}
// check if something needs resolving before bothering to load all
// the package information (which is slow)
if pkg_json_remote_pkgs.iter().all(|pkg| {
self
.npm_resolution
.resolve_pkg_id_from_pkg_req(&pkg.req)
.is_ok()
}) {
log::debug!(
"All package.json deps resolvable. Skipping top level install."
);
return Ok(true); // everything is already resolvable
}
let pkg_reqs = pkg_json_remote_pkgs
.iter()
.map(|pkg| pkg.req.clone())
.collect::<Vec<_>>();
self.add_package_reqs_no_cache(&pkg_reqs).await?;
Ok(false)
}
}

17
cli/npm/managed/installer.rs → cli/npm/installer/resolution.rs
View file

@ -8,12 +8,14 @@ use deno_core::error::AnyError;
use deno_error::JsErrorBox;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_npm::resolution::AddPkgReqsOptions;
use deno_npm::resolution::NpmResolutionError;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmResolutionPackage;
use deno_resolver::npm::managed::NpmResolution;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
@ -35,9 +37,10 @@ pub struct AddPkgReqsResult {
}
/// Updates the npm resolution with the provided package requirements.
#[derive(Debug)]
pub struct NpmResolutionInstaller {
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
resolution: Arc<NpmResolution>,
resolution: Arc<NpmResolutionCell>,
maybe_lockfile: Option<Arc<CliLockfile>>,
update_queue: TaskQueue,
}
@ -45,7 +48,7 @@ pub struct NpmResolutionInstaller {
impl NpmResolutionInstaller {
pub fn new(
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
resolution: Arc<NpmResolution>,
resolution: Arc<NpmResolutionCell>,
maybe_lockfile: Option<Arc<CliLockfile>>,
) -> Self {
Self {
@ -56,6 +59,14 @@ impl NpmResolutionInstaller {
}
}
pub async fn cache_package_info(
&self,
package_name: &str,
) -> Result<Arc<NpmPackageInfo>, NpmRegistryPackageInfoLoadError> {
// this will internally cache the package information
self.registry_info_provider.package_info(package_name).await
}
pub async fn add_package_reqs(
&self,
package_reqs: &[PackageReq],

233
cli/npm/managed.rs Normal file
View file

@ -0,0 +1,233 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_error::JsError;
use deno_error::JsErrorBox;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_resolver::npm::ManagedNpmResolverRc;
use deno_runtime::deno_process::NpmProcessStateProvider;
use thiserror::Error;
use super::CliNpmRegistryInfoProvider;
use crate::args::CliLockfile;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::sys::CliSys;
pub type CliManagedNpmResolverCreateOptions =
ManagedNpmResolverCreateOptions<CliSys>;
#[derive(Debug, Clone)]
pub enum CliNpmResolverManagedSnapshotOption {
ResolveFromLockfile(Arc<CliLockfile>),
Specified(Option<ValidSerializedNpmResolutionSnapshot>),
}
#[derive(Debug)]
enum SyncState {
Pending(Option<CliNpmResolverManagedSnapshotOption>),
Err(ResolveSnapshotError),
Success,
}
#[derive(Debug)]
pub struct NpmResolutionInitializer {
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_resolution: Arc<NpmResolutionCell>,
queue: tokio::sync::Mutex<()>,
sync_state: Mutex<SyncState>,
}
impl NpmResolutionInitializer {
pub fn new(
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_resolution: Arc<NpmResolutionCell>,
snapshot_option: CliNpmResolverManagedSnapshotOption,
) -> Self {
Self {
npm_registry_info_provider,
npm_resolution,
queue: tokio::sync::Mutex::new(()),
sync_state: Mutex::new(SyncState::Pending(Some(snapshot_option))),
}
}
#[cfg(debug_assertions)]
pub fn debug_assert_initialized(&self) {
if !matches!(*self.sync_state.lock(), SyncState::Success) {
panic!("debug assert: npm resolution must be initialized before calling this code");
}
}
pub async fn ensure_initialized(&self) -> Result<(), JsErrorBox> {
// fast exit if not pending
{
match &*self.sync_state.lock() {
SyncState::Pending(_) => {}
SyncState::Err(err) => return Err(JsErrorBox::from_err(err.clone())),
SyncState::Success => return Ok(()),
}
}
// only allow one task in here at a time
let _guard = self.queue.lock().await;
let snapshot_option = {
let mut sync_state = self.sync_state.lock();
match &mut *sync_state {
SyncState::Pending(snapshot_option) => {
// this should never panic, but if it does it means that a
// previous future was dropped while initialization occurred...
// that should never happen because this is initialized during
// startup
snapshot_option.take().unwrap()
}
// another thread updated the state while we were waiting
SyncState::Err(resolve_snapshot_error) => {
return Err(JsErrorBox::from_err(resolve_snapshot_error.clone()));
}
SyncState::Success => {
return Ok(());
}
}
};
match resolve_snapshot(&self.npm_registry_info_provider, snapshot_option)
.await
{
Ok(maybe_snapshot) => {
if let Some(snapshot) = maybe_snapshot {
self
.npm_resolution
.set_snapshot(NpmResolutionSnapshot::new(snapshot));
}
let mut sync_state = self.sync_state.lock();
*sync_state = SyncState::Success;
Ok(())
}
Err(err) => {
let mut sync_state = self.sync_state.lock();
*sync_state = SyncState::Err(err.clone());
Err(JsErrorBox::from_err(err))
}
}
}
}
#[derive(Debug, Error, Clone, JsError)]
#[error("failed reading lockfile '{}'", lockfile_path.display())]
#[class(inherit)]
pub struct ResolveSnapshotError {
lockfile_path: PathBuf,
#[inherit]
#[source]
source: SnapshotFromLockfileError,
}
impl ResolveSnapshotError {
pub fn maybe_integrity_check_error(
&self,
) -> Option<&deno_npm::resolution::IntegrityCheckFailedError> {
match &self.source {
SnapshotFromLockfileError::SnapshotFromLockfile(
deno_npm::resolution::SnapshotFromLockfileError::IntegrityCheckFailed(
err,
),
) => Some(err),
_ => None,
}
}
}
async fn resolve_snapshot(
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
snapshot: CliNpmResolverManagedSnapshotOption,
) -> Result<Option<ValidSerializedNpmResolutionSnapshot>, ResolveSnapshotError>
{
match snapshot {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(lockfile) => {
if !lockfile.overwrite() {
let snapshot = snapshot_from_lockfile(
lockfile.clone(),
&registry_info_provider.as_npm_registry_api(),
)
.await
.map_err(|source| ResolveSnapshotError {
lockfile_path: lockfile.filename.clone(),
source,
})?;
Ok(Some(snapshot))
} else {
Ok(None)
}
}
CliNpmResolverManagedSnapshotOption::Specified(snapshot) => Ok(snapshot),
}
}
#[derive(Debug, Error, Clone, JsError)]
pub enum SnapshotFromLockfileError {
#[error(transparent)]
#[class(inherit)]
IncompleteError(
#[from] deno_npm::resolution::IncompleteSnapshotFromLockfileError,
),
#[error(transparent)]
#[class(inherit)]
SnapshotFromLockfile(#[from] deno_npm::resolution::SnapshotFromLockfileError),
}
async fn snapshot_from_lockfile(
lockfile: Arc<CliLockfile>,
api: &dyn NpmRegistryApi,
) -> Result<ValidSerializedNpmResolutionSnapshot, SnapshotFromLockfileError> {
let (incomplete_snapshot, skip_integrity_check) = {
let lock = lockfile.lock();
(
deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?,
lock.overwrite,
)
};
let snapshot = deno_npm::resolution::snapshot_from_lockfile(
deno_npm::resolution::SnapshotFromLockfileParams {
incomplete_snapshot,
api,
skip_integrity_check,
},
)
.await?;
Ok(snapshot)
}
pub fn npm_process_state(
snapshot: ValidSerializedNpmResolutionSnapshot,
node_modules_path: Option<&Path>,
) -> String {
serde_json::to_string(&NpmProcessState {
kind: NpmProcessStateKind::Snapshot(snapshot.into_serialized()),
local_node_modules_path: node_modules_path
.map(|p| p.to_string_lossy().to_string()),
})
.unwrap()
}
#[derive(Debug)]
pub struct CliManagedNpmProcessStateProvider(pub ManagedNpmResolverRc<CliSys>);
impl NpmProcessStateProvider for CliManagedNpmProcessStateProvider {
fn get_npm_process_state(&self) -> String {
npm_process_state(
self.0.resolution().serialized_valid_snapshot(),
self.0.root_node_modules_path(),
)
}
}

55
cli/npm/managed/installers/mod.rs
View file

@ -1,55 +0,0 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::PathBuf;
use std::sync::Arc;
use deno_npm::NpmSystemInfo;
use deno_resolver::npm::managed::NpmResolution;
pub use self::common::NpmPackageFsInstaller;
use self::global::GlobalNpmPackageInstaller;
use self::local::LocalNpmPackageInstaller;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
mod common;
mod global;
mod local;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_installer(
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Arc<dyn NpmPackageFsInstaller> {
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageInstaller::new(
npm_cache,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
lifecycle_scripts,
)),
None => Arc::new(GlobalNpmPackageInstaller::new(
npm_cache,
tarball_cache,
resolution,
system_info,
lifecycle_scripts,
)),
}
}

804
cli/npm/managed/mod.rs
View file

@ -1,804 +0,0 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsErrorBox;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::PackageReqNotFoundError;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::managed::create_npm_fs_resolver;
use deno_resolver::npm::managed::NpmPackageFsResolver;
use deno_resolver::npm::managed::NpmPackageFsResolverPackageFolderError;
use deno_resolver::npm::managed::NpmResolution;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::colors;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use installer::AddPkgReqsResult;
use installer::NpmResolutionInstaller;
use installers::create_npm_fs_installer;
use installers::NpmPackageFsInstaller;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::NpmPackageFolderResolver;
use super::CliNpmCache;
use super::CliNpmCacheHttpClient;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
use super::InnerCliNpmResolverRef;
use super::ResolvePkgFolderFromDenoReqError;
use crate::args::CliLockfile;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
mod installer;
mod installers;
pub enum CliNpmResolverManagedSnapshotOption {
ResolveFromLockfile(Arc<CliLockfile>),
Specified(Option<ValidSerializedNpmResolutionSnapshot>),
}
pub struct CliManagedNpmResolverCreateOptions {
pub snapshot: CliNpmResolverManagedSnapshotOption,
pub maybe_lockfile: Option<Arc<CliLockfile>>,
pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
pub npm_cache_dir: Arc<NpmCacheDir>,
pub sys: CliSys,
pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting,
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
pub npm_system_info: NpmSystemInfo,
pub npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
pub npmrc: Arc<ResolvedNpmRc>,
pub lifecycle_scripts: LifecycleScriptsConfig,
}
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let npm_api = create_api(npm_cache.clone(), http_client.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
Ok(snapshot) => snapshot,
Err(err) => {
log::warn!("failed to resolve snapshot: {}", err);
None
}
};
create_inner(
http_client,
npm_cache,
options.npm_cache_dir,
options.npm_install_deps_provider,
npm_api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
options.lifecycle_scripts,
)
})
.await
.unwrap()
}
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let api = create_api(npm_cache.clone(), http_client.clone(), &options);
let snapshot = resolve_snapshot(&api, options.snapshot).await?;
Ok(create_inner(
http_client,
npm_cache,
options.npm_cache_dir,
options.npm_install_deps_provider,
api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
options.lifecycle_scripts,
))
}
#[allow(clippy::too_many_arguments)]
fn create_inner(
http_client: Arc<CliNpmCacheHttpClient>,
npm_cache: Arc<CliNpmCache>,
npm_cache_dir: Arc<NpmCacheDir>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
sys: CliSys,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_rc: Arc<ResolvedNpmRc>,
node_modules_dir_path: Option<PathBuf>,
npm_system_info: NpmSystemInfo,
snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Arc<dyn CliNpmResolver> {
let resolution = Arc::new(NpmResolution::from_serialized(snapshot));
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
http_client,
sys.clone(),
npm_rc.clone(),
));
let fs_installer = create_npm_fs_installer(
npm_cache.clone(),
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
sys.clone(),
tarball_cache.clone(),
node_modules_dir_path.clone(),
npm_system_info.clone(),
lifecycle_scripts.clone(),
);
let fs_resolver = create_npm_fs_resolver(
&npm_cache_dir,
&npm_rc,
resolution.clone(),
sys.clone(),
node_modules_dir_path,
);
Arc::new(ManagedCliNpmResolver::new(
fs_installer,
fs_resolver,
maybe_lockfile,
registry_info_provider,
npm_cache,
npm_cache_dir,
npm_install_deps_provider,
npm_rc,
resolution,
sys,
tarball_cache,
text_only_progress_bar,
npm_system_info,
lifecycle_scripts,
))
}
fn create_cache(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.sys.clone(),
NpmCacheSetting::from_cache_setting(&options.cache_setting),
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
http_client: Arc<CliNpmCacheHttpClient>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmRegistryInfoProvider> {
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
http_client,
options.npmrc.clone(),
))
}
async fn resolve_snapshot(
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
snapshot: CliNpmResolverManagedSnapshotOption,
) -> Result<Option<ValidSerializedNpmResolutionSnapshot>, AnyError> {
match snapshot {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(lockfile) => {
if !lockfile.overwrite() {
let snapshot = snapshot_from_lockfile(
lockfile.clone(),
&registry_info_provider.as_npm_registry_api(),
)
.await
.with_context(|| {
format!("failed reading lockfile '{}'", lockfile.filename.display())
})?;
Ok(Some(snapshot))
} else {
Ok(None)
}
}
CliNpmResolverManagedSnapshotOption::Specified(snapshot) => Ok(snapshot),
}
}
async fn snapshot_from_lockfile(
lockfile: Arc<CliLockfile>,
api: &dyn NpmRegistryApi,
) -> Result<ValidSerializedNpmResolutionSnapshot, AnyError> {
let (incomplete_snapshot, skip_integrity_check) = {
let lock = lockfile.lock();
(
deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?,
lock.overwrite,
)
};
let snapshot = deno_npm::resolution::snapshot_from_lockfile(
deno_npm::resolution::SnapshotFromLockfileParams {
incomplete_snapshot,
api,
skip_integrity_check,
},
)
.await?;
Ok(snapshot)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PackageCaching<'a> {
Only(Cow<'a, [PackageReq]>),
All,
}
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
fs_installer: Arc<dyn NpmPackageFsInstaller>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_cache_dir: Arc<NpmCacheDir>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
npm_rc: Arc<ResolvedNpmRc>,
sys: CliSys,
resolution: Arc<NpmResolution>,
resolution_installer: NpmResolutionInstaller,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
top_level_install_flag: AtomicFlag,
lifecycle_scripts: LifecycleScriptsConfig,
}
impl std::fmt::Debug for ManagedCliNpmResolver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ManagedNpmResolver")
.field("<omitted>", &"<omitted>")
.finish()
}
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum ResolvePkgFolderFromPkgIdError {
#[class(inherit)]
#[error("{0}")]
NpmPackageFsResolverPackageFolder(
#[from] NpmPackageFsResolverPackageFolderError,
),
#[class(inherit)]
#[error("{0}")]
Io(#[from] std::io::Error),
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum ResolvePkgFolderFromDenoModuleError {
#[class(inherit)]
#[error("{0}")]
PackageNvNotFound(#[from] deno_npm::resolution::PackageNvNotFoundError),
#[class(inherit)]
#[error("{0}")]
ResolvePkgFolderFromPkgId(#[from] ResolvePkgFolderFromPkgIdError),
}
impl ManagedCliNpmResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
fs_installer: Arc<dyn NpmPackageFsInstaller>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_cache_dir: Arc<NpmCacheDir>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
npm_rc: Arc<ResolvedNpmRc>,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
let resolution_installer = NpmResolutionInstaller::new(
registry_info_provider.clone(),
resolution.clone(),
maybe_lockfile.clone(),
);
Self {
fs_installer,
fs_resolver,
maybe_lockfile,
registry_info_provider,
npm_cache,
npm_cache_dir,
npm_install_deps_provider,
npm_rc,
text_only_progress_bar,
resolution,
resolution_installer,
sys,
tarball_cache,
npm_system_info,
top_level_install_flag: Default::default(),
lifecycle_scripts,
}
}
pub fn resolve_pkg_folder_from_pkg_id(
&self,
pkg_id: &NpmPackageId,
) -> Result<PathBuf, ResolvePkgFolderFromPkgIdError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path = canonicalize_path_maybe_not_exists(&self.sys, &path)?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
path.display()
);
Ok(path)
}
/// Resolves the package id from the provided specifier.
pub fn resolve_pkg_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<Option<NpmPackageId>, AnyError> {
let Some(cache_folder_id) = self
.fs_resolver
.resolve_package_cache_folder_id_from_specifier(specifier)?
else {
return Ok(None);
};
Ok(Some(
self
.resolution
.resolve_pkg_id_from_pkg_cache_folder_id(&cache_folder_id)?,
))
}
pub fn resolve_pkg_reqs_from_pkg_id(
&self,
id: &NpmPackageId,
) -> Vec<PackageReq> {
self.resolution.resolve_pkg_reqs_from_pkg_id(id)
}
/// Attempts to get the package size in bytes.
pub fn package_size(
&self,
package_id: &NpmPackageId,
) -> Result<u64, AnyError> {
let package_folder = self.fs_resolver.package_folder(package_id)?;
Ok(crate::util::fs::dir_size(&package_folder)?)
}
pub fn all_system_packages(
&self,
system_info: &NpmSystemInfo,
) -> Vec<NpmResolutionPackage> {
self.resolution.all_system_packages(system_info)
}
/// Checks if the provided package req's folder is cached.
pub fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool {
self
.resolve_pkg_id_from_pkg_req(req)
.ok()
.and_then(|id| self.fs_resolver.package_folder(&id).ok())
.map(|folder| folder.exists())
.unwrap_or(false)
}
/// Adds package requirements to the resolver and ensures everything is setup.
/// This includes setting up the `node_modules` directory, if applicable.
pub async fn add_and_cache_package_reqs(
&self,
packages: &[PackageReq],
) -> Result<(), JsErrorBox> {
self
.add_package_reqs_raw(
packages,
Some(PackageCaching::Only(packages.into())),
)
.await
.dependencies_result
}
pub async fn add_package_reqs_no_cache(
&self,
packages: &[PackageReq],
) -> Result<(), JsErrorBox> {
self
.add_package_reqs_raw(packages, None)
.await
.dependencies_result
}
pub async fn add_package_reqs(
&self,
packages: &[PackageReq],
caching: PackageCaching<'_>,
) -> Result<(), JsErrorBox> {
self
.add_package_reqs_raw(packages, Some(caching))
.await
.dependencies_result
}
pub async fn add_package_reqs_raw<'a>(
&self,
packages: &[PackageReq],
caching: Option<PackageCaching<'a>>,
) -> AddPkgReqsResult {
if packages.is_empty() {
return AddPkgReqsResult {
dependencies_result: Ok(()),
results: vec![],
};
}
let mut result = self.resolution_installer.add_package_reqs(packages).await;
if result.dependencies_result.is_ok() {
if let Some(lockfile) = self.maybe_lockfile.as_ref() {
result.dependencies_result = lockfile.error_if_changed();
}
}
if result.dependencies_result.is_ok() {
if let Some(caching) = caching {
result.dependencies_result = self.cache_packages(caching).await;
}
}
result
}
/// Sets package requirements to the resolver, removing old requirements and adding new ones.
///
/// This will retrieve and resolve package information, but not cache any package files.
pub async fn set_package_reqs(
&self,
packages: &[PackageReq],
) -> Result<(), AnyError> {
self.resolution_installer.set_package_reqs(packages).await
}
pub fn snapshot(&self) -> NpmResolutionSnapshot {
self.resolution.snapshot()
}
pub fn top_package_req_for_name(&self, name: &str) -> Option<PackageReq> {
let package_reqs = self.resolution.package_reqs();
let mut entries = package_reqs
.iter()
.filter(|(_, nv)| nv.name == name)
.collect::<Vec<_>>();
entries.sort_by_key(|(_, nv)| &nv.version);
Some(entries.last()?.0.clone())
}
pub fn serialized_valid_snapshot_for_system(
&self,
system_info: &NpmSystemInfo,
) -> ValidSerializedNpmResolutionSnapshot {
self
.resolution
.serialized_valid_snapshot_for_system(system_info)
}
pub async fn inject_synthetic_types_node_package(
&self,
) -> Result<(), JsErrorBox> {
let reqs = &[PackageReq::from_str("@types/node").unwrap()];
// add and ensure this isn't added to the lockfile
self
.add_package_reqs(reqs, PackageCaching::Only(reqs.into()))
.await?;
Ok(())
}
pub async fn cache_packages(
&self,
caching: PackageCaching<'_>,
) -> Result<(), JsErrorBox> {
self.fs_installer.cache_packages(caching).await
}
pub fn resolve_pkg_folder_from_deno_module(
&self,
nv: &PackageNv,
) -> Result<PathBuf, ResolvePkgFolderFromDenoModuleError> {
let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(nv)?;
Ok(self.resolve_pkg_folder_from_pkg_id(&pkg_id)?)
}
pub fn resolve_pkg_id_from_pkg_req(
&self,
req: &PackageReq,
) -> Result<NpmPackageId, PackageReqNotFoundError> {
self.resolution.resolve_pkg_id_from_pkg_req(req)
}
pub fn ensure_no_pkg_json_dep_errors(
&self,
) -> Result<(), Box<PackageJsonDepValueParseWithLocationError>> {
for err in self.npm_install_deps_provider.pkg_json_dep_errors() {
match err.source.as_kind() {
deno_package_json::PackageJsonDepValueParseErrorKind::VersionReq(_) => {
return Err(Box::new(err.clone()));
}
deno_package_json::PackageJsonDepValueParseErrorKind::Unsupported {
..
} => {
// only warn for this one
log::warn!(
"{} {}\n at {}",
colors::yellow("Warning"),
err.source,
err.location,
)
}
}
}
Ok(())
}
/// Ensures that the top level `package.json` dependencies are installed.
/// This may set up the `node_modules` directory.
///
/// Returns `true` if the top level packages are already installed. A
/// return value of `false` means that new packages were added to the NPM resolution.
pub async fn ensure_top_level_package_json_install(
&self,
) -> Result<bool, JsErrorBox> {
if !self.top_level_install_flag.raise() {
return Ok(true); // already did this
}
let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs();
if pkg_json_remote_pkgs.is_empty() {
return Ok(true);
}
// check if something needs resolving before bothering to load all
// the package information (which is slow)
if pkg_json_remote_pkgs.iter().all(|pkg| {
self
.resolution
.resolve_pkg_id_from_pkg_req(&pkg.req)
.is_ok()
}) {
log::debug!(
"All package.json deps resolvable. Skipping top level install."
);
return Ok(true); // everything is already resolvable
}
let pkg_reqs = pkg_json_remote_pkgs
.iter()
.map(|pkg| pkg.req.clone())
.collect::<Vec<_>>();
self.add_package_reqs_no_cache(&pkg_reqs).await?;
Ok(false)
}
pub async fn cache_package_info(
&self,
package_name: &str,
) -> Result<Arc<NpmPackageInfo>, AnyError> {
// this will internally cache the package information
self
.registry_info_provider
.package_info(package_name)
.await
.map_err(|err| err.into())
}
pub fn maybe_node_modules_path(&self) -> Option<&Path> {
self.fs_resolver.node_modules_path()
}
pub fn global_cache_root_path(&self) -> &Path {
self.npm_cache_dir.root_dir()
}
pub fn global_cache_root_url(&self) -> &Url {
self.npm_cache_dir.root_dir_url()
}
}
fn npm_process_state(
snapshot: ValidSerializedNpmResolutionSnapshot,
node_modules_path: Option<&Path>,
) -> String {
serde_json::to_string(&NpmProcessState {
kind: NpmProcessStateKind::Snapshot(snapshot.into_serialized()),
local_node_modules_path: node_modules_path
.map(|p| p.to_string_lossy().to_string()),
})
.unwrap()
}
impl NpmPackageFolderResolver for ManagedCliNpmResolver {
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<PathBuf, PackageFolderResolveError> {
let path = self
.fs_resolver
.resolve_package_folder_from_package(name, referrer)?;
let path =
canonicalize_path_maybe_not_exists(&self.sys, &path).map_err(|err| {
PackageFolderResolveIoError {
package_name: name.to_string(),
referrer: referrer.clone(),
source: err,
}
})?;
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
}
impl NpmProcessStateProvider for ManagedCliNpmResolver {
fn get_npm_process_state(&self) -> String {
npm_process_state(
self.resolution.serialized_valid_snapshot(),
self.fs_resolver.node_modules_path(),
)
}
}
impl CliNpmReqResolver for ManagedCliNpmResolver {
fn resolve_pkg_folder_from_deno_module_req(
&self,
req: &PackageReq,
_referrer: &ModuleSpecifier,
) -> Result<PathBuf, ResolvePkgFolderFromDenoReqError> {
let pkg_id = self.resolve_pkg_id_from_pkg_req(req).map_err(|err| {
ResolvePkgFolderFromDenoReqError::Managed(Box::new(err))
})?;
self
.resolve_pkg_folder_from_pkg_id(&pkg_id)
.map_err(|err| ResolvePkgFolderFromDenoReqError::Managed(Box::new(err)))
}
}
impl CliNpmResolver for ManagedCliNpmResolver {
fn into_npm_pkg_folder_resolver(
self: Arc<Self>,
) -> Arc<dyn NpmPackageFolderResolver> {
self
}
fn into_npm_req_resolver(self: Arc<Self>) -> Arc<dyn CliNpmReqResolver> {
self
}
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider> {
self
}
fn clone_snapshotted(&self) -> Arc<dyn CliNpmResolver> {
// create a new snapshotted npm resolution and resolver
let npm_resolution =
Arc::new(NpmResolution::new(self.resolution.snapshot()));
Arc::new(ManagedCliNpmResolver::new(
create_npm_fs_installer(
self.npm_cache.clone(),
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
npm_resolution.clone(),
self.sys.clone(),
self.tarball_cache.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
self.npm_system_info.clone(),
self.lifecycle_scripts.clone(),
),
create_npm_fs_resolver(
&self.npm_cache_dir,
&self.npm_rc,
npm_resolution.clone(),
self.sys.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
),
self.maybe_lockfile.clone(),
self.registry_info_provider.clone(),
self.npm_cache.clone(),
self.npm_cache_dir.clone(),
self.npm_install_deps_provider.clone(),
self.npm_rc.clone(),
npm_resolution,
self.sys.clone(),
self.tarball_cache.clone(),
self.text_only_progress_bar.clone(),
self.npm_system_info.clone(),
self.lifecycle_scripts.clone(),
))
}
fn as_inner(&self) -> InnerCliNpmResolverRef {
InnerCliNpmResolverRef::Managed(self)
}
fn root_node_modules_path(&self) -> Option<&Path> {
self.fs_resolver.node_modules_path()
}
fn check_state_hash(&self) -> Option<u64> {
// We could go further and check all the individual
// npm packages, but that's probably overkill.
let mut package_reqs = self
.resolution
.package_reqs()
.into_iter()
.collect::<Vec<_>>();
package_reqs.sort_by(|a, b| a.0.cmp(&b.0)); // determinism
let mut hasher = FastInsecureHasher::new_without_deno_version();
// ensure the cache gets busted when turning nodeModulesDir on or off
// as this could cause changes in resolution
hasher.write_hashable(self.fs_resolver.node_modules_path().is_some());
for (pkg_req, pkg_nv) in package_reqs {
hasher.write_hashable(&pkg_req);
hasher.write_hashable(&pkg_nv);
}
Some(hasher.finish())
}
}

105
cli/npm/mod.rs
View file

@ -1,38 +1,28 @@
// Copyright 2018-2025 the Deno authors. MIT license.
mod byonm;
pub mod installer;
mod managed;
mod permission_checker;
use std::path::Path;
use std::sync::Arc;
use dashmap::DashMap;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsErrorBox;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_runtime::deno_process::NpmProcessStateProviderRc;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use http::HeaderName;
use http::HeaderValue;
use node_resolver::NpmPackageFolderResolver;
pub use self::byonm::CliByonmNpmResolver;
pub use self::byonm::CliByonmNpmResolverCreateOptions;
pub use self::managed::CliManagedNpmResolverCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub use self::managed::PackageCaching;
pub use self::managed::ResolvePkgFolderFromDenoModuleError;
pub use self::permission_checker::NpmRegistryReadPermissionChecker;
pub use self::permission_checker::NpmRegistryReadPermissionCheckerMode;
pub use self::managed::NpmResolutionInitializer;
pub use self::managed::ResolveSnapshotError;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::sys::CliSys;
@ -43,6 +33,10 @@ pub type CliNpmTarballCache =
pub type CliNpmCache = deno_npm_cache::NpmCache<CliSys>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheHttpClient, CliSys>;
pub type CliNpmResolver = deno_resolver::npm::NpmResolver<CliSys>;
pub type CliManagedNpmResolver = deno_resolver::npm::ManagedNpmResolver<CliSys>;
pub type CliNpmResolverCreateOptions =
deno_resolver::npm::NpmResolverCreateOptions<CliSys>;
#[derive(Debug)]
pub struct CliNpmCacheHttpClient {
@ -62,6 +56,19 @@ impl CliNpmCacheHttpClient {
}
}
pub fn create_npm_process_state_provider(
npm_resolver: &CliNpmResolver,
) -> NpmProcessStateProviderRc {
match npm_resolver {
CliNpmResolver::Byonm(byonm_npm_resolver) => Arc::new(
byonm::CliByonmNpmProcessStateProvider(byonm_npm_resolver.clone()),
),
CliNpmResolver::Managed(managed_npm_resolver) => Arc::new(
managed::CliManagedNpmProcessStateProvider(managed_npm_resolver.clone()),
),
}
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
async fn download_with_retries_on_any_tokio_runtime(
@ -103,76 +110,6 @@ impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
}
}
pub enum CliNpmResolverCreateOptions {
Managed(CliManagedNpmResolverCreateOptions),
Byonm(CliByonmNpmResolverCreateOptions),
}
pub async fn create_cli_npm_resolver_for_lsp(
options: CliNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
use CliNpmResolverCreateOptions::*;
match options {
Managed(options) => {
managed::create_managed_npm_resolver_for_lsp(options).await
}
Byonm(options) => Arc::new(ByonmNpmResolver::new(options)),
}
}
pub async fn create_cli_npm_resolver(
options: CliNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
use CliNpmResolverCreateOptions::*;
match options {
Managed(options) => managed::create_managed_npm_resolver(options).await,
Byonm(options) => Ok(Arc::new(ByonmNpmResolver::new(options))),
}
}
pub enum InnerCliNpmResolverRef<'a> {
Managed(&'a ManagedCliNpmResolver),
#[allow(dead_code)]
Byonm(&'a CliByonmNpmResolver),
}
pub trait CliNpmResolver: CliNpmReqResolver {
fn into_npm_pkg_folder_resolver(
self: Arc<Self>,
) -> Arc<dyn NpmPackageFolderResolver>;
fn into_npm_req_resolver(self: Arc<Self>) -> Arc<dyn CliNpmReqResolver>;
fn into_process_state_provider(
self: Arc<Self>,
) -> Arc<dyn NpmProcessStateProvider>;
fn into_maybe_byonm(self: Arc<Self>) -> Option<Arc<CliByonmNpmResolver>> {
None
}
fn clone_snapshotted(&self) -> Arc<dyn CliNpmResolver>;
fn as_inner(&self) -> InnerCliNpmResolverRef;
fn as_managed(&self) -> Option<&ManagedCliNpmResolver> {
match self.as_inner() {
InnerCliNpmResolverRef::Managed(inner) => Some(inner),
InnerCliNpmResolverRef::Byonm(_) => None,
}
}
fn as_byonm(&self) -> Option<&CliByonmNpmResolver> {
match self.as_inner() {
InnerCliNpmResolverRef::Managed(_) => None,
InnerCliNpmResolverRef::Byonm(inner) => Some(inner),
}
}
fn root_node_modules_path(&self) -> Option<&Path>;
/// Returns a hash returning the state of the npm resolver
/// or `None` if the state currently can't be determined.
fn check_state_hash(&self) -> Option<u64>;
}
#[derive(Debug)]
pub struct NpmFetchResolver {
nv_by_req: DashMap<PackageReq, Option<PackageNv>>,

232
cli/resolver.rs
View file

@ -1,6 +1,7 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
@ -8,8 +9,6 @@ use dashmap::DashSet;
use deno_ast::MediaType;
use deno_config::workspace::MappedResolutionDiagnostic;
use deno_config::workspace::MappedResolutionError;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_core::ModuleSourceCode;
use deno_core::ModuleSpecifier;
@ -19,6 +18,7 @@ use deno_graph::source::UnknownBuiltInNodeModuleError;
use deno_graph::NpmLoadError;
use deno_graph::NpmResolvePkgReqsResult;
use deno_npm::resolution::NpmResolutionError;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::sloppy_imports::SloppyImportsCachedFs;
use deno_resolver::sloppy_imports::SloppyImportsResolver;
use deno_runtime::colors;
@ -33,24 +33,33 @@ use thiserror::Error;
use crate::args::NpmCachingStrategy;
use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS;
use crate::node::CliNodeCodeTranslator;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::PackageCaching;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::sys::CliSys;
use crate::util::sync::AtomicFlag;
use crate::util::text_encoding::from_utf8_lossy_cow;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<CliSys>;
pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver<CliSys>;
pub type CliCjsTracker =
deno_resolver::cjs::CjsTracker<DenoInNpmPackageChecker, CliSys>;
pub type CliIsCjsResolver =
deno_resolver::cjs::IsCjsResolver<DenoInNpmPackageChecker, CliSys>;
pub type CliSloppyImportsCachedFs = SloppyImportsCachedFs<CliSys>;
pub type CliSloppyImportsResolver =
SloppyImportsResolver<CliSloppyImportsCachedFs>;
pub type CliDenoResolver = deno_resolver::DenoResolver<
DenoInNpmPackageChecker,
RealIsBuiltInNodeModuleChecker,
CliNpmResolver,
CliSloppyImportsCachedFs,
CliSys,
>;
pub type CliNpmReqResolver =
deno_resolver::npm::NpmReqResolver<RealIsBuiltInNodeModuleChecker, CliSys>;
pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver<
DenoInNpmPackageChecker,
RealIsBuiltInNodeModuleChecker,
CliNpmResolver,
CliSys,
>;
pub struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
@ -69,14 +78,70 @@ pub struct NotSupportedKindInNpmError {
// todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone)
#[derive(Clone)]
pub struct NpmModuleLoader {
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
node_code_translator: Arc<CliNodeCodeTranslator>,
}
#[derive(Debug, Error, deno_error::JsError)]
pub enum NpmModuleLoadError {
#[class(inherit)]
#[error(transparent)]
NotSupportedKindInNpm(#[from] NotSupportedKindInNpmError),
#[class(inherit)]
#[error(transparent)]
ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError),
#[class(inherit)]
#[error(transparent)]
TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError),
#[class(inherit)]
#[error("{}", format_message(file_path, maybe_referrer))]
Fs {
file_path: PathBuf,
maybe_referrer: Option<ModuleSpecifier>,
#[source]
#[inherit]
source: deno_runtime::deno_io::fs::FsError,
},
}
fn format_message(
file_path: &std::path::Path,
maybe_referrer: &Option<ModuleSpecifier>,
) -> String {
if file_path.is_dir() {
// directory imports are not allowed when importing from an
// ES module, so provide the user with a helpful error message
let dir_path = file_path;
let mut msg = "Directory import ".to_string();
msg.push_str(&dir_path.to_string_lossy());
if let Some(referrer) = maybe_referrer {
msg.push_str(" is not supported resolving import from ");
msg.push_str(referrer.as_str());
let entrypoint_name = ["index.mjs", "index.js", "index.cjs"]
.iter()
.find(|e| dir_path.join(e).is_file());
if let Some(entrypoint_name) = entrypoint_name {
msg.push_str("\nDid you mean to import ");
msg.push_str(entrypoint_name);
msg.push_str(" within the directory?");
}
}
msg
} else {
let mut msg = "Unable to load ".to_string();
msg.push_str(&file_path.to_string_lossy());
if let Some(referrer) = maybe_referrer {
msg.push_str(" imported from ");
msg.push_str(referrer.as_str());
}
msg
}
}
impl NpmModuleLoader {
pub fn new(
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
fs: Arc<dyn deno_fs::FileSystem>,
node_code_translator: Arc<CliNodeCodeTranslator>,
) -> Self {
@ -91,50 +156,26 @@ impl NpmModuleLoader {
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
) -> Result<ModuleCodeStringSource, AnyError> {
) -> Result<ModuleCodeStringSource, NpmModuleLoadError> {
let file_path = specifier.to_file_path().unwrap();
let code = self
.fs
.read_file_async(file_path.clone(), None)
.await
.map_err(AnyError::from)
.with_context(|| {
if file_path.is_dir() {
// directory imports are not allowed when importing from an
// ES module, so provide the user with a helpful error message
let dir_path = file_path;
let mut msg = "Directory import ".to_string();
msg.push_str(&dir_path.to_string_lossy());
if let Some(referrer) = &maybe_referrer {
msg.push_str(" is not supported resolving import from ");
msg.push_str(referrer.as_str());
let entrypoint_name = ["index.mjs", "index.js", "index.cjs"]
.iter()
.find(|e| dir_path.join(e).is_file());
if let Some(entrypoint_name) = entrypoint_name {
msg.push_str("\nDid you mean to import ");
msg.push_str(entrypoint_name);
msg.push_str(" within the directory?");
}
}
msg
} else {
let mut msg = "Unable to load ".to_string();
msg.push_str(&file_path.to_string_lossy());
if let Some(referrer) = &maybe_referrer {
msg.push_str(" imported from ");
msg.push_str(referrer.as_str());
}
msg
}
.map_err(|source| NpmModuleLoadError::Fs {
file_path,
maybe_referrer: maybe_referrer.cloned(),
source,
})?;
let media_type = MediaType::from_specifier(specifier);
if media_type.is_emittable() {
return Err(AnyError::from(NotSupportedKindInNpmError {
media_type,
specifier: specifier.clone(),
}));
return Err(NpmModuleLoadError::NotSupportedKindInNpm(
NotSupportedKindInNpmError {
media_type,
specifier: specifier.clone(),
},
));
}
let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? {
@ -164,48 +205,30 @@ impl NpmModuleLoader {
}
}
pub struct CliResolverOptions {
pub deno_resolver: Arc<CliDenoResolver>,
pub npm_resolver: Option<Arc<dyn CliNpmResolver>>,
pub bare_node_builtins_enabled: bool,
}
#[derive(Debug, Default)]
pub struct FoundPackageJsonDepFlag(AtomicFlag);
/// A resolver that takes care of resolution, taking into account loaded
/// import map, JSX settings.
#[derive(Debug)]
pub struct CliResolver {
deno_resolver: Arc<CliDenoResolver>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
found_package_json_dep_flag: AtomicFlag,
bare_node_builtins_enabled: bool,
found_package_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
warned_pkgs: DashSet<PackageReq>,
}
impl CliResolver {
pub fn new(options: CliResolverOptions) -> Self {
pub fn new(
deno_resolver: Arc<CliDenoResolver>,
found_package_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
) -> Self {
Self {
deno_resolver: options.deno_resolver,
npm_resolver: options.npm_resolver,
found_package_json_dep_flag: Default::default(),
bare_node_builtins_enabled: options.bare_node_builtins_enabled,
deno_resolver,
found_package_json_dep_flag,
warned_pkgs: Default::default(),
}
}
// todo(dsherret): move this off CliResolver as CliResolver is acting
// like a factory by doing this (it's beyond its responsibility)
pub fn create_graph_npm_resolver(
&self,
npm_caching: NpmCachingStrategy,
) -> WorkerCliNpmGraphResolver {
WorkerCliNpmGraphResolver {
npm_resolver: self.npm_resolver.as_ref(),
found_package_json_dep_flag: &self.found_package_json_dep_flag,
bare_node_builtins_enabled: self.bare_node_builtins_enabled,
npm_caching,
}
}
pub fn resolve(
&self,
raw_specifier: &str,
@ -233,7 +256,7 @@ impl CliResolver {
if resolution.found_package_json_dep {
// mark that we need to do an "npm install" later
self.found_package_json_dep_flag.raise();
self.found_package_json_dep_flag.0.raise();
}
if let Some(diagnostic) = resolution.maybe_diagnostic {
@ -260,15 +283,31 @@ impl CliResolver {
}
#[derive(Debug)]
pub struct WorkerCliNpmGraphResolver<'a> {
npm_resolver: Option<&'a Arc<dyn CliNpmResolver>>,
found_package_json_dep_flag: &'a AtomicFlag,
pub struct CliNpmGraphResolver {
npm_installer: Option<Arc<NpmInstaller>>,
found_package_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
bare_node_builtins_enabled: bool,
npm_caching: NpmCachingStrategy,
}
impl CliNpmGraphResolver {
pub fn new(
npm_installer: Option<Arc<NpmInstaller>>,
found_package_json_dep_flag: Arc<FoundPackageJsonDepFlag>,
bare_node_builtins_enabled: bool,
npm_caching: NpmCachingStrategy,
) -> Self {
Self {
npm_installer,
found_package_json_dep_flag,
bare_node_builtins_enabled,
npm_caching,
}
}
}
#[async_trait(?Send)]
impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
impl deno_graph::source::NpmResolver for CliNpmGraphResolver {
fn resolve_builtin_node_module(
&self,
specifier: &ModuleSpecifier,
@ -298,17 +337,12 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
}
fn load_and_cache_npm_package_info(&self, package_name: &str) {
match self.npm_resolver {
Some(npm_resolver) if npm_resolver.as_managed().is_some() => {
let npm_resolver = npm_resolver.clone();
let package_name = package_name.to_string();
deno_core::unsync::spawn(async move {
if let Some(managed) = npm_resolver.as_managed() {
let _ignore = managed.cache_package_info(&package_name).await;
}
});
}
_ => {}
if let Some(npm_installer) = &self.npm_installer {
let npm_installer = npm_installer.clone();
let package_name = package_name.to_string();
deno_core::unsync::spawn(async move {
let _ignore = npm_installer.cache_package_info(&package_name).await;
});
}
}
@ -316,17 +350,11 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
&self,
package_reqs: &[PackageReq],
) -> NpmResolvePkgReqsResult {
match &self.npm_resolver {
Some(npm_resolver) => {
let npm_resolver = match npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(npm_resolver) => npm_resolver,
// if we are using byonm, then this should never be called because
// we don't use deno_graph's npm resolution in this case
InnerCliNpmResolverRef::Byonm(_) => unreachable!(),
};
let top_level_result = if self.found_package_json_dep_flag.is_raised() {
npm_resolver
match &self.npm_installer {
Some(npm_installer) => {
let top_level_result = if self.found_package_json_dep_flag.0.is_raised()
{
npm_installer
.ensure_top_level_package_json_install()
.await
.map(|_| ())
@ -334,15 +362,13 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
Ok(())
};
let result = npm_resolver
let result = npm_installer
.add_package_reqs_raw(
package_reqs,
match self.npm_caching {
NpmCachingStrategy::Eager => {
Some(crate::npm::PackageCaching::All)
}
NpmCachingStrategy::Eager => Some(PackageCaching::All),
NpmCachingStrategy::Lazy => {
Some(crate::npm::PackageCaching::Only(package_reqs.into()))
Some(PackageCaching::Only(package_reqs.into()))
}
NpmCachingStrategy::Manual => None,
},

100
cli/standalone/binary.rs
View file

@ -37,7 +37,15 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsErrorBox;
use deno_graph::ModuleGraph;
use deno_lib::cache::DenoDir;
use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity;
use deno_lib::standalone::virtual_fs::VfsEntry;
use deno_lib::standalone::virtual_fs::VfsFileSubDataKind;
use deno_lib::standalone::virtual_fs::VirtualDirectory;
use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries;
use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
@ -74,27 +82,21 @@ use super::virtual_fs::output_vfs;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsFileSubDataKind;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
use super::virtual_fs::VirtualDirectoryEntries;
use super::virtual_fs::WindowsSystemRootablePath;
use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::CompileFlags;
use crate::args::NpmInstallDepsProvider;
use crate::args::PermissionFlags;
use crate::args::UnstableConfig;
use crate::cache::DenoDir;
use crate::cache::FastInsecureHasher;
use crate::emit::Emitter;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::resolver::CjsTracker;
use crate::resolver::CliCjsTracker;
use crate::shared::ReleaseChannel;
use crate::standalone::virtual_fs::VfsEntry;
use crate::sys::CliSys;
use crate::util::archive;
use crate::util::fs::canonicalize_path;
use crate::util::fs::canonicalize_path_maybe_not_exists;
@ -202,6 +204,7 @@ pub struct Metadata {
pub node_modules: Option<NodeModules>,
pub unstable_config: UnstableConfig,
pub otel_config: OtelConfig,
pub vfs_case_sensitivity: FileSystemCaseSensitivity,
}
#[allow(clippy::too_many_arguments)]
@ -276,7 +279,7 @@ impl StandaloneModules {
pub fn resolve_specifier<'a>(
&'a self,
specifier: &'a ModuleSpecifier,
) -> Result<Option<&'a ModuleSpecifier>, AnyError> {
) -> Result<Option<&'a ModuleSpecifier>, JsErrorBox> {
if specifier.scheme() == "file" {
Ok(Some(specifier))
} else {
@ -379,7 +382,11 @@ pub fn extract_standalone(
root_path: root_path.clone(),
start_file_offset: 0,
};
Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root))
Arc::new(FileBackedVfs::new(
Cow::Borrowed(vfs_files_data),
fs_root,
metadata.vfs_case_sensitivity,
))
};
Ok(Some(StandaloneData {
metadata,
@ -404,13 +411,13 @@ pub struct WriteBinOptions<'a> {
}
pub struct DenoCompileBinaryWriter<'a> {
cjs_tracker: &'a CjsTracker,
cjs_tracker: &'a CliCjsTracker,
cli_options: &'a CliOptions,
deno_dir: &'a DenoDir,
deno_dir: &'a DenoDir<CliSys>,
emitter: &'a Emitter,
file_fetcher: &'a CliFileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
npm_resolver: &'a CliNpmResolver,
workspace_resolver: &'a WorkspaceResolver,
npm_system_info: NpmSystemInfo,
}
@ -418,13 +425,13 @@ pub struct DenoCompileBinaryWriter<'a> {
impl<'a> DenoCompileBinaryWriter<'a> {
#[allow(clippy::too_many_arguments)]
pub fn new(
cjs_tracker: &'a CjsTracker,
cjs_tracker: &'a CliCjsTracker,
cli_options: &'a CliOptions,
deno_dir: &'a DenoDir,
deno_dir: &'a DenoDir<CliSys>,
emitter: &'a Emitter,
file_fetcher: &'a CliFileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
npm_resolver: &'a CliNpmResolver,
workspace_resolver: &'a WorkspaceResolver,
npm_system_info: NpmSystemInfo,
) -> Self {
@ -593,10 +600,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => None,
};
let mut vfs = VfsBuilder::new();
let npm_snapshot = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
let npm_snapshot = match &self.npm_resolver {
CliNpmResolver::Managed(managed) => {
let snapshot = managed
.resolution()
.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?;
Some(snapshot)
@ -604,7 +612,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None
}
}
InnerCliNpmResolverRef::Byonm(_) => {
CliNpmResolver::Byonm(_) => {
self.fill_npm_vfs(&mut vfs)?;
None
}
@ -745,8 +753,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
);
}
let node_modules = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(_) => {
let node_modules = match &self.npm_resolver {
CliNpmResolver::Managed(_) => {
npm_snapshot.as_ref().map(|_| NodeModules::Managed {
node_modules_dir: self.npm_resolver.root_node_modules_path().map(
|path| {
@ -759,7 +767,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
),
})
}
InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm {
CliNpmResolver::Byonm(resolver) => Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
root_dir_url
@ -851,6 +859,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
npm_lazy_caching: self.cli_options.unstable_npm_lazy_caching(),
},
otel_config: self.cli_options.otel_config(),
vfs_case_sensitivity: vfs.case_sensitivity,
};
write_binary_bytes(
@ -873,16 +882,17 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
}
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
match &self.npm_resolver {
CliNpmResolver::Managed(npm_resolver) => {
if let Some(node_modules_path) = npm_resolver.root_node_modules_path() {
maybe_warn_different_system(&self.npm_system_info);
builder.add_dir_recursive(node_modules_path)?;
Ok(())
} else {
// we'll flatten to remove any custom registries later
let mut packages =
npm_resolver.all_system_packages(&self.npm_system_info);
let mut packages = npm_resolver
.resolution()
.all_system_packages(&self.npm_system_info);
packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
for package in packages {
let folder =
@ -892,7 +902,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
Ok(())
}
}
InnerCliNpmResolverRef::Byonm(_) => {
CliNpmResolver::Byonm(_) => {
maybe_warn_different_system(&self.npm_system_info);
for pkg_json in self.cli_options.workspace().package_jsons() {
builder.add_file_at_path(&pkg_json.path)?;
@ -935,8 +945,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
&self,
mut vfs: VfsBuilder,
) -> BuiltVfs {
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
match &self.npm_resolver {
CliNpmResolver::Managed(npm_resolver) => {
if npm_resolver.root_node_modules_path().is_some() {
return vfs.build();
}
@ -984,11 +994,15 @@ impl<'a> DenoCompileBinaryWriter<'a> {
// it's better to not expose the user's cache directory, so take it out
// of there
let case_sensitivity = vfs.case_sensitivity();
let parent = global_cache_root_path.parent().unwrap();
let parent_dir = vfs.get_dir_mut(parent).unwrap();
let index = parent_dir
.entries
.binary_search(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME)
.binary_search(
DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME,
case_sensitivity,
)
.unwrap();
let npm_global_cache_dir_entry = parent_dir.entries.remove(index);
@ -996,9 +1010,19 @@ impl<'a> DenoCompileBinaryWriter<'a> {
// this is not as optimized as it could be
let mut last_name =
Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME);
for ancestor in parent.ancestors() {
let dir = vfs.get_dir_mut(ancestor).unwrap();
if let Ok(index) = dir.entries.binary_search(&last_name) {
for ancestor in
parent.ancestors().map(Some).chain(std::iter::once(None))
{
let dir = if let Some(ancestor) = ancestor {
vfs.get_dir_mut(ancestor).unwrap()
} else if cfg!(windows) {
vfs.get_system_root_dir_mut()
} else {
break;
};
if let Ok(index) =
dir.entries.binary_search(&last_name, case_sensitivity)
{
dir.entries.remove(index);
}
last_name = Cow::Owned(dir.name.clone());
@ -1009,10 +1033,12 @@ impl<'a> DenoCompileBinaryWriter<'a> {
// now build the vfs and add the global cache dir entry there
let mut built_vfs = vfs.build();
built_vfs.entries.insert(npm_global_cache_dir_entry);
built_vfs
.entries
.insert(npm_global_cache_dir_entry, case_sensitivity);
built_vfs
}
InnerCliNpmResolverRef::Byonm(_) => vfs.build(),
CliNpmResolver::Byonm(_) => vfs.build(),
}
}
}

2
cli/standalone/file_system.rs
View file

@ -9,6 +9,7 @@ use std::sync::Arc;
use std::time::Duration;
use std::time::SystemTime;
use deno_lib::standalone::virtual_fs::VfsFileSubDataKind;
use deno_runtime::deno_fs::AccessCheckCb;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsDirEntry;
@ -30,7 +31,6 @@ use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::FileBackedVfsDirEntry;
use super::virtual_fs::FileBackedVfsFile;
use super::virtual_fs::FileBackedVfsMetadata;
use super::virtual_fs::VfsFileSubDataKind;
#[derive(Debug, Clone)]
pub struct DenoCompileFileSystem(Arc<FileBackedVfs>);

192
cli/standalone/mod.rs
View file

@ -36,12 +36,24 @@ use deno_core::RequestedModuleType;
use deno_core::ResolutionKind;
use deno_core::SourceCodeCacheInfo;
use deno_error::JsErrorBox;
use deno_lib::cache::DenoDirProvider;
use deno_lib::npm::NpmRegistryReadPermissionChecker;
use deno_lib::npm::NpmRegistryReadPermissionCheckerMode;
use deno_lib::standalone::virtual_fs::VfsFileSubDataKind;
use deno_lib::worker::CreateModuleLoaderResult;
use deno_lib::worker::LibMainWorkerFactory;
use deno_lib::worker::LibMainWorkerOptions;
use deno_lib::worker::ModuleLoaderFactory;
use deno_lib::worker::StorageKeyResolver;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_package_json::PackageJsonDepValue;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::create_in_npm_pkg_checker;
use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions;
use deno_resolver::npm::managed::NpmResolutionCell;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::CreateInNpmPkgCheckerOptions;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
@ -66,16 +78,13 @@ use node_resolver::ResolutionMode;
use serialization::DenoCompileModuleSource;
use serialization::SourceMapStore;
use virtual_fs::FileBackedVfs;
use virtual_fs::VfsFileSubDataKind;
use crate::args::create_default_npmrc;
use crate::args::get_root_cert_store;
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CaData;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoDirProvider;
use crate::cache::FastInsecureHasher;
use crate::cache::NodeAnalysisCache;
use crate::http_util::HttpClientProvider;
@ -83,15 +92,14 @@ use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_npm_process_state_provider;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedNpmResolverCreateOptions;
use crate::npm::CliNpmResolver;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::NpmRegistryReadPermissionChecker;
use crate::npm::NpmRegistryReadPermissionCheckerMode;
use crate::resolver::CjsTracker;
use crate::npm::NpmResolutionInitializer;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::NpmModuleLoader;
use crate::sys::CliSys;
@ -102,8 +110,6 @@ use crate::util::v8::construct_v8_flags;
use crate::worker::CliCodeCache;
use crate::worker::CliMainWorkerFactory;
use crate::worker::CliMainWorkerOptions;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
pub mod binary;
mod code_cache;
@ -119,16 +125,16 @@ use self::binary::Metadata;
pub use self::file_system::DenoCompileFileSystem;
struct SharedModuleLoaderState {
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
code_cache: Option<Arc<dyn CliCodeCache>>,
fs: Arc<dyn deno_fs::FileSystem>,
modules: StandaloneModules,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<CliNodeResolver>,
npm_module_loader: Arc<NpmModuleLoader>,
npm_registry_permission_checker: NpmRegistryReadPermissionChecker,
npm_registry_permission_checker: NpmRegistryReadPermissionChecker<CliSys>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_resolver: CliNpmResolver,
source_maps: SourceMapStore,
vfs: Arc<FileBackedVfs>,
workspace_resolver: WorkspaceResolver,
@ -408,7 +414,8 @@ impl ModuleLoader for EmbeddedModuleLoader {
let code_source = shared
.npm_module_loader
.load(&original_specifier, maybe_referrer.as_ref())
.await?;
.await
.map_err(JsErrorBox::from_err)?;
let code_cache_entry = shared.get_code_cache(
&code_source.found_url,
code_source.code.as_bytes(),
@ -471,7 +478,8 @@ impl ModuleLoader for EmbeddedModuleLoader {
let source = shared
.node_code_translator
.translate_cjs_to_esm(&module_specifier, Some(source))
.await?;
.await
.map_err(JsErrorBox::from_err)?;
let module_source = match source {
Cow::Owned(source) => ModuleSourceCode::String(source.into()),
Cow::Borrowed(source) => {
@ -687,18 +695,12 @@ pub async fn run(
ca_data: metadata.ca_data.map(CaData::Bytes),
cell: Default::default(),
});
let progress_bar = ProgressBar::new(ProgressBarStyle::TextOnly);
let http_client_provider = Arc::new(HttpClientProvider::new(
Some(root_cert_store_provider.clone()),
metadata.unsafely_ignore_certificate_errors.clone(),
));
// use a dummy npm registry url
let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap();
let root_dir_url =
Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap());
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let npm_global_cache_dir = root_path.join(".deno_compile_node_modules");
let cache_setting = CacheSetting::Only;
let pkg_json_resolver = Arc::new(CliPackageJsonResolver::new(sys.clone()));
let npm_registry_permission_checker = {
let mode = match &metadata.node_modules {
@ -737,35 +739,25 @@ pub async fn run(
let maybe_node_modules_path = node_modules_dir
.map(|node_modules_dir| root_path.join(node_modules_dir));
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed(
DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed(
ManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: npm_cache_dir.root_dir_url(),
maybe_node_modules_path: maybe_node_modules_path.as_deref(),
},
));
let npm_resolution =
Arc::new(NpmResolutionCell::new(NpmResolutionSnapshot::new(snapshot)));
let npm_resolver =
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliNpmResolver::new(CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some(
snapshot,
)),
maybe_lockfile: None,
http_client_provider: http_client_provider.clone(),
npm_resolution,
npm_cache_dir,
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
text_only_progress_bar: progress_bar,
cache_setting,
maybe_node_modules_path,
npm_system_info: Default::default(),
npmrc,
lifecycle_scripts: Default::default(),
},
))
.await?;
));
(in_npm_pkg_checker, npm_resolver)
}
Some(binary::NodeModules::Byonm {
@ -774,15 +766,14 @@ pub async fn run(
let root_node_modules_dir =
root_node_modules_dir.map(|p| vfs.root().join(p));
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = create_cli_npm_resolver(
DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = CliNpmResolver::new(
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
sys: sys.clone(),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir,
}),
)
.await?;
);
(in_npm_pkg_checker, npm_resolver)
}
None => {
@ -795,33 +786,24 @@ pub async fn run(
npmrc.get_all_known_registries_urls(),
));
let in_npm_pkg_checker =
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed(
DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed(
ManagedInNpmPkgCheckerCreateOptions {
root_cache_dir_url: npm_cache_dir.root_dir_url(),
maybe_node_modules_path: None,
},
));
let npm_resolution = Arc::new(NpmResolutionCell::default());
let npm_resolver =
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliNpmResolver::new(CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
http_client_provider: http_client_provider.clone(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
npm_resolution,
sys: sys.clone(),
cache_setting,
text_only_progress_bar: progress_bar,
npm_cache_dir,
maybe_lockfile: None,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npmrc: create_default_npmrc(),
lifecycle_scripts: Default::default(),
},
))
.await?;
));
(in_npm_pkg_checker, npm_resolver)
}
};
@ -830,11 +812,12 @@ pub async fn run(
let node_resolver = Arc::new(NodeResolver::new(
in_npm_pkg_checker.clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
npm_resolver.clone(),
pkg_json_resolver.clone(),
sys.clone(),
node_resolver::ConditionsFromResolutionMode::default(),
));
let cjs_tracker = Arc::new(CjsTracker::new(
let cjs_tracker = Arc::new(CliCjsTracker::new(
in_npm_pkg_checker.clone(),
pkg_json_resolver.clone(),
if metadata.unstable_config.detect_cjs {
@ -849,11 +832,10 @@ pub async fn run(
let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db());
let npm_req_resolver =
Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
sys: sys.clone(),
in_npm_pkg_checker: in_npm_pkg_checker.clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
npm_resolver: npm_resolver.clone(),
}));
let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new(
node_analysis_cache,
@ -865,7 +847,7 @@ pub async fn run(
cjs_esm_code_analyzer,
in_npm_pkg_checker,
node_resolver.clone(),
npm_resolver.clone().into_npm_pkg_folder_resolver(),
npm_resolver.clone(),
pkg_json_resolver.clone(),
sys.clone(),
));
@ -985,53 +967,67 @@ pub async fn run(
}
checker
});
let worker_factory = CliMainWorkerFactory::new(
let lib_main_worker_options = LibMainWorkerOptions {
argv: metadata.argv,
log_level: WorkerLogLevel::Info,
enable_op_summary_metrics: false,
enable_testing_features: false,
has_node_modules_dir,
inspect_brk: false,
inspect_wait: false,
strace_ops: None,
is_inspecting: false,
skip_op_registration: true,
location: metadata.location,
argv0: NpmPackageReqReference::from_specifier(&main_module)
.ok()
.map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref))
.or(std::env::args().next()),
node_debug: std::env::var("NODE_DEBUG").ok(),
origin_data_folder_path: None,
seed: metadata.seed,
unsafely_ignore_certificate_errors: metadata
.unsafely_ignore_certificate_errors,
node_ipc: None,
serve_port: None,
serve_host: None,
deno_version: crate::version::DENO_VERSION_INFO.deno,
deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent,
otel_config: metadata.otel_config,
startup_snapshot: crate::js::deno_isolate_init(),
};
let lib_main_worker_factory = LibMainWorkerFactory::new(
Arc::new(BlobStore::default()),
code_cache,
code_cache.map(|c| c.as_code_cache()),
feature_checker,
fs,
None,
None,
None,
Box::new(module_loader_factory),
node_resolver,
npm_resolver,
node_resolver.clone(),
create_npm_process_state_provider(&npm_resolver),
pkg_json_resolver,
root_cert_store_provider,
permissions,
StorageKeyResolver::empty(),
sys.clone(),
lib_main_worker_options,
);
// todo(dsherret): use LibMainWorker directly here and don't use CliMainWorkerFactory
let cli_main_worker_options = CliMainWorkerOptions {
create_hmr_runner: None,
create_coverage_collector: None,
needs_test_modules: false,
default_npm_caching_strategy: crate::args::NpmCachingStrategy::Lazy,
};
let worker_factory = CliMainWorkerFactory::new(
lib_main_worker_factory,
None,
None,
node_resolver,
None,
npm_resolver,
sys,
crate::args::DenoSubcommand::Run(Default::default()),
CliMainWorkerOptions {
argv: metadata.argv,
log_level: WorkerLogLevel::Info,
enable_op_summary_metrics: false,
enable_testing_features: false,
has_node_modules_dir,
hmr: false,
inspect_brk: false,
inspect_wait: false,
strace_ops: None,
is_inspecting: false,
skip_op_registration: true,
location: metadata.location,
argv0: NpmPackageReqReference::from_specifier(&main_module)
.ok()
.map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref))
.or(std::env::args().next()),
node_debug: std::env::var("NODE_DEBUG").ok(),
origin_data_folder_path: None,
seed: metadata.seed,
unsafely_ignore_certificate_errors: metadata
.unsafely_ignore_certificate_errors,
create_hmr_runner: None,
create_coverage_collector: None,
node_ipc: None,
serve_port: None,
serve_host: None,
},
metadata.otel_config,
crate::args::NpmCachingStrategy::Lazy,
cli_main_worker_options,
permissions,
);
// Initialize v8 once from the main thread.

11
cli/standalone/serialization.rs
View file

@ -17,6 +17,8 @@ use deno_core::url::Url;
use deno_core::FastString;
use deno_core::ModuleSourceCode;
use deno_core::ModuleType;
use deno_error::JsErrorBox;
use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
@ -28,8 +30,6 @@ use indexmap::IndexMap;
use super::binary::Metadata;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VirtualDirectoryEntries;
use crate::standalone::virtual_fs::VirtualDirectory;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
@ -442,12 +442,15 @@ impl RemoteModulesStore {
pub fn resolve_specifier<'a>(
&'a self,
specifier: &'a Url,
) -> Result<Option<&'a Url>, AnyError> {
) -> Result<Option<&'a Url>, JsErrorBox> {
let mut count = 0;
let mut current = specifier;
loop {
if count > 10 {
bail!("Too many redirects resolving '{}'", specifier);
return Err(JsErrorBox::generic(format!(
"Too many redirects resolving '{}'",
specifier
)));
}
match self.specifiers.get(current) {
Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => {

448
cli/standalone/virtual_fs.rs
View file

@ -2,6 +2,7 @@
use std::borrow::Cow;
use std::cell::RefCell;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs::File;
@ -22,6 +23,17 @@ use deno_core::parking_lot::Mutex;
use deno_core::BufMutView;
use deno_core::BufView;
use deno_core::ResourceHandleFd;
use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity;
use deno_lib::standalone::virtual_fs::OffsetWithLength;
use deno_lib::standalone::virtual_fs::VfsEntry;
use deno_lib::standalone::virtual_fs::VfsEntryRef;
use deno_lib::standalone::virtual_fs::VfsFileSubDataKind;
use deno_lib::standalone::virtual_fs::VirtualDirectory;
use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries;
use deno_lib::standalone::virtual_fs::VirtualFile;
use deno_lib::standalone::virtual_fs::VirtualSymlink;
use deno_lib::standalone::virtual_fs::VirtualSymlinkParts;
use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath;
use deno_path_util::normalize_path;
use deno_path_util::strip_unc_prefix;
use deno_runtime::deno_fs::FsDirEntry;
@ -40,53 +52,14 @@ use crate::util::display::human_size;
use crate::util::display::DisplayTreeNode;
use crate::util::fs::canonicalize_path;
#[derive(Debug, PartialEq, Eq)]
pub enum WindowsSystemRootablePath {
/// The root of the system above any drive letters.
WindowSystemRoot,
Path(PathBuf),
}
impl WindowsSystemRootablePath {
pub fn join(&self, name_component: &str) -> PathBuf {
// this method doesn't handle multiple components
debug_assert!(
!name_component.contains('\\'),
"Invalid component: {}",
name_component
);
debug_assert!(
!name_component.contains('/'),
"Invalid component: {}",
name_component
);
match self {
WindowsSystemRootablePath::WindowSystemRoot => {
// windows drive letter
PathBuf::from(&format!("{}\\", name_component))
}
WindowsSystemRootablePath::Path(path) => path.join(name_component),
}
}
}
#[derive(Debug)]
pub struct BuiltVfs {
pub root_path: WindowsSystemRootablePath,
pub case_sensitivity: FileSystemCaseSensitivity,
pub entries: VirtualDirectoryEntries,
pub files: Vec<Vec<u8>>,
}
#[derive(Debug, Copy, Clone)]
pub enum VfsFileSubDataKind {
/// Raw bytes of the file.
Raw,
/// Bytes to use for module loading. For example, for TypeScript
/// files this will be the transpiled JavaScript source.
ModuleGraph,
}
#[derive(Debug)]
pub struct VfsBuilder {
executable_root: VirtualDirectory,
@ -95,6 +68,7 @@ pub struct VfsBuilder {
file_offsets: HashMap<String, u64>,
/// The minimum root directory that should be included in the VFS.
min_root_dir: Option<WindowsSystemRootablePath>,
case_sensitivity: FileSystemCaseSensitivity,
}
impl VfsBuilder {
@ -108,9 +82,23 @@ impl VfsBuilder {
current_offset: 0,
file_offsets: Default::default(),
min_root_dir: Default::default(),
// This is not exactly correct because file systems on these OSes
// may be case-sensitive or not based on the directory, but this
// is a good enough approximation and limitation. In the future,
// we may want to store this information per directory instead
// depending on the feedback we get.
case_sensitivity: if cfg!(windows) || cfg!(target_os = "macos") {
FileSystemCaseSensitivity::Insensitive
} else {
FileSystemCaseSensitivity::Sensitive
},
}
}
pub fn case_sensitivity(&self) -> FileSystemCaseSensitivity {
self.case_sensitivity
}
/// Add a directory that might be the minimum root directory
/// of the VFS.
///
@ -215,21 +203,21 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir.entries.binary_search(&name) {
Ok(index) => index,
Err(insert_index) => {
current_dir.entries.0.insert(
insert_index,
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Default::default(),
}),
);
insert_index
}
};
match &mut current_dir.entries.0[index] {
VfsEntry::Dir(dir) => {
let index = current_dir.entries.insert_or_modify(
&name,
self.case_sensitivity,
|| {
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Default::default(),
})
},
|_| {
// ignore
},
);
match current_dir.entries.get_mut_by_index(index) {
Some(VfsEntry::Dir(dir)) => {
current_dir = dir;
}
_ => unreachable!(),
@ -252,7 +240,9 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let entry = current_dir.entries.get_mut_by_name(&name)?;
let entry = current_dir
.entries
.get_mut_by_name(&name, self.case_sensitivity)?;
match entry {
VfsEntry::Dir(dir) => {
current_dir = dir;
@ -303,7 +293,8 @@ impl VfsBuilder {
sub_data_kind: VfsFileSubDataKind,
) -> Result<(), AnyError> {
log::debug!("Adding file '{}'", path.display());
let checksum = util::checksum::gen(&[&data]);
let checksum = deno_lib::util::checksum::gen(&[&data]);
let case_sensitivity = self.case_sensitivity;
let offset = if let Some(offset) = self.file_offsets.get(&checksum) {
// duplicate file, reuse an old offset
*offset
@ -318,32 +309,28 @@ impl VfsBuilder {
offset,
len: data.len() as u64,
};
match dir.entries.binary_search(&name) {
Ok(index) => {
let entry = &mut dir.entries.0[index];
match entry {
VfsEntry::File(virtual_file) => match sub_data_kind {
VfsFileSubDataKind::Raw => {
virtual_file.offset = offset_and_len;
}
VfsFileSubDataKind::ModuleGraph => {
virtual_file.module_graph_offset = offset_and_len;
}
},
VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(),
}
}
Err(insert_index) => {
dir.entries.0.insert(
insert_index,
VfsEntry::File(VirtualFile {
name: name.to_string(),
offset: offset_and_len,
module_graph_offset: offset_and_len,
}),
);
}
}
dir.entries.insert_or_modify(
&name,
case_sensitivity,
|| {
VfsEntry::File(VirtualFile {
name: name.to_string(),
offset: offset_and_len,
module_graph_offset: offset_and_len,
})
},
|entry| match entry {
VfsEntry::File(virtual_file) => match sub_data_kind {
VfsFileSubDataKind::Raw => {
virtual_file.offset = offset_and_len;
}
VfsFileSubDataKind::ModuleGraph => {
virtual_file.module_graph_offset = offset_and_len;
}
},
VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(),
},
);
// new file, update the list of files
if self.current_offset == offset {
@ -379,21 +366,23 @@ impl VfsBuilder {
std::fs::read_link(path)
.with_context(|| format!("Reading symlink '{}'", path.display()))?,
);
let case_sensitivity = self.case_sensitivity;
let target = normalize_path(path.parent().unwrap().join(&target));
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search(&name) {
Ok(_) => {} // previously inserted
Err(insert_index) => {
dir.entries.0.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: VirtualSymlinkParts::from_path(&target),
}),
);
}
}
dir.entries.insert_or_modify(
&name,
case_sensitivity,
|| {
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: VirtualSymlinkParts::from_path(&target),
})
},
|_| {
// ignore previously inserted
},
);
let target_metadata =
std::fs::symlink_metadata(&target).with_context(|| {
format!("Reading symlink target '{}'", target.display())
@ -424,16 +413,20 @@ impl VfsBuilder {
dir: &mut VirtualDirectory,
parts: &[String],
) {
for entry in &mut dir.entries.0 {
for entry in dir.entries.iter_mut() {
match entry {
VfsEntry::Dir(dir) => {
strip_prefix_from_symlinks(dir, parts);
}
VfsEntry::File(_) => {}
VfsEntry::Symlink(symlink) => {
let old_parts = std::mem::take(&mut symlink.dest_parts.0);
symlink.dest_parts.0 =
old_parts.into_iter().skip(parts.len()).collect();
let parts = symlink
.dest_parts
.take_parts()
.into_iter()
.skip(parts.len())
.collect();
symlink.dest_parts.set_parts(parts);
}
}
}
@ -452,13 +445,13 @@ impl VfsBuilder {
if self.min_root_dir.as_ref() == Some(&current_path) {
break;
}
match &current_dir.entries.0[0] {
match current_dir.entries.iter().next().unwrap() {
VfsEntry::Dir(dir) => {
if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
// special directory we want to maintain
break;
}
match current_dir.entries.0.remove(0) {
match current_dir.entries.remove(0) {
VfsEntry::Dir(dir) => {
current_path =
WindowsSystemRootablePath::Path(current_path.join(&dir.name));
@ -473,11 +466,12 @@ impl VfsBuilder {
if let WindowsSystemRootablePath::Path(path) = &current_path {
strip_prefix_from_symlinks(
&mut current_dir,
&VirtualSymlinkParts::from_path(path).0,
VirtualSymlinkParts::from_path(path).parts(),
);
}
BuiltVfs {
root_path: current_path,
case_sensitivity: self.case_sensitivity,
entries: current_dir.entries,
files: self.files,
}
@ -552,7 +546,7 @@ fn vfs_as_display_tree(
All(Size),
Subset(Vec<DirEntryOutput<'a>>),
File(Size),
Symlink(&'a [String]),
Symlink(&'a VirtualSymlinkParts),
}
impl<'a> EntryOutput<'a> {
@ -601,7 +595,7 @@ fn vfs_as_display_tree(
format!("{} ({})", name, format_size(*size))
}
EntryOutput::Symlink(parts) => {
format!("{} --> {}", name, parts.join("/"))
format!("{} --> {}", name, parts.display())
}
},
children: match self {
@ -744,7 +738,7 @@ fn vfs_as_display_tree(
EntryOutput::File(file_size(file, seen_offsets))
}
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
EntryOutput::Symlink(&virtual_symlink.dest_parts)
}
},
})
@ -781,7 +775,7 @@ fn vfs_as_display_tree(
}
VfsEntry::File(file) => EntryOutput::File(file_size(file, seen_offsets)),
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
EntryOutput::Symlink(&virtual_symlink.dest_parts)
}
}
}
@ -847,187 +841,6 @@ fn vfs_as_display_tree(
}
}
#[derive(Debug)]
enum VfsEntryRef<'a> {
Dir(&'a VirtualDirectory),
File(&'a VirtualFile),
Symlink(&'a VirtualSymlink),
}
impl VfsEntryRef<'_> {
pub fn as_metadata(&self) -> FileBackedVfsMetadata {
FileBackedVfsMetadata {
file_type: match self {
Self::Dir(_) => sys_traits::FileType::Dir,
Self::File(_) => sys_traits::FileType::File,
Self::Symlink(_) => sys_traits::FileType::Symlink,
},
name: self.name().to_string(),
len: match self {
Self::Dir(_) => 0,
Self::File(file) => file.offset.len,
Self::Symlink(_) => 0,
},
}
}
pub fn name(&self) -> &str {
match self {
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
}
// todo(dsherret): we should store this more efficiently in the binary
#[derive(Debug, Serialize, Deserialize)]
pub enum VfsEntry {
Dir(VirtualDirectory),
File(VirtualFile),
Symlink(VirtualSymlink),
}
impl VfsEntry {
pub fn name(&self) -> &str {
match self {
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
fn as_ref(&self) -> VfsEntryRef {
match self {
VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir),
VfsEntry::File(file) => VfsEntryRef::File(file),
VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink),
}
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct VirtualDirectoryEntries(Vec<VfsEntry>);
impl VirtualDirectoryEntries {
pub fn new(mut entries: Vec<VfsEntry>) -> Self {
// needs to be sorted by name
entries.sort_by(|a, b| a.name().cmp(b.name()));
Self(entries)
}
pub fn take_inner(&mut self) -> Vec<VfsEntry> {
std::mem::take(&mut self.0)
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn get_by_name(&self, name: &str) -> Option<&VfsEntry> {
self.binary_search(name).ok().map(|index| &self.0[index])
}
pub fn get_mut_by_name(&mut self, name: &str) -> Option<&mut VfsEntry> {
self
.binary_search(name)
.ok()
.map(|index| &mut self.0[index])
}
pub fn binary_search(&self, name: &str) -> Result<usize, usize> {
self.0.binary_search_by(|e| e.name().cmp(name))
}
pub fn insert(&mut self, entry: VfsEntry) {
match self.binary_search(entry.name()) {
Ok(index) => {
self.0[index] = entry;
}
Err(insert_index) => {
self.0.insert(insert_index, entry);
}
}
}
pub fn remove(&mut self, index: usize) -> VfsEntry {
self.0.remove(index)
}
pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> {
self.0.iter()
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
#[serde(rename = "n")]
pub name: String,
// should be sorted by name
#[serde(rename = "e")]
pub entries: VirtualDirectoryEntries,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct OffsetWithLength {
#[serde(rename = "o")]
pub offset: u64,
#[serde(rename = "l")]
pub len: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VirtualFile {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "o")]
pub offset: OffsetWithLength,
/// Offset file to use for module loading when it differs from the
/// raw file. Often this will be the same offset as above for data
/// such as JavaScript files, but for TypeScript files the `offset`
/// will be the original raw bytes when included as an asset and this
/// offset will be to the transpiled JavaScript source.
#[serde(rename = "m")]
pub module_graph_offset: OffsetWithLength,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlinkParts(Vec<String>);
impl VirtualSymlinkParts {
pub fn from_path(path: &Path) -> Self {
Self(
path
.components()
.filter(|c| !matches!(c, std::path::Component::RootDir))
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect(),
)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "p")]
pub dest_parts: VirtualSymlinkParts,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts.0 {
dest.push(part);
}
dest
}
}
#[derive(Debug)]
pub struct VfsRoot {
pub dir: VirtualDirectory,
@ -1039,19 +852,21 @@ impl VfsRoot {
fn find_entry<'a>(
&'a self,
path: &Path,
case_sensitivity: FileSystemCaseSensitivity,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
self.find_entry_inner(path, &mut HashSet::new())
self.find_entry_inner(path, &mut HashSet::new(), case_sensitivity)
}
fn find_entry_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
case_sensitivity: FileSystemCaseSensitivity,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let mut path = Cow::Borrowed(path);
loop {
let (resolved_path, entry) =
self.find_entry_no_follow_inner(&path, seen)?;
self.find_entry_no_follow_inner(&path, seen, case_sensitivity)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
if !seen.insert(path.to_path_buf()) {
@ -1072,14 +887,16 @@ impl VfsRoot {
fn find_entry_no_follow(
&self,
path: &Path,
case_sensitivity: FileSystemCaseSensitivity,
) -> std::io::Result<(PathBuf, VfsEntryRef)> {
self.find_entry_no_follow_inner(path, &mut HashSet::new())
self.find_entry_no_follow_inner(path, &mut HashSet::new(), case_sensitivity)
}
fn find_entry_no_follow_inner<'a>(
&'a self,
path: &Path,
seen: &mut HashSet<PathBuf>,
case_sensitivity: FileSystemCaseSensitivity,
) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> {
let relative_path = match path.strip_prefix(&self.root_path) {
Ok(p) => p,
@ -1101,7 +918,8 @@ impl VfsRoot {
}
VfsEntryRef::Symlink(symlink) => {
let dest = symlink.resolve_dest_from_root(&self.root_path);
let (resolved_path, entry) = self.find_entry_inner(&dest, seen)?;
let (resolved_path, entry) =
self.find_entry_inner(&dest, seen, case_sensitivity)?;
final_path = resolved_path; // overwrite with the new resolved path
match entry {
VfsEntryRef::Dir(dir) => {
@ -1126,7 +944,7 @@ impl VfsRoot {
let component = component.to_string_lossy();
current_entry = current_dir
.entries
.get_by_name(&component)
.get_by_name(&component, case_sensitivity)
.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::NotFound, "path not found")
})?
@ -1361,6 +1179,21 @@ pub struct FileBackedVfsMetadata {
}
impl FileBackedVfsMetadata {
pub fn from_vfs_entry_ref(vfs_entry: VfsEntryRef) -> Self {
FileBackedVfsMetadata {
file_type: match vfs_entry {
VfsEntryRef::Dir(_) => sys_traits::FileType::Dir,
VfsEntryRef::File(_) => sys_traits::FileType::File,
VfsEntryRef::Symlink(_) => sys_traits::FileType::Symlink,
},
name: vfs_entry.name().to_string(),
len: match vfs_entry {
VfsEntryRef::Dir(_) => 0,
VfsEntryRef::File(file) => file.offset.len,
VfsEntryRef::Symlink(_) => 0,
},
}
}
pub fn as_fs_stat(&self) -> FsStat {
FsStat {
is_directory: self.file_type == sys_traits::FileType::Dir,
@ -1392,13 +1225,19 @@ impl FileBackedVfsMetadata {
pub struct FileBackedVfs {
vfs_data: Cow<'static, [u8]>,
fs_root: VfsRoot,
case_sensitivity: FileSystemCaseSensitivity,
}
impl FileBackedVfs {
pub fn new(data: Cow<'static, [u8]>, fs_root: VfsRoot) -> Self {
pub fn new(
data: Cow<'static, [u8]>,
fs_root: VfsRoot,
case_sensitivity: FileSystemCaseSensitivity,
) -> Self {
Self {
vfs_data: data,
fs_root,
case_sensitivity,
}
}
@ -1446,12 +1285,14 @@ impl FileBackedVfs {
let path = path.to_path_buf();
Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry {
parent_path: path.to_path_buf(),
metadata: entry.as_ref().as_metadata(),
metadata: FileBackedVfsMetadata::from_vfs_entry_ref(entry.as_ref()),
}))
}
pub fn read_link(&self, path: &Path) -> std::io::Result<PathBuf> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
let (_, entry) = self
.fs_root
.find_entry_no_follow(path, self.case_sensitivity)?;
match entry {
VfsEntryRef::Symlink(symlink) => {
Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path))
@ -1464,17 +1305,19 @@ impl FileBackedVfs {
}
pub fn lstat(&self, path: &Path) -> std::io::Result<FileBackedVfsMetadata> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
Ok(entry.as_metadata())
let (_, entry) = self
.fs_root
.find_entry_no_follow(path, self.case_sensitivity)?;
Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry))
}
pub fn stat(&self, path: &Path) -> std::io::Result<FileBackedVfsMetadata> {
let (_, entry) = self.fs_root.find_entry(path)?;
Ok(entry.as_metadata())
let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?;
Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry))
}
pub fn canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
let (path, _) = self.fs_root.find_entry(path)?;
let (path, _) = self.fs_root.find_entry(path, self.case_sensitivity)?;
Ok(path)
}
@ -1536,7 +1379,7 @@ impl FileBackedVfs {
}
pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> {
let (_, entry) = self.fs_root.find_entry(path)?;
let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?;
match entry {
VfsEntryRef::Dir(dir) => Ok(dir),
VfsEntryRef::Symlink(_) => unreachable!(),
@ -1548,7 +1391,7 @@ impl FileBackedVfs {
}
pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> {
let (_, entry) = self.fs_root.find_entry(path)?;
let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?;
match entry {
VfsEntryRef::Dir(_) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
@ -1754,6 +1597,7 @@ mod test {
root_path: dest_path.to_path_buf(),
start_file_offset: 0,
},
FileSystemCaseSensitivity::Sensitive,
),
)
}

2
cli/sys.rs
View file

@ -29,6 +29,8 @@ pub enum CliSys {
DenoCompile(DenoCompileFileSystem),
}
impl deno_lib::sys::DenoLibSys for CliSys {}
impl Default for CliSys {
fn default() -> Self {
Self::Real(sys_traits::impls::RealSys)

20
cli/task_runner.rs
View file

@ -25,9 +25,8 @@ use tokio::task::LocalSet;
use tokio_util::sync::CancellationToken;
use crate::node::CliNodeResolver;
use crate::npm::CliManagedNpmResolver;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::npm::ManagedCliNpmResolver;
pub fn get_script_with_args(script: &str, argv: &[String]) -> String {
let additional_args = argv
@ -414,15 +413,15 @@ impl ShellCommand for NodeModulesFileRunCommand {
}
pub fn resolve_custom_commands(
npm_resolver: &dyn CliNpmResolver,
npm_resolver: &CliNpmResolver,
node_resolver: &CliNodeResolver,
) -> Result<HashMap<String, Rc<dyn ShellCommand>>, AnyError> {
let mut commands = match npm_resolver.as_inner() {
InnerCliNpmResolverRef::Byonm(npm_resolver) => {
let mut commands = match npm_resolver {
CliNpmResolver::Byonm(npm_resolver) => {
let node_modules_dir = npm_resolver.root_node_modules_path().unwrap();
resolve_npm_commands_from_bin_dir(node_modules_dir)
}
InnerCliNpmResolverRef::Managed(npm_resolver) => {
CliNpmResolver::Managed(npm_resolver) => {
resolve_managed_npm_commands(npm_resolver, node_resolver)?
}
};
@ -521,13 +520,12 @@ fn resolve_execution_path_from_npx_shim(
}
fn resolve_managed_npm_commands(
npm_resolver: &ManagedCliNpmResolver,
npm_resolver: &CliManagedNpmResolver,
node_resolver: &CliNodeResolver,
) -> Result<HashMap<String, Rc<dyn ShellCommand>>, AnyError> {
let mut result = HashMap::new();
let snapshot = npm_resolver.snapshot();
for id in snapshot.top_level_packages() {
let package_folder = npm_resolver.resolve_pkg_folder_from_pkg_id(id)?;
for id in npm_resolver.resolution().top_level_packages() {
let package_folder = npm_resolver.resolve_pkg_folder_from_pkg_id(&id)?;
let bin_commands =
node_resolver.resolve_binary_commands(&package_folder)?;
for bin_command in bin_commands {
@ -598,7 +596,7 @@ async fn listen_ctrl_c(kill_signal: KillSignal) {
#[cfg(unix)]
async fn listen_and_forward_all_signals(kill_signal: KillSignal) {
use deno_core::futures::FutureExt;
use deno_runtime::signal::SIGNAL_NUMS;
use deno_runtime::deno_os::signal::SIGNAL_NUMS;
// listen and forward every signal we support
let mut futures = Vec::with_capacity(SIGNAL_NUMS.len());

33
cli/tools/bench/mod.rs
View file

@ -48,6 +48,7 @@ use crate::util::fs::collect_specifiers;
use crate::util::path::is_script_ext;
use crate::util::path::matches_pattern_or_exact_path;
use crate::worker::CliMainWorkerFactory;
use crate::worker::CreateCustomWorkerError;
mod mitata;
mod reporters;
@ -164,7 +165,7 @@ async fn bench_specifier(
.await
{
Ok(()) => Ok(()),
Err(CoreError::Js(error)) => {
Err(CreateCustomWorkerError::Core(CoreError::Js(error))) => {
sender.send(BenchEvent::UncaughtError(
specifier.to_string(),
Box::new(error),
@ -182,7 +183,7 @@ async fn bench_specifier_inner(
specifier: ModuleSpecifier,
sender: &UnboundedSender<BenchEvent>,
filter: TestFilter,
) -> Result<(), CoreError> {
) -> Result<(), CreateCustomWorkerError> {
let mut worker = worker_factory
.create_custom_worker(
WorkerExecutionMode::Bench,
@ -201,7 +202,7 @@ async fn bench_specifier_inner(
// Ensure that there are no pending exceptions before we start running tests
worker.run_up_to_duration(Duration::from_millis(0)).await?;
worker.dispatch_load_event()?;
worker.dispatch_load_event().map_err(CoreError::Js)?;
let benchmarks = {
let state_rc = worker.js_runtime.op_state();
@ -236,11 +237,13 @@ async fn bench_specifier_inner(
used_only,
names: benchmarks.iter().map(|(d, _)| d.name.clone()).collect(),
}))
.map_err(JsErrorBox::from_err)?;
.map_err(JsErrorBox::from_err)
.map_err(CoreError::JsBox)?;
for (desc, function) in benchmarks {
sender
.send(BenchEvent::Wait(desc.id))
.map_err(JsErrorBox::from_err)?;
.map_err(JsErrorBox::from_err)
.map_err(CoreError::JsBox)?;
let call = worker.js_runtime.call(&function);
let result = worker
.js_runtime
@ -249,18 +252,26 @@ async fn bench_specifier_inner(
let scope = &mut worker.js_runtime.handle_scope();
let result = v8::Local::new(scope, result);
let result = serde_v8::from_v8::<BenchResult>(scope, result)
.map_err(JsErrorBox::from_err)?;
.map_err(JsErrorBox::from_err)
.map_err(CoreError::JsBox)?;
sender
.send(BenchEvent::Result(desc.id, result))
.map_err(JsErrorBox::from_err)?;
.map_err(JsErrorBox::from_err)
.map_err(CoreError::JsBox)?;
}
// Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
// event loop to continue beyond what's needed to await results.
worker.dispatch_beforeunload_event()?;
worker.dispatch_process_beforeexit_event()?;
worker.dispatch_unload_event()?;
worker.dispatch_process_exit_event()?;
worker
.dispatch_beforeunload_event()
.map_err(CoreError::Js)?;
worker
.dispatch_process_beforeexit_event()
.map_err(CoreError::Js)?;
worker.dispatch_unload_event().map_err(CoreError::Js)?;
worker
.dispatch_process_exit_event()
.map_err(CoreError::Js)?;
// Ensure the worker has settled so we can catch any remaining unhandled rejections. We don't
// want to wait forever here.

102
cli/tools/check.rs
View file

@ -13,6 +13,7 @@ use deno_graph::Module;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleLoadError;
use deno_semver::npm::NpmPackageNvReference;
use deno_terminal::colors;
use once_cell::sync::Lazy;
use regex::Regex;
@ -34,6 +35,7 @@ use crate::graph_util::maybe_additional_sloppy_imports_message;
use crate::graph_util::BuildFastCheckGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeResolver;
use crate::npm::installer::NpmInstaller;
use crate::npm::CliNpmResolver;
use crate::sys::CliSys;
use crate::tsc;
@ -109,8 +111,9 @@ pub struct TypeChecker {
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
npm_installer: Option<Arc<NpmInstaller>>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_resolver: CliNpmResolver,
sys: CliSys,
}
@ -136,13 +139,15 @@ pub enum CheckError {
}
impl TypeChecker {
#[allow(clippy::too_many_arguments)]
pub fn new(
caches: Arc<Caches>,
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolver: CliNpmResolver,
sys: CliSys,
) -> Self {
Self {
@ -151,6 +156,7 @@ impl TypeChecker {
cli_options,
module_graph_builder,
node_resolver,
npm_installer,
npm_resolver,
sys,
}
@ -184,6 +190,29 @@ impl TypeChecker {
mut graph: ModuleGraph,
options: CheckOptions,
) -> Result<(Arc<ModuleGraph>, Diagnostics), CheckError> {
fn check_state_hash(resolver: &CliNpmResolver) -> Option<u64> {
match resolver {
CliNpmResolver::Byonm(_) => {
// not feasible and probably slower to compute
None
}
CliNpmResolver::Managed(resolver) => {
// we should probably go further and check all the individual npm packages
let mut package_reqs = resolver.resolution().package_reqs();
package_reqs.sort_by(|a, b| a.0.cmp(&b.0)); // determinism
let mut hasher = FastInsecureHasher::new_without_deno_version();
// ensure the cache gets busted when turning nodeModulesDir on or off
// as this could cause changes in resolution
hasher.write_hashable(resolver.root_node_modules_path().is_some());
for (pkg_req, pkg_nv) in package_reqs {
hasher.write_hashable(&pkg_req);
hasher.write_hashable(&pkg_nv);
}
Some(hasher.finish())
}
}
}
if !options.type_check_mode.is_true() || graph.roots.is_empty() {
return Ok((graph.into(), Default::default()));
}
@ -191,9 +220,9 @@ impl TypeChecker {
// node built-in specifiers use the @types/node package to determine
// types, so inject that now (the caller should do this after the lockfile
// has been written)
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
if let Some(npm_installer) = &self.npm_installer {
if graph.has_node_specifier {
npm_resolver.inject_synthetic_types_node_package().await?;
npm_installer.inject_synthetic_types_node_package().await?;
}
}
@ -233,9 +262,11 @@ impl TypeChecker {
maybe_check_hash,
} = get_tsc_roots(
&self.sys,
&self.npm_resolver,
&self.node_resolver,
&graph,
check_js,
self.npm_resolver.check_state_hash(),
check_state_hash(&self.npm_resolver),
type_check_mode,
&ts_config,
);
@ -345,8 +376,11 @@ struct TscRoots {
/// redirects resolved. We need to include all the emittable files in
/// the roots, so they get type checked and optionally emitted,
/// otherwise they would be ignored if only imported into JavaScript.
#[allow(clippy::too_many_arguments)]
fn get_tsc_roots(
sys: &CliSys,
npm_resolver: &CliNpmResolver,
node_resolver: &CliNodeResolver,
graph: &ModuleGraph,
check_js: bool,
npm_cache_state_hash: Option<u64>,
@ -429,6 +463,7 @@ fn get_tsc_roots(
if let Some(hasher) = hasher {
hasher.write_str(module.specifier.as_str());
}
None
}
}
@ -465,17 +500,33 @@ fn get_tsc_roots(
let mut pending = VecDeque::new();
// put in the global types first so that they're resolved before anything else
let get_import_specifiers = || {
graph
.imports
for (referrer, import) in graph.imports.iter() {
for specifier in import
.dependencies
.values()
.flat_map(|i| i.dependencies.values())
.filter_map(|dep| dep.get_type().or_else(|| dep.get_code()))
};
for specifier in get_import_specifiers() {
let specifier = graph.resolve(specifier);
if seen.insert(specifier) {
pending.push_back((specifier, false));
{
let specifier = graph.resolve(specifier);
if seen.insert(specifier) {
if let Ok(nv_ref) = NpmPackageNvReference::from_specifier(specifier) {
let Some(resolved) =
resolve_npm_nv_ref(npm_resolver, node_resolver, &nv_ref, referrer)
else {
result.missing_diagnostics.push(
tsc::Diagnostic::from_missing_error(
specifier,
None,
maybe_additional_sloppy_imports_message(sys, specifier),
),
);
continue;
};
let mt = MediaType::from_specifier(&resolved);
result.roots.push((resolved, mt));
} else {
pending.push_back((specifier, false));
}
}
}
}
@ -596,6 +647,29 @@ fn get_tsc_roots(
result
}
fn resolve_npm_nv_ref(
npm_resolver: &CliNpmResolver,
node_resolver: &CliNodeResolver,
nv_ref: &NpmPackageNvReference,
referrer: &ModuleSpecifier,
) -> Option<ModuleSpecifier> {
let pkg_dir = npm_resolver
.as_managed()
.unwrap()
.resolve_pkg_folder_from_deno_module(nv_ref.nv())
.ok()?;
let resolved = node_resolver
.resolve_package_subpath_from_deno_module(
&pkg_dir,
nv_ref.sub_path(),
Some(referrer),
node_resolver::ResolutionMode::Import,
node_resolver::NodeResolutionKind::Types,
)
.ok()?;
Some(resolved)
}
/// Matches the `@ts-check` pragma.
static TS_CHECK_RE: Lazy<Regex> =
lazy_regex::lazy_regex!(r#"(?i)^\s*@ts-check(?:\s+|$)"#);

2
cli/tools/clean.rs
View file

@ -4,8 +4,8 @@ use std::path::Path;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_lib::cache::DenoDir;
use crate::cache::DenoDir;
use crate::colors;
use crate::display;
use crate::sys::CliSys;

32
cli/tools/coverage/mod.rs
View file

@ -18,10 +18,13 @@ use deno_config::glob::PathOrPatternSet;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::error::CoreError;
use deno_core::serde_json;
use deno_core::sourcemap::SourceMap;
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use deno_error::JsErrorBox;
use deno_resolver::npm::DenoInNpmPackageChecker;
use node_resolver::InNpmPackageChecker;
use regex::Regex;
use text_lines::TextLines;
@ -52,7 +55,7 @@ pub struct CoverageCollector {
#[async_trait::async_trait(?Send)]
impl crate::worker::CoverageCollector for CoverageCollector {
async fn start_collecting(&mut self) -> Result<(), AnyError> {
async fn start_collecting(&mut self) -> Result<(), CoreError> {
self.enable_debugger().await?;
self.enable_profiler().await?;
self
@ -66,7 +69,7 @@ impl crate::worker::CoverageCollector for CoverageCollector {
Ok(())
}
async fn stop_collecting(&mut self) -> Result<(), AnyError> {
async fn stop_collecting(&mut self) -> Result<(), CoreError> {
fs::create_dir_all(&self.dir)?;
let script_coverages = self.take_precise_coverage().await?.result;
@ -87,7 +90,8 @@ impl crate::worker::CoverageCollector for CoverageCollector {
let filepath = self.dir.join(filename);
let mut out = BufWriter::new(File::create(&filepath)?);
let coverage = serde_json::to_string(&script_coverage)?;
let coverage = serde_json::to_string(&script_coverage)
.map_err(JsErrorBox::from_err)?;
let formatted_coverage =
format_json(&filepath, &coverage, &Default::default())
.ok()
@ -110,7 +114,7 @@ impl CoverageCollector {
Self { dir, session }
}
async fn enable_debugger(&mut self) -> Result<(), AnyError> {
async fn enable_debugger(&mut self) -> Result<(), CoreError> {
self
.session
.post_message::<()>("Debugger.enable", None)
@ -118,7 +122,7 @@ impl CoverageCollector {
Ok(())
}
async fn enable_profiler(&mut self) -> Result<(), AnyError> {
async fn enable_profiler(&mut self) -> Result<(), CoreError> {
self
.session
.post_message::<()>("Profiler.enable", None)
@ -126,7 +130,7 @@ impl CoverageCollector {
Ok(())
}
async fn disable_debugger(&mut self) -> Result<(), AnyError> {
async fn disable_debugger(&mut self) -> Result<(), CoreError> {
self
.session
.post_message::<()>("Debugger.disable", None)
@ -134,7 +138,7 @@ impl CoverageCollector {
Ok(())
}
async fn disable_profiler(&mut self) -> Result<(), AnyError> {
async fn disable_profiler(&mut self) -> Result<(), CoreError> {
self
.session
.post_message::<()>("Profiler.disable", None)
@ -145,26 +149,28 @@ impl CoverageCollector {
async fn start_precise_coverage(
&mut self,
parameters: cdp::StartPreciseCoverageArgs,
) -> Result<cdp::StartPreciseCoverageResponse, AnyError> {
) -> Result<cdp::StartPreciseCoverageResponse, CoreError> {
let return_value = self
.session
.post_message("Profiler.startPreciseCoverage", Some(parameters))
.await?;
let return_object = serde_json::from_value(return_value)?;
let return_object =
serde_json::from_value(return_value).map_err(JsErrorBox::from_err)?;
Ok(return_object)
}
async fn take_precise_coverage(
&mut self,
) -> Result<cdp::TakePreciseCoverageResponse, AnyError> {
) -> Result<cdp::TakePreciseCoverageResponse, CoreError> {
let return_value = self
.session
.post_message::<()>("Profiler.takePreciseCoverage", None)
.await?;
let return_object = serde_json::from_value(return_value)?;
let return_object =
serde_json::from_value(return_value).map_err(JsErrorBox::from_err)?;
Ok(return_object)
}
@ -464,7 +470,7 @@ fn filter_coverages(
coverages: Vec<cdp::ScriptCoverage>,
include: Vec<String>,
exclude: Vec<String>,
in_npm_pkg_checker: &dyn InNpmPackageChecker,
in_npm_pkg_checker: &DenoInNpmPackageChecker,
) -> Vec<cdp::ScriptCoverage> {
let include: Vec<Regex> =
include.iter().map(|e| Regex::new(e).unwrap()).collect();
@ -532,7 +538,7 @@ pub fn cover_files(
script_coverages,
coverage_flags.include,
coverage_flags.exclude,
in_npm_pkg_checker.as_ref(),
in_npm_pkg_checker,
);
if script_coverages.is_empty() {
return Err(anyhow!("No covered files included in the report"));

61
cli/tools/info.rs
View file

@ -18,6 +18,7 @@ use deno_graph::Module;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_lib::util::checksum;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageId;
@ -32,9 +33,7 @@ use crate::args::InfoFlags;
use crate::display;
use crate::factory::CliFactory;
use crate::graph_util::graph_exit_integrity_errors;
use crate::npm::CliNpmResolver;
use crate::npm::ManagedCliNpmResolver;
use crate::util::checksum;
use crate::npm::CliManagedNpmResolver;
use crate::util::display::DisplayTreeNode;
const JSON_SCHEMA_VERSION: u8 = 1;
@ -138,6 +137,10 @@ pub async fn info(
lockfile.write_if_changed()?;
}
let maybe_npm_info = npm_resolver
.as_managed()
.map(|r| (r, r.resolution().snapshot()));
if info_flags.json {
let mut json_graph = serde_json::json!(graph);
if let Some(output) = json_graph.as_object_mut() {
@ -148,11 +151,19 @@ pub async fn info(
);
}
add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref(), npmrc);
add_npm_packages_to_json(
&mut json_graph,
maybe_npm_info.as_ref().map(|(_, s)| s),
npmrc,
);
display::write_json_to_stdout(&json_graph)?;
} else {
let mut output = String::new();
GraphDisplayContext::write(&graph, npm_resolver.as_ref(), &mut output)?;
GraphDisplayContext::write(
&graph,
maybe_npm_info.as_ref().map(|(r, s)| (*r, s)),
&mut output,
)?;
display::write_to_stdout_ignore_sigpipe(output.as_bytes())?;
}
} else {
@ -180,7 +191,7 @@ fn print_cache_info(
let registry_cache = dir.registries_folder_path();
let mut origin_dir = dir.origin_data_folder_path();
let deno_dir = dir.root_path_for_display().to_string();
let web_cache_dir = crate::worker::get_cache_storage_dir();
let web_cache_dir = deno_lib::worker::get_cache_storage_dir();
if let Some(location) = &location {
origin_dir =
@ -251,15 +262,14 @@ fn print_cache_info(
fn add_npm_packages_to_json(
json: &mut serde_json::Value,
npm_resolver: &dyn CliNpmResolver,
npm_snapshot: Option<&NpmResolutionSnapshot>,
npmrc: &ResolvedNpmRc,
) {
let Some(npm_resolver) = npm_resolver.as_managed() else {
let Some(npm_snapshot) = npm_snapshot else {
return; // does not include byonm to deno info's output
};
// ideally deno_graph could handle this, but for now we just modify the json here
let snapshot = npm_resolver.snapshot();
let json = json.as_object_mut().unwrap();
let modules = json.get_mut("modules").and_then(|m| m.as_array_mut());
if let Some(modules) = modules {
@ -273,7 +283,7 @@ fn add_npm_packages_to_json(
.and_then(|k| k.as_str())
.and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok())
.and_then(|package_ref| {
snapshot
npm_snapshot
.resolve_package_from_deno_module(package_ref.nv())
.ok()
});
@ -295,7 +305,8 @@ fn add_npm_packages_to_json(
if let Some(specifier) = dep.get("specifier").and_then(|s| s.as_str())
{
if let Ok(npm_ref) = NpmPackageReqReference::from_str(specifier) {
if let Ok(pkg) = snapshot.resolve_pkg_from_pkg_req(npm_ref.req())
if let Ok(pkg) =
npm_snapshot.resolve_pkg_from_pkg_req(npm_ref.req())
{
dep.insert(
"npmPackage".to_string(),
@ -321,8 +332,9 @@ fn add_npm_packages_to_json(
}
}
let mut sorted_packages =
snapshot.all_packages_for_every_system().collect::<Vec<_>>();
let mut sorted_packages = npm_snapshot
.all_packages_for_every_system()
.collect::<Vec<_>>();
sorted_packages.sort_by(|a, b| a.id.cmp(&b.id));
let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len());
for pkg in sorted_packages {
@ -356,7 +368,7 @@ struct NpmInfo {
impl NpmInfo {
pub fn build<'a>(
graph: &'a ModuleGraph,
npm_resolver: &'a ManagedCliNpmResolver,
npm_resolver: &'a CliManagedNpmResolver,
npm_snapshot: &'a NpmResolutionSnapshot,
) -> Self {
let mut info = NpmInfo::default();
@ -382,12 +394,15 @@ impl NpmInfo {
fn fill_package_info<'a>(
&mut self,
package: &NpmResolutionPackage,
npm_resolver: &'a ManagedCliNpmResolver,
npm_resolver: &'a CliManagedNpmResolver,
npm_snapshot: &'a NpmResolutionSnapshot,
) {
self.packages.insert(package.id.clone(), package.clone());
if let Ok(size) = npm_resolver.package_size(&package.id) {
self.package_sizes.insert(package.id.clone(), size);
if let Ok(folder) = npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)
{
if let Ok(size) = crate::util::fs::dir_size(&folder) {
self.package_sizes.insert(package.id.clone(), size);
}
}
for id in package.dependencies.values() {
if !self.packages.contains_key(id) {
@ -416,13 +431,15 @@ struct GraphDisplayContext<'a> {
impl<'a> GraphDisplayContext<'a> {
pub fn write<TWrite: Write>(
graph: &'a ModuleGraph,
npm_resolver: &'a dyn CliNpmResolver,
managed_npm_info: Option<(
&'a CliManagedNpmResolver,
&'a NpmResolutionSnapshot,
)>,
writer: &mut TWrite,
) -> Result<(), AnyError> {
let npm_info = match npm_resolver.as_managed() {
Some(npm_resolver) => {
let npm_snapshot = npm_resolver.snapshot();
NpmInfo::build(graph, npm_resolver, &npm_snapshot)
let npm_info = match managed_npm_info {
Some((npm_resolver, npm_snapshot)) => {
NpmInfo::build(graph, npm_resolver, npm_snapshot)
}
None => NpmInfo::default(),
};

4
cli/tools/installer.rs
View file

@ -300,8 +300,8 @@ async fn install_local(
InstallFlagsLocal::TopLevel => {
let factory = CliFactory::from_flags(flags);
// surface any errors in the package.json
if let Some(npm_resolver) = factory.npm_resolver().await?.as_managed() {
npm_resolver.ensure_no_pkg_json_dep_errors()?;
if let Some(npm_installer) = factory.npm_installer_if_managed()? {
npm_installer.ensure_no_pkg_json_dep_errors()?;
}
crate::tools::registry::cache_top_level_deps(&factory, None).await?;

4
cli/tools/jupyter/mod.rs
View file

@ -67,7 +67,7 @@ pub async fn kernel(
// TODO(bartlomieju): should we run with all permissions?
let permissions =
PermissionsContainer::allow_all(factory.permission_desc_parser()?.clone());
let npm_resolver = factory.npm_resolver().await?.clone();
let npm_installer = factory.npm_installer_if_managed()?.cloned();
let resolver = factory.resolver().await?.clone();
let worker_factory = factory.create_cli_main_worker_factory().await?;
let (stdio_tx, stdio_rx) = mpsc::unbounded_channel();
@ -115,7 +115,7 @@ pub async fn kernel(
let worker = worker.into_main_worker();
let mut repl_session = repl::ReplSession::initialize(
cli_options,
npm_resolver,
npm_installer,
resolver,
worker,
main_module,

591
cli/tools/lint/ast_buffer/buffer.rs
View file

@ -14,9 +14,14 @@ pub enum PropFlags {
Ref,
RefArr,
String,
Number,
Bool,
Null,
Undefined,
Object,
Regex,
BigInt,
Array,
}
impl From<PropFlags> for u8 {
@ -33,21 +38,29 @@ impl TryFrom<u8> for PropFlags {
0 => Ok(PropFlags::Ref),
1 => Ok(PropFlags::RefArr),
2 => Ok(PropFlags::String),
3 => Ok(PropFlags::Bool),
4 => Ok(PropFlags::Null),
5 => Ok(PropFlags::Undefined),
3 => Ok(PropFlags::Number),
4 => Ok(PropFlags::Bool),
5 => Ok(PropFlags::Null),
6 => Ok(PropFlags::Undefined),
7 => Ok(PropFlags::Object),
8 => Ok(PropFlags::Regex),
9 => Ok(PropFlags::BigInt),
10 => Ok(PropFlags::Array),
_ => Err("Unknown Prop flag"),
}
}
}
pub type Index = u32;
const GROUP_KIND: u8 = 1;
const MASK_U32_1: u32 = 0b11111111_00000000_00000000_00000000;
const MASK_U32_2: u32 = 0b00000000_11111111_00000000_00000000;
const MASK_U32_3: u32 = 0b00000000_00000000_11111111_00000000;
const MASK_U32_4: u32 = 0b00000000_00000000_00000000_11111111;
// TODO: There is probably a native Rust function to do this.
pub fn append_u32(result: &mut Vec<u8>, value: u32) {
#[inline]
fn append_u32(result: &mut Vec<u8>, value: u32) {
let v1: u8 = ((value & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((value & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((value & MASK_U32_3) >> 8) as u8;
@ -59,25 +72,11 @@ pub fn append_u32(result: &mut Vec<u8>, value: u32) {
result.push(v4);
}
pub fn append_usize(result: &mut Vec<u8>, value: usize) {
fn append_usize(result: &mut Vec<u8>, value: usize) {
let raw = u32::try_from(value).unwrap();
append_u32(result, raw);
}
pub fn write_usize(result: &mut [u8], value: usize, idx: usize) {
let raw = u32::try_from(value).unwrap();
let v1: u8 = ((raw & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((raw & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((raw & MASK_U32_3) >> 8) as u8;
let v4: u8 = (raw & MASK_U32_4) as u8;
result[idx] = v1;
result[idx + 1] = v2;
result[idx + 2] = v3;
result[idx + 3] = v4;
}
#[derive(Debug)]
pub struct StringTable {
id: usize,
@ -119,71 +118,47 @@ impl StringTable {
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct NodeRef(pub usize);
/// Represents an offset to a node whose schema hasn't been committed yet
pub struct NodeRef(pub Index);
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct PendingNodeRef(pub NodeRef);
pub struct PendingRef(pub Index);
#[derive(Debug)]
pub struct BoolPos(pub usize);
#[derive(Debug)]
pub struct FieldPos(pub usize);
#[derive(Debug)]
pub struct FieldArrPos(pub usize);
#[derive(Debug)]
pub struct StrPos(pub usize);
#[derive(Debug)]
pub struct UndefPos(pub usize);
#[derive(Debug)]
pub struct NullPos(pub usize);
#[derive(Debug)]
pub enum NodePos {
Bool(BoolPos),
#[allow(dead_code)]
Field(FieldPos),
#[allow(dead_code)]
FieldArr(FieldArrPos),
Str(StrPos),
Undef(UndefPos),
#[allow(dead_code)]
Null(NullPos),
pub trait AstBufSerializer {
fn serialize(&mut self) -> Vec<u8>;
}
pub trait AstBufSerializer<K, P>
where
K: Into<u8> + Display,
P: Into<u8> + Display,
{
fn header(&mut self, kind: K, parent: NodeRef, span: &Span)
-> PendingNodeRef;
fn ref_field(&mut self, prop: P) -> FieldPos;
fn ref_vec_field(&mut self, prop: P, len: usize) -> FieldArrPos;
fn str_field(&mut self, prop: P) -> StrPos;
fn bool_field(&mut self, prop: P) -> BoolPos;
fn undefined_field(&mut self, prop: P) -> UndefPos;
#[allow(dead_code)]
fn null_field(&mut self, prop: P) -> NullPos;
fn commit_schema(&mut self, offset: PendingNodeRef) -> NodeRef;
fn write_ref(&mut self, pos: FieldPos, value: NodeRef);
fn write_maybe_ref(&mut self, pos: FieldPos, value: Option<NodeRef>);
fn write_refs(&mut self, pos: FieldArrPos, value: Vec<NodeRef>);
fn write_str(&mut self, pos: StrPos, value: &str);
fn write_bool(&mut self, pos: BoolPos, value: bool);
fn serialize(&mut self) -> Vec<u8>;
/// <type u8>
/// <prop offset u32>
/// <child idx u32>
/// <next idx u32>
/// <parent idx u32>
#[derive(Debug)]
struct Node {
kind: u8,
prop_offset: u32,
child: u32,
next: u32,
parent: u32,
}
#[derive(Debug)]
pub struct SerializeCtx {
buf: Vec<u8>,
start_buf: NodeRef,
root_idx: Index,
nodes: Vec<Node>,
prop_stack: Vec<Vec<u8>>,
field_count: Vec<usize>,
field_buf: Vec<u8>,
prev_sibling_stack: Vec<Index>,
/// Vec of spans
spans: Vec<u32>,
/// Maps string id to the actual string
str_table: StringTable,
kind_map: Vec<usize>,
prop_map: Vec<usize>,
field_count: u8,
/// Maps kind id to string id
kind_name_map: Vec<usize>,
/// Maps prop id to string id
prop_name_map: Vec<usize>,
}
/// This is the internal context used to allocate and fill the buffer. The point
@ -198,20 +173,24 @@ impl SerializeCtx {
let kind_size = kind_len as usize;
let prop_size = prop_len as usize;
let mut ctx = Self {
start_buf: NodeRef(0),
buf: vec![],
spans: Vec::with_capacity(512),
root_idx: 0,
nodes: Vec::with_capacity(512),
prop_stack: vec![vec![]],
prev_sibling_stack: vec![0],
field_count: vec![0],
field_buf: Vec::with_capacity(1024),
str_table: StringTable::new(),
kind_map: vec![0; kind_size],
prop_map: vec![0; prop_size],
field_count: 0,
kind_name_map: vec![0; kind_size],
prop_name_map: vec![0; prop_size],
};
let empty_str = ctx.str_table.insert("");
// Placeholder node is always 0
ctx.append_node(0, NodeRef(0), &DUMMY_SP, 0);
ctx.kind_map[0] = empty_str;
ctx.start_buf = NodeRef(ctx.buf.len());
ctx.append_node(0, &DUMMY_SP);
ctx.kind_name_map[0] = empty_str;
ctx.kind_name_map[1] = empty_str;
// Insert default props that are always present
let type_str = ctx.str_table.insert("type");
@ -220,258 +199,306 @@ impl SerializeCtx {
let length_str = ctx.str_table.insert("length");
// These values are expected to be in this order on the JS side
ctx.prop_map[0] = empty_str;
ctx.prop_map[1] = type_str;
ctx.prop_map[2] = parent_str;
ctx.prop_map[3] = range_str;
ctx.prop_map[4] = length_str;
ctx.prop_name_map[0] = empty_str;
ctx.prop_name_map[1] = type_str;
ctx.prop_name_map[2] = parent_str;
ctx.prop_name_map[3] = range_str;
ctx.prop_name_map[4] = length_str;
ctx
}
pub fn set_root_idx(&mut self, idx: Index) {
self.root_idx = idx;
}
/// Allocate a node's header
fn field_header<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
fn field_header<P>(&mut self, prop: P, prop_flags: PropFlags)
where
P: Into<u8> + Display + Clone,
{
self.field_count += 1;
let offset = self.buf.len();
let flags: u8 = prop_flags.into();
let n: u8 = prop.clone().into();
self.buf.push(n);
if let Some(v) = self.prop_map.get::<usize>(n.into()) {
if let Some(v) = self.prop_name_map.get::<usize>(n.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{prop}"));
self.prop_map[n as usize] = id;
self.prop_name_map[n as usize] = id;
}
}
let flags: u8 = prop_flags.into();
self.buf.push(flags);
// Increment field counter
let idx = self.field_count.len() - 1;
let count = self.field_count[idx];
self.field_count[idx] = count + 1;
offset
let buf = self.prop_stack.last_mut().unwrap();
buf.push(n);
buf.push(flags);
}
/// Allocate a property pointing to another node.
fn field<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
fn get_node(&mut self, id: Index) -> &mut Node {
self.nodes.get_mut(id as usize).unwrap()
}
fn set_parent(&mut self, child_id: Index, parent_id: Index) {
let child = self.get_node(child_id);
child.parent = parent_id;
}
fn set_child(&mut self, parent_id: Index, child_id: Index) {
let parent = self.get_node(parent_id);
parent.child = child_id;
}
fn set_next(&mut self, node_id: Index, next_id: Index) {
let node = self.get_node(node_id);
node.next = next_id;
}
fn update_ref_links(&mut self, parent_id: Index, child_id: Index) {
let last_idx = self.prev_sibling_stack.len() - 1;
let parent = self.get_node(parent_id);
if parent.child == 0 {
parent.child = child_id;
} else {
let prev_id = self.prev_sibling_stack[last_idx];
self.set_next(prev_id, child_id);
}
self.prev_sibling_stack[last_idx] = child_id;
self.set_parent(child_id, parent_id);
}
pub fn append_node<K>(&mut self, kind: K, span: &Span) -> PendingRef
where
P: Into<u8> + Display + Clone,
K: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, prop_flags);
append_usize(&mut self.buf, 0);
offset
self.append_inner(kind, span.lo.0, span.hi.0)
}
fn append_node(
pub fn append_inner<K>(
&mut self,
kind: u8,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> PendingNodeRef {
let offset = self.buf.len();
// Node type fits in a u8
self.buf.push(kind);
// Offset to the parent node. Will be 0 if none exists
append_usize(&mut self.buf, parent.0);
// Span, the start and end location of this node
append_u32(&mut self.buf, span.lo.0);
append_u32(&mut self.buf, span.hi.0);
// No node has more than <10 properties
debug_assert!(prop_count < 10);
self.buf.push(prop_count as u8);
PendingNodeRef(NodeRef(offset))
}
pub fn commit_schema(&mut self, node_ref: PendingNodeRef) -> NodeRef {
let mut offset = node_ref.0 .0;
// type + parentId + span lo + span hi
offset += 1 + 4 + 4 + 4;
self.buf[offset] = self.field_count;
self.field_count = 0;
node_ref.0
}
/// Allocate the node header. It's always the same for every node.
/// <type u8>
/// <parent offset u32>
/// <span lo u32>
/// <span high u32>
/// <property count u8> (There is no node with more than 10 properties)
pub fn header<N>(
&mut self,
kind: N,
parent: NodeRef,
span: &Span,
) -> PendingNodeRef
kind: K,
span_lo: u32,
span_hi: u32,
) -> PendingRef
where
N: Into<u8> + Display + Clone,
K: Into<u8> + Display + Clone,
{
let n: u8 = kind.clone().into();
let kind_u8: u8 = kind.clone().into();
if let Some(v) = self.kind_map.get::<usize>(n.into()) {
let id: Index = self.nodes.len() as u32;
self.nodes.push(Node {
kind: kind_u8,
prop_offset: 0,
child: 0,
next: 0,
parent: 0,
});
if let Some(v) = self.kind_name_map.get::<usize>(kind_u8.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{kind}"));
self.kind_map[n as usize] = id;
let s_id = self.str_table.insert(&format!("{kind}"));
self.kind_name_map[kind_u8 as usize] = s_id;
}
}
// Prop count will be filled with the actual value when the
// schema is committed.
self.append_node(n, parent, span, 0)
self.field_count.push(0);
self.prop_stack.push(vec![]);
self.prev_sibling_stack.push(0);
// write spans
self.spans.push(span_lo);
self.spans.push(span_hi);
PendingRef(id)
}
/// Allocate a reference property that will hold the offset of
/// another node.
pub fn ref_field<P>(&mut self, prop: P) -> usize
pub fn commit_node(&mut self, id: PendingRef) -> NodeRef {
let mut buf = self.prop_stack.pop().unwrap();
let count = self.field_count.pop().unwrap();
let offset = self.field_buf.len();
// All nodes have <10 fields
self.field_buf.push(count as u8);
self.field_buf.append(&mut buf);
let node = self.nodes.get_mut(id.0 as usize).unwrap();
node.prop_offset = offset as u32;
self.prev_sibling_stack.pop();
NodeRef(id.0)
}
// Allocate an object field
pub fn open_obj(&mut self) {
self.field_count.push(0);
self.prop_stack.push(vec![]);
}
pub fn commit_obj<P>(&mut self, prop: P)
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::Ref)
let mut buf = self.prop_stack.pop().unwrap();
let count = self.field_count.pop().unwrap();
let offset = self.field_buf.len();
append_usize(&mut self.field_buf, count);
self.field_buf.append(&mut buf);
self.field_header(prop, PropFlags::Object);
let buf = self.prop_stack.last_mut().unwrap();
append_usize(buf, offset);
}
/// Allocate a property that is a vec of node offsets pointing to other
/// nodes.
pub fn ref_vec_field<P>(&mut self, prop: P, len: usize) -> usize
/// Allocate an null field
pub fn write_null<P>(&mut self, prop: P)
where
P: Into<u8> + Display + Clone,
{
let offset = self.field(prop, PropFlags::RefArr);
self.field_header(prop, PropFlags::Null);
for _ in 0..len {
append_u32(&mut self.buf, 0);
}
offset
let buf = self.prop_stack.last_mut().unwrap();
append_u32(buf, 0);
}
// Allocate a property representing a string. Strings are deduplicated
// in the message and the property will only contain the string id.
pub fn str_field<P>(&mut self, prop: P) -> usize
/// Allocate an null field
pub fn write_undefined<P>(&mut self, prop: P)
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::String)
self.field_header(prop, PropFlags::Undefined);
let buf = self.prop_stack.last_mut().unwrap();
append_u32(buf, 0);
}
/// Allocate a bool field
pub fn bool_field<P>(&mut self, prop: P) -> usize
/// Allocate a number field
pub fn write_num<P>(&mut self, prop: P, value: &str)
where
P: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, PropFlags::Bool);
self.buf.push(0);
offset
self.field_header(prop, PropFlags::Number);
let id = self.str_table.insert(value);
let buf = self.prop_stack.last_mut().unwrap();
append_usize(buf, id);
}
/// Allocate an undefined field
pub fn undefined_field<P>(&mut self, prop: P) -> usize
/// Allocate a bigint field
pub fn write_bigint<P>(&mut self, prop: P, value: &str)
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Undefined)
self.field_header(prop, PropFlags::BigInt);
let id = self.str_table.insert(value);
let buf = self.prop_stack.last_mut().unwrap();
append_usize(buf, id);
}
/// Allocate an undefined field
#[allow(dead_code)]
pub fn null_field<P>(&mut self, prop: P) -> usize
/// Allocate a RegExp field
pub fn write_regex<P>(&mut self, prop: P, value: &str)
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Null)
}
self.field_header(prop, PropFlags::Regex);
/// Replace the placeholder of a reference field with the actual offset
/// to the node we want to point to.
pub fn write_ref(&mut self, field_offset: usize, value: NodeRef) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
write_usize(&mut self.buf, value.0, field_offset + 2);
}
/// Helper for writing optional node offsets
pub fn write_maybe_ref(
&mut self,
field_offset: usize,
value: Option<NodeRef>,
) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
let ref_value = if let Some(v) = value { v } else { NodeRef(0) };
write_usize(&mut self.buf, ref_value.0, field_offset + 2);
}
/// Write a vec of node offsets into the property. The necessary space
/// has been reserved earlier.
pub fn write_refs(&mut self, field_offset: usize, value: Vec<NodeRef>) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::RefArr {
panic!("Trying to write a ref into a non-ref array field")
}
}
let mut offset = field_offset + 2;
write_usize(&mut self.buf, value.len(), offset);
offset += 4;
for item in value {
write_usize(&mut self.buf, item.0, offset);
offset += 4;
}
let id = self.str_table.insert(value);
let buf = self.prop_stack.last_mut().unwrap();
append_usize(buf, id);
}
/// Store the string in our string table and save the id of the string
/// in the current field.
pub fn write_str(&mut self, field_offset: usize, value: &str) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::String {
panic!("Trying to write a ref into a non-string field")
}
}
pub fn write_str<P>(&mut self, prop: P, value: &str)
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::String);
let id = self.str_table.insert(value);
write_usize(&mut self.buf, id, field_offset + 2);
let buf = self.prop_stack.last_mut().unwrap();
append_usize(buf, id);
}
/// Write a bool to a field.
pub fn write_bool(&mut self, field_offset: usize, value: bool) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Bool {
panic!("Trying to write a ref into a non-bool field")
}
}
pub fn write_bool<P>(&mut self, prop: P, value: bool)
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Bool);
self.buf[field_offset + 2] = if value { 1 } else { 0 };
let n = if value { 1 } else { 0 };
let buf = self.prop_stack.last_mut().unwrap();
append_u32(buf, n);
}
/// Replace the placeholder of a reference field with the actual offset
/// to the node we want to point to.
pub fn write_ref<P>(&mut self, prop: P, parent: &PendingRef, value: NodeRef)
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Ref);
let buf = self.prop_stack.last_mut().unwrap();
append_u32(buf, value.0);
if parent.0 > 0 {
self.update_ref_links(parent.0, value.0);
}
}
/// Helper for writing optional node offsets
pub fn write_maybe_ref<P>(
&mut self,
prop: P,
parent: &PendingRef,
value: Option<NodeRef>,
) where
P: Into<u8> + Display + Clone,
{
if let Some(v) = value {
self.write_ref(prop, parent, v);
} else {
self.write_null(prop);
};
}
/// Write a vec of node offsets into the property. The necessary space
/// has been reserved earlier.
pub fn write_ref_vec<P>(
&mut self,
prop: P,
parent_ref: &PendingRef,
value: Vec<NodeRef>,
) where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::RefArr);
let group_id = self.append_node(GROUP_KIND, &DUMMY_SP);
let group_id = self.commit_node(group_id).0;
let buf = self.prop_stack.last_mut().unwrap();
append_u32(buf, group_id);
self.update_ref_links(parent_ref.0, group_id);
let mut prev_id = 0;
for (i, item) in value.iter().enumerate() {
self.set_parent(item.0, group_id);
if i == 0 {
self.set_child(group_id, item.0);
} else {
self.set_next(prev_id, item.0);
}
prev_id = item.0;
}
}
/// Serialize all information we have into a buffer that can be sent to JS.
@ -481,6 +508,8 @@ impl SerializeCtx {
/// <string table>
/// <node kind map> <- node kind id maps to string id
/// <node prop map> <- node property id maps to string id
/// <spans> <- List of spans, rarely needed
/// <offset spans>
/// <offset kind map>
/// <offset prop map>
/// <offset str table>
@ -490,7 +519,13 @@ impl SerializeCtx {
// The buffer starts with the serialized AST first, because that
// contains absolute offsets. By butting this at the start of the
// message we don't have to waste time updating any offsets.
buf.append(&mut self.buf);
for node in &self.nodes {
buf.push(node.kind);
append_u32(&mut buf, node.prop_offset);
append_u32(&mut buf, node.child);
append_u32(&mut buf, node.next);
append_u32(&mut buf, node.parent);
}
// Next follows the string table. We'll keep track of the offset
// in the message of where the string table begins
@ -507,8 +542,8 @@ impl SerializeCtx {
// Write the total number of entries in the kind -> str mapping table
// TODO: make this a u8
append_usize(&mut buf, self.kind_map.len());
for v in &self.kind_map {
append_usize(&mut buf, self.kind_name_map.len());
for v in &self.kind_name_map {
append_usize(&mut buf, *v);
}
@ -517,19 +552,35 @@ impl SerializeCtx {
// as u8.
let offset_prop_map = buf.len();
// Write the total number of entries in the kind -> str mapping table
append_usize(&mut buf, self.prop_map.len());
for v in &self.prop_map {
append_usize(&mut buf, self.prop_name_map.len());
for v in &self.prop_name_map {
append_usize(&mut buf, *v);
}
// Spans are rarely needed, so they're stored in a separate array.
// They're indexed by the node id.
let offset_spans = buf.len();
for v in &self.spans {
append_u32(&mut buf, *v);
}
// The field value table. They're detached from nodes as they're not
// as frequently needed as the nodes themselves. The most common
// operation is traversal and we can traverse nodes without knowing
// about the fields.
let offset_props = buf.len();
buf.append(&mut self.field_buf);
// Putting offsets of relevant parts of the buffer at the end. This
// allows us to hop to the relevant part by merely looking at the last
// for values in the message. Each value represents an offset into the
// buffer.
append_usize(&mut buf, offset_props);
append_usize(&mut buf, offset_spans);
append_usize(&mut buf, offset_kind_map);
append_usize(&mut buf, offset_prop_map);
append_usize(&mut buf, offset_str_table);
append_usize(&mut buf, self.start_buf.0);
append_u32(&mut buf, self.root_idx);
buf
}

3540
cli/tools/lint/ast_buffer/swc.rs
View file

File diff suppressed because it is too large Load diff

2356
cli/tools/lint/ast_buffer/ts_estree.rs
View file

File diff suppressed because it is too large Load diff

6
cli/tools/registry/pm.rs
View file

@ -861,10 +861,8 @@ async fn npm_install_after_modification(
// make a new CliFactory to pick up the updated config file
let cli_factory = CliFactory::from_flags(flags);
// surface any errors in the package.json
let npm_resolver = cli_factory.npm_resolver().await?;
if let Some(npm_resolver) = npm_resolver.as_managed() {
npm_resolver.ensure_no_pkg_json_dep_errors()?;
}
let npm_installer = cli_factory.npm_installer()?;
npm_installer.ensure_no_pkg_json_dep_errors()?;
// npm install
cache_deps::cache_top_level_deps(&cli_factory, jsr_resolver).await?;

15
cli/tools/registry/pm/cache_deps.rs
View file

@ -12,16 +12,19 @@ use crate::factory::CliFactory;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::npm::installer::PackageCaching;
pub async fn cache_top_level_deps(
// todo(dsherret): don't pass the factory into this function. Instead use ctor deps
factory: &CliFactory,
jsr_resolver: Option<Arc<crate::jsr::JsrFetchResolver>>,
) -> Result<(), AnyError> {
let npm_resolver = factory.npm_resolver().await?;
let npm_installer = factory.npm_installer_if_managed()?;
let cli_options = factory.cli_options()?;
if let Some(npm_resolver) = npm_resolver.as_managed() {
npm_resolver.ensure_top_level_package_json_install().await?;
if let Some(npm_installer) = &npm_installer {
npm_installer
.ensure_top_level_package_json_install()
.await?;
if let Some(lockfile) = cli_options.maybe_lockfile() {
lockfile.error_if_changed()?;
}
@ -138,10 +141,8 @@ pub async fn cache_top_level_deps(
maybe_graph_error = graph_builder.graph_roots_valid(graph, &roots);
}
if let Some(npm_resolver) = npm_resolver.as_managed() {
npm_resolver
.cache_packages(crate::npm::PackageCaching::All)
.await?;
if let Some(npm_installer) = &npm_installer {
npm_installer.cache_packages(PackageCaching::All).await?;
}
maybe_graph_error?;

43
cli/tools/registry/pm/deps.rs
View file

@ -42,6 +42,7 @@ use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::jsr::JsrFetchResolver;
use crate::module_loader::ModuleLoadPreparer;
use crate::npm::installer::NpmInstaller;
use crate::npm::CliNpmResolver;
use crate::npm::NpmFetchResolver;
use crate::util::sync::AtomicFlag;
@ -450,7 +451,8 @@ pub struct DepManager {
// TODO(nathanwhit): probably shouldn't be pub
pub(crate) jsr_fetch_resolver: Arc<JsrFetchResolver>,
pub(crate) npm_fetch_resolver: Arc<NpmFetchResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_resolver: CliNpmResolver,
npm_installer: Arc<NpmInstaller>,
permissions_container: PermissionsContainer,
main_module_graph_container: Arc<MainModuleGraphContainer>,
lockfile: Option<Arc<CliLockfile>>,
@ -460,7 +462,8 @@ pub struct DepManagerArgs {
pub module_load_preparer: Arc<ModuleLoadPreparer>,
pub jsr_fetch_resolver: Arc<JsrFetchResolver>,
pub npm_fetch_resolver: Arc<NpmFetchResolver>,
pub npm_resolver: Arc<dyn CliNpmResolver>,
pub npm_installer: Arc<NpmInstaller>,
pub npm_resolver: CliNpmResolver,
pub permissions_container: PermissionsContainer,
pub main_module_graph_container: Arc<MainModuleGraphContainer>,
pub lockfile: Option<Arc<CliLockfile>>,
@ -477,6 +480,7 @@ impl DepManager {
module_load_preparer,
jsr_fetch_resolver,
npm_fetch_resolver,
npm_installer,
npm_resolver,
permissions_container,
main_module_graph_container,
@ -490,6 +494,7 @@ impl DepManager {
dependencies_resolved: AtomicFlag::lowered(),
module_load_preparer,
npm_fetch_resolver,
npm_installer,
npm_resolver,
permissions_container,
main_module_graph_container,
@ -546,9 +551,10 @@ impl DepManager {
let npm_resolver = self.npm_resolver.as_managed().unwrap();
if self.deps.iter().all(|dep| match dep.kind {
DepKind::Npm => {
npm_resolver.resolve_pkg_id_from_pkg_req(&dep.req).is_ok()
}
DepKind::Npm => npm_resolver
.resolution()
.resolve_pkg_id_from_pkg_req(&dep.req)
.is_ok(),
DepKind::Jsr => graph.packages.mappings().contains_key(&dep.req),
}) {
self.dependencies_resolved.raise();
@ -556,7 +562,10 @@ impl DepManager {
return Ok(());
}
npm_resolver.ensure_top_level_package_json_install().await?;
self
.npm_installer
.ensure_top_level_package_json_install()
.await?;
let mut roots = Vec::new();
let mut info_futures = FuturesUnordered::new();
for dep in &self.deps {
@ -622,7 +631,12 @@ impl DepManager {
let graph = self.main_module_graph_container.graph();
let mut resolved = Vec::with_capacity(self.deps.len());
let snapshot = self.npm_resolver.as_managed().unwrap().snapshot();
let snapshot = self
.npm_resolver
.as_managed()
.unwrap()
.resolution()
.snapshot();
let resolved_npm = snapshot.package_reqs();
let resolved_jsr = graph.packages.mappings();
for dep in &self.deps {
@ -669,10 +683,21 @@ impl DepManager {
.and_then(|info| {
let latest_tag = info.dist_tags.get("latest")?;
let lower_bound = &semver_compatible.as_ref()?.version;
if latest_tag > lower_bound {
if latest_tag >= lower_bound {
Some(latest_tag.clone())
} else {
latest_version(Some(latest_tag), info.versions.keys())
latest_version(
Some(latest_tag),
info.versions.iter().filter_map(
|(version, version_info)| {
if version_info.deprecated.is_none() {
Some(version)
} else {
None
}
},
),
)
}
})
.map(|version| PackageNv {

9
cli/tools/registry/pm/outdated.rs
View file

@ -280,9 +280,15 @@ fn choose_new_version_req(
if preferred.version <= resolved?.version {
return None;
}
let exact = if let Some(range) = dep.req.version_req.range() {
range.0[0].start == range.0[0].end
} else {
false
};
Some(
VersionReq::parse_from_specifier(
format!("^{}", preferred.version).as_str(),
format!("{}{}", if exact { "" } else { "^" }, preferred.version)
.as_str(),
)
.unwrap(),
)
@ -445,6 +451,7 @@ async fn dep_manager_args(
jsr_fetch_resolver,
npm_fetch_resolver,
npm_resolver: factory.npm_resolver().await?.clone(),
npm_installer: factory.npm_installer()?.clone(),
permissions_container: factory.root_permissions_container()?.clone(),
main_module_graph_container: factory
.main_module_graph_container()

4
cli/tools/repl/mod.rs
View file

@ -164,7 +164,7 @@ pub async fn run(
let cli_options = factory.cli_options()?;
let main_module = cli_options.resolve_main_module()?;
let permissions = factory.root_permissions_container()?;
let npm_resolver = factory.npm_resolver().await?.clone();
let npm_installer = factory.npm_installer_if_managed()?.cloned();
let resolver = factory.resolver().await?.clone();
let file_fetcher = factory.file_fetcher()?;
let worker_factory = factory.create_cli_main_worker_factory().await?;
@ -187,7 +187,7 @@ pub async fn run(
let worker = worker.into_main_worker();
let session = ReplSession::initialize(
cli_options,
npm_resolver,
npm_installer,
resolver,
worker,
main_module.clone(),

16
cli/tools/repl/session.rs
View file

@ -45,7 +45,7 @@ use crate::args::CliOptions;
use crate::cdp;
use crate::colors;
use crate::lsp::ReplLanguageServer;
use crate::npm::CliNpmResolver;
use crate::npm::installer::NpmInstaller;
use crate::resolver::CliResolver;
use crate::tools::test::report_tests;
use crate::tools::test::reporters::PrettyTestReporter;
@ -181,7 +181,7 @@ struct ReplJsxState {
}
pub struct ReplSession {
npm_resolver: Arc<dyn CliNpmResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
resolver: Arc<CliResolver>,
pub worker: MainWorker,
session: LocalInspectorSession,
@ -200,7 +200,7 @@ pub struct ReplSession {
impl ReplSession {
pub async fn initialize(
cli_options: &CliOptions,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
resolver: Arc<CliResolver>,
mut worker: MainWorker,
main_module: ModuleSpecifier,
@ -265,7 +265,7 @@ impl ReplSession {
)?;
let experimental_decorators = transpile_options.use_ts_decorators;
let mut repl_session = ReplSession {
npm_resolver,
npm_installer,
resolver,
worker,
session,
@ -704,8 +704,8 @@ impl ReplSession {
&mut self,
program: &swc_ast::Program,
) -> Result<(), AnyError> {
let Some(npm_resolver) = self.npm_resolver.as_managed() else {
return Ok(()); // don't auto-install for byonm
let Some(npm_installer) = &self.npm_installer else {
return Ok(());
};
let mut collector = ImportCollector::new();
@ -737,13 +737,13 @@ impl ReplSession {
let has_node_specifier =
resolved_imports.iter().any(|url| url.scheme() == "node");
if !npm_imports.is_empty() || has_node_specifier {
npm_resolver
npm_installer
.add_and_cache_package_reqs(&npm_imports)
.await?;
// prevent messages in the repl about @types/node not being cached
if has_node_specifier {
npm_resolver.inject_synthetic_types_node_package().await?;
npm_installer.inject_synthetic_types_node_package().await?;
}
}
Ok(())

12
cli/tools/run/mod.rs
View file

@ -12,6 +12,7 @@ use crate::args::EvalFlags;
use crate::args::Flags;
use crate::args::WatchFlagsWithPaths;
use crate::factory::CliFactory;
use crate::npm::installer::PackageCaching;
use crate::util;
use crate::util::file_watcher::WatcherRestartMode;
@ -202,18 +203,17 @@ pub async fn maybe_npm_install(factory: &CliFactory) -> Result<(), AnyError> {
// ensure an "npm install" is done if the user has explicitly
// opted into using a managed node_modules directory
if cli_options.node_modules_dir()? == Some(NodeModulesDirMode::Auto) {
if let Some(npm_resolver) = factory.npm_resolver().await?.as_managed() {
let already_done =
npm_resolver.ensure_top_level_package_json_install().await?;
if let Some(npm_installer) = factory.npm_installer_if_managed()? {
let already_done = npm_installer
.ensure_top_level_package_json_install()
.await?;
if !already_done
&& matches!(
cli_options.default_npm_caching_strategy(),
crate::graph_util::NpmCachingStrategy::Eager
)
{
npm_resolver
.cache_packages(crate::npm::PackageCaching::All)
.await?;
npm_installer.cache_packages(PackageCaching::All).await?;
}
}
}

10
cli/tools/serve.rs
View file

@ -43,7 +43,8 @@ pub async fn serve(
maybe_npm_install(&factory).await?;
let worker_factory = factory.create_cli_main_worker_factory().await?;
let worker_factory =
Arc::new(factory.create_cli_main_worker_factory().await?);
let hmr = serve_flags
.watch
.map(|watch_flags| watch_flags.hmr)
@ -58,7 +59,7 @@ pub async fn serve(
}
async fn do_serve(
worker_factory: CliMainWorkerFactory,
worker_factory: Arc<CliMainWorkerFactory>,
main_module: ModuleSpecifier,
worker_count: Option<usize>,
hmr: bool,
@ -116,7 +117,7 @@ async fn do_serve(
async fn run_worker(
worker_count: usize,
worker_factory: CliMainWorkerFactory,
worker_factory: Arc<CliMainWorkerFactory>,
main_module: ModuleSpecifier,
hmr: bool,
) -> Result<i32, AnyError> {
@ -164,7 +165,8 @@ async fn serve_with_watch(
maybe_npm_install(&factory).await?;
let _ = watcher_communicator.watch_paths(cli_options.watch_paths());
let worker_factory = factory.create_cli_main_worker_factory().await?;
let worker_factory =
Arc::new(factory.create_cli_main_worker_factory().await?);
do_serve(worker_factory, main_module.clone(), worker_count, hmr)
.await?;

25
cli/tools/task.rs
View file

@ -36,6 +36,8 @@ use crate::args::TaskFlags;
use crate::colors;
use crate::factory::CliFactory;
use crate::node::CliNodeResolver;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::PackageCaching;
use crate::npm::CliNpmResolver;
use crate::task_runner;
use crate::task_runner::run_future_forwarding_signals;
@ -203,6 +205,7 @@ pub async fn execute_script(
}]
};
let npm_installer = factory.npm_installer_if_managed()?;
let npm_resolver = factory.npm_resolver().await?;
let node_resolver = factory.node_resolver().await?;
let env_vars = task_runner::real_env_vars();
@ -216,7 +219,8 @@ pub async fn execute_script(
let task_runner = TaskRunner {
task_flags: &task_flags,
npm_resolver: npm_resolver.as_ref(),
npm_installer: npm_installer.map(|n| n.as_ref()),
npm_resolver,
node_resolver: node_resolver.as_ref(),
env_vars,
cli_options,
@ -266,7 +270,8 @@ struct RunSingleOptions<'a> {
struct TaskRunner<'a> {
task_flags: &'a TaskFlags,
npm_resolver: &'a dyn CliNpmResolver,
npm_installer: Option<&'a NpmInstaller>,
npm_resolver: &'a CliNpmResolver,
node_resolver: &'a CliNodeResolver,
env_vars: HashMap<String, String>,
cli_options: &'a CliOptions,
@ -458,11 +463,11 @@ impl<'a> TaskRunner<'a> {
return Ok(0);
};
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
npm_resolver.ensure_top_level_package_json_install().await?;
npm_resolver
.cache_packages(crate::npm::PackageCaching::All)
if let Some(npm_installer) = self.npm_installer {
npm_installer
.ensure_top_level_package_json_install()
.await?;
npm_installer.cache_packages(PackageCaching::All).await?;
}
let cwd = match &self.task_flags.cwd {
@ -497,11 +502,11 @@ impl<'a> TaskRunner<'a> {
argv: &[String],
) -> Result<i32, deno_core::anyhow::Error> {
// ensure the npm packages are installed if using a managed resolver
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
npm_resolver.ensure_top_level_package_json_install().await?;
npm_resolver
.cache_packages(crate::npm::PackageCaching::All)
if let Some(npm_installer) = self.npm_installer {
npm_installer
.ensure_top_level_package_json_install()
.await?;
npm_installer.cache_packages(PackageCaching::All).await?;
}
let cwd = match &self.task_flags.cwd {

49
cli/tools/test/mod.rs
View file

@ -87,6 +87,7 @@ use crate::util::path::is_script_ext;
use crate::util::path::matches_pattern_or_exact_path;
use crate::worker::CliMainWorkerFactory;
use crate::worker::CoverageCollector;
use crate::worker::CreateCustomWorkerError;
mod channel;
pub mod fmt;
@ -614,7 +615,10 @@ async fn configure_main_worker(
permissions_container: PermissionsContainer,
worker_sender: TestEventWorkerSender,
options: &TestSpecifierOptions,
) -> Result<(Option<Box<dyn CoverageCollector>>, MainWorker), CoreError> {
) -> Result<
(Option<Box<dyn CoverageCollector>>, MainWorker),
CreateCustomWorkerError,
> {
let mut worker = worker_factory
.create_custom_worker(
WorkerExecutionMode::Test,
@ -647,7 +651,7 @@ async fn configure_main_worker(
&worker.js_runtime.op_state(),
TestEvent::UncaughtError(specifier.to_string(), Box::new(err)),
)
.map_err(JsErrorBox::from_err)?;
.map_err(|e| CoreError::JsBox(JsErrorBox::from_err(e)))?;
Ok(())
}
Err(err) => Err(err),
@ -687,7 +691,7 @@ pub async fn test_specifier(
.await
{
Ok(()) => Ok(()),
Err(CoreError::Js(err)) => {
Err(TestSpecifierError::Core(CoreError::Js(err))) => {
send_test_event(
&worker.js_runtime.op_state(),
TestEvent::UncaughtError(specifier.to_string(), Box::new(err)),
@ -698,6 +702,16 @@ pub async fn test_specifier(
}
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum TestSpecifierError {
#[class(inherit)]
#[error(transparent)]
Core(#[from] CoreError),
#[class(inherit)]
#[error(transparent)]
RunTestsForWorker(#[from] RunTestsForWorkerErr),
}
/// Test a single specifier as documentation containing test programs, an executable test module or
/// both.
#[allow(clippy::too_many_arguments)]
@ -707,19 +721,21 @@ async fn test_specifier_inner(
specifier: ModuleSpecifier,
fail_fast_tracker: FailFastTracker,
options: TestSpecifierOptions,
) -> Result<(), CoreError> {
) -> Result<(), TestSpecifierError> {
// Ensure that there are no pending exceptions before we start running tests
worker.run_up_to_duration(Duration::from_millis(0)).await?;
worker.dispatch_load_event()?;
worker.dispatch_load_event().map_err(CoreError::Js)?;
run_tests_for_worker(worker, &specifier, &options, &fail_fast_tracker)
.await?;
// Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
// event loop to continue beyond what's needed to await results.
worker.dispatch_beforeunload_event()?;
worker.dispatch_unload_event()?;
worker
.dispatch_beforeunload_event()
.map_err(CoreError::Js)?;
worker.dispatch_unload_event().map_err(CoreError::Js)?;
// Ensure all output has been flushed
_ = worker
@ -780,12 +796,25 @@ pub fn send_test_event(
.send(event)
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum RunTestsForWorkerErr {
#[class(inherit)]
#[error(transparent)]
ChannelClosed(#[from] ChannelClosedError),
#[class(inherit)]
#[error(transparent)]
Core(#[from] CoreError),
#[class(inherit)]
#[error(transparent)]
SerdeV8(#[from] serde_v8::Error),
}
pub async fn run_tests_for_worker(
worker: &mut MainWorker,
specifier: &ModuleSpecifier,
options: &TestSpecifierOptions,
fail_fast_tracker: &FailFastTracker,
) -> Result<(), AnyError> {
) -> Result<(), RunTestsForWorkerErr> {
let state_rc = worker.js_runtime.op_state();
// Take whatever tests have been registered
let TestContainer(tests, test_functions) =
@ -814,7 +843,7 @@ async fn run_tests_for_worker_inner(
test_functions: Vec<v8::Global<v8::Function>>,
options: &TestSpecifierOptions,
fail_fast_tracker: &FailFastTracker,
) -> Result<(), AnyError> {
) -> Result<(), RunTestsForWorkerErr> {
let unfiltered = tests.len();
let state_rc = worker.js_runtime.op_state();
@ -1109,7 +1138,7 @@ async fn wait_for_activity_to_stabilize(
before: RuntimeActivityStats,
sanitize_ops: bool,
sanitize_resources: bool,
) -> Result<Option<RuntimeActivityDiff>, AnyError> {
) -> Result<Option<RuntimeActivityDiff>, CoreError> {
// First, check to see if there's any diff at all. If not, just continue.
let after = stats.clone().capture(filter);
let mut diff = RuntimeActivityStats::diff(&before, &after);

120
cli/tsc/99_main_compiler.js
View file

@ -500,6 +500,8 @@ delete Object.prototype.__proto__;
// Microsoft/TypeScript#26825 but that doesn't seem to be working here,
// so we will ignore complaints about this compiler setting.
5070,
// TS6053: File '{0}' not found.
6053,
// TS7016: Could not find a declaration file for module '...'. '...'
// implicitly has an 'any' type. This is due to `allowJs` being off by
// default but importing of a JavaScript module.
@ -705,15 +707,14 @@ delete Object.prototype.__proto__;
resolveTypeReferenceDirectiveReferences(
typeDirectiveReferences,
containingFilePath,
redirectedReference,
_redirectedReference,
options,
containingSourceFile,
_reusedNames,
) {
const isCjs =
containingSourceFile?.impliedNodeFormat === ts.ModuleKind.CommonJS;
/** @type {Array<ts.ResolvedTypeReferenceDirectiveWithFailedLookupLocations>} */
const result = typeDirectiveReferences.map((arg) => {
const toResolve = typeDirectiveReferences.map((arg) => {
/** @type {ts.FileReference} */
const fileReference = typeof arg === "string"
? {
@ -722,46 +723,50 @@ delete Object.prototype.__proto__;
fileName: arg,
}
: arg;
if (fileReference.fileName.startsWith("npm:")) {
/** @type {[string, ts.Extension] | undefined} */
const resolved = ops.op_resolve(
containingFilePath,
[
[
fileReference.resolutionMode == null
? isCjs
: fileReference.resolutionMode === ts.ModuleKind.CommonJS,
fileReference.fileName,
],
],
)?.[0];
if (resolved) {
return {
resolvedTypeReferenceDirective: {
primary: true,
resolvedFileName: resolved[0],
// todo(dsherret): we should probably be setting this
isExternalLibraryImport: undefined,
},
};
} else {
return {
resolvedTypeReferenceDirective: undefined,
};
}
return [
fileReference.resolutionMode == null
? isCjs
: fileReference.resolutionMode === ts.ModuleKind.CommonJS,
fileReference.fileName,
];
});
/** @type {Array<[string, ts.Extension | null] | undefined>} */
const resolved = ops.op_resolve(
containingFilePath,
toResolve,
);
/** @type {Array<ts.ResolvedTypeReferenceDirectiveWithFailedLookupLocations>} */
const result = resolved.map((item) => {
if (item && item[1]) {
const [resolvedFileName, extension] = item;
return {
resolvedTypeReferenceDirective: {
primary: true,
resolvedFileName,
extension,
isExternalLibraryImport: false,
},
};
} else {
return ts.resolveTypeReferenceDirective(
fileReference.fileName,
containingFilePath,
options,
host,
redirectedReference,
undefined,
containingSourceFile?.impliedNodeFormat ??
fileReference.resolutionMode,
);
return {
resolvedTypeReferenceDirective: undefined,
};
}
});
if (logDebug) {
debug(
"resolveTypeReferenceDirectiveReferences ",
typeDirectiveReferences,
containingFilePath,
options,
containingSourceFile?.fileName,
" => ",
result,
);
}
return result;
},
resolveModuleNameLiterals(
@ -785,7 +790,7 @@ delete Object.prototype.__proto__;
debug(` base: ${base}`);
debug(` specifiers: ${specifiers.map((s) => s[1]).join(", ")}`);
}
/** @type {Array<[string, ts.Extension] | undefined>} */
/** @type {Array<[string, ts.Extension | null] | undefined>} */
const resolved = ops.op_resolve(
base,
specifiers,
@ -793,7 +798,7 @@ delete Object.prototype.__proto__;
if (resolved) {
/** @type {Array<ts.ResolvedModuleWithFailedLookupLocations>} */
const result = resolved.map((item) => {
if (item) {
if (item && item[1]) {
const [resolvedFileName, extension] = item;
return {
resolvedModule: {
@ -1116,6 +1121,36 @@ delete Object.prototype.__proto__;
if (IGNORED_DIAGNOSTICS.includes(diagnostic.code)) {
return false;
}
// ignore diagnostics resulting from the `ImportMeta` declaration in deno merging with
// the one in @types/node. the types of the filename and dirname properties are different,
// which causes tsc to error.
const importMetaFilenameDirnameModifiersRe =
/^All declarations of '(filename|dirname)'/;
const importMetaFilenameDirnameTypesRe =
/^Subsequent property declarations must have the same type.\s+Property '(filename|dirname)'/;
// Declarations of X must have identical modifiers.
if (diagnostic.code === 2687) {
if (
typeof diagnostic.messageText === "string" &&
(importMetaFilenameDirnameModifiersRe.test(diagnostic.messageText)) &&
(diagnostic.file?.fileName.startsWith("asset:///") ||
diagnostic.file?.fileName?.includes("@types/node"))
) {
return false;
}
}
// Subsequent property declarations must have the same type.
if (diagnostic.code === 2717) {
if (
typeof diagnostic.messageText === "string" &&
(importMetaFilenameDirnameTypesRe.test(diagnostic.messageText)) &&
(diagnostic.file?.fileName.startsWith("asset:///") ||
diagnostic.file?.fileName?.includes("@types/node"))
) {
return false;
}
}
// make the diagnostic for using an `export =` in an es module a warning
if (diagnostic.code === 1203) {
diagnostic.category = ts.DiagnosticCategory.Warning;
@ -1410,7 +1445,6 @@ delete Object.prototype.__proto__;
"ErrorConstructor",
"gc",
"Global",
"ImportMeta",
"localStorage",
"queueMicrotask",
"RequestInit",

105
cli/tsc/dts/lib.deno.window.d.ts vendored
View file

@ -119,9 +119,86 @@ declare var onunload: ((this: Window, ev: Event) => any) | null;
declare var onunhandledrejection:
| ((this: Window, ev: PromiseRejectionEvent) => any)
| null;
/** @category Storage */
/**
* Deno's `localStorage` API provides a way to store key-value pairs in a
* web-like environment, similar to the Web Storage API found in browsers.
* It allows developers to persist data across sessions in a Deno application.
* This API is particularly useful for applications that require a simple
* and effective way to store data locally.
*
* - Key-Value Storage: Stores data as key-value pairs.
* - Persistent: Data is retained even after the application is closed.
* - Synchronous API: Operations are performed synchronously.
*
* `localStorage` is similar to {@linkcode sessionStorage}, and shares the same
* API methods, visible in the {@linkcode Storage} type.
*
* When using the `--location` flag, the origin for the location is used to
* uniquely store the data. That means a location of http://example.com/a.ts
* and http://example.com/b.ts and http://example.com:80/ would all share the
* same storage, but https://example.com/ would be different.
*
* For more information, see the reference guide for
* [Web Storage](https://docs.deno.com/runtime/reference/web_platform_apis/#web-storage)
* and using
* [the `--location` flag](https://docs.deno.com/runtime/reference/web_platform_apis/#location-flag).
*
* @example
* ```ts
* // Set a value in localStorage
* localStorage.setItem("key", "value");
*
* // Get a value from localStorage
* const value = localStorage.getItem("key");
* console.log(value); // Output: "value"
*
* // Remove a value from localStorage
* localStorage.removeItem("key");
*
* // Clear all values from localStorage
* localStorage.clear();
* ```
*
* @see https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage
* @category Storage */
declare var localStorage: Storage;
/** @category Storage */
/**
* Deno's `sessionStorage` API operates similarly to the {@linkcode localStorage} API,
* but it is intended for storing data temporarily for the duration of a session.
* Data stored in sessionStorage is cleared when the application session or
* process ends. This makes it suitable for temporary data that you do not need
* to persist across user sessions.
*
* - Key-Value Storage: Stores data as key-value pairs.
* - Session-Based: Data is only available for the duration of the page session.
* - Synchronous API: Operations are performed synchronously.
*
* `sessionStorage` is similar to {@linkcode localStorage}, and shares the same API
* methods, visible in the {@linkcode Storage} type.
*
* For more information, see the reference guide for
* [Web Storage](https://docs.deno.com/runtime/reference/web_platform_apis/#web-storage)
*
* @example
* ```ts
* // Set a value in sessionStorage
* sessionStorage.setItem("key", "value");
*
* // Get a value from sessionStorage
* const value = sessionStorage.getItem("key");
* console.log(value); // Output: "value"
*
* // Remove a value from sessionStorage
* sessionStorage.removeItem("key");
*
* // Clear all the values from sessionStorage
* sessionStorage.clear();
* ```
*
* @see https://developer.mozilla.org/en-US/docs/Web/API/Window/sessionStorage
* @category Storage
*/
declare var sessionStorage: Storage;
/** @category Cache */
declare var caches: CacheStorage;
@ -149,6 +226,12 @@ declare var navigator: Navigator;
*
* If the stdin is not interactive, it does nothing.
*
* @example
* ```ts
* // Displays the message "Acknowledge me! [Enter]" and waits for the enter key to be pressed before continuing.
* alert("Acknowledge me!");
* ```
* @see https://developer.mozilla.org/en-US/docs/Web/API/Window/alert
* @category Platform
*
* @param message
@ -162,6 +245,15 @@ declare function alert(message?: string): void;
*
* If the stdin is not interactive, it returns false.
*
* @example
* ```ts
* const shouldProceed = confirm("Do you want to proceed?");
*
* // If the user presses 'y' or 'Y', the result will be true
* // If the user presses 'n' or 'N', the result will be false
* console.log("Should proceed?", shouldProceed);
* ```
* @see https://developer.mozilla.org/en-US/docs/Web/API/Window/confirm
* @category Platform
*
* @param message
@ -179,6 +271,15 @@ declare function confirm(message?: string): boolean;
*
* If the stdin is not interactive, it returns null.
*
* @example
* ```ts
* const pet = prompt("Cats or dogs?", "It's fine to love both!");
*
* // Displays the user's input or the default value of "It's fine to love both!"
* console.log("Best pet:", pet);
* ```
* @see https://developer.mozilla.org/en-US/docs/Web/API/Window/prompt
*
* @category Platform
*
* @param message

44
cli/tsc/mod.rs
View file

@ -28,6 +28,9 @@ use deno_graph::GraphKind;
use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::ResolutionResolved;
use deno_lib::util::checksum;
use deno_lib::worker::create_isolate_create_params;
use deno_resolver::npm::managed::ResolvePkgFolderFromDenoModuleError;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::errors::NodeJsErrorCode;
@ -45,11 +48,9 @@ use crate::cache::FastInsecureHasher;
use crate::cache::ModuleInfoCache;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliCjsTracker;
use crate::sys::CliSys;
use crate::util::checksum;
use crate::util::path::mapped_specifier_for_tsc;
use crate::worker::create_isolate_create_params;
mod diagnostics;
@ -299,13 +300,13 @@ pub fn into_specifier_and_media_type(
#[derive(Debug)]
pub struct TypeCheckingCjsTracker {
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
module_info_cache: Arc<ModuleInfoCache>,
}
impl TypeCheckingCjsTracker {
pub fn new(
cjs_tracker: Arc<CjsTracker>,
cjs_tracker: Arc<CliCjsTracker>,
module_info_cache: Arc<ModuleInfoCache>,
) -> Self {
Self {
@ -357,7 +358,7 @@ impl TypeCheckingCjsTracker {
pub struct RequestNpmState {
pub cjs_tracker: Arc<TypeCheckingCjsTracker>,
pub node_resolver: Arc<CliNodeResolver>,
pub npm_resolver: Arc<dyn CliNpmResolver>,
pub npm_resolver: CliNpmResolver,
}
/// A structure representing a request to be sent to the tsc runtime.
@ -709,9 +710,7 @@ pub enum ResolveError {
PackageSubpathResolve(PackageSubpathResolveError),
#[class(inherit)]
#[error("{0}")]
ResolvePkgFolderFromDenoModule(
#[from] crate::npm::ResolvePkgFolderFromDenoModuleError,
),
ResolvePkgFolderFromDenoModule(#[from] ResolvePkgFolderFromDenoModuleError),
#[class(inherit)]
#[error("{0}")]
ResolveNonGraphSpecifierTypes(#[from] ResolveNonGraphSpecifierTypesError),
@ -746,7 +745,7 @@ fn op_resolve(
state: &mut OpState,
#[string] base: String,
#[serde] specifiers: Vec<(bool, String)>,
) -> Result<Vec<(String, &'static str)>, ResolveError> {
) -> Result<Vec<(String, Option<&'static str>)>, ResolveError> {
op_resolve_inner(state, ResolveArgs { base, specifiers })
}
@ -754,9 +753,9 @@ fn op_resolve(
fn op_resolve_inner(
state: &mut OpState,
args: ResolveArgs,
) -> Result<Vec<(String, &'static str)>, ResolveError> {
) -> Result<Vec<(String, Option<&'static str>)>, ResolveError> {
let state = state.borrow_mut::<State>();
let mut resolved: Vec<(String, &'static str)> =
let mut resolved: Vec<(String, Option<&'static str>)> =
Vec::with_capacity(args.specifiers.len());
let referrer = if let Some(remapped_specifier) =
state.maybe_remapped_specifier(&args.base)
@ -770,14 +769,14 @@ fn op_resolve_inner(
if specifier.starts_with("node:") {
resolved.push((
MISSING_DEPENDENCY_SPECIFIER.to_string(),
MediaType::Dts.as_ts_extension(),
Some(MediaType::Dts.as_ts_extension()),
));
continue;
}
if specifier.starts_with("asset:///") {
let ext = MediaType::from_str(&specifier).as_ts_extension();
resolved.push((specifier, ext));
resolved.push((specifier, Some(ext)));
continue;
}
@ -857,14 +856,15 @@ fn op_resolve_inner(
(
specifier_str,
match media_type {
MediaType::Css => ".js", // surface these as .js for typescript
media_type => media_type.as_ts_extension(),
MediaType::Css => Some(".js"), // surface these as .js for typescript
MediaType::Unknown => None,
media_type => Some(media_type.as_ts_extension()),
},
)
}
None => (
MISSING_DEPENDENCY_SPECIFIER.to_string(),
MediaType::Dts.as_ts_extension(),
Some(MediaType::Dts.as_ts_extension()),
),
};
log::debug!("Resolved {} from {} to {:?}", specifier, referrer, result);
@ -1441,7 +1441,10 @@ mod tests {
},
)
.expect("should have invoked op");
assert_eq!(actual, vec![("https://deno.land/x/b.ts".into(), ".ts")]);
assert_eq!(
actual,
vec![("https://deno.land/x/b.ts".into(), Some(".ts"))]
);
}
#[tokio::test]
@ -1460,7 +1463,10 @@ mod tests {
},
)
.expect("should have not errored");
assert_eq!(actual, vec![(MISSING_DEPENDENCY_SPECIFIER.into(), ".d.ts")]);
assert_eq!(
actual,
vec![(MISSING_DEPENDENCY_SPECIFIER.into(), Some(".d.ts"))]
);
}
#[tokio::test]

84
cli/util/file_watcher.rs
View file

@ -25,6 +25,7 @@ use notify::Watcher;
use tokio::select;
use tokio::sync::broadcast::error::RecvError;
use tokio::sync::mpsc;
use tokio::sync::mpsc::error::SendError;
use tokio::sync::mpsc::UnboundedReceiver;
use tokio::time::sleep;
@ -141,36 +142,60 @@ fn create_print_after_restart_fn(clear_screen: bool) -> impl Fn() {
}
}
#[derive(Debug)]
pub struct WatcherCommunicatorOptions {
/// Send a list of paths that should be watched for changes.
pub paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,
/// Listen for a list of paths that were changed.
pub changed_paths_rx: tokio::sync::broadcast::Receiver<Option<Vec<PathBuf>>>,
pub changed_paths_tx: tokio::sync::broadcast::Sender<Option<Vec<PathBuf>>>,
/// Send a message to force a restart.
pub restart_tx: tokio::sync::mpsc::UnboundedSender<()>,
pub restart_mode: WatcherRestartMode,
pub banner: String,
}
/// An interface to interact with Deno's CLI file watcher.
#[derive(Debug)]
pub struct WatcherCommunicator {
/// Send a list of paths that should be watched for changes.
paths_to_watch_tx: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,
/// Listen for a list of paths that were changed.
changed_paths_rx: tokio::sync::broadcast::Receiver<Option<Vec<PathBuf>>>,
changed_paths_tx: tokio::sync::broadcast::Sender<Option<Vec<PathBuf>>>,
/// Send a message to force a restart.
restart_tx: tokio::sync::mpsc::UnboundedSender<()>,
restart_mode: Mutex<WatcherRestartMode>,
banner: String,
}
impl WatcherCommunicator {
pub fn watch_paths(&self, paths: Vec<PathBuf>) -> Result<(), AnyError> {
pub fn new(options: WatcherCommunicatorOptions) -> Self {
Self {
paths_to_watch_tx: options.paths_to_watch_tx,
changed_paths_rx: options.changed_paths_rx,
changed_paths_tx: options.changed_paths_tx,
restart_tx: options.restart_tx,
restart_mode: Mutex::new(options.restart_mode),
banner: options.banner,
}
}
pub fn watch_paths(
&self,
paths: Vec<PathBuf>,
) -> Result<(), SendError<Vec<PathBuf>>> {
if paths.is_empty() {
return Ok(());
}
self.paths_to_watch_tx.send(paths).map_err(AnyError::from)
self.paths_to_watch_tx.send(paths)
}
pub fn force_restart(&self) -> Result<(), AnyError> {
pub fn force_restart(&self) -> Result<(), SendError<()>> {
// Change back to automatic mode, so that HMR can set up watching
// from scratch.
*self.restart_mode.lock() = WatcherRestartMode::Automatic;
self.restart_tx.send(()).map_err(AnyError::from)
self.restart_tx.send(())
}
pub async fn watch_for_changed_paths(
@ -184,6 +209,22 @@ impl WatcherCommunicator {
*self.restart_mode.lock() = restart_mode;
}
pub fn send(
&self,
paths: Option<Vec<PathBuf>>,
) -> Result<(), SendError<Option<Vec<PathBuf>>>> {
match *self.restart_mode.lock() {
WatcherRestartMode::Automatic => {
self.restart_tx.send(()).map_err(|_| SendError(None))
}
WatcherRestartMode::Manual => self
.changed_paths_tx
.send(paths)
.map(|_| ())
.map_err(|e| SendError(e.0)),
}
}
pub fn print(&self, msg: String) {
log::info!("{} {}", self.banner, colors::gray(msg));
}
@ -272,13 +313,15 @@ where
} = print_config;
let print_after_restart = create_print_after_restart_fn(clear_screen);
let watcher_communicator = Arc::new(WatcherCommunicator {
paths_to_watch_tx: paths_to_watch_tx.clone(),
changed_paths_rx: changed_paths_rx.resubscribe(),
restart_tx: restart_tx.clone(),
restart_mode: Mutex::new(restart_mode),
banner: colors::intense_blue(banner).to_string(),
});
let watcher_communicator =
Arc::new(WatcherCommunicator::new(WatcherCommunicatorOptions {
paths_to_watch_tx: paths_to_watch_tx.clone(),
changed_paths_rx: changed_paths_rx.resubscribe(),
changed_paths_tx,
restart_tx: restart_tx.clone(),
restart_mode,
banner: colors::intense_blue(banner).to_string(),
}));
info!("{} {} started.", colors::intense_blue(banner), job_name);
let changed_paths = Rc::new(RefCell::new(None));
@ -292,15 +335,8 @@ where
.borrow_mut()
.clone_from(&received_changed_paths);
match *watcher_.restart_mode.lock() {
WatcherRestartMode::Automatic => {
let _ = restart_tx.send(());
}
WatcherRestartMode::Manual => {
// TODO(bartlomieju): should we fail on sending changed paths?
let _ = changed_paths_tx.send(received_changed_paths);
}
}
// TODO(bartlomieju): should we fail on sending changed paths?
let _ = watcher_.send(received_changed_paths);
}
});

1
cli/util/mod.rs
View file

@ -2,7 +2,6 @@
// Note: Only add code in this folder that has no application specific logic
pub mod archive;
pub mod checksum;
pub mod console;
pub mod diff;
pub mod display;

2
cli/util/result.rs
View file

@ -36,7 +36,7 @@ pub fn any_and_jserrorbox_downcast_ref<
})
.or_else(|| {
err.downcast_ref::<CoreError>().and_then(|e| match e {
CoreError::JsNative(e) => e.as_any().downcast_ref::<E>(),
CoreError::JsBox(e) => e.as_any().downcast_ref::<E>(),
_ => None,
})
})

629
cli/worker.rs
View file

@ -1,83 +1,37 @@
// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::error::CoreError;
use deno_core::futures::FutureExt;
use deno_core::url::Url;
use deno_core::v8;
use deno_core::CompiledWasmModuleStore;
use deno_core::Extension;
use deno_core::FeatureChecker;
use deno_core::ModuleLoader;
use deno_core::PollEventLoopOptions;
use deno_core::SharedArrayBufferStore;
use deno_error::JsErrorBox;
use deno_lib::worker::LibMainWorker;
use deno_lib::worker::LibMainWorkerFactory;
use deno_runtime::code_cache;
use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeExtInitServices;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeRequireLoaderRc;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::fmt_errors::format_js_error;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::ops::process::NpmProcessStateProviderRc;
use deno_runtime::ops::worker_host::CreateWebWorkerCb;
use deno_runtime::web_worker::WebWorker;
use deno_runtime::web_worker::WebWorkerOptions;
use deno_runtime::web_worker::WebWorkerServiceOptions;
use deno_runtime::worker::MainWorker;
use deno_runtime::worker::WorkerOptions;
use deno_runtime::worker::WorkerServiceOptions;
use deno_runtime::BootstrapOptions;
use deno_runtime::WorkerExecutionMode;
use deno_runtime::WorkerLogLevel;
use deno_semver::npm::NpmPackageReqReference;
use deno_telemetry::OtelConfig;
use deno_terminal::colors;
use node_resolver::errors::ResolvePkgJsonBinExportError;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use sys_traits::EnvCurrentDir;
use tokio::select;
use crate::args::CliLockfile;
use crate::args::DenoSubcommand;
use crate::args::NpmCachingStrategy;
use crate::args::StorageKeyResolver;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::installer::NpmInstaller;
use crate::npm::installer::PackageCaching;
use crate::npm::CliNpmResolver;
use crate::sys::CliSys;
use crate::util::checksum;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::file_watcher::WatcherRestartMode;
use crate::version;
pub struct CreateModuleLoaderResult {
pub module_loader: Rc<dyn ModuleLoader>,
pub node_require_loader: Rc<dyn NodeRequireLoader>,
}
pub trait ModuleLoaderFactory: Send + Sync {
fn create_for_main(
&self,
root_permissions: PermissionsContainer,
) -> CreateModuleLoaderResult;
fn create_for_worker(
&self,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
) -> CreateModuleLoaderResult;
}
#[async_trait::async_trait(?Send)]
pub trait HmrRunner: Send + Sync {
@ -97,8 +51,8 @@ pub trait CliCodeCache: code_cache::CodeCache {
#[async_trait::async_trait(?Send)]
pub trait CoverageCollector: Send + Sync {
async fn start_collecting(&mut self) -> Result<(), AnyError>;
async fn stop_collecting(&mut self) -> Result<(), AnyError>;
async fn start_collecting(&mut self) -> Result<(), CoreError>;
async fn stop_collecting(&mut self) -> Result<(), CoreError>;
}
pub type CreateHmrRunnerCb = Box<
@ -112,86 +66,31 @@ pub type CreateCoverageCollectorCb = Box<
>;
pub struct CliMainWorkerOptions {
pub argv: Vec<String>,
pub log_level: WorkerLogLevel,
pub enable_op_summary_metrics: bool,
pub enable_testing_features: bool,
pub has_node_modules_dir: bool,
pub hmr: bool,
pub inspect_brk: bool,
pub inspect_wait: bool,
pub strace_ops: Option<Vec<String>>,
pub is_inspecting: bool,
pub location: Option<Url>,
pub argv0: Option<String>,
pub node_debug: Option<String>,
pub origin_data_folder_path: Option<PathBuf>,
pub seed: Option<u64>,
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub skip_op_registration: bool,
pub create_hmr_runner: Option<CreateHmrRunnerCb>,
pub create_coverage_collector: Option<CreateCoverageCollectorCb>,
pub node_ipc: Option<i64>,
pub serve_port: Option<u16>,
pub serve_host: Option<String>,
pub default_npm_caching_strategy: NpmCachingStrategy,
pub needs_test_modules: bool,
}
struct SharedWorkerState {
blob_store: Arc<BlobStore>,
broadcast_channel: InMemoryBroadcastChannel,
code_cache: Option<Arc<dyn CliCodeCache>>,
compiled_wasm_module_store: CompiledWasmModuleStore,
feature_checker: Arc<FeatureChecker>,
fs: Arc<dyn deno_fs::FileSystem>,
maybe_file_watcher_communicator: Option<Arc<WatcherCommunicator>>,
maybe_inspector_server: Option<Arc<InspectorServer>>,
maybe_lockfile: Option<Arc<CliLockfile>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
root_permissions: PermissionsContainer,
shared_array_buffer_store: SharedArrayBufferStore,
storage_key_resolver: StorageKeyResolver,
sys: CliSys,
options: CliMainWorkerOptions,
subcommand: DenoSubcommand,
otel_config: OtelConfig,
default_npm_caching_strategy: NpmCachingStrategy,
}
impl SharedWorkerState {
pub fn create_node_init_services(
&self,
node_require_loader: NodeRequireLoaderRc,
) -> NodeExtInitServices<CliSys> {
NodeExtInitServices {
node_require_loader,
node_resolver: self.node_resolver.clone(),
npm_resolver: self.npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver: self.pkg_json_resolver.clone(),
sys: self.sys.clone(),
}
}
pub fn npm_process_state_provider(&self) -> NpmProcessStateProviderRc {
self.npm_resolver.clone().into_process_state_provider()
}
/// Data shared between the factory and workers.
struct SharedState {
pub create_hmr_runner: Option<CreateHmrRunnerCb>,
pub create_coverage_collector: Option<CreateCoverageCollectorCb>,
pub maybe_file_watcher_communicator: Option<Arc<WatcherCommunicator>>,
}
pub struct CliMainWorker {
main_module: ModuleSpecifier,
worker: MainWorker,
shared: Arc<SharedWorkerState>,
worker: LibMainWorker,
shared: Arc<SharedState>,
}
impl CliMainWorker {
#[inline]
pub fn into_main_worker(self) -> MainWorker {
self.worker
self.worker.into_main_worker()
}
pub async fn setup_repl(&mut self) -> Result<(), AnyError> {
pub async fn setup_repl(&mut self) -> Result<(), CoreError> {
self.worker.run_event_loop(false).await?;
Ok(())
}
@ -201,16 +100,13 @@ impl CliMainWorker {
self.maybe_setup_coverage_collector().await?;
let mut maybe_hmr_runner = self.maybe_setup_hmr_runner().await?;
log::debug!("main_module {}", self.main_module);
log::debug!("main_module {}", self.worker.main_module());
self.execute_main_module().await?;
self.worker.dispatch_load_event()?;
loop {
if let Some(hmr_runner) = maybe_hmr_runner.as_mut() {
let watcher_communicator =
self.shared.maybe_file_watcher_communicator.clone().unwrap();
let hmr_future = hmr_runner.run().boxed_local();
let event_loop_future = self.worker.run_event_loop(false).boxed_local();
@ -224,7 +120,11 @@ impl CliMainWorker {
}
}
if let Err(e) = result {
watcher_communicator
self
.shared
.maybe_file_watcher_communicator
.as_ref()
.unwrap()
.change_restart_mode(WatcherRestartMode::Automatic);
return Err(e);
}
@ -250,7 +150,7 @@ impl CliMainWorker {
if let Some(coverage_collector) = maybe_coverage_collector.as_mut() {
self
.worker
.js_runtime
.js_runtime()
.with_event_loop_future(
coverage_collector.stop_collecting().boxed_local(),
PollEventLoopOptions::default(),
@ -260,7 +160,7 @@ impl CliMainWorker {
if let Some(hmr_runner) = maybe_hmr_runner.as_mut() {
self
.worker
.js_runtime
.js_runtime()
.with_event_loop_future(
hmr_runner.stop().boxed_local(),
PollEventLoopOptions::default(),
@ -271,7 +171,7 @@ impl CliMainWorker {
Ok(self.worker.exit_code())
}
pub async fn run_for_watcher(self) -> Result<(), AnyError> {
pub async fn run_for_watcher(self) -> Result<(), CoreError> {
/// The FileWatcherModuleExecutor provides module execution with safe dispatching of life-cycle events by tracking the
/// state of any pending events and emitting accordingly on drop in the case of a future
/// cancellation.
@ -290,7 +190,7 @@ impl CliMainWorker {
/// Execute the given main module emitting load and unload events before and after execution
/// respectively.
pub async fn execute(&mut self) -> Result<(), AnyError> {
pub async fn execute(&mut self) -> Result<(), CoreError> {
self.inner.execute_main_module().await?;
self.inner.worker.dispatch_load_event()?;
self.pending_unload = true;
@ -332,24 +232,20 @@ impl CliMainWorker {
executor.execute().await
}
#[inline]
pub async fn execute_main_module(&mut self) -> Result<(), CoreError> {
let id = self.worker.preload_main_module(&self.main_module).await?;
self.worker.evaluate_module(id).await
self.worker.execute_main_module().await
}
#[inline]
pub async fn execute_side_module(&mut self) -> Result<(), CoreError> {
let id = self.worker.preload_side_module(&self.main_module).await?;
self.worker.evaluate_module(id).await
self.worker.execute_side_module().await
}
pub async fn maybe_setup_hmr_runner(
&mut self,
) -> Result<Option<Box<dyn HmrRunner>>, AnyError> {
if !self.shared.options.hmr {
return Ok(None);
}
let Some(setup_hmr_runner) = self.shared.options.create_hmr_runner.as_ref()
else {
) -> Result<Option<Box<dyn HmrRunner>>, CoreError> {
let Some(setup_hmr_runner) = self.shared.create_hmr_runner.as_ref() else {
return Ok(None);
};
@ -359,7 +255,7 @@ impl CliMainWorker {
self
.worker
.js_runtime
.js_runtime()
.with_event_loop_future(
hmr_runner.start().boxed_local(),
PollEventLoopOptions::default(),
@ -370,9 +266,9 @@ impl CliMainWorker {
pub async fn maybe_setup_coverage_collector(
&mut self,
) -> Result<Option<Box<dyn CoverageCollector>>, AnyError> {
) -> Result<Option<Box<dyn CoverageCollector>>, CoreError> {
let Some(create_coverage_collector) =
self.shared.options.create_coverage_collector.as_ref()
self.shared.create_coverage_collector.as_ref()
else {
return Ok(None);
};
@ -381,7 +277,7 @@ impl CliMainWorker {
let mut coverage_collector = create_coverage_collector(session);
self
.worker
.js_runtime
.js_runtime()
.with_event_loop_future(
coverage_collector.start_collecting().boxed_local(),
PollEventLoopOptions::default(),
@ -395,70 +291,103 @@ impl CliMainWorker {
name: &'static str,
source_code: &'static str,
) -> Result<v8::Global<v8::Value>, CoreError> {
self.worker.js_runtime.execute_script(name, source_code)
self.worker.js_runtime().execute_script(name, source_code)
}
}
// TODO(bartlomieju): this should be moved to some other place, added to avoid string
// duplication between worker setups and `deno info` output.
pub fn get_cache_storage_dir() -> PathBuf {
// Note: we currently use temp_dir() to avoid managing storage size.
std::env::temp_dir().join("deno_cache")
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum ResolveBinaryEntrypointError {
#[class(inherit)]
#[error(transparent)]
ResolvePkgJsonBinExport(ResolvePkgJsonBinExportError),
#[class(generic)]
#[error("{original:#}\n\nFallback failed: {fallback:#}")]
Fallback {
fallback: ResolveBinaryEntrypointFallbackError,
original: ResolvePkgJsonBinExportError,
},
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum ResolveBinaryEntrypointFallbackError {
#[class(inherit)]
#[error(transparent)]
PackageSubpathResolve(node_resolver::errors::PackageSubpathResolveError),
#[class(generic)]
#[error("Cannot find module '{0}'")]
ModuleNotFound(ModuleSpecifier),
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum CreateCustomWorkerError {
#[class(inherit)]
#[error(transparent)]
Io(#[from] std::io::Error),
#[class(inherit)]
#[error(transparent)]
Core(#[from] CoreError),
#[class(inherit)]
#[error(transparent)]
ResolvePkgFolderFromDenoReq(
#[from] deno_resolver::npm::ResolvePkgFolderFromDenoReqError,
),
#[class(inherit)]
#[error(transparent)]
UrlParse(#[from] deno_core::url::ParseError),
#[class(inherit)]
#[error(transparent)]
ResolveBinaryEntrypoint(#[from] ResolveBinaryEntrypointError),
#[class(inherit)]
#[error(transparent)]
NpmPackageReq(JsErrorBox),
#[class(inherit)]
#[error(transparent)]
AtomicWriteFileWithRetries(
#[from] crate::args::AtomicWriteFileWithRetriesError,
),
}
#[derive(Clone)]
pub struct CliMainWorkerFactory {
shared: Arc<SharedWorkerState>,
lib_main_worker_factory: LibMainWorkerFactory<CliSys>,
maybe_lockfile: Option<Arc<CliLockfile>>,
node_resolver: Arc<CliNodeResolver>,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolver: CliNpmResolver,
root_permissions: PermissionsContainer,
shared: Arc<SharedState>,
sys: CliSys,
default_npm_caching_strategy: NpmCachingStrategy,
needs_test_modules: bool,
}
impl CliMainWorkerFactory {
#[allow(clippy::too_many_arguments)]
pub fn new(
blob_store: Arc<BlobStore>,
code_cache: Option<Arc<dyn CliCodeCache>>,
feature_checker: Arc<FeatureChecker>,
fs: Arc<dyn deno_fs::FileSystem>,
lib_main_worker_factory: LibMainWorkerFactory<CliSys>,
maybe_file_watcher_communicator: Option<Arc<WatcherCommunicator>>,
maybe_inspector_server: Option<Arc<InspectorServer>>,
maybe_lockfile: Option<Arc<CliLockfile>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
root_permissions: PermissionsContainer,
storage_key_resolver: StorageKeyResolver,
npm_installer: Option<Arc<NpmInstaller>>,
npm_resolver: CliNpmResolver,
sys: CliSys,
subcommand: DenoSubcommand,
options: CliMainWorkerOptions,
otel_config: OtelConfig,
default_npm_caching_strategy: NpmCachingStrategy,
root_permissions: PermissionsContainer,
) -> Self {
Self {
shared: Arc::new(SharedWorkerState {
blob_store,
broadcast_channel: Default::default(),
code_cache,
compiled_wasm_module_store: Default::default(),
feature_checker,
fs,
lib_main_worker_factory,
maybe_lockfile,
node_resolver,
npm_installer,
npm_resolver,
root_permissions,
sys,
shared: Arc::new(SharedState {
create_hmr_runner: options.create_hmr_runner,
create_coverage_collector: options.create_coverage_collector,
maybe_file_watcher_communicator,
maybe_inspector_server,
maybe_lockfile,
module_loader_factory,
node_resolver,
npm_resolver,
pkg_json_resolver,
root_cert_store_provider,
root_permissions,
shared_array_buffer_store: Default::default(),
storage_key_resolver,
sys,
options,
subcommand,
otel_config,
default_npm_caching_strategy,
}),
default_npm_caching_strategy: options.default_npm_caching_strategy,
needs_test_modules: options.needs_test_modules,
}
}
@ -466,12 +395,12 @@ impl CliMainWorkerFactory {
&self,
mode: WorkerExecutionMode,
main_module: ModuleSpecifier,
) -> Result<CliMainWorker, CoreError> {
) -> Result<CliMainWorker, CreateCustomWorkerError> {
self
.create_custom_worker(
mode,
main_module,
self.shared.root_permissions.clone(),
self.root_permissions.clone(),
vec![],
Default::default(),
)
@ -485,48 +414,42 @@ impl CliMainWorkerFactory {
permissions: PermissionsContainer,
custom_extensions: Vec<Extension>,
stdio: deno_runtime::deno_io::Stdio,
) -> Result<CliMainWorker, CoreError> {
let shared = &self.shared;
let CreateModuleLoaderResult {
module_loader,
node_require_loader,
} = shared
.module_loader_factory
.create_for_main(permissions.clone());
) -> Result<CliMainWorker, CreateCustomWorkerError> {
let main_module = if let Ok(package_ref) =
NpmPackageReqReference::from_specifier(&main_module)
{
if let Some(npm_resolver) = shared.npm_resolver.as_managed() {
if let Some(npm_installer) = &self.npm_installer {
let reqs = &[package_ref.req().clone()];
npm_resolver
npm_installer
.add_package_reqs(
reqs,
if matches!(
shared.default_npm_caching_strategy,
self.default_npm_caching_strategy,
NpmCachingStrategy::Lazy
) {
crate::npm::PackageCaching::Only(reqs.into())
PackageCaching::Only(reqs.into())
} else {
crate::npm::PackageCaching::All
PackageCaching::All
},
)
.await?;
.await
.map_err(CreateCustomWorkerError::NpmPackageReq)?;
}
// use a fake referrer that can be used to discover the package.json if necessary
let referrer = ModuleSpecifier::from_directory_path(
self.shared.fs.cwd().map_err(JsErrorBox::from_err)?,
)
.unwrap()
.join("package.json")?;
let package_folder = shared
.npm_resolver
.resolve_pkg_folder_from_deno_module_req(package_ref.req(), &referrer)
.map_err(JsErrorBox::from_err)?;
let referrer =
ModuleSpecifier::from_directory_path(self.sys.env_current_dir()?)
.unwrap()
.join("package.json")?;
let package_folder =
self.npm_resolver.resolve_pkg_folder_from_deno_module_req(
package_ref.req(),
&referrer,
)?;
let main_module = self
.resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?;
if let Some(lockfile) = &shared.maybe_lockfile {
if let Some(lockfile) = &self.maybe_lockfile {
// For npm binary commands, ensure that the lockfile gets updated
// so that we can re-use the npm resolution the next time it runs
// for better performance
@ -538,119 +461,18 @@ impl CliMainWorkerFactory {
main_module
};
let maybe_inspector_server = shared.maybe_inspector_server.clone();
let create_web_worker_cb =
create_web_worker_callback(shared.clone(), stdio.clone());
let maybe_storage_key = shared
.storage_key_resolver
.resolve_storage_key(&main_module);
let origin_storage_dir = maybe_storage_key.as_ref().map(|key| {
shared
.options
.origin_data_folder_path
.as_ref()
.unwrap() // must be set if storage key resolver returns a value
.join(checksum::gen(&[key.as_bytes()]))
});
let cache_storage_dir = maybe_storage_key.map(|key| {
// TODO(@satyarohith): storage quota management
get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()]))
});
// TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
// list of enabled features.
let feature_checker = shared.feature_checker.clone();
let mut unstable_features =
Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len());
for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS {
if feature_checker.check(granular_flag.name) {
unstable_features.push(granular_flag.id);
}
}
let services = WorkerServiceOptions {
root_cert_store_provider: Some(shared.root_cert_store_provider.clone()),
module_loader,
fs: shared.fs.clone(),
node_services: Some(
shared.create_node_init_services(node_require_loader),
),
npm_process_state_provider: Some(shared.npm_process_state_provider()),
blob_store: shared.blob_store.clone(),
broadcast_channel: shared.broadcast_channel.clone(),
fetch_dns_resolver: Default::default(),
shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(
shared.compiled_wasm_module_store.clone(),
),
feature_checker,
let mut worker = self.lib_main_worker_factory.create_custom_worker(
mode,
main_module,
permissions,
v8_code_cache: shared.code_cache.clone().map(|c| c.as_code_cache()),
};
let options = WorkerOptions {
bootstrap: BootstrapOptions {
deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(),
args: shared.options.argv.clone(),
cpu_count: std::thread::available_parallelism()
.map(|p| p.get())
.unwrap_or(1),
log_level: shared.options.log_level,
enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
enable_testing_features: shared.options.enable_testing_features,
locale: deno_core::v8::icu::get_language_tag(),
location: shared.options.location.clone(),
no_color: !colors::use_color(),
is_stdout_tty: deno_terminal::is_stdout_tty(),
is_stderr_tty: deno_terminal::is_stderr_tty(),
color_level: colors::get_color_level(),
unstable_features,
user_agent: version::DENO_VERSION_INFO.user_agent.to_string(),
inspect: shared.options.is_inspecting,
has_node_modules_dir: shared.options.has_node_modules_dir,
argv0: shared.options.argv0.clone(),
node_debug: shared.options.node_debug.clone(),
node_ipc_fd: shared.options.node_ipc,
mode,
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: true,
},
extensions: custom_extensions,
startup_snapshot: crate::js::deno_isolate_init(),
create_params: create_isolate_create_params(),
unsafely_ignore_certificate_errors: shared
.options
.unsafely_ignore_certificate_errors
.clone(),
seed: shared.options.seed,
format_js_error_fn: Some(Arc::new(format_js_error)),
create_web_worker_cb,
maybe_inspector_server,
should_break_on_first_statement: shared.options.inspect_brk,
should_wait_for_inspector_session: shared.options.inspect_wait,
strace_ops: shared.options.strace_ops.clone(),
cache_storage_dir,
origin_storage_dir,
custom_extensions,
stdio,
skip_op_registration: shared.options.skip_op_registration,
enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled(
),
};
)?;
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),
services,
options,
);
if self.shared.subcommand.needs_test() {
if self.needs_test_modules {
macro_rules! test_file {
($($file:literal),*) => {
$(worker.js_runtime.lazy_load_es_module_with_code(
$(worker.js_runtime().lazy_load_es_module_with_code(
concat!("ext:cli/", $file),
deno_core::ascii_str_include!(concat!("js/", $file)),
)?;)*
@ -668,9 +490,8 @@ impl CliMainWorkerFactory {
}
Ok(CliMainWorker {
main_module,
worker,
shared: shared.clone(),
shared: self.shared.clone(),
})
}
@ -678,9 +499,8 @@ impl CliMainWorkerFactory {
&self,
package_folder: &Path,
sub_path: Option<&str>,
) -> Result<ModuleSpecifier, AnyError> {
) -> Result<ModuleSpecifier, ResolveBinaryEntrypointError> {
match self
.shared
.node_resolver
.resolve_binary_export(package_folder, sub_path)
{
@ -691,10 +511,13 @@ impl CliMainWorkerFactory {
self.resolve_binary_entrypoint_fallback(package_folder, sub_path);
match result {
Ok(Some(specifier)) => Ok(specifier),
Ok(None) => Err(original_err.into()),
Err(fallback_err) => {
bail!("{:#}\n\nFallback failed: {:#}", original_err, fallback_err)
}
Ok(None) => Err(
ResolveBinaryEntrypointError::ResolvePkgJsonBinExport(original_err),
),
Err(fallback_err) => Err(ResolveBinaryEntrypointError::Fallback {
original: original_err,
fallback: fallback_err,
}),
}
}
}
@ -705,7 +528,7 @@ impl CliMainWorkerFactory {
&self,
package_folder: &Path,
sub_path: Option<&str>,
) -> Result<Option<ModuleSpecifier>, AnyError> {
) -> Result<Option<ModuleSpecifier>, ResolveBinaryEntrypointFallbackError> {
// only fallback if the user specified a sub path
if sub_path.is_none() {
// it's confusing to users if the package doesn't have any binary
@ -715,7 +538,6 @@ impl CliMainWorkerFactory {
}
let specifier = self
.shared
.node_resolver
.resolve_package_subpath_from_deno_module(
package_folder,
@ -723,7 +545,8 @@ impl CliMainWorkerFactory {
/* referrer */ None,
ResolutionMode::Import,
NodeResolutionKind::Execution,
)?;
)
.map_err(ResolveBinaryEntrypointFallbackError::PackageSubpathResolve)?;
if specifier
.to_file_path()
.map(|p| p.exists())
@ -731,141 +554,27 @@ impl CliMainWorkerFactory {
{
Ok(Some(specifier))
} else {
bail!("Cannot find module '{}'", specifier)
Err(ResolveBinaryEntrypointFallbackError::ModuleNotFound(
specifier,
))
}
}
}
fn create_web_worker_callback(
shared: Arc<SharedWorkerState>,
stdio: deno_runtime::deno_io::Stdio,
) -> Arc<CreateWebWorkerCb> {
Arc::new(move |args| {
let maybe_inspector_server = shared.maybe_inspector_server.clone();
let CreateModuleLoaderResult {
module_loader,
node_require_loader,
} = shared.module_loader_factory.create_for_worker(
args.parent_permissions.clone(),
args.permissions.clone(),
);
let create_web_worker_cb =
create_web_worker_callback(shared.clone(), stdio.clone());
let maybe_storage_key = shared
.storage_key_resolver
.resolve_storage_key(&args.main_module);
let cache_storage_dir = maybe_storage_key.map(|key| {
// TODO(@satyarohith): storage quota management
get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()]))
});
// TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
// list of enabled features.
let feature_checker = shared.feature_checker.clone();
let mut unstable_features =
Vec::with_capacity(crate::UNSTABLE_GRANULAR_FLAGS.len());
for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS {
if feature_checker.check(granular_flag.name) {
unstable_features.push(granular_flag.id);
}
}
let services = WebWorkerServiceOptions {
root_cert_store_provider: Some(shared.root_cert_store_provider.clone()),
module_loader,
fs: shared.fs.clone(),
node_services: Some(
shared.create_node_init_services(node_require_loader),
),
blob_store: shared.blob_store.clone(),
broadcast_channel: shared.broadcast_channel.clone(),
shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(
shared.compiled_wasm_module_store.clone(),
),
maybe_inspector_server,
feature_checker,
npm_process_state_provider: Some(shared.npm_process_state_provider()),
permissions: args.permissions,
};
let options = WebWorkerOptions {
name: args.name,
main_module: args.main_module.clone(),
worker_id: args.worker_id,
bootstrap: BootstrapOptions {
deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(),
args: shared.options.argv.clone(),
cpu_count: std::thread::available_parallelism()
.map(|p| p.get())
.unwrap_or(1),
log_level: shared.options.log_level,
enable_op_summary_metrics: shared.options.enable_op_summary_metrics,
enable_testing_features: shared.options.enable_testing_features,
locale: deno_core::v8::icu::get_language_tag(),
location: Some(args.main_module),
no_color: !colors::use_color(),
color_level: colors::get_color_level(),
is_stdout_tty: deno_terminal::is_stdout_tty(),
is_stderr_tty: deno_terminal::is_stderr_tty(),
unstable_features,
user_agent: version::DENO_VERSION_INFO.user_agent.to_string(),
inspect: shared.options.is_inspecting,
has_node_modules_dir: shared.options.has_node_modules_dir,
argv0: shared.options.argv0.clone(),
node_debug: shared.options.node_debug.clone(),
node_ipc_fd: None,
mode: WorkerExecutionMode::Worker,
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: args.close_on_idle,
},
extensions: vec![],
startup_snapshot: crate::js::deno_isolate_init(),
create_params: create_isolate_create_params(),
unsafely_ignore_certificate_errors: shared
.options
.unsafely_ignore_certificate_errors
.clone(),
seed: shared.options.seed,
create_web_worker_cb,
format_js_error_fn: Some(Arc::new(format_js_error)),
worker_type: args.worker_type,
stdio: stdio.clone(),
cache_storage_dir,
strace_ops: shared.options.strace_ops.clone(),
close_on_idle: args.close_on_idle,
maybe_worker_metadata: args.maybe_worker_metadata,
enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled(
),
};
WebWorker::bootstrap_from_options(services, options)
})
}
/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs.
/// Instead probe for the total memory on the system and use it instead
/// as a default.
pub fn create_isolate_create_params() -> Option<v8::CreateParams> {
let maybe_mem_info = deno_runtime::sys_info::mem_info();
maybe_mem_info.map(|mem_info| {
v8::CreateParams::default()
.heap_limits_from_system_memory(mem_info.total, 0)
})
}
#[allow(clippy::print_stdout)]
#[allow(clippy::print_stderr)]
#[cfg(test)]
mod tests {
use std::rc::Rc;
use deno_core::resolve_path;
use deno_core::FsModuleLoader;
use deno_fs::RealFs;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::permissions::RuntimePermissionDescriptorParser;
use deno_runtime::worker::WorkerOptions;
use deno_runtime::worker::WorkerServiceOptions;
use super::*;
@ -881,8 +590,12 @@ mod tests {
..Default::default()
};
MainWorker::bootstrap_from_options::<CliSys>(
main_module,
MainWorker::bootstrap_from_options::<
DenoInNpmPackageChecker,
CliNpmResolver,
CliSys,
>(
&main_module,
WorkerServiceOptions {
module_loader: Rc::new(FsModuleLoader),
permissions: PermissionsContainer::new(

2
ext/broadcast_channel/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_broadcast_channel"
version = "0.179.0"
version = "0.180.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/cache/Cargo.toml vendored
View file

@ -2,7 +2,7 @@
[package]
name = "deno_cache"
version = "0.117.0"
version = "0.118.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/canvas/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_canvas"
version = "0.54.0"
version = "0.55.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/console/01_console.js
View file

@ -216,7 +216,7 @@ const styles = {
regexp: "red",
module: "underline",
internalError: "red",
temporal: "magenta",
temporal: "cyan",
};
const defaultFG = 39;

2
ext/console/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_console"
version = "0.185.0"
version = "0.186.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

Some files were not shown because too many files have changed in this diff Show more