foster/denoland-deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-02-01 12:16:11 -05:00
Code Issues Packages 1 Wiki Activity

Merge branch 'main' into open-flag-on-serve

Browse source
This commit is contained in:
HasanAlrimawi 2024-12-16 09:45:43 +02:00 • committed by GitHub
commit 3ff08abde9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
118 changed files with 3154 additions and 1449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/cargo_publish.yml vendored
View file

@ -35,7 +35,7 @@ jobs:
- name: Install deno
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
deno-version: v2.x
- name: Publish
env:

67
.github/workflows/ci.generate.ts vendored
View file

@ -59,6 +59,15 @@ const Runners = {
const prCacheKeyPrefix =
`${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`;
const prCacheKey = `${prCacheKeyPrefix}\${{ github.sha }}`;
const prCachePath = [
// this must match for save and restore (https://github.com/actions/cache/issues/1444)
"./target",
"!./target/*/gn_out",
"!./target/*/gn_root",
"!./target/*/*.zip",
"!./target/*/*.tar.gz",
].join("\n");
// Note that you may need to add more version to the `apt-get remove` line below if you change this
const llvmVersion = 19;
@ -196,7 +205,7 @@ const installNodeStep = {
const installDenoStep = {
name: "Install Deno",
uses: "denoland/setup-deno@v2",
with: { "deno-version": "v1.x" },
with: { "deno-version": "v2.x" },
};
const authenticateWithGoogleCloud = {
@ -475,6 +484,27 @@ const ci = {
" -czvf target/release/deno_src.tar.gz -C .. deno",
].join("\n"),
},
{
name: "Cache Cargo home",
uses: "actions/cache@v4",
with: {
// See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
// Note that with the new sparse registry format, we no longer have to cache a `.git` dir
path: [
"~/.cargo/.crates.toml",
"~/.cargo/.crates2.json",
"~/.cargo/bin",
"~/.cargo/registry/index",
"~/.cargo/registry/cache",
"~/.cargo/git/db",
].join("\n"),
key:
`${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ hashFiles('Cargo.lock') }}`,
// We will try to restore from the closest cargo-home we can find
"restore-keys":
`${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-`,
},
},
installRustStep,
{
if:
@ -598,23 +628,6 @@ const ci = {
installBenchTools,
].join("\n"),
},
{
name: "Cache Cargo home",
uses: "actions/cache@v4",
with: {
// See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
// Note that with the new sparse registry format, we no longer have to cache a `.git` dir
path: [
"~/.cargo/registry/index",
"~/.cargo/registry/cache",
].join("\n"),
key:
`${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ hashFiles('Cargo.lock') }}`,
// We will try to restore from the closest cargo-home we can find
"restore-keys":
`${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}`,
},
},
{
// Restore cache from the latest 'main' branch build.
name: "Restore cache build output (PR)",
@ -622,13 +635,7 @@ const ci = {
if:
"github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')",
with: {
path: [
"./target",
"!./target/*/gn_out",
"!./target/*/gn_root",
"!./target/*/*.zip",
"!./target/*/*.tar.gz",
].join("\n"),
path: prCachePath,
key: "never_saved",
"restore-keys": prCacheKeyPrefix,
},
@ -1080,14 +1087,8 @@ const ci = {
if:
"(matrix.job == 'test' || matrix.job == 'lint') && github.ref == 'refs/heads/main'",
with: {
path: [
"./target",
"!./target/*/gn_out",
"!./target/*/*.zip",
"!./target/*/*.sha256sum",
"!./target/*/*.tar.gz",
].join("\n"),
key: prCacheKeyPrefix + "${{ github.sha }}",
path: prCachePath,
key: prCacheKey,
},
},
]),

26
.github/workflows/ci.yml vendored
View file

@ -174,13 +174,26 @@ jobs:
mkdir -p target/release
tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
-czvf target/release/deno_src.tar.gz -C .. deno
- name: Cache Cargo home
uses: actions/cache@v4
with:
path: |-
~/.cargo/.crates.toml
~/.cargo/.crates2.json
~/.cargo/bin
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
if: '!(matrix.skip)'
- uses: dsherret/rust-toolchain-file@v1
if: '!(matrix.skip)'
- if: '!(matrix.skip) && (matrix.job == ''lint'' || matrix.job == ''test'' || matrix.job == ''bench'')'
name: Install Deno
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
deno-version: v2.x
- name: Install Python
uses: actions/setup-python@v5
with:
@ -355,15 +368,6 @@ jobs:
- name: Install benchmark tools
if: '!(matrix.skip) && (matrix.job == ''bench'')'
run: ./tools/install_prebuilt.js wrk hyperfine
- name: Cache Cargo home
uses: actions/cache@v4
with:
path: |-
~/.cargo/registry/index
~/.cargo/registry/cache
key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}'
if: '!(matrix.skip)'
- name: Restore cache build output (PR)
uses: actions/cache/restore@v4
if: '!(matrix.skip) && (github.ref != ''refs/heads/main'' && !startsWith(github.ref, ''refs/tags/''))'
@ -682,8 +686,8 @@ jobs:
path: |-
./target
!./target/*/gn_out
!./target/*/gn_root
!./target/*/*.zip
!./target/*/*.sha256sum
!./target/*/*.tar.gz
key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
publish-canary:

45
.github/workflows/npm_publish.yml vendored Normal file
View file

@ -0,0 +1,45 @@
name: npm_publish
on:
workflow_dispatch:
inputs:
version:
description: 'Version'
type: string
release:
types: [published]
permissions:
id-token: write
jobs:
build:
name: npm publish
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Configure git
run: |
git config --global core.symlinks true
git config --global fetch.parallel 32
- name: Clone repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Install Deno
uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- name: Install Node
uses: actions/setup-node@v4
with:
node-version: '22.x'
registry-url: 'https://registry.npmjs.org'
- name: Publish
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: ./tools/release/npm/build.ts ${{ github.event.inputs.version }} --publish

2
.github/workflows/promote_to_release.yml vendored
View file

@ -42,7 +42,7 @@ jobs:
- name: Install deno
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
deno-version: v2.x
- name: Install rust-codesign
run: |-

2
.github/workflows/start_release.yml vendored
View file

@ -36,7 +36,7 @@ jobs:
- name: Install deno
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
deno-version: v2.x
- name: Create Gist URL
env:

2
.github/workflows/version_bump.yml vendored
View file

@ -41,7 +41,7 @@ jobs:
- name: Install deno
uses: denoland/setup-deno@v2
with:
deno-version: v1.x
deno-version: v2.x
- name: Run version bump
run: |

9
Cargo.lock generated
View file

@ -658,9 +658,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.6.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]]
name = "cache_control"
@ -4022,9 +4022,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.7"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
"futures-channel",
@ -4035,7 +4035,6 @@ dependencies = [
"pin-project-lite",
"socket2",
"tokio",
"tower",
"tower-service",
"tracing",
]

2
Cargo.toml
View file

@ -142,7 +142,7 @@ http_v02 = { package = "http", version = "0.2.9" }
httparse = "1.8.0"
hyper = { version = "1.4.1", features = ["full"] }
hyper-rustls = { version = "0.27.2", default-features = false, features = ["http1", "http2", "tls12", "ring"] }
hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy", "server", "server-auto"] }
hyper-util = { version = "0.1.10", features = ["tokio", "client", "client-legacy", "server", "server-auto"] }
hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] }
indexmap = { version = "2", features = ["serde"] }
ipnet = "2.3"

2
cli/Cargo.toml
View file

@ -80,7 +80,7 @@ deno_npm.workspace = true
deno_npm_cache.workspace = true
deno_package_json.workspace = true
deno_path_util.workspace = true
deno_resolver.workspace = true
deno_resolver = { workspace = true, features = ["sync"] }
deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_semver.workspace = true
deno_task_shell = "=0.20.2"

9
cli/args/deno_json.rs
View file

@ -64,6 +64,15 @@ impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> {
}
}
pub fn import_map_deps(
import_map: &serde_json::Value,
) -> HashSet<JsrDepPackageReq> {
let values = imports_values(import_map.get("imports"))
.into_iter()
.chain(scope_values(import_map.get("scopes")));
values_to_set(values)
}
pub fn deno_json_deps(
config: &deno_config::deno_json::ConfigFile,
) -> HashSet<JsrDepPackageReq> {

47
cli/args/flags.rs
View file

@ -37,6 +37,7 @@ use deno_path_util::url_to_file_path;
use deno_runtime::deno_permissions::PermissionsOptions;
use deno_runtime::deno_permissions::SysDescriptor;
use deno_telemetry::OtelConfig;
use deno_telemetry::OtelConsoleConfig;
use log::debug;
use log::Level;
use serde::Deserialize;
@ -988,21 +989,41 @@ impl Flags {
args
}
pub fn otel_config(&self) -> Option<OtelConfig> {
if self
pub fn otel_config(&self) -> OtelConfig {
let has_unstable_flag = self
.unstable_config
.features
.contains(&String::from("otel"))
{
Some(OtelConfig {
runtime_name: Cow::Borrowed("deno"),
runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno),
deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC")
.is_ok(),
..Default::default()
})
} else {
None
.contains(&String::from("otel"));
let otel_var = |name| match std::env::var(name) {
Ok(s) if s.to_lowercase() == "true" => Some(true),
Ok(s) if s.to_lowercase() == "false" => Some(false),
_ => None,
};
let disabled =
!has_unstable_flag || otel_var("OTEL_SDK_DISABLED").unwrap_or(false);
let default = !disabled && otel_var("OTEL_DENO").unwrap_or(false);
OtelConfig {
tracing_enabled: !disabled
&& otel_var("OTEL_DENO_TRACING").unwrap_or(default),
console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() {
Ok(_) if disabled => OtelConsoleConfig::Ignore,
Ok("ignore") => OtelConsoleConfig::Ignore,
Ok("capture") => OtelConsoleConfig::Capture,
Ok("replace") => OtelConsoleConfig::Replace,
_ => {
if default {
OtelConsoleConfig::Capture
} else {
OtelConsoleConfig::Ignore
}
}
},
deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC")
.as_deref()
== Ok("1"),
}
}

9
cli/args/lockfile.rs
View file

@ -9,11 +9,13 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::parking_lot::MutexGuard;
use deno_core::serde_json;
use deno_lockfile::WorkspaceMemberConfig;
use deno_package_json::PackageJsonDepValue;
use deno_runtime::deno_node::PackageJson;
use deno_semver::jsr::JsrDepPackageReq;
use crate::args::deno_json::import_map_deps;
use crate::cache;
use crate::util::fs::atomic_write_file_with_retries;
use crate::Flags;
@ -101,6 +103,7 @@ impl CliLockfile {
pub fn discover(
flags: &Flags,
workspace: &Workspace,
maybe_external_import_map: Option<&serde_json::Value>,
) -> Result<Option<CliLockfile>, AnyError> {
fn pkg_json_deps(
maybe_pkg_json: Option<&PackageJson>,
@ -171,7 +174,11 @@ impl CliLockfile {
let config = deno_lockfile::WorkspaceConfig {
root: WorkspaceMemberConfig {
package_json_deps: pkg_json_deps(root_folder.pkg_json.as_deref()),
dependencies: deno_json_deps(root_folder.deno_json.as_deref()),
dependencies: if let Some(map) = maybe_external_import_map {
import_map_deps(map)
} else {
deno_json_deps(root_folder.deno_json.as_deref())
},
},
members: workspace
.config_folders()

58
cli/args/mod.rs
View file

@ -31,6 +31,7 @@ use deno_npm_cache::NpmCacheSetting;
use deno_path_util::normalize_path;
use deno_semver::npm::NpmPackageReqReference;
use deno_telemetry::OtelConfig;
use deno_telemetry::OtelRuntimeConfig;
use import_map::resolve_import_map_value_from_specifier;
pub use deno_config::deno_json::BenchConfig;
@ -807,6 +808,7 @@ pub struct CliOptions {
maybe_node_modules_folder: Option<PathBuf>,
npmrc: Arc<ResolvedNpmRc>,
maybe_lockfile: Option<Arc<CliLockfile>>,
maybe_external_import_map: Option<(PathBuf, serde_json::Value)>,
overrides: CliOptionOverrides,
pub start_dir: Arc<WorkspaceDirectory>,
pub deno_dir_provider: Arc<DenoDirProvider>,
@ -820,6 +822,7 @@ impl CliOptions {
npmrc: Arc<ResolvedNpmRc>,
start_dir: Arc<WorkspaceDirectory>,
force_global_cache: bool,
maybe_external_import_map: Option<(PathBuf, serde_json::Value)>,
) -> Result<Self, AnyError> {
if let Some(insecure_allowlist) =
flags.unsafely_ignore_certificate_errors.as_ref()
@ -857,6 +860,7 @@ impl CliOptions {
maybe_node_modules_folder,
overrides: Default::default(),
main_module_cell: std::sync::OnceLock::new(),
maybe_external_import_map,
start_dir,
deno_dir_provider,
})
@ -932,7 +936,33 @@ impl CliOptions {
let (npmrc, _) = discover_npmrc_from_workspace(&start_dir.workspace)?;
let maybe_lock_file = CliLockfile::discover(&flags, &start_dir.workspace)?;
fn load_external_import_map(
deno_json: &ConfigFile,
) -> Result<Option<(PathBuf, serde_json::Value)>, AnyError> {
if !deno_json.is_an_import_map() {
if let Some(path) = deno_json.to_import_map_path()? {
let contents = std::fs::read_to_string(&path).with_context(|| {
format!("Unable to read import map at '{}'", path.display())
})?;
let map = serde_json::from_str(&contents)?;
return Ok(Some((path, map)));
}
}
Ok(None)
}
let external_import_map =
if let Some(deno_json) = start_dir.workspace.root_deno_json() {
load_external_import_map(deno_json)?
} else {
None
};
let maybe_lock_file = CliLockfile::discover(
&flags,
&start_dir.workspace,
external_import_map.as_ref().map(|(_, v)| v),
)?;
log::debug!("Finished config loading.");
@ -943,6 +973,7 @@ impl CliOptions {
npmrc,
Arc::new(start_dir),
false,
external_import_map,
)
}
@ -1063,7 +1094,7 @@ impl CliOptions {
file_fetcher: &FileFetcher,
pkg_json_dep_resolution: PackageJsonDepResolution,
) -> Result<WorkspaceResolver, AnyError> {
let overrode_no_import_map = self
let overrode_no_import_map: bool = self
.overrides
.import_map_specifier
.as_ref()
@ -1091,7 +1122,19 @@ impl CliOptions {
value,
})
}
None => None,
None => {
if let Some((path, import_map)) =
self.maybe_external_import_map.as_ref()
{
let path_url = deno_path_util::url_from_file_path(path)?;
Some(deno_config::workspace::SpecifiedImportMap {
base_url: path_url,
value: import_map.clone(),
})
} else {
None
}
}
}
};
Ok(self.workspace().create_resolver(
@ -1130,7 +1173,7 @@ impl CliOptions {
}
}
pub fn otel_config(&self) -> Option<OtelConfig> {
pub fn otel_config(&self) -> OtelConfig {
self.flags.otel_config()
}
@ -2000,6 +2043,13 @@ pub enum NpmCachingStrategy {
Manual,
}
pub(crate) fn otel_runtime_config() -> OtelRuntimeConfig {
OtelRuntimeConfig {
runtime_name: Cow::Borrowed("deno"),
runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno),
}
}
#[cfg(test)]
mod test {
use pretty_assertions::assert_eq;

24
cli/lsp/documents.rs
View file

@ -65,6 +65,12 @@ pub enum LanguageId {
Html,
Css,
Yaml,
Sql,
Svelte,
Vue,
Astro,
Vento,
Nunjucks,
Unknown,
}
@ -81,6 +87,12 @@ impl LanguageId {
LanguageId::Html => Some("html"),
LanguageId::Css => Some("css"),
LanguageId::Yaml => Some("yaml"),
LanguageId::Sql => Some("sql"),
LanguageId::Svelte => Some("svelte"),
LanguageId::Vue => Some("vue"),
LanguageId::Astro => Some("astro"),
LanguageId::Vento => Some("vto"),
LanguageId::Nunjucks => Some("njk"),
LanguageId::Unknown => None,
}
}
@ -96,6 +108,12 @@ impl LanguageId {
LanguageId::Html => Some("text/html"),
LanguageId::Css => Some("text/css"),
LanguageId::Yaml => Some("application/yaml"),
LanguageId::Sql => None,
LanguageId::Svelte => None,
LanguageId::Vue => None,
LanguageId::Astro => None,
LanguageId::Vento => None,
LanguageId::Nunjucks => None,
LanguageId::Unknown => None,
}
}
@ -123,6 +141,12 @@ impl FromStr for LanguageId {
"html" => Ok(Self::Html),
"css" => Ok(Self::Css),
"yaml" => Ok(Self::Yaml),
"sql" => Ok(Self::Sql),
"svelte" => Ok(Self::Svelte),
"vue" => Ok(Self::Vue),
"astro" => Ok(Self::Astro),
"vento" => Ok(Self::Vento),
"nunjucks" => Ok(Self::Nunjucks),
_ => Ok(Self::Unknown),
}
}

1
cli/lsp/language_server.rs
View file

@ -3676,6 +3676,7 @@ impl Inner {
.unwrap_or_else(create_default_npmrc),
workspace,
force_global_cache,
None,
)?;
let open_docs = self.documents.documents(DocumentsFilter::OpenDiagnosable);

10
cli/main.rs
View file

@ -437,20 +437,18 @@ fn resolve_flags_and_init(
if err.kind() == clap::error::ErrorKind::DisplayVersion =>
{
// Ignore results to avoid BrokenPipe errors.
util::logger::init(None);
util::logger::init(None, None);
let _ = err.print();
deno_runtime::exit(0);
}
Err(err) => {
util::logger::init(None);
util::logger::init(None, None);
exit_for_error(AnyError::from(err))
}
};
if let Some(otel_config) = flags.otel_config() {
deno_telemetry::init(otel_config)?;
}
util::logger::init(flags.log_level);
deno_telemetry::init(crate::args::otel_runtime_config())?;
util::logger::init(flags.log_level, Some(flags.otel_config()));
// TODO(bartlomieju): remove in Deno v2.5 and hard error then.
if flags.unstable_config.legacy_flag_enabled {

11
cli/mainrt.rs
View file

@ -87,17 +87,18 @@ fn main() {
let future = async move {
match standalone {
Ok(Some(data)) => {
if let Some(otel_config) = data.metadata.otel_config.clone() {
deno_telemetry::init(otel_config)?;
}
util::logger::init(data.metadata.log_level);
deno_telemetry::init(crate::args::otel_runtime_config())?;
util::logger::init(
data.metadata.log_level,
Some(data.metadata.otel_config.clone()),
);
load_env_vars(&data.metadata.env_vars_from_env_file);
let exit_code = standalone::run(data).await?;
deno_runtime::exit(exit_code);
}
Ok(None) => Ok(()),
Err(err) => {
util::logger::init(None);
util::logger::init(None, None);
Err(err)
}
}

351
cli/standalone/binary.rs
View file

@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmSystemInfo;
use deno_path_util::url_from_directory_path;
use deno_path_util::url_from_file_path;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::RealFs;
@ -76,6 +79,7 @@ use crate::resolver::CjsTracker;
use crate::shared::ReleaseChannel;
use crate::standalone::virtual_fs::VfsEntry;
use crate::util::archive;
use crate::util::fs::canonicalize_path;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -88,31 +92,28 @@ use super::serialization::DeserializedDataSection;
use super::serialization::RemoteModulesStore;
use super::serialization::RemoteModulesStoreBuilder;
use super::virtual_fs::output_vfs;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsFileSubDataKind;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
use super::virtual_fs::WindowsSystemRootablePath;
pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str =
".deno_compile_node_modules";
/// A URL that can be designated as the base for relative URLs.
///
/// After creation, this URL may be used to get the key for a
/// module in the binary.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url);
impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> {
fn from(url: &'a Url) -> Self {
Self(url)
}
pub enum StandaloneRelativeFileBaseUrl<'a> {
WindowsSystemRoot,
Path(&'a Url),
}
impl<'a> StandaloneRelativeFileBaseUrl<'a> {
pub fn new(url: &'a Url) -> Self {
debug_assert_eq!(url.scheme(), "file");
Self(url)
}
/// Gets the module map key of the provided specifier.
///
/// * Descendant file specifiers will be made relative to the base.
@ -122,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> {
if target.scheme() != "file" {
return Cow::Borrowed(target.as_str());
}
let base = match self {
Self::Path(base) => base,
Self::WindowsSystemRoot => return Cow::Borrowed(target.path()),
};
match self.0.make_relative(target) {
match base.make_relative(target) {
Some(relative) => {
if relative.starts_with("../") {
Cow::Borrowed(target.as_str())
} else {
Cow::Owned(relative)
}
// This is not a great scenario to have because it means that the
// specifier is outside the vfs and could cause the binary to act
// strangely. If you encounter this, the fix is to add more paths
// to the vfs builder by calling `add_possible_min_root_dir`.
debug_assert!(
!relative.starts_with("../"),
"{} -> {} ({})",
base.as_str(),
target.as_str(),
relative,
);
Cow::Owned(relative)
}
None => Cow::Borrowed(target.as_str()),
}
}
pub fn inner(&self) -> &Url {
self.0
}
}
#[derive(Deserialize, Serialize)]
@ -192,7 +200,7 @@ pub struct Metadata {
pub entrypoint_key: String,
pub node_modules: Option<NodeModules>,
pub unstable_config: UnstableConfig,
pub otel_config: Option<OtelConfig>, // None means disabled.
pub otel_config: OtelConfig,
}
fn write_binary_bytes(
@ -201,7 +209,7 @@ fn write_binary_bytes(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
vfs: &BuiltVfs,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let data_section_bytes =
@ -372,7 +380,6 @@ pub struct WriteBinOptions<'a> {
pub writer: File,
pub display_output_filename: &'a str,
pub graph: &'a ModuleGraph,
pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>,
pub entrypoint: &'a ModuleSpecifier,
pub include_files: &'a [ModuleSpecifier],
pub compile_flags: &'a CompileFlags,
@ -556,7 +563,6 @@ impl<'a> DenoCompileBinaryWriter<'a> {
writer,
display_output_filename,
graph,
root_dir_url,
entrypoint,
include_files,
compile_flags,
@ -568,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> {
Some(CaData::Bytes(bytes)) => Some(bytes.clone()),
None => None,
};
let root_path = root_dir_url.inner().to_file_path().unwrap();
let (maybe_npm_vfs, node_modules, npm_snapshot) =
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
let npm_vfs_builder = self
.build_npm_vfs(&root_path)
.context("Building npm vfs.")?;
(
Some(npm_vfs_builder),
Some(NodeModules::Managed {
node_modules_dir: self
.npm_resolver
.root_node_modules_path()
.map(|path| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(path).unwrap(),
)
.into_owned()
}),
}),
Some(snapshot),
)
} else {
(None, None, None)
}
let mut vfs = VfsBuilder::new();
let npm_snapshot = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?;
Some(snapshot)
} else {
None
}
InnerCliNpmResolverRef::Byonm(resolver) => {
let npm_vfs_builder = self.build_npm_vfs(&root_path)?;
(
Some(npm_vfs_builder),
Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(node_modules_dir)
.unwrap(),
)
.into_owned()
},
),
}),
None,
)
}
};
let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs {
npm_vfs
} else {
VfsBuilder::new(root_path.clone())?
}
InnerCliNpmResolverRef::Byonm(_) => {
self.fill_npm_vfs(&mut vfs)?;
None
}
};
for include_file in include_files {
let path = deno_path_util::url_to_file_path(include_file)?;
if path.is_dir() {
// TODO(#26941): we should analyze if any of these are
// modules in order to include their dependencies
vfs
.add_dir_recursive(&path)
.with_context(|| format!("Including {}", path.display()))?;
} else {
vfs
.add_file_at_path(&path)
.with_context(|| format!("Including {}", path.display()))?;
}
vfs
.add_file_at_path(&path)
.with_context(|| format!("Including {}", path.display()))?;
}
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() {
@ -707,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
remote_modules_store.add_redirects(&graph.redirects);
if let Some(import_map) = self.workspace_resolver.maybe_import_map() {
if let Ok(file_path) = url_to_file_path(import_map.base_url()) {
if let Some(import_map_parent_dir) = file_path.parent() {
// tell the vfs about the import map's parent directory in case it
// falls outside what the root of where the VFS will be based
vfs.add_possible_min_root_dir(import_map_parent_dir);
}
}
}
if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() {
// ensure the vfs doesn't go below the node_modules directory's parent
if let Some(parent) = node_modules_dir.parent() {
vfs.add_possible_min_root_dir(parent);
}
}
let vfs = self.build_vfs_consolidating_global_npm_cache(vfs);
let root_dir_url = match &vfs.root_path {
WindowsSystemRootablePath::Path(dir) => {
Some(url_from_directory_path(dir)?)
}
WindowsSystemRootablePath::WindowSystemRoot => None,
};
let root_dir_url = match &root_dir_url {
Some(url) => StandaloneRelativeFileBaseUrl::Path(url),
None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot,
};
let node_modules = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(_) => {
npm_snapshot.as_ref().map(|_| NodeModules::Managed {
node_modules_dir: self.npm_resolver.root_node_modules_path().map(
|path| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(path).unwrap(),
)
.into_owned()
},
),
})
}
InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(node_modules_dir)
.unwrap(),
)
.into_owned()
},
),
}),
};
let env_vars_from_env_file = match self.cli_options.env_file_name() {
Some(env_filenames) => {
let mut aggregated_env_vars = IndexMap::new();
@ -721,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => Default::default(),
};
output_vfs(&vfs, display_output_filename);
let metadata = Metadata {
argv: compile_flags.args.clone(),
seed: self.cli_options.seed(),
@ -785,21 +803,19 @@ impl<'a> DenoCompileBinaryWriter<'a> {
otel_config: self.cli_options.otel_config(),
};
output_vfs(&vfs, display_output_filename);
write_binary_bytes(
writer,
original_bin,
&metadata,
npm_snapshot.map(|s| s.into_serialized()),
&remote_modules_store,
vfs,
&vfs,
compile_flags,
)
.context("Writing binary bytes")
}
fn build_npm_vfs(&self, root_path: &Path) -> Result<VfsBuilder, AnyError> {
fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> {
fn maybe_warn_different_system(system_info: &NpmSystemInfo) {
if system_info != &NpmSystemInfo::default() {
log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning"));
@ -810,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
if let Some(node_modules_path) = npm_resolver.root_node_modules_path() {
maybe_warn_different_system(&self.npm_system_info);
let mut builder = VfsBuilder::new(root_path.to_path_buf())?;
builder.add_dir_recursive(node_modules_path)?;
Ok(builder)
Ok(())
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let global_cache_root_path = npm_resolver.global_cache_root_path();
let mut builder =
VfsBuilder::new(global_cache_root_path.to_path_buf())?;
// we'll flatten to remove any custom registries later
let mut packages =
npm_resolver.all_system_packages(&self.npm_system_info);
packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
@ -827,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?;
builder.add_dir_recursive(&folder)?;
}
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
builder.with_root_dir(|root_dir| {
root_dir.name = ".deno_compile_node_modules".to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
log::debug!(
"Flattening {} into node_modules",
entry.name()
);
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
{
panic!(
"Unhandled scenario where a duplicate entry was found: {:?}",
existing
);
}
}
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => {
new_entries.push(entry);
}
}
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
});
builder.set_new_root_path(root_path.to_path_buf())?;
Ok(builder)
Ok(())
}
}
InnerCliNpmResolverRef::Byonm(_) => {
maybe_warn_different_system(&self.npm_system_info);
let mut builder = VfsBuilder::new(root_path.to_path_buf())?;
for pkg_json in self.cli_options.workspace().package_jsons() {
builder.add_file_at_path(&pkg_json.path)?;
}
@ -908,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
}
}
Ok(builder)
Ok(())
}
}
}
fn build_vfs_consolidating_global_npm_cache(
&self,
mut vfs: VfsBuilder,
) -> BuiltVfs {
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
if npm_resolver.root_node_modules_path().is_some() {
return vfs.build();
}
let global_cache_root_path = npm_resolver.global_cache_root_path();
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else {
return vfs.build();
};
root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
log::debug!("Flattening {} into node_modules", entry.name());
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
{
panic!(
"Unhandled scenario where a duplicate entry was found: {:?}",
existing
);
}
}
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => {
new_entries.push(entry);
}
}
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
// it's better to not expose the user's cache directory, so take it out
// of there
let parent = global_cache_root_path.parent().unwrap();
let parent_dir = vfs.get_dir_mut(parent).unwrap();
let index = parent_dir
.entries
.iter()
.position(|entry| {
entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME
})
.unwrap();
let npm_global_cache_dir_entry = parent_dir.entries.remove(index);
// go up from the ancestors removing empty directories...
// this is not as optimized as it could be
let mut last_name =
Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME);
for ancestor in parent.ancestors() {
let dir = vfs.get_dir_mut(ancestor).unwrap();
if let Some(index) = dir
.entries
.iter()
.position(|entry| entry.name() == last_name)
{
dir.entries.remove(index);
}
last_name = Cow::Owned(dir.name.clone());
if !dir.entries.is_empty() {
break;
}
}
// now build the vfs and add the global cache dir entry there
let mut built_vfs = vfs.build();
built_vfs.root.insert_entry(npm_global_cache_dir_entry);
built_vfs
}
InnerCliNpmResolverRef::Byonm(_) => vfs.build(),
}
}
}
fn get_denort_path(deno_exe: PathBuf) -> Option<OsString> {

12
cli/standalone/serialization.rs
View file

@ -23,6 +23,7 @@ use deno_semver::package::PackageReq;
use crate::standalone::virtual_fs::VirtualDirectory;
use super::binary::Metadata;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::VfsBuilder;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
@ -39,7 +40,7 @@ pub fn serialize_binary_data_section(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
vfs: &BuiltVfs,
) -> Result<Vec<u8>, AnyError> {
fn write_bytes_with_len(bytes: &mut Vec<u8>, data: &[u8]) {
bytes.extend_from_slice(&(data.len() as u64).to_le_bytes());
@ -73,12 +74,11 @@ pub fn serialize_binary_data_section(
}
// 4. VFS
{
let (vfs, vfs_files) = vfs.into_dir_and_files();
let vfs = serde_json::to_string(&vfs)?;
write_bytes_with_len(&mut bytes, vfs.as_bytes());
let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::<u64>();
let serialized_vfs = serde_json::to_string(&vfs.root)?;
write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes());
let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::<u64>();
bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes());
for file in &vfs_files {
for file in &vfs.files {
bytes.extend_from_slice(file);
}
}

671
cli/standalone/virtual_fs.rs
View file

@ -15,17 +15,21 @@ use std::rc::Rc;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::BufMutView;
use deno_core::BufView;
use deno_core::ResourceHandleFd;
use deno_path_util::normalize_path;
use deno_path_util::strip_unc_prefix;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_io;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use indexmap::IndexSet;
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
@ -34,6 +38,38 @@ use crate::util;
use crate::util::display::DisplayTreeNode;
use crate::util::fs::canonicalize_path;
use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME;
#[derive(Debug, PartialEq, Eq)]
pub enum WindowsSystemRootablePath {
/// The root of the system above any drive letters.
WindowSystemRoot,
Path(PathBuf),
}
impl WindowsSystemRootablePath {
pub fn join(&self, name_component: &str) -> PathBuf {
// this method doesn't handle multiple components
debug_assert!(!name_component.contains('\\'));
debug_assert!(!name_component.contains('/'));
match self {
WindowsSystemRootablePath::WindowSystemRoot => {
// windows drive letter
PathBuf::from(&format!("{}\\", name_component))
}
WindowsSystemRootablePath::Path(path) => path.join(name_component),
}
}
}
#[derive(Debug)]
pub struct BuiltVfs {
pub root_path: WindowsSystemRootablePath,
pub root: VirtualDirectory,
pub files: Vec<Vec<u8>>,
}
#[derive(Debug, Copy, Clone)]
pub enum VfsFileSubDataKind {
/// Raw bytes of the file.
@ -43,84 +79,84 @@ pub enum VfsFileSubDataKind {
ModuleGraph,
}
#[derive(Error, Debug)]
#[error(
"Failed to strip prefix '{}' from '{}'", root_path.display(), target.display()
)]
pub struct StripRootError {
root_path: PathBuf,
target: PathBuf,
}
#[derive(Debug)]
pub struct VfsBuilder {
root_path: PathBuf,
root_dir: VirtualDirectory,
executable_root: VirtualDirectory,
files: Vec<Vec<u8>>,
current_offset: u64,
file_offsets: HashMap<String, u64>,
/// The minimum root directory that should be included in the VFS.
min_root_dir: Option<WindowsSystemRootablePath>,
}
impl VfsBuilder {
pub fn new(root_path: PathBuf) -> Result<Self, AnyError> {
let root_path = canonicalize_path(&root_path)
.with_context(|| format!("Canonicalizing {}", root_path.display()))?;
log::debug!("Building vfs with root '{}'", root_path.display());
Ok(Self {
root_dir: VirtualDirectory {
name: root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
pub fn new() -> Self {
Self {
executable_root: VirtualDirectory {
name: "/".to_string(),
entries: Vec::new(),
},
root_path,
files: Vec::new(),
current_offset: 0,
file_offsets: Default::default(),
})
min_root_dir: Default::default(),
}
}
pub fn set_new_root_path(
&mut self,
root_path: PathBuf,
) -> Result<(), AnyError> {
let root_path = canonicalize_path(&root_path)?;
self.root_path = root_path;
self.root_dir = VirtualDirectory {
name: self
.root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
entries: vec![VfsEntry::Dir(VirtualDirectory {
name: std::mem::take(&mut self.root_dir.name),
entries: std::mem::take(&mut self.root_dir.entries),
})],
};
Ok(())
}
/// Add a directory that might be the minimum root directory
/// of the VFS.
///
/// For example, say the user has a deno.json and specifies an
/// import map in a parent directory. The import map won't be
/// included in the VFS, but its base will meaning we need to
/// tell the VFS builder to include the base of the import map
/// by calling this method.
pub fn add_possible_min_root_dir(&mut self, path: &Path) {
self.add_dir_raw(path);
pub fn with_root_dir<R>(
&mut self,
with_root: impl FnOnce(&mut VirtualDirectory) -> R,
) -> R {
with_root(&mut self.root_dir)
match &self.min_root_dir {
Some(WindowsSystemRootablePath::WindowSystemRoot) => {
// already the root dir
}
Some(WindowsSystemRootablePath::Path(current_path)) => {
let mut common_components = Vec::new();
for (a, b) in current_path.components().zip(path.components()) {
if a != b {
break;
}
common_components.push(a);
}
if common_components.is_empty() {
if cfg!(windows) {
self.min_root_dir =
Some(WindowsSystemRootablePath::WindowSystemRoot);
} else {
self.min_root_dir =
Some(WindowsSystemRootablePath::Path(PathBuf::from("/")));
}
} else {
self.min_root_dir = Some(WindowsSystemRootablePath::Path(
common_components.iter().collect(),
));
}
}
None => {
self.min_root_dir =
Some(WindowsSystemRootablePath::Path(path.to_path_buf()));
}
}
}
pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if path != target_path {
self.add_symlink(path, &target_path)?;
}
self.add_dir_recursive_internal(&target_path)
let target_path = self.resolve_target_path(path)?;
self.add_dir_recursive_not_symlink(&target_path)
}
fn add_dir_recursive_internal(
fn add_dir_recursive_not_symlink(
&mut self,
path: &Path,
) -> Result<(), AnyError> {
self.add_dir(path)?;
self.add_dir_raw(path);
let read_dir = std::fs::read_dir(path)
.with_context(|| format!("Reading {}", path.display()))?;
@ -133,49 +169,26 @@ impl VfsBuilder {
let path = entry.path();
if file_type.is_dir() {
self.add_dir_recursive_internal(&path)?;
self.add_dir_recursive_not_symlink(&path)?;
} else if file_type.is_file() {
self.add_file_at_path_not_symlink(&path)?;
} else if file_type.is_symlink() {
match util::fs::canonicalize_path(&path) {
Ok(target) => {
if let Err(StripRootError { .. }) = self.add_symlink(&path, &target)
{
if target.is_file() {
// this may change behavior, so warn the user about it
log::warn!(
"{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.",
crate::colors::yellow("Warning"),
self.root_path.display(),
path.display(),
target.display(),
);
// inline the symlink and make the target file
let file_bytes = std::fs::read(&target)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file_with_data_inner(
&path,
file_bytes,
VfsFileSubDataKind::Raw,
)?;
} else {
log::warn!(
"{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.",
crate::colors::yellow("Warning"),
self.root_path.display(),
path.display(),
target.display(),
);
}
match self.add_symlink(&path) {
Ok(target) => match target {
SymlinkTarget::File(target) => {
self.add_file_at_path_not_symlink(&target)?
}
}
SymlinkTarget::Dir(target) => {
self.add_dir_recursive_not_symlink(&target)?;
}
},
Err(err) => {
log::warn!(
"{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}",
crate::colors::yellow("Warning"),
path.display(),
err
);
"{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}",
crate::colors::yellow("Warning"),
path.display(),
err
);
}
}
}
@ -184,15 +197,15 @@ impl VfsBuilder {
Ok(())
}
fn add_dir(
&mut self,
path: &Path,
) -> Result<&mut VirtualDirectory, StripRootError> {
fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory {
log::debug!("Ensuring directory '{}'", path.display());
let path = self.path_relative_root(path)?;
let mut current_dir = &mut self.root_dir;
debug_assert!(path.is_absolute());
let mut current_dir = &mut self.executable_root;
for component in path.components() {
if matches!(component, std::path::Component::RootDir) {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
@ -218,15 +231,44 @@ impl VfsBuilder {
};
}
Ok(current_dir)
current_dir
}
pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory {
&mut self.executable_root
}
pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> {
debug_assert!(path.is_absolute());
let mut current_dir = &mut self.executable_root;
for component in path.components() {
if matches!(component, std::path::Component::RootDir) {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(_) => return None,
};
match &mut current_dir.entries[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
_ => unreachable!(),
};
}
Some(current_dir)
}
pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if target_path != path {
self.add_symlink(path, &target_path)?;
}
self.add_file_at_path_not_symlink(&target_path)
let file_bytes = std::fs::read(path)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw)
}
fn add_file_at_path_not_symlink(
@ -244,11 +286,15 @@ impl VfsBuilder {
data: Vec<u8>,
sub_data_kind: VfsFileSubDataKind,
) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if target_path != path {
self.add_symlink(path, &target_path)?;
let metadata = std::fs::symlink_metadata(path).with_context(|| {
format!("Resolving target path for '{}'", path.display())
})?;
if metadata.is_symlink() {
let target = self.add_symlink(path)?.into_path_buf();
self.add_file_with_data_inner(&target, data, sub_data_kind)
} else {
self.add_file_with_data_inner(path, data, sub_data_kind)
}
self.add_file_with_data_inner(&target_path, data, sub_data_kind)
}
fn add_file_with_data_inner(
@ -267,7 +313,7 @@ impl VfsBuilder {
self.current_offset
};
let dir = self.add_dir(path.parent().unwrap())?;
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
let offset_and_len = OffsetWithLength {
offset,
@ -309,74 +355,162 @@ impl VfsBuilder {
Ok(())
}
fn add_symlink(
fn resolve_target_path(&mut self, path: &Path) -> Result<PathBuf, AnyError> {
let metadata = std::fs::symlink_metadata(path).with_context(|| {
format!("Resolving target path for '{}'", path.display())
})?;
if metadata.is_symlink() {
Ok(self.add_symlink(path)?.into_path_buf())
} else {
Ok(path.to_path_buf())
}
}
fn add_symlink(&mut self, path: &Path) -> Result<SymlinkTarget, AnyError> {
self.add_symlink_inner(path, &mut IndexSet::new())
}
fn add_symlink_inner(
&mut self,
path: &Path,
target: &Path,
) -> Result<(), StripRootError> {
log::debug!(
"Adding symlink '{}' to '{}'",
path.display(),
target.display()
visited: &mut IndexSet<PathBuf>,
) -> Result<SymlinkTarget, AnyError> {
log::debug!("Adding symlink '{}'", path.display());
let target = strip_unc_prefix(
std::fs::read_link(path)
.with_context(|| format!("Reading symlink '{}'", path.display()))?,
);
let relative_target = self.path_relative_root(target)?;
let relative_path = match self.path_relative_root(path) {
Ok(path) => path,
Err(StripRootError { .. }) => {
// ignore if the original path is outside the root directory
return Ok(());
}
};
if relative_target == relative_path {
// it's the same, ignore
return Ok(());
}
let dir = self.add_dir(path.parent().unwrap())?;
let target = normalize_path(path.parent().unwrap().join(&target));
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => Ok(()), // previously inserted
Ok(_) => {} // previously inserted
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: relative_target
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
dest_parts: VirtualSymlinkParts::from_path(&target),
}),
);
Ok(())
}
}
let target_metadata =
std::fs::symlink_metadata(&target).with_context(|| {
format!("Reading symlink target '{}'", target.display())
})?;
if target_metadata.is_symlink() {
if !visited.insert(target.clone()) {
// todo: probably don't error in this scenario
bail!(
"Circular symlink detected: {} -> {}",
visited
.iter()
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(" -> "),
target.display()
);
}
self.add_symlink_inner(&target, visited)
} else if target_metadata.is_dir() {
Ok(SymlinkTarget::Dir(target))
} else {
Ok(SymlinkTarget::File(target))
}
}
pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec<Vec<u8>>) {
(self.root_dir, self.files)
}
pub fn build(self) -> BuiltVfs {
fn strip_prefix_from_symlinks(
dir: &mut VirtualDirectory,
parts: &[String],
) {
for entry in &mut dir.entries {
match entry {
VfsEntry::Dir(dir) => {
strip_prefix_from_symlinks(dir, parts);
}
VfsEntry::File(_) => {}
VfsEntry::Symlink(symlink) => {
let old_parts = std::mem::take(&mut symlink.dest_parts.0);
symlink.dest_parts.0 =
old_parts.into_iter().skip(parts.len()).collect();
}
}
}
}
fn path_relative_root(&self, path: &Path) -> Result<PathBuf, StripRootError> {
match path.strip_prefix(&self.root_path) {
Ok(p) => Ok(p.to_path_buf()),
Err(_) => Err(StripRootError {
root_path: self.root_path.clone(),
target: path.to_path_buf(),
}),
let mut current_dir = self.executable_root;
let mut current_path = if cfg!(windows) {
WindowsSystemRootablePath::WindowSystemRoot
} else {
WindowsSystemRootablePath::Path(PathBuf::from("/"))
};
loop {
if current_dir.entries.len() != 1 {
break;
}
if self.min_root_dir.as_ref() == Some(&current_path) {
break;
}
match &current_dir.entries[0] {
VfsEntry::Dir(dir) => {
if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
// special directory we want to maintain
break;
}
match current_dir.entries.remove(0) {
VfsEntry::Dir(dir) => {
current_path =
WindowsSystemRootablePath::Path(current_path.join(&dir.name));
current_dir = dir;
}
_ => unreachable!(),
};
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => break,
}
}
if let WindowsSystemRootablePath::Path(path) = &current_path {
strip_prefix_from_symlinks(
&mut current_dir,
&VirtualSymlinkParts::from_path(path).0,
);
}
BuiltVfs {
root_path: current_path,
root: current_dir,
files: self.files,
}
}
}
pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) {
#[derive(Debug)]
enum SymlinkTarget {
File(PathBuf),
Dir(PathBuf),
}
impl SymlinkTarget {
pub fn into_path_buf(self) -> PathBuf {
match self {
Self::File(path) => path,
Self::Dir(path) => path,
}
}
}
pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) {
if !log::log_enabled!(log::Level::Info) {
return; // no need to compute if won't output
}
if builder.root_dir.entries.is_empty() {
if vfs.root.entries.is_empty() {
return; // nothing to output
}
let mut text = String::new();
let display_tree = vfs_as_display_tree(builder, executable_name);
let display_tree = vfs_as_display_tree(vfs, executable_name);
display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string
log::info!(
"\n{}\n",
@ -386,7 +520,7 @@ pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) {
}
fn vfs_as_display_tree(
builder: &VfsBuilder,
vfs: &BuiltVfs,
executable_name: &str,
) -> DisplayTreeNode {
enum EntryOutput<'a> {
@ -398,20 +532,38 @@ fn vfs_as_display_tree(
impl<'a> EntryOutput<'a> {
pub fn as_display_tree(&self, name: String) -> DisplayTreeNode {
let mut children = match self {
EntryOutput::Subset(vec) => vec
.iter()
.map(|e| e.output.as_display_tree(e.name.to_string()))
.collect(),
EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => {
vec![]
}
};
// we only want to collapse leafs so that nodes of the
// same depth have the same indentation
let collapse_single_child =
children.len() == 1 && children[0].children.is_empty();
DisplayTreeNode {
text: match self {
EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name,
EntryOutput::All => format!("{}/*", name),
EntryOutput::Subset(_) => {
if collapse_single_child {
format!("{}/{}", name, children[0].text)
} else {
name
}
}
EntryOutput::File => name,
EntryOutput::Symlink(parts) => {
format!("{} --> {}", name, parts.join("/"))
}
},
children: match self {
EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())],
EntryOutput::Subset(vec) => vec
.iter()
.map(|e| e.output.as_display_tree(e.name.to_string()))
.collect(),
EntryOutput::File | EntryOutput::Symlink(_) => vec![],
children: if collapse_single_child {
children.remove(0).children
} else {
children
},
}
}
@ -422,37 +574,81 @@ fn vfs_as_display_tree(
output: EntryOutput<'a>,
}
fn include_all_entries<'a>(
dir: &Path,
vfs_dir: &'a VirtualDirectory,
) -> EntryOutput<'a> {
EntryOutput::Subset(
fn show_global_node_modules_dir(
vfs_dir: &VirtualDirectory,
) -> Vec<DirEntryOutput> {
fn show_subset_deep(
vfs_dir: &VirtualDirectory,
depth: usize,
) -> EntryOutput {
if depth == 0 {
EntryOutput::All
} else {
EntryOutput::Subset(show_subset(vfs_dir, depth))
}
}
fn show_subset(
vfs_dir: &VirtualDirectory,
depth: usize,
) -> Vec<DirEntryOutput> {
vfs_dir
.entries
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(&dir.join(entry.name()), entry),
output: match entry {
VfsEntry::Dir(virtual_directory) => {
show_subset_deep(virtual_directory, depth - 1)
}
VfsEntry::File(_) => EntryOutput::File,
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
}
},
})
.collect(),
)
.collect()
}
// in this scenario, we want to show
// .deno_compile_node_modules/localhost/<package_name>/<version>/*
show_subset(vfs_dir, 3)
}
fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> {
fn include_all_entries<'a>(
dir_path: &WindowsSystemRootablePath,
vfs_dir: &'a VirtualDirectory,
) -> Vec<DirEntryOutput<'a>> {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return show_global_node_modules_dir(vfs_dir);
}
vfs_dir
.entries
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(dir_path.join(entry.name()), entry),
})
.collect()
}
fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput {
match entry {
VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory),
VfsEntry::File(_) => EntryOutput::File,
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts)
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
}
}
}
fn analyze_dir<'a>(
dir: &Path,
vfs_dir: &'a VirtualDirectory,
) -> EntryOutput<'a> {
let real_entry_count = std::fs::read_dir(dir)
fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir));
}
let real_entry_count = std::fs::read_dir(&dir)
.ok()
.map(|entries| entries.flat_map(|e| e.ok()).count())
.unwrap_or(0);
@ -462,7 +658,7 @@ fn vfs_as_display_tree(
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(&dir.join(entry.name()), entry),
output: analyze_entry(dir.join(entry.name()), entry),
})
.collect::<Vec<_>>();
if children
@ -474,15 +670,23 @@ fn vfs_as_display_tree(
EntryOutput::Subset(children)
}
} else {
include_all_entries(dir, vfs_dir)
EntryOutput::Subset(include_all_entries(
&WindowsSystemRootablePath::Path(dir),
vfs_dir,
))
}
}
// always include all the entries for the root directory, otherwise the
// user might not have context about what's being shown
let output = include_all_entries(&builder.root_path, &builder.root_dir);
output
.as_display_tree(deno_terminal::colors::italic(executable_name).to_string())
let child_entries = include_all_entries(&vfs.root_path, &vfs.root);
DisplayTreeNode {
text: deno_terminal::colors::italic(executable_name).to_string(),
children: child_entries
.iter()
.map(|entry| entry.output.as_display_tree(entry.name.to_string()))
.collect(),
}
}
#[derive(Debug)]
@ -603,6 +807,20 @@ pub struct VirtualDirectory {
pub entries: Vec<VfsEntry>,
}
impl VirtualDirectory {
pub fn insert_entry(&mut self, entry: VfsEntry) {
let name = entry.name();
match self.entries.binary_search_by(|e| e.name().cmp(name)) {
Ok(index) => {
self.entries[index] = entry;
}
Err(insert_index) => {
self.entries.insert(insert_index, entry);
}
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct OffsetWithLength {
#[serde(rename = "o")]
@ -626,18 +844,33 @@ pub struct VirtualFile {
pub module_graph_offset: OffsetWithLength,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlinkParts(Vec<String>);
impl VirtualSymlinkParts {
pub fn from_path(path: &Path) -> Self {
Self(
path
.components()
.filter(|c| !matches!(c, std::path::Component::RootDir))
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect(),
)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "p")]
pub dest_parts: Vec<String>,
pub dest_parts: VirtualSymlinkParts,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts {
for part in &self.dest_parts.0 {
dest.push(part);
}
dest
@ -709,10 +942,10 @@ impl VfsRoot {
let mut final_path = self.root_path.clone();
let mut current_entry = VfsEntryRef::Dir(&self.dir);
for component in relative_path.components() {
let component = component.as_os_str().to_string_lossy();
let component = component.as_os_str();
let current_dir = match current_entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
final_path.push(component);
dir
}
VfsEntryRef::Symlink(symlink) => {
@ -721,7 +954,7 @@ impl VfsRoot {
final_path = resolved_path; // overwrite with the new resolved path
match entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
final_path.push(component);
dir
}
_ => {
@ -739,6 +972,7 @@ impl VfsRoot {
));
}
};
let component = component.to_string_lossy();
match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
@ -1136,6 +1370,7 @@ impl FileBackedVfs {
mod test {
use console_static_text::ansi::strip_ansi_codes;
use std::io::Write;
use test_util::assert_contains;
use test_util::TempDir;
use super::*;
@ -1159,8 +1394,11 @@ mod test {
// will canonicalize the root path
let src_path = temp_dir.path().canonicalize().join("src");
src_path.create_dir_all();
src_path.join("sub_dir").create_dir_all();
src_path.join("e.txt").write("e");
src_path.symlink_file("e.txt", "sub_dir/e.txt");
let src_path = src_path.to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
let mut builder = VfsBuilder::new();
builder
.add_file_with_data_inner(
&src_path.join("a.txt"),
@ -1190,18 +1428,9 @@ mod test {
VfsFileSubDataKind::Raw,
)
.unwrap();
builder.add_file_at_path(&src_path.join("e.txt")).unwrap();
builder
.add_file_with_data_inner(
&src_path.join("e.txt"),
"e".into(),
VfsFileSubDataKind::Raw,
)
.unwrap();
builder
.add_symlink(
&src_path.join("sub_dir").join("e.txt"),
&src_path.join("e.txt"),
)
.add_symlink(&src_path.join("sub_dir").join("e.txt"))
.unwrap();
// get the virtual fs
@ -1262,7 +1491,7 @@ mod test {
// build and create the virtual fs
let src_path = temp_dir_path.join("src").to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
let mut builder = VfsBuilder::new();
builder.add_dir_recursive(&src_path).unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
@ -1300,10 +1529,10 @@ mod test {
temp_dir: &TempDir,
) -> (PathBuf, FileBackedVfs) {
let virtual_fs_file = temp_dir.path().join("virtual_fs");
let (root_dir, files) = builder.into_dir_and_files();
let vfs = builder.build();
{
let mut file = std::fs::File::create(&virtual_fs_file).unwrap();
for file_data in &files {
for file_data in &vfs.files {
file.write_all(file_data).unwrap();
}
}
@ -1314,7 +1543,7 @@ mod test {
FileBackedVfs::new(
Cow::Owned(data),
VfsRoot {
dir: root_dir,
dir: vfs.root,
root_path: dest_path.to_path_buf(),
start_file_offset: 0,
},
@ -1327,41 +1556,22 @@ mod test {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().canonicalize().join("src");
src_path.create_dir_all();
src_path.symlink_file("a.txt", "b.txt");
src_path.symlink_file("b.txt", "c.txt");
src_path.symlink_file("c.txt", "a.txt");
let src_path = src_path.to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
builder
.add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt"))
.unwrap();
builder
.add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt"))
.unwrap();
builder
.add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt"))
.unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(
virtual_fs
.file_entry(&dest_path.join("a.txt"))
.err()
.unwrap()
.to_string(),
"circular symlinks",
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(),
dest_path.join("b.txt")
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(),
dest_path.join("c.txt")
);
let mut builder = VfsBuilder::new();
let err = builder
.add_symlink(src_path.join("a.txt").as_path())
.unwrap_err();
assert_contains!(err.to_string(), "Circular symlink detected",);
}
#[tokio::test]
async fn test_open_file() {
let temp_dir = TempDir::new();
let temp_path = temp_dir.path().canonicalize();
let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap();
let mut builder = VfsBuilder::new();
builder
.add_file_with_data_inner(
temp_path.join("a.txt").as_path(),
@ -1436,8 +1646,7 @@ mod test {
temp_dir.write("c/a.txt", "contents");
temp_dir.symlink_file("c/a.txt", "c/b.txt");
assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works
let mut vfs_builder =
VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap();
let mut vfs_builder = VfsBuilder::new();
// full dir
vfs_builder
.add_dir_recursive(temp_dir.path().join("a").as_path())
@ -1451,16 +1660,14 @@ mod test {
.add_dir_recursive(temp_dir.path().join("c").as_path())
.unwrap();
temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory
let node = vfs_as_display_tree(&vfs_builder, "executable");
let node = vfs_as_display_tree(&vfs_builder.build(), "executable");
let mut text = String::new();
node.print(&mut text).unwrap();
assert_eq!(
strip_ansi_codes(&text),
r#"executable
├─┬ a
│ └── *
├─┬ b
│ └── a.txt
├── a/*
├── b/a.txt
└─┬ c
├── a.txt
└── b.txt --> c/a.txt

167
cli/tools/compile.rs
View file

@ -5,7 +5,6 @@ use crate::args::CompileFlags;
use crate::args::Flags;
use crate::factory::CliFactory;
use crate::http_util::HttpClientProvider;
use crate::standalone::binary::StandaloneRelativeFileBaseUrl;
use crate::standalone::binary::WriteBinOptions;
use crate::standalone::is_standalone_binary;
use deno_ast::MediaType;
@ -17,8 +16,11 @@ use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_graph::GraphKind;
use deno_path_util::url_from_file_path;
use deno_path_util::url_to_file_path;
use deno_terminal::colors;
use rand::Rng;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@ -84,29 +86,6 @@ pub async fn compile(
let ts_config_for_emit = cli_options
.resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?;
check_warn_tsconfig(&ts_config_for_emit);
let root_dir_url = resolve_root_dir_from_specifiers(
cli_options.workspace().root_dir(),
graph
.specifiers()
.map(|(s, _)| s)
.chain(
cli_options
.node_modules_dir_path()
.and_then(|p| ModuleSpecifier::from_directory_path(p).ok())
.iter(),
)
.chain(include_files.iter())
.chain(
// sometimes the import map path is outside the root dir
cli_options
.workspace()
.to_import_map_path()
.ok()
.and_then(|p| p.and_then(|p| url_from_file_path(&p).ok()))
.iter(),
),
);
log::debug!("Binary root dir: {}", root_dir_url);
log::info!(
"{} {} to {}",
colors::green("Compile"),
@ -138,7 +117,6 @@ pub async fn compile(
.unwrap()
.to_string_lossy(),
graph: &graph,
root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url),
entrypoint,
include_files: &include_files,
compile_flags: &compile_flags,
@ -261,15 +239,58 @@ fn get_module_roots_and_include_files(
}
}
let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1);
let mut include_files = Vec::with_capacity(compile_flags.include.len());
fn analyze_path(
url: &ModuleSpecifier,
module_roots: &mut Vec<ModuleSpecifier>,
include_files: &mut Vec<ModuleSpecifier>,
searched_paths: &mut HashSet<PathBuf>,
) -> Result<(), AnyError> {
let Ok(path) = url_to_file_path(url) else {
return Ok(());
};
let mut pending = VecDeque::from([path]);
while let Some(path) = pending.pop_front() {
if !searched_paths.insert(path.clone()) {
continue;
}
if !path.is_dir() {
let url = url_from_file_path(&path)?;
include_files.push(url.clone());
if is_module_graph_module(&url) {
module_roots.push(url);
}
continue;
}
for entry in std::fs::read_dir(&path).with_context(|| {
format!("Failed reading directory '{}'", path.display())
})? {
let entry = entry.with_context(|| {
format!("Failed reading entry in directory '{}'", path.display())
})?;
pending.push_back(entry.path());
}
}
Ok(())
}
let mut searched_paths = HashSet::new();
let mut module_roots = Vec::new();
let mut include_files = Vec::new();
module_roots.push(entrypoint.clone());
for side_module in &compile_flags.include {
let url = resolve_url_or_path(side_module, initial_cwd)?;
if is_module_graph_module(&url) {
module_roots.push(url);
module_roots.push(url.clone());
if url.scheme() == "file" {
include_files.push(url);
}
} else {
include_files.push(url);
analyze_path(
&url,
&mut module_roots,
&mut include_files,
&mut searched_paths,
)?;
}
}
Ok((module_roots, include_files))
@ -335,57 +356,6 @@ fn get_os_specific_filepath(
}
}
fn resolve_root_dir_from_specifiers<'a>(
starting_dir: &ModuleSpecifier,
specifiers: impl Iterator<Item = &'a ModuleSpecifier>,
) -> ModuleSpecifier {
fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str {
let min_length = a.len().min(b.len());
let mut last_slash = 0;
for i in 0..min_length {
if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' {
last_slash = i;
} else if a.as_bytes()[i] != b.as_bytes()[i] {
break;
}
}
// Return the common root path up to the last common slash.
// This returns a slice of the original string 'a', up to and including the last matching '/'.
let common = &a[..=last_slash];
if cfg!(windows) && common == "file:///" {
a
} else {
common
}
}
fn is_file_system_root(url: &str) -> bool {
let Some(path) = url.strip_prefix("file:///") else {
return false;
};
if cfg!(windows) {
let Some((_drive, path)) = path.split_once('/') else {
return true;
};
path.is_empty()
} else {
path.is_empty()
}
}
let mut found_dir = starting_dir.as_str();
if !is_file_system_root(found_dir) {
for specifier in specifiers {
if specifier.scheme() == "file" {
found_dir = select_common_root(found_dir, specifier.as_str());
}
}
}
ModuleSpecifier::parse(found_dir).unwrap()
}
#[cfg(test)]
mod test {
pub use super::*;
@ -462,41 +432,4 @@ mod test {
run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe");
run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2");
}
#[test]
fn test_resolve_root_dir_from_specifiers() {
fn resolve(start: &str, specifiers: &[&str]) -> String {
let specifiers = specifiers
.iter()
.map(|s| ModuleSpecifier::parse(s).unwrap())
.collect::<Vec<_>>();
resolve_root_dir_from_specifiers(
&ModuleSpecifier::parse(start).unwrap(),
specifiers.iter(),
)
.to_string()
}
assert_eq!(
resolve("file:///a/b/e", &["file:///a/b/c/d"]),
"file:///a/b/"
);
assert_eq!(
resolve("file:///a/b/c/", &["file:///a/b/c/d"]),
"file:///a/b/c/"
);
assert_eq!(
resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]),
"file:///a/b/c/"
);
assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///");
if cfg!(windows) {
assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/");
// this will ignore the other one because it's on a separate drive
assert_eq!(
resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]),
"file:///c:/a/b/c/"
);
}
}
}

61
cli/tools/registry/pm/deps.rs
View file

@ -2,6 +2,7 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
@ -11,6 +12,7 @@ use deno_config::deno_json::ConfigFileRc;
use deno_config::workspace::Workspace;
use deno_config::workspace::WorkspaceDirectory;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::future::try_join;
use deno_core::futures::stream::FuturesOrdered;
@ -43,10 +45,10 @@ use crate::npm::NpmFetchResolver;
use super::ConfigUpdater;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ImportMapKind {
Inline,
Outline,
Outline(PathBuf),
}
#[derive(Clone)]
@ -62,9 +64,12 @@ impl DepLocation {
pub fn file_path(&self) -> Cow<std::path::Path> {
match self {
DepLocation::DenoJson(arc, _, _) => {
Cow::Owned(arc.specifier.to_file_path().unwrap())
}
DepLocation::DenoJson(arc, _, kind) => match kind {
ImportMapKind::Inline => {
Cow::Owned(arc.specifier.to_file_path().unwrap())
}
ImportMapKind::Outline(path) => Cow::Borrowed(path.as_path()),
},
DepLocation::PackageJson(arc, _) => Cow::Borrowed(arc.path.as_ref()),
}
}
@ -238,22 +243,30 @@ fn to_import_map_value_from_imports(
fn deno_json_import_map(
deno_json: &ConfigFile,
) -> Result<Option<(ImportMapWithDiagnostics, ImportMapKind)>, AnyError> {
let (value, kind) =
if deno_json.json.imports.is_some() || deno_json.json.scopes.is_some() {
(
to_import_map_value_from_imports(deno_json),
ImportMapKind::Inline,
)
} else {
match deno_json.to_import_map_path()? {
Some(path) => {
let text = std::fs::read_to_string(&path)?;
let value = serde_json::from_str(&text)?;
(value, ImportMapKind::Outline)
}
None => return Ok(None),
let (value, kind) = if deno_json.json.imports.is_some()
|| deno_json.json.scopes.is_some()
{
(
to_import_map_value_from_imports(deno_json),
ImportMapKind::Inline,
)
} else {
match deno_json.to_import_map_path()? {
Some(path) => {
let err_context = || {
format!(
"loading import map at '{}' (from \"importMap\" field in '{}')",
path.display(),
deno_json.specifier
)
};
let text = std::fs::read_to_string(&path).with_context(err_context)?;
let value = serde_json::from_str(&text).with_context(err_context)?;
(value, ImportMapKind::Outline(path))
}
};
None => return Ok(None),
}
};
import_map::parse_from_value(deno_json.specifier.clone(), value)
.map_err(Into::into)
@ -303,7 +316,7 @@ fn add_deps_from_deno_json(
location: DepLocation::DenoJson(
deno_json.clone(),
key_path,
import_map_kind,
import_map_kind.clone(),
),
kind,
req,
@ -747,11 +760,7 @@ impl DepManager {
let dep = &mut self.deps[dep_id.0];
dep.req.version_req = version_req.clone();
match &dep.location {
DepLocation::DenoJson(arc, key_path, import_map_kind) => {
if matches!(import_map_kind, ImportMapKind::Outline) {
// not supported
continue;
}
DepLocation::DenoJson(arc, key_path, _) => {
let updater =
get_or_create_updater(&mut config_updaters, &dep.location)?;

51
cli/util/logger.rs
View file

@ -1,24 +1,34 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use super::draw_thread::DrawThread;
use deno_telemetry::OtelConfig;
use deno_telemetry::OtelConsoleConfig;
use std::io::Write;
use super::draw_thread::DrawThread;
struct CliLogger(env_logger::Logger);
struct CliLogger {
otel_console_config: OtelConsoleConfig,
logger: env_logger::Logger,
}
impl CliLogger {
pub fn new(logger: env_logger::Logger) -> Self {
Self(logger)
pub fn new(
logger: env_logger::Logger,
otel_console_config: OtelConsoleConfig,
) -> Self {
Self {
logger,
otel_console_config,
}
}
pub fn filter(&self) -> log::LevelFilter {
self.0.filter()
self.logger.filter()
}
}
impl log::Log for CliLogger {
fn enabled(&self, metadata: &log::Metadata) -> bool {
self.0.enabled(metadata)
self.logger.enabled(metadata)
}
fn log(&self, record: &log::Record) {
@ -28,18 +38,30 @@ impl log::Log for CliLogger {
// could potentially block other threads that access the draw
// thread's state
DrawThread::hide();
self.0.log(record);
deno_telemetry::handle_log(record);
match self.otel_console_config {
OtelConsoleConfig::Ignore => {
self.logger.log(record);
}
OtelConsoleConfig::Capture => {
self.logger.log(record);
deno_telemetry::handle_log(record);
}
OtelConsoleConfig::Replace => {
deno_telemetry::handle_log(record);
}
}
DrawThread::show();
}
}
fn flush(&self) {
self.0.flush();
self.logger.flush();
}
}
pub fn init(maybe_level: Option<log::Level>) {
pub fn init(maybe_level: Option<log::Level>, otel_config: Option<OtelConfig>) {
let log_level = maybe_level.unwrap_or(log::Level::Info);
let logger = env_logger::Builder::from_env(
env_logger::Env::new()
@ -93,7 +115,12 @@ pub fn init(maybe_level: Option<log::Level>) {
})
.build();
let cli_logger = CliLogger::new(logger);
let cli_logger = CliLogger::new(
logger,
otel_config
.map(|c| c.console)
.unwrap_or(OtelConsoleConfig::Ignore),
);
let max_level = cli_logger.filter();
let r = log::set_boxed_logger(Box::new(cli_logger));
if r.is_ok() {

4
cli/worker.rs
View file

@ -154,7 +154,7 @@ struct SharedWorkerState {
storage_key_resolver: StorageKeyResolver,
options: CliMainWorkerOptions,
subcommand: DenoSubcommand,
otel_config: Option<OtelConfig>, // `None` means OpenTelemetry is disabled.
otel_config: OtelConfig,
default_npm_caching_strategy: NpmCachingStrategy,
}
@ -426,7 +426,7 @@ impl CliMainWorkerFactory {
storage_key_resolver: StorageKeyResolver,
subcommand: DenoSubcommand,
options: CliMainWorkerOptions,
otel_config: Option<OtelConfig>,
otel_config: OtelConfig,
default_npm_caching_strategy: NpmCachingStrategy,
) -> Self {
Self {

3
ext/fetch/lib.rs
View file

@ -206,9 +206,6 @@ pub enum FetchError {
RequestBuilderHook(deno_core::error::AnyError),
#[error(transparent)]
Io(#[from] std::io::Error),
// Only used for node upgrade
#[error(transparent)]
Hyper(#[from] hyper::Error),
}
pub type CancelableResponseFuture =

4
ext/node/lib.rs
View file

@ -364,9 +364,9 @@ deno_core::extension!(deno_node,
ops::zlib::brotli::op_create_brotli_decompress,
ops::zlib::brotli::op_brotli_decompress_stream,
ops::zlib::brotli::op_brotli_decompress_stream_end,
ops::http::op_node_http_request<P>,
ops::http::op_node_http_fetch_response_upgrade,
ops::http::op_node_http_fetch_send,
ops::http::op_node_http_request_with_conn<P>,
ops::http::op_node_http_await_response,
ops::http2::op_http2_connect,
ops::http2::op_http2_poll_client_connection,
ops::http2::op_http2_client_request,

278
ext/node/ops/http.rs
View file

@ -2,18 +2,20 @@
use std::borrow::Cow;
use std::cell::RefCell;
use std::fmt::Debug;
use std::pin::Pin;
use std::rc::Rc;
use std::task::Context;
use std::task::Poll;
use bytes::Bytes;
use deno_core::error::bad_resource;
use deno_core::error::type_error;
use deno_core::futures::stream::Peekable;
use deno_core::futures::Future;
use deno_core::futures::FutureExt;
use deno_core::futures::Stream;
use deno_core::futures::StreamExt;
use deno_core::futures::TryFutureExt;
use deno_core::op2;
use deno_core::serde::Serialize;
use deno_core::unsync::spawn;
@ -25,17 +27,17 @@ use deno_core::ByteString;
use deno_core::CancelFuture;
use deno_core::CancelHandle;
use deno_core::CancelTryFuture;
use deno_core::Canceled;
use deno_core::OpState;
use deno_core::RcRef;
use deno_core::Resource;
use deno_core::ResourceId;
use deno_fetch::get_or_create_client_from_state;
use deno_fetch::FetchCancelHandle;
use deno_fetch::FetchError;
use deno_fetch::FetchRequestResource;
use deno_fetch::FetchReturn;
use deno_fetch::HttpClientResource;
use deno_fetch::ResBody;
use deno_net::io::TcpStreamResource;
use deno_net::ops_tls::TlsStreamResource;
use deno_permissions::PermissionCheckError;
use http::header::HeaderMap;
use http::header::HeaderName;
use http::header::HeaderValue;
@ -44,41 +46,140 @@ use http::header::CONTENT_LENGTH;
use http::Method;
use http_body_util::BodyExt;
use hyper::body::Frame;
use hyper::body::Incoming;
use hyper_util::rt::TokioIo;
use std::cmp::min;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
#[op2(stack_trace)]
#[derive(Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct NodeHttpResponse {
pub status: u16,
pub status_text: String,
pub headers: Vec<(ByteString, ByteString)>,
pub url: String,
pub response_rid: ResourceId,
pub content_length: Option<u64>,
pub remote_addr_ip: Option<String>,
pub remote_addr_port: Option<u16>,
pub error: Option<String>,
}
type CancelableResponseResult =
Result<Result<http::Response<Incoming>, hyper::Error>, Canceled>;
pub struct NodeHttpClientResponse {
response: Pin<Box<dyn Future<Output = CancelableResponseResult>>>,
url: String,
}
impl Debug for NodeHttpClientResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NodeHttpClientResponse")
.field("url", &self.url)
.finish()
}
}
impl deno_core::Resource for NodeHttpClientResponse {
fn name(&self) -> Cow<str> {
"nodeHttpClientResponse".into()
}
}
#[derive(Debug, thiserror::Error)]
pub enum ConnError {
#[error(transparent)]
Resource(deno_core::error::AnyError),
#[error(transparent)]
Permission(#[from] PermissionCheckError),
#[error("Invalid URL {0}")]
InvalidUrl(Url),
#[error(transparent)]
InvalidHeaderName(#[from] http::header::InvalidHeaderName),
#[error(transparent)]
InvalidHeaderValue(#[from] http::header::InvalidHeaderValue),
#[error(transparent)]
Url(#[from] url::ParseError),
#[error(transparent)]
Method(#[from] http::method::InvalidMethod),
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("TLS stream is currently in use")]
TlsStreamBusy,
#[error("TCP stream is currently in use")]
TcpStreamBusy,
#[error(transparent)]
ReuniteTcp(#[from] tokio::net::tcp::ReuniteError),
#[error(transparent)]
Canceled(#[from] deno_core::Canceled),
#[error(transparent)]
Hyper(#[from] hyper::Error),
}
#[op2(async, stack_trace)]
#[serde]
pub fn op_node_http_request<P>(
state: &mut OpState,
pub async fn op_node_http_request_with_conn<P>(
state: Rc<RefCell<OpState>>,
#[serde] method: ByteString,
#[string] url: String,
#[serde] headers: Vec<(ByteString, ByteString)>,
#[smi] client_rid: Option<u32>,
#[smi] body: Option<ResourceId>,
) -> Result<FetchReturn, FetchError>
#[smi] conn_rid: ResourceId,
encrypted: bool,
) -> Result<FetchReturn, ConnError>
where
P: crate::NodePermissions + 'static,
{
let client = if let Some(rid) = client_rid {
let r = state
let (_handle, mut sender) = if encrypted {
let resource_rc = state
.borrow_mut()
.resource_table
.get::<HttpClientResource>(rid)
.map_err(FetchError::Resource)?;
r.client.clone()
.take::<TlsStreamResource>(conn_rid)
.map_err(ConnError::Resource)?;
let resource =
Rc::try_unwrap(resource_rc).map_err(|_e| ConnError::TlsStreamBusy)?;
let (read_half, write_half) = resource.into_inner();
let tcp_stream = read_half.unsplit(write_half);
let io = TokioIo::new(tcp_stream);
let (sender, conn) = hyper::client::conn::http1::handshake(io).await?;
(
tokio::task::spawn(async move { conn.with_upgrades().await }),
sender,
)
} else {
get_or_create_client_from_state(state)?
let resource_rc = state
.borrow_mut()
.resource_table
.take::<TcpStreamResource>(conn_rid)
.map_err(ConnError::Resource)?;
let resource =
Rc::try_unwrap(resource_rc).map_err(|_| ConnError::TcpStreamBusy)?;
let (read_half, write_half) = resource.into_inner();
let tcp_stream = read_half.reunite(write_half)?;
let io = TokioIo::new(tcp_stream);
let (sender, conn) = hyper::client::conn::http1::handshake(io).await?;
// Spawn a task to poll the connection, driving the HTTP state
(
tokio::task::spawn(async move {
conn.with_upgrades().await?;
Ok::<_, _>(())
}),
sender,
)
};
// Create the request.
let method = Method::from_bytes(&method)?;
let mut url = Url::parse(&url)?;
let maybe_authority = deno_fetch::extract_authority(&mut url);
let mut url_parsed = Url::parse(&url)?;
let maybe_authority = deno_fetch::extract_authority(&mut url_parsed);
{
let permissions = state.borrow_mut::<P>();
permissions.check_net_url(&url, "ClientRequest")?;
let mut state_ = state.borrow_mut();
let permissions = state_.borrow_mut::<P>();
permissions.check_net_url(&url_parsed, "ClientRequest")?;
}
let mut header_map = HeaderMap::new();
@ -93,9 +194,10 @@ where
(
BodyExt::boxed(NodeHttpResourceToBodyAdapter::new(
state
.borrow_mut()
.resource_table
.take_any(body)
.map_err(FetchError::Resource)?,
.map_err(ConnError::Resource)?,
)),
None,
)
@ -117,10 +219,13 @@ where
let mut request = http::Request::new(body);
*request.method_mut() = method.clone();
*request.uri_mut() = url
.as_str()
let path = url_parsed.path();
let query = url_parsed.query();
*request.uri_mut() = query
.map(|q| format!("{}?{}", path, q))
.unwrap_or_else(|| path.to_string())
.parse()
.map_err(|_| FetchError::InvalidUrl(url.clone()))?;
.map_err(|_| ConnError::InvalidUrl(url_parsed.clone()))?;
*request.headers_mut() = header_map;
if let Some((username, password)) = maybe_authority {
@ -136,86 +241,44 @@ where
let cancel_handle = CancelHandle::new_rc();
let cancel_handle_ = cancel_handle.clone();
let fut = async move {
client
.send(request)
.map_err(Into::into)
.or_cancel(cancel_handle_)
.await
};
let fut =
async move { sender.send_request(request).or_cancel(cancel_handle_).await };
let request_rid = state.resource_table.add(FetchRequestResource {
future: Box::pin(fut),
url,
});
let rid = state
.borrow_mut()
.resource_table
.add(NodeHttpClientResponse {
response: Box::pin(fut),
url: url.clone(),
});
let cancel_handle_rid =
state.resource_table.add(FetchCancelHandle(cancel_handle));
let cancel_handle_rid = state
.borrow_mut()
.resource_table
.add(FetchCancelHandle(cancel_handle));
Ok(FetchReturn {
request_rid,
request_rid: rid,
cancel_handle_rid: Some(cancel_handle_rid),
})
}
#[derive(Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct NodeHttpFetchResponse {
pub status: u16,
pub status_text: String,
pub headers: Vec<(ByteString, ByteString)>,
pub url: String,
pub response_rid: ResourceId,
pub content_length: Option<u64>,
pub remote_addr_ip: Option<String>,
pub remote_addr_port: Option<u16>,
pub error: Option<String>,
}
#[op2(async)]
#[serde]
pub async fn op_node_http_fetch_send(
pub async fn op_node_http_await_response(
state: Rc<RefCell<OpState>>,
#[smi] rid: ResourceId,
) -> Result<NodeHttpFetchResponse, FetchError> {
let request = state
) -> Result<NodeHttpResponse, ConnError> {
let resource = state
.borrow_mut()
.resource_table
.take::<FetchRequestResource>(rid)
.map_err(FetchError::Resource)?;
let request = Rc::try_unwrap(request)
.ok()
.expect("multiple op_node_http_fetch_send ongoing");
let res = match request.future.await {
Ok(Ok(res)) => res,
Ok(Err(err)) => {
// We're going to try and rescue the error cause from a stream and return it from this fetch.
// If any error in the chain is a hyper body error, return that as a special result we can use to
// reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`).
// TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead
if let FetchError::ClientSend(err_src) = &err {
if let Some(client_err) = std::error::Error::source(&err_src.source) {
if let Some(err_src) = client_err.downcast_ref::<hyper::Error>() {
if let Some(err_src) = std::error::Error::source(err_src) {
return Ok(NodeHttpFetchResponse {
error: Some(err_src.to_string()),
..Default::default()
});
}
}
}
}
return Err(err);
}
Err(_) => return Err(FetchError::RequestCanceled),
};
.take::<NodeHttpClientResponse>(rid)
.map_err(ConnError::Resource)?;
let resource = Rc::try_unwrap(resource)
.map_err(|_| ConnError::Resource(bad_resource("NodeHttpClientResponse")))?;
let res = resource.response.await??;
let status = res.status();
let url = request.url.into();
let mut res_headers = Vec::new();
for (key, val) in res.headers().iter() {
res_headers.push((key.as_str().into(), val.as_bytes().into()));
@ -232,16 +295,22 @@ pub async fn op_node_http_fetch_send(
(None, None)
};
let (parts, body) = res.into_parts();
let body = body.map_err(deno_core::anyhow::Error::from);
let body = body.boxed();
let res = http::Response::from_parts(parts, body);
let response_rid = state
.borrow_mut()
.resource_table
.add(NodeHttpFetchResponseResource::new(res, content_length));
.add(NodeHttpResponseResource::new(res, content_length));
Ok(NodeHttpFetchResponse {
Ok(NodeHttpResponse {
status: status.as_u16(),
status_text: status.canonical_reason().unwrap_or("").to_string(),
headers: res_headers,
url,
url: resource.url,
response_rid,
content_length,
remote_addr_ip,
@ -255,12 +324,12 @@ pub async fn op_node_http_fetch_send(
pub async fn op_node_http_fetch_response_upgrade(
state: Rc<RefCell<OpState>>,
#[smi] rid: ResourceId,
) -> Result<ResourceId, FetchError> {
) -> Result<ResourceId, ConnError> {
let raw_response = state
.borrow_mut()
.resource_table
.take::<NodeHttpFetchResponseResource>(rid)
.map_err(FetchError::Resource)?;
.take::<NodeHttpResponseResource>(rid)
.map_err(ConnError::Resource)?;
let raw_response = Rc::try_unwrap(raw_response)
.expect("Someone is holding onto NodeHttpFetchResponseResource");
@ -283,7 +352,7 @@ pub async fn op_node_http_fetch_response_upgrade(
}
read_tx.write_all(&buf[..read]).await?;
}
Ok::<_, FetchError>(())
Ok::<_, ConnError>(())
});
spawn(async move {
let mut buf = [0; 1024];
@ -294,7 +363,7 @@ pub async fn op_node_http_fetch_response_upgrade(
}
upgraded_tx.write_all(&buf[..read]).await?;
}
Ok::<_, FetchError>(())
Ok::<_, ConnError>(())
});
}
@ -379,13 +448,13 @@ impl Default for NodeHttpFetchResponseReader {
}
#[derive(Debug)]
pub struct NodeHttpFetchResponseResource {
pub struct NodeHttpResponseResource {
pub response_reader: AsyncRefCell<NodeHttpFetchResponseReader>,
pub cancel: CancelHandle,
pub size: Option<u64>,
}
impl NodeHttpFetchResponseResource {
impl NodeHttpResponseResource {
pub fn new(response: http::Response<ResBody>, size: Option<u64>) -> Self {
Self {
response_reader: AsyncRefCell::new(NodeHttpFetchResponseReader::Start(
@ -400,14 +469,14 @@ impl NodeHttpFetchResponseResource {
let reader = self.response_reader.into_inner();
match reader {
NodeHttpFetchResponseReader::Start(resp) => {
Ok(hyper::upgrade::on(resp).await?)
hyper::upgrade::on(resp).await
}
_ => unreachable!(),
}
}
}
impl Resource for NodeHttpFetchResponseResource {
impl Resource for NodeHttpResponseResource {
fn name(&self) -> Cow<str> {
"fetchResponse".into()
}
@ -454,9 +523,7 @@ impl Resource for NodeHttpFetchResponseResource {
// safely call `await` on it without creating a race condition.
Some(_) => match reader.as_mut().next().await.unwrap() {
Ok(chunk) => assert!(chunk.is_empty()),
Err(err) => {
break Err(deno_core::error::type_error(err.to_string()))
}
Err(err) => break Err(type_error(err.to_string())),
},
None => break Ok(BufView::empty()),
}
@ -464,7 +531,7 @@ impl Resource for NodeHttpFetchResponseResource {
};
let cancel_handle = RcRef::map(self, |r| &r.cancel);
fut.try_or_cancel(cancel_handle).await.map_err(Into::into)
fut.try_or_cancel(cancel_handle).await
})
}
@ -514,8 +581,9 @@ impl Stream for NodeHttpResourceToBodyAdapter {
Poll::Ready(res) => match res {
Ok(buf) if buf.is_empty() => Poll::Ready(None),
Ok(buf) => {
let bytes: Bytes = buf.to_vec().into();
this.1 = Some(this.0.clone().read(64 * 1024));
Poll::Ready(Some(Ok(buf.to_vec().into())))
Poll::Ready(Some(Ok(bytes)))
}
Err(err) => Poll::Ready(Some(Err(err))),
},

57
ext/node/polyfills/_http_outgoing.ts
View file

@ -491,19 +491,53 @@ Object.defineProperties(
return ret;
},
/** Right after socket is ready, we need to writeHeader() to setup the request and
* client. This is invoked by onSocket(). */
_flushHeaders() {
if (!this._headerSent) {
this._headerSent = true;
this._writeHeader();
}
},
// deno-lint-ignore no-explicit-any
_send(data: any, encoding?: string | null, callback?: () => void) {
if (!this._headerSent && this._header !== null) {
this._writeHeader();
this._headerSent = true;
// if socket is ready, write the data after headers are written.
// if socket is not ready, buffer data in outputbuffer.
if (
this.socket && !this.socket.connecting && this.outputData.length === 0
) {
if (!this._headerSent) {
this._writeHeader();
this._headerSent = true;
}
return this._writeRaw(data, encoding, callback);
} else {
this.outputData.push({ data, encoding, callback });
}
return this._writeRaw(data, encoding, callback);
return false;
},
_writeHeader() {
throw new ERR_METHOD_NOT_IMPLEMENTED("_writeHeader()");
},
_flushBuffer() {
const outputLength = this.outputData.length;
if (outputLength <= 0 || !this.socket || !this._bodyWriter) {
return undefined;
}
const { data, encoding, callback } = this.outputData.shift();
const ret = this._writeRaw(data, encoding, callback);
if (this.outputData.length > 0) {
this.once("drain", this._flushBuffer);
}
return ret;
},
_writeRaw(
// deno-lint-ignore no-explicit-any
data: any,
@ -517,11 +551,15 @@ Object.defineProperties(
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
}
if (data.buffer.byteLength > 0) {
this._bodyWriter.write(data).then(() => {
callback?.();
this.emit("drain");
}).catch((e) => {
this._requestSendError = e;
this._bodyWriter.ready.then(() => {
if (this._bodyWriter.desiredSize > 0) {
this._bodyWriter.write(data).then(() => {
callback?.();
this.emit("drain");
}).catch((e) => {
this._requestSendError = e;
});
}
});
}
return false;
@ -658,7 +696,6 @@ Object.defineProperties(
const { header } = state;
this._header = header + "\r\n";
this._headerSent = false;
// Wait until the first body chunk, or close(), is sent to flush,
// UNLESS we're sending Expect: 100-continue.

7
ext/node/polyfills/_tls_wrap.ts
View file

@ -154,6 +154,13 @@ export class TLSSocket extends net.Socket {
const afterConnect = handle.afterConnect;
handle.afterConnect = async (req: any, status: number) => {
options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined
if (tlssock._isNpmAgent) {
// skips the TLS handshake for @npmcli/agent as it's handled by
// onSocket handler of ClientRequest object.
tlssock.emit("secure");
tlssock.removeListener("end", onConnectEnd);
return afterConnect.call(handle, req, status);
}
try {
const conn = await Deno.startTls(handle[kStreamBaseField], options);

159
ext/node/polyfills/http.ts
View file

@ -5,16 +5,17 @@
import { core, primordials } from "ext:core/mod.js";
import {
op_node_http_await_response,
op_node_http_fetch_response_upgrade,
op_node_http_fetch_send,
op_node_http_request,
op_node_http_request_with_conn,
op_tls_start,
} from "ext:core/ops";
import { TextEncoder } from "ext:deno_web/08_text_encoding.js";
import { setTimeout } from "ext:deno_web/02_timers.js";
import {
_normalizeArgs,
// createConnection,
createConnection,
ListenOptions,
Socket,
} from "node:net";
@ -48,9 +49,10 @@ import { kOutHeaders } from "ext:deno_node/internal/http.ts";
import { _checkIsHttpToken as checkIsHttpToken } from "node:_http_common";
import { Agent, globalAgent } from "node:_http_agent";
import { urlToHttpOptions } from "ext:deno_node/internal/url.ts";
import { kEmptyObject } from "ext:deno_node/internal/util.mjs";
import { kEmptyObject, once } from "ext:deno_node/internal/util.mjs";
import { constants, TCP } from "ext:deno_node/internal_binding/tcp_wrap.ts";
import { notImplemented, warnNotImplemented } from "ext:deno_node/_utils.ts";
import { kStreamBaseField } from "ext:deno_node/internal_binding/stream_wrap.ts";
import { notImplemented } from "ext:deno_node/_utils.ts";
import {
connResetException,
ERR_HTTP_HEADERS_SENT,
@ -62,7 +64,6 @@ import {
} from "ext:deno_node/internal/errors.ts";
import { getTimerDuration } from "ext:deno_node/internal/timers.mjs";
import { serve, upgradeHttpRaw } from "ext:deno_http/00_serve.ts";
import { createHttpClient } from "ext:deno_fetch/22_http_client.js";
import { headersEntries } from "ext:deno_fetch/20_headers.js";
import { timerId } from "ext:deno_web/03_abort_signal.js";
import { clearTimeout as webClearTimeout } from "ext:deno_web/02_timers.js";
@ -148,6 +149,10 @@ class FakeSocket extends EventEmitter {
}
}
function emitErrorEvent(request, error) {
request.emit("error", error);
}
/** ClientRequest represents the http(s) request from the client */
class ClientRequest extends OutgoingMessage {
defaultProtocol = "http:";
@ -160,6 +165,8 @@ class ClientRequest extends OutgoingMessage {
useChunkedEncodingByDefault: boolean;
path: string;
_req: { requestRid: number; cancelHandleRid: number | null } | undefined;
_encrypted = false;
socket: Socket;
constructor(
input: string | URL,
@ -382,17 +389,11 @@ class ClientRequest extends OutgoingMessage {
delete optsWithoutSignal.signal;
}
if (options!.createConnection) {
warnNotImplemented("ClientRequest.options.createConnection");
}
if (options!.lookup) {
notImplemented("ClientRequest.options.lookup");
}
// initiate connection
// TODO(crowlKats): finish this
/*if (this.agent) {
if (this.agent) {
this.agent.addRequest(this, optsWithoutSignal);
} else {
// No agent, default to Connection:close.
@ -422,8 +423,7 @@ class ClientRequest extends OutgoingMessage {
debug("CLIENT use net.createConnection", optsWithoutSignal);
this.onSocket(createConnection(optsWithoutSignal));
}
}*/
this.onSocket(new FakeSocket({ encrypted: this._encrypted }));
}
}
_writeHeader() {
@ -437,9 +437,6 @@ class ClientRequest extends OutgoingMessage {
}
}
const client = this._getClient() ?? createHttpClient({ http2: false });
this._client = client;
if (
this.method === "POST" || this.method === "PATCH" || this.method === "PUT"
) {
@ -455,17 +452,29 @@ class ClientRequest extends OutgoingMessage {
this._bodyWriteRid = resourceForReadableStream(readable);
}
this._req = op_node_http_request(
this.method,
url,
headers,
client[internalRidSymbol],
this._bodyWriteRid,
);
(async () => {
try {
const res = await op_node_http_fetch_send(this._req.requestRid);
const parsedUrl = new URL(url);
let baseConnRid =
this.socket._handle[kStreamBaseField][internalRidSymbol];
if (this._encrypted) {
[baseConnRid] = op_tls_start({
rid: baseConnRid,
hostname: parsedUrl.hostname,
caCerts: [],
alpnProtocols: ["http/1.0", "http/1.1"],
});
}
this._req = await op_node_http_request_with_conn(
this.method,
url,
headers,
this._bodyWriteRid,
baseConnRid,
this._encrypted,
);
this._flushBuffer();
const res = await op_node_http_await_response(this._req!.requestRid);
if (this._req.cancelHandleRid !== null) {
core.tryClose(this._req.cancelHandleRid);
}
@ -473,7 +482,6 @@ class ClientRequest extends OutgoingMessage {
this._timeout.removeEventListener("abort", this._timeoutCb);
webClearTimeout(this._timeout[timerId]);
}
this._client.close();
const incoming = new IncomingMessageForClient(this.socket);
incoming.req = this;
this.res = incoming;
@ -512,12 +520,9 @@ class ClientRequest extends OutgoingMessage {
if (this.method === "CONNECT") {
throw new Error("not implemented CONNECT");
}
const upgradeRid = await op_node_http_fetch_response_upgrade(
res.responseRid,
);
assert(typeof res.remoteAddrIp !== "undefined");
assert(typeof res.remoteAddrIp !== "undefined");
const conn = new UpgradedConn(
upgradeRid,
{
@ -543,13 +548,11 @@ class ClientRequest extends OutgoingMessage {
this._closed = true;
this.emit("close");
} else {
{
incoming._bodyRid = res.responseRid;
}
incoming._bodyRid = res.responseRid;
this.emit("response", incoming);
}
} catch (err) {
if (this._req.cancelHandleRid !== null) {
if (this._req && this._req.cancelHandleRid !== null) {
core.tryClose(this._req.cancelHandleRid);
}
@ -592,11 +595,54 @@ class ClientRequest extends OutgoingMessage {
return undefined;
}
// TODO(bartlomieju): handle error
onSocket(socket, _err) {
onSocket(socket, err) {
nextTick(() => {
this.socket = socket;
this.emit("socket", socket);
// deno-lint-ignore no-this-alias
const req = this;
if (req.destroyed || err) {
req.destroyed = true;
// deno-lint-ignore no-inner-declarations
function _destroy(req, err) {
if (!req.aborted && !err) {
err = new connResetException("socket hang up");
}
if (err) {
emitErrorEvent(req, err);
}
req._closed = true;
req.emit("close");
}
if (socket) {
if (!err && req.agent && !socket.destroyed) {
socket.emit("free");
} else {
finished(socket.destroy(err || req[kError]), (er) => {
if (er?.code === "ERR_STREAM_PREMATURE_CLOSE") {
er = null;
}
_destroy(req, er || err);
});
return;
}
}
_destroy(req, err || req[kError]);
} else {
// Note: this code is specific to deno to initiate a request.
const onConnect = () => {
// Flush the internal buffers once socket is ready.
this._flushHeaders();
};
this.socket = socket;
this.emit("socket", socket);
if (socket.readyState === "opening") {
socket.on("connect", onConnect);
} else {
onConnect();
}
}
});
}
@ -618,14 +664,20 @@ class ClientRequest extends OutgoingMessage {
if (chunk) {
this.write_(chunk, encoding, null, true);
} else if (!this._headerSent) {
this._contentLength = 0;
this._implicitHeader();
this._send("", "latin1");
if (
(this.socket && !this.socket.connecting) || // socket is not connecting, or
(!this.socket && this.outputData.length === 0) // no data to send
) {
this._contentLength = 0;
this._implicitHeader();
this._send("", "latin1");
}
}
(async () => {
const finish = async () => {
try {
await this._bodyWriter.ready;
await this._bodyWriter?.close();
} catch (_) {
} catch {
// The readable stream resource is dropped right after
// read is complete closing the writable stream resource.
// If we try to close the writer again, it will result in an
@ -633,10 +685,20 @@ class ClientRequest extends OutgoingMessage {
}
try {
cb?.();
} catch (_) {
} catch {
//
}
})();
};
if (this.socket && this._bodyWriter) {
finish();
} else {
this.on("drain", () => {
if (this.outputData.length === 0) {
finish();
}
});
}
return this;
}
@ -658,11 +720,6 @@ class ClientRequest extends OutgoingMessage {
}
this.destroyed = true;
const rid = this._client?.[internalRidSymbol];
if (rid) {
core.tryClose(rid);
}
// Request might be closed before we actually made it
if (this._req !== undefined && this._req.cancelHandleRid !== null) {
core.tryClose(this._req.cancelHandleRid);

2
ext/node/polyfills/https.ts
View file

@ -112,7 +112,7 @@ export const globalAgent = new Agent({
/** HttpsClientRequest class loosely follows http.ClientRequest class API. */
class HttpsClientRequest extends ClientRequest {
override _encrypted: true;
override _encrypted = true;
override defaultProtocol = "https:";
override _getClient(): Deno.HttpClient | undefined {
if (caCerts === null) {

10
ext/node/polyfills/internal_binding/cares_wrap.ts
View file

@ -36,7 +36,6 @@ import {
} from "ext:deno_node/internal_binding/async_wrap.ts";
import { ares_strerror } from "ext:deno_node/internal_binding/ares.ts";
import { notImplemented } from "ext:deno_node/_utils.ts";
import { isWindows } from "ext:deno_node/_util/os.ts";
interface LookupAddress {
address: string;
@ -68,7 +67,7 @@ export function getaddrinfo(
_hints: number,
verbatim: boolean,
): number {
let addresses: string[] = [];
const addresses: string[] = [];
// TODO(cmorten): use hints
// REF: https://nodejs.org/api/dns.html#dns_supported_getaddrinfo_flags
@ -107,13 +106,6 @@ export function getaddrinfo(
});
}
// TODO(@bartlomieju): Forces IPv4 as a workaround for Deno not
// aligning with Node on implicit binding on Windows
// REF: https://github.com/denoland/deno/issues/10762
if (isWindows && hostname === "localhost") {
addresses = addresses.filter((address) => isIPv4(address));
}
req.oncomplete(error, addresses);
})();

40
ext/node/polyfills/net.ts
View file

@ -986,16 +986,20 @@ function _lookupAndConnect(
} else {
self._unrefTimer();
defaultTriggerAsyncIdScope(
self[asyncIdSymbol],
_internalConnect,
self,
ip,
port,
addressType,
localAddress,
localPort,
);
defaultTriggerAsyncIdScope(self[asyncIdSymbol], nextTick, () => {
if (self.connecting) {
defaultTriggerAsyncIdScope(
self[asyncIdSymbol],
_internalConnect,
self,
ip,
port,
addressType,
localAddress,
localPort,
);
}
});
}
},
);
@ -1197,6 +1201,9 @@ export class Socket extends Duplex {
_host: string | null = null;
// deno-lint-ignore no-explicit-any
_parent: any = null;
// The flag for detecting if it's called in @npmcli/agent
// See discussions in https://github.com/denoland/deno/pull/25470 for more details.
_isNpmAgent = false;
autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined;
constructor(options: SocketOptions | number) {
@ -1217,6 +1224,19 @@ export class Socket extends Duplex {
super(options);
// Note: If the socket is created from @npmcli/agent, the 'socket' event
// on ClientRequest object happens after 'connect' event on Socket object.
// That swaps the sequence of op_node_http_request_with_conn() call and
// initial socket read. That causes op_node_http_request_with_conn() not
// working.
// To avoid the above situation, we detect the socket created from
// @npmcli/agent and pause the socket (and also skips the startTls call
// if it's TLSSocket)
this._isNpmAgent = new Error().stack?.includes("@npmcli/agent") || false;
if (this._isNpmAgent) {
this.pause();
}
if (options.handle) {
this._handle = options.handle;
this[asyncIdSymbol] = _getNewAsyncId(this._handle);

32
ext/telemetry/lib.rs
View file

@ -97,13 +97,28 @@ deno_core::extension!(
);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OtelConfig {
pub struct OtelRuntimeConfig {
pub runtime_name: Cow<'static, str>,
pub runtime_version: Cow<'static, str>,
}
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct OtelConfig {
pub tracing_enabled: bool,
pub console: OtelConsoleConfig,
pub deterministic: bool,
}
impl OtelConfig {
pub fn as_v8(&self) -> Box<[u8]> {
Box::new([
self.tracing_enabled as u8,
self.console as u8,
self.deterministic as u8,
])
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[repr(u8)]
pub enum OtelConsoleConfig {
@ -112,14 +127,9 @@ pub enum OtelConsoleConfig {
Replace = 2,
}
impl Default for OtelConfig {
impl Default for OtelConsoleConfig {
fn default() -> Self {
Self {
runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")),
runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")),
console: OtelConsoleConfig::Capture,
deterministic: false,
}
Self::Ignore
}
}
@ -411,16 +421,14 @@ static BUILT_IN_INSTRUMENTATION_SCOPE: OnceCell<
opentelemetry::InstrumentationScope,
> = OnceCell::new();
pub fn init(config: OtelConfig) -> anyhow::Result<()> {
pub fn init(config: OtelRuntimeConfig) -> anyhow::Result<()> {
// Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_*
// crates don't do this automatically.
// TODO(piscisaureus): enable GRPC support.
let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() {
Ok("http/protobuf") => Protocol::HttpBinary,
Ok("http/json") => Protocol::HttpJson,
Ok("") | Err(env::VarError::NotPresent) => {
return Ok(());
}
Ok("") | Err(env::VarError::NotPresent) => Protocol::HttpBinary,
Ok(protocol) => {
return Err(anyhow!(
"Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}",

10
ext/telemetry/telemetry.ts
View file

@ -950,15 +950,15 @@ const otelConsoleConfig = {
};
export function bootstrap(
config: [] | [
config: [
0 | 1,
typeof otelConsoleConfig[keyof typeof otelConsoleConfig],
number,
0 | 1,
],
): void {
if (config.length === 0) return;
const { 0: consoleConfig, 1: deterministic } = config;
const { 0: tracingEnabled, 1: consoleConfig, 2: deterministic } = config;
TRACING_ENABLED = true;
TRACING_ENABLED = tracingEnabled === 1;
DETERMINISTIC = deterministic === 1;
switch (consoleConfig) {

15
ext/web/01_dom_exception.js
View file

@ -9,14 +9,15 @@
import { primordials } from "ext:core/mod.js";
const {
Error,
ErrorPrototype,
ErrorCaptureStackTrace,
ObjectDefineProperty,
ObjectCreate,
ObjectEntries,
ObjectHasOwn,
ObjectPrototypeIsPrototypeOf,
ObjectSetPrototypeOf,
ReflectConstruct,
Symbol,
SymbolFor,
} = primordials;
@ -107,12 +108,14 @@ class DOMException {
);
const code = nameToCodeMapping[name] ?? 0;
this[_message] = message;
this[_name] = name;
this[_code] = code;
this[webidl.brand] = webidl.brand;
// execute Error constructor to have stack property and [[ErrorData]] internal slot
const error = ReflectConstruct(Error, [], new.target);
error[_message] = message;
error[_name] = name;
error[_code] = code;
error[webidl.brand] = webidl.brand;
ErrorCaptureStackTrace(this, DOMException);
return error;
}
get message() {

5
resolvers/deno/Cargo.toml
View file

@ -13,11 +13,14 @@ description = "Deno resolution algorithm"
[lib]
path = "lib.rs"
[features]
sync = ["dashmap"]
[dependencies]
anyhow.workspace = true
base32.workspace = true
boxed_error.workspace = true
dashmap.workspace = true
dashmap = { workspace = true, optional = true }
deno_config.workspace = true
deno_media_type.workspace = true
deno_package_json.workspace = true

24
resolvers/deno/cjs.rs
View file

@ -1,13 +1,11 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::sync::Arc;
use dashmap::DashMap;
use crate::sync::MaybeDashMap;
use deno_media_type::MediaType;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::InNpmPackageChecker;
use node_resolver::PackageJsonResolver;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::PackageJsonResolverRc;
use node_resolver::ResolutionMode;
use url::Url;
@ -19,13 +17,13 @@ use url::Url;
#[derive(Debug)]
pub struct CjsTracker<TEnv: NodeResolverEnv> {
is_cjs_resolver: IsCjsResolver<TEnv>,
known: DashMap<Url, ResolutionMode>,
known: MaybeDashMap<Url, ResolutionMode>,
}
impl<TEnv: NodeResolverEnv> CjsTracker<TEnv> {
pub fn new(
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver<TEnv>>,
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
mode: IsCjsResolutionMode,
) -> Self {
Self {
@ -127,15 +125,15 @@ pub enum IsCjsResolutionMode {
/// Resolves whether a module is CJS or ESM.
#[derive(Debug)]
pub struct IsCjsResolver<TEnv: NodeResolverEnv> {
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver<TEnv>>,
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
mode: IsCjsResolutionMode,
}
impl<TEnv: NodeResolverEnv> IsCjsResolver<TEnv> {
pub fn new(
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pkg_json_resolver: Arc<PackageJsonResolver<TEnv>>,
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
mode: IsCjsResolutionMode,
) -> Self {
Self {
@ -185,7 +183,7 @@ impl<TEnv: NodeResolverEnv> IsCjsResolver<TEnv> {
specifier: &Url,
media_type: MediaType,
is_script: Option<bool>,
known_cache: &DashMap<Url, ResolutionMode>,
known_cache: &MaybeDashMap<Url, ResolutionMode>,
) -> Option<ResolutionMode> {
if specifier.scheme() != "file" {
return Some(ResolutionMode::Import);

3
resolvers/deno/clippy.toml
View file

@ -47,6 +47,5 @@ disallowed-methods = [
{ path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" },
]
disallowed-types = [
# todo(dsherret): consider for the future
# { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" },
{ path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" },
]

47
resolvers/deno/lib.rs
View file

@ -4,7 +4,6 @@
#![deny(clippy::print_stdout)]
use std::path::PathBuf;
use std::sync::Arc;
use boxed_error::Boxed;
use deno_config::workspace::MappedResolution;
@ -19,20 +18,20 @@ use fs::DenoResolverFs;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::NodeResolveError;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::InNpmPackageChecker;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::NodeResolution;
use node_resolver::NodeResolutionKind;
use node_resolver::NodeResolver;
use node_resolver::NodeResolverRc;
use node_resolver::ResolutionMode;
use npm::MissingPackageNodeModulesFolderError;
use npm::NodeModulesOutOfDateError;
use npm::NpmReqResolver;
use npm::NpmReqResolverRc;
use npm::ResolveIfForNpmPackageErrorKind;
use npm::ResolvePkgFolderFromDenoReqError;
use npm::ResolveReqWithSubPathErrorKind;
use sloppy_imports::SloppyImportResolverFs;
use sloppy_imports::SloppyImportsResolutionKind;
use sloppy_imports::SloppyImportsResolver;
use sloppy_imports::SloppyImportsResolverRc;
use thiserror::Error;
use url::Url;
@ -40,6 +39,10 @@ pub mod cjs;
pub mod fs;
pub mod npm;
pub mod sloppy_imports;
mod sync;
#[allow(clippy::disallowed_types)]
pub type WorkspaceResolverRc = crate::sync::MaybeArc<WorkspaceResolver>;
#[derive(Debug, Clone)]
pub struct DenoResolution {
@ -80,8 +83,8 @@ pub struct NodeAndNpmReqResolver<
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
> {
pub node_resolver: Arc<NodeResolver<TNodeResolverEnv>>,
pub npm_req_resolver: Arc<NpmReqResolver<Fs, TNodeResolverEnv>>,
pub node_resolver: NodeResolverRc<TNodeResolverEnv>,
pub npm_req_resolver: NpmReqResolverRc<Fs, TNodeResolverEnv>,
}
pub struct DenoResolverOptions<
@ -90,12 +93,12 @@ pub struct DenoResolverOptions<
TNodeResolverEnv: NodeResolverEnv,
TSloppyImportResolverFs: SloppyImportResolverFs,
> {
pub in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pub in_npm_pkg_checker: InNpmPackageCheckerRc,
pub node_and_req_resolver:
Option<NodeAndNpmReqResolver<Fs, TNodeResolverEnv>>,
pub sloppy_imports_resolver:
Option<Arc<SloppyImportsResolver<TSloppyImportResolverFs>>>,
pub workspace_resolver: Arc<WorkspaceResolver>,
Option<SloppyImportsResolverRc<TSloppyImportResolverFs>>,
pub workspace_resolver: WorkspaceResolverRc,
/// Whether "bring your own node_modules" is enabled where Deno does not
/// setup the node_modules directories automatically, but instead uses
/// what already exists on the file system.
@ -111,11 +114,11 @@ pub struct DenoResolver<
TNodeResolverEnv: NodeResolverEnv,
TSloppyImportResolverFs: SloppyImportResolverFs,
> {
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_and_npm_resolver: Option<NodeAndNpmReqResolver<Fs, TNodeResolverEnv>>,
sloppy_imports_resolver:
Option<Arc<SloppyImportsResolver<TSloppyImportResolverFs>>>,
workspace_resolver: Arc<WorkspaceResolver>,
Option<SloppyImportsResolverRc<TSloppyImportResolverFs>>,
workspace_resolver: WorkspaceResolverRc,
is_byonm: bool,
maybe_vendor_specifier: Option<Url>,
}
@ -355,16 +358,16 @@ impl<
})
.map_err(|err| {
match err.into_kind() {
ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder(
err,
) => err.into(),
ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq(
err,
) => err.into(),
ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => {
err.into()
ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder(
err,
) => err.into(),
ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq(
err,
) => err.into(),
ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => {
err.into()
}
}
}
});
}
}

73
resolvers/deno/npm/byonm.rs
View file

@ -3,10 +3,10 @@
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_package_json::PackageJson;
use deno_package_json::PackageJsonDepValue;
use deno_package_json::PackageJsonRc;
use deno_path_util::url_to_file_path;
use deno_semver::package::PackageReq;
use deno_semver::Version;
@ -49,6 +49,10 @@ pub struct ByonmNpmResolverCreateOptions<
pub pkg_json_resolver: PackageJsonResolverRc<TEnv>,
}
#[allow(clippy::disallowed_types)]
pub type ByonmNpmResolverRc<Fs, TEnv> =
crate::sync::MaybeArc<ByonmNpmResolver<Fs, TEnv>>;
#[derive(Debug)]
pub struct ByonmNpmResolver<Fs: DenoResolverFs, TEnv: NodeResolverEnv> {
fs: Fs,
@ -84,7 +88,7 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
fn load_pkg_json(
&self,
path: &Path,
) -> Result<Option<Arc<PackageJson>>, PackageJsonLoadError> {
) -> Result<Option<PackageJsonRc>, PackageJsonLoadError> {
self.pkg_json_resolver.load_package_json(path)
}
@ -93,7 +97,7 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
&self,
dep_name: &str,
referrer: &Url,
) -> Option<Arc<PackageJson>> {
) -> Option<PackageJsonRc> {
let referrer_path = url_to_file_path(referrer).ok()?;
let mut current_folder = referrer_path.parent()?;
loop {
@ -173,7 +177,7 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
&self,
req: &PackageReq,
referrer: &Url,
) -> Result<Option<(Arc<PackageJson>, String)>, PackageJsonLoadError> {
) -> Result<Option<(PackageJsonRc, String)>, PackageJsonLoadError> {
fn resolve_alias_from_pkg_json(
req: &PackageReq,
pkg_json: &PackageJson,
@ -205,9 +209,9 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
}
// attempt to resolve the npm specifier from the referrer's package.json,
if let Ok(file_path) = url_to_file_path(referrer) {
let mut current_path = file_path.as_path();
while let Some(dir_path) = current_path.parent() {
let maybe_referrer_path = url_to_file_path(referrer).ok();
if let Some(file_path) = maybe_referrer_path {
for dir_path in file_path.as_path().ancestors().skip(1) {
let package_json_path = dir_path.join("package.json");
if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? {
if let Some(alias) =
@ -216,11 +220,10 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
return Ok(Some((pkg_json, alias)));
}
}
current_path = dir_path;
}
}
// otherwise, fall fallback to the project's package.json
// fall fallback to the project's package.json
if let Some(root_node_modules_dir) = &self.root_node_modules_dir {
let root_pkg_json_path =
root_node_modules_dir.parent().unwrap().join("package.json");
@ -232,6 +235,58 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
}
}
// now try to resolve based on the closest node_modules directory
let maybe_referrer_path = url_to_file_path(referrer).ok();
let search_node_modules = |node_modules: &Path| {
if req.version_req.tag().is_some() {
return None;
}
let pkg_folder = node_modules.join(&req.name);
if let Ok(Some(dep_pkg_json)) =
self.load_pkg_json(&pkg_folder.join("package.json"))
{
if dep_pkg_json.name.as_ref() == Some(&req.name) {
let matches_req = dep_pkg_json
.version
.as_ref()
.and_then(|v| Version::parse_from_npm(v).ok())
.map(|version| req.version_req.matches(&version))
.unwrap_or(true);
if matches_req {
return Some((dep_pkg_json, req.name.clone()));
}
}
}
None
};
if let Some(file_path) = &maybe_referrer_path {
for dir_path in file_path.as_path().ancestors().skip(1) {
if let Some(result) =
search_node_modules(&dir_path.join("node_modules"))
{
return Ok(Some(result));
}
}
}
// and finally check the root node_modules directory
if let Some(root_node_modules_dir) = &self.root_node_modules_dir {
let already_searched = maybe_referrer_path
.as_ref()
.and_then(|referrer_path| {
root_node_modules_dir
.parent()
.map(|root_dir| referrer_path.starts_with(root_dir))
})
.unwrap_or(false);
if !already_searched {
if let Some(result) = search_node_modules(root_node_modules_dir) {
return Ok(Some(result));
}
}
}
Ok(None)
}

29
resolvers/deno/npm/mod.rs
View file

@ -2,7 +2,6 @@
use std::fmt::Debug;
use std::path::PathBuf;
use std::sync::Arc;
use boxed_error::Boxed;
use deno_semver::npm::NpmPackageReqReference;
@ -15,10 +14,10 @@ use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::PackageResolveErrorKind;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::InNpmPackageChecker;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::NodeResolution;
use node_resolver::NodeResolutionKind;
use node_resolver::NodeResolver;
use node_resolver::NodeResolverRc;
use node_resolver::ResolutionMode;
use thiserror::Error;
use url::Url;
@ -28,6 +27,7 @@ use crate::fs::DenoResolverFs;
pub use byonm::ByonmInNpmPackageChecker;
pub use byonm::ByonmNpmResolver;
pub use byonm::ByonmNpmResolverCreateOptions;
pub use byonm::ByonmNpmResolverRc;
pub use byonm::ByonmResolvePkgFolderFromDenoReqError;
pub use local::normalize_pkg_name_for_node_modules_deno_folder;
@ -81,6 +81,9 @@ pub enum ResolvePkgFolderFromDenoReqError {
Byonm(#[from] ByonmResolvePkgFolderFromDenoReqError),
}
#[allow(clippy::disallowed_types)]
pub type CliNpmReqResolverRc = crate::sync::MaybeArc<dyn CliNpmReqResolver>;
// todo(dsherret): a temporary trait until we extract
// out the CLI npm resolver into here
pub trait CliNpmReqResolver: Debug + Send + Sync {
@ -98,21 +101,25 @@ pub struct NpmReqResolverOptions<
/// The resolver when "bring your own node_modules" is enabled where Deno
/// does not setup the node_modules directories automatically, but instead
/// uses what already exists on the file system.
pub byonm_resolver: Option<Arc<ByonmNpmResolver<Fs, TNodeResolverEnv>>>,
pub byonm_resolver: Option<ByonmNpmResolverRc<Fs, TNodeResolverEnv>>,
pub fs: Fs,
pub in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
pub node_resolver: Arc<NodeResolver<TNodeResolverEnv>>,
pub npm_req_resolver: Arc<dyn CliNpmReqResolver>,
pub in_npm_pkg_checker: InNpmPackageCheckerRc,
pub node_resolver: NodeResolverRc<TNodeResolverEnv>,
pub npm_req_resolver: CliNpmReqResolverRc,
}
#[allow(clippy::disallowed_types)]
pub type NpmReqResolverRc<Fs, TNodeResolverEnv> =
crate::sync::MaybeArc<NpmReqResolver<Fs, TNodeResolverEnv>>;
#[derive(Debug)]
pub struct NpmReqResolver<Fs: DenoResolverFs, TNodeResolverEnv: NodeResolverEnv>
{
byonm_resolver: Option<Arc<ByonmNpmResolver<Fs, TNodeResolverEnv>>>,
byonm_resolver: Option<ByonmNpmResolverRc<Fs, TNodeResolverEnv>>,
fs: Fs,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
node_resolver: Arc<NodeResolver<TNodeResolverEnv>>,
npm_resolver: Arc<dyn CliNpmReqResolver>,
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_resolver: NodeResolverRc<TNodeResolverEnv>,
npm_resolver: CliNpmReqResolverRc,
}
impl<Fs: DenoResolverFs, TNodeResolverEnv: NodeResolverEnv>

4
resolvers/deno/sloppy_imports.rs
View file

@ -101,6 +101,10 @@ pub trait SloppyImportResolverFs {
}
}
#[allow(clippy::disallowed_types)]
pub type SloppyImportsResolverRc<TSloppyImportResolverFs> =
crate::sync::MaybeArc<SloppyImportsResolver<TSloppyImportResolverFs>>;
#[derive(Debug)]
pub struct SloppyImportsResolver<Fs: SloppyImportResolverFs> {
fs: Fs,

34
resolvers/deno/sync.rs Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
pub use inner::*;
#[cfg(feature = "sync")]
mod inner {
#![allow(clippy::disallowed_types)]
pub use std::sync::Arc as MaybeArc;
pub use dashmap::DashMap as MaybeDashMap;
}
#[cfg(not(feature = "sync"))]
mod inner {
use std::hash::RandomState;
pub use std::rc::Rc as MaybeArc;
// Wrapper struct that exposes a subset of `DashMap` API.
#[derive(Default)]
struct MaybeDashMap<K, V, S = RandomState>(RefCell<HashMap<K, V, S>>);
impl MaybeDashMap<K, V, S> {
pub fn get(&'a self, key: &K) -> Option<&'a V> {
let inner = self.0.borrow();
inner.get(key)
}
pub fn insert(&self, key: K, value: V) -> Option<V> {
let inner = self.0.borrow_mut();
inner.insert(key, value)
}
}
}

1
resolvers/node/lib.rs
View file

@ -26,6 +26,7 @@ pub use resolution::resolve_specifier_into_node_modules;
pub use resolution::NodeResolution;
pub use resolution::NodeResolutionKind;
pub use resolution::NodeResolver;
pub use resolution::NodeResolverRc;
pub use resolution::ResolutionMode;
pub use resolution::DEFAULT_CONDITIONS;
pub use resolution::REQUIRE_CONDITIONS;

24
runtime/errors.rs
View file

@ -712,7 +712,6 @@ fn get_fetch_error(error: &FetchError) -> &'static str {
FetchError::ClientSend(_) => "TypeError",
FetchError::RequestBuilderHook(_) => "TypeError",
FetchError::Io(e) => get_io_error_class(e),
FetchError::Hyper(e) => get_hyper_error_class(e),
}
}
@ -1083,6 +1082,7 @@ mod node {
pub use deno_node::ops::crypto::SignEd25519Error;
pub use deno_node::ops::crypto::VerifyEd25519Error;
pub use deno_node::ops::fs::FsError;
pub use deno_node::ops::http::ConnError;
pub use deno_node::ops::http2::Http2Error;
pub use deno_node::ops::idna::IdnaError;
pub use deno_node::ops::ipc::IpcError;
@ -1538,6 +1538,24 @@ mod node {
pub fn get_verify_ed25519_error(_: &VerifyEd25519Error) -> &'static str {
"TypeError"
}
pub fn get_conn_error(e: &ConnError) -> &'static str {
match e {
ConnError::Resource(e) => get_error_class_name(e).unwrap_or("Error"),
ConnError::Permission(e) => get_permission_check_error_class(e),
ConnError::InvalidUrl(_) => "TypeError",
ConnError::InvalidHeaderName(_) => "TypeError",
ConnError::InvalidHeaderValue(_) => "TypeError",
ConnError::Url(e) => get_url_parse_error_class(e),
ConnError::Method(_) => "TypeError",
ConnError::Io(e) => get_io_error_class(e),
ConnError::Hyper(e) => super::get_hyper_error_class(e),
ConnError::TlsStreamBusy => "Busy",
ConnError::TcpStreamBusy => "Busy",
ConnError::ReuniteTcp(_) => "Error",
ConnError::Canceled(_) => "Error",
}
}
}
fn get_os_error(error: &OsError) -> &'static str {
@ -1730,6 +1748,10 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> {
e.downcast_ref::<node::VerifyEd25519Error>()
.map(node::get_verify_ed25519_error)
})
.or_else(|| {
e.downcast_ref::<node::ConnError>()
.map(node::get_conn_error)
})
.or_else(|| e.downcast_ref::<NApiError>().map(get_napi_error_class))
.or_else(|| e.downcast_ref::<WebError>().map(get_web_error_class))
.or_else(|| {

11
runtime/worker_bootstrap.rs
View file

@ -119,8 +119,7 @@ pub struct BootstrapOptions {
// Used by `deno serve`
pub serve_port: Option<u16>,
pub serve_host: Option<String>,
// OpenTelemetry output options. If `None`, OpenTelemetry is disabled.
pub otel_config: Option<OtelConfig>,
pub otel_config: OtelConfig,
}
impl Default for BootstrapOptions {
@ -155,7 +154,7 @@ impl Default for BootstrapOptions {
mode: WorkerExecutionMode::None,
serve_port: Default::default(),
serve_host: Default::default(),
otel_config: None,
otel_config: Default::default(),
}
}
}
@ -225,11 +224,7 @@ impl BootstrapOptions {
self.serve_host.as_deref(),
serve_is_main,
serve_worker_count,
if let Some(otel_config) = self.otel_config.as_ref() {
Box::new([otel_config.console as u8, otel_config.deterministic as u8])
} else {
Box::new([])
},
self.otel_config.as_v8(),
);
bootstrap.serialize(ser).unwrap()

31
tests/integration/compile_tests.rs
View file

@ -846,21 +846,6 @@ testing[WILDCARD]this
.assert_matches_text("2\n");
}
#[test]
fn compile_npm_file_system() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/npm_fs/main.ts",
copy_temp_dir: Some("compile/npm_fs"),
compile_args: vec!["-A"],
run_args: vec![],
output_file: "compile/npm_fs/main.out",
node_modules_local: true,
input_name: Some("binary"),
expected_name: "binary",
exit_code: 0,
});
}
#[test]
fn compile_npm_bin_esm() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() {
});
}
#[test]
fn compile_npm_vfs_implicit_read_permissions() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/vfs_implicit_read_permission/main.ts",
copy_temp_dir: Some("compile/vfs_implicit_read_permission"),
compile_args: vec![],
run_args: vec![],
output_file: "compile/vfs_implicit_read_permission/main.out",
node_modules_local: false,
input_name: Some("binary"),
expected_name: "binary",
exit_code: 0,
});
}
#[test]
fn compile_npm_no_permissions() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() {
let symlink_target_dir = temp_dir.path().join("some_folder");
project_dir.join("node_modules").create_dir_all();
symlink_target_dir.create_dir_all();
symlink_target_dir.join("file.txt").write("5");
let symlink_target_file = temp_dir.path().join("target.txt");
symlink_target_file.write("5");
let symlink_dir = project_dir.join("node_modules").join("symlink_dir");

213
tests/integration/lsp_tests.rs
View file

@ -11544,8 +11544,7 @@ fn lsp_json_import_with_query_string() {
fn lsp_format_markdown() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
let markdown_file =
source_file(temp_dir.path().join("file.md"), "# Hello World");
let file = source_file(temp_dir.path().join("file.md"), "# Hello World");
let mut client = context.new_lsp_command().build();
client.initialize_default();
@ -11553,7 +11552,7 @@ fn lsp_format_markdown() {
"textDocument/formatting",
json!({
"textDocument": {
"uri": markdown_file.url()
"uri": file.url()
},
"options": {
"tabSize": 2,
@ -11587,14 +11586,13 @@ fn lsp_format_markdown() {
fn lsp_format_html() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
let html_file =
source_file(temp_dir.path().join("file.html"), " <html></html>");
let file = source_file(temp_dir.path().join("file.html"), " <html></html>");
let mut client = context.new_lsp_command().build();
client.initialize_default();
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": html_file.url() },
"textDocument": { "uri": file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
@ -11627,13 +11625,13 @@ fn lsp_format_html() {
fn lsp_format_css() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
let css_file = source_file(temp_dir.path().join("file.css"), " foo {}");
let file = source_file(temp_dir.path().join("file.css"), " foo {}");
let mut client = context.new_lsp_command().build();
client.initialize_default();
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": css_file.url() },
"textDocument": { "uri": file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
@ -11666,13 +11664,13 @@ fn lsp_format_css() {
fn lsp_format_yaml() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
let yaml_file = source_file(temp_dir.path().join("file.yaml"), " foo: 1");
let file = source_file(temp_dir.path().join("file.yaml"), " foo: 1");
let mut client = context.new_lsp_command().build();
client.initialize_default();
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": yaml_file.url() },
"textDocument": { "uri": file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
@ -11701,6 +11699,201 @@ fn lsp_format_yaml() {
client.shutdown();
}
#[test]
fn lsp_format_sql() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
temp_dir.write(
"deno.json",
json!({
"unstable": ["fmt-sql"],
})
.to_string(),
);
let file = source_file(
temp_dir.path().join("file.sql"),
" CREATE TABLE item (id int NOT NULL IDENTITY(1, 1))",
);
let mut client = context.new_lsp_command().build();
client.initialize_default();
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
{
"range": {
"start": { "line": 0, "character": 52 },
"end": { "line": 0, "character": 52 },
},
"newText": "\n",
},
]),
);
client.shutdown();
}
#[test]
fn lsp_format_component() {
let context = TestContextBuilder::new().use_temp_cwd().build();
let temp_dir = context.temp_dir();
temp_dir.write(
"deno.json",
json!({
"unstable": ["fmt-component"],
})
.to_string(),
);
let svelte_file = source_file(
temp_dir.path().join("file.svelte"),
" <script module>\n // foo\n</script>\n",
);
let vue_file = source_file(
temp_dir.path().join("file.vue"),
" <script setup>\n// foo\n</script>\n",
);
let astro_file = source_file(
temp_dir.path().join("file.astro"),
" ---\n// foo\n---\n<html></html>\n",
);
let vento_file = source_file(
temp_dir.path().join("file.vto"),
" {{ layout \"foo.vto\" }}\n <h1>Foo!</h1>\n{{ /layout }}\n",
);
let nunjucks_file = source_file(
temp_dir.path().join("file.njk"),
" {% block header %}\n Foo\n{% endblock %}\n",
);
let mut client = context.new_lsp_command().build();
client.initialize_default();
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": svelte_file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
]),
);
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": vue_file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
]),
);
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": astro_file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
]),
);
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": vento_file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
]),
);
let res = client.write_request(
"textDocument/formatting",
json!({
"textDocument": { "uri": nunjucks_file.url() },
"options": {
"tabSize": 2,
"insertSpaces": true,
},
}),
);
assert_eq!(
res,
json!([
{
"range": {
"start": { "line": 0, "character": 0 },
"end": { "line": 0, "character": 2 },
},
"newText": "",
},
]),
);
client.shutdown();
}
#[test]
fn lsp_format_with_config() {
let context = TestContextBuilder::new().use_temp_cwd().build();

8
tests/node_compat/config.jsonc
View file

@ -565,9 +565,7 @@
"test-handle-wrap-close-abort.js",
"test-http-abort-before-end.js",
"test-http-addrequest-localaddress.js",
"test-http-agent-false.js",
"test-http-agent-getname.js",
"test-http-agent-keepalive-delay.js",
"test-http-agent-maxtotalsockets.js",
"test-http-agent-no-protocol.js",
"test-http-agent-null.js",
@ -590,7 +588,6 @@
"test-http-client-race.js",
"test-http-client-read-in-error.js",
"test-http-client-reject-unexpected-agent.js",
"test-http-client-timeout-connect-listener.js",
"test-http-client-timeout-with-data.js",
"test-http-client-unescaped-path.js",
"test-http-client-upload-buf.js",
@ -604,7 +601,6 @@
"test-http-date-header.js",
"test-http-decoded-auth.js",
"test-http-default-encoding.js",
"test-http-dump-req-when-res-ends.js",
"test-http-end-throw-socket-handling.js",
"test-http-eof-on-connect.js",
"test-http-extra-response.js",
@ -622,7 +618,6 @@
"test-http-hex-write.js",
"test-http-highwatermark.js",
"test-http-host-headers.js",
"test-http-hostname-typechecking.js",
"test-http-incoming-message-destroy.js",
"test-http-invalid-path-chars.js",
"test-http-invalidheaderfield.js",
@ -1292,10 +1287,7 @@
"test-buffer-creation-regression.js",
"test-child-process-exit.js",
"test-http-server-keep-alive-timeout-slow-server.js",
"test-net-better-error-messages-port.js",
"test-net-connect-handle-econnrefused.js",
"test-net-connect-local-error.js",
"test-net-reconnect-error.js",
"test-net-response-size.js",
"test-net-server-bind.js",
"test-tls-lookup.js",

10
tests/node_compat/runner/TODO.md
View file

@ -1,7 +1,7 @@
<!-- deno-fmt-ignore-file -->
# Remaining Node Tests
1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining).
1155 tests out of 3681 have been ported from Node 20.11.1 (31.38% ported, 69.14% remaining).
NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead.
@ -792,6 +792,8 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
- [parallel/test-http-agent-destroyed-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-destroyed-socket.js)
- [parallel/test-http-agent-domain-reused-gc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-domain-reused-gc.js)
- [parallel/test-http-agent-error-on-idle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-error-on-idle.js)
- [parallel/test-http-agent-false.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-false.js)
- [parallel/test-http-agent-keepalive-delay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive-delay.js)
- [parallel/test-http-agent-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive.js)
- [parallel/test-http-agent-maxsockets-respected.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets-respected.js)
- [parallel/test-http-agent-maxsockets.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets.js)
@ -848,6 +850,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
- [parallel/test-http-client-set-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-set-timeout.js)
- [parallel/test-http-client-spurious-aborted.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-spurious-aborted.js)
- [parallel/test-http-client-timeout-agent.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-agent.js)
- [parallel/test-http-client-timeout-connect-listener.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-connect-listener.js)
- [parallel/test-http-client-timeout-event.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-event.js)
- [parallel/test-http-client-timeout-on-connect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-on-connect.js)
- [parallel/test-http-client-timeout-option-listeners.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-option-listeners.js)
@ -865,6 +868,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
- [parallel/test-http-destroyed-socket-write2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-destroyed-socket-write2.js)
- [parallel/test-http-dns-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dns-error.js)
- [parallel/test-http-double-content-length.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-double-content-length.js)
- [parallel/test-http-dump-req-when-res-ends.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dump-req-when-res-ends.js)
- [parallel/test-http-early-hints-invalid-argument.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints-invalid-argument.js)
- [parallel/test-http-early-hints.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints.js)
- [parallel/test-http-exceptions.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-exceptions.js)
@ -876,6 +880,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
- [parallel/test-http-header-badrequest.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-badrequest.js)
- [parallel/test-http-header-overflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-overflow.js)
- [parallel/test-http-host-header-ipv6-fail.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-host-header-ipv6-fail.js)
- [parallel/test-http-hostname-typechecking.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-hostname-typechecking.js)
- [parallel/test-http-incoming-matchKnownFields.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-matchKnownFields.js)
- [parallel/test-http-incoming-message-connection-setter.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-connection-setter.js)
- [parallel/test-http-incoming-message-options.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-options.js)
@ -2508,9 +2513,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
- [sequential/test-inspector-port-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-inspector-port-cluster.js)
- [sequential/test-module-loading.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-module-loading.js)
- [sequential/test-net-GH-5504.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-GH-5504.js)
- [sequential/test-net-better-error-messages-port.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-better-error-messages-port.js)
- [sequential/test-net-connect-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-econnrefused.js)
- [sequential/test-net-connect-handle-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-handle-econnrefused.js)
- [sequential/test-net-listen-shared-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-listen-shared-ports.js)
- [sequential/test-net-localport.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-localport.js)
- [sequential/test-net-reconnect-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-reconnect-error.js)
- [sequential/test-net-server-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-server-address.js)
- [sequential/test-next-tick-error-spin.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-next-tick-error-spin.js)
- [sequential/test-perf-hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-perf-hooks.js)

53
tests/node_compat/test/parallel/test-http-agent-false.js
View file

@ -1,53 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
'use strict';
const common = require('../common');
const http = require('http');
// Sending `agent: false` when `port: null` is also passed in (i.e. the result
// of a `url.parse()` call with the default port used, 80 or 443), should not
// result in an assertion error...
const opts = {
host: '127.0.0.1',
port: null,
path: '/',
method: 'GET',
agent: false
};
// We just want an "error" (no local HTTP server on port 80) or "response"
// to happen (user happens ot have HTTP server running on port 80).
// As long as the process doesn't crash from a C++ assertion then we're good.
const req = http.request(opts);
// Will be called by either the response event or error event, not both
const oneResponse = common.mustCall();
req.on('response', oneResponse);
req.on('error', oneResponse);
req.end();

43
tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js
View file

@ -1,43 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
'use strict';
const common = require('../common');
const assert = require('assert');
const http = require('http');
const { Agent } = require('_http_agent');
const agent = new Agent({
keepAlive: true,
keepAliveMsecs: 1000,
});
const server = http.createServer(common.mustCall((req, res) => {
res.end('ok');
}));
server.listen(0, common.mustCall(() => {
const createConnection = agent.createConnection;
agent.createConnection = (options, ...args) => {
assert.strictEqual(options.keepAlive, true);
assert.strictEqual(options.keepAliveInitialDelay, agent.keepAliveMsecs);
return createConnection.call(agent, options, ...args);
};
http.get({
host: 'localhost',
port: server.address().port,
agent: agent,
path: '/'
}, common.mustCall((res) => {
// for emit end event
res.on('data', () => {});
res.on('end', () => {
server.close();
});
}));
}));

49
tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js
View file

@ -1,49 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
'use strict';
const common = require('../common');
// This test ensures that `ClientRequest.prototype.setTimeout()` does
// not add a listener for the `'connect'` event to the socket if the
// socket is already connected.
const assert = require('assert');
const http = require('http');
// Maximum allowed value for timeouts.
const timeout = 2 ** 31 - 1;
const server = http.createServer((req, res) => {
res.end();
});
server.listen(0, common.mustCall(() => {
const agent = new http.Agent({ keepAlive: true, maxSockets: 1 });
const options = { port: server.address().port, agent: agent };
doRequest(options, common.mustCall(() => {
const req = doRequest(options, common.mustCall(() => {
agent.destroy();
server.close();
}));
req.on('socket', common.mustCall((socket) => {
assert.strictEqual(socket.listenerCount('connect'), 0);
}));
}));
}));
function doRequest(options, callback) {
const req = http.get(options, (res) => {
res.on('end', callback);
res.resume();
});
req.setTimeout(timeout);
return req;
}

73
tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js
View file

@ -1,73 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
'use strict';
const { mustCall } = require('../common');
const fs = require('fs');
const http = require('http');
const { strictEqual } = require('assert');
const server = http.createServer(mustCall(function(req, res) {
strictEqual(req.socket.listenerCount('data'), 1);
req.socket.once('data', mustCall(function() {
// Ensure that a chunk of data is received before calling `res.end()`.
res.end('hello world');
}));
// This checks if the request gets dumped
// resume will be triggered by res.end().
req.on('resume', mustCall(function() {
// There is no 'data' event handler anymore
// it gets automatically removed when dumping the request.
strictEqual(req.listenerCount('data'), 0);
req.on('data', mustCall());
}));
// We explicitly pause the stream
// so that the following on('data') does not cause
// a resume.
req.pause();
req.on('data', function() {});
// Start sending the response.
res.flushHeaders();
}));
server.listen(0, mustCall(function() {
const req = http.request({
method: 'POST',
port: server.address().port
});
// Send the http request without waiting
// for the body.
req.flushHeaders();
req.on('response', mustCall(function(res) {
// Pipe the body as soon as we get the headers of the
// response back.
fs.createReadStream(__filename).pipe(req);
res.resume();
// On some platforms the `'end'` event might not be emitted because the
// socket could be destroyed by the other peer while data is still being
// sent. In this case the 'aborted'` event is emitted instead of `'end'`.
// `'close'` is used here because it is always emitted and does not
// invalidate the test.
res.on('close', function() {
server.close();
});
}));
req.on('error', function() {
// An error can happen if there is some data still
// being sent, as the other side is calling .destroy()
// this is safe to ignore.
});
}));

49
tests/node_compat/test/parallel/test-http-hostname-typechecking.js
View file

@ -1,49 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
'use strict';
const common = require('../common');
const assert = require('assert');
const http = require('http');
// All of these values should cause http.request() to throw synchronously
// when passed as the value of either options.hostname or options.host
const vals = [{}, [], NaN, Infinity, -Infinity, true, false, 1, 0, new Date()];
vals.forEach((v) => {
const received = common.invalidArgTypeHelper(v);
assert.throws(
() => http.request({ hostname: v }),
{
code: 'ERR_INVALID_ARG_TYPE',
name: 'TypeError',
message: 'The "options.hostname" property must be of ' +
'type string or one of undefined or null.' +
received
}
);
assert.throws(
() => http.request({ host: v }),
{
code: 'ERR_INVALID_ARG_TYPE',
name: 'TypeError',
message: 'The "options.host" property must be of ' +
'type string or one of undefined or null.' +
received
}
);
});
// These values are OK and should not throw synchronously.
// Only testing for 'hostname' validation so ignore connection errors.
const dontCare = () => {};
['', undefined, null].forEach((v) => {
http.request({ hostname: v }).on('error', dontCare).end();
http.request({ host: v }).on('error', dontCare).end();
});

24
tests/node_compat/test/sequential/test-net-better-error-messages-port.js
View file

@ -1,24 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
'use strict';
const common = require('../common');
const net = require('net');
const assert = require('assert');
const c = net.createConnection(common.PORT);
c.on('connect', common.mustNotCall());
c.on('error', common.mustCall(function(error) {
// Family autoselection might be skipped if only a single address is returned by DNS.
const failedAttempt = Array.isArray(error.errors) ? error.errors[0] : error;
assert.strictEqual(failedAttempt.code, 'ECONNREFUSED');
assert.strictEqual(failedAttempt.port, common.PORT);
assert.match(failedAttempt.address, /^(127\.0\.0\.1|::1)$/);
}));

39
tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js
View file

@ -1,39 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
'use strict';
const common = require('../common');
const net = require('net');
const assert = require('assert');
const c = net.createConnection(common.PORT);
c.on('connect', common.mustNotCall());
c.on('error', common.mustCall((e) => {
assert.strictEqual(c.connecting, false);
assert.strictEqual(e.code, 'ECONNREFUSED');
}));

50
tests/node_compat/test/sequential/test-net-reconnect-error.js
View file

@ -1,50 +0,0 @@
// deno-fmt-ignore-file
// deno-lint-ignore-file
// Copyright Joyent and Node contributors. All rights reserved. MIT license.
// Taken from Node 20.11.1
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually.
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
'use strict';
const common = require('../common');
const net = require('net');
const assert = require('assert');
const N = 20;
let disconnectCount = 0;
const c = net.createConnection(common.PORT);
c.on('connect', common.mustNotCall('client should not have connected'));
c.on('error', common.mustCall((error) => {
// Family autoselection might be skipped if only a single address is returned by DNS.
const actualError = Array.isArray(error.errors) ? error.errors[0] : error;
assert.strictEqual(actualError.code, 'ECONNREFUSED');
}, N + 1));
c.on('close', common.mustCall(() => {
if (disconnectCount++ < N)
c.connect(common.PORT); // reconnect
}, N + 1));

1
tests/specs/cli/otel_basic/main.ts
View file

@ -13,6 +13,7 @@ const server = Deno.serve(
const command = new Deno.Command(Deno.execPath(), {
args: ["run", "-A", "-q", "--unstable-otel", Deno.args[0]],
env: {
OTEL_DENO: "true",
DENO_UNSTABLE_OTEL_DETERMINISTIC: "1",
OTEL_EXPORTER_OTLP_PROTOCOL: "http/json",
OTEL_EXPORTER_OTLP_ENDPOINT: `http://localhost:${port}`,

22
tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc Normal file
View file

@ -0,0 +1,22 @@
{
"tempDir": true,
"steps": [{
"if": "unix",
"args": "compile --output main main.ts",
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "main.out"
}, {
"if": "windows",
"args": "compile --output main.exe main.ts",
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "main.out"
}]
}

47
tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out Normal file
View file

@ -0,0 +1,47 @@
[WILDCARD]
Compile file:///[WILDLINE]/main.ts to [WILDLINE]
Embedded File System
main[WILDLINE]
├─┬ .deno_compile_node_modules
│ └─┬ localhost
│ ├─┬ ansi-regex
│ │ ├── 3.0.1/*
│ │ └── 5.0.1/*
│ ├── ansi-styles/4.3.0/*
│ ├── camelcase/5.3.1/*
│ ├── cliui/6.0.0/*
│ ├── color-convert/2.0.1/*
│ ├── color-name/1.1.4/*
│ ├── cowsay/1.5.0/*
│ ├── decamelize/1.2.0/*
│ ├── emoji-regex/8.0.0/*
│ ├── find-up/4.1.0/*
│ ├── get-caller-file/2.0.5/*
│ ├── get-stdin/8.0.0/*
│ ├─┬ is-fullwidth-code-point
│ │ ├── 2.0.0/*
│ │ └── 3.0.0/*
│ ├── locate-path/5.0.0/*
│ ├── p-limit/2.3.0/*
│ ├── p-locate/4.1.0/*
│ ├── p-try/2.2.0/*
│ ├── path-exists/4.0.0/*
│ ├── require-directory/2.1.1/*
│ ├── require-main-filename/2.0.0/*
│ ├── set-blocking/2.0.0/*
│ ├─┬ string-width
│ │ ├── 2.1.1/*
│ │ └── 4.2.3/*
│ ├─┬ strip-ansi
│ │ ├── 4.0.0/*
│ │ └── 6.0.1/*
│ ├── strip-final-newline/2.0.0/*
│ ├── which-module/2.0.0/*
│ ├── wrap-ansi/6.2.0/*
│ ├── y18n/4.0.3/*
│ ├── yargs/15.4.1/*
│ └── yargs-parser/18.1.3/*
└── main.ts

0
tests/testdata/compile/vfs_implicit_read_permission/main.out → tests/specs/compile/global_npm_cache_implicit_read_permission/main.out
View file

0
tests/testdata/compile/vfs_implicit_read_permission/main.ts → tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts
View file

3
tests/specs/compile/include/data_files/non_existent.out
View file

@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]'
Caused by:
0: Including [WILDLINE]does_not_exist.txt
1: [WILDLINE]
1: Reading [WILDLINE]does_not_exist.txt
2: [WILDLINE]

25
tests/specs/compile/include/folder_ts_file/__test__.jsonc Normal file
View file

@ -0,0 +1,25 @@
{
"tempDir": true,
"steps": [{
"if": "unix",
// notice how the math folder is not included
"args": "compile --allow-read=data --include src --output main main.js",
"output": "[WILDCARD]"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "output.out",
"exitCode": 0
}, {
"if": "windows",
"args": "compile --allow-read=data --include src --output main.exe main.js",
"output": "[WILDCARD]"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "output.out",
"exitCode": 0
}]
}

14
tests/specs/compile/include/folder_ts_file/main.js Normal file
View file

@ -0,0 +1,14 @@
const mathDir = import.meta.dirname + "/math";
const files = Array.from(
Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name),
);
files.sort();
for (const file of files) {
console.log(file);
}
function nonAnalyzable() {
return "./src/main.ts";
}
await import(nonAnalyzable());

3
tests/specs/compile/include/folder_ts_file/math/add.ts Normal file
View file

@ -0,0 +1,3 @@
export function add(a: number, b: number) {
return a + b;
}

2
tests/specs/compile/include/folder_ts_file/output.out Normal file
View file

@ -0,0 +1,2 @@
[WILDLINE]add.ts
3

2
tests/specs/compile/include/folder_ts_file/src/main.ts Normal file
View file

@ -0,0 +1,2 @@
import { add } from "../math/add.ts";
console.log(add(1, 2));

4
tests/specs/compile/include/symlink_twice/__test__.jsonc
View file

@ -6,7 +6,7 @@
}, {
"if": "unix",
"args": "compile --allow-read=data --include . --output main link.js",
"output": "[WILDCARD]"
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
@ -16,7 +16,7 @@
}, {
"if": "windows",
"args": "compile --allow-read=data --include . --output main.exe link.js",
"output": "[WILDCARD]"
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",

9
tests/specs/compile/include/symlink_twice/compile.out Normal file
View file

@ -0,0 +1,9 @@
Compile [WILDLINE]
Embedded File System
main[WILDLINE]
├── index.js
├── link.js --> index.js
└── setup.js

1
tests/specs/compile/include/symlink_twice/setup.js
View file

@ -1,3 +1,2 @@
Deno.mkdirSync("data");
Deno.writeTextFileSync("index.js", "console.log(1);");
Deno.symlinkSync("index.js", "link.js");

24
tests/specs/compile/npm_fs/__test__.jsonc Normal file
View file

@ -0,0 +1,24 @@
{
"tempDir": true,
// use this so the vfs output is all in the same folder
"canonicalizedTempDir": true,
"steps": [{
"if": "unix",
"args": "compile -A --output main main.ts",
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "main.out"
}, {
"if": "windows",
"args": "compile -A --output main.exe main.ts",
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "main.out"
}]
}

8
tests/specs/compile/npm_fs/compile.out Normal file
View file

@ -0,0 +1,8 @@
[WILDCARD]
Embedded File System
main[WILDLINE]
├── main.ts
└── node_modules/*

3
tests/specs/compile/npm_fs/deno.json Normal file
View file

@ -0,0 +1,3 @@
{
"nodeModulesDir": "auto"
}

0
tests/testdata/compile/npm_fs/main.out → tests/specs/compile/npm_fs/main.out
View file

0
tests/testdata/compile/npm_fs/main.ts → tests/specs/compile/npm_fs/main.ts
View file

10
tests/specs/lockfile/external_import_map/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "run -A main.ts",
"output": "[WILDCARD]"
}, {
"args": ["eval", "console.log(Deno.readTextFileSync('deno.lock').trim())"],
"output": "deno.lock.out"
}]
}

3
tests/specs/lockfile/external_import_map/deno.json Normal file
View file

@ -0,0 +1,3 @@
{
"importMap": "import_map.json"
}

17
tests/specs/lockfile/external_import_map/deno.lock.out Normal file
View file

@ -0,0 +1,17 @@
{
"version": "4",
"specifiers": {
"jsr:@denotest/add@1.0.0": "1.0.0"
},
"jsr": {
"@denotest/add@1.0.0": {
"integrity": "[WILDLINE]"
}
},
"workspace": {
"dependencies": [
"jsr:@denotest/add@1.0.0",
"npm:@denotest/esm-basic@1.0.0"
]
}
}

10
tests/specs/lockfile/external_import_map/import_map.json Normal file
View file

@ -0,0 +1,10 @@
{
"imports": {
"@denotest/add": "jsr:@denotest/add@1.0.0"
},
"scopes": {
"/foo/": {
"@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0"
}
}
}

2
tests/specs/lockfile/external_import_map/main.ts Normal file
View file

@ -0,0 +1,2 @@
import { add } from "@denotest/add";
console.log(add(1, 2));

16
tests/specs/mod.rs
View file

@ -118,6 +118,12 @@ struct MultiStepMetaData {
/// steps.
#[serde(default)]
pub temp_dir: bool,
/// Whether the temporary directory should be canonicalized.
///
/// This should be used sparingly, but is sometimes necessary
/// on the CI.
#[serde(default)]
pub canonicalized_temp_dir: bool,
/// Whether the temporary directory should be symlinked to another path.
#[serde(default)]
pub symlinked_temp_dir: bool,
@ -144,6 +150,8 @@ struct SingleTestMetaData {
#[serde(default)]
pub temp_dir: bool,
#[serde(default)]
pub canonicalized_temp_dir: bool,
#[serde(default)]
pub symlinked_temp_dir: bool,
#[serde(default)]
pub repeat: Option<usize>,
@ -159,6 +167,7 @@ impl SingleTestMetaData {
base: self.base,
cwd: None,
temp_dir: self.temp_dir,
canonicalized_temp_dir: self.canonicalized_temp_dir,
symlinked_temp_dir: self.symlinked_temp_dir,
repeat: self.repeat,
envs: Default::default(),
@ -326,6 +335,13 @@ fn test_context_from_metadata(
builder = builder.cwd(cwd.to_string_lossy());
}
if metadata.canonicalized_temp_dir {
// not actually deprecated, we just want to discourage its use
#[allow(deprecated)]
{
builder = builder.use_canonicalized_temp_dir();
}
}
if metadata.symlinked_temp_dir {
// not actually deprecated, we just want to discourage its use
// because it's mostly used for testing purposes locally

18
tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc generated Normal file
View file

@ -0,0 +1,18 @@
{
"tests": {
"matches": {
"args": "run -A matches.ts",
"output": "5\n"
},
"not_matches": {
"args": "run -A not_matches.ts",
"output": "not_matches.out",
"exitCode": 1
},
"not_matches_aliased": {
"args": "run -A not_matches_aliased.ts",
"output": "not_matches_aliased.out",
"exitCode": 1
}
}
}

3
tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts generated Normal file
View file

@ -0,0 +1,3 @@
import { add } from "npm:package@1";
console.log(add(2, 3));

3
tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js generated vendored Normal file
View file

@ -0,0 +1,3 @@
module.exports.add = function(a, b) {
return a + b;
};

4
tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json generated vendored Normal file
View file

@ -0,0 +1,4 @@
{
"name": "not-same-name",
"version": "1.0.0"
}

3
tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js generated vendored Normal file
View file

@ -0,0 +1,3 @@
module.exports.add = function(a, b) {
return a + b;
};

4
tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json generated vendored Normal file
View file

@ -0,0 +1,4 @@
{
"name": "package",
"version": "1.0.0"
}

2
tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out generated Normal file
View file

@ -0,0 +1,2 @@
error: Could not find a matching package for 'npm:package@2' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file.
at file:///[WILDLINE]

3
tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts generated Normal file
View file

@ -0,0 +1,3 @@
import { add } from "npm:package@2"; // won't match 2
console.log(add(2, 3));

2
tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out generated Normal file
View file

@ -0,0 +1,2 @@
error: Could not find a matching package for 'npm:aliased@1' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file.
at file:///[WILDLINE]

3
tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts generated Normal file
View file

@ -0,0 +1,3 @@
import { add } from "npm:aliased@1";
console.log(add(2, 3));

2
tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json generated Normal file
View file

@ -0,0 +1,2 @@
{
}

15
tests/specs/schema.json
View file

@ -36,6 +36,9 @@
"flaky": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
@ -66,6 +69,12 @@
"tempDir": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
"base": {
"type": "string"
},
@ -94,6 +103,12 @@
"tempDir": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
"base": {
"type": "string"
},

24
tests/specs/update/external_import_map/__test__.jsonc Normal file
View file

@ -0,0 +1,24 @@
{
"tempDir": true,
"steps": [
{
"args": "i",
"output": "[WILDCARD]"
},
{
"args": "outdated",
"output": "outdated.out"
},
{
"args": "outdated --update --latest",
"output": "update.out"
},
{
"args": [
"eval",
"console.log(Deno.readTextFileSync('import_map.json').trim())"
],
"output": "import_map.json.out"
}
]
}

3
tests/specs/update/external_import_map/deno.json Normal file
View file

@ -0,0 +1,3 @@
{
"importMap": "import_map.json"
}

33
tests/specs/update/external_import_map/deno.lock generated Normal file
View file

@ -0,0 +1,33 @@
{
"version": "4",
"specifiers": {
"jsr:@denotest/add@0.2": "0.2.0",
"jsr:@denotest/subtract@0.2": "0.2.0",
"npm:@denotest/breaking-change-between-versions@1.0.0": "1.0.0",
"npm:@denotest/has-patch-versions@0.1": "0.1.0"
},
"jsr": {
"@denotest/add@0.2.0": {
"integrity": "a9076d30ecb42b2fc6dd95e7055fbf4e6358b53f550741bd7f60089d19f68848"
},
"@denotest/subtract@0.2.0": {
"integrity": "c9650fc559ab2430effc0c7fb1540e3aa89888fbdd926335ccfdeac57eb3a64d"
}
},
"npm": {
"@denotest/breaking-change-between-versions@1.0.0": {
"integrity": "sha512-bzMGYx+DxxPlI74n/VsDAN7Db1BY7Sz2XqxXruMo9dEznsBZu7Ez3i8YQ8n0leTxAiiMk1RCG4zQHPG1aj3xRw=="
},
"@denotest/has-patch-versions@0.1.0": {
"integrity": "sha512-H/MBo0jKDdMsX4AAGEGQbZj70nfNe3oUNZXbohYHhqf9EfpLnXp/7FC29ZdfV4+p6VjEcOGdCtXc6rilE6iYpg=="
}
},
"workspace": {
"dependencies": [
"jsr:@denotest/add@0.2",
"jsr:@denotest/subtract@0.2",
"npm:@denotest/breaking-change-between-versions@1.0.0",
"npm:@denotest/has-patch-versions@0.1"
]
}
}

Some files were not shown because too many files have changed in this diff Show more