From 56035f34a23891869d244732a28b95c7675bf7a8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 9 Dec 2024 11:01:57 -0500 Subject: [PATCH 01/33] fix(task): do not always kill child on ctrl+c on windows (#27269) We don't need to forward the kill signal because ctrl+c events are sent to the process group. Closes https://github.com/denoland/deno/issues/27266 --- cli/task_runner.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/task_runner.rs b/cli/task_runner.rs index aabdaf5777..d6589a1832 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -585,7 +585,13 @@ pub async fn run_future_forwarding_signals( async fn listen_ctrl_c(kill_signal: KillSignal) { while let Ok(()) = tokio::signal::ctrl_c().await { - kill_signal.send(deno_task_shell::SignalKind::SIGINT) + // On windows, ctrl+c is sent to the process group, so the signal would + // have already been sent to the child process. We still want to listen + // for ctrl+c here to keep the process alive when receiving it, but no + // need to forward the signal because it's already been sent. + if !cfg!(windows) { + kill_signal.send(deno_task_shell::SignalKind::SIGINT) + } } } From d7dfd4b540f376159e5e3a0f98a5d76a0557a05e Mon Sep 17 00:00:00 2001 From: Benjamin Swerdlow Date: Mon, 9 Dec 2024 10:52:13 -0800 Subject: [PATCH 02/33] refactor: Make `deno_runtime::shared` module public (#27242) Signed-off-by: Benjamin Swerdlow --- runtime/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/lib.rs b/runtime/lib.rs index 1ce325964f..1f449dc69a 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -46,7 +46,7 @@ pub use worker_bootstrap::BootstrapOptions; pub use worker_bootstrap::WorkerExecutionMode; pub use worker_bootstrap::WorkerLogLevel; -mod shared; +pub mod shared; pub use shared::runtime; pub struct UnstableGranularFlag { From 883abfa1bfeb4eded1ebdb8cc7364c254dcaad2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 9 Dec 2024 19:04:47 +0000 Subject: [PATCH 03/33] fix(ext/node): handle Float16Array in node:v8 module (#27285) Closes https://github.com/denoland/deno/issues/26580 --- ext/node/polyfills/v8.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/node/polyfills/v8.ts b/ext/node/polyfills/v8.ts index 9df199865e..e24a79ab5a 100644 --- a/ext/node/polyfills/v8.ts +++ b/ext/node/polyfills/v8.ts @@ -227,6 +227,7 @@ function arrayBufferViewTypeToIndex(abView: ArrayBufferView) { // Index 10 is FastBuffer. if (type === "[object BigInt64Array]") return 11; if (type === "[object BigUint64Array]") return 12; + if (type === "[object Float16Array]") return 13; return -1; } export class DefaultSerializer extends Serializer { @@ -276,6 +277,7 @@ function arrayBufferViewIndexToType(index: number): any { if (index === 10) return Buffer; if (index === 11) return BigInt64Array; if (index === 12) return BigUint64Array; + if (index === 13) return Float16Array; return undefined; } From 44d76975d555faed794bf874a235290c8e362357 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:01:09 -0800 Subject: [PATCH 04/33] fix(node): update list of builtin node modules, add missing export to _http_common (#27294) Fixes https://github.com/denoland/deno/issues/27289 We exported these but forgot to add them to the list of builtins used by the resolver, so we weren't resolving bare imports of some modules (e.g. `"_http_common"`) Also adds a missing export of `HTTPParser` from `_http_common` --- cli/lsp/completions.rs | 9 ++++++--- ext/node/polyfill.rs | 11 +++++++++++ ext/node/polyfills/_http_common.ts | 4 ++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index 95e5113620..31f0b066ed 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -743,13 +743,16 @@ fn get_node_completions( } let items = SUPPORTED_BUILTIN_NODE_MODULES .iter() - .map(|name| { + .filter_map(|name| { + if name.starts_with('_') { + return None; + } let specifier = format!("node:{}", name); let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit { range: *range, new_text: specifier.clone(), })); - lsp::CompletionItem { + Some(lsp::CompletionItem { label: specifier, kind: Some(lsp::CompletionItemKind::FILE), detail: Some("(node)".to_string()), @@ -758,7 +761,7 @@ fn get_node_completions( IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), ), ..Default::default() - } + }) }) .collect(); Some(CompletionList { diff --git a/ext/node/polyfill.rs b/ext/node/polyfill.rs index a14b75bac0..762e32f7ba 100644 --- a/ext/node/polyfill.rs +++ b/ext/node/polyfill.rs @@ -25,6 +25,17 @@ macro_rules! generate_builtin_node_module_lists { // NOTE(bartlomieju): keep this list in sync with `ext/node/polyfills/01_require.js` generate_builtin_node_module_lists! { + "_http_agent", + "_http_common", + "_http_outgoing", + "_http_server", + "_stream_duplex", + "_stream_passthrough", + "_stream_readable", + "_stream_transform", + "_stream_writable", + "_tls_common", + "_tls_wrap", "assert", "assert/strict", "async_hooks", diff --git a/ext/node/polyfills/_http_common.ts b/ext/node/polyfills/_http_common.ts index 8fb7758a55..86143b4dea 100644 --- a/ext/node/polyfills/_http_common.ts +++ b/ext/node/polyfills/_http_common.ts @@ -7,6 +7,7 @@ const { SafeRegExp, Symbol, } = primordials; +import { HTTPParser } from "ext:deno_node/internal_binding/http_parser.ts"; export const CRLF = "\r\n"; export const kIncomingMessage = Symbol("IncomingMessage"); @@ -79,6 +80,8 @@ export { checkIsHttpToken as _checkIsHttpToken, }; +export { HTTPParser }; + export default { _checkInvalidHeaderChar: checkInvalidHeaderChar, _checkIsHttpToken: checkIsHttpToken, @@ -87,4 +90,5 @@ export default { continueExpression, kIncomingMessage, methods, + HTTPParser, }; From da3a676d1ca6031db22c46e9504e7f967791607d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 9 Dec 2024 16:19:08 -0500 Subject: [PATCH 05/33] fix: do not error when subpath has an @ symbol (#27290) Closes https://github.com/denoland/deno/issues/27243 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- cli/tools/registry/pm.rs | 10 ++++------ .../add/add_with_subpath/wrong_constraint_jsr.out | 4 ++-- .../add/add_with_subpath/wrong_constraint_npm.out | 4 ++-- .../error_version_after_subpath/main.out | 2 +- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c42621100..30824206b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,9 +2192,9 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756be7351289726087408984db18b9eb5e0186907673f39f858d119d0162071" +checksum = "7d1259270d66a5e6d29bb75c9289656541874f79ae9ff6c9f1c790846d5c07ba" dependencies = [ "deno_error", "monch", @@ -4569,7 +4569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4698acd06d..f8b01164ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ deno_npm = "=0.26.0" deno_path_util = "=0.2.1" deno_permissions = { version = "0.41.0", path = "./runtime/permissions" } deno_runtime = { version = "0.190.0", path = "./runtime" } -deno_semver = "=0.6.0" +deno_semver = "=0.6.1" deno_terminal = "0.2.0" napi_sym = { version = "0.111.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index 5718cd3ec1..6f89ec7aae 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -432,9 +432,8 @@ pub async fn add( let mut package_reqs = Vec::with_capacity(add_flags.packages.len()); for entry_text in add_flags.packages.iter() { - let req = AddRmPackageReq::parse(entry_text).with_context(|| { - format!("Failed to parse package required: {}", entry_text) - })?; + let req = AddRmPackageReq::parse(entry_text) + .with_context(|| format!("Failed to parse package: {}", entry_text))?; match req { Ok(add_req) => package_reqs.push(add_req), @@ -805,9 +804,8 @@ pub async fn remove( let mut removed_packages = vec![]; for package in &remove_flags.packages { - let req = AddRmPackageReq::parse(package).with_context(|| { - format!("Failed to parse package required: {}", package) - })?; + let req = AddRmPackageReq::parse(package) + .with_context(|| format!("Failed to parse package: {}", package))?; let mut parsed_pkg_name = None; for config in configs.iter_mut().flatten() { match &req { diff --git a/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out b/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out index 7de7008e93..5f6661920a 100644 --- a/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out +++ b/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out @@ -1,4 +1,4 @@ -error: Failed to parse package required: jsr:@std/testing/bdd@1 +error: Failed to parse package: jsr:@std/testing/bdd@1 Caused by: - Invalid package specifier 'jsr:@std/testing/bdd@1'. Did you mean to write 'jsr:@std/testing@1/bdd'? + Invalid package specifier 'jsr:@std/testing/bdd@1'. Did you mean to write 'jsr:@std/testing@1/bdd'? If not, add a version requirement to the specifier. diff --git a/tests/specs/add/add_with_subpath/wrong_constraint_npm.out b/tests/specs/add/add_with_subpath/wrong_constraint_npm.out index 4adcf9ef6a..2298810bec 100644 --- a/tests/specs/add/add_with_subpath/wrong_constraint_npm.out +++ b/tests/specs/add/add_with_subpath/wrong_constraint_npm.out @@ -1,4 +1,4 @@ -error: Failed to parse package required: npm:preact/hooks@10 +error: Failed to parse package: npm:preact/hooks@10 Caused by: - Invalid package specifier 'npm:preact/hooks@10'. Did you mean to write 'npm:preact@10/hooks'? + Invalid package specifier 'npm:preact/hooks@10'. Did you mean to write 'npm:preact@10/hooks'? If not, add a version requirement to the specifier. diff --git a/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out b/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out index 4e5f03e23c..1804936f0f 100644 --- a/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out +++ b/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out @@ -1,2 +1,2 @@ -error: Invalid package specifier 'npm:react-dom/server@18.2.0'. Did you mean to write 'npm:react-dom@18.2.0/server'? +error: Invalid package specifier 'npm:react-dom/server@18.2.0'. Did you mean to write 'npm:react-dom@18.2.0/server'? If not, add a version requirement to the specifier. at [WILDCARD]/error_version_after_subpath/main.js:1:8 From 1c0f236923d72bdda109c9d111d017ebd42188f7 Mon Sep 17 00:00:00 2001 From: snek Date: Mon, 9 Dec 2024 22:33:07 +0100 Subject: [PATCH 06/33] fix(unstable): don't unwrap optional state in otel (#27292) otel global state may not be initialized if otel is not enabled, so bail out instead of panicking. Fixes: https://github.com/denoland/deno/issues/27272 --- ext/telemetry/lib.rs | 14 +++++++------- ext/telemetry/telemetry.ts | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/telemetry/lib.rs b/ext/telemetry/lib.rs index 9612401014..06210a70e7 100644 --- a/ext/telemetry/lib.rs +++ b/ext/telemetry/lib.rs @@ -732,9 +732,9 @@ fn op_otel_instrumentation_scope_enter( #[op2(fast)] fn op_otel_instrumentation_scope_enter_builtin(state: &mut OpState) { - state.put(InstrumentationScope( - BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap().clone(), - )); + if let Some(scope) = BUILT_IN_INSTRUMENTATION_SCOPE.get() { + state.put(InstrumentationScope(scope.clone())); + } } #[op2(fast)] @@ -749,6 +749,9 @@ fn op_otel_log( let Some(Processors { logs, .. }) = OTEL_PROCESSORS.get() else { return; }; + let Some(instrumentation_scope) = BUILT_IN_INSTRUMENTATION_SCOPE.get() else { + return; + }; // Convert the integer log level that ext/console uses to the corresponding // OpenTelemetry log severity. @@ -776,10 +779,7 @@ fn op_otel_log( ); } - logs.emit( - &mut log_record, - BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap(), - ); + logs.emit(&mut log_record, instrumentation_scope); } fn owned_string<'s>( diff --git a/ext/telemetry/telemetry.ts b/ext/telemetry/telemetry.ts index e9e38d1592..acdfd4d715 100644 --- a/ext/telemetry/telemetry.ts +++ b/ext/telemetry/telemetry.ts @@ -220,6 +220,7 @@ function submitSpan( startTime: number, endTime: number, ) { + if (!TRACING_ENABLED) return; if (!(traceFlags & TRACE_FLAG_SAMPLED)) return; // TODO(@lucacasonato): `resource` is ignored for now, should we implement it? From d99b2d6f7db1f88e353021ed55e9492a3997ed8e Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 9 Dec 2024 19:28:53 -0500 Subject: [PATCH 07/33] chore: reduce allocations in a few places (#27288) Probably doesn't have much impact. I didn't measure any of these, but reducing allocations should always be good. --- Cargo.lock | 14 ++++++-- Cargo.toml | 1 + cli/util/path.rs | 5 +-- cli/util/progress_bar/renderer.rs | 9 ++++-- resolvers/npm_cache/tarball_extract.rs | 5 +-- runtime/permissions/Cargo.toml | 1 + runtime/permissions/lib.rs | 45 ++++++++++++++++---------- 7 files changed, 54 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30824206b6..26936169e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -668,6 +668,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bf2a5fb3207c12b5d208ebc145f967fea5cac41a021c37417ccc31ba40f39ee" +[[package]] +name = "capacity_builder" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c0f637033edd76ceb881faaee372868a383f0ed7a4a59e8fdf90db2502f3d3" +dependencies = [ + "itoa", +] + [[package]] name = "caseless" version = "0.2.1" @@ -2088,6 +2097,7 @@ dependencies = [ name = "deno_permissions" version = "0.41.0" dependencies = [ + "capacity_builder", "deno_core", "deno_path_util", "deno_terminal 0.2.0", @@ -4375,9 +4385,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jni-sys" diff --git a/Cargo.toml b/Cargo.toml index f8b01164ac..c3703d8c5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ boxed_error = "0.2.2" brotli = "6.0.0" bytes = "1.4.0" cache_control = "=0.2.0" +capacity_builder = "0.1.0" cbc = { version = "=0.1.2", features = ["alloc"] } # Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS. # Instead use util::time::utc_now() diff --git a/cli/util/path.rs b/cli/util/path.rs index df66b83766..de72843406 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::fmt::Write; use std::path::Path; use std::path::PathBuf; @@ -58,8 +59,8 @@ pub fn get_atomic_file_path(file_path: &Path) -> PathBuf { } fn gen_rand_path_component() -> String { - (0..4).fold(String::new(), |mut output, _| { - output.push_str(&format!("{:02x}", rand::random::())); + (0..4).fold(String::with_capacity(8), |mut output, _| { + write!(&mut output, "{:02x}", rand::random::()).unwrap(); output }) } diff --git a/cli/util/progress_bar/renderer.rs b/cli/util/progress_bar/renderer.rs index 6b08dada12..db3d37140f 100644 --- a/cli/util/progress_bar/renderer.rs +++ b/cli/util/progress_bar/renderer.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::fmt::Write; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; @@ -81,12 +82,14 @@ impl ProgressBarRenderer for BarProgressBarRenderer { let elapsed_text = get_elapsed_text(data.duration); let mut text = String::new(); if !display_entry.message.is_empty() { - text.push_str(&format!( - "{} {}{}\n", + writeln!( + &mut text, + "{} {}{}", colors::green("Download"), display_entry.message, bytes_text, - )); + ) + .unwrap(); } text.push_str(&elapsed_text); let max_width = (data.terminal_width as i32 - 5).clamp(10, 75) as usize; diff --git a/resolvers/npm_cache/tarball_extract.rs b/resolvers/npm_cache/tarball_extract.rs index 262618d905..c4c614b35f 100644 --- a/resolvers/npm_cache/tarball_extract.rs +++ b/resolvers/npm_cache/tarball_extract.rs @@ -219,8 +219,9 @@ fn get_atomic_dir_path(file_path: &Path) -> PathBuf { } fn gen_rand_path_component() -> String { - (0..4).fold(String::new(), |mut output, _| { - output.push_str(&format!("{:02x}", rand::random::())); + use std::fmt::Write; + (0..4).fold(String::with_capacity(8), |mut output, _| { + write!(&mut output, "{:02x}", rand::random::()).unwrap(); output }) } diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index 0140f0594e..5be45377a0 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -14,6 +14,7 @@ name = "deno_permissions" path = "lib.rs" [dependencies] +capacity_builder.workspace = true deno_core.workspace = true deno_path_util.workspace = true deno_terminal.workspace = true diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index a0b901d200..a1a217738d 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use capacity_builder::StringBuilder; use deno_core::parking_lot::Mutex; use deno_core::serde::de; use deno_core::serde::Deserialize; @@ -179,13 +180,18 @@ impl PermissionState { (Ok(()), false, false) } PermissionState::Prompt if prompt => { - let msg = format!( - "{} access{}", - name, - info() - .map(|info| { format!(" to {info}") }) - .unwrap_or_default(), - ); + let msg = { + let info = info(); + StringBuilder::build(|builder| { + builder.append(name); + builder.append(" access"); + if let Some(info) = &info { + builder.append(" to "); + builder.append(info); + } + }) + .unwrap() + }; match permission_prompt(&msg, name, api_name, true) { PromptResponse::Allow => { Self::log_perm_access(name, info); @@ -344,11 +350,11 @@ pub trait QueryDescriptor: Debug { fn overlaps_deny(&self, other: &Self::DenyDesc) -> bool; } -fn format_display_name(display_name: Cow) -> String { +fn format_display_name(display_name: Cow) -> Cow { if display_name.starts_with('<') && display_name.ends_with('>') { - display_name.into_owned() + display_name } else { - format!("\"{}\"", display_name) + Cow::Owned(format!("\"{}\"", display_name)) } } @@ -424,7 +430,7 @@ impl UnaryPermission { .check2( TQuery::flag_name(), api_name, - || desc.map(|d| format_display_name(d.display_name())), + || desc.map(|d| format_display_name(d.display_name()).into_owned()), self.prompt, ); if prompted { @@ -487,12 +493,17 @@ impl UnaryPermission { if !self.prompt { return PermissionState::Denied; } - let mut message = String::with_capacity(40); - message.push_str(&format!("{} access", TQuery::flag_name())); - if let Some(desc) = desc { - message - .push_str(&format!(" to {}", format_display_name(desc.display_name()))); - } + let maybe_formatted_display_name = + desc.map(|d| format_display_name(d.display_name())); + let message = StringBuilder::build(|builder| { + builder.append(TQuery::flag_name()); + builder.append(" access"); + if let Some(display_name) = &maybe_formatted_display_name { + builder.append(" to "); + builder.append(display_name) + } + }) + .unwrap(); match permission_prompt( &message, TQuery::flag_name(), From 94c7653d0e3ce8dcdc08866b92ff698a65b4b0d7 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 9 Dec 2024 20:11:52 -0500 Subject: [PATCH 08/33] fix(compile): correct read length for transpiled typescript files (#27301) Extracted out of https://github.com/denoland/deno/pull/27296/files It's hard to test for this, but a test for this is in that other PR. --- cli/standalone/virtual_fs.rs | 75 +++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 66fc835534..d5c8efb235 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -266,17 +266,20 @@ impl VfsBuilder { let dir = self.add_dir(path.parent().unwrap())?; let name = path.file_name().unwrap().to_string_lossy(); - let data_len = data.len(); + let offset_and_len = OffsetWithLength { + offset, + len: data.len() as u64, + }; match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { Ok(index) => { let entry = &mut dir.entries[index]; match entry { VfsEntry::File(virtual_file) => match sub_data_kind { VfsFileSubDataKind::Raw => { - virtual_file.offset = offset; + virtual_file.offset = offset_and_len; } VfsFileSubDataKind::ModuleGraph => { - virtual_file.module_graph_offset = offset; + virtual_file.module_graph_offset = offset_and_len; } }, VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), @@ -287,9 +290,8 @@ impl VfsBuilder { insert_index, VfsEntry::File(VirtualFile { name: name.to_string(), - offset, - module_graph_offset: offset, - len: data.len() as u64, + offset: offset_and_len, + module_graph_offset: offset_and_len, }), ); } @@ -298,7 +300,7 @@ impl VfsBuilder { // new file, update the list of files if self.current_offset == offset { self.files.push(data); - self.current_offset += data_len as u64; + self.current_offset += offset_and_len.len; } Ok(()) @@ -403,7 +405,7 @@ impl<'a> VfsEntryRef<'a> { mtime: None, ctime: None, blksize: 0, - size: file.len, + size: file.offset.len, dev: 0, ino: 0, mode: 0, @@ -472,27 +474,41 @@ impl VfsEntry { #[derive(Debug, Serialize, Deserialize)] pub struct VirtualDirectory { + #[serde(rename = "n")] pub name: String, // should be sorted by name + #[serde(rename = "e")] pub entries: Vec, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct OffsetWithLength { + #[serde(rename = "o")] + pub offset: u64, + #[serde(rename = "l")] + pub len: u64, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VirtualFile { + #[serde(rename = "n")] pub name: String, - pub offset: u64, + #[serde(rename = "o")] + pub offset: OffsetWithLength, /// Offset file to use for module loading when it differs from the /// raw file. Often this will be the same offset as above for data /// such as JavaScript files, but for TypeScript files the `offset` /// will be the original raw bytes when included as an asset and this /// offset will be to the transpiled JavaScript source. - pub module_graph_offset: u64, - pub len: u64, + #[serde(rename = "m")] + pub module_graph_offset: OffsetWithLength, } #[derive(Debug, Serialize, Deserialize)] pub struct VirtualSymlink { + #[serde(rename = "n")] pub name: String, + #[serde(rename = "p")] pub dest_parts: Vec, } @@ -636,7 +652,7 @@ impl FileBackedVfsFile { Ok(pos) } SeekFrom::End(offset) => { - if offset < 0 && -offset as u64 > self.file.len { + if offset < 0 && -offset as u64 > self.file.offset.len { let msg = "An attempt was made to move the file pointer before the beginning of the file."; Err( std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg) @@ -645,9 +661,9 @@ impl FileBackedVfsFile { } else { let mut current_pos = self.pos.lock(); *current_pos = if offset >= 0 { - self.file.len - (offset as u64) + self.file.offset.len - (offset as u64) } else { - self.file.len + (-offset as u64) + self.file.offset.len + (-offset as u64) }; Ok(*current_pos) } @@ -671,7 +687,7 @@ impl FileBackedVfsFile { let mut pos = self.pos.lock(); let read_pos = *pos; // advance the position due to the read - *pos = std::cmp::min(self.file.len, *pos + buf.len() as u64); + *pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64); read_pos }; self @@ -685,13 +701,13 @@ impl FileBackedVfsFile { let mut pos = self.pos.lock(); let read_pos = *pos; // todo(dsherret): should this always set it to the end of the file? - if *pos < self.file.len { + if *pos < self.file.offset.len { // advance the position due to the read - *pos = self.file.len; + *pos = self.file.offset.len; } read_pos }; - if read_pos > self.file.len { + if read_pos > self.file.offset.len { return Ok(Cow::Borrowed(&[])); } if read_pos == 0 { @@ -701,7 +717,7 @@ impl FileBackedVfsFile { .read_file_all(&self.file, VfsFileSubDataKind::Raw)?, ) } else { - let size = (self.file.len - read_pos) as usize; + let size = (self.file.offset.len - read_pos) as usize; let mut buf = vec![0; size]; self.vfs.read_file(&self.file, read_pos, &mut buf)?; Ok(Cow::Owned(buf)) @@ -921,7 +937,11 @@ impl FileBackedVfs { file: &VirtualFile, sub_data_kind: VfsFileSubDataKind, ) -> std::io::Result> { - let read_range = self.get_read_range(file, sub_data_kind, 0, file.len)?; + let read_len = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset.len, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset.len, + }; + let read_range = self.get_read_range(file, sub_data_kind, 0, read_len)?; match &self.vfs_data { Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), @@ -952,19 +972,20 @@ impl FileBackedVfs { pos: u64, len: u64, ) -> std::io::Result> { - if pos > file.len { + let file_offset_and_len = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, + }; + if pos > file_offset_and_len.len { return Err(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, "unexpected EOF", )); } - let offset = match sub_data_kind { - VfsFileSubDataKind::Raw => file.offset, - VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, - }; - let file_offset = self.fs_root.start_file_offset + offset; + let file_offset = + self.fs_root.start_file_offset + file_offset_and_len.offset; let start = file_offset + pos; - let end = file_offset + std::cmp::min(pos + len, file.len); + let end = file_offset + std::cmp::min(pos + len, file_offset_and_len.len); Ok(start as usize..end as usize) } From c3af09821a196fe989c4b028680063febbc6ee73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 10 Dec 2024 13:41:16 +0000 Subject: [PATCH 09/33] fix(outdated): show a suggestion for updating (#27304) Show command to update dependencies based on the output of "deno outdated" command. Closes https://github.com/denoland/deno/issues/27256 --- cli/tools/registry/pm/outdated.rs | 19 +++++++++++++++++++ tests/specs/update/deno_json/outdated.out | 3 +++ .../update/deno_json/outdated_compatible.out | 3 +++ .../print_outdated/member_a.out | 3 +++ .../print_outdated/member_b.out | 3 +++ .../print_outdated/recursive.out | 3 +++ .../mixed_workspace/print_outdated/root.out | 3 +++ tests/specs/update/package_json/outdated.out | 3 +++ .../package_json/outdated_compatible.out | 3 +++ 9 files changed, 43 insertions(+) diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index 2a29014267..073d772f8a 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -100,6 +100,24 @@ fn print_outdated_table(packages: &[OutdatedPackage]) { println!("└{package_fill}┴{current_fill}┴{update_fill}┴{latest_fill}┘",); } +#[allow(clippy::print_stdout)] +fn print_suggestion(compatible: bool) { + println!(); + let (cmd, txt) = if compatible { + ("", "compatible") + } else { + (" --latest", "available") + }; + println!( + "{}", + color_print::cformat!( + "Run deno outdated --update{} to update to the latest {} versions,\nor deno outdated --help for more information.", + cmd, + txt, + ) + ); +} + fn print_outdated( deps: &mut DepManager, compatible: bool, @@ -148,6 +166,7 @@ fn print_outdated( if !outdated.is_empty() { outdated.sort(); print_outdated_table(&outdated); + print_suggestion(compatible); } Ok(()) diff --git a/tests/specs/update/deno_json/outdated.out b/tests/specs/update/deno_json/outdated.out index 07ff9f3416..dd86ed8e87 100644 --- a/tests/specs/update/deno_json/outdated.out +++ b/tests/specs/update/deno_json/outdated.out @@ -13,3 +13,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/deno_json/outdated_compatible.out b/tests/specs/update/deno_json/outdated_compatible.out index 54511a537f..8b67e0750e 100644 --- a/tests/specs/update/deno_json/outdated_compatible.out +++ b/tests/specs/update/deno_json/outdated_compatible.out @@ -5,3 +5,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update to update to the latest compatible versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/member_a.out b/tests/specs/update/mixed_workspace/print_outdated/member_a.out index 8699aac2bf..9abfd127c0 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/member_a.out +++ b/tests/specs/update/mixed_workspace/print_outdated/member_a.out @@ -7,3 +7,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/member_b.out b/tests/specs/update/mixed_workspace/print_outdated/member_b.out index fc8ef320a8..80548e4ac3 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/member_b.out +++ b/tests/specs/update/mixed_workspace/print_outdated/member_b.out @@ -5,3 +5,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/bin │ 0.6.0 │ 0.6.0 │ 1.0.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/recursive.out b/tests/specs/update/mixed_workspace/print_outdated/recursive.out index ca03776a9c..1c4f4d06e2 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/recursive.out +++ b/tests/specs/update/mixed_workspace/print_outdated/recursive.out @@ -13,3 +13,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/root.out b/tests/specs/update/mixed_workspace/print_outdated/root.out index c7934edc79..ef8df6dbe1 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/root.out +++ b/tests/specs/update/mixed_workspace/print_outdated/root.out @@ -3,3 +3,6 @@ ├────────────────────────┼─────────┼────────┼────────┤ │ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ └────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/package_json/outdated.out b/tests/specs/update/package_json/outdated.out index d672aace7f..cf61af8065 100644 --- a/tests/specs/update/package_json/outdated.out +++ b/tests/specs/update/package_json/outdated.out @@ -7,3 +7,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/package_json/outdated_compatible.out b/tests/specs/update/package_json/outdated_compatible.out index 8a682c461c..812e9ba0f4 100644 --- a/tests/specs/update/package_json/outdated_compatible.out +++ b/tests/specs/update/package_json/outdated_compatible.out @@ -3,3 +3,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update to update to the latest compatible versions, +or deno outdated --help for more information. From 59dd5d21d47ca6238497075754d6bb4e0fbf045c Mon Sep 17 00:00:00 2001 From: Mohammad Sulaiman Date: Tue, 10 Dec 2024 16:52:13 +0200 Subject: [PATCH 10/33] fix: replace the @deno-types with @ts-types (#27310) --- cli/lsp/analysis.rs | 4 ++-- cli/lsp/diagnostics.rs | 12 ++++++------ cli/lsp/tsc.rs | 8 +++----- tests/integration/lsp_tests.rs | 26 +++++++++++++------------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 853708221f..c128372dcd 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -1387,7 +1387,7 @@ impl CodeActionCollection { character: import_start.column_index as u32, }; let new_text = format!( - "{}// @deno-types=\"{}\"\n", + "{}// @ts-types=\"{}\"\n", if position.character == 0 { "" } else { "\n" }, &types_specifier_text ); @@ -1400,7 +1400,7 @@ impl CodeActionCollection { }; Some(lsp::CodeAction { title: format!( - "Add @deno-types directive for \"{}\"", + "Add @ts-types directive for \"{}\"", &types_specifier_text ), kind: Some(lsp::CodeActionKind::QUICKFIX), diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index ac4d8c01e4..01fc3bf69e 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -1355,7 +1355,7 @@ fn diagnose_resolution( } // don't bother warning about sloppy import redirects from .js to .d.ts // because explaining how to fix this via a diagnostic involves using - // @deno-types and that's a bit complicated to explain + // @ts-types and that's a bit complicated to explain let is_sloppy_import_dts_redirect = doc_specifier.scheme() == "file" && doc.media_type().is_declaration() && !MediaType::from_specifier(specifier).is_declaration(); @@ -1523,7 +1523,7 @@ fn diagnose_dependency( .iter() .map(|i| documents::to_lsp_range(&i.specifier_range)) .collect(); - // TODO(nayeemrmn): This is a crude way of detecting `@deno-types` which has + // TODO(nayeemrmn): This is a crude way of detecting `@ts-types` which has // a different specifier and therefore needs a separate call to // `diagnose_resolution()`. It would be much cleaner if that were modelled as // a separate dependency: https://github.com/denoland/deno_graph/issues/247. @@ -1540,7 +1540,7 @@ fn diagnose_dependency( snapshot, dependency_key, if dependency.maybe_code.is_none() - // If not @deno-types, diagnose the types if the code errored because + // If not @ts-types, diagnose the types if the code errored because // it's likely resolving into the node_modules folder, which might be // erroring correctly due to resolution only being for bundlers. Let this // fail at runtime if necessary, but don't bother erroring in the editor @@ -1951,7 +1951,7 @@ let c: number = "a"; &[( "a.ts", r#" - // @deno-types="bad.d.ts" + // @ts-types="bad.d.ts" import "bad.js"; import "bad.js"; "#, @@ -2005,11 +2005,11 @@ let c: number = "a"; "range": { "start": { "line": 1, - "character": 23 + "character": 21 }, "end": { "line": 1, - "character": 33 + "character": 31 } }, "severity": 1, diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index d3d821ebb3..931e008a72 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -3419,7 +3419,7 @@ fn parse_code_actions( &specifier_rewrite.new_deno_types_specifier { text_edit.new_text = format!( - "// @deno-types=\"{}\"\n{}", + "// @ts-types=\"{}\"\n{}", deno_types_specifier, &text_edit.new_text ); } @@ -3594,10 +3594,8 @@ impl CompletionEntryDetails { if let Some(deno_types_specifier) = &specifier_rewrite.new_deno_types_specifier { - *new_text = format!( - "// @deno-types=\"{}\"\n{}", - deno_types_specifier, new_text - ); + *new_text = + format!("// @ts-types=\"{}\"\n{}", deno_types_specifier, new_text); } } } diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index decc635ffa..6142c55888 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -2701,7 +2701,7 @@ fn lsp_hover_dependency() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @deno-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\nimport * as h from \"./mod🦕.ts\";\n\nconsole.log(a, b, c, d, e, f, g, h);\n" + "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @ts-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\nimport * as h from \"./mod🦕.ts\";\n\nconsole.log(a, b, c, d, e, f, g, h);\n" } }), ); @@ -6269,7 +6269,7 @@ fn lsp_code_actions_deno_types_for_npm() { res, json!([ { - "title": "Add @deno-types directive for \"@types/react\"", + "title": "Add @ts-types directive for \"@types/react\"", "kind": "quickfix", "edit": { "changes": { @@ -6279,7 +6279,7 @@ fn lsp_code_actions_deno_types_for_npm() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/react\"\n", + "newText": "// @ts-types=\"@types/react\"\n", }, ], }, @@ -6322,7 +6322,7 @@ fn lsp_code_actions_deno_types_for_npm() { res, json!([ { - "title": "Add @deno-types directive for \"npm:@types/react@^18.3.10\"", + "title": "Add @ts-types directive for \"npm:@types/react@^18.3.10\"", "kind": "quickfix", "edit": { "changes": { @@ -6332,7 +6332,7 @@ fn lsp_code_actions_deno_types_for_npm() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"npm:@types/react@^18.3.10\"\n", + "newText": "// @ts-types=\"npm:@types/react@^18.3.10\"\n", }, ], }, @@ -8310,7 +8310,7 @@ fn lsp_npm_auto_import_with_deno_types() { temp_dir.write( "other.ts", r#" - // @deno-types="@types/lz-string" + // @ts-types="@types/lz-string" import "lz-string"; "#, ); @@ -8358,7 +8358,7 @@ fn lsp_npm_auto_import_with_deno_types() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/lz-string\"\nimport { compressToBase64 } from \"lz-string\";\n", + "newText": "// @ts-types=\"@types/lz-string\"\nimport { compressToBase64 } from \"lz-string\";\n", }, ], }), @@ -8391,7 +8391,7 @@ fn lsp_npm_auto_import_with_deno_types() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/react\"\nimport { createRef } from \"react\";\n", + "newText": "// @ts-types=\"@types/react\"\nimport { createRef } from \"react\";\n", }, ], }), @@ -10226,7 +10226,7 @@ fn lsp_cache_location() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @deno-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" + "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @ts-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" } })); assert_eq!(diagnostics.all().len(), 6); @@ -10319,7 +10319,7 @@ fn lsp_tls_cert() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"https://localhost:5545/xTypeScriptTypes.js\";\n// @deno-types=\"https://localhost:5545/type_definitions/foo.d.ts\"\nimport * as b from \"https://localhost:5545/type_definitions/foo.js\";\nimport * as c from \"https://localhost:5545/subdir/type_reference.js\";\nimport * as d from \"https://localhost:5545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" + "text": "import * as a from \"https://localhost:5545/xTypeScriptTypes.js\";\n// @ts-types=\"https://localhost:5545/type_definitions/foo.d.ts\"\nimport * as b from \"https://localhost:5545/type_definitions/foo.js\";\nimport * as c from \"https://localhost:5545/subdir/type_reference.js\";\nimport * as d from \"https://localhost:5545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" } })); let diagnostics = diagnostics.all(); @@ -10784,7 +10784,7 @@ fn lsp_diagnostics_deno_types() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "/// \n/// \n/// Date: Tue, 10 Dec 2024 11:13:14 -0500 Subject: [PATCH 11/33] perf(compile): improve FileBackedVfsFile (#27299) --- cli/standalone/virtual_fs.rs | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index d5c8efb235..814774b71b 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::cell::RefCell; use std::collections::HashMap; use std::collections::HashSet; use std::fs::File; @@ -637,10 +638,9 @@ impl VfsRoot { } } -#[derive(Clone)] struct FileBackedVfsFile { file: VirtualFile, - pos: Arc>, + pos: RefCell, vfs: Arc, } @@ -648,7 +648,7 @@ impl FileBackedVfsFile { fn seek(&self, pos: SeekFrom) -> FsResult { match pos { SeekFrom::Start(pos) => { - *self.pos.lock() = pos; + *self.pos.borrow_mut() = pos; Ok(pos) } SeekFrom::End(offset) => { @@ -659,7 +659,7 @@ impl FileBackedVfsFile { .into(), ) } else { - let mut current_pos = self.pos.lock(); + let mut current_pos = self.pos.borrow_mut(); *current_pos = if offset >= 0 { self.file.offset.len - (offset as u64) } else { @@ -669,7 +669,7 @@ impl FileBackedVfsFile { } } SeekFrom::Current(offset) => { - let mut current_pos = self.pos.lock(); + let mut current_pos = self.pos.borrow_mut(); if offset >= 0 { *current_pos += offset as u64; } else if -offset as u64 > *current_pos { @@ -684,7 +684,7 @@ impl FileBackedVfsFile { fn read_to_buf(&self, buf: &mut [u8]) -> FsResult { let read_pos = { - let mut pos = self.pos.lock(); + let mut pos = self.pos.borrow_mut(); let read_pos = *pos; // advance the position due to the read *pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64); @@ -698,7 +698,7 @@ impl FileBackedVfsFile { fn read_to_end(&self) -> FsResult> { let read_pos = { - let mut pos = self.pos.lock(); + let mut pos = self.pos.borrow_mut(); let read_pos = *pos; // todo(dsherret): should this always set it to the end of the file? if *pos < self.file.offset.len { @@ -734,12 +734,9 @@ impl deno_io::fs::File for FileBackedVfsFile { self: Rc, mut buf: BufMutView, ) -> FsResult<(usize, BufMutView)> { - let inner = (*self).clone(); - tokio::task::spawn(async move { - let nread = inner.read_to_buf(&mut buf)?; - Ok((nread, buf)) - }) - .await? + // this is fast, no need to spawn a task + let nread = self.read_to_buf(&mut buf)?; + Ok((nread, buf)) } fn write_sync(self: Rc, _buf: &[u8]) -> FsResult { @@ -763,8 +760,8 @@ impl deno_io::fs::File for FileBackedVfsFile { self.read_to_end() } async fn read_all_async(self: Rc) -> FsResult> { - let inner = (*self).clone(); - tokio::task::spawn_blocking(move || inner.read_to_end()).await? + // this is fast, no need to spawn a task + self.read_to_end() } fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { From 7bab83d6c089d92120bad8a0a15237ca0eb4cb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 10 Dec 2024 18:22:42 +0000 Subject: [PATCH 12/33] fix(outdated): respect --quiet flag for hints (#27317) --- cli/tools/registry/pm/outdated.rs | 5 ++--- tests/specs/update/deno_json/__test__.jsonc | 5 +++++ tests/specs/update/deno_json/outdated_quiet.out | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 tests/specs/update/deno_json/outdated_quiet.out diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index 073d772f8a..9389f59678 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -100,15 +100,14 @@ fn print_outdated_table(packages: &[OutdatedPackage]) { println!("└{package_fill}┴{current_fill}┴{update_fill}┴{latest_fill}┘",); } -#[allow(clippy::print_stdout)] fn print_suggestion(compatible: bool) { - println!(); + log::info!(""); let (cmd, txt) = if compatible { ("", "compatible") } else { (" --latest", "available") }; - println!( + log::info!( "{}", color_print::cformat!( "Run deno outdated --update{} to update to the latest {} versions,\nor deno outdated --help for more information.", diff --git a/tests/specs/update/deno_json/__test__.jsonc b/tests/specs/update/deno_json/__test__.jsonc index 8b4aa26b5c..c57115fb02 100644 --- a/tests/specs/update/deno_json/__test__.jsonc +++ b/tests/specs/update/deno_json/__test__.jsonc @@ -26,6 +26,11 @@ { "args": "outdated", "output": "outdated.out" + }, + { + // Respect `--quiet flag and don't print hint how to update + "args": "outdated --quiet", + "output": "outdated_quiet.out" } ] }, diff --git a/tests/specs/update/deno_json/outdated_quiet.out b/tests/specs/update/deno_json/outdated_quiet.out new file mode 100644 index 0000000000..07ff9f3416 --- /dev/null +++ b/tests/specs/update/deno_json/outdated_quiet.out @@ -0,0 +1,15 @@ +┌────────────────────────────────────────────────┬─────────┬────────┬────────┐ +│ Package │ Current │ Update │ Latest │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/multiple-exports │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/add │ 0.2.0 │ 0.2.1 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/bin │ 0.6.0 │ 0.6.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ +└────────────────────────────────────────────────┴─────────┴────────┴────────┘ From 4ab668ed06db07c16fc7df47911d290d385e0307 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 10 Dec 2024 15:55:28 -0500 Subject: [PATCH 13/33] fix(lint): do not error providing --allow-import (#27321) Closes https://github.com/denoland/deno/issues/27318 --- cli/args/flags.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index cdeaa1b335..1bc2135ea5 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -2904,6 +2904,7 @@ To ignore linting on an entire file, you can add an ignore comment at the top of .arg(watch_arg(false)) .arg(watch_exclude_arg()) .arg(no_clear_screen_arg()) + .arg(allow_import_arg()) }) } @@ -5083,6 +5084,7 @@ fn lint_parse( unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionOnly); ext_arg_parse(flags, matches); config_args_parse(flags, matches); + allow_import_parse(flags, matches); let files = match matches.remove_many::("files") { Some(f) => f.collect(), @@ -7141,6 +7143,7 @@ mod tests { let r = flags_from_vec(svec![ "deno", "lint", + "--allow-import", "--watch", "script_1.ts", "script_2.ts" @@ -7162,6 +7165,10 @@ mod tests { compact: false, watch: Some(Default::default()), }), + permissions: PermissionFlags { + allow_import: Some(vec![]), + ..Default::default() + }, ..Flags::default() } ); From fe1be715d8dbd7c4c3a08c0db738ff9883470cae Mon Sep 17 00:00:00 2001 From: ingalless Date: Tue, 10 Dec 2024 22:41:24 +0000 Subject: [PATCH 14/33] chore(tests): Deprecate remaining usages of itest in check tests (#26962) This PR progresses #22907 by deprecating the usage of `itest` in `tests/integration/check_tests.rs` --- tests/integration/check_tests.rs | 14 -------------- tests/specs/check/check_all/__test__.jsonc | 5 +++++ .../all => specs/check/check_all}/check_all.out | 0 .../all => specs/check/check_all}/check_all.ts | 0 tests/specs/check/check_all_local/__test__.jsonc | 5 +++++ .../specs/check/check_all_local/check_all_local.ts | 3 +++ tools/lint.js | 2 +- 7 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 tests/specs/check/check_all/__test__.jsonc rename tests/{testdata/check/all => specs/check/check_all}/check_all.out (100%) rename tests/{testdata/check/all => specs/check/check_all}/check_all.ts (100%) create mode 100644 tests/specs/check/check_all_local/__test__.jsonc create mode 100644 tests/specs/check/check_all_local/check_all_local.ts diff --git a/tests/integration/check_tests.rs b/tests/integration/check_tests.rs index 178ac6493f..b98d719fca 100644 --- a/tests/integration/check_tests.rs +++ b/tests/integration/check_tests.rs @@ -3,23 +3,9 @@ use deno_lockfile::NewLockfileOptions; use deno_semver::jsr::JsrDepPackageReq; use test_util as util; -use test_util::itest; use util::TestContext; use util::TestContextBuilder; -itest!(check_all { - args: "check --allow-import --quiet --all check/all/check_all.ts", - output: "check/all/check_all.out", - http_server: true, - exit_code: 1, -}); - -itest!(check_all_local { - args: "check --allow-import --quiet check/all/check_all.ts", - output_str: Some(""), - http_server: true, -}); - #[test] fn cache_switching_config_then_no_config() { let context = TestContext::default(); diff --git a/tests/specs/check/check_all/__test__.jsonc b/tests/specs/check/check_all/__test__.jsonc new file mode 100644 index 0000000000..101e5aab9e --- /dev/null +++ b/tests/specs/check/check_all/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "args": "check --allow-import --quiet --all check_all.ts", + "output": "check_all.out", + "exitCode": 1 +} diff --git a/tests/testdata/check/all/check_all.out b/tests/specs/check/check_all/check_all.out similarity index 100% rename from tests/testdata/check/all/check_all.out rename to tests/specs/check/check_all/check_all.out diff --git a/tests/testdata/check/all/check_all.ts b/tests/specs/check/check_all/check_all.ts similarity index 100% rename from tests/testdata/check/all/check_all.ts rename to tests/specs/check/check_all/check_all.ts diff --git a/tests/specs/check/check_all_local/__test__.jsonc b/tests/specs/check/check_all_local/__test__.jsonc new file mode 100644 index 0000000000..0cd1c16acb --- /dev/null +++ b/tests/specs/check/check_all_local/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "args": "check --allow-import --quiet check_all_local.ts", + "output": "", + "exitCode": 0 +} diff --git a/tests/specs/check/check_all_local/check_all_local.ts b/tests/specs/check/check_all_local/check_all_local.ts new file mode 100644 index 0000000000..2ae8c2692c --- /dev/null +++ b/tests/specs/check/check_all_local/check_all_local.ts @@ -0,0 +1,3 @@ +import * as a from "http://localhost:4545/subdir/type_error.ts"; + +console.log(a.a); diff --git a/tools/lint.js b/tools/lint.js index 2312cde272..5bc3f2654f 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -211,7 +211,7 @@ async function ensureNoNewITests() { "bench_tests.rs": 0, "cache_tests.rs": 0, "cert_tests.rs": 0, - "check_tests.rs": 2, + "check_tests.rs": 0, "compile_tests.rs": 0, "coverage_tests.rs": 0, "eval_tests.rs": 0, From 7c8b55b584b028542bbf33a253e50deea1e98345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 10 Dec 2024 23:25:29 +0000 Subject: [PATCH 15/33] fix(outdated): error when there are no config files (#27306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit changes "deno outdated" subcommand to error out if run in a directory that has no config file (including parent directories). This matches "pnpm" behavior. Also added tests for filtering that yields no results, to ensure that it exists cleanly, that also matches "pnpm" behavior. Closes https://github.com/denoland/deno/issues/27287 --------- Signed-off-by: Bartek Iwańczuk Co-authored-by: David Sherret --- cli/tools/registry/pm/outdated.rs | 10 ++++++++++ tests/specs/update/deno_json/__test__.jsonc | 10 ++++++++++ tests/specs/update/mixed_workspace/__test__.jsonc | 10 ++++++++++ tests/specs/update/no_config_file/__test__.jsonc | 6 ++++++ tests/specs/update/package_json/__test__.jsonc | 10 ++++++++++ 5 files changed, 46 insertions(+) create mode 100644 tests/specs/update/no_config_file/__test__.jsonc diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index 9389f59678..aef65a5de0 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::sync::Arc; +use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; @@ -197,6 +198,15 @@ pub async fn outdated( let jsr_fetch_resolver = Arc::new(JsrFetchResolver::new(file_fetcher.clone())); + if !cli_options.start_dir.has_deno_json() + && !cli_options.start_dir.has_pkg_json() + { + bail!( + "No deno.json or package.json in \"{}\".", + cli_options.initial_cwd().display(), + ); + } + let args = dep_manager_args( &factory, cli_options, diff --git a/tests/specs/update/deno_json/__test__.jsonc b/tests/specs/update/deno_json/__test__.jsonc index c57115fb02..7983d2c56f 100644 --- a/tests/specs/update/deno_json/__test__.jsonc +++ b/tests/specs/update/deno_json/__test__.jsonc @@ -27,6 +27,11 @@ "args": "outdated", "output": "outdated.out" }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" + }, { // Respect `--quiet flag and don't print hint how to update "args": "outdated --quiet", @@ -43,6 +48,11 @@ { "args": "outdated --compatible", "output": "outdated_compatible.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated --compatible foobar", + "output": "" } ] }, diff --git a/tests/specs/update/mixed_workspace/__test__.jsonc b/tests/specs/update/mixed_workspace/__test__.jsonc index 8c846467d4..9810e15bdd 100644 --- a/tests/specs/update/mixed_workspace/__test__.jsonc +++ b/tests/specs/update/mixed_workspace/__test__.jsonc @@ -26,6 +26,11 @@ { "args": "outdated", "output": "print_outdated/root.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, @@ -38,6 +43,11 @@ { "args": "outdated --recursive", "output": "print_outdated/recursive.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, diff --git a/tests/specs/update/no_config_file/__test__.jsonc b/tests/specs/update/no_config_file/__test__.jsonc new file mode 100644 index 0000000000..1032e79279 --- /dev/null +++ b/tests/specs/update/no_config_file/__test__.jsonc @@ -0,0 +1,6 @@ +{ + "tempDir": true, + "args": "outdated", + "exitCode": 1, + "output": "error: No deno.json or package.json in \"[WILDLINE]\".\n" +} diff --git a/tests/specs/update/package_json/__test__.jsonc b/tests/specs/update/package_json/__test__.jsonc index 19d576dfc0..b86b9956cc 100644 --- a/tests/specs/update/package_json/__test__.jsonc +++ b/tests/specs/update/package_json/__test__.jsonc @@ -25,6 +25,11 @@ { "args": "outdated", "output": "outdated.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, @@ -37,6 +42,11 @@ { "args": "outdated --compatible", "output": "outdated_compatible.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated --compatible foobar", + "output": "" } ] }, From dd42a64c431f6c9bf39ae3ac8783551e6b59a18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 11 Dec 2024 01:06:04 +0000 Subject: [PATCH 16/33] refactor(lint): manage schema files for linter in Deno repo (#27324) This commit provides schema files for lint rules and lint tags in this repo instead of pulling them from `deno_lint` repository. A unit test was added to ensure all available rules are listed in the schema file. A unit test for tags can be done once https://github.com/denoland/deno/pull/27162 lands. --- cli/schemas/config-file.v1.json | 6 +- cli/schemas/lint-rules.v1.json | 112 ++++++++++++++++++++++++++++++++ cli/schemas/lint-tags.v1.json | 4 ++ cli/tools/lint/mod.rs | 65 ++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 cli/schemas/lint-rules.v1.json create mode 100644 cli/schemas/lint-tags.v1.json diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index a64cb2ff65..1e3abb2c0d 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -291,7 +291,7 @@ "type": "array", "description": "List of tag names that will be run. Empty list disables all tags and will only use rules from `include`.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/tags.v1.json" + "$ref": "lint-tags.v1.json" }, "minItems": 0, "uniqueItems": true @@ -300,7 +300,7 @@ "type": "array", "description": "List of rule names that will be excluded from configured tag sets. If the same rule is in `include` it will be run.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" + "$ref": "lint-rules.v1.json" }, "minItems": 0, "uniqueItems": true @@ -309,7 +309,7 @@ "type": "array", "description": "List of rule names that will be run. Even if the same rule is in `exclude` it will be run.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" + "$ref": "lint-rules.v1.json" }, "minItems": 0, "uniqueItems": true diff --git a/cli/schemas/lint-rules.v1.json b/cli/schemas/lint-rules.v1.json new file mode 100644 index 0000000000..71d1784958 --- /dev/null +++ b/cli/schemas/lint-rules.v1.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "enum": [ + "adjacent-overload-signatures", + "ban-ts-comment", + "ban-types", + "ban-unknown-rule-code", + "ban-untagged-ignore", + "ban-untagged-todo", + "ban-unused-ignore", + "camelcase", + "constructor-super", + "default-param-last", + "eqeqeq", + "explicit-function-return-type", + "explicit-module-boundary-types", + "for-direction", + "fresh-handler-export", + "fresh-server-event-handlers", + "getter-return", + "guard-for-in", + "no-array-constructor", + "no-async-promise-executor", + "no-await-in-loop", + "no-await-in-sync-fn", + "no-boolean-literal-for-arguments", + "no-case-declarations", + "no-class-assign", + "no-compare-neg-zero", + "no-cond-assign", + "no-console", + "no-const-assign", + "no-constant-condition", + "no-control-regex", + "no-debugger", + "no-delete-var", + "no-deprecated-deno-api", + "no-dupe-args", + "no-dupe-class-members", + "no-dupe-else-if", + "no-dupe-keys", + "no-duplicate-case", + "no-empty", + "no-empty-character-class", + "no-empty-enum", + "no-empty-interface", + "no-empty-pattern", + "no-eval", + "no-ex-assign", + "no-explicit-any", + "no-external-import", + "no-extra-boolean-cast", + "no-extra-non-null-assertion", + "no-fallthrough", + "no-func-assign", + "no-global-assign", + "no-implicit-declare-namespace-export", + "no-import-assertions", + "no-import-assign", + "no-inferrable-types", + "no-inner-declarations", + "no-invalid-regexp", + "no-invalid-triple-slash-reference", + "no-irregular-whitespace", + "no-misused-new", + "no-namespace", + "no-new-symbol", + "no-node-globals", + "no-non-null-asserted-optional-chain", + "no-non-null-assertion", + "no-obj-calls", + "no-octal", + "no-process-globals", + "no-prototype-builtins", + "no-redeclare", + "no-regex-spaces", + "no-self-assign", + "no-self-compare", + "no-setter-return", + "no-shadow-restricted-names", + "no-sloppy-imports", + "no-slow-types", + "no-sparse-arrays", + "no-sync-fn-in-async-fn", + "no-this-alias", + "no-this-before-super", + "no-throw-literal", + "no-top-level-await", + "no-undef", + "no-unreachable", + "no-unsafe-finally", + "no-unsafe-negation", + "no-unused-labels", + "no-unused-vars", + "no-var", + "no-window", + "no-window-prefix", + "no-with", + "prefer-as-const", + "prefer-ascii", + "prefer-const", + "prefer-namespace-keyword", + "prefer-primordials", + "require-await", + "require-yield", + "single-var-declarator", + "triple-slash-reference", + "use-isnan", + "valid-typeof", + "verbatim-module-syntax" + ] +} diff --git a/cli/schemas/lint-tags.v1.json b/cli/schemas/lint-tags.v1.json new file mode 100644 index 0000000000..4b4f0e48db --- /dev/null +++ b/cli/schemas/lint-tags.v1.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "enum": ["fresh", "jsr", "jsx", "react", "recommended"] +} diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index fcefb45874..596359bdc0 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -556,3 +556,68 @@ struct LintError { file_path: String, message: String, } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde::Deserialize; + use test_util as util; + + #[derive(Serialize, Deserialize)] + struct RulesSchema { + #[serde(rename = "$schema")] + schema: String, + + #[serde(rename = "enum")] + rules: Vec, + } + + fn get_all_rules() -> Vec { + let rule_provider = LintRuleProvider::new(None, None); + let configured_rules = + rule_provider.resolve_lint_rules(Default::default(), None); + let mut all_rules = configured_rules + .all_rule_codes + .into_iter() + .map(|s| s.to_string()) + .collect::>(); + all_rules.sort(); + + all_rules + } + + // TODO(bartlomieju): do the same for tags, once https://github.com/denoland/deno/pull/27162 lands + #[test] + fn all_lint_rules_are_listed_in_schema_file() { + let all_rules = get_all_rules(); + + let rules_schema_path = + util::root_path().join("cli/schemas/lint-rules.v1.json"); + let rules_schema_file = + std::fs::read_to_string(&rules_schema_path).unwrap(); + + let schema: RulesSchema = serde_json::from_str(&rules_schema_file).unwrap(); + + const UPDATE_ENV_VAR_NAME: &str = "UPDATE_EXPECTED"; + + if std::env::var(UPDATE_ENV_VAR_NAME).ok().is_none() { + assert_eq!( + schema.rules, all_rules, + "Lint rules schema file not up to date. Run again with {}=1 to update the expected output", + UPDATE_ENV_VAR_NAME + ); + return; + } + + std::fs::write( + &rules_schema_path, + serde_json::to_string_pretty(&RulesSchema { + schema: schema.schema, + rules: all_rules, + }) + .unwrap(), + ) + .unwrap(); + } +} From 6f506208f60f710952b4ed363e557cb8a36e92b4 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:24:23 -0800 Subject: [PATCH 17/33] feat(unstable): support caching npm dependencies only as they're needed (#27300) Currently deno eagerly caches all npm packages in the workspace's npm resolution. So, for instance, running a file `foo.ts` that imports `npm:chalk` will also install all dependencies listed in `package.json` and all `npm` dependencies listed in the lockfile. This PR refactors things to give more control over when and what npm packages are automatically cached while building the module graph. After this PR, by default the current behavior is unchanged _except_ for `deno install --entrypoint`, which will only cache npm packages used by the given entrypoint. For the other subcommands, this behavior can be enabled with `--unstable-npm-lazy-caching` Fixes #25782. --------- Signed-off-by: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Co-authored-by: Luca Casonato --- cli/args/flags.rs | 76 ++++++++++--------- cli/args/lockfile.rs | 7 +- cli/args/mod.rs | 35 +++++++-- cli/factory.rs | 1 + cli/graph_util.rs | 26 ++++++- cli/lsp/language_server.rs | 7 +- cli/lsp/resolver.rs | 7 +- cli/module_loader.rs | 1 + cli/npm/managed/mod.rs | 65 ++++++++++++---- cli/npm/managed/resolution.rs | 4 + cli/npm/managed/resolvers/common.rs | 6 +- cli/npm/managed/resolvers/global.rs | 18 ++++- cli/npm/managed/resolvers/local.rs | 12 ++- cli/npm/mod.rs | 1 + cli/resolver.rs | 23 +++++- cli/standalone/binary.rs | 1 + cli/standalone/mod.rs | 1 + cli/tools/bench/mod.rs | 6 +- cli/tools/compile.rs | 6 +- cli/tools/doc.rs | 6 +- cli/tools/info.rs | 7 +- cli/tools/installer.rs | 7 +- cli/tools/registry/pm/cache_deps.rs | 41 ++++++---- cli/tools/repl/session.rs | 4 +- cli/tools/run/mod.rs | 18 ++++- cli/tools/task.rs | 12 +++ cli/tools/test/mod.rs | 6 +- cli/worker.rs | 17 ++++- .../__test__.jsonc | 13 ++++ .../install-entrypoint.out | 6 ++ .../entrypoint_only_used_packages/install.out | 2 + .../entrypoint_only_used_packages/main.ts | 3 + .../package.json | 6 ++ .../run/jsx_import_source/__test__.jsonc | 1 + tests/specs/run/lazy_npm/__test__.jsonc | 9 +++ tests/specs/run/lazy_npm/deno.json | 3 + tests/specs/run/lazy_npm/main.out | 7 ++ tests/specs/run/lazy_npm/main.ts | 3 + tests/specs/run/lazy_npm/package.json | 6 ++ 39 files changed, 371 insertions(+), 109 deletions(-) create mode 100644 tests/specs/install/entrypoint_only_used_packages/__test__.jsonc create mode 100644 tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out create mode 100644 tests/specs/install/entrypoint_only_used_packages/install.out create mode 100644 tests/specs/install/entrypoint_only_used_packages/main.ts create mode 100644 tests/specs/install/entrypoint_only_used_packages/package.json create mode 100644 tests/specs/run/lazy_npm/__test__.jsonc create mode 100644 tests/specs/run/lazy_npm/deno.json create mode 100644 tests/specs/run/lazy_npm/main.out create mode 100644 tests/specs/run/lazy_npm/main.ts create mode 100644 tests/specs/run/lazy_npm/package.json diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 1bc2135ea5..9418739564 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -245,7 +245,7 @@ pub struct InstallFlagsGlobal { } #[derive(Clone, Debug, Eq, PartialEq)] -pub enum InstallKind { +pub enum InstallFlags { Local(InstallFlagsLocal), Global(InstallFlagsGlobal), } @@ -257,11 +257,6 @@ pub enum InstallFlagsLocal { Entrypoints(Vec), } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct InstallFlags { - pub kind: InstallKind, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub struct JSONReferenceFlags { pub json: deno_core::serde_json::Value, @@ -600,6 +595,7 @@ pub struct UnstableConfig { pub bare_node_builtins: bool, pub detect_cjs: bool, pub sloppy_imports: bool, + pub npm_lazy_caching: bool, pub features: Vec, // --unstabe-kv --unstable-cron } @@ -4407,6 +4403,16 @@ impl CommandExt for Command { }) .help_heading(UNSTABLE_HEADING) .display_order(next_display_order()) + ).arg( + Arg::new("unstable-npm-lazy-caching") + .long("unstable-npm-lazy-caching") + .help("Enable unstable lazy caching of npm dependencies, downloading them only as needed (disabled: all npm packages in package.json are installed on startup; enabled: only npm packages that are actually referenced in an import are installed") + .env("DENO_UNSTABLE_NPM_LAZY_CACHING") + .value_parser(FalseyValueParser::new()) + .action(ArgAction::SetTrue) + .hide(true) + .help_heading(UNSTABLE_HEADING) + .display_order(next_display_order()), ); for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS.iter() { @@ -4920,15 +4926,14 @@ fn install_parse( let module_url = cmd_values.next().unwrap(); let args = cmd_values.collect(); - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + flags.subcommand = + DenoSubcommand::Install(InstallFlags::Global(InstallFlagsGlobal { name, module_url, args, root, force, - }), - }); + })); return Ok(()); } @@ -4937,22 +4942,19 @@ fn install_parse( allow_scripts_arg_parse(flags, matches)?; if matches.get_flag("entrypoint") { let entrypoints = matches.remove_many::("cmd").unwrap_or_default(); - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Entrypoints( - entrypoints.collect(), - )), - }); + flags.subcommand = DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Entrypoints(entrypoints.collect()), + )); } else if let Some(add_files) = matches .remove_many("cmd") .map(|packages| add_parse_inner(matches, Some(packages))) { - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Add(add_files)), - }) + flags.subcommand = DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Add(add_files), + )) } else { - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::TopLevel), - }); + flags.subcommand = + DenoSubcommand::Install(InstallFlags::Local(InstallFlagsLocal::TopLevel)); } Ok(()) } @@ -5998,6 +6000,8 @@ fn unstable_args_parse( flags.unstable_config.detect_cjs = matches.get_flag("unstable-detect-cjs"); flags.unstable_config.sloppy_imports = matches.get_flag("unstable-sloppy-imports"); + flags.unstable_config.npm_lazy_caching = + matches.get_flag("unstable-npm-lazy-caching"); if matches!(cfg, UnstableArgsConfig::ResolutionAndRuntime) { for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { @@ -8606,15 +8610,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: None, module_url: "jsr:@std/http/file-server".to_string(), args: vec![], root: None, force: false, - }), - }), + } + ),), ..Flags::default() } ); @@ -8628,15 +8632,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: None, module_url: "jsr:@std/http/file-server".to_string(), args: vec![], root: None, force: false, - }), - }), + } + ),), ..Flags::default() } ); @@ -8649,15 +8653,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: Some("file_server".to_string()), module_url: "jsr:@std/http/file-server".to_string(), args: svec!["foo", "bar"], root: Some("/foo".to_string()), force: true, - }), - }), + } + ),), import_map_path: Some("import_map.json".to_string()), no_remote: true, config_flag: ConfigFlag::Path("tsconfig.json".to_owned()), @@ -11211,9 +11215,9 @@ mod tests { ..Flags::default() }, "install" => Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Add(flags)), - }), + subcommand: DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Add(flags), + )), ..Flags::default() }, _ => unreachable!(), diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index 6c1a2ca0ef..74eab78f1c 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -20,7 +20,6 @@ use crate::Flags; use crate::args::DenoSubcommand; use crate::args::InstallFlags; -use crate::args::InstallKind; use deno_lockfile::Lockfile; @@ -136,10 +135,8 @@ impl CliLockfile { if flags.no_lock || matches!( flags.subcommand, - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(..), - .. - }) | DenoSubcommand::Uninstall(_) + DenoSubcommand::Install(InstallFlags::Global(..)) + | DenoSubcommand::Uninstall(_) ) { return Ok(None); diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 0b049cf409..314c0ff17a 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -970,9 +970,7 @@ impl CliOptions { match self.sub_command() { DenoSubcommand::Cache(_) => GraphKind::All, DenoSubcommand::Check(_) => GraphKind::TypesOnly, - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(_), - }) => GraphKind::All, + DenoSubcommand::Install(InstallFlags::Local(_)) => GraphKind::All, _ => self.type_check_mode().as_graph_kind(), } } @@ -1549,11 +1547,11 @@ impl CliOptions { DenoSubcommand::Check(check_flags) => { Some(files_to_urls(&check_flags.files)) } - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(flags), - }) => Url::parse(&flags.module_url) - .ok() - .map(|url| vec![Cow::Owned(url)]), + DenoSubcommand::Install(InstallFlags::Global(flags)) => { + Url::parse(&flags.module_url) + .ok() + .map(|url| vec![Cow::Owned(url)]) + } DenoSubcommand::Doc(DocFlags { source_files: DocSourceFileFlag::Paths(paths), .. @@ -1689,6 +1687,7 @@ impl CliOptions { "detect-cjs", "fmt-component", "fmt-sql", + "lazy-npm-caching", ]) .collect(); @@ -1767,6 +1766,19 @@ impl CliOptions { ), } } + + pub fn unstable_npm_lazy_caching(&self) -> bool { + self.flags.unstable_config.npm_lazy_caching + || self.workspace().has_unstable("npm-lazy-caching") + } + + pub fn default_npm_caching_strategy(&self) -> NpmCachingStrategy { + if self.flags.unstable_config.npm_lazy_caching { + NpmCachingStrategy::Lazy + } else { + NpmCachingStrategy::Eager + } + } } /// Resolves the path to use for a local node_modules folder. @@ -1981,6 +1993,13 @@ fn load_env_variables_from_env_file(filename: Option<&Vec>) { } } +#[derive(Debug, Clone, Copy)] +pub enum NpmCachingStrategy { + Eager, + Lazy, + Manual, +} + #[cfg(test)] mod test { use pretty_assertions::assert_eq; diff --git a/cli/factory.rs b/cli/factory.rs index 6937b750f9..f08bf7e4b1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -984,6 +984,7 @@ impl CliFactory { cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, self.cli_options()?.otel_config(), + self.cli_options()?.default_npm_caching_strategy(), )) } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 22117990d2..b655dda0f6 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -4,6 +4,7 @@ use crate::args::config_to_deno_graph_workspace_member; use crate::args::jsr_url; use crate::args::CliLockfile; use crate::args::CliOptions; +pub use crate::args::NpmCachingStrategy; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::cache; use crate::cache::FetchCacher; @@ -218,6 +219,7 @@ pub struct CreateGraphOptions<'a> { pub is_dynamic: bool, /// Specify `None` to use the default CLI loader. pub loader: Option<&'a mut dyn Loader>, + pub npm_caching: NpmCachingStrategy, } pub struct ModuleGraphCreator { @@ -246,10 +248,11 @@ impl ModuleGraphCreator { &self, graph_kind: GraphKind, roots: Vec, + npm_caching: NpmCachingStrategy, ) -> Result { let mut cache = self.module_graph_builder.create_graph_loader(); self - .create_graph_with_loader(graph_kind, roots, &mut cache) + .create_graph_with_loader(graph_kind, roots, &mut cache, npm_caching) .await } @@ -258,6 +261,7 @@ impl ModuleGraphCreator { graph_kind: GraphKind, roots: Vec, loader: &mut dyn Loader, + npm_caching: NpmCachingStrategy, ) -> Result { self .create_graph_with_options(CreateGraphOptions { @@ -265,6 +269,7 @@ impl ModuleGraphCreator { graph_kind, roots, loader: Some(loader), + npm_caching, }) .await } @@ -317,6 +322,7 @@ impl ModuleGraphCreator { graph_kind: deno_graph::GraphKind::All, roots, loader: Some(&mut publish_loader), + npm_caching: self.options.default_npm_caching_strategy(), }) .await?; self.graph_valid(&graph)?; @@ -376,6 +382,7 @@ impl ModuleGraphCreator { graph_kind, roots, loader: None, + npm_caching: self.options.default_npm_caching_strategy(), }) .await?; @@ -565,7 +572,8 @@ impl ModuleGraphBuilder { }; let cli_resolver = &self.resolver; let graph_resolver = self.create_graph_resolver()?; - let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); + let graph_npm_resolver = + cli_resolver.create_graph_npm_resolver(options.npm_caching); let maybe_file_watcher_reporter = self .maybe_file_watcher_reporter .as_ref() @@ -592,6 +600,7 @@ impl ModuleGraphBuilder { resolver: Some(&graph_resolver), locker: locker.as_mut().map(|l| l as _), }, + options.npm_caching, ) .await } @@ -602,6 +611,7 @@ impl ModuleGraphBuilder { roots: Vec, loader: &'a mut dyn deno_graph::source::Loader, options: deno_graph::BuildOptions<'a>, + npm_caching: NpmCachingStrategy, ) -> Result<(), AnyError> { // ensure an "npm install" is done if the user has explicitly // opted into using a node_modules directory @@ -612,7 +622,13 @@ impl ModuleGraphBuilder { .unwrap_or(false) { if let Some(npm_resolver) = self.npm_resolver.as_managed() { - npm_resolver.ensure_top_level_package_json_install().await?; + let already_done = + npm_resolver.ensure_top_level_package_json_install().await?; + if !already_done && matches!(npm_caching, NpmCachingStrategy::Eager) { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } } } @@ -701,7 +717,9 @@ impl ModuleGraphBuilder { let parser = self.parsed_source_cache.as_capturing_parser(); let cli_resolver = &self.resolver; let graph_resolver = self.create_graph_resolver()?; - let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); + let graph_npm_resolver = cli_resolver.create_graph_npm_resolver( + self.cli_options.default_npm_caching_strategy(), + ); graph.build_fast_check_type_graph( deno_graph::BuildFastCheckTypeGraphOptions { diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 0caaa94107..3c4cb0930e 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -270,7 +270,12 @@ impl LanguageServer { open_docs: &open_docs, }; let graph = module_graph_creator - .create_graph_with_loader(GraphKind::All, roots.clone(), &mut loader) + .create_graph_with_loader( + GraphKind::All, + roots.clone(), + &mut loader, + graph_util::NpmCachingStrategy::Eager, + ) .await?; graph_util::graph_valid( &graph, diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 363ad43700..28c7b04fc9 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -133,7 +133,8 @@ impl LspScopeResolver { cache.for_specifier(config_data.map(|d| d.scope.as_ref())), config_data.and_then(|d| d.lockfile.clone()), ))); - let npm_graph_resolver = cli_resolver.create_graph_npm_resolver(); + let npm_graph_resolver = cli_resolver + .create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager); let maybe_jsx_import_source_config = config_data.and_then(|d| d.maybe_jsx_import_source_config()); let graph_imports = config_data @@ -343,7 +344,9 @@ impl LspResolver { file_referrer: Option<&ModuleSpecifier>, ) -> WorkerCliNpmGraphResolver { let resolver = self.get_scope_resolver(file_referrer); - resolver.resolver.create_graph_npm_resolver() + resolver + .resolver + .create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager) } pub fn as_is_cjs_resolver( diff --git a/cli/module_loader.rs b/cli/module_loader.rs index c5f80d68e0..5e4ff875dc 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -156,6 +156,7 @@ impl ModuleLoadPreparer { graph_kind: graph.graph_kind(), roots: roots.to_vec(), loader: Some(&mut cache), + npm_caching: self.options.default_npm_caching_strategy(), }, ) .await?; diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 5ed25f8272..2c6e6d318a 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -296,6 +296,12 @@ pub fn create_managed_in_npm_pkg_checker( Arc::new(ManagedInNpmPackageChecker { root_dir }) } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackageCaching<'a> { + Only(Cow<'a, [PackageReq]>), + All, +} + /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { @@ -420,19 +426,44 @@ impl ManagedCliNpmResolver { /// Adds package requirements to the resolver and ensures everything is setup. /// This includes setting up the `node_modules` directory, if applicable. - pub async fn add_package_reqs( + pub async fn add_and_cache_package_reqs( &self, packages: &[PackageReq], ) -> Result<(), AnyError> { self - .add_package_reqs_raw(packages) + .add_package_reqs_raw( + packages, + Some(PackageCaching::Only(packages.into())), + ) .await .dependencies_result } - pub async fn add_package_reqs_raw( + pub async fn add_package_reqs_no_cache( &self, packages: &[PackageReq], + ) -> Result<(), AnyError> { + self + .add_package_reqs_raw(packages, None) + .await + .dependencies_result + } + + pub async fn add_package_reqs( + &self, + packages: &[PackageReq], + caching: PackageCaching<'_>, + ) -> Result<(), AnyError> { + self + .add_package_reqs_raw(packages, Some(caching)) + .await + .dependencies_result + } + + pub async fn add_package_reqs_raw<'a>( + &self, + packages: &[PackageReq], + caching: Option>, ) -> AddPkgReqsResult { if packages.is_empty() { return AddPkgReqsResult { @@ -449,7 +480,9 @@ impl ManagedCliNpmResolver { } } if result.dependencies_result.is_ok() { - result.dependencies_result = self.cache_packages().await; + if let Some(caching) = caching { + result.dependencies_result = self.cache_packages(caching).await; + } } result @@ -491,16 +524,20 @@ impl ManagedCliNpmResolver { pub async fn inject_synthetic_types_node_package( &self, ) -> Result<(), AnyError> { + let reqs = &[PackageReq::from_str("@types/node").unwrap()]; // add and ensure this isn't added to the lockfile self - .add_package_reqs(&[PackageReq::from_str("@types/node").unwrap()]) + .add_package_reqs(reqs, PackageCaching::Only(reqs.into())) .await?; Ok(()) } - pub async fn cache_packages(&self) -> Result<(), AnyError> { - self.fs_resolver.cache_packages().await + pub async fn cache_packages( + &self, + caching: PackageCaching<'_>, + ) -> Result<(), AnyError> { + self.fs_resolver.cache_packages(caching).await } pub fn resolve_pkg_folder_from_deno_module( @@ -545,18 +582,18 @@ impl ManagedCliNpmResolver { /// Ensures that the top level `package.json` dependencies are installed. /// This may set up the `node_modules` directory. /// - /// Returns `true` if any changes (such as caching packages) were made. - /// If this returns `false`, `node_modules` has _not_ been set up. + /// Returns `true` if the top level packages are already installed. A + /// return value of `false` means that new packages were added to the NPM resolution. pub async fn ensure_top_level_package_json_install( &self, ) -> Result { if !self.top_level_install_flag.raise() { - return Ok(false); // already did this + return Ok(true); // already did this } let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs(); if pkg_json_remote_pkgs.is_empty() { - return Ok(false); + return Ok(true); } // check if something needs resolving before bothering to load all @@ -570,14 +607,16 @@ impl ManagedCliNpmResolver { log::debug!( "All package.json deps resolvable. Skipping top level install." ); - return Ok(false); // everything is already resolvable + return Ok(true); // everything is already resolvable } let pkg_reqs = pkg_json_remote_pkgs .iter() .map(|pkg| pkg.req.clone()) .collect::>(); - self.add_package_reqs(&pkg_reqs).await.map(|_| true) + self.add_package_reqs_no_cache(&pkg_reqs).await?; + + Ok(false) } pub async fn cache_package_info( diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 66cc6a7428..73c5c31caf 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -255,6 +255,10 @@ impl NpmResolution { .read() .as_valid_serialized_for_system(system_info) } + + pub fn subset(&self, package_reqs: &[PackageReq]) -> NpmResolutionSnapshot { + self.snapshot.read().subset(package_reqs) + } } async fn add_package_reqs_to_snapshot( diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 332756daa4..68e95fb39a 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use super::super::PackageCaching; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::anyhow::Context; @@ -57,7 +58,10 @@ pub trait NpmPackageFsResolver: Send + Sync { specifier: &ModuleSpecifier, ) -> Result, AnyError>; - async fn cache_packages(&self) -> Result<(), AnyError>; + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn ensure_read_permission<'a>( diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 2b48c3d2fc..4e79941af6 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::sync::Arc; use crate::colors; +use crate::npm::managed::PackageCaching; use crate::npm::CliNpmCache; use crate::npm::CliNpmTarballCache; use async_trait::async_trait; @@ -150,10 +151,19 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { ) } - async fn cache_packages(&self) -> Result<(), AnyError> { - let package_partitions = self - .resolution - .all_system_packages_partitioned(&self.system_info); + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError> { + let package_partitions = match caching { + PackageCaching::All => self + .resolution + .all_system_packages_partitioned(&self.system_info), + PackageCaching::Only(reqs) => self + .resolution + .subset(&reqs) + .all_system_packages_partitioned(&self.system_info), + }; cache_packages(&package_partitions.packages, &self.tarball_cache).await?; // create the copy package folders diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index d383a5413f..1e83717f15 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use crate::args::LifecycleScriptsConfig; use crate::colors; +use crate::npm::managed::PackageCaching; use crate::npm::CliNpmCache; use crate::npm::CliNpmTarballCache; use async_trait::async_trait; @@ -253,9 +254,16 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { )) } - async fn cache_packages(&self) -> Result<(), AnyError> { + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError> { + let snapshot = match caching { + PackageCaching::All => self.resolution.snapshot(), + PackageCaching::Only(reqs) => self.resolution.subset(&reqs), + }; sync_resolution_with_fs( - &self.resolution.snapshot(), + &snapshot, &self.cache, &self.npm_install_deps_provider, &self.progress_bar, diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 48d90d7dd0..b39e0a340d 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -41,6 +41,7 @@ pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions; pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; +pub use self::managed::PackageCaching; pub type CliNpmTarballCache = deno_npm_cache::TarballCache; pub type CliNpmCache = deno_npm_cache::NpmCache; diff --git a/cli/resolver.rs b/cli/resolver.rs index 15ca4aa2b6..f5c3f68f36 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -32,6 +32,7 @@ use std::path::PathBuf; use std::sync::Arc; use thiserror::Error; +use crate::args::NpmCachingStrategy; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::node::CliNodeCodeTranslator; use crate::npm::CliNpmResolver; @@ -240,11 +241,15 @@ impl CliResolver { // todo(dsherret): move this off CliResolver as CliResolver is acting // like a factory by doing this (it's beyond its responsibility) - pub fn create_graph_npm_resolver(&self) -> WorkerCliNpmGraphResolver { + pub fn create_graph_npm_resolver( + &self, + npm_caching: NpmCachingStrategy, + ) -> WorkerCliNpmGraphResolver { WorkerCliNpmGraphResolver { npm_resolver: self.npm_resolver.as_ref(), found_package_json_dep_flag: &self.found_package_json_dep_flag, bare_node_builtins_enabled: self.bare_node_builtins_enabled, + npm_caching, } } @@ -304,6 +309,7 @@ pub struct WorkerCliNpmGraphResolver<'a> { npm_resolver: Option<&'a Arc>, found_package_json_dep_flag: &'a AtomicFlag, bare_node_builtins_enabled: bool, + npm_caching: NpmCachingStrategy, } #[async_trait(?Send)] @@ -373,7 +379,20 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { Ok(()) }; - let result = npm_resolver.add_package_reqs_raw(package_reqs).await; + let result = npm_resolver + .add_package_reqs_raw( + package_reqs, + match self.npm_caching { + NpmCachingStrategy::Eager => { + Some(crate::npm::PackageCaching::All) + } + NpmCachingStrategy::Lazy => { + Some(crate::npm::PackageCaching::Only(package_reqs.into())) + } + NpmCachingStrategy::Manual => None, + }, + ) + .await; NpmResolvePkgReqsResult { results: result diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 632f27da6f..dec1d89110 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -779,6 +779,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { detect_cjs: self.cli_options.unstable_detect_cjs(), sloppy_imports: self.cli_options.unstable_sloppy_imports(), features: self.cli_options.unstable_features(), + npm_lazy_caching: self.cli_options.unstable_npm_lazy_caching(), }, otel_config: self.cli_options.otel_config(), }; diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 53efab2964..22e0b6d115 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -924,6 +924,7 @@ pub async fn run(data: StandaloneData) -> Result { serve_host: None, }, metadata.otel_config, + crate::args::NpmCachingStrategy::Lazy, ); // Initialize v8 once from the main thread. diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index 1d49fa061d..5983590531 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -538,7 +538,11 @@ pub async fn run_benchmarks_with_watch( )?; let graph = module_graph_creator - .create_graph(graph_kind, collected_bench_modules.clone()) + .create_graph( + graph_kind, + collected_bench_modules.clone(), + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; module_graph_creator.graph_valid(&graph)?; let bench_modules = &graph.roots; diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 4fa9963683..5cd19ecf71 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -69,7 +69,11 @@ pub async fn compile( // create a module graph with types information in it. We don't want to // store that in the binary so create a code only module graph from scratch. module_graph_creator - .create_graph(GraphKind::CodeOnly, module_roots) + .create_graph( + GraphKind::CodeOnly, + module_roots, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await? } else { graph diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 9a24e458ac..647a36dc48 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -131,7 +131,11 @@ pub async fn doc( |_| true, )?; let graph = module_graph_creator - .create_graph(GraphKind::TypesOnly, module_specifiers.clone()) + .create_graph( + GraphKind::TypesOnly, + module_specifiers.clone(), + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; graph_exit_integrity_errors(&graph); diff --git a/cli/tools/info.rs b/cli/tools/info.rs index f0cd37772d..f58732a904 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -123,7 +123,12 @@ pub async fn info( let mut loader = module_graph_builder.create_graph_loader(); loader.enable_loading_cache_info(); // for displaying the cache information let graph = module_graph_creator - .create_graph_with_loader(GraphKind::All, vec![specifier], &mut loader) + .create_graph_with_loader( + GraphKind::All, + vec![specifier], + &mut loader, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; // write out the lockfile if there is one diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index df5981e6e7..d7c484beba 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -9,7 +9,6 @@ use crate::args::Flags; use crate::args::InstallFlags; use crate::args::InstallFlagsGlobal; use crate::args::InstallFlagsLocal; -use crate::args::InstallKind; use crate::args::TypeCheckMode; use crate::args::UninstallFlags; use crate::args::UninstallKind; @@ -339,11 +338,11 @@ pub async fn install_command( flags: Arc, install_flags: InstallFlags, ) -> Result<(), AnyError> { - match install_flags.kind { - InstallKind::Global(global_flags) => { + match install_flags { + InstallFlags::Global(global_flags) => { install_global(flags, global_flags).await } - InstallKind::Local(local_flags) => { + InstallFlags::Local(local_flags) => { if let InstallFlagsLocal::Add(add_flags) = &local_flags { check_if_installs_a_single_package_globally(Some(add_flags))?; } diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index dbec8a3b3f..814c76cb27 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use crate::factory::CliFactory; use crate::graph_container::ModuleGraphContainer; use crate::graph_container::ModuleGraphUpdatePermit; +use crate::graph_util::CreateGraphOptions; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::StreamExt; @@ -18,18 +19,16 @@ pub async fn cache_top_level_deps( ) -> Result<(), AnyError> { let npm_resolver = factory.npm_resolver().await?; let cli_options = factory.cli_options()?; - let root_permissions = factory.root_permissions_container()?; if let Some(npm_resolver) = npm_resolver.as_managed() { - if !npm_resolver.ensure_top_level_package_json_install().await? { - if let Some(lockfile) = cli_options.maybe_lockfile() { - lockfile.error_if_changed()?; - } - - npm_resolver.cache_packages().await?; + npm_resolver.ensure_top_level_package_json_install().await?; + if let Some(lockfile) = cli_options.maybe_lockfile() { + lockfile.error_if_changed()?; } } // cache as many entries in the import map as we can let resolver = factory.workspace_resolver().await?; + + let mut maybe_graph_error = Ok(()); if let Some(import_map) = resolver.maybe_import_map() { let jsr_resolver = if let Some(resolver) = jsr_resolver { resolver @@ -122,19 +121,29 @@ pub async fn cache_top_level_deps( } drop(info_futures); - factory - .module_load_preparer() - .await? - .prepare_module_load( + let graph_builder = factory.module_graph_builder().await?; + graph_builder + .build_graph_with_npm_resolution( graph, - &roots, - false, - deno_config::deno_json::TsTypeLib::DenoWorker, - root_permissions.clone(), - None, + CreateGraphOptions { + loader: None, + graph_kind: graph.graph_kind(), + is_dynamic: false, + roots: roots.clone(), + npm_caching: crate::graph_util::NpmCachingStrategy::Manual, + }, ) .await?; + maybe_graph_error = graph_builder.graph_roots_valid(graph, &roots); + } + + if let Some(npm_resolver) = npm_resolver.as_managed() { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; } + maybe_graph_error?; + Ok(()) } diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index 26e1eeac2f..02594f1519 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -727,7 +727,9 @@ impl ReplSession { let has_node_specifier = resolved_imports.iter().any(|url| url.scheme() == "node"); if !npm_imports.is_empty() || has_node_specifier { - npm_resolver.add_package_reqs(&npm_imports).await?; + npm_resolver + .add_and_cache_package_reqs(&npm_imports) + .await?; // prevent messages in the repl about @types/node not being cached if has_node_specifier { diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 8fab544eca..d3f7b093d4 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -198,13 +198,23 @@ pub async fn eval_command( } pub async fn maybe_npm_install(factory: &CliFactory) -> Result<(), AnyError> { + let cli_options = factory.cli_options()?; // ensure an "npm install" is done if the user has explicitly // opted into using a managed node_modules directory - if factory.cli_options()?.node_modules_dir()? - == Some(NodeModulesDirMode::Auto) - { + if cli_options.node_modules_dir()? == Some(NodeModulesDirMode::Auto) { if let Some(npm_resolver) = factory.npm_resolver().await?.as_managed() { - npm_resolver.ensure_top_level_package_json_install().await?; + let already_done = + npm_resolver.ensure_top_level_package_json_install().await?; + if !already_done + && matches!( + cli_options.default_npm_caching_strategy(), + crate::graph_util::NpmCachingStrategy::Eager + ) + { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } } } Ok(()) diff --git a/cli/tools/task.rs b/cli/tools/task.rs index fc1410aa0e..25d1d66710 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -440,6 +440,13 @@ impl<'a> TaskRunner<'a> { kill_signal: KillSignal, argv: &'a [String], ) -> Result { + if let Some(npm_resolver) = self.npm_resolver.as_managed() { + npm_resolver.ensure_top_level_package_json_install().await?; + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } + let cwd = match &self.task_flags.cwd { Some(path) => canonicalize_path(&PathBuf::from(path)) .context("failed canonicalizing --cwd")?, @@ -450,6 +457,7 @@ impl<'a> TaskRunner<'a> { self.npm_resolver, self.node_resolver, )?; + self .run_single(RunSingleOptions { task_name, @@ -473,6 +481,9 @@ impl<'a> TaskRunner<'a> { // ensure the npm packages are installed if using a managed resolver if let Some(npm_resolver) = self.npm_resolver.as_managed() { npm_resolver.ensure_top_level_package_json_install().await?; + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; } let cwd = match &self.task_flags.cwd { @@ -492,6 +503,7 @@ impl<'a> TaskRunner<'a> { self.npm_resolver, self.node_resolver, )?; + for task_name in &task_names { if let Some(script) = scripts.get(task_name) { let exit_code = self diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index 6357ebcae2..2e46bdd4da 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -1716,7 +1716,11 @@ pub async fn run_tests_with_watch( &cli_options.permissions_options(), )?; let graph = module_graph_creator - .create_graph(graph_kind, test_modules) + .create_graph( + graph_kind, + test_modules, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; module_graph_creator.graph_valid(&graph)?; let test_modules = &graph.roots; diff --git a/cli/worker.rs b/cli/worker.rs index 161d8bcc21..81b8cd2f83 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -50,6 +50,7 @@ use tokio::select; use crate::args::CliLockfile; use crate::args::DenoSubcommand; +use crate::args::NpmCachingStrategy; use crate::args::StorageKeyResolver; use crate::errors; use crate::npm::CliNpmResolver; @@ -154,6 +155,7 @@ struct SharedWorkerState { options: CliMainWorkerOptions, subcommand: DenoSubcommand, otel_config: Option, // `None` means OpenTelemetry is disabled. + default_npm_caching_strategy: NpmCachingStrategy, } impl SharedWorkerState { @@ -425,6 +427,7 @@ impl CliMainWorkerFactory { subcommand: DenoSubcommand, options: CliMainWorkerOptions, otel_config: Option, + default_npm_caching_strategy: NpmCachingStrategy, ) -> Self { Self { shared: Arc::new(SharedWorkerState { @@ -448,6 +451,7 @@ impl CliMainWorkerFactory { options, subcommand, otel_config, + default_npm_caching_strategy, }), } } @@ -487,8 +491,19 @@ impl CliMainWorkerFactory { NpmPackageReqReference::from_specifier(&main_module) { if let Some(npm_resolver) = shared.npm_resolver.as_managed() { + let reqs = &[package_ref.req().clone()]; npm_resolver - .add_package_reqs(&[package_ref.req().clone()]) + .add_package_reqs( + reqs, + if matches!( + shared.default_npm_caching_strategy, + NpmCachingStrategy::Lazy + ) { + crate::npm::PackageCaching::Only(reqs.into()) + } else { + crate::npm::PackageCaching::All + }, + ) .await?; } diff --git a/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc b/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc new file mode 100644 index 0000000000..9d24eb4de4 --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc @@ -0,0 +1,13 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "install --unstable-npm-lazy-caching --entrypoint main.ts", + "output": "install-entrypoint.out" + }, + { + "args": "install", + "output": "install.out" + } + ] +} diff --git a/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out b/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out new file mode 100644 index 0000000000..5f6c8247a5 --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out @@ -0,0 +1,6 @@ +[UNORDERED_START] +Download http://localhost:4260/@denotest%2fadd +Download http://localhost:4260/@denotest%2fsay-hello +Download http://localhost:4260/@denotest/add/1.0.0.tgz +[UNORDERED_END] +Initialize @denotest/add@1.0.0 diff --git a/tests/specs/install/entrypoint_only_used_packages/install.out b/tests/specs/install/entrypoint_only_used_packages/install.out new file mode 100644 index 0000000000..5ea41bed4c --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/install.out @@ -0,0 +1,2 @@ +Download http://localhost:4260/@denotest/say-hello/1.0.0.tgz +Initialize @denotest/say-hello@1.0.0 diff --git a/tests/specs/install/entrypoint_only_used_packages/main.ts b/tests/specs/install/entrypoint_only_used_packages/main.ts new file mode 100644 index 0000000000..1ca631410f --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/main.ts @@ -0,0 +1,3 @@ +import { add } from "@denotest/add"; + +console.log(add(1, 2)); diff --git a/tests/specs/install/entrypoint_only_used_packages/package.json b/tests/specs/install/entrypoint_only_used_packages/package.json new file mode 100644 index 0000000000..2623bcc82e --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "@denotest/add": "1.0.0", + "@denotest/say-hello": "1.0.0" + } +} diff --git a/tests/specs/run/jsx_import_source/__test__.jsonc b/tests/specs/run/jsx_import_source/__test__.jsonc index 55a895fc8f..cbda2dd32e 100644 --- a/tests/specs/run/jsx_import_source/__test__.jsonc +++ b/tests/specs/run/jsx_import_source/__test__.jsonc @@ -1,4 +1,5 @@ { + "tempDir": true, "tests": { "jsx_import_source_error": { "args": "run --config jsx/deno-jsx-error.jsonc --check jsx_import_source_no_pragma.tsx", diff --git a/tests/specs/run/lazy_npm/__test__.jsonc b/tests/specs/run/lazy_npm/__test__.jsonc new file mode 100644 index 0000000000..8212addd5c --- /dev/null +++ b/tests/specs/run/lazy_npm/__test__.jsonc @@ -0,0 +1,9 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "run --unstable-npm-lazy-caching -A main.ts", + "output": "main.out" + } + ] +} diff --git a/tests/specs/run/lazy_npm/deno.json b/tests/specs/run/lazy_npm/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/run/lazy_npm/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/specs/run/lazy_npm/main.out b/tests/specs/run/lazy_npm/main.out new file mode 100644 index 0000000000..d1aab68e90 --- /dev/null +++ b/tests/specs/run/lazy_npm/main.out @@ -0,0 +1,7 @@ +[UNORDERED_START] +Download http://localhost:4260/@denotest%2fadd +Download http://localhost:4260/@denotest%2fsay-hello +Download http://localhost:4260/@denotest/add/1.0.0.tgz +[UNORDERED_END] +Initialize @denotest/add@1.0.0 +3 diff --git a/tests/specs/run/lazy_npm/main.ts b/tests/specs/run/lazy_npm/main.ts new file mode 100644 index 0000000000..1ca631410f --- /dev/null +++ b/tests/specs/run/lazy_npm/main.ts @@ -0,0 +1,3 @@ +import { add } from "@denotest/add"; + +console.log(add(1, 2)); diff --git a/tests/specs/run/lazy_npm/package.json b/tests/specs/run/lazy_npm/package.json new file mode 100644 index 0000000000..2623bcc82e --- /dev/null +++ b/tests/specs/run/lazy_npm/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "@denotest/add": "1.0.0", + "@denotest/say-hello": "1.0.0" + } +} From 9df6be8916d9e260ba82dca1024cc84563eb3cbe Mon Sep 17 00:00:00 2001 From: denobot <33910674+denobot@users.noreply.github.com> Date: Wed, 11 Dec 2024 07:47:31 -0500 Subject: [PATCH 18/33] chore: forward v2.1.4 release commit to main (#27329) This is the release commit being forwarded back to main for 2.1.4 Co-authored-by: bartlomieju --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 8 ++--- Cargo.lock | 62 ++++++++++++++++---------------- Cargo.toml | 60 +++++++++++++++---------------- Releases.md | 20 +++++++++++ bench_util/Cargo.toml | 2 +- cli/Cargo.toml | 2 +- ext/broadcast_channel/Cargo.toml | 2 +- ext/cache/Cargo.toml | 2 +- ext/canvas/Cargo.toml | 2 +- ext/console/Cargo.toml | 2 +- ext/cron/Cargo.toml | 2 +- ext/crypto/Cargo.toml | 2 +- ext/fetch/Cargo.toml | 2 +- ext/ffi/Cargo.toml | 2 +- ext/fs/Cargo.toml | 2 +- ext/http/Cargo.toml | 2 +- ext/io/Cargo.toml | 2 +- ext/kv/Cargo.toml | 2 +- ext/napi/Cargo.toml | 2 +- ext/napi/sym/Cargo.toml | 2 +- ext/net/Cargo.toml | 2 +- ext/node/Cargo.toml | 2 +- ext/telemetry/Cargo.toml | 2 +- ext/tls/Cargo.toml | 2 +- ext/url/Cargo.toml | 2 +- ext/web/Cargo.toml | 2 +- ext/webgpu/Cargo.toml | 2 +- ext/webidl/Cargo.toml | 2 +- ext/websocket/Cargo.toml | 2 +- ext/webstorage/Cargo.toml | 2 +- resolvers/deno/Cargo.toml | 2 +- resolvers/node/Cargo.toml | 2 +- resolvers/npm_cache/Cargo.toml | 2 +- runtime/Cargo.toml | 2 +- runtime/permissions/Cargo.toml | 2 +- 36 files changed, 117 insertions(+), 97 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index cb83842c2f..9dd1f3017b 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 29; +const cacheVersion = 30; const ubuntuX86Runner = "ubuntu-24.04"; const ubuntuX86XlRunner = "ubuntu-24.04-xl"; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dc71ffa2d..838373cb78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,8 +361,8 @@ jobs: path: |- ~/.cargo/registry/index ~/.cargo/registry/cache - key: '29-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '29-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -375,7 +375,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '29-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -685,7 +685,7 @@ jobs: !./target/*/*.zip !./target/*/*.sha256sum !./target/*/*.tar.gz - key: '29-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-24.04 diff --git a/Cargo.lock b/Cargo.lock index 26936169e0..d82040854c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1215,7 +1215,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anstream", "async-trait", @@ -1388,7 +1388,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.175.0" +version = "0.176.0" dependencies = [ "bencher", "deno_core", @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.175.0" +version = "0.176.0" dependencies = [ "async-trait", "deno_core", @@ -1408,7 +1408,7 @@ dependencies = [ [[package]] name = "deno_cache" -version = "0.113.0" +version = "0.114.0" dependencies = [ "async-trait", "deno_core", @@ -1441,7 +1441,7 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.50.0" +version = "0.51.0" dependencies = [ "deno_core", "deno_webgpu", @@ -1476,7 +1476,7 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_core", ] @@ -1524,7 +1524,7 @@ checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_cron" -version = "0.61.0" +version = "0.62.0" dependencies = [ "anyhow", "async-trait", @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "deno_crypto" -version = "0.195.0" +version = "0.196.0" dependencies = [ "aes", "aes-gcm", @@ -1626,7 +1626,7 @@ dependencies = [ [[package]] name = "deno_fetch" -version = "0.205.0" +version = "0.206.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.168.0" +version = "0.169.0" dependencies = [ "deno_core", "deno_permissions", @@ -1681,7 +1681,7 @@ dependencies = [ [[package]] name = "deno_fs" -version = "0.91.0" +version = "0.92.0" dependencies = [ "async-trait", "base32", @@ -1734,7 +1734,7 @@ dependencies = [ [[package]] name = "deno_http" -version = "0.179.0" +version = "0.180.0" dependencies = [ "async-compression", "async-trait", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.91.0" +version = "0.92.0" dependencies = [ "async-trait", "deno_core", @@ -1794,7 +1794,7 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.89.0" +version = "0.90.0" dependencies = [ "anyhow", "async-trait", @@ -1867,7 +1867,7 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.112.0" +version = "0.113.0" dependencies = [ "deno_core", "deno_permissions", @@ -1895,7 +1895,7 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.173.0" +version = "0.174.0" dependencies = [ "deno_core", "deno_permissions", @@ -1912,7 +1912,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.118.0" +version = "0.119.0" dependencies = [ "aead-gcm-stream", "aes", @@ -2023,7 +2023,7 @@ dependencies = [ [[package]] name = "deno_npm_cache" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "async-trait", @@ -2095,7 +2095,7 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.41.0" +version = "0.42.0" dependencies = [ "capacity_builder", "deno_core", @@ -2114,7 +2114,7 @@ dependencies = [ [[package]] name = "deno_resolver" -version = "0.13.0" +version = "0.14.0" dependencies = [ "anyhow", "base32", @@ -2133,7 +2133,7 @@ dependencies = [ [[package]] name = "deno_runtime" -version = "0.190.0" +version = "0.191.0" dependencies = [ "color-print", "deno_ast", @@ -2234,7 +2234,7 @@ dependencies = [ [[package]] name = "deno_telemetry" -version = "0.3.0" +version = "0.4.0" dependencies = [ "async-trait", "deno_core", @@ -2275,7 +2275,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.168.0" +version = "0.169.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2325,7 +2325,7 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_bench_util", "deno_console", @@ -2337,7 +2337,7 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.212.0" +version = "0.213.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2359,7 +2359,7 @@ dependencies = [ [[package]] name = "deno_webgpu" -version = "0.148.0" +version = "0.149.0" dependencies = [ "deno_core", "raw-window-handle", @@ -2372,7 +2372,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2380,7 +2380,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.186.0" +version = "0.187.0" dependencies = [ "bytes", "deno_core", @@ -2402,7 +2402,7 @@ dependencies = [ [[package]] name = "deno_webstorage" -version = "0.176.0" +version = "0.177.0" dependencies = [ "deno_core", "deno_web", @@ -4917,7 +4917,7 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.111.0" +version = "0.112.0" dependencies = [ "quote", "serde", @@ -4972,7 +4972,7 @@ dependencies = [ [[package]] name = "node_resolver" -version = "0.20.0" +version = "0.21.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index c3703d8c5c..712bdae8b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,17 +50,17 @@ repository = "https://github.com/denoland/deno" deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.324.0" } -deno_bench_util = { version = "0.175.0", path = "./bench_util" } +deno_bench_util = { version = "0.176.0", path = "./bench_util" } deno_config = { version = "=0.39.3", features = ["workspace", "sync"] } deno_lockfile = "=0.23.2" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.26.0" deno_path_util = "=0.2.1" -deno_permissions = { version = "0.41.0", path = "./runtime/permissions" } -deno_runtime = { version = "0.190.0", path = "./runtime" } +deno_permissions = { version = "0.42.0", path = "./runtime/permissions" } +deno_runtime = { version = "0.191.0", path = "./runtime" } deno_semver = "=0.6.1" deno_terminal = "0.2.0" -napi_sym = { version = "0.111.0", path = "./ext/napi/sym" } +napi_sym = { version = "0.112.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.4" @@ -69,34 +69,34 @@ denokv_remote = "0.8.4" denokv_sqlite = { default-features = false, version = "0.8.4" } # exts -deno_broadcast_channel = { version = "0.175.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.113.0", path = "./ext/cache" } -deno_canvas = { version = "0.50.0", path = "./ext/canvas" } -deno_console = { version = "0.181.0", path = "./ext/console" } -deno_cron = { version = "0.61.0", path = "./ext/cron" } -deno_crypto = { version = "0.195.0", path = "./ext/crypto" } -deno_fetch = { version = "0.205.0", path = "./ext/fetch" } -deno_ffi = { version = "0.168.0", path = "./ext/ffi" } -deno_fs = { version = "0.91.0", path = "./ext/fs" } -deno_http = { version = "0.179.0", path = "./ext/http" } -deno_io = { version = "0.91.0", path = "./ext/io" } -deno_kv = { version = "0.89.0", path = "./ext/kv" } -deno_napi = { version = "0.112.0", path = "./ext/napi" } -deno_net = { version = "0.173.0", path = "./ext/net" } -deno_node = { version = "0.118.0", path = "./ext/node" } -deno_telemetry = { version = "0.3.0", path = "./ext/telemetry" } -deno_tls = { version = "0.168.0", path = "./ext/tls" } -deno_url = { version = "0.181.0", path = "./ext/url" } -deno_web = { version = "0.212.0", path = "./ext/web" } -deno_webgpu = { version = "0.148.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.181.0", path = "./ext/webidl" } -deno_websocket = { version = "0.186.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.176.0", path = "./ext/webstorage" } +deno_broadcast_channel = { version = "0.176.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.114.0", path = "./ext/cache" } +deno_canvas = { version = "0.51.0", path = "./ext/canvas" } +deno_console = { version = "0.182.0", path = "./ext/console" } +deno_cron = { version = "0.62.0", path = "./ext/cron" } +deno_crypto = { version = "0.196.0", path = "./ext/crypto" } +deno_fetch = { version = "0.206.0", path = "./ext/fetch" } +deno_ffi = { version = "0.169.0", path = "./ext/ffi" } +deno_fs = { version = "0.92.0", path = "./ext/fs" } +deno_http = { version = "0.180.0", path = "./ext/http" } +deno_io = { version = "0.92.0", path = "./ext/io" } +deno_kv = { version = "0.90.0", path = "./ext/kv" } +deno_napi = { version = "0.113.0", path = "./ext/napi" } +deno_net = { version = "0.174.0", path = "./ext/net" } +deno_node = { version = "0.119.0", path = "./ext/node" } +deno_telemetry = { version = "0.4.0", path = "./ext/telemetry" } +deno_tls = { version = "0.169.0", path = "./ext/tls" } +deno_url = { version = "0.182.0", path = "./ext/url" } +deno_web = { version = "0.213.0", path = "./ext/web" } +deno_webgpu = { version = "0.149.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.182.0", path = "./ext/webidl" } +deno_websocket = { version = "0.187.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.177.0", path = "./ext/webstorage" } # resolvers -deno_npm_cache = { version = "0.1.0", path = "./resolvers/npm_cache" } -deno_resolver = { version = "0.13.0", path = "./resolvers/deno" } -node_resolver = { version = "0.20.0", path = "./resolvers/node" } +deno_npm_cache = { version = "0.2.0", path = "./resolvers/npm_cache" } +deno_resolver = { version = "0.14.0", path = "./resolvers/deno" } +node_resolver = { version = "0.21.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" diff --git a/Releases.md b/Releases.md index e395934ac1..aaae202a37 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,26 @@ https://github.com/denoland/deno/releases We also have one-line install commands at: https://github.com/denoland/deno_install +### 2.1.4 / 2024.12.11 + +- feat(unstable): support caching npm dependencies only as they're needed + (#27300) +- fix(compile): correct read length for transpiled typescript files (#27301) +- fix(ext/node): accept file descriptor in fs.readFile(Sync) (#27252) +- fix(ext/node): handle Float16Array in node:v8 module (#27285) +- fix(lint): do not error providing --allow-import (#27321) +- fix(node): update list of builtin node modules, add missing export to + _http_common (#27294) +- fix(outdated): error when there are no config files (#27306) +- fix(outdated): respect --quiet flag for hints (#27317) +- fix(outdated): show a suggestion for updating (#27304) +- fix(task): do not always kill child on ctrl+c on windows (#27269) +- fix(unstable): don't unwrap optional state in otel (#27292) +- fix: do not error when subpath has an @ symbol (#27290) +- fix: do not panic when fetching invalid file url on Windows (#27259) +- fix: replace the @deno-types with @ts-types (#27310) +- perf(compile): improve FileBackedVfsFile (#27299) + ### 2.1.3 / 2024.12.05 - feat(unstable): add metrics to otel (#27143) diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index 65c51c24d5..9833996fd3 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.175.0" +version = "0.176.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4b4b17292c..cf76dfe69d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.1.3" +version = "2.1.4" authors.workspace = true default-run = "deno" edition.workspace = true diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index fccec9a66a..5b238aad25 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.175.0" +version = "0.176.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index bf067c8e32..d03779d364 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.113.0" +version = "0.114.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 5ec468ec18..c851b7724a 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.50.0" +version = "0.51.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index 7e827efc61..4a26917933 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index 3762ace1a5..d8f2d949f7 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.61.0" +version = "0.62.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index f13e0a3c66..63656bf642 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.195.0" +version = "0.196.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 959fac574a..716d268a04 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.205.0" +version = "0.206.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 23c71d0030..d54249329d 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index f9e3359d07..d11520ad8c 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.91.0" +version = "0.92.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index 4a398db859..27a91ca61b 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.179.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index f5d9e47245..1b73bad348 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.91.0" +version = "0.92.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index c0e030e3c8..c69a962fa3 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.89.0" +version = "0.90.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index 5bd86d31b4..783b4b7cff 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.112.0" +version = "0.113.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml index 478443e78f..a3dd56e2bf 100644 --- a/ext/napi/sym/Cargo.toml +++ b/ext/napi/sym/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "napi_sym" -version = "0.111.0" +version = "0.112.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index efe46a79e7..f5aa32c8ce 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.173.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 9cd09f6fde..1bfe3a4d8d 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.118.0" +version = "0.119.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/telemetry/Cargo.toml b/ext/telemetry/Cargo.toml index 87b330b322..f3d4bbd336 100644 --- a/ext/telemetry/Cargo.toml +++ b/ext/telemetry/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_telemetry" -version = "0.3.0" +version = "0.4.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index 239c8f0834..fc75b0a05d 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index 8d0f951869..d30332d0ff 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index c05bc2889c..ac2b14fbed 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.212.0" +version = "0.213.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index a6eea01ced..9621c085e8 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.148.0" +version = "0.149.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index 07ee546be8..38eff7b66f 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index 5ba2172414..cb72618cad 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.186.0" +version = "0.187.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index 5e74700b84..700a252016 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.176.0" +version = "0.177.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index ce5bee3839..2966b5fef6 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_resolver" -version = "0.13.0" +version = "0.14.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 111d67ad59..e175bcfafa 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "node_resolver" -version = "0.20.0" +version = "0.21.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/npm_cache/Cargo.toml b/resolvers/npm_cache/Cargo.toml index fab102dee8..1cc7237025 100644 --- a/resolvers/npm_cache/Cargo.toml +++ b/resolvers/npm_cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_npm_cache" -version = "0.1.0" +version = "0.2.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 317e619bf7..cb12abb141 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_runtime" -version = "0.190.0" +version = "0.191.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index 5be45377a0..dc46b03310 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_permissions" -version = "0.41.0" +version = "0.42.0" authors.workspace = true edition.workspace = true license.workspace = true From c6fa62896d2b1f6b4661659fe977ef4e2feffd9a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 11 Dec 2024 09:40:50 -0500 Subject: [PATCH 19/33] fix(compile): output contents of embedded file system (#27302) --- cli/standalone/binary.rs | 59 +++--- cli/standalone/virtual_fs.rs | 168 ++++++++++++++++++ cli/tools/compile.rs | 53 +++--- cli/tools/info.rs | 100 ++--------- cli/util/display.rs | 73 ++++++++ tests/integration/compile_tests.rs | 5 + .../compile/env_vars_support/compile.out | 6 + .../compile/package_json_type/compile.out | 4 + .../main_compile_file.out | 6 +- .../main_compile_folder.out | 6 +- 10 files changed, 344 insertions(+), 136 deletions(-) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index dec1d89110..7728728a26 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -87,6 +87,7 @@ use super::serialization::DenoCompileModuleData; use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; +use super::virtual_fs::output_vfs; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; @@ -367,6 +368,16 @@ pub fn extract_standalone( })) } +pub struct WriteBinOptions<'a> { + pub writer: File, + pub display_output_filename: &'a str, + pub graph: &'a ModuleGraph, + pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>, + pub entrypoint: &'a ModuleSpecifier, + pub include_files: &'a [ModuleSpecifier], + pub compile_flags: &'a CompileFlags, +} + pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CjsTracker, cli_options: &'a CliOptions, @@ -407,18 +418,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub async fn write_bin( &self, - writer: File, - graph: &ModuleGraph, - root_dir_url: StandaloneRelativeFileBaseUrl<'_>, - entrypoint: &ModuleSpecifier, - include_files: &[ModuleSpecifier], - compile_flags: &CompileFlags, + options: WriteBinOptions<'_>, ) -> Result<(), AnyError> { // Select base binary based on target - let mut original_binary = self.get_base_binary(compile_flags).await?; + let mut original_binary = + self.get_base_binary(options.compile_flags).await?; - if compile_flags.no_terminal { - let target = compile_flags.resolve_target(); + if options.compile_flags.no_terminal { + let target = options.compile_flags.resolve_target(); if !target.contains("windows") { bail!( "The `--no-terminal` flag is only available when targeting Windows (current: {})", @@ -428,8 +435,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { set_windows_binary_to_gui(&mut original_binary) .context("Setting windows binary to GUI.")?; } - if compile_flags.icon.is_some() { - let target = compile_flags.resolve_target(); + if options.compile_flags.icon.is_some() { + let target = options.compile_flags.resolve_target(); if !target.contains("windows") { bail!( "The `--icon` flag is only available when targeting Windows (current: {})", @@ -437,17 +444,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self - .write_standalone_binary( - writer, - original_binary, - graph, - root_dir_url, - entrypoint, - include_files, - compile_flags, - ) - .await + self.write_standalone_binary(options, original_binary).await } async fn get_base_binary( @@ -552,14 +549,18 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] async fn write_standalone_binary( &self, - writer: File, + options: WriteBinOptions<'_>, original_bin: Vec, - graph: &ModuleGraph, - root_dir_url: StandaloneRelativeFileBaseUrl<'_>, - entrypoint: &ModuleSpecifier, - include_files: &[ModuleSpecifier], - compile_flags: &CompileFlags, ) -> Result<(), AnyError> { + let WriteBinOptions { + writer, + display_output_filename, + graph, + root_dir_url, + entrypoint, + include_files, + compile_flags, + } = options; let ca_data = match self.cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?, @@ -784,6 +785,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { otel_config: self.cli_options.otel_config(), }; + output_vfs(&vfs, display_output_filename); + write_binary_bytes( writer, original_bin, diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 814774b71b..ce7c0bb625 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -31,6 +31,7 @@ use serde::Serialize; use thiserror::Error; use crate::util; +use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; #[derive(Debug, Copy, Clone)] @@ -51,6 +52,7 @@ pub struct StripRootError { target: PathBuf, } +#[derive(Debug)] pub struct VfsBuilder { root_path: PathBuf, root_dir: VirtualDirectory, @@ -364,6 +366,125 @@ impl VfsBuilder { } } +pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { + if !log::log_enabled!(log::Level::Info) { + return; // no need to compute if won't output + } + + if builder.root_dir.entries.is_empty() { + return; // nothing to output + } + + let mut text = String::new(); + let display_tree = vfs_as_display_tree(builder, executable_name); + display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string + log::info!( + "\n{}\n", + deno_terminal::colors::bold("Embedded File System") + ); + log::info!("{}\n", text.trim()); +} + +fn vfs_as_display_tree( + builder: &VfsBuilder, + executable_name: &str, +) -> DisplayTreeNode { + enum EntryOutput<'a> { + All, + Subset(Vec>), + File, + Symlink(&'a [String]), + } + + impl<'a> EntryOutput<'a> { + pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + DisplayTreeNode { + text: match self { + EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name, + EntryOutput::Symlink(parts) => { + format!("{} --> {}", name, parts.join("/")) + } + }, + children: match self { + EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())], + EntryOutput::Subset(vec) => vec + .iter() + .map(|e| e.output.as_display_tree(e.name.to_string())) + .collect(), + EntryOutput::File | EntryOutput::Symlink(_) => vec![], + }, + } + } + } + + pub struct DirEntryOutput<'a> { + name: &'a str, + output: EntryOutput<'a>, + } + + fn include_all_entries<'a>( + dir: &Path, + vfs_dir: &'a VirtualDirectory, + ) -> EntryOutput<'a> { + EntryOutput::Subset( + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(&dir.join(entry.name()), entry), + }) + .collect(), + ) + } + + fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> { + match entry { + VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts) + } + } + } + + fn analyze_dir<'a>( + dir: &Path, + vfs_dir: &'a VirtualDirectory, + ) -> EntryOutput<'a> { + let real_entry_count = std::fs::read_dir(dir) + .ok() + .map(|entries| entries.flat_map(|e| e.ok()).count()) + .unwrap_or(0); + if real_entry_count == vfs_dir.entries.len() { + let children = vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(&dir.join(entry.name()), entry), + }) + .collect::>(); + if children + .iter() + .all(|c| !matches!(c.output, EntryOutput::Subset(_))) + { + EntryOutput::All + } else { + EntryOutput::Subset(children) + } + } else { + include_all_entries(dir, vfs_dir) + } + } + + // always include all the entries for the root directory, otherwise the + // user might not have context about what's being shown + let output = include_all_entries(&builder.root_path, &builder.root_dir); + output + .as_display_tree(deno_terminal::colors::italic(executable_name).to_string()) +} + #[derive(Debug)] enum VfsEntryRef<'a> { Dir(&'a VirtualDirectory), @@ -1013,6 +1134,7 @@ impl FileBackedVfs { #[cfg(test)] mod test { + use console_static_text::ansi::strip_ansi_codes; use std::io::Write; use test_util::TempDir; @@ -1299,4 +1421,50 @@ mod test { .unwrap(); assert_eq!(all_buf.to_vec(), b"123456789"); } + + #[test] + fn test_vfs_as_display_tree() { + let temp_dir = TempDir::new(); + temp_dir.write("root.txt", ""); + temp_dir.create_dir_all("a"); + temp_dir.write("a/a.txt", ""); + temp_dir.write("a/b.txt", ""); + temp_dir.create_dir_all("b"); + temp_dir.write("b/a.txt", ""); + temp_dir.write("b/b.txt", ""); + temp_dir.create_dir_all("c"); + temp_dir.write("c/a.txt", "contents"); + temp_dir.symlink_file("c/a.txt", "c/b.txt"); + assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works + let mut vfs_builder = + VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap(); + // full dir + vfs_builder + .add_dir_recursive(temp_dir.path().join("a").as_path()) + .unwrap(); + // part of the dir + vfs_builder + .add_file_at_path(temp_dir.path().join("b/a.txt").as_path()) + .unwrap(); + // symlink + vfs_builder + .add_dir_recursive(temp_dir.path().join("c").as_path()) + .unwrap(); + temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory + let node = vfs_as_display_tree(&vfs_builder, "executable"); + let mut text = String::new(); + node.print(&mut text).unwrap(); + assert_eq!( + strip_ansi_codes(&text), + r#"executable +├─┬ a +│ └── * +├─┬ b +│ └── a.txt +└─┬ c + ├── a.txt + └── b.txt --> c/a.txt +"# + ); + } } diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 5cd19ecf71..4d0607ba71 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -6,6 +6,7 @@ use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; use crate::standalone::binary::StandaloneRelativeFileBaseUrl; +use crate::standalone::binary::WriteBinOptions; use crate::standalone::is_standalone_binary; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; @@ -15,6 +16,7 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; +use deno_path_util::url_from_file_path; use deno_terminal::colors; use rand::Rng; use std::path::Path; @@ -93,7 +95,16 @@ pub async fn compile( .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) .iter(), ) - .chain(include_files.iter()), + .chain(include_files.iter()) + .chain( + // sometimes the import map path is outside the root dir + cli_options + .workspace() + .to_import_map_path() + .ok() + .and_then(|p| p.and_then(|p| url_from_file_path(&p).ok())) + .iter(), + ), ); log::debug!("Binary root dir: {}", root_dir_url); log::info!( @@ -120,14 +131,18 @@ pub async fn compile( })?; let write_result = binary_writer - .write_bin( - file, - &graph, - StandaloneRelativeFileBaseUrl::from(&root_dir_url), + .write_bin(WriteBinOptions { + writer: file, + display_output_filename: &output_path + .file_name() + .unwrap() + .to_string_lossy(), + graph: &graph, + root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url), entrypoint, - &include_files, - &compile_flags, - ) + include_files: &include_files, + compile_flags: &compile_flags, + }) .await .with_context(|| { format!( @@ -368,17 +383,6 @@ fn resolve_root_dir_from_specifiers<'a>( } } } - let found_dir = if is_file_system_root(found_dir) { - found_dir - } else { - // include the parent dir name because it helps create some context - found_dir - .strip_suffix('/') - .unwrap_or(found_dir) - .rfind('/') - .map(|i| &found_dir[..i + 1]) - .unwrap_or(found_dir) - }; ModuleSpecifier::parse(found_dir).unwrap() } @@ -473,14 +477,17 @@ mod test { .to_string() } - assert_eq!(resolve("file:///a/b/c", &["file:///a/b/c/d"]), "file:///a/"); assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d"]), + resolve("file:///a/b/e", &["file:///a/b/c/d"]), "file:///a/b/" ); + assert_eq!( + resolve("file:///a/b/c/", &["file:///a/b/c/d"]), + "file:///a/b/c/" + ); assert_eq!( resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]), - "file:///a/b/" + "file:///a/b/c/" ); assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///"); if cfg!(windows) { @@ -488,7 +495,7 @@ mod test { // this will ignore the other one because it's on a separate drive assert_eq!( resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]), - "file:///c:/a/b/" + "file:///c:/a/b/c/" ); } } diff --git a/cli/tools/info.rs b/cli/tools/info.rs index f58732a904..7a35f597c3 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::collections::HashSet; -use std::fmt; use std::fmt::Write; use std::sync::Arc; @@ -35,6 +34,7 @@ use crate::graph_util::graph_exit_integrity_errors; use crate::npm::CliNpmResolver; use crate::npm::ManagedCliNpmResolver; use crate::util::checksum; +use crate::util::display::DisplayTreeNode; const JSON_SCHEMA_VERSION: u8 = 1; @@ -342,76 +342,6 @@ fn add_npm_packages_to_json( json.insert("npmPackages".to_string(), json_packages.into()); } -struct TreeNode { - text: String, - children: Vec, -} - -impl TreeNode { - pub fn from_text(text: String) -> Self { - Self { - text, - children: Default::default(), - } - } -} - -fn print_tree_node( - tree_node: &TreeNode, - writer: &mut TWrite, -) -> fmt::Result { - fn print_children( - writer: &mut TWrite, - prefix: &str, - children: &[TreeNode], - ) -> fmt::Result { - const SIBLING_CONNECTOR: char = '├'; - const LAST_SIBLING_CONNECTOR: char = '└'; - const CHILD_DEPS_CONNECTOR: char = '┬'; - const CHILD_NO_DEPS_CONNECTOR: char = '─'; - const VERTICAL_CONNECTOR: char = '│'; - const EMPTY_CONNECTOR: char = ' '; - - let child_len = children.len(); - for (index, child) in children.iter().enumerate() { - let is_last = index + 1 == child_len; - let sibling_connector = if is_last { - LAST_SIBLING_CONNECTOR - } else { - SIBLING_CONNECTOR - }; - let child_connector = if child.children.is_empty() { - CHILD_NO_DEPS_CONNECTOR - } else { - CHILD_DEPS_CONNECTOR - }; - writeln!( - writer, - "{} {}", - colors::gray(format!("{prefix}{sibling_connector}─{child_connector}")), - child.text - )?; - let child_prefix = format!( - "{}{}{}", - prefix, - if is_last { - EMPTY_CONNECTOR - } else { - VERTICAL_CONNECTOR - }, - EMPTY_CONNECTOR - ); - print_children(writer, &child_prefix, &child.children)?; - } - - Ok(()) - } - - writeln!(writer, "{}", tree_node.text)?; - print_children(writer, "", &tree_node.children)?; - Ok(()) -} - /// Precached information about npm packages that are used in deno info. #[derive(Default)] struct NpmInfo { @@ -568,7 +498,7 @@ impl<'a> GraphDisplayContext<'a> { )?; writeln!(writer)?; let root_node = self.build_module_info(root, false); - print_tree_node(&root_node, writer)?; + root_node.print(writer)?; Ok(()) } Err(err) => { @@ -584,7 +514,7 @@ impl<'a> GraphDisplayContext<'a> { } } - fn build_dep_info(&mut self, dep: &Dependency) -> Vec { + fn build_dep_info(&mut self, dep: &Dependency) -> Vec { let mut children = Vec::with_capacity(2); if !dep.maybe_code.is_none() { if let Some(child) = self.build_resolved_info(&dep.maybe_code, false) { @@ -599,7 +529,11 @@ impl<'a> GraphDisplayContext<'a> { children } - fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { + fn build_module_info( + &mut self, + module: &Module, + type_dep: bool, + ) -> DisplayTreeNode { enum PackageOrSpecifier { Package(Box), Specifier(ModuleSpecifier), @@ -645,7 +579,7 @@ impl<'a> GraphDisplayContext<'a> { format!("{} {}", header_text, maybe_size_to_text(maybe_size)) }; - let mut tree_node = TreeNode::from_text(header_text); + let mut tree_node = DisplayTreeNode::from_text(header_text); if !was_seen { match &package_or_specifier { @@ -683,14 +617,14 @@ impl<'a> GraphDisplayContext<'a> { fn build_npm_deps( &mut self, package: &NpmResolutionPackage, - ) -> Vec { + ) -> Vec { let mut deps = package.dependencies.values().collect::>(); deps.sort(); let mut children = Vec::with_capacity(deps.len()); for dep_id in deps.into_iter() { let maybe_size = self.npm_info.package_sizes.get(dep_id).cloned(); let size_str = maybe_size_to_text(maybe_size); - let mut child = TreeNode::from_text(format!( + let mut child = DisplayTreeNode::from_text(format!( "npm:/{} {}", dep_id.as_serialized(), size_str @@ -715,7 +649,7 @@ impl<'a> GraphDisplayContext<'a> { &mut self, err: &ModuleError, specifier: &ModuleSpecifier, - ) -> TreeNode { + ) -> DisplayTreeNode { self.seen.insert(specifier.to_string()); match err { ModuleError::InvalidTypeAssertion { .. } => { @@ -758,8 +692,8 @@ impl<'a> GraphDisplayContext<'a> { &self, specifier: &ModuleSpecifier, error_msg: &str, - ) -> TreeNode { - TreeNode::from_text(format!( + ) -> DisplayTreeNode { + DisplayTreeNode::from_text(format!( "{} {}", colors::red(specifier), colors::red_bold(error_msg) @@ -770,7 +704,7 @@ impl<'a> GraphDisplayContext<'a> { &mut self, resolution: &Resolution, type_dep: bool, - ) -> Option { + ) -> Option { match resolution { Resolution::Ok(resolved) => { let specifier = &resolved.specifier; @@ -778,14 +712,14 @@ impl<'a> GraphDisplayContext<'a> { Some(match self.graph.try_get(resolved_specifier) { Ok(Some(module)) => self.build_module_info(module, type_dep), Err(err) => self.build_error_info(err, resolved_specifier), - Ok(None) => TreeNode::from_text(format!( + Ok(None) => DisplayTreeNode::from_text(format!( "{} {}", colors::red(specifier), colors::red_bold("(missing)") )), }) } - Resolution::Err(err) => Some(TreeNode::from_text(format!( + Resolution::Err(err) => Some(DisplayTreeNode::from_text(format!( "{} {}", colors::italic(err.to_string()), colors::red_bold("(resolve error)") diff --git a/cli/util/display.rs b/cli/util/display.rs index d18b045d8d..8795d3db68 100644 --- a/cli/util/display.rs +++ b/cli/util/display.rs @@ -2,6 +2,7 @@ use deno_core::error::AnyError; use deno_core::serde_json; +use deno_runtime::colors; use std::io::Write; /// A function that converts a float to a string the represents a human @@ -85,6 +86,78 @@ where Ok(()) } +pub struct DisplayTreeNode { + pub text: String, + pub children: Vec, +} + +impl DisplayTreeNode { + pub fn from_text(text: String) -> Self { + Self { + text, + children: Default::default(), + } + } + + pub fn print( + &self, + writer: &mut TWrite, + ) -> std::fmt::Result { + fn print_children( + writer: &mut TWrite, + prefix: &str, + children: &[DisplayTreeNode], + ) -> std::fmt::Result { + const SIBLING_CONNECTOR: char = '├'; + const LAST_SIBLING_CONNECTOR: char = '└'; + const CHILD_DEPS_CONNECTOR: char = '┬'; + const CHILD_NO_DEPS_CONNECTOR: char = '─'; + const VERTICAL_CONNECTOR: char = '│'; + const EMPTY_CONNECTOR: char = ' '; + + let child_len = children.len(); + for (index, child) in children.iter().enumerate() { + let is_last = index + 1 == child_len; + let sibling_connector = if is_last { + LAST_SIBLING_CONNECTOR + } else { + SIBLING_CONNECTOR + }; + let child_connector = if child.children.is_empty() { + CHILD_NO_DEPS_CONNECTOR + } else { + CHILD_DEPS_CONNECTOR + }; + writeln!( + writer, + "{} {}", + colors::gray(format!( + "{prefix}{sibling_connector}─{child_connector}" + )), + child.text + )?; + let child_prefix = format!( + "{}{}{}", + prefix, + if is_last { + EMPTY_CONNECTOR + } else { + VERTICAL_CONNECTOR + }, + EMPTY_CONNECTOR + ); + print_children(writer, &child_prefix, &child.children)?; + } + + Ok(()) + } + + writeln!(writer, "{}", self.text)?; + print_children(writer, "", &self.children)?; + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index fa6364a136..62c5cf8fab 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -1119,6 +1119,11 @@ Compile file:///[WILDCARD]/main.ts to [WILDCARD] Warning Failed resolving symlink. Ignoring. Path: [WILDCARD] Message: [WILDCARD]) + +Embedded File System + +[WILDCARD] + "#, ); diff --git a/tests/specs/compile/env_vars_support/compile.out b/tests/specs/compile/env_vars_support/compile.out index 2d004e7cb1..02332ea0eb 100644 --- a/tests/specs/compile/env_vars_support/compile.out +++ b/tests/specs/compile/env_vars_support/compile.out @@ -2,3 +2,9 @@ Warning Parsing failed within the specified environment file: environment.env at Check [WILDCARD]main.ts Compile [WILDCARD]main.ts to out[WILDCARD] Warning Environment variables from the file "environment.env" were embedded in the generated executable file + +Embedded File System + +out[WILDLINE] +└── main.ts + diff --git a/tests/specs/compile/package_json_type/compile.out b/tests/specs/compile/package_json_type/compile.out index 913e363c3e..0370192809 100644 --- a/tests/specs/compile/package_json_type/compile.out +++ b/tests/specs/compile/package_json_type/compile.out @@ -1,2 +1,6 @@ Check file:///[WILDLINE]/main.js Compile file:///[WILDLINE] + +Embedded File System + +[WILDCARD] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index e5b39a7527..70de321361 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,2 +1,6 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Warning Symlink target is outside '[WILDCARD]compile'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. +Warning Symlink target is outside '[WILDCARD]node_modules_symlink_outside'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. + +Embedded File System + +[WILDCARD] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 1a7eb0a4f2..205c6a9281 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -3,4 +3,8 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Warning Symlink target is outside '[WILDLINE]compile'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. +Warning Symlink target is outside '[WILDLINE]node_modules_symlink_outside'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. + +Embedded File System + +[WILDCARD] From 5f8be055db1b276d2bb2d986ac0780997ddc68c4 Mon Sep 17 00:00:00 2001 From: snek Date: Thu, 12 Dec 2024 09:17:26 +0100 Subject: [PATCH 20/33] refactor(unstable): otel configuration (#27333) split up otel config into user configurable and runtime configurable parts. user configurable part is now set via env vars parsed according to the otel spec. otel is now enabled via `OTEL_DENO=true`, and `--unstable-otel` only acts as a guard. Fixes: https://github.com/denoland/deno/issues/27273 --- cli/args/flags.rs | 47 +++++++++++++++++++-------- cli/args/mod.rs | 10 +++++- cli/main.rs | 10 +++--- cli/mainrt.rs | 11 ++++--- cli/standalone/binary.rs | 2 +- cli/util/logger.rs | 51 +++++++++++++++++++++++------- cli/worker.rs | 4 +-- ext/telemetry/lib.rs | 32 ++++++++++++------- ext/telemetry/telemetry.ts | 10 +++--- runtime/worker_bootstrap.rs | 11 ++----- tests/specs/cli/otel_basic/main.ts | 1 + 11 files changed, 124 insertions(+), 65 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 9418739564..418edcf34b 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -37,6 +37,7 @@ use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsOptions; use deno_runtime::deno_permissions::SysDescriptor; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use log::debug; use log::Level; use serde::Deserialize; @@ -986,21 +987,41 @@ impl Flags { args } - pub fn otel_config(&self) -> Option { - if self + pub fn otel_config(&self) -> OtelConfig { + let has_unstable_flag = self .unstable_config .features - .contains(&String::from("otel")) - { - Some(OtelConfig { - runtime_name: Cow::Borrowed("deno"), - runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), - deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") - .is_ok(), - ..Default::default() - }) - } else { - None + .contains(&String::from("otel")); + + let otel_var = |name| match std::env::var(name) { + Ok(s) if s.to_lowercase() == "true" => Some(true), + Ok(s) if s.to_lowercase() == "false" => Some(false), + _ => None, + }; + + let disabled = + !has_unstable_flag || otel_var("OTEL_SDK_DISABLED").unwrap_or(false); + let default = !disabled && otel_var("OTEL_DENO").unwrap_or(false); + + OtelConfig { + tracing_enabled: !disabled + && otel_var("OTEL_DENO_TRACING").unwrap_or(default), + console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() { + Ok(_) if disabled => OtelConsoleConfig::Ignore, + Ok("ignore") => OtelConsoleConfig::Ignore, + Ok("capture") => OtelConsoleConfig::Capture, + Ok("replace") => OtelConsoleConfig::Replace, + _ => { + if default { + OtelConsoleConfig::Capture + } else { + OtelConsoleConfig::Ignore + } + } + }, + deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") + .as_deref() + == Ok("1"), } } diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 314c0ff17a..71f79e12e0 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -31,6 +31,7 @@ use deno_npm_cache::NpmCacheSetting; use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelRuntimeConfig; use import_map::resolve_import_map_value_from_specifier; pub use deno_config::deno_json::BenchConfig; @@ -1130,7 +1131,7 @@ impl CliOptions { } } - pub fn otel_config(&self) -> Option { + pub fn otel_config(&self) -> OtelConfig { self.flags.otel_config() } @@ -2000,6 +2001,13 @@ pub enum NpmCachingStrategy { Manual, } +pub(crate) fn otel_runtime_config() -> OtelRuntimeConfig { + OtelRuntimeConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + } +} + #[cfg(test)] mod test { use pretty_assertions::assert_eq; diff --git a/cli/main.rs b/cli/main.rs index d47f1e363c..0594739fd8 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -437,20 +437,18 @@ fn resolve_flags_and_init( if err.kind() == clap::error::ErrorKind::DisplayVersion => { // Ignore results to avoid BrokenPipe errors. - util::logger::init(None); + util::logger::init(None, None); let _ = err.print(); deno_runtime::exit(0); } Err(err) => { - util::logger::init(None); + util::logger::init(None, None); exit_for_error(AnyError::from(err)) } }; - if let Some(otel_config) = flags.otel_config() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(flags.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init(flags.log_level, Some(flags.otel_config())); // TODO(bartlomieju): remove in Deno v2.5 and hard error then. if flags.unstable_config.legacy_flag_enabled { diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 7ad3b3744b..18142bd0e7 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -87,17 +87,18 @@ fn main() { let future = async move { match standalone { Ok(Some(data)) => { - if let Some(otel_config) = data.metadata.otel_config.clone() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(data.metadata.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init( + data.metadata.log_level, + Some(data.metadata.otel_config.clone()), + ); load_env_vars(&data.metadata.env_vars_from_env_file); let exit_code = standalone::run(data).await?; deno_runtime::exit(exit_code); } Ok(None) => Ok(()), Err(err) => { - util::logger::init(None); + util::logger::init(None, None); Err(err) } } diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 7728728a26..1e7b0d3f70 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -192,7 +192,7 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, - pub otel_config: Option, // None means disabled. + pub otel_config: OtelConfig, } fn write_binary_bytes( diff --git a/cli/util/logger.rs b/cli/util/logger.rs index 2b8987c3e7..783f8a5f68 100644 --- a/cli/util/logger.rs +++ b/cli/util/logger.rs @@ -1,24 +1,34 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use super::draw_thread::DrawThread; +use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use std::io::Write; -use super::draw_thread::DrawThread; - -struct CliLogger(env_logger::Logger); +struct CliLogger { + otel_console_config: OtelConsoleConfig, + logger: env_logger::Logger, +} impl CliLogger { - pub fn new(logger: env_logger::Logger) -> Self { - Self(logger) + pub fn new( + logger: env_logger::Logger, + otel_console_config: OtelConsoleConfig, + ) -> Self { + Self { + logger, + otel_console_config, + } } pub fn filter(&self) -> log::LevelFilter { - self.0.filter() + self.logger.filter() } } impl log::Log for CliLogger { fn enabled(&self, metadata: &log::Metadata) -> bool { - self.0.enabled(metadata) + self.logger.enabled(metadata) } fn log(&self, record: &log::Record) { @@ -28,18 +38,30 @@ impl log::Log for CliLogger { // could potentially block other threads that access the draw // thread's state DrawThread::hide(); - self.0.log(record); - deno_telemetry::handle_log(record); + + match self.otel_console_config { + OtelConsoleConfig::Ignore => { + self.logger.log(record); + } + OtelConsoleConfig::Capture => { + self.logger.log(record); + deno_telemetry::handle_log(record); + } + OtelConsoleConfig::Replace => { + deno_telemetry::handle_log(record); + } + } + DrawThread::show(); } } fn flush(&self) { - self.0.flush(); + self.logger.flush(); } } -pub fn init(maybe_level: Option) { +pub fn init(maybe_level: Option, otel_config: Option) { let log_level = maybe_level.unwrap_or(log::Level::Info); let logger = env_logger::Builder::from_env( env_logger::Env::new() @@ -93,7 +115,12 @@ pub fn init(maybe_level: Option) { }) .build(); - let cli_logger = CliLogger::new(logger); + let cli_logger = CliLogger::new( + logger, + otel_config + .map(|c| c.console) + .unwrap_or(OtelConsoleConfig::Ignore), + ); let max_level = cli_logger.filter(); let r = log::set_boxed_logger(Box::new(cli_logger)); if r.is_ok() { diff --git a/cli/worker.rs b/cli/worker.rs index 81b8cd2f83..0bbc27b29f 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -154,7 +154,7 @@ struct SharedWorkerState { storage_key_resolver: StorageKeyResolver, options: CliMainWorkerOptions, subcommand: DenoSubcommand, - otel_config: Option, // `None` means OpenTelemetry is disabled. + otel_config: OtelConfig, default_npm_caching_strategy: NpmCachingStrategy, } @@ -426,7 +426,7 @@ impl CliMainWorkerFactory { storage_key_resolver: StorageKeyResolver, subcommand: DenoSubcommand, options: CliMainWorkerOptions, - otel_config: Option, + otel_config: OtelConfig, default_npm_caching_strategy: NpmCachingStrategy, ) -> Self { Self { diff --git a/ext/telemetry/lib.rs b/ext/telemetry/lib.rs index 06210a70e7..816e838743 100644 --- a/ext/telemetry/lib.rs +++ b/ext/telemetry/lib.rs @@ -97,13 +97,28 @@ deno_core::extension!( ); #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OtelConfig { +pub struct OtelRuntimeConfig { pub runtime_name: Cow<'static, str>, pub runtime_version: Cow<'static, str>, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct OtelConfig { + pub tracing_enabled: bool, pub console: OtelConsoleConfig, pub deterministic: bool, } +impl OtelConfig { + pub fn as_v8(&self) -> Box<[u8]> { + Box::new([ + self.tracing_enabled as u8, + self.console as u8, + self.deterministic as u8, + ]) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[repr(u8)] pub enum OtelConsoleConfig { @@ -112,14 +127,9 @@ pub enum OtelConsoleConfig { Replace = 2, } -impl Default for OtelConfig { +impl Default for OtelConsoleConfig { fn default() -> Self { - Self { - runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")), - runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), - console: OtelConsoleConfig::Capture, - deterministic: false, - } + Self::Ignore } } @@ -411,16 +421,14 @@ static BUILT_IN_INSTRUMENTATION_SCOPE: OnceCell< opentelemetry::InstrumentationScope, > = OnceCell::new(); -pub fn init(config: OtelConfig) -> anyhow::Result<()> { +pub fn init(config: OtelRuntimeConfig) -> anyhow::Result<()> { // Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_* // crates don't do this automatically. // TODO(piscisaureus): enable GRPC support. let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() { Ok("http/protobuf") => Protocol::HttpBinary, Ok("http/json") => Protocol::HttpJson, - Ok("") | Err(env::VarError::NotPresent) => { - return Ok(()); - } + Ok("") | Err(env::VarError::NotPresent) => Protocol::HttpBinary, Ok(protocol) => { return Err(anyhow!( "Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}", diff --git a/ext/telemetry/telemetry.ts b/ext/telemetry/telemetry.ts index acdfd4d715..d1335f65b5 100644 --- a/ext/telemetry/telemetry.ts +++ b/ext/telemetry/telemetry.ts @@ -950,15 +950,15 @@ const otelConsoleConfig = { }; export function bootstrap( - config: [] | [ + config: [ + 0 | 1, typeof otelConsoleConfig[keyof typeof otelConsoleConfig], - number, + 0 | 1, ], ): void { - if (config.length === 0) return; - const { 0: consoleConfig, 1: deterministic } = config; + const { 0: tracingEnabled, 1: consoleConfig, 2: deterministic } = config; - TRACING_ENABLED = true; + TRACING_ENABLED = tracingEnabled === 1; DETERMINISTIC = deterministic === 1; switch (consoleConfig) { diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index 4a8c5dba86..2020c2bc8d 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -119,8 +119,7 @@ pub struct BootstrapOptions { // Used by `deno serve` pub serve_port: Option, pub serve_host: Option, - // OpenTelemetry output options. If `None`, OpenTelemetry is disabled. - pub otel_config: Option, + pub otel_config: OtelConfig, } impl Default for BootstrapOptions { @@ -155,7 +154,7 @@ impl Default for BootstrapOptions { mode: WorkerExecutionMode::None, serve_port: Default::default(), serve_host: Default::default(), - otel_config: None, + otel_config: Default::default(), } } } @@ -225,11 +224,7 @@ impl BootstrapOptions { self.serve_host.as_deref(), serve_is_main, serve_worker_count, - if let Some(otel_config) = self.otel_config.as_ref() { - Box::new([otel_config.console as u8, otel_config.deterministic as u8]) - } else { - Box::new([]) - }, + self.otel_config.as_v8(), ); bootstrap.serialize(ser).unwrap() diff --git a/tests/specs/cli/otel_basic/main.ts b/tests/specs/cli/otel_basic/main.ts index ccba126cc1..634727cea7 100644 --- a/tests/specs/cli/otel_basic/main.ts +++ b/tests/specs/cli/otel_basic/main.ts @@ -13,6 +13,7 @@ const server = Deno.serve( const command = new Deno.Command(Deno.execPath(), { args: ["run", "-A", "-q", "--unstable-otel", Deno.args[0]], env: { + OTEL_DENO: "true", DENO_UNSTABLE_OTEL_DETERMINISTIC: "1", OTEL_EXPORTER_OTLP_PROTOCOL: "http/json", OTEL_EXPORTER_OTLP_ENDPOINT: `http://localhost:${port}`, From 4cfa34052d0d6650f4a2b6a6a0c3363e7f3232e3 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 13:07:35 -0500 Subject: [PATCH 21/33] fix(compile): analyze modules in directory specified in --include (#27296) I ended up changing the file system implementation to determine its root directory as the last step of building it instead of being the first step which makes it much more reliable. --- cli/standalone/binary.rs | 349 +++++---- cli/standalone/serialization.rs | 12 +- cli/standalone/virtual_fs.rs | 671 ++++++++++++------ cli/tools/compile.rs | 167 ++--- tests/integration/compile_tests.rs | 31 +- .../__test__.jsonc | 22 + .../compile.out | 47 ++ .../main.out | 0 .../main.ts | 0 .../include/data_files/non_existent.out | 3 +- .../include/folder_ts_file/__test__.jsonc | 25 + .../compile/include/folder_ts_file/main.js | 14 + .../include/folder_ts_file/math/add.ts | 3 + .../compile/include/folder_ts_file/output.out | 2 + .../include/folder_ts_file/src/main.ts | 2 + .../include/symlink_twice/__test__.jsonc | 4 +- .../compile/include/symlink_twice/compile.out | 9 + .../compile/include/symlink_twice/setup.js | 1 - tests/specs/compile/npm_fs/__test__.jsonc | 24 + tests/specs/compile/npm_fs/compile.out | 8 + tests/specs/compile/npm_fs/deno.json | 3 + .../compile/npm_fs/main.out | 0 .../compile/npm_fs/main.ts | 0 tests/specs/mod.rs | 16 + tests/specs/schema.json | 15 + .../main_compile_file.out | 1 - .../main_compile_folder.out | 9 +- tests/util/server/src/lib.rs | 10 +- 28 files changed, 907 insertions(+), 541 deletions(-) create mode 100644 tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc create mode 100644 tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out rename tests/{testdata/compile/vfs_implicit_read_permission => specs/compile/global_npm_cache_implicit_read_permission}/main.out (100%) rename tests/{testdata/compile/vfs_implicit_read_permission => specs/compile/global_npm_cache_implicit_read_permission}/main.ts (100%) create mode 100644 tests/specs/compile/include/folder_ts_file/__test__.jsonc create mode 100644 tests/specs/compile/include/folder_ts_file/main.js create mode 100644 tests/specs/compile/include/folder_ts_file/math/add.ts create mode 100644 tests/specs/compile/include/folder_ts_file/output.out create mode 100644 tests/specs/compile/include/folder_ts_file/src/main.ts create mode 100644 tests/specs/compile/include/symlink_twice/compile.out create mode 100644 tests/specs/compile/npm_fs/__test__.jsonc create mode 100644 tests/specs/compile/npm_fs/compile.out create mode 100644 tests/specs/compile/npm_fs/deno.json rename tests/{testdata => specs}/compile/npm_fs/main.out (100%) rename tests/{testdata => specs}/compile/npm_fs/main.ts (100%) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 1e7b0d3f70..85a22cf837 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; @@ -76,6 +79,7 @@ use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; +use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -88,31 +92,28 @@ use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::output_vfs; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; +use super::virtual_fs::WindowsSystemRootablePath; + +pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = + ".deno_compile_node_modules"; /// A URL that can be designated as the base for relative URLs. /// /// After creation, this URL may be used to get the key for a /// module in the binary. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); - -impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { - fn from(url: &'a Url) -> Self { - Self(url) - } +pub enum StandaloneRelativeFileBaseUrl<'a> { + WindowsSystemRoot, + Path(&'a Url), } impl<'a> StandaloneRelativeFileBaseUrl<'a> { - pub fn new(url: &'a Url) -> Self { - debug_assert_eq!(url.scheme(), "file"); - Self(url) - } - /// Gets the module map key of the provided specifier. /// /// * Descendant file specifiers will be made relative to the base. @@ -122,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> { if target.scheme() != "file" { return Cow::Borrowed(target.as_str()); } + let base = match self { + Self::Path(base) => base, + Self::WindowsSystemRoot => return Cow::Borrowed(target.path()), + }; - match self.0.make_relative(target) { + match base.make_relative(target) { Some(relative) => { - if relative.starts_with("../") { - Cow::Borrowed(target.as_str()) - } else { - Cow::Owned(relative) - } + // This is not a great scenario to have because it means that the + // specifier is outside the vfs and could cause the binary to act + // strangely. If you encounter this, the fix is to add more paths + // to the vfs builder by calling `add_possible_min_root_dir`. + debug_assert!( + !relative.starts_with("../"), + "{} -> {} ({})", + base.as_str(), + target.as_str(), + relative, + ); + Cow::Owned(relative) } None => Cow::Borrowed(target.as_str()), } } - - pub fn inner(&self) -> &Url { - self.0 - } } #[derive(Deserialize, Serialize)] @@ -201,7 +209,7 @@ fn write_binary_bytes( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = @@ -372,7 +380,6 @@ pub struct WriteBinOptions<'a> { pub writer: File, pub display_output_filename: &'a str, pub graph: &'a ModuleGraph, - pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>, pub entrypoint: &'a ModuleSpecifier, pub include_files: &'a [ModuleSpecifier], pub compile_flags: &'a CompileFlags, @@ -556,7 +563,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { writer, display_output_filename, graph, - root_dir_url, entrypoint, include_files, compile_flags, @@ -568,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; - let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self - .build_npm_vfs(&root_path) - .context("Building npm vfs.")?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self - .npm_resolver - .root_node_modules_path() - .map(|path| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), - ) - .into_owned() - }), - }), - Some(snapshot), - ) - } else { - (None, None, None) - } + let mut vfs = VfsBuilder::new(); + let npm_snapshot = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; + Some(snapshot) + } else { + None } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; - let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { - npm_vfs - } else { - VfsBuilder::new(root_path.clone())? + } + InnerCliNpmResolverRef::Byonm(_) => { + self.fill_npm_vfs(&mut vfs)?; + None + } }; for include_file in include_files { let path = deno_path_util::url_to_file_path(include_file)?; - if path.is_dir() { - // TODO(#26941): we should analyze if any of these are - // modules in order to include their dependencies - vfs - .add_dir_recursive(&path) - .with_context(|| format!("Including {}", path.display()))?; - } else { - vfs - .add_file_at_path(&path) - .with_context(|| format!("Including {}", path.display()))?; - } + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { @@ -707,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); + if let Some(import_map) = self.workspace_resolver.maybe_import_map() { + if let Ok(file_path) = url_to_file_path(import_map.base_url()) { + if let Some(import_map_parent_dir) = file_path.parent() { + // tell the vfs about the import map's parent directory in case it + // falls outside what the root of where the VFS will be based + vfs.add_possible_min_root_dir(import_map_parent_dir); + } + } + } + if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() { + // ensure the vfs doesn't go below the node_modules directory's parent + if let Some(parent) = node_modules_dir.parent() { + vfs.add_possible_min_root_dir(parent); + } + } + + let vfs = self.build_vfs_consolidating_global_npm_cache(vfs); + let root_dir_url = match &vfs.root_path { + WindowsSystemRootablePath::Path(dir) => { + Some(url_from_directory_path(dir)?) + } + WindowsSystemRootablePath::WindowSystemRoot => None, + }; + let root_dir_url = match &root_dir_url { + Some(url) => StandaloneRelativeFileBaseUrl::Path(url), + None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot, + }; + + let node_modules = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(_) => { + npm_snapshot.as_ref().map(|_| NodeModules::Managed { + node_modules_dir: self.npm_resolver.root_node_modules_path().map( + |path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }, + ), + }) + } + InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), + ) + .into_owned() + }, + ), + }), + }; + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); @@ -721,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => Default::default(), }; + output_vfs(&vfs, display_output_filename); + let metadata = Metadata { argv: compile_flags.args.clone(), seed: self.cli_options.seed(), @@ -785,21 +803,19 @@ impl<'a> DenoCompileBinaryWriter<'a> { otel_config: self.cli_options.otel_config(), }; - output_vfs(&vfs, display_output_filename); - write_binary_bytes( writer, original_bin, &metadata, npm_snapshot.map(|s| s.into_serialized()), &remote_modules_store, - vfs, + &vfs, compile_flags, ) .context("Writing binary bytes") } - fn build_npm_vfs(&self, root_path: &Path) -> Result { + fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -810,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Managed(npm_resolver) => { if let Some(node_modules_path) = npm_resolver.root_node_modules_path() { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; builder.add_dir_recursive(node_modules_path)?; - Ok(builder) + Ok(()) } else { - // DO NOT include the user's registry url as it may contain credentials, - // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_path(); - let mut builder = - VfsBuilder::new(global_cache_root_path.to_path_buf())?; + // we'll flatten to remove any custom registries later let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -827,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?; builder.add_dir_recursive(&folder)?; } - - // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder - // that will be used by denort when loading the npm cache. This avoids us exposing - // the user's private registry information and means we don't have to bother - // serializing all the different registry config into the binary. - builder.with_root_dir(|root_dir| { - root_dir.name = ".deno_compile_node_modules".to_string(); - let mut new_entries = Vec::with_capacity(root_dir.entries.len()); - let mut localhost_entries = IndexMap::new(); - for entry in std::mem::take(&mut root_dir.entries) { - match entry { - VfsEntry::Dir(dir) => { - for entry in dir.entries { - log::debug!( - "Flattening {} into node_modules", - entry.name() - ); - if let Some(existing) = - localhost_entries.insert(entry.name().to_string(), entry) - { - panic!( - "Unhandled scenario where a duplicate entry was found: {:?}", - existing - ); - } - } - } - VfsEntry::File(_) | VfsEntry::Symlink(_) => { - new_entries.push(entry); - } - } - } - new_entries.push(VfsEntry::Dir(VirtualDirectory { - name: "localhost".to_string(), - entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), - })); - // needs to be sorted by name - new_entries.sort_by(|a, b| a.name().cmp(b.name())); - root_dir.entries = new_entries; - }); - - builder.set_new_root_path(root_path.to_path_buf())?; - - Ok(builder) + Ok(()) } } InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } @@ -908,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } } - Ok(builder) + Ok(()) } } } + + fn build_vfs_consolidating_global_npm_cache( + &self, + mut vfs: VfsBuilder, + ) -> BuiltVfs { + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(npm_resolver) => { + if npm_resolver.root_node_modules_path().is_some() { + return vfs.build(); + } + + let global_cache_root_path = npm_resolver.global_cache_root_path(); + + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder + // that will be used by denort when loading the npm cache. This avoids us exposing + // the user's private registry information and means we don't have to bother + // serializing all the different registry config into the binary. + let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else { + return vfs.build(); + }; + + root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string(); + let mut new_entries = Vec::with_capacity(root_dir.entries.len()); + let mut localhost_entries = IndexMap::new(); + for entry in std::mem::take(&mut root_dir.entries) { + match entry { + VfsEntry::Dir(dir) => { + for entry in dir.entries { + log::debug!("Flattening {} into node_modules", entry.name()); + if let Some(existing) = + localhost_entries.insert(entry.name().to_string(), entry) + { + panic!( + "Unhandled scenario where a duplicate entry was found: {:?}", + existing + ); + } + } + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => { + new_entries.push(entry); + } + } + } + new_entries.push(VfsEntry::Dir(VirtualDirectory { + name: "localhost".to_string(), + entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), + })); + // needs to be sorted by name + new_entries.sort_by(|a, b| a.name().cmp(b.name())); + root_dir.entries = new_entries; + + // it's better to not expose the user's cache directory, so take it out + // of there + let parent = global_cache_root_path.parent().unwrap(); + let parent_dir = vfs.get_dir_mut(parent).unwrap(); + let index = parent_dir + .entries + .iter() + .position(|entry| { + entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME + }) + .unwrap(); + let npm_global_cache_dir_entry = parent_dir.entries.remove(index); + + // go up from the ancestors removing empty directories... + // this is not as optimized as it could be + let mut last_name = + Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME); + for ancestor in parent.ancestors() { + let dir = vfs.get_dir_mut(ancestor).unwrap(); + if let Some(index) = dir + .entries + .iter() + .position(|entry| entry.name() == last_name) + { + dir.entries.remove(index); + } + last_name = Cow::Owned(dir.name.clone()); + if !dir.entries.is_empty() { + break; + } + } + + // now build the vfs and add the global cache dir entry there + let mut built_vfs = vfs.build(); + built_vfs.root.insert_entry(npm_global_cache_dir_entry); + built_vfs + } + InnerCliNpmResolverRef::Byonm(_) => vfs.build(), + } + } } fn get_denort_path(deno_exe: PathBuf) -> Option { diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index a5eb649bfd..6062e21019 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -23,6 +23,7 @@ use deno_semver::package::PackageReq; use crate::standalone::virtual_fs::VirtualDirectory; use super::binary::Metadata; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::VfsBuilder; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; @@ -39,7 +40,7 @@ pub fn serialize_binary_data_section( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, ) -> Result, AnyError> { fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); @@ -73,12 +74,11 @@ pub fn serialize_binary_data_section( } // 4. VFS { - let (vfs, vfs_files) = vfs.into_dir_and_files(); - let vfs = serde_json::to_string(&vfs)?; - write_bytes_with_len(&mut bytes, vfs.as_bytes()); - let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + let serialized_vfs = serde_json::to_string(&vfs.root)?; + write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes()); + let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); - for file in &vfs_files { + for file in &vfs.files { bytes.extend_from_slice(file); } } diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index ce7c0bb625..8ddd179c7a 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -15,17 +15,21 @@ use std::rc::Rc; use std::sync::Arc; use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_path_util::normalize_path; +use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; use deno_runtime::deno_io; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; +use indexmap::IndexSet; use serde::Deserialize; use serde::Serialize; use thiserror::Error; @@ -34,6 +38,38 @@ use crate::util; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; +use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!(!name_component.contains('\\')); + debug_assert!(!name_component.contains('/')); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug)] +pub struct BuiltVfs { + pub root_path: WindowsSystemRootablePath, + pub root: VirtualDirectory, + pub files: Vec>, +} + #[derive(Debug, Copy, Clone)] pub enum VfsFileSubDataKind { /// Raw bytes of the file. @@ -43,84 +79,84 @@ pub enum VfsFileSubDataKind { ModuleGraph, } -#[derive(Error, Debug)] -#[error( - "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() -)] -pub struct StripRootError { - root_path: PathBuf, - target: PathBuf, -} - #[derive(Debug)] pub struct VfsBuilder { - root_path: PathBuf, - root_dir: VirtualDirectory, + executable_root: VirtualDirectory, files: Vec>, current_offset: u64, file_offsets: HashMap, + /// The minimum root directory that should be included in the VFS. + min_root_dir: Option, } impl VfsBuilder { - pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path) - .with_context(|| format!("Canonicalizing {}", root_path.display()))?; - log::debug!("Building vfs with root '{}'", root_path.display()); - Ok(Self { - root_dir: VirtualDirectory { - name: root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), + pub fn new() -> Self { + Self { + executable_root: VirtualDirectory { + name: "/".to_string(), entries: Vec::new(), }, - root_path, files: Vec::new(), current_offset: 0, file_offsets: Default::default(), - }) + min_root_dir: Default::default(), + } } - pub fn set_new_root_path( - &mut self, - root_path: PathBuf, - ) -> Result<(), AnyError> { - let root_path = canonicalize_path(&root_path)?; - self.root_path = root_path; - self.root_dir = VirtualDirectory { - name: self - .root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), - entries: vec![VfsEntry::Dir(VirtualDirectory { - name: std::mem::take(&mut self.root_dir.name), - entries: std::mem::take(&mut self.root_dir.entries), - })], - }; - Ok(()) - } + /// Add a directory that might be the minimum root directory + /// of the VFS. + /// + /// For example, say the user has a deno.json and specifies an + /// import map in a parent directory. The import map won't be + /// included in the VFS, but its base will meaning we need to + /// tell the VFS builder to include the base of the import map + /// by calling this method. + pub fn add_possible_min_root_dir(&mut self, path: &Path) { + self.add_dir_raw(path); - pub fn with_root_dir( - &mut self, - with_root: impl FnOnce(&mut VirtualDirectory) -> R, - ) -> R { - with_root(&mut self.root_dir) + match &self.min_root_dir { + Some(WindowsSystemRootablePath::WindowSystemRoot) => { + // already the root dir + } + Some(WindowsSystemRootablePath::Path(current_path)) => { + let mut common_components = Vec::new(); + for (a, b) in current_path.components().zip(path.components()) { + if a != b { + break; + } + common_components.push(a); + } + if common_components.is_empty() { + if cfg!(windows) { + self.min_root_dir = + Some(WindowsSystemRootablePath::WindowSystemRoot); + } else { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(PathBuf::from("/"))); + } + } else { + self.min_root_dir = Some(WindowsSystemRootablePath::Path( + common_components.iter().collect(), + )); + } + } + None => { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(path.to_path_buf())); + } + } } pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if path != target_path { - self.add_symlink(path, &target_path)?; - } - self.add_dir_recursive_internal(&target_path) + let target_path = self.resolve_target_path(path)?; + self.add_dir_recursive_not_symlink(&target_path) } - fn add_dir_recursive_internal( + fn add_dir_recursive_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { - self.add_dir(path)?; + self.add_dir_raw(path); let read_dir = std::fs::read_dir(path) .with_context(|| format!("Reading {}", path.display()))?; @@ -133,49 +169,26 @@ impl VfsBuilder { let path = entry.path(); if file_type.is_dir() { - self.add_dir_recursive_internal(&path)?; + self.add_dir_recursive_not_symlink(&path)?; } else if file_type.is_file() { self.add_file_at_path_not_symlink(&path)?; } else if file_type.is_symlink() { - match util::fs::canonicalize_path(&path) { - Ok(target) => { - if let Err(StripRootError { .. }) = self.add_symlink(&path, &target) - { - if target.is_file() { - // this may change behavior, so warn the user about it - log::warn!( - "{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - // inline the symlink and make the target file - let file_bytes = std::fs::read(&target) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner( - &path, - file_bytes, - VfsFileSubDataKind::Raw, - )?; - } else { - log::warn!( - "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - } + match self.add_symlink(&path) { + Ok(target) => match target { + SymlinkTarget::File(target) => { + self.add_file_at_path_not_symlink(&target)? } - } + SymlinkTarget::Dir(target) => { + self.add_dir_recursive_not_symlink(&target)?; + } + }, Err(err) => { log::warn!( - "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", - crate::colors::yellow("Warning"), - path.display(), - err - ); + "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", + crate::colors::yellow("Warning"), + path.display(), + err + ); } } } @@ -184,15 +197,15 @@ impl VfsBuilder { Ok(()) } - fn add_dir( - &mut self, - path: &Path, - ) -> Result<&mut VirtualDirectory, StripRootError> { + fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { log::debug!("Ensuring directory '{}'", path.display()); - let path = self.path_relative_root(path)?; - let mut current_dir = &mut self.root_dir; + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } let name = component.as_os_str().to_string_lossy(); let index = match current_dir .entries @@ -218,15 +231,44 @@ impl VfsBuilder { }; } - Ok(current_dir) + current_dir + } + + pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { + &mut self.executable_root + } + + pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let index = match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&name)) + { + Ok(index) => index, + Err(_) => return None, + }; + match &mut current_dir.entries[index] { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + Some(current_dir) } pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; - } - self.add_file_at_path_not_symlink(&target_path) + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw) } fn add_file_at_path_not_symlink( @@ -244,11 +286,15 @@ impl VfsBuilder { data: Vec, sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + let target = self.add_symlink(path)?.into_path_buf(); + self.add_file_with_data_inner(&target, data, sub_data_kind) + } else { + self.add_file_with_data_inner(path, data, sub_data_kind) } - self.add_file_with_data_inner(&target_path, data, sub_data_kind) } fn add_file_with_data_inner( @@ -267,7 +313,7 @@ impl VfsBuilder { self.current_offset }; - let dir = self.add_dir(path.parent().unwrap())?; + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); let offset_and_len = OffsetWithLength { offset, @@ -309,74 +355,162 @@ impl VfsBuilder { Ok(()) } - fn add_symlink( + fn resolve_target_path(&mut self, path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + Ok(self.add_symlink(path)?.into_path_buf()) + } else { + Ok(path.to_path_buf()) + } + } + + fn add_symlink(&mut self, path: &Path) -> Result { + self.add_symlink_inner(path, &mut IndexSet::new()) + } + + fn add_symlink_inner( &mut self, path: &Path, - target: &Path, - ) -> Result<(), StripRootError> { - log::debug!( - "Adding symlink '{}' to '{}'", - path.display(), - target.display() + visited: &mut IndexSet, + ) -> Result { + log::debug!("Adding symlink '{}'", path.display()); + let target = strip_unc_prefix( + std::fs::read_link(path) + .with_context(|| format!("Reading symlink '{}'", path.display()))?, ); - let relative_target = self.path_relative_root(target)?; - let relative_path = match self.path_relative_root(path) { - Ok(path) => path, - Err(StripRootError { .. }) => { - // ignore if the original path is outside the root directory - return Ok(()); - } - }; - if relative_target == relative_path { - // it's the same, ignore - return Ok(()); - } - let dir = self.add_dir(path.parent().unwrap())?; + let target = normalize_path(path.parent().unwrap().join(&target)); + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => Ok(()), // previously inserted + Ok(_) => {} // previously inserted Err(insert_index) => { dir.entries.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: relative_target - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(), + dest_parts: VirtualSymlinkParts::from_path(&target), }), ); - Ok(()) } } + let target_metadata = + std::fs::symlink_metadata(&target).with_context(|| { + format!("Reading symlink target '{}'", target.display()) + })?; + if target_metadata.is_symlink() { + if !visited.insert(target.clone()) { + // todo: probably don't error in this scenario + bail!( + "Circular symlink detected: {} -> {}", + visited + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> "), + target.display() + ); + } + self.add_symlink_inner(&target, visited) + } else if target_metadata.is_dir() { + Ok(SymlinkTarget::Dir(target)) + } else { + Ok(SymlinkTarget::File(target)) + } } - pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { - (self.root_dir, self.files) - } + pub fn build(self) -> BuiltVfs { + fn strip_prefix_from_symlinks( + dir: &mut VirtualDirectory, + parts: &[String], + ) { + for entry in &mut dir.entries { + match entry { + VfsEntry::Dir(dir) => { + strip_prefix_from_symlinks(dir, parts); + } + VfsEntry::File(_) => {} + VfsEntry::Symlink(symlink) => { + let old_parts = std::mem::take(&mut symlink.dest_parts.0); + symlink.dest_parts.0 = + old_parts.into_iter().skip(parts.len()).collect(); + } + } + } + } - fn path_relative_root(&self, path: &Path) -> Result { - match path.strip_prefix(&self.root_path) { - Ok(p) => Ok(p.to_path_buf()), - Err(_) => Err(StripRootError { - root_path: self.root_path.clone(), - target: path.to_path_buf(), - }), + let mut current_dir = self.executable_root; + let mut current_path = if cfg!(windows) { + WindowsSystemRootablePath::WindowSystemRoot + } else { + WindowsSystemRootablePath::Path(PathBuf::from("/")) + }; + loop { + if current_dir.entries.len() != 1 { + break; + } + if self.min_root_dir.as_ref() == Some(¤t_path) { + break; + } + match ¤t_dir.entries[0] { + VfsEntry::Dir(dir) => { + if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + // special directory we want to maintain + break; + } + match current_dir.entries.remove(0) { + VfsEntry::Dir(dir) => { + current_path = + WindowsSystemRootablePath::Path(current_path.join(&dir.name)); + current_dir = dir; + } + _ => unreachable!(), + }; + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => break, + } + } + if let WindowsSystemRootablePath::Path(path) = ¤t_path { + strip_prefix_from_symlinks( + &mut current_dir, + &VirtualSymlinkParts::from_path(path).0, + ); + } + BuiltVfs { + root_path: current_path, + root: current_dir, + files: self.files, } } } -pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { +#[derive(Debug)] +enum SymlinkTarget { + File(PathBuf), + Dir(PathBuf), +} + +impl SymlinkTarget { + pub fn into_path_buf(self) -> PathBuf { + match self { + Self::File(path) => path, + Self::Dir(path) => path, + } + } +} + +pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { if !log::log_enabled!(log::Level::Info) { return; // no need to compute if won't output } - if builder.root_dir.entries.is_empty() { + if vfs.root.entries.is_empty() { return; // nothing to output } let mut text = String::new(); - let display_tree = vfs_as_display_tree(builder, executable_name); + let display_tree = vfs_as_display_tree(vfs, executable_name); display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string log::info!( "\n{}\n", @@ -386,7 +520,7 @@ pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { } fn vfs_as_display_tree( - builder: &VfsBuilder, + vfs: &BuiltVfs, executable_name: &str, ) -> DisplayTreeNode { enum EntryOutput<'a> { @@ -398,20 +532,38 @@ fn vfs_as_display_tree( impl<'a> EntryOutput<'a> { pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + let mut children = match self { + EntryOutput::Subset(vec) => vec + .iter() + .map(|e| e.output.as_display_tree(e.name.to_string())) + .collect(), + EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => { + vec![] + } + }; + // we only want to collapse leafs so that nodes of the + // same depth have the same indentation + let collapse_single_child = + children.len() == 1 && children[0].children.is_empty(); DisplayTreeNode { text: match self { - EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name, + EntryOutput::All => format!("{}/*", name), + EntryOutput::Subset(_) => { + if collapse_single_child { + format!("{}/{}", name, children[0].text) + } else { + name + } + } + EntryOutput::File => name, EntryOutput::Symlink(parts) => { format!("{} --> {}", name, parts.join("/")) } }, - children: match self { - EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())], - EntryOutput::Subset(vec) => vec - .iter() - .map(|e| e.output.as_display_tree(e.name.to_string())) - .collect(), - EntryOutput::File | EntryOutput::Symlink(_) => vec![], + children: if collapse_single_child { + children.remove(0).children + } else { + children }, } } @@ -422,37 +574,81 @@ fn vfs_as_display_tree( output: EntryOutput<'a>, } - fn include_all_entries<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - EntryOutput::Subset( + fn show_global_node_modules_dir( + vfs_dir: &VirtualDirectory, + ) -> Vec { + fn show_subset_deep( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> EntryOutput { + if depth == 0 { + EntryOutput::All + } else { + EntryOutput::Subset(show_subset(vfs_dir, depth)) + } + } + + fn show_subset( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> Vec { vfs_dir .entries .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: match entry { + VfsEntry::Dir(virtual_directory) => { + show_subset_deep(virtual_directory, depth - 1) + } + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + } + }, }) - .collect(), - ) + .collect() + } + + // in this scenario, we want to show + // .deno_compile_node_modules/localhost///* + show_subset(vfs_dir, 3) } - fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> { + fn include_all_entries<'a>( + dir_path: &WindowsSystemRootablePath, + vfs_dir: &'a VirtualDirectory, + ) -> Vec> { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return show_global_node_modules_dir(vfs_dir); + } + + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(dir_path.join(entry.name()), entry), + }) + .collect() + } + + fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput { match entry { VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), VfsEntry::File(_) => EntryOutput::File, VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts) + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) } } } - fn analyze_dir<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - let real_entry_count = std::fs::read_dir(dir) + fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir)); + } + + let real_entry_count = std::fs::read_dir(&dir) .ok() .map(|entries| entries.flat_map(|e| e.ok()).count()) .unwrap_or(0); @@ -462,7 +658,7 @@ fn vfs_as_display_tree( .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: analyze_entry(dir.join(entry.name()), entry), }) .collect::>(); if children @@ -474,15 +670,23 @@ fn vfs_as_display_tree( EntryOutput::Subset(children) } } else { - include_all_entries(dir, vfs_dir) + EntryOutput::Subset(include_all_entries( + &WindowsSystemRootablePath::Path(dir), + vfs_dir, + )) } } // always include all the entries for the root directory, otherwise the // user might not have context about what's being shown - let output = include_all_entries(&builder.root_path, &builder.root_dir); - output - .as_display_tree(deno_terminal::colors::italic(executable_name).to_string()) + let child_entries = include_all_entries(&vfs.root_path, &vfs.root); + DisplayTreeNode { + text: deno_terminal::colors::italic(executable_name).to_string(), + children: child_entries + .iter() + .map(|entry| entry.output.as_display_tree(entry.name.to_string())) + .collect(), + } } #[derive(Debug)] @@ -603,6 +807,20 @@ pub struct VirtualDirectory { pub entries: Vec, } +impl VirtualDirectory { + pub fn insert_entry(&mut self, entry: VfsEntry) { + let name = entry.name(); + match self.entries.binary_search_by(|e| e.name().cmp(name)) { + Ok(index) => { + self.entries[index] = entry; + } + Err(insert_index) => { + self.entries.insert(insert_index, entry); + } + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct OffsetWithLength { #[serde(rename = "o")] @@ -626,18 +844,33 @@ pub struct VirtualFile { pub module_graph_offset: OffsetWithLength, } +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct VirtualSymlink { #[serde(rename = "n")] pub name: String, #[serde(rename = "p")] - pub dest_parts: Vec, + pub dest_parts: VirtualSymlinkParts, } impl VirtualSymlink { pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { let mut dest = root.to_path_buf(); - for part in &self.dest_parts { + for part in &self.dest_parts.0 { dest.push(part); } dest @@ -709,10 +942,10 @@ impl VfsRoot { let mut final_path = self.root_path.clone(); let mut current_entry = VfsEntryRef::Dir(&self.dir); for component in relative_path.components() { - let component = component.as_os_str().to_string_lossy(); + let component = component.as_os_str(); let current_dir = match current_entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } VfsEntryRef::Symlink(symlink) => { @@ -721,7 +954,7 @@ impl VfsRoot { final_path = resolved_path; // overwrite with the new resolved path match entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } _ => { @@ -739,6 +972,7 @@ impl VfsRoot { )); } }; + let component = component.to_string_lossy(); match current_dir .entries .binary_search_by(|e| e.name().cmp(&component)) @@ -1136,6 +1370,7 @@ impl FileBackedVfs { mod test { use console_static_text::ansi::strip_ansi_codes; use std::io::Write; + use test_util::assert_contains; use test_util::TempDir; use super::*; @@ -1159,8 +1394,11 @@ mod test { // will canonicalize the root path let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.join("sub_dir").create_dir_all(); + src_path.join("e.txt").write("e"); + src_path.symlink_file("e.txt", "sub_dir/e.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( &src_path.join("a.txt"), @@ -1190,18 +1428,9 @@ mod test { VfsFileSubDataKind::Raw, ) .unwrap(); + builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); builder - .add_file_with_data_inner( - &src_path.join("e.txt"), - "e".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_symlink( - &src_path.join("sub_dir").join("e.txt"), - &src_path.join("e.txt"), - ) + .add_symlink(&src_path.join("sub_dir").join("e.txt")) .unwrap(); // get the virtual fs @@ -1262,7 +1491,7 @@ mod test { // build and create the virtual fs let src_path = temp_dir_path.join("src").to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder.add_dir_recursive(&src_path).unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); @@ -1300,10 +1529,10 @@ mod test { temp_dir: &TempDir, ) -> (PathBuf, FileBackedVfs) { let virtual_fs_file = temp_dir.path().join("virtual_fs"); - let (root_dir, files) = builder.into_dir_and_files(); + let vfs = builder.build(); { let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); - for file_data in &files { + for file_data in &vfs.files { file.write_all(file_data).unwrap(); } } @@ -1314,7 +1543,7 @@ mod test { FileBackedVfs::new( Cow::Owned(data), VfsRoot { - dir: root_dir, + dir: vfs.root, root_path: dest_path.to_path_buf(), start_file_offset: 0, }, @@ -1327,41 +1556,22 @@ mod test { let temp_dir = TempDir::new(); let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.symlink_file("a.txt", "b.txt"); + src_path.symlink_file("b.txt", "c.txt"); + src_path.symlink_file("c.txt", "a.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); - builder - .add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt")) - .unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - assert_eq!( - virtual_fs - .file_entry(&dest_path.join("a.txt")) - .err() - .unwrap() - .to_string(), - "circular symlinks", - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(), - dest_path.join("b.txt") - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(), - dest_path.join("c.txt") - ); + let mut builder = VfsBuilder::new(); + let err = builder + .add_symlink(src_path.join("a.txt").as_path()) + .unwrap_err(); + assert_contains!(err.to_string(), "Circular symlink detected",); } #[tokio::test] async fn test_open_file() { let temp_dir = TempDir::new(); let temp_path = temp_dir.path().canonicalize(); - let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( temp_path.join("a.txt").as_path(), @@ -1436,8 +1646,7 @@ mod test { temp_dir.write("c/a.txt", "contents"); temp_dir.symlink_file("c/a.txt", "c/b.txt"); assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works - let mut vfs_builder = - VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap(); + let mut vfs_builder = VfsBuilder::new(); // full dir vfs_builder .add_dir_recursive(temp_dir.path().join("a").as_path()) @@ -1451,16 +1660,14 @@ mod test { .add_dir_recursive(temp_dir.path().join("c").as_path()) .unwrap(); temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory - let node = vfs_as_display_tree(&vfs_builder, "executable"); + let node = vfs_as_display_tree(&vfs_builder.build(), "executable"); let mut text = String::new(); node.print(&mut text).unwrap(); assert_eq!( strip_ansi_codes(&text), r#"executable -├─┬ a -│ └── * -├─┬ b -│ └── a.txt +├── a/* +├── b/a.txt └─┬ c ├── a.txt └── b.txt --> c/a.txt diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 4d0607ba71..7a463a7b09 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,7 +5,6 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; -use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::binary::WriteBinOptions; use crate::standalone::is_standalone_binary; use deno_ast::MediaType; @@ -17,8 +16,11 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_terminal::colors; use rand::Rng; +use std::collections::HashSet; +use std::collections::VecDeque; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -84,29 +86,6 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let root_dir_url = resolve_root_dir_from_specifiers( - cli_options.workspace().root_dir(), - graph - .specifiers() - .map(|(s, _)| s) - .chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ) - .chain(include_files.iter()) - .chain( - // sometimes the import map path is outside the root dir - cli_options - .workspace() - .to_import_map_path() - .ok() - .and_then(|p| p.and_then(|p| url_from_file_path(&p).ok())) - .iter(), - ), - ); - log::debug!("Binary root dir: {}", root_dir_url); log::info!( "{} {} to {}", colors::green("Compile"), @@ -138,7 +117,6 @@ pub async fn compile( .unwrap() .to_string_lossy(), graph: &graph, - root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url), entrypoint, include_files: &include_files, compile_flags: &compile_flags, @@ -261,15 +239,58 @@ fn get_module_roots_and_include_files( } } - let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); - let mut include_files = Vec::with_capacity(compile_flags.include.len()); + fn analyze_path( + url: &ModuleSpecifier, + module_roots: &mut Vec, + include_files: &mut Vec, + searched_paths: &mut HashSet, + ) -> Result<(), AnyError> { + let Ok(path) = url_to_file_path(url) else { + return Ok(()); + }; + let mut pending = VecDeque::from([path]); + while let Some(path) = pending.pop_front() { + if !searched_paths.insert(path.clone()) { + continue; + } + if !path.is_dir() { + let url = url_from_file_path(&path)?; + include_files.push(url.clone()); + if is_module_graph_module(&url) { + module_roots.push(url); + } + continue; + } + for entry in std::fs::read_dir(&path).with_context(|| { + format!("Failed reading directory '{}'", path.display()) + })? { + let entry = entry.with_context(|| { + format!("Failed reading entry in directory '{}'", path.display()) + })?; + pending.push_back(entry.path()); + } + } + Ok(()) + } + + let mut searched_paths = HashSet::new(); + let mut module_roots = Vec::new(); + let mut include_files = Vec::new(); module_roots.push(entrypoint.clone()); for side_module in &compile_flags.include { let url = resolve_url_or_path(side_module, initial_cwd)?; if is_module_graph_module(&url) { - module_roots.push(url); + module_roots.push(url.clone()); + if url.scheme() == "file" { + include_files.push(url); + } } else { - include_files.push(url); + analyze_path( + &url, + &mut module_roots, + &mut include_files, + &mut searched_paths, + )?; } } Ok((module_roots, include_files)) @@ -335,57 +356,6 @@ fn get_os_specific_filepath( } } -fn resolve_root_dir_from_specifiers<'a>( - starting_dir: &ModuleSpecifier, - specifiers: impl Iterator, -) -> ModuleSpecifier { - fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str { - let min_length = a.len().min(b.len()); - - let mut last_slash = 0; - for i in 0..min_length { - if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' { - last_slash = i; - } else if a.as_bytes()[i] != b.as_bytes()[i] { - break; - } - } - - // Return the common root path up to the last common slash. - // This returns a slice of the original string 'a', up to and including the last matching '/'. - let common = &a[..=last_slash]; - if cfg!(windows) && common == "file:///" { - a - } else { - common - } - } - - fn is_file_system_root(url: &str) -> bool { - let Some(path) = url.strip_prefix("file:///") else { - return false; - }; - if cfg!(windows) { - let Some((_drive, path)) = path.split_once('/') else { - return true; - }; - path.is_empty() - } else { - path.is_empty() - } - } - - let mut found_dir = starting_dir.as_str(); - if !is_file_system_root(found_dir) { - for specifier in specifiers { - if specifier.scheme() == "file" { - found_dir = select_common_root(found_dir, specifier.as_str()); - } - } - } - ModuleSpecifier::parse(found_dir).unwrap() -} - #[cfg(test)] mod test { pub use super::*; @@ -462,41 +432,4 @@ mod test { run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); } - - #[test] - fn test_resolve_root_dir_from_specifiers() { - fn resolve(start: &str, specifiers: &[&str]) -> String { - let specifiers = specifiers - .iter() - .map(|s| ModuleSpecifier::parse(s).unwrap()) - .collect::>(); - resolve_root_dir_from_specifiers( - &ModuleSpecifier::parse(start).unwrap(), - specifiers.iter(), - ) - .to_string() - } - - assert_eq!( - resolve("file:///a/b/e", &["file:///a/b/c/d"]), - "file:///a/b/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d"]), - "file:///a/b/c/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]), - "file:///a/b/c/" - ); - assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///"); - if cfg!(windows) { - assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/"); - // this will ignore the other one because it's on a separate drive - assert_eq!( - resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]), - "file:///c:/a/b/c/" - ); - } - } } diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index 62c5cf8fab..a34d2cdd1d 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -846,21 +846,6 @@ testing[WILDCARD]this .assert_matches_text("2\n"); } -#[test] -fn compile_npm_file_system() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/npm_fs/main.ts", - copy_temp_dir: Some("compile/npm_fs"), - compile_args: vec!["-A"], - run_args: vec![], - output_file: "compile/npm_fs/main.out", - node_modules_local: true, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_bin_esm() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() { }); } -#[test] -fn compile_npm_vfs_implicit_read_permissions() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/vfs_implicit_read_permission/main.ts", - copy_temp_dir: Some("compile/vfs_implicit_read_permission"), - compile_args: vec![], - run_args: vec![], - output_file: "compile/vfs_implicit_read_permission/main.out", - node_modules_local: false, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_no_permissions() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() { let symlink_target_dir = temp_dir.path().join("some_folder"); project_dir.join("node_modules").create_dir_all(); symlink_target_dir.create_dir_all(); + symlink_target_dir.join("file.txt").write("5"); let symlink_target_file = temp_dir.path().join("target.txt"); symlink_target_file.write("5"); let symlink_dir = project_dir.join("node_modules").join("symlink_dir"); diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc new file mode 100644 index 0000000000..d346c3ad20 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc @@ -0,0 +1,22 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out new file mode 100644 index 0000000000..c29c878593 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -0,0 +1,47 @@ +[WILDCARD] +Compile file:///[WILDLINE]/main.ts to [WILDLINE] + +Embedded File System + +main[WILDLINE] +├─┬ .deno_compile_node_modules +│ └─┬ localhost +│ ├─┬ ansi-regex +│ │ ├── 3.0.1/* +│ │ └── 5.0.1/* +│ ├── ansi-styles/4.3.0/* +│ ├── camelcase/5.3.1/* +│ ├── cliui/6.0.0/* +│ ├── color-convert/2.0.1/* +│ ├── color-name/1.1.4/* +│ ├── cowsay/1.5.0/* +│ ├── decamelize/1.2.0/* +│ ├── emoji-regex/8.0.0/* +│ ├── find-up/4.1.0/* +│ ├── get-caller-file/2.0.5/* +│ ├── get-stdin/8.0.0/* +│ ├─┬ is-fullwidth-code-point +│ │ ├── 2.0.0/* +│ │ └── 3.0.0/* +│ ├── locate-path/5.0.0/* +│ ├── p-limit/2.3.0/* +│ ├── p-locate/4.1.0/* +│ ├── p-try/2.2.0/* +│ ├── path-exists/4.0.0/* +│ ├── require-directory/2.1.1/* +│ ├── require-main-filename/2.0.0/* +│ ├── set-blocking/2.0.0/* +│ ├─┬ string-width +│ │ ├── 2.1.1/* +│ │ └── 4.2.3/* +│ ├─┬ strip-ansi +│ │ ├── 4.0.0/* +│ │ └── 6.0.1/* +│ ├── strip-final-newline/2.0.0/* +│ ├── which-module/2.0.0/* +│ ├── wrap-ansi/6.2.0/* +│ ├── y18n/4.0.3/* +│ ├── yargs/15.4.1/* +│ └── yargs-parser/18.1.3/* +└── main.ts + diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.out similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.out rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.out diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.ts b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.ts rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts diff --git a/tests/specs/compile/include/data_files/non_existent.out b/tests/specs/compile/include/data_files/non_existent.out index a88b441ba8..54bc69ef09 100644 --- a/tests/specs/compile/include/data_files/non_existent.out +++ b/tests/specs/compile/include/data_files/non_existent.out @@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]' Caused by: 0: Including [WILDLINE]does_not_exist.txt - 1: [WILDLINE] + 1: Reading [WILDLINE]does_not_exist.txt + 2: [WILDLINE] diff --git a/tests/specs/compile/include/folder_ts_file/__test__.jsonc b/tests/specs/compile/include/folder_ts_file/__test__.jsonc new file mode 100644 index 0000000000..f02ed1efc3 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/__test__.jsonc @@ -0,0 +1,25 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + // notice how the math folder is not included + "args": "compile --allow-read=data --include src --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=data --include src --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/include/folder_ts_file/main.js b/tests/specs/compile/include/folder_ts_file/main.js new file mode 100644 index 0000000000..23b490e390 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/main.js @@ -0,0 +1,14 @@ +const mathDir = import.meta.dirname + "/math"; +const files = Array.from( + Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name), +); +files.sort(); +for (const file of files) { + console.log(file); +} + +function nonAnalyzable() { + return "./src/main.ts"; +} + +await import(nonAnalyzable()); diff --git a/tests/specs/compile/include/folder_ts_file/math/add.ts b/tests/specs/compile/include/folder_ts_file/math/add.ts new file mode 100644 index 0000000000..3b399665dc --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/math/add.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a + b; +} diff --git a/tests/specs/compile/include/folder_ts_file/output.out b/tests/specs/compile/include/folder_ts_file/output.out new file mode 100644 index 0000000000..959e3d5c76 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/output.out @@ -0,0 +1,2 @@ +[WILDLINE]add.ts +3 diff --git a/tests/specs/compile/include/folder_ts_file/src/main.ts b/tests/specs/compile/include/folder_ts_file/src/main.ts new file mode 100644 index 0000000000..38868c3d82 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/src/main.ts @@ -0,0 +1,2 @@ +import { add } from "../math/add.ts"; +console.log(add(1, 2)); diff --git a/tests/specs/compile/include/symlink_twice/__test__.jsonc b/tests/specs/compile/include/symlink_twice/__test__.jsonc index ebdf824f43..f0f57292a6 100644 --- a/tests/specs/compile/include/symlink_twice/__test__.jsonc +++ b/tests/specs/compile/include/symlink_twice/__test__.jsonc @@ -6,7 +6,7 @@ }, { "if": "unix", "args": "compile --allow-read=data --include . --output main link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "unix", "commandName": "./main", @@ -16,7 +16,7 @@ }, { "if": "windows", "args": "compile --allow-read=data --include . --output main.exe link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "windows", "commandName": "./main.exe", diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out new file mode 100644 index 0000000000..c57eb9b2f1 --- /dev/null +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -0,0 +1,9 @@ +Compile [WILDLINE] + +Embedded File System + +main[WILDLINE] +├── index.js +├── link.js --> index.js +└── setup.js + diff --git a/tests/specs/compile/include/symlink_twice/setup.js b/tests/specs/compile/include/symlink_twice/setup.js index 3e713dd63e..4c7cebfaf5 100644 --- a/tests/specs/compile/include/symlink_twice/setup.js +++ b/tests/specs/compile/include/symlink_twice/setup.js @@ -1,3 +1,2 @@ -Deno.mkdirSync("data"); Deno.writeTextFileSync("index.js", "console.log(1);"); Deno.symlinkSync("index.js", "link.js"); diff --git a/tests/specs/compile/npm_fs/__test__.jsonc b/tests/specs/compile/npm_fs/__test__.jsonc new file mode 100644 index 0000000000..a8198bfb5d --- /dev/null +++ b/tests/specs/compile/npm_fs/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + // use this so the vfs output is all in the same folder + "canonicalizedTempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out new file mode 100644 index 0000000000..4944146788 --- /dev/null +++ b/tests/specs/compile/npm_fs/compile.out @@ -0,0 +1,8 @@ +[WILDCARD] + +Embedded File System + +main[WILDLINE] +├── main.ts +└── node_modules/* + diff --git a/tests/specs/compile/npm_fs/deno.json b/tests/specs/compile/npm_fs/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/compile/npm_fs/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/testdata/compile/npm_fs/main.out b/tests/specs/compile/npm_fs/main.out similarity index 100% rename from tests/testdata/compile/npm_fs/main.out rename to tests/specs/compile/npm_fs/main.out diff --git a/tests/testdata/compile/npm_fs/main.ts b/tests/specs/compile/npm_fs/main.ts similarity index 100% rename from tests/testdata/compile/npm_fs/main.ts rename to tests/specs/compile/npm_fs/main.ts diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs index f5820e4d88..985a6c7c40 100644 --- a/tests/specs/mod.rs +++ b/tests/specs/mod.rs @@ -118,6 +118,12 @@ struct MultiStepMetaData { /// steps. #[serde(default)] pub temp_dir: bool, + /// Whether the temporary directory should be canonicalized. + /// + /// This should be used sparingly, but is sometimes necessary + /// on the CI. + #[serde(default)] + pub canonicalized_temp_dir: bool, /// Whether the temporary directory should be symlinked to another path. #[serde(default)] pub symlinked_temp_dir: bool, @@ -144,6 +150,8 @@ struct SingleTestMetaData { #[serde(default)] pub temp_dir: bool, #[serde(default)] + pub canonicalized_temp_dir: bool, + #[serde(default)] pub symlinked_temp_dir: bool, #[serde(default)] pub repeat: Option, @@ -159,6 +167,7 @@ impl SingleTestMetaData { base: self.base, cwd: None, temp_dir: self.temp_dir, + canonicalized_temp_dir: self.canonicalized_temp_dir, symlinked_temp_dir: self.symlinked_temp_dir, repeat: self.repeat, envs: Default::default(), @@ -326,6 +335,13 @@ fn test_context_from_metadata( builder = builder.cwd(cwd.to_string_lossy()); } + if metadata.canonicalized_temp_dir { + // not actually deprecated, we just want to discourage its use + #[allow(deprecated)] + { + builder = builder.use_canonicalized_temp_dir(); + } + } if metadata.symlinked_temp_dir { // not actually deprecated, we just want to discourage its use // because it's mostly used for testing purposes locally diff --git a/tests/specs/schema.json b/tests/specs/schema.json index 2b35d9bd7d..77ffc59530 100644 --- a/tests/specs/schema.json +++ b/tests/specs/schema.json @@ -36,6 +36,9 @@ "flaky": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, "symlinkedTempDir": { "type": "boolean" }, @@ -66,6 +69,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, @@ -94,6 +103,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index 70de321361..633c2cca62 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,5 +1,4 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Warning Symlink target is outside '[WILDCARD]node_modules_symlink_outside'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. Embedded File System diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 205c6a9281..61f0a2456a 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -3,8 +3,13 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Warning Symlink target is outside '[WILDLINE]node_modules_symlink_outside'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. Embedded File System -[WILDCARD] +bin[WILDLINE] +├─┬ compile +│ └─┬ node_modules_symlink_outside +│ ├── main.ts +│ └── node_modules/* +└── some_folder/* + diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 953896cffd..531944bf6a 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -816,15 +816,17 @@ pub fn wildcard_match_detailed( } let actual_next_text = ¤t_text[max_current_text_found_index..]; - let max_next_text_len = 40; - let next_text_len = - std::cmp::min(max_next_text_len, actual_next_text.len()); + let next_text_len = actual_next_text + .chars() + .take(40) + .map(|c| c.len_utf8()) + .sum::(); output_lines.push(format!( "==== NEXT ACTUAL TEXT ====\n{}{}", colors::red(annotate_whitespace( &actual_next_text[..next_text_len] )), - if actual_next_text.len() > max_next_text_len { + if actual_next_text.len() > next_text_len { "[TRUNCATED]" } else { "" From b7564636b5ded00819cd1f89cf828f555d9085e2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 18:43:43 -0500 Subject: [PATCH 22/33] chore: add npm distribution at `deno` package (#27346) --- .github/workflows/cargo_publish.yml | 2 +- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/npm_publish.yml | 45 +++ .github/workflows/promote_to_release.yml | 2 +- .github/workflows/start_release.yml | 2 +- .github/workflows/version_bump.yml | 2 +- tools/deno.lock.json | 436 +++++++++++++++++++---- tools/release/npm/.gitignore | 1 + tools/release/npm/bin.cjs | 54 +++ tools/release/npm/build.ts | 237 ++++++++++++ tools/release/npm/install.cjs | 5 + tools/release/npm/install_api.cjs | 196 ++++++++++ 13 files changed, 904 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/npm_publish.yml create mode 100644 tools/release/npm/.gitignore create mode 100644 tools/release/npm/bin.cjs create mode 100644 tools/release/npm/build.ts create mode 100644 tools/release/npm/install.cjs create mode 100644 tools/release/npm/install_api.cjs diff --git a/.github/workflows/cargo_publish.yml b/.github/workflows/cargo_publish.yml index 3af97f4662..eb72e3739f 100644 --- a/.github/workflows/cargo_publish.yml +++ b/.github/workflows/cargo_publish.yml @@ -35,7 +35,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Publish env: diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 9dd1f3017b..d43db76414 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -196,7 +196,7 @@ const installNodeStep = { const installDenoStep = { name: "Install Deno", uses: "denoland/setup-deno@v2", - with: { "deno-version": "v1.x" }, + with: { "deno-version": "v2.x" }, }; const authenticateWithGoogleCloud = { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838373cb78..29dd694c6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: name: Install Deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/npm_publish.yml b/.github/workflows/npm_publish.yml new file mode 100644 index 0000000000..5e58005926 --- /dev/null +++ b/.github/workflows/npm_publish.yml @@ -0,0 +1,45 @@ +name: npm_publish + +on: + workflow_dispatch: + inputs: + version: + description: 'Version' + type: string + release: + types: [published] + +permissions: + id-token: write + +jobs: + build: + name: npm publish + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Configure git + run: | + git config --global core.symlinks true + git config --global fetch.parallel 32 + + - name: Clone repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: '22.x' + registry-url: 'https://registry.npmjs.org' + + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: ./tools/release/npm/build.ts ${{ github.event.inputs.version }} --publish diff --git a/.github/workflows/promote_to_release.yml b/.github/workflows/promote_to_release.yml index 79fefa6d6c..4079118d92 100644 --- a/.github/workflows/promote_to_release.yml +++ b/.github/workflows/promote_to_release.yml @@ -42,7 +42,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install rust-codesign run: |- diff --git a/.github/workflows/start_release.yml b/.github/workflows/start_release.yml index 40a44bb61a..35446c1adb 100644 --- a/.github/workflows/start_release.yml +++ b/.github/workflows/start_release.yml @@ -36,7 +36,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Create Gist URL env: diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 9038fe0d22..306a8642ad 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -41,7 +41,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Run version bump run: | diff --git a/tools/deno.lock.json b/tools/deno.lock.json index 46c09ce24b..3d21e5c9f6 100644 --- a/tools/deno.lock.json +++ b/tools/deno.lock.json @@ -1,81 +1,365 @@ { - "version": "3", - "packages": { - "specifiers": { - "jsr:@david/dax@0.41.0": "jsr:@david/dax@0.41.0", - "jsr:@david/which@^0.4.1": "jsr:@david/which@0.4.1", - "jsr:@deno/patchver@0.1.0": "jsr:@deno/patchver@0.1.0", - "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", - "jsr:@std/bytes@^0.221.0": "jsr:@std/bytes@0.221.0", - "jsr:@std/fmt@1": "jsr:@std/fmt@1.0.0", - "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", - "jsr:@std/fs@0.221.0": "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/io@^0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0": "jsr:@std/streams@0.221.0", - "jsr:@std/yaml@^0.221": "jsr:@std/yaml@0.221.0" + "version": "4", + "specifiers": { + "jsr:@david/dax@0.41.0": "0.41.0", + "jsr:@david/dax@0.42": "0.42.0", + "jsr:@david/path@0.2": "0.2.0", + "jsr:@david/which@~0.4.1": "0.4.1", + "jsr:@deno/patchver@0.1.0": "0.1.0", + "jsr:@std/assert@0.221": "0.221.0", + "jsr:@std/bytes@0.221": "0.221.0", + "jsr:@std/fmt@0.221": "0.221.0", + "jsr:@std/fmt@1": "1.0.0", + "jsr:@std/fs@0.221.0": "0.221.0", + "jsr:@std/fs@1": "1.0.5", + "jsr:@std/io@0.221": "0.221.0", + "jsr:@std/io@0.221.0": "0.221.0", + "jsr:@std/path@0.221": "0.221.0", + "jsr:@std/path@0.221.0": "0.221.0", + "jsr:@std/path@1": "1.0.8", + "jsr:@std/path@^1.0.7": "1.0.8", + "jsr:@std/streams@0.221": "0.221.0", + "jsr:@std/streams@0.221.0": "0.221.0", + "jsr:@std/yaml@0.221": "0.221.0", + "npm:decompress@4.2.1": "4.2.1" + }, + "jsr": { + "@david/dax@0.41.0": { + "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", + "dependencies": [ + "jsr:@david/which", + "jsr:@std/fmt@0.221", + "jsr:@std/fs@0.221.0", + "jsr:@std/io@0.221.0", + "jsr:@std/path@0.221.0", + "jsr:@std/streams@0.221.0" + ] }, - "jsr": { - "@david/dax@0.41.0": { - "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", - "dependencies": [ - "jsr:@david/which@^0.4.1", - "jsr:@std/fmt@^0.221.0", - "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0" - ] - }, - "@david/which@0.4.1": { - "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" - }, - "@deno/patchver@0.1.0": { - "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" - }, - "@std/assert@0.221.0": { - "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" - }, - "@std/bytes@0.221.0": { - "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" - }, - "@std/fmt@0.221.0": { - "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" - }, - "@std/fmt@1.0.0": { - "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" - }, - "@std/fs@0.221.0": { - "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, - "@std/io@0.221.0": { - "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/bytes@^0.221.0" - ] - }, - "@std/path@0.221.0": { - "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", - "dependencies": [ - "jsr:@std/assert@^0.221.0" - ] - }, - "@std/streams@0.221.0": { - "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", - "dependencies": [ - "jsr:@std/io@^0.221.0" - ] - }, - "@std/yaml@0.221.0": { - "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" - } + "@david/dax@0.42.0": { + "integrity": "0c547c9a20577a6072b90def194c159c9ddab82280285ebfd8268a4ebefbd80b", + "dependencies": [ + "jsr:@david/path", + "jsr:@david/which", + "jsr:@std/fmt@1", + "jsr:@std/fs@1", + "jsr:@std/io@0.221", + "jsr:@std/path@1", + "jsr:@std/streams@0.221" + ] + }, + "@david/path@0.2.0": { + "integrity": "f2d7aa7f02ce5a55e27c09f9f1381794acb09d328f8d3c8a2e3ab3ffc294dccd", + "dependencies": [ + "jsr:@std/fs@1", + "jsr:@std/path@1" + ] + }, + "@david/which@0.4.1": { + "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" + }, + "@deno/patchver@0.1.0": { + "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" + }, + "@std/assert@0.221.0": { + "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" + }, + "@std/bytes@0.221.0": { + "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" + }, + "@std/fmt@0.221.0": { + "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" + }, + "@std/fmt@1.0.0": { + "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" + }, + "@std/fs@0.221.0": { + "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/path@0.221" + ] + }, + "@std/fs@1.0.5": { + "integrity": "41806ad6823d0b5f275f9849a2640d87e4ef67c51ee1b8fb02426f55e02fd44e", + "dependencies": [ + "jsr:@std/path@^1.0.7" + ] + }, + "@std/io@0.221.0": { + "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/bytes" + ] + }, + "@std/path@0.221.0": { + "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", + "dependencies": [ + "jsr:@std/assert" + ] + }, + "@std/path@1.0.8": { + "integrity": "548fa456bb6a04d3c1a1e7477986b6cffbce95102d0bb447c67c4ee70e0364be" + }, + "@std/streams@0.221.0": { + "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", + "dependencies": [ + "jsr:@std/io@0.221" + ] + }, + "@std/yaml@0.221.0": { + "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" + } + }, + "npm": { + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl@1.2.3": { + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "dependencies": [ + "readable-stream", + "safe-buffer@5.2.1" + ] + }, + "buffer-alloc-unsafe@1.1.0": { + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, + "buffer-alloc@1.2.0": { + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dependencies": [ + "buffer-alloc-unsafe", + "buffer-fill" + ] + }, + "buffer-crc32@0.2.13": { + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" + }, + "buffer-fill@1.0.0": { + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==" + }, + "buffer@5.7.1": { + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "commander@2.20.3": { + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "decompress-tar@4.1.1": { + "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", + "dependencies": [ + "file-type@5.2.0", + "is-stream", + "tar-stream" + ] + }, + "decompress-tarbz2@4.1.1": { + "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", + "dependencies": [ + "decompress-tar", + "file-type@6.2.0", + "is-stream", + "seek-bzip", + "unbzip2-stream" + ] + }, + "decompress-targz@4.1.1": { + "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", + "dependencies": [ + "decompress-tar", + "file-type@5.2.0", + "is-stream" + ] + }, + "decompress-unzip@4.0.1": { + "integrity": "sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==", + "dependencies": [ + "file-type@3.9.0", + "get-stream", + "pify@2.3.0", + "yauzl" + ] + }, + "decompress@4.2.1": { + "integrity": "sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==", + "dependencies": [ + "decompress-tar", + "decompress-tarbz2", + "decompress-targz", + "decompress-unzip", + "graceful-fs", + "make-dir", + "pify@2.3.0", + "strip-dirs" + ] + }, + "end-of-stream@1.4.4": { + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": [ + "once" + ] + }, + "fd-slicer@1.1.0": { + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": [ + "pend" + ] + }, + "file-type@3.9.0": { + "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==" + }, + "file-type@5.2.0": { + "integrity": "sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==" + }, + "file-type@6.2.0": { + "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==" + }, + "fs-constants@1.0.0": { + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "get-stream@2.3.1": { + "integrity": "sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==", + "dependencies": [ + "object-assign", + "pinkie-promise" + ] + }, + "graceful-fs@4.2.11": { + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "ieee754@1.2.1": { + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-natural-number@4.0.1": { + "integrity": "sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==" + }, + "is-stream@1.1.0": { + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "make-dir@1.3.0": { + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dependencies": [ + "pify@3.0.0" + ] + }, + "object-assign@4.1.1": { + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "once@1.4.0": { + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": [ + "wrappy" + ] + }, + "pend@1.2.0": { + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "pify@2.3.0": { + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==" + }, + "pify@3.0.0": { + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==" + }, + "pinkie-promise@2.0.1": { + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": [ + "pinkie" + ] + }, + "pinkie@2.0.4": { + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray", + "process-nextick-args", + "safe-buffer@5.1.2", + "string_decoder", + "util-deprecate" + ] + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "seek-bzip@1.0.6": { + "integrity": "sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==", + "dependencies": [ + "commander" + ] + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer@5.1.2" + ] + }, + "strip-dirs@2.1.0": { + "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", + "dependencies": [ + "is-natural-number" + ] + }, + "tar-stream@1.6.2": { + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dependencies": [ + "bl", + "buffer-alloc", + "end-of-stream", + "fs-constants", + "readable-stream", + "to-buffer", + "xtend" + ] + }, + "through@2.3.8": { + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "to-buffer@1.1.1": { + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" + }, + "unbzip2-stream@1.4.3": { + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": [ + "buffer", + "through" + ] + }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "wrappy@1.0.2": { + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "xtend@4.0.2": { + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "yauzl@2.10.0": { + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": [ + "buffer-crc32", + "fd-slicer" + ] } }, "remote": { diff --git a/tools/release/npm/.gitignore b/tools/release/npm/.gitignore new file mode 100644 index 0000000000..1521c8b765 --- /dev/null +++ b/tools/release/npm/.gitignore @@ -0,0 +1 @@ +dist diff --git a/tools/release/npm/bin.cjs b/tools/release/npm/bin.cjs new file mode 100644 index 0000000000..984aa350f1 --- /dev/null +++ b/tools/release/npm/bin.cjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// @ts-check +const path = require("path"); +const child_process = require("child_process"); +const os = require("os"); +const fs = require("fs"); + +const exePath = path.join( + __dirname, + os.platform() === "win32" ? "deno.exe" : "deno", +); + +if (!fs.existsSync(exePath)) { + try { + const resolvedExePath = require("./install_api.cjs").runInstall(); + runDenoExe(resolvedExePath); + } catch (err) { + if (err !== undefined && typeof err.message === "string") { + console.error(err.message); + } else { + console.error(err); + } + process.exit(1); + } +} else { + runDenoExe(exePath); +} + +/** @param exePath {string} */ +function runDenoExe(exePath) { + const result = child_process.spawnSync( + exePath, + process.argv.slice(2), + { stdio: "inherit" }, + ); + if (result.error) { + throw result.error; + } + + throwIfNoExePath(); + + process.exitCode = result.status; + + function throwIfNoExePath() { + if (!fs.existsSync(exePath)) { + throw new Error( + "Could not find exe at path '" + exePath + + "'. Maybe try running deno again.", + ); + } + } +} diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts new file mode 100644 index 0000000000..b1f1c45cbf --- /dev/null +++ b/tools/release/npm/build.ts @@ -0,0 +1,237 @@ +#!/usr/bin/env -S deno run -A --lock=tools/deno.lock.json +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// NOTICE: This deployment/npm folder was lifted from https://github.com/dprint/dprint/blob/0ba79811cc96d2dee8e0cf766a8c8c0fc44879c2/deployment/npm/ +// with permission (Copyright 2019-2023 David Sherret) +import $ from "jsr:@david/dax@^0.42.0"; +// @ts-types="npm:@types/decompress@4.2.7" +import decompress from "npm:decompress@4.2.1"; +import { parseArgs } from "@std/cli/parse-args"; + +interface Package { + zipFileName: string; + os: "win32" | "darwin" | "linux"; + cpu: "x64" | "arm64"; + libc?: "glibc" | "musl"; +} + +const args = parseArgs(Deno.args, { + boolean: ["publish"], +}); +const packages: Package[] = [{ + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "x64", +}, { + // use x64_64 until there's an arm64 build + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-apple-darwin.zip", + os: "darwin", + cpu: "x64", +}, { + zipFileName: "deno-aarch64-apple-darwin.zip", + os: "darwin", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-unknown-linux-gnu.zip", + os: "linux", + cpu: "x64", + libc: "glibc", +}, { + zipFileName: "deno-aarch64-unknown-linux-gnu.zip", + os: "linux", + cpu: "arm64", + libc: "glibc", +}]; + +const markdownText = `# Deno + +[Deno](https://www.deno.com) +([/ˈdiːnoʊ/](http://ipa-reader.xyz/?text=%CB%88di%CB%90no%CA%8A), pronounced +\`dee-no\`) is a JavaScript, TypeScript, and WebAssembly runtime with secure +defaults and a great developer experience. It's built on [V8](https://v8.dev/), +[Rust](https://www.rust-lang.org/), and [Tokio](https://tokio.rs/). + +Learn more about the Deno runtime +[in the documentation](https://docs.deno.com/runtime/manual). +`; + +const currentDir = $.path(import.meta.url).parentOrThrow(); +const rootDir = currentDir.parentOrThrow().parentOrThrow().parentOrThrow(); +const outputDir = currentDir.join("./dist"); +const scopeDir = outputDir.join("@deno"); +const denoDir = outputDir.join("deno"); +const version = resolveVersion(); + +$.logStep(`Publishing ${version}...`); + +await $`rm -rf ${outputDir}`; +await $`mkdir -p ${denoDir} ${scopeDir}`; + +// setup Deno packages +{ + $.logStep(`Setting up deno ${version}...`); + const pkgJson = { + "name": "deno", + "version": version, + "description": "A modern runtime for JavaScript and TypeScript.", + "bin": "bin.cjs", + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + "keywords": [ + "runtime", + "typescript", + ], + "author": "the Deno authors", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.com", + // for yarn berry (https://github.com/dprint/dprint/issues/686) + "preferUnplugged": true, + "scripts": { + "postinstall": "node ./install.cjs", + }, + optionalDependencies: packages + .map((pkg) => `@deno/${getPackageNameNoScope(pkg)}`) + .reduce((obj, pkgName) => ({ ...obj, [pkgName]: version }), {}), + }; + currentDir.join("bin.cjs").copyFileToDirSync(denoDir); + currentDir.join("install_api.cjs").copyFileToDirSync(denoDir); + currentDir.join("install.cjs").copyFileToDirSync(denoDir); + denoDir.join("package.json").writeJsonPrettySync(pkgJson); + rootDir.join("LICENSE.md").copyFileSync(denoDir.join("LICENSE")); + denoDir.join("README.md").writeTextSync(markdownText); + // ensure the test files don't get published + denoDir.join(".npmignore").writeTextSync("deno\ndeno.exe\n"); + + // setup each binary package + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Setting up @deno/${pkgName}...`); + const pkgDir = scopeDir.join(pkgName); + const zipPath = pkgDir.join("output.zip"); + + await $`mkdir -p ${pkgDir}`; + + // download and extract the zip file + const zipUrl = + `https://github.com/denoland/deno/releases/download/v${version}/${pkg.zipFileName}`; + await $.request(zipUrl).showProgress().pipeToPath(zipPath); + await decompress(zipPath.toString(), pkgDir.toString()); + zipPath.removeSync(); + + // create the package.json and readme + pkgDir.join("README.md").writeTextSync( + `# @denoland/${pkgName}\n\n${pkgName} distribution of [Deno](https://deno.land).\n`, + ); + pkgDir.join("package.json").writeJsonPrettySync({ + "name": `@deno/${pkgName}`, + "version": version, + "description": `${pkgName} distribution of Deno`, + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + // force yarn to unpack + "preferUnplugged": true, + "author": "David Sherret", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.land", + "os": [pkg.os], + "cpu": [pkg.cpu], + libc: pkg.libc == null ? undefined : [pkg.libc], + }); + } +} + +// verify that the package is created correctly +{ + $.logStep("Verifying packages..."); + const testPlatform = Deno.build.os == "windows" + ? (Deno.build.arch === "x86_64" ? "@deno/win32-x64" : "@deno/win32-arm64") + : Deno.build.os === "darwin" + ? (Deno.build.arch === "x86_64" ? "@deno/darwin-x64" : "@deno/darwin-arm64") + : "@deno/linux-x64-glibc"; + outputDir.join("package.json").writeJsonPrettySync({ + workspaces: [ + "deno", + // There seems to be a bug with npm workspaces where this doesn't + // work, so for now make some assumptions and only include the package + // that works on the CI for the current operating system + // ...packages.map(p => `@deno/${getPackageNameNoScope(p)}`), + testPlatform, + ], + }); + + const denoExe = Deno.build.os === "windows" ? "deno.exe" : "deno"; + await $`npm install`.cwd(denoDir); + + // ensure the post-install script adds the executable to the deno package, + // which is necessary for faster caching and to ensure the vscode extension + // picks it up + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist after post install"); + } + + // run once after post install created deno, once with a simulated readonly file system, once creating the cache and once with + await $`node bin.cjs -v && rm ${denoExe} && DENO_SIMULATED_READONLY_FILE_SYSTEM=1 node bin.cjs -v && node bin.cjs -v && node bin.cjs -v` + .cwd(denoDir); + + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist when lazily initialized"); + } +} + +// publish if necessary +if (args.publish) { + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Publishing @deno/${pkgName}...`); + if (await checkPackagePublished(`@deno/${pkgName}`)) { + $.logLight(" Already published."); + continue; + } + const pkgDir = scopeDir.join(pkgName); + await $`cd ${pkgDir} && npm publish --provenance --access public`; + } + + $.logStep(`Publishing deno...`); + await $`cd ${denoDir} && npm publish --provenance --access public`; +} + +function getPackageNameNoScope(name: Package) { + const libc = name.libc == null ? "" : `-${name.libc}`; + return `${name.os}-${name.cpu}${libc}`; +} + +function resolveVersion() { + const firstArg = args._[0]; + if ( + firstArg != null && + typeof firstArg === "string" && + firstArg.trim().length > 0 + ) { + return firstArg; + } + const version = (rootDir.join("cli/Cargo.toml").readTextSync().match( + /version = "(.*?)"/, + ))?.[1]; + if (version == null) { + throw new Error("Could not resolve version."); + } + return version; +} + +async function checkPackagePublished(pkgName: string) { + const result = await $`npm info ${pkgName}@${version}`.quiet().noThrow(); + return result.code === 0; +} diff --git a/tools/release/npm/install.cjs b/tools/release/npm/install.cjs new file mode 100644 index 0000000000..8bf9aabe47 --- /dev/null +++ b/tools/release/npm/install.cjs @@ -0,0 +1,5 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +require("./install_api.cjs").runInstall(); diff --git a/tools/release/npm/install_api.cjs b/tools/release/npm/install_api.cjs new file mode 100644 index 0000000000..026d8ccc45 --- /dev/null +++ b/tools/release/npm/install_api.cjs @@ -0,0 +1,196 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +/** @type {string | undefined} */ +let cachedIsMusl = undefined; + +module.exports = { + runInstall() { + const denoFileName = os.platform() === "win32" ? "deno.exe" : "deno"; + const targetExecutablePath = path.join( + __dirname, + denoFileName, + ); + + if (fs.existsSync(targetExecutablePath)) { + return targetExecutablePath; + } + + const target = getTarget(); + const sourcePackagePath = path.dirname( + require.resolve("@deno/" + target + "/package.json"), + ); + const sourceExecutablePath = path.join(sourcePackagePath, denoFileName); + + if (!fs.existsSync(sourceExecutablePath)) { + throw new Error( + "Could not find executable for @deno/" + target + " at " + + sourceExecutablePath, + ); + } + + try { + if (process.env.DPRINT_SIMULATED_READONLY_FILE_SYSTEM === "1") { + console.warn("Simulating readonly file system for testing."); + throw new Error("Throwing for testing purposes."); + } + + // in order to make things faster the next time we run and to allow the + // deno vscode extension to easily pick this up, copy the executable + // into the deno package folder + hardLinkOrCopy(sourceExecutablePath, targetExecutablePath); + if (os.platform() !== "win32") { + // chomd +x + chmodX(targetExecutablePath); + } + return targetExecutablePath; + } catch (err) { + // this may fail on readonly file systems... in this case, fall + // back to using the resolved package path + if (process.env.DENO_DEBUG === "1") { + console.warn( + "Failed to copy executable from " + + sourceExecutablePath + " to " + targetExecutablePath + + ". Using resolved package path instead.", + err, + ); + } + // use the path found in the specific package + try { + chmodX(sourceExecutablePath); + } catch (_err) { + // ignore + } + return sourceExecutablePath; + } + }, +}; + +/** @filePath {string} */ +function chmodX(filePath) { + const perms = fs.statSync(filePath).mode; + fs.chmodSync(filePath, perms | 0o111); +} + +function getTarget() { + const platform = os.platform(); + if (platform === "linux") { + return platform + "-" + getArch() + "-" + getLinuxFamily(); + } else { + return platform + "-" + getArch(); + } +} + +function getArch() { + const arch = os.arch(); + if (arch !== "arm64" && arch !== "x64") { + throw new Error( + "Unsupported architecture " + os.arch() + + ". Only x64 and aarch64 binaries are available.", + ); + } + return arch; +} + +function getLinuxFamily() { + if (getIsMusl()) { + throw new Error( + "Musl is not supported. It's one of our priorities. Please upvote this issue: https://github.com/denoland/deno/issues/3711", + ); + // return "musl"; + } + return "glibc"; + + function getIsMusl() { + // code adapted from https://github.com/lovell/detect-libc + // Copyright Apache 2.0 license, the detect-libc maintainers + if (cachedIsMusl == null) { + cachedIsMusl = innerGet(); + } + return cachedIsMusl; + + function innerGet() { + try { + if (os.platform() !== "linux") { + return false; + } + return isProcessReportMusl() || isConfMusl(); + } catch (err) { + // just in case + console.warn("Error checking if musl.", err); + return false; + } + } + + function isProcessReportMusl() { + if (!process.report) { + return false; + } + const rawReport = process.report.getReport(); + const report = typeof rawReport === "string" + ? JSON.parse(rawReport) + : rawReport; + if (!report || !(report.sharedObjects instanceof Array)) { + return false; + } + return report.sharedObjects.some((o) => + o.includes("libc.musl-") || o.includes("ld-musl-") + ); + } + + function isConfMusl() { + const output = getCommandOutput(); + const [_, ldd1] = output.split(/[\r\n]+/); + return ldd1 && ldd1.includes("musl"); + } + + function getCommandOutput() { + try { + const command = + "getconf GNU_LIBC_VERSION 2>&1 || true; ldd --version 2>&1 || true"; + return require("child_process").execSync(command, { encoding: "utf8" }); + } catch (_err) { + return ""; + } + } + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function hardLinkOrCopy(sourcePath, destinationPath) { + try { + fs.linkSync(sourcePath, destinationPath); + } catch { + atomicCopyFile(sourcePath, destinationPath); + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function atomicCopyFile(sourcePath, destinationPath) { + const crypto = require("crypto"); + const rand = crypto.randomBytes(4).toString("hex"); + const tempFilePath = destinationPath + "." + rand; + fs.copyFileSync(sourcePath, tempFilePath); + try { + fs.renameSync(tempFilePath, destinationPath); + } catch (err) { + // will maybe throw when another process had already done this + // so just ignore and delete the created temporary file + try { + fs.unlinkSync(tempFilePath); + } catch (_err2) { + // ignore + } + throw err; + } +} From f4f64cbe2defd7fd4edf9b4ccd7e1ffc96f05917 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 18:58:14 -0500 Subject: [PATCH 23/33] fix(npm): search node_modules folder for package matching npm specifier (#27345) --- resolvers/deno/lib.rs | 18 +++--- resolvers/deno/npm/byonm.rs | 61 +++++++++++++++++-- .../__test__.jsonc | 18 ++++++ .../matches.ts | 3 + .../node_modules/aliased/index.js | 3 + .../node_modules/aliased/package.json | 4 ++ .../node_modules/package/index.js | 3 + .../node_modules/package/package.json | 4 ++ .../not_matches.out | 2 + .../not_matches.ts | 3 + .../not_matches_aliased.out | 2 + .../not_matches_aliased.ts | 3 + .../package.json | 2 + 13 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 661caf836d..a74ca614a3 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -355,16 +355,16 @@ impl< }) .map_err(|err| { match err.into_kind() { - ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { - err.into() + ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { + err.into() + } } - } }); } } diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index 771f23ce23..6e1be35ca0 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -205,9 +205,9 @@ impl ByonmNpmResolver { } // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = url_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { + let maybe_referrer_path = url_to_file_path(referrer).ok(); + if let Some(file_path) = maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { let package_json_path = dir_path.join("package.json"); if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { if let Some(alias) = @@ -216,11 +216,10 @@ impl ByonmNpmResolver { return Ok(Some((pkg_json, alias))); } } - current_path = dir_path; } } - // otherwise, fall fallback to the project's package.json + // fall fallback to the project's package.json if let Some(root_node_modules_dir) = &self.root_node_modules_dir { let root_pkg_json_path = root_node_modules_dir.parent().unwrap().join("package.json"); @@ -232,6 +231,58 @@ impl ByonmNpmResolver { } } + // now try to resolve based on the closest node_modules directory + let maybe_referrer_path = url_to_file_path(referrer).ok(); + let search_node_modules = |node_modules: &Path| { + if req.version_req.tag().is_some() { + return None; + } + + let pkg_folder = node_modules.join(&req.name); + if let Ok(Some(dep_pkg_json)) = + self.load_pkg_json(&pkg_folder.join("package.json")) + { + if dep_pkg_json.name.as_ref() == Some(&req.name) { + let matches_req = dep_pkg_json + .version + .as_ref() + .and_then(|v| Version::parse_from_npm(v).ok()) + .map(|version| req.version_req.matches(&version)) + .unwrap_or(true); + if matches_req { + return Some((dep_pkg_json, req.name.clone())); + } + } + } + None + }; + if let Some(file_path) = &maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { + if let Some(result) = + search_node_modules(&dir_path.join("node_modules")) + { + return Ok(Some(result)); + } + } + } + + // and finally check the root node_modules directory + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let already_searched = maybe_referrer_path + .as_ref() + .and_then(|referrer_path| { + root_node_modules_dir + .parent() + .map(|root_dir| referrer_path.starts_with(root_dir)) + }) + .unwrap_or(false); + if !already_searched { + if let Some(result) = search_node_modules(root_node_modules_dir) { + return Ok(Some(result)); + } + } + } + Ok(None) } diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc new file mode 100644 index 0000000000..e2c5495387 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc @@ -0,0 +1,18 @@ +{ + "tests": { + "matches": { + "args": "run -A matches.ts", + "output": "5\n" + }, + "not_matches": { + "args": "run -A not_matches.ts", + "output": "not_matches.out", + "exitCode": 1 + }, + "not_matches_aliased": { + "args": "run -A not_matches_aliased.ts", + "output": "not_matches_aliased.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts new file mode 100644 index 0000000000..986e7baf4c --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json new file mode 100644 index 0000000000..618960872f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json @@ -0,0 +1,4 @@ +{ + "name": "not-same-name", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json new file mode 100644 index 0000000000..5723987e9f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json @@ -0,0 +1,4 @@ +{ + "name": "package", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out new file mode 100644 index 0000000000..c549e13ff2 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:package@2' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts new file mode 100644 index 0000000000..a337bd7d86 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@2"; // won't match 2 + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out new file mode 100644 index 0000000000..93b52f64bc --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:aliased@1' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts new file mode 100644 index 0000000000..85fa11e319 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts @@ -0,0 +1,3 @@ +import { add } from "npm:aliased@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json @@ -0,0 +1,2 @@ +{ +} From a63f8452e99d1896703098799f246d07fa0f30c2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 19:30:21 -0500 Subject: [PATCH 24/33] chore: mark ./tools/release/npm/build.ts as executable (#27349) --- tools/release/npm/build.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/release/npm/build.ts diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts old mode 100644 new mode 100755 From 960776cd32331495034fcda21b194d897e780892 Mon Sep 17 00:00:00 2001 From: Satya Rohith Date: Fri, 13 Dec 2024 06:14:42 +0530 Subject: [PATCH 25/33] fix(ext/node): support createConnection option in node:http.request() (#25470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit changes "node:http" module to add support for the "createConnection" option when the "request()" API is called. Closes https://github.com/denoland/deno/issues/19507 --------- Signed-off-by: Yoshiya Hinosawa Signed-off-by: Satya Rohith Co-authored-by: Yoshiya Hinosawa Co-authored-by: Bartek Iwańczuk Co-authored-by: crowlkats --- ext/fetch/lib.rs | 3 - ext/node/lib.rs | 4 +- ext/node/ops/http.rs | 278 +++++++++++------- ext/node/polyfills/_http_outgoing.ts | 57 +++- ext/node/polyfills/_tls_wrap.ts | 7 + ext/node/polyfills/http.ts | 159 ++++++---- ext/node/polyfills/https.ts | 2 +- .../polyfills/internal_binding/cares_wrap.ts | 10 +- ext/node/polyfills/net.ts | 40 ++- runtime/errors.rs | 24 +- tests/node_compat/config.jsonc | 8 - tests/node_compat/runner/TODO.md | 10 +- .../test/parallel/test-http-agent-false.js | 53 ---- .../test-http-agent-keepalive-delay.js | 43 --- ...st-http-client-timeout-connect-listener.js | 49 --- .../test-http-dump-req-when-res-ends.js | 73 ----- .../test-http-hostname-typechecking.js | 49 --- .../test-net-better-error-messages-port.js | 24 -- .../test-net-connect-handle-econnrefused.js | 39 --- .../sequential/test-net-reconnect-error.js | 50 ---- tests/unit_node/http_test.ts | 80 ++++- 21 files changed, 465 insertions(+), 597 deletions(-) delete mode 100644 tests/node_compat/test/parallel/test-http-agent-false.js delete mode 100644 tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js delete mode 100644 tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js delete mode 100644 tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js delete mode 100644 tests/node_compat/test/parallel/test-http-hostname-typechecking.js delete mode 100644 tests/node_compat/test/sequential/test-net-better-error-messages-port.js delete mode 100644 tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js delete mode 100644 tests/node_compat/test/sequential/test-net-reconnect-error.js diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index a3f5d03e64..919c6d3044 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -206,9 +206,6 @@ pub enum FetchError { RequestBuilderHook(deno_core::error::AnyError), #[error(transparent)] Io(#[from] std::io::Error), - // Only used for node upgrade - #[error(transparent)] - Hyper(#[from] hyper::Error), } pub type CancelableResponseFuture = diff --git a/ext/node/lib.rs b/ext/node/lib.rs index bf593ad432..1e6c920c9e 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -364,9 +364,9 @@ deno_core::extension!(deno_node, ops::zlib::brotli::op_create_brotli_decompress, ops::zlib::brotli::op_brotli_decompress_stream, ops::zlib::brotli::op_brotli_decompress_stream_end, - ops::http::op_node_http_request

, ops::http::op_node_http_fetch_response_upgrade, - ops::http::op_node_http_fetch_send, + ops::http::op_node_http_request_with_conn

, + ops::http::op_node_http_await_response, ops::http2::op_http2_connect, ops::http2::op_http2_poll_client_connection, ops::http2::op_http2_client_request, diff --git a/ext/node/ops/http.rs b/ext/node/ops/http.rs index f4adb94060..eb28e68aee 100644 --- a/ext/node/ops/http.rs +++ b/ext/node/ops/http.rs @@ -2,18 +2,20 @@ use std::borrow::Cow; use std::cell::RefCell; +use std::fmt::Debug; use std::pin::Pin; use std::rc::Rc; use std::task::Context; use std::task::Poll; use bytes::Bytes; +use deno_core::error::bad_resource; +use deno_core::error::type_error; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::futures::Stream; use deno_core::futures::StreamExt; -use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::serde::Serialize; use deno_core::unsync::spawn; @@ -25,17 +27,17 @@ use deno_core::ByteString; use deno_core::CancelFuture; use deno_core::CancelHandle; use deno_core::CancelTryFuture; +use deno_core::Canceled; use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; -use deno_fetch::get_or_create_client_from_state; use deno_fetch::FetchCancelHandle; -use deno_fetch::FetchError; -use deno_fetch::FetchRequestResource; use deno_fetch::FetchReturn; -use deno_fetch::HttpClientResource; use deno_fetch::ResBody; +use deno_net::io::TcpStreamResource; +use deno_net::ops_tls::TlsStreamResource; +use deno_permissions::PermissionCheckError; use http::header::HeaderMap; use http::header::HeaderName; use http::header::HeaderValue; @@ -44,41 +46,140 @@ use http::header::CONTENT_LENGTH; use http::Method; use http_body_util::BodyExt; use hyper::body::Frame; +use hyper::body::Incoming; use hyper_util::rt::TokioIo; use std::cmp::min; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; -#[op2(stack_trace)] +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NodeHttpResponse { + pub status: u16, + pub status_text: String, + pub headers: Vec<(ByteString, ByteString)>, + pub url: String, + pub response_rid: ResourceId, + pub content_length: Option, + pub remote_addr_ip: Option, + pub remote_addr_port: Option, + pub error: Option, +} + +type CancelableResponseResult = + Result, hyper::Error>, Canceled>; + +pub struct NodeHttpClientResponse { + response: Pin>>, + url: String, +} + +impl Debug for NodeHttpClientResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeHttpClientResponse") + .field("url", &self.url) + .finish() + } +} + +impl deno_core::Resource for NodeHttpClientResponse { + fn name(&self) -> Cow { + "nodeHttpClientResponse".into() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ConnError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error("Invalid URL {0}")] + InvalidUrl(Url), + #[error(transparent)] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Method(#[from] http::method::InvalidMethod), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("TLS stream is currently in use")] + TlsStreamBusy, + #[error("TCP stream is currently in use")] + TcpStreamBusy, + #[error(transparent)] + ReuniteTcp(#[from] tokio::net::tcp::ReuniteError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error(transparent)] + Hyper(#[from] hyper::Error), +} + +#[op2(async, stack_trace)] #[serde] -pub fn op_node_http_request

( - state: &mut OpState, +pub async fn op_node_http_request_with_conn

( + state: Rc>, #[serde] method: ByteString, #[string] url: String, #[serde] headers: Vec<(ByteString, ByteString)>, - #[smi] client_rid: Option, #[smi] body: Option, -) -> Result + #[smi] conn_rid: ResourceId, + encrypted: bool, +) -> Result where P: crate::NodePermissions + 'static, { - let client = if let Some(rid) = client_rid { - let r = state + let (_handle, mut sender) = if encrypted { + let resource_rc = state + .borrow_mut() .resource_table - .get::(rid) - .map_err(FetchError::Resource)?; - r.client.clone() + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_e| ConnError::TlsStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.unsplit(write_half); + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + ( + tokio::task::spawn(async move { conn.with_upgrades().await }), + sender, + ) } else { - get_or_create_client_from_state(state)? + let resource_rc = state + .borrow_mut() + .resource_table + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_| ConnError::TcpStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.reunite(write_half)?; + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + + // Spawn a task to poll the connection, driving the HTTP state + ( + tokio::task::spawn(async move { + conn.with_upgrades().await?; + Ok::<_, _>(()) + }), + sender, + ) }; + // Create the request. let method = Method::from_bytes(&method)?; - let mut url = Url::parse(&url)?; - let maybe_authority = deno_fetch::extract_authority(&mut url); + let mut url_parsed = Url::parse(&url)?; + let maybe_authority = deno_fetch::extract_authority(&mut url_parsed); { - let permissions = state.borrow_mut::

(); - permissions.check_net_url(&url, "ClientRequest")?; + let mut state_ = state.borrow_mut(); + let permissions = state_.borrow_mut::

(); + permissions.check_net_url(&url_parsed, "ClientRequest")?; } let mut header_map = HeaderMap::new(); @@ -93,9 +194,10 @@ where ( BodyExt::boxed(NodeHttpResourceToBodyAdapter::new( state + .borrow_mut() .resource_table .take_any(body) - .map_err(FetchError::Resource)?, + .map_err(ConnError::Resource)?, )), None, ) @@ -117,10 +219,13 @@ where let mut request = http::Request::new(body); *request.method_mut() = method.clone(); - *request.uri_mut() = url - .as_str() + let path = url_parsed.path(); + let query = url_parsed.query(); + *request.uri_mut() = query + .map(|q| format!("{}?{}", path, q)) + .unwrap_or_else(|| path.to_string()) .parse() - .map_err(|_| FetchError::InvalidUrl(url.clone()))?; + .map_err(|_| ConnError::InvalidUrl(url_parsed.clone()))?; *request.headers_mut() = header_map; if let Some((username, password)) = maybe_authority { @@ -136,86 +241,44 @@ where let cancel_handle = CancelHandle::new_rc(); let cancel_handle_ = cancel_handle.clone(); - let fut = async move { - client - .send(request) - .map_err(Into::into) - .or_cancel(cancel_handle_) - .await - }; + let fut = + async move { sender.send_request(request).or_cancel(cancel_handle_).await }; - let request_rid = state.resource_table.add(FetchRequestResource { - future: Box::pin(fut), - url, - }); + let rid = state + .borrow_mut() + .resource_table + .add(NodeHttpClientResponse { + response: Box::pin(fut), + url: url.clone(), + }); - let cancel_handle_rid = - state.resource_table.add(FetchCancelHandle(cancel_handle)); + let cancel_handle_rid = state + .borrow_mut() + .resource_table + .add(FetchCancelHandle(cancel_handle)); Ok(FetchReturn { - request_rid, + request_rid: rid, cancel_handle_rid: Some(cancel_handle_rid), }) } -#[derive(Default, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct NodeHttpFetchResponse { - pub status: u16, - pub status_text: String, - pub headers: Vec<(ByteString, ByteString)>, - pub url: String, - pub response_rid: ResourceId, - pub content_length: Option, - pub remote_addr_ip: Option, - pub remote_addr_port: Option, - pub error: Option, -} - #[op2(async)] #[serde] -pub async fn op_node_http_fetch_send( +pub async fn op_node_http_await_response( state: Rc>, #[smi] rid: ResourceId, -) -> Result { - let request = state +) -> Result { + let resource = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; - - let request = Rc::try_unwrap(request) - .ok() - .expect("multiple op_node_http_fetch_send ongoing"); - - let res = match request.future.await { - Ok(Ok(res)) => res, - Ok(Err(err)) => { - // We're going to try and rescue the error cause from a stream and return it from this fetch. - // If any error in the chain is a hyper body error, return that as a special result we can use to - // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). - // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - - if let FetchError::ClientSend(err_src) = &err { - if let Some(client_err) = std::error::Error::source(&err_src.source) { - if let Some(err_src) = client_err.downcast_ref::() { - if let Some(err_src) = std::error::Error::source(err_src) { - return Ok(NodeHttpFetchResponse { - error: Some(err_src.to_string()), - ..Default::default() - }); - } - } - } - } - - return Err(err); - } - Err(_) => return Err(FetchError::RequestCanceled), - }; + .take::(rid) + .map_err(ConnError::Resource)?; + let resource = Rc::try_unwrap(resource) + .map_err(|_| ConnError::Resource(bad_resource("NodeHttpClientResponse")))?; + let res = resource.response.await??; let status = res.status(); - let url = request.url.into(); let mut res_headers = Vec::new(); for (key, val) in res.headers().iter() { res_headers.push((key.as_str().into(), val.as_bytes().into())); @@ -232,16 +295,22 @@ pub async fn op_node_http_fetch_send( (None, None) }; + let (parts, body) = res.into_parts(); + let body = body.map_err(deno_core::anyhow::Error::from); + let body = body.boxed(); + + let res = http::Response::from_parts(parts, body); + let response_rid = state .borrow_mut() .resource_table - .add(NodeHttpFetchResponseResource::new(res, content_length)); + .add(NodeHttpResponseResource::new(res, content_length)); - Ok(NodeHttpFetchResponse { + Ok(NodeHttpResponse { status: status.as_u16(), status_text: status.canonical_reason().unwrap_or("").to_string(), headers: res_headers, - url, + url: resource.url, response_rid, content_length, remote_addr_ip, @@ -255,12 +324,12 @@ pub async fn op_node_http_fetch_send( pub async fn op_node_http_fetch_response_upgrade( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let raw_response = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; + .take::(rid) + .map_err(ConnError::Resource)?; let raw_response = Rc::try_unwrap(raw_response) .expect("Someone is holding onto NodeHttpFetchResponseResource"); @@ -283,7 +352,7 @@ pub async fn op_node_http_fetch_response_upgrade( } read_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); spawn(async move { let mut buf = [0; 1024]; @@ -294,7 +363,7 @@ pub async fn op_node_http_fetch_response_upgrade( } upgraded_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); } @@ -379,13 +448,13 @@ impl Default for NodeHttpFetchResponseReader { } #[derive(Debug)] -pub struct NodeHttpFetchResponseResource { +pub struct NodeHttpResponseResource { pub response_reader: AsyncRefCell, pub cancel: CancelHandle, pub size: Option, } -impl NodeHttpFetchResponseResource { +impl NodeHttpResponseResource { pub fn new(response: http::Response, size: Option) -> Self { Self { response_reader: AsyncRefCell::new(NodeHttpFetchResponseReader::Start( @@ -400,14 +469,14 @@ impl NodeHttpFetchResponseResource { let reader = self.response_reader.into_inner(); match reader { NodeHttpFetchResponseReader::Start(resp) => { - Ok(hyper::upgrade::on(resp).await?) + hyper::upgrade::on(resp).await } _ => unreachable!(), } } } -impl Resource for NodeHttpFetchResponseResource { +impl Resource for NodeHttpResponseResource { fn name(&self) -> Cow { "fetchResponse".into() } @@ -454,9 +523,7 @@ impl Resource for NodeHttpFetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => { - break Err(deno_core::error::type_error(err.to_string())) - } + Err(err) => break Err(type_error(err.to_string())), }, None => break Ok(BufView::empty()), } @@ -464,7 +531,7 @@ impl Resource for NodeHttpFetchResponseResource { }; let cancel_handle = RcRef::map(self, |r| &r.cancel); - fut.try_or_cancel(cancel_handle).await.map_err(Into::into) + fut.try_or_cancel(cancel_handle).await }) } @@ -514,8 +581,9 @@ impl Stream for NodeHttpResourceToBodyAdapter { Poll::Ready(res) => match res { Ok(buf) if buf.is_empty() => Poll::Ready(None), Ok(buf) => { + let bytes: Bytes = buf.to_vec().into(); this.1 = Some(this.0.clone().read(64 * 1024)); - Poll::Ready(Some(Ok(buf.to_vec().into()))) + Poll::Ready(Some(Ok(bytes))) } Err(err) => Poll::Ready(Some(Err(err))), }, diff --git a/ext/node/polyfills/_http_outgoing.ts b/ext/node/polyfills/_http_outgoing.ts index 4da6b73e87..5a9a8ad7e6 100644 --- a/ext/node/polyfills/_http_outgoing.ts +++ b/ext/node/polyfills/_http_outgoing.ts @@ -491,19 +491,53 @@ Object.defineProperties( return ret; }, + /** Right after socket is ready, we need to writeHeader() to setup the request and + * client. This is invoked by onSocket(). */ + _flushHeaders() { + if (!this._headerSent) { + this._headerSent = true; + this._writeHeader(); + } + }, + // deno-lint-ignore no-explicit-any _send(data: any, encoding?: string | null, callback?: () => void) { - if (!this._headerSent && this._header !== null) { - this._writeHeader(); - this._headerSent = true; + // if socket is ready, write the data after headers are written. + // if socket is not ready, buffer data in outputbuffer. + if ( + this.socket && !this.socket.connecting && this.outputData.length === 0 + ) { + if (!this._headerSent) { + this._writeHeader(); + this._headerSent = true; + } + + return this._writeRaw(data, encoding, callback); + } else { + this.outputData.push({ data, encoding, callback }); } - return this._writeRaw(data, encoding, callback); + return false; }, _writeHeader() { throw new ERR_METHOD_NOT_IMPLEMENTED("_writeHeader()"); }, + _flushBuffer() { + const outputLength = this.outputData.length; + if (outputLength <= 0 || !this.socket || !this._bodyWriter) { + return undefined; + } + + const { data, encoding, callback } = this.outputData.shift(); + const ret = this._writeRaw(data, encoding, callback); + if (this.outputData.length > 0) { + this.once("drain", this._flushBuffer); + } + + return ret; + }, + _writeRaw( // deno-lint-ignore no-explicit-any data: any, @@ -517,11 +551,15 @@ Object.defineProperties( data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } if (data.buffer.byteLength > 0) { - this._bodyWriter.write(data).then(() => { - callback?.(); - this.emit("drain"); - }).catch((e) => { - this._requestSendError = e; + this._bodyWriter.ready.then(() => { + if (this._bodyWriter.desiredSize > 0) { + this._bodyWriter.write(data).then(() => { + callback?.(); + this.emit("drain"); + }).catch((e) => { + this._requestSendError = e; + }); + } }); } return false; @@ -658,7 +696,6 @@ Object.defineProperties( const { header } = state; this._header = header + "\r\n"; - this._headerSent = false; // Wait until the first body chunk, or close(), is sent to flush, // UNLESS we're sending Expect: 100-continue. diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index 4c7424a328..9e5def9f2b 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -154,6 +154,13 @@ export class TLSSocket extends net.Socket { const afterConnect = handle.afterConnect; handle.afterConnect = async (req: any, status: number) => { options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined + if (tlssock._isNpmAgent) { + // skips the TLS handshake for @npmcli/agent as it's handled by + // onSocket handler of ClientRequest object. + tlssock.emit("secure"); + tlssock.removeListener("end", onConnectEnd); + return afterConnect.call(handle, req, status); + } try { const conn = await Deno.startTls(handle[kStreamBaseField], options); diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index 948a3527bd..e911535be5 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -5,16 +5,17 @@ import { core, primordials } from "ext:core/mod.js"; import { + op_node_http_await_response, op_node_http_fetch_response_upgrade, - op_node_http_fetch_send, - op_node_http_request, + op_node_http_request_with_conn, + op_tls_start, } from "ext:core/ops"; import { TextEncoder } from "ext:deno_web/08_text_encoding.js"; import { setTimeout } from "ext:deno_web/02_timers.js"; import { _normalizeArgs, - // createConnection, + createConnection, ListenOptions, Socket, } from "node:net"; @@ -48,9 +49,10 @@ import { kOutHeaders } from "ext:deno_node/internal/http.ts"; import { _checkIsHttpToken as checkIsHttpToken } from "node:_http_common"; import { Agent, globalAgent } from "node:_http_agent"; import { urlToHttpOptions } from "ext:deno_node/internal/url.ts"; -import { kEmptyObject } from "ext:deno_node/internal/util.mjs"; +import { kEmptyObject, once } from "ext:deno_node/internal/util.mjs"; import { constants, TCP } from "ext:deno_node/internal_binding/tcp_wrap.ts"; -import { notImplemented, warnNotImplemented } from "ext:deno_node/_utils.ts"; +import { kStreamBaseField } from "ext:deno_node/internal_binding/stream_wrap.ts"; +import { notImplemented } from "ext:deno_node/_utils.ts"; import { connResetException, ERR_HTTP_HEADERS_SENT, @@ -62,7 +64,6 @@ import { } from "ext:deno_node/internal/errors.ts"; import { getTimerDuration } from "ext:deno_node/internal/timers.mjs"; import { serve, upgradeHttpRaw } from "ext:deno_http/00_serve.ts"; -import { createHttpClient } from "ext:deno_fetch/22_http_client.js"; import { headersEntries } from "ext:deno_fetch/20_headers.js"; import { timerId } from "ext:deno_web/03_abort_signal.js"; import { clearTimeout as webClearTimeout } from "ext:deno_web/02_timers.js"; @@ -148,6 +149,10 @@ class FakeSocket extends EventEmitter { } } +function emitErrorEvent(request, error) { + request.emit("error", error); +} + /** ClientRequest represents the http(s) request from the client */ class ClientRequest extends OutgoingMessage { defaultProtocol = "http:"; @@ -160,6 +165,8 @@ class ClientRequest extends OutgoingMessage { useChunkedEncodingByDefault: boolean; path: string; _req: { requestRid: number; cancelHandleRid: number | null } | undefined; + _encrypted = false; + socket: Socket; constructor( input: string | URL, @@ -382,17 +389,11 @@ class ClientRequest extends OutgoingMessage { delete optsWithoutSignal.signal; } - if (options!.createConnection) { - warnNotImplemented("ClientRequest.options.createConnection"); - } - if (options!.lookup) { notImplemented("ClientRequest.options.lookup"); } - // initiate connection - // TODO(crowlKats): finish this - /*if (this.agent) { + if (this.agent) { this.agent.addRequest(this, optsWithoutSignal); } else { // No agent, default to Connection:close. @@ -422,8 +423,7 @@ class ClientRequest extends OutgoingMessage { debug("CLIENT use net.createConnection", optsWithoutSignal); this.onSocket(createConnection(optsWithoutSignal)); } - }*/ - this.onSocket(new FakeSocket({ encrypted: this._encrypted })); + } } _writeHeader() { @@ -437,9 +437,6 @@ class ClientRequest extends OutgoingMessage { } } - const client = this._getClient() ?? createHttpClient({ http2: false }); - this._client = client; - if ( this.method === "POST" || this.method === "PATCH" || this.method === "PUT" ) { @@ -455,17 +452,29 @@ class ClientRequest extends OutgoingMessage { this._bodyWriteRid = resourceForReadableStream(readable); } - this._req = op_node_http_request( - this.method, - url, - headers, - client[internalRidSymbol], - this._bodyWriteRid, - ); - (async () => { try { - const res = await op_node_http_fetch_send(this._req.requestRid); + const parsedUrl = new URL(url); + let baseConnRid = + this.socket._handle[kStreamBaseField][internalRidSymbol]; + if (this._encrypted) { + [baseConnRid] = op_tls_start({ + rid: baseConnRid, + hostname: parsedUrl.hostname, + caCerts: [], + alpnProtocols: ["http/1.0", "http/1.1"], + }); + } + this._req = await op_node_http_request_with_conn( + this.method, + url, + headers, + this._bodyWriteRid, + baseConnRid, + this._encrypted, + ); + this._flushBuffer(); + const res = await op_node_http_await_response(this._req!.requestRid); if (this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -473,7 +482,6 @@ class ClientRequest extends OutgoingMessage { this._timeout.removeEventListener("abort", this._timeoutCb); webClearTimeout(this._timeout[timerId]); } - this._client.close(); const incoming = new IncomingMessageForClient(this.socket); incoming.req = this; this.res = incoming; @@ -512,12 +520,9 @@ class ClientRequest extends OutgoingMessage { if (this.method === "CONNECT") { throw new Error("not implemented CONNECT"); } - const upgradeRid = await op_node_http_fetch_response_upgrade( res.responseRid, ); - assert(typeof res.remoteAddrIp !== "undefined"); - assert(typeof res.remoteAddrIp !== "undefined"); const conn = new UpgradedConn( upgradeRid, { @@ -543,13 +548,11 @@ class ClientRequest extends OutgoingMessage { this._closed = true; this.emit("close"); } else { - { - incoming._bodyRid = res.responseRid; - } + incoming._bodyRid = res.responseRid; this.emit("response", incoming); } } catch (err) { - if (this._req.cancelHandleRid !== null) { + if (this._req && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -592,11 +595,54 @@ class ClientRequest extends OutgoingMessage { return undefined; } - // TODO(bartlomieju): handle error - onSocket(socket, _err) { + onSocket(socket, err) { nextTick(() => { - this.socket = socket; - this.emit("socket", socket); + // deno-lint-ignore no-this-alias + const req = this; + if (req.destroyed || err) { + req.destroyed = true; + + // deno-lint-ignore no-inner-declarations + function _destroy(req, err) { + if (!req.aborted && !err) { + err = new connResetException("socket hang up"); + } + if (err) { + emitErrorEvent(req, err); + } + req._closed = true; + req.emit("close"); + } + + if (socket) { + if (!err && req.agent && !socket.destroyed) { + socket.emit("free"); + } else { + finished(socket.destroy(err || req[kError]), (er) => { + if (er?.code === "ERR_STREAM_PREMATURE_CLOSE") { + er = null; + } + _destroy(req, er || err); + }); + return; + } + } + + _destroy(req, err || req[kError]); + } else { + // Note: this code is specific to deno to initiate a request. + const onConnect = () => { + // Flush the internal buffers once socket is ready. + this._flushHeaders(); + }; + this.socket = socket; + this.emit("socket", socket); + if (socket.readyState === "opening") { + socket.on("connect", onConnect); + } else { + onConnect(); + } + } }); } @@ -618,14 +664,20 @@ class ClientRequest extends OutgoingMessage { if (chunk) { this.write_(chunk, encoding, null, true); } else if (!this._headerSent) { - this._contentLength = 0; - this._implicitHeader(); - this._send("", "latin1"); + if ( + (this.socket && !this.socket.connecting) || // socket is not connecting, or + (!this.socket && this.outputData.length === 0) // no data to send + ) { + this._contentLength = 0; + this._implicitHeader(); + this._send("", "latin1"); + } } - (async () => { + const finish = async () => { try { + await this._bodyWriter.ready; await this._bodyWriter?.close(); - } catch (_) { + } catch { // The readable stream resource is dropped right after // read is complete closing the writable stream resource. // If we try to close the writer again, it will result in an @@ -633,10 +685,20 @@ class ClientRequest extends OutgoingMessage { } try { cb?.(); - } catch (_) { + } catch { // } - })(); + }; + + if (this.socket && this._bodyWriter) { + finish(); + } else { + this.on("drain", () => { + if (this.outputData.length === 0) { + finish(); + } + }); + } return this; } @@ -658,11 +720,6 @@ class ClientRequest extends OutgoingMessage { } this.destroyed = true; - const rid = this._client?.[internalRidSymbol]; - if (rid) { - core.tryClose(rid); - } - // Request might be closed before we actually made it if (this._req !== undefined && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); diff --git a/ext/node/polyfills/https.ts b/ext/node/polyfills/https.ts index f60c5e471a..fd700173eb 100644 --- a/ext/node/polyfills/https.ts +++ b/ext/node/polyfills/https.ts @@ -112,7 +112,7 @@ export const globalAgent = new Agent({ /** HttpsClientRequest class loosely follows http.ClientRequest class API. */ class HttpsClientRequest extends ClientRequest { - override _encrypted: true; + override _encrypted = true; override defaultProtocol = "https:"; override _getClient(): Deno.HttpClient | undefined { if (caCerts === null) { diff --git a/ext/node/polyfills/internal_binding/cares_wrap.ts b/ext/node/polyfills/internal_binding/cares_wrap.ts index 6feb7faf0d..cbd0bb8ef6 100644 --- a/ext/node/polyfills/internal_binding/cares_wrap.ts +++ b/ext/node/polyfills/internal_binding/cares_wrap.ts @@ -36,7 +36,6 @@ import { } from "ext:deno_node/internal_binding/async_wrap.ts"; import { ares_strerror } from "ext:deno_node/internal_binding/ares.ts"; import { notImplemented } from "ext:deno_node/_utils.ts"; -import { isWindows } from "ext:deno_node/_util/os.ts"; interface LookupAddress { address: string; @@ -68,7 +67,7 @@ export function getaddrinfo( _hints: number, verbatim: boolean, ): number { - let addresses: string[] = []; + const addresses: string[] = []; // TODO(cmorten): use hints // REF: https://nodejs.org/api/dns.html#dns_supported_getaddrinfo_flags @@ -107,13 +106,6 @@ export function getaddrinfo( }); } - // TODO(@bartlomieju): Forces IPv4 as a workaround for Deno not - // aligning with Node on implicit binding on Windows - // REF: https://github.com/denoland/deno/issues/10762 - if (isWindows && hostname === "localhost") { - addresses = addresses.filter((address) => isIPv4(address)); - } - req.oncomplete(error, addresses); })(); diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index 2b01125190..b2b0c9857c 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -986,16 +986,20 @@ function _lookupAndConnect( } else { self._unrefTimer(); - defaultTriggerAsyncIdScope( - self[asyncIdSymbol], - _internalConnect, - self, - ip, - port, - addressType, - localAddress, - localPort, - ); + defaultTriggerAsyncIdScope(self[asyncIdSymbol], nextTick, () => { + if (self.connecting) { + defaultTriggerAsyncIdScope( + self[asyncIdSymbol], + _internalConnect, + self, + ip, + port, + addressType, + localAddress, + localPort, + ); + } + }); } }, ); @@ -1197,6 +1201,9 @@ export class Socket extends Duplex { _host: string | null = null; // deno-lint-ignore no-explicit-any _parent: any = null; + // The flag for detecting if it's called in @npmcli/agent + // See discussions in https://github.com/denoland/deno/pull/25470 for more details. + _isNpmAgent = false; autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined; constructor(options: SocketOptions | number) { @@ -1217,6 +1224,19 @@ export class Socket extends Duplex { super(options); + // Note: If the socket is created from @npmcli/agent, the 'socket' event + // on ClientRequest object happens after 'connect' event on Socket object. + // That swaps the sequence of op_node_http_request_with_conn() call and + // initial socket read. That causes op_node_http_request_with_conn() not + // working. + // To avoid the above situation, we detect the socket created from + // @npmcli/agent and pause the socket (and also skips the startTls call + // if it's TLSSocket) + this._isNpmAgent = new Error().stack?.includes("@npmcli/agent") || false; + if (this._isNpmAgent) { + this.pause(); + } + if (options.handle) { this._handle = options.handle; this[asyncIdSymbol] = _getNewAsyncId(this._handle); diff --git a/runtime/errors.rs b/runtime/errors.rs index 22ba640bcf..3f8e900851 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -712,7 +712,6 @@ fn get_fetch_error(error: &FetchError) -> &'static str { FetchError::ClientSend(_) => "TypeError", FetchError::RequestBuilderHook(_) => "TypeError", FetchError::Io(e) => get_io_error_class(e), - FetchError::Hyper(e) => get_hyper_error_class(e), } } @@ -1083,6 +1082,7 @@ mod node { pub use deno_node::ops::crypto::SignEd25519Error; pub use deno_node::ops::crypto::VerifyEd25519Error; pub use deno_node::ops::fs::FsError; + pub use deno_node::ops::http::ConnError; pub use deno_node::ops::http2::Http2Error; pub use deno_node::ops::idna::IdnaError; pub use deno_node::ops::ipc::IpcError; @@ -1538,6 +1538,24 @@ mod node { pub fn get_verify_ed25519_error(_: &VerifyEd25519Error) -> &'static str { "TypeError" } + + pub fn get_conn_error(e: &ConnError) -> &'static str { + match e { + ConnError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + ConnError::Permission(e) => get_permission_check_error_class(e), + ConnError::InvalidUrl(_) => "TypeError", + ConnError::InvalidHeaderName(_) => "TypeError", + ConnError::InvalidHeaderValue(_) => "TypeError", + ConnError::Url(e) => get_url_parse_error_class(e), + ConnError::Method(_) => "TypeError", + ConnError::Io(e) => get_io_error_class(e), + ConnError::Hyper(e) => super::get_hyper_error_class(e), + ConnError::TlsStreamBusy => "Busy", + ConnError::TcpStreamBusy => "Busy", + ConnError::ReuniteTcp(_) => "Error", + ConnError::Canceled(_) => "Error", + } + } } fn get_os_error(error: &OsError) -> &'static str { @@ -1730,6 +1748,10 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { e.downcast_ref::() .map(node::get_verify_ed25519_error) }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_conn_error) + }) .or_else(|| e.downcast_ref::().map(get_napi_error_class)) .or_else(|| e.downcast_ref::().map(get_web_error_class)) .or_else(|| { diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 105341109c..cda2923789 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -565,9 +565,7 @@ "test-handle-wrap-close-abort.js", "test-http-abort-before-end.js", "test-http-addrequest-localaddress.js", - "test-http-agent-false.js", "test-http-agent-getname.js", - "test-http-agent-keepalive-delay.js", "test-http-agent-maxtotalsockets.js", "test-http-agent-no-protocol.js", "test-http-agent-null.js", @@ -590,7 +588,6 @@ "test-http-client-race.js", "test-http-client-read-in-error.js", "test-http-client-reject-unexpected-agent.js", - "test-http-client-timeout-connect-listener.js", "test-http-client-timeout-with-data.js", "test-http-client-unescaped-path.js", "test-http-client-upload-buf.js", @@ -604,7 +601,6 @@ "test-http-date-header.js", "test-http-decoded-auth.js", "test-http-default-encoding.js", - "test-http-dump-req-when-res-ends.js", "test-http-end-throw-socket-handling.js", "test-http-eof-on-connect.js", "test-http-extra-response.js", @@ -622,7 +618,6 @@ "test-http-hex-write.js", "test-http-highwatermark.js", "test-http-host-headers.js", - "test-http-hostname-typechecking.js", "test-http-incoming-message-destroy.js", "test-http-invalid-path-chars.js", "test-http-invalidheaderfield.js", @@ -1292,10 +1287,7 @@ "test-buffer-creation-regression.js", "test-child-process-exit.js", "test-http-server-keep-alive-timeout-slow-server.js", - "test-net-better-error-messages-port.js", - "test-net-connect-handle-econnrefused.js", "test-net-connect-local-error.js", - "test-net-reconnect-error.js", "test-net-response-size.js", "test-net-server-bind.js", "test-tls-lookup.js", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index 09d68aded7..8ad00c9bfd 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -1,7 +1,7 @@ # Remaining Node Tests -1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining). +1155 tests out of 3681 have been ported from Node 20.11.1 (31.38% ported, 69.14% remaining). NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead. @@ -792,6 +792,8 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-agent-destroyed-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-destroyed-socket.js) - [parallel/test-http-agent-domain-reused-gc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-domain-reused-gc.js) - [parallel/test-http-agent-error-on-idle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-error-on-idle.js) +- [parallel/test-http-agent-false.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-false.js) +- [parallel/test-http-agent-keepalive-delay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive-delay.js) - [parallel/test-http-agent-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive.js) - [parallel/test-http-agent-maxsockets-respected.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets-respected.js) - [parallel/test-http-agent-maxsockets.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets.js) @@ -848,6 +850,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-client-set-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-set-timeout.js) - [parallel/test-http-client-spurious-aborted.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-spurious-aborted.js) - [parallel/test-http-client-timeout-agent.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-agent.js) +- [parallel/test-http-client-timeout-connect-listener.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-connect-listener.js) - [parallel/test-http-client-timeout-event.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-event.js) - [parallel/test-http-client-timeout-on-connect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-on-connect.js) - [parallel/test-http-client-timeout-option-listeners.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-option-listeners.js) @@ -865,6 +868,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-destroyed-socket-write2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-destroyed-socket-write2.js) - [parallel/test-http-dns-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dns-error.js) - [parallel/test-http-double-content-length.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-double-content-length.js) +- [parallel/test-http-dump-req-when-res-ends.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dump-req-when-res-ends.js) - [parallel/test-http-early-hints-invalid-argument.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints-invalid-argument.js) - [parallel/test-http-early-hints.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints.js) - [parallel/test-http-exceptions.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-exceptions.js) @@ -876,6 +880,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-header-badrequest.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-badrequest.js) - [parallel/test-http-header-overflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-overflow.js) - [parallel/test-http-host-header-ipv6-fail.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-host-header-ipv6-fail.js) +- [parallel/test-http-hostname-typechecking.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-hostname-typechecking.js) - [parallel/test-http-incoming-matchKnownFields.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-matchKnownFields.js) - [parallel/test-http-incoming-message-connection-setter.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-connection-setter.js) - [parallel/test-http-incoming-message-options.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-options.js) @@ -2508,9 +2513,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [sequential/test-inspector-port-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-inspector-port-cluster.js) - [sequential/test-module-loading.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-module-loading.js) - [sequential/test-net-GH-5504.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-GH-5504.js) +- [sequential/test-net-better-error-messages-port.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-better-error-messages-port.js) - [sequential/test-net-connect-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-econnrefused.js) +- [sequential/test-net-connect-handle-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-handle-econnrefused.js) - [sequential/test-net-listen-shared-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-listen-shared-ports.js) - [sequential/test-net-localport.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-localport.js) +- [sequential/test-net-reconnect-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-reconnect-error.js) - [sequential/test-net-server-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-server-address.js) - [sequential/test-next-tick-error-spin.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-next-tick-error-spin.js) - [sequential/test-perf-hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-perf-hooks.js) diff --git a/tests/node_compat/test/parallel/test-http-agent-false.js b/tests/node_compat/test/parallel/test-http-agent-false.js deleted file mode 100644 index 60dc16d9b0..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-false.js +++ /dev/null @@ -1,53 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const http = require('http'); - -// Sending `agent: false` when `port: null` is also passed in (i.e. the result -// of a `url.parse()` call with the default port used, 80 or 443), should not -// result in an assertion error... -const opts = { - host: '127.0.0.1', - port: null, - path: '/', - method: 'GET', - agent: false -}; - -// We just want an "error" (no local HTTP server on port 80) or "response" -// to happen (user happens ot have HTTP server running on port 80). -// As long as the process doesn't crash from a C++ assertion then we're good. -const req = http.request(opts); - -// Will be called by either the response event or error event, not both -const oneResponse = common.mustCall(); -req.on('response', oneResponse); -req.on('error', oneResponse); -req.end(); diff --git a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js b/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js deleted file mode 100644 index 7cc6120d73..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js +++ /dev/null @@ -1,43 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); -const { Agent } = require('_http_agent'); - -const agent = new Agent({ - keepAlive: true, - keepAliveMsecs: 1000, -}); - -const server = http.createServer(common.mustCall((req, res) => { - res.end('ok'); -})); - -server.listen(0, common.mustCall(() => { - const createConnection = agent.createConnection; - agent.createConnection = (options, ...args) => { - assert.strictEqual(options.keepAlive, true); - assert.strictEqual(options.keepAliveInitialDelay, agent.keepAliveMsecs); - return createConnection.call(agent, options, ...args); - }; - http.get({ - host: 'localhost', - port: server.address().port, - agent: agent, - path: '/' - }, common.mustCall((res) => { - // for emit end event - res.on('data', () => {}); - res.on('end', () => { - server.close(); - }); - })); -})); diff --git a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js b/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js deleted file mode 100644 index c151d16556..0000000000 --- a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); - -// This test ensures that `ClientRequest.prototype.setTimeout()` does -// not add a listener for the `'connect'` event to the socket if the -// socket is already connected. - -const assert = require('assert'); -const http = require('http'); - -// Maximum allowed value for timeouts. -const timeout = 2 ** 31 - 1; - -const server = http.createServer((req, res) => { - res.end(); -}); - -server.listen(0, common.mustCall(() => { - const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }); - const options = { port: server.address().port, agent: agent }; - - doRequest(options, common.mustCall(() => { - const req = doRequest(options, common.mustCall(() => { - agent.destroy(); - server.close(); - })); - - req.on('socket', common.mustCall((socket) => { - assert.strictEqual(socket.listenerCount('connect'), 0); - })); - })); -})); - -function doRequest(options, callback) { - const req = http.get(options, (res) => { - res.on('end', callback); - res.resume(); - }); - - req.setTimeout(timeout); - return req; -} diff --git a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js b/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js deleted file mode 100644 index 3b94250f5a..0000000000 --- a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js +++ /dev/null @@ -1,73 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const { mustCall } = require('../common'); - -const fs = require('fs'); -const http = require('http'); -const { strictEqual } = require('assert'); - -const server = http.createServer(mustCall(function(req, res) { - strictEqual(req.socket.listenerCount('data'), 1); - req.socket.once('data', mustCall(function() { - // Ensure that a chunk of data is received before calling `res.end()`. - res.end('hello world'); - })); - // This checks if the request gets dumped - // resume will be triggered by res.end(). - req.on('resume', mustCall(function() { - // There is no 'data' event handler anymore - // it gets automatically removed when dumping the request. - strictEqual(req.listenerCount('data'), 0); - req.on('data', mustCall()); - })); - - // We explicitly pause the stream - // so that the following on('data') does not cause - // a resume. - req.pause(); - req.on('data', function() {}); - - // Start sending the response. - res.flushHeaders(); -})); - -server.listen(0, mustCall(function() { - const req = http.request({ - method: 'POST', - port: server.address().port - }); - - // Send the http request without waiting - // for the body. - req.flushHeaders(); - - req.on('response', mustCall(function(res) { - // Pipe the body as soon as we get the headers of the - // response back. - fs.createReadStream(__filename).pipe(req); - - res.resume(); - - // On some platforms the `'end'` event might not be emitted because the - // socket could be destroyed by the other peer while data is still being - // sent. In this case the 'aborted'` event is emitted instead of `'end'`. - // `'close'` is used here because it is always emitted and does not - // invalidate the test. - res.on('close', function() { - server.close(); - }); - })); - - req.on('error', function() { - // An error can happen if there is some data still - // being sent, as the other side is calling .destroy() - // this is safe to ignore. - }); -})); diff --git a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js b/tests/node_compat/test/parallel/test-http-hostname-typechecking.js deleted file mode 100644 index e42384504b..0000000000 --- a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); - -// All of these values should cause http.request() to throw synchronously -// when passed as the value of either options.hostname or options.host -const vals = [{}, [], NaN, Infinity, -Infinity, true, false, 1, 0, new Date()]; - -vals.forEach((v) => { - const received = common.invalidArgTypeHelper(v); - assert.throws( - () => http.request({ hostname: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.hostname" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); - - assert.throws( - () => http.request({ host: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.host" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); -}); - -// These values are OK and should not throw synchronously. -// Only testing for 'hostname' validation so ignore connection errors. -const dontCare = () => {}; -['', undefined, null].forEach((v) => { - http.request({ hostname: v }).on('error', dontCare).end(); - http.request({ host: v }).on('error', dontCare).end(); -}); diff --git a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js b/tests/node_compat/test/sequential/test-net-better-error-messages-port.js deleted file mode 100644 index f718ca3f84..0000000000 --- a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js +++ /dev/null @@ -1,24 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall()); - -c.on('error', common.mustCall(function(error) { - // Family autoselection might be skipped if only a single address is returned by DNS. - const failedAttempt = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(failedAttempt.code, 'ECONNREFUSED'); - assert.strictEqual(failedAttempt.port, common.PORT); - assert.match(failedAttempt.address, /^(127\.0\.0\.1|::1)$/); -})); diff --git a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js b/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js deleted file mode 100644 index 629705564b..0000000000 --- a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js +++ /dev/null @@ -1,39 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); -c.on('connect', common.mustNotCall()); -c.on('error', common.mustCall((e) => { - assert.strictEqual(c.connecting, false); - assert.strictEqual(e.code, 'ECONNREFUSED'); -})); diff --git a/tests/node_compat/test/sequential/test-net-reconnect-error.js b/tests/node_compat/test/sequential/test-net-reconnect-error.js deleted file mode 100644 index 450a0798bf..0000000000 --- a/tests/node_compat/test/sequential/test-net-reconnect-error.js +++ /dev/null @@ -1,50 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); -const N = 20; -let disconnectCount = 0; - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall('client should not have connected')); - -c.on('error', common.mustCall((error) => { - // Family autoselection might be skipped if only a single address is returned by DNS. - const actualError = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(actualError.code, 'ECONNREFUSED'); -}, N + 1)); - -c.on('close', common.mustCall(() => { - if (disconnectCount++ < N) - c.connect(common.PORT); // reconnect -}, N + 1)); diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index 048ddf30f5..e6c36eea19 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -499,7 +499,6 @@ Deno.test("[node/http] send request with non-chunked body", async () => { assert(socket.writable); assert(socket.readable); socket.setKeepAlive(); - socket.destroy(); socket.setTimeout(100); }); req.write("hello "); @@ -512,6 +511,11 @@ Deno.test("[node/http] send request with non-chunked body", async () => { // in order to not cause a flaky test sanitizer failure await new Promise((resolve) => setTimeout(resolve, 100)), ]); + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body", async () => { @@ -559,6 +563,11 @@ Deno.test("[node/http] send request with chunked body", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body as default", async () => { @@ -604,6 +613,11 @@ Deno.test("[node/http] send request with chunked body as default", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] ServerResponse _implicitHeader", async () => { @@ -689,7 +703,7 @@ Deno.test("[node/http] ClientRequest handle non-string headers", async () => { assertEquals(headers!["1"], "2"); }); -Deno.test("[node/http] ClientRequest uses HTTP/1.1", async () => { +Deno.test("[node/https] ClientRequest uses HTTP/1.1", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = https.request("https://localhost:5545/http_version", { @@ -800,8 +814,9 @@ Deno.test("[node/http] ClientRequest search params", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = http.request({ - host: "localhost:4545", - path: "search_params?foo=bar", + host: "localhost", + port: 4545, + path: "/search_params?foo=bar", }, (resp) => { resp.on("data", (chunk) => { body += chunk; @@ -1011,28 +1026,50 @@ Deno.test( Deno.test( "[node/http] client destroy before sending request should not error", - () => { + async () => { + const { resolve, promise } = Promise.withResolvers(); const request = http.request("http://localhost:5929/"); // Calling this would throw request.destroy(); + request.on("error", (e) => { + assertEquals(e.message, "socket hang up"); + }); + request.on("close", () => resolve()); + await promise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }, ); +const isWindows = Deno.build.os === "windows"; + Deno.test( "[node/http] destroyed requests should not be sent", + { sanitizeResources: !isWindows, sanitizeOps: !isWindows }, async () => { let receivedRequest = false; - const server = Deno.serve(() => { + const requestClosed = Promise.withResolvers(); + const ac = new AbortController(); + const server = Deno.serve({ port: 0, signal: ac.signal }, () => { receivedRequest = true; return new Response(null); }); const request = http.request(`http://localhost:${server.addr.port}/`); request.destroy(); request.end("hello"); - - await new Promise((r) => setTimeout(r, 500)); + request.on("error", (err) => { + assert(err.message.includes("socket hang up")); + ac.abort(); + }); + request.on("close", () => { + requestClosed.resolve(); + }); + await requestClosed.promise; assertEquals(receivedRequest, false); - await server.shutdown(); + await server.finished; }, ); @@ -1060,22 +1097,33 @@ Deno.test("[node/https] node:https exports globalAgent", async () => { ); }); -Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", () => { +Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", async () => { { - const req = http.request("http://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + await promise; } { - const req = https.request("https://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + + await promise; } }); From e46f42d052f05eb6dc9eabff3ec11483519ff1c4 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 20:12:13 -0500 Subject: [PATCH 26/33] ci: try fix cache (#27348) --- .github/workflows/ci.generate.ts | 29 +++++++++++++---------------- .github/workflows/ci.yml | 4 ++-- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index d43db76414..5c2de96006 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -59,6 +59,15 @@ const Runners = { const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; +const prCacheKey = `${prCacheKeyPrefix}\${{ github.sha }}`; +const prCachePath = [ + // this must match for save and restore (https://github.com/actions/cache/issues/1444) + "./target", + "!./target/*/gn_out", + "!./target/*/gn_root", + "!./target/*/*.zip", + "!./target/*/*.tar.gz", +].join("\n"); // Note that you may need to add more version to the `apt-get remove` line below if you change this const llvmVersion = 19; @@ -612,7 +621,7 @@ const ci = { `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ hashFiles('Cargo.lock') }}`, // We will try to restore from the closest cargo-home we can find "restore-keys": - `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}`, + `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-`, }, }, { @@ -622,13 +631,7 @@ const ci = { if: "github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/gn_root", - "!./target/*/*.zip", - "!./target/*/*.tar.gz", - ].join("\n"), + path: prCachePath, key: "never_saved", "restore-keys": prCacheKeyPrefix, }, @@ -1080,14 +1083,8 @@ const ci = { if: "(matrix.job == 'test' || matrix.job == 'lint') && github.ref == 'refs/heads/main'", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/*.zip", - "!./target/*/*.sha256sum", - "!./target/*/*.tar.gz", - ].join("\n"), - key: prCacheKeyPrefix + "${{ github.sha }}", + path: prCachePath, + key: prCacheKey, }, }, ]), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29dd694c6c..44eb15cb95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -362,7 +362,7 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -682,8 +682,8 @@ jobs: path: |- ./target !./target/*/gn_out + !./target/*/gn_root !./target/*/*.zip - !./target/*/*.sha256sum !./target/*/*.tar.gz key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: From 3ddbea62c25891f832354134a2b24e738c45fe10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 13 Dec 2024 02:31:07 +0000 Subject: [PATCH 27/33] ci: use self-hosted mac arm runner only on main branch (#27347) We can have 2 concurrent runs now, so it's worth trying it out on `main`. --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 5c2de96006..1f9dc45340 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -42,7 +42,7 @@ const Runners = { os: "macos", arch: "aarch64", runner: - `\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, + `\${{ github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, windowsX86: { os: "windows", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44eb15cb95..91ba16c6ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,12 +68,12 @@ jobs: skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' job: test profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' From bf888d942a8a0f70094f9d8cd90155fe19bc65ca Mon Sep 17 00:00:00 2001 From: Kenta Moriuchi Date: Fri, 13 Dec 2024 21:22:29 +0900 Subject: [PATCH 28/33] feat(ext/web): add `[[ErrorData]]` slot to `DOMException` (#27342) --- ext/web/01_dom_exception.js | 15 +++++++++------ tests/unit_node/util_test.ts | 2 +- tests/wpt/runner/expectation.json | 15 +++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/ext/web/01_dom_exception.js b/ext/web/01_dom_exception.js index 38e4d088e5..db2996e0c6 100644 --- a/ext/web/01_dom_exception.js +++ b/ext/web/01_dom_exception.js @@ -9,14 +9,15 @@ import { primordials } from "ext:core/mod.js"; const { + Error, ErrorPrototype, - ErrorCaptureStackTrace, ObjectDefineProperty, ObjectCreate, ObjectEntries, ObjectHasOwn, ObjectPrototypeIsPrototypeOf, ObjectSetPrototypeOf, + ReflectConstruct, Symbol, SymbolFor, } = primordials; @@ -107,12 +108,14 @@ class DOMException { ); const code = nameToCodeMapping[name] ?? 0; - this[_message] = message; - this[_name] = name; - this[_code] = code; - this[webidl.brand] = webidl.brand; + // execute Error constructor to have stack property and [[ErrorData]] internal slot + const error = ReflectConstruct(Error, [], new.target); + error[_message] = message; + error[_name] = name; + error[_code] = code; + error[webidl.brand] = webidl.brand; - ErrorCaptureStackTrace(this, DOMException); + return error; } get message() { diff --git a/tests/unit_node/util_test.ts b/tests/unit_node/util_test.ts index 6267018b12..af174b0f4d 100644 --- a/tests/unit_node/util_test.ts +++ b/tests/unit_node/util_test.ts @@ -224,7 +224,7 @@ Deno.test({ fn() { assert(util.types.isNativeError(new Error())); assert(util.types.isNativeError(new TypeError())); - assert(!util.types.isNativeError(new DOMException())); + assert(util.types.isNativeError(new DOMException())); }, }); diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index 5776fdb486..6025ce42a4 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -3594,16 +3594,13 @@ "DOMException-constructor-behavior.any.html": true, "DOMException-constructor-behavior.any.worker.html": true, "DOMException-custom-bindings.any.html": true, - "DOMException-custom-bindings.any.worker.html": true + "DOMException-custom-bindings.any.worker.html": true, + "exceptions.html": false }, "class-string-interface.any.html": true, "class-string-interface.any.worker.html": true, - "class-string-iterator-prototype-object.any.html": [ - "Object.prototype.toString applied after deleting @@toStringTag" - ], - "class-string-iterator-prototype-object.any.worker.html": [ - "Object.prototype.toString applied after deleting @@toStringTag" - ], + "class-string-iterator-prototype-object.any.html": true, + "class-string-iterator-prototype-object.any.worker.html": true, "class-string-named-properties-object.window.html": false, "global-immutable-prototype.any.html": [ "Setting to a different prototype" @@ -9754,7 +9751,6 @@ "structured-cloning-error-stack-optional.sub.window.html": [ "page-created Error (cross-site iframe)", "page-created Error (same-origin iframe)", - "page-created DOMException (structuredClone())", "page-created DOMException (cross-site iframe)", "page-created DOMException (same-origin iframe)", "JS-engine-created TypeError (cross-site iframe)", @@ -9762,8 +9758,7 @@ "web API-created TypeError (cross-site iframe)", "web API-created TypeError (same-origin iframe)", "web API-created DOMException (cross-site iframe)", - "web API-created DOMException (same-origin iframe)", - "page-created DOMException (worker)" + "web API-created DOMException (same-origin iframe)" ], "transfer-errors.window.html": false, "window-postmessage.window.html": false From 05de351e9ac57e9d6c6e79ddb340892bfd99f5ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 13 Dec 2024 12:42:10 +0000 Subject: [PATCH 29/33] Revert "ci: use self-hosted mac arm runner only on main branch (#27347)" (#27354) This reverts commit 3ddbea62c25891f832354134a2b24e738c45fe10. Looks like VMs are now slightly different than before and require additional setup. Reverting for now to unblock `main` branch. --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 1f9dc45340..5c2de96006 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -42,7 +42,7 @@ const Runners = { os: "macos", arch: "aarch64", runner: - `\${{ github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, + `\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, windowsX86: { os: "windows", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91ba16c6ee..44eb15cb95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,12 +68,12 @@ jobs: skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: '${{ github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' job: test profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' From 369d68c8483c02034fb139fac76778ec838afb00 Mon Sep 17 00:00:00 2001 From: Yusuke Tanaka Date: Fri, 13 Dec 2024 21:43:03 +0900 Subject: [PATCH 30/33] chore: bump hyper-util to 0.1.10 (#27330) This commit upgrades hyper-util to 0.1.10, the current latest version. This also removes exact version specifier in hyper-util to allow library consumers (e.g. one who wants to use `deno_fetch` as a dependency) to decide which version to use as long as its version is 0.1.z where z >= 10. Specifically, hyper-util 0.1.10 is required by Deno Deploy to tweak `http2_max_header_list_size` (see [hyper-util v0.1.10 changelog](https://github.com/hyperium/hyper-util/releases/tag/v0.1.10)) --- Cargo.lock | 9 ++++----- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d82040854c..077f1e1e18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,9 +658,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cache_control" @@ -4022,9 +4022,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -4035,7 +4035,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 712bdae8b8..27038110d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,7 +142,7 @@ http_v02 = { package = "http", version = "0.2.9" } httparse = "1.8.0" hyper = { version = "1.4.1", features = ["full"] } hyper-rustls = { version = "0.27.2", default-features = false, features = ["http1", "http2", "tls12", "ring"] } -hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy", "server", "server-auto"] } +hyper-util = { version = "0.1.10", features = ["tokio", "client", "client-legacy", "server", "server-auto"] } hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] } indexmap = { version = "2", features = ["serde"] } ipnet = "2.3" From 32b57f7b82d26c40586fad1d08d5bd4f3061e6a8 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Fri, 13 Dec 2024 14:14:55 +0000 Subject: [PATCH 31/33] fix(lsp): sql and component file formatting (#27350) --- cli/lsp/documents.rs | 24 ++++ tests/integration/lsp_tests.rs | 213 +++++++++++++++++++++++++++++++-- tests/util/server/src/lsp.rs | 8 ++ 3 files changed, 235 insertions(+), 10 deletions(-) diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index b9ec8ffc46..bdb64c9da3 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -65,6 +65,12 @@ pub enum LanguageId { Html, Css, Yaml, + Sql, + Svelte, + Vue, + Astro, + Vento, + Nunjucks, Unknown, } @@ -81,6 +87,12 @@ impl LanguageId { LanguageId::Html => Some("html"), LanguageId::Css => Some("css"), LanguageId::Yaml => Some("yaml"), + LanguageId::Sql => Some("sql"), + LanguageId::Svelte => Some("svelte"), + LanguageId::Vue => Some("vue"), + LanguageId::Astro => Some("astro"), + LanguageId::Vento => Some("vto"), + LanguageId::Nunjucks => Some("njk"), LanguageId::Unknown => None, } } @@ -96,6 +108,12 @@ impl LanguageId { LanguageId::Html => Some("text/html"), LanguageId::Css => Some("text/css"), LanguageId::Yaml => Some("application/yaml"), + LanguageId::Sql => None, + LanguageId::Svelte => None, + LanguageId::Vue => None, + LanguageId::Astro => None, + LanguageId::Vento => None, + LanguageId::Nunjucks => None, LanguageId::Unknown => None, } } @@ -123,6 +141,12 @@ impl FromStr for LanguageId { "html" => Ok(Self::Html), "css" => Ok(Self::Css), "yaml" => Ok(Self::Yaml), + "sql" => Ok(Self::Sql), + "svelte" => Ok(Self::Svelte), + "vue" => Ok(Self::Vue), + "astro" => Ok(Self::Astro), + "vento" => Ok(Self::Vento), + "nunjucks" => Ok(Self::Nunjucks), _ => Ok(Self::Unknown), } } diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 6142c55888..13a3c0d69b 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -11544,8 +11544,7 @@ fn lsp_json_import_with_query_string() { fn lsp_format_markdown() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let markdown_file = - source_file(temp_dir.path().join("file.md"), "# Hello World"); + let file = source_file(temp_dir.path().join("file.md"), "# Hello World"); let mut client = context.new_lsp_command().build(); client.initialize_default(); @@ -11553,7 +11552,7 @@ fn lsp_format_markdown() { "textDocument/formatting", json!({ "textDocument": { - "uri": markdown_file.url() + "uri": file.url() }, "options": { "tabSize": 2, @@ -11587,14 +11586,13 @@ fn lsp_format_markdown() { fn lsp_format_html() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let html_file = - source_file(temp_dir.path().join("file.html"), " "); + let file = source_file(temp_dir.path().join("file.html"), " "); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": html_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11627,13 +11625,13 @@ fn lsp_format_html() { fn lsp_format_css() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let css_file = source_file(temp_dir.path().join("file.css"), " foo {}"); + let file = source_file(temp_dir.path().join("file.css"), " foo {}"); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": css_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11666,13 +11664,13 @@ fn lsp_format_css() { fn lsp_format_yaml() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let yaml_file = source_file(temp_dir.path().join("file.yaml"), " foo: 1"); + let file = source_file(temp_dir.path().join("file.yaml"), " foo: 1"); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": yaml_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11701,6 +11699,201 @@ fn lsp_format_yaml() { client.shutdown(); } +#[test] +fn lsp_format_sql() { + let context = TestContextBuilder::new().use_temp_cwd().build(); + let temp_dir = context.temp_dir(); + temp_dir.write( + "deno.json", + json!({ + "unstable": ["fmt-sql"], + }) + .to_string(), + ); + let file = source_file( + temp_dir.path().join("file.sql"), + " CREATE TABLE item (id int NOT NULL IDENTITY(1, 1))", + ); + let mut client = context.new_lsp_command().build(); + client.initialize_default(); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + { + "range": { + "start": { "line": 0, "character": 52 }, + "end": { "line": 0, "character": 52 }, + }, + "newText": "\n", + }, + ]), + ); + client.shutdown(); +} + +#[test] +fn lsp_format_component() { + let context = TestContextBuilder::new().use_temp_cwd().build(); + let temp_dir = context.temp_dir(); + temp_dir.write( + "deno.json", + json!({ + "unstable": ["fmt-component"], + }) + .to_string(), + ); + let svelte_file = source_file( + temp_dir.path().join("file.svelte"), + " \n", + ); + let vue_file = source_file( + temp_dir.path().join("file.vue"), + " \n", + ); + let astro_file = source_file( + temp_dir.path().join("file.astro"), + " ---\n// foo\n---\n\n", + ); + let vento_file = source_file( + temp_dir.path().join("file.vto"), + " {{ layout \"foo.vto\" }}\n

Foo!

\n{{ /layout }}\n", + ); + let nunjucks_file = source_file( + temp_dir.path().join("file.njk"), + " {% block header %}\n Foo\n{% endblock %}\n", + ); + let mut client = context.new_lsp_command().build(); + client.initialize_default(); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": svelte_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": vue_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": astro_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": vento_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": nunjucks_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + client.shutdown(); +} + #[test] fn lsp_format_with_config() { let context = TestContextBuilder::new().use_temp_cwd().build(); diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index d34deb2161..92169ee644 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -1290,6 +1290,14 @@ impl SourceFile { "html" => "html", "css" => "css", "yaml" => "yaml", + "sql" => "sql", + "svelte" => "svelte", + "vue" => "vue", + "astro" => "astro", + "vto" => "vento", + "vento" => "vento", + "njk" => "nunjucks", + "nunjucks" => "nunjucks", other => panic!("unsupported file extension: {other}"), }; Self { From 3946956b8cbda33b84d6f9dc80c96f09c3afcd37 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:19:37 -0800 Subject: [PATCH 32/33] fix(lockfile): include dependencies listed in external import map in lockfile (#27337) --- cli/args/deno_json.rs | 9 ++++ cli/args/lockfile.rs | 9 +++- cli/args/mod.rs | 48 +++++++++++++++++-- cli/lsp/language_server.rs | 1 + .../external_import_map/__test__.jsonc | 10 ++++ .../lockfile/external_import_map/deno.json | 3 ++ .../external_import_map/deno.lock.out | 17 +++++++ .../external_import_map/import_map.json | 10 ++++ .../lockfile/external_import_map/main.ts | 2 + 9 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 tests/specs/lockfile/external_import_map/__test__.jsonc create mode 100644 tests/specs/lockfile/external_import_map/deno.json create mode 100644 tests/specs/lockfile/external_import_map/deno.lock.out create mode 100644 tests/specs/lockfile/external_import_map/import_map.json create mode 100644 tests/specs/lockfile/external_import_map/main.ts diff --git a/cli/args/deno_json.rs b/cli/args/deno_json.rs index c2ba31fd36..8853107eef 100644 --- a/cli/args/deno_json.rs +++ b/cli/args/deno_json.rs @@ -64,6 +64,15 @@ impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> { } } +pub fn import_map_deps( + import_map: &serde_json::Value, +) -> HashSet { + let values = imports_values(import_map.get("imports")) + .into_iter() + .chain(scope_values(import_map.get("scopes"))); + values_to_set(values) +} + pub fn deno_json_deps( config: &deno_config::deno_json::ConfigFile, ) -> HashSet { diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index 74eab78f1c..1075f93a6f 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -9,11 +9,13 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::MutexGuard; +use deno_core::serde_json; use deno_lockfile::WorkspaceMemberConfig; use deno_package_json::PackageJsonDepValue; use deno_runtime::deno_node::PackageJson; use deno_semver::jsr::JsrDepPackageReq; +use crate::args::deno_json::import_map_deps; use crate::cache; use crate::util::fs::atomic_write_file_with_retries; use crate::Flags; @@ -101,6 +103,7 @@ impl CliLockfile { pub fn discover( flags: &Flags, workspace: &Workspace, + maybe_external_import_map: Option<&serde_json::Value>, ) -> Result, AnyError> { fn pkg_json_deps( maybe_pkg_json: Option<&PackageJson>, @@ -171,7 +174,11 @@ impl CliLockfile { let config = deno_lockfile::WorkspaceConfig { root: WorkspaceMemberConfig { package_json_deps: pkg_json_deps(root_folder.pkg_json.as_deref()), - dependencies: deno_json_deps(root_folder.deno_json.as_deref()), + dependencies: if let Some(map) = maybe_external_import_map { + import_map_deps(map) + } else { + deno_json_deps(root_folder.deno_json.as_deref()) + }, }, members: workspace .config_folders() diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 71f79e12e0..450aa11652 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -808,6 +808,7 @@ pub struct CliOptions { maybe_node_modules_folder: Option, npmrc: Arc, maybe_lockfile: Option>, + maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, overrides: CliOptionOverrides, pub start_dir: Arc, pub deno_dir_provider: Arc, @@ -821,6 +822,7 @@ impl CliOptions { npmrc: Arc, start_dir: Arc, force_global_cache: bool, + maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, ) -> Result { if let Some(insecure_allowlist) = flags.unsafely_ignore_certificate_errors.as_ref() @@ -858,6 +860,7 @@ impl CliOptions { maybe_node_modules_folder, overrides: Default::default(), main_module_cell: std::sync::OnceLock::new(), + maybe_external_import_map, start_dir, deno_dir_provider, }) @@ -933,7 +936,33 @@ impl CliOptions { let (npmrc, _) = discover_npmrc_from_workspace(&start_dir.workspace)?; - let maybe_lock_file = CliLockfile::discover(&flags, &start_dir.workspace)?; + fn load_external_import_map( + deno_json: &ConfigFile, + ) -> Result, AnyError> { + if !deno_json.is_an_import_map() { + if let Some(path) = deno_json.to_import_map_path()? { + let contents = std::fs::read_to_string(&path).with_context(|| { + format!("Unable to read import map at '{}'", path.display()) + })?; + let map = serde_json::from_str(&contents)?; + return Ok(Some((path, map))); + } + } + Ok(None) + } + + let external_import_map = + if let Some(deno_json) = start_dir.workspace.root_deno_json() { + load_external_import_map(deno_json)? + } else { + None + }; + + let maybe_lock_file = CliLockfile::discover( + &flags, + &start_dir.workspace, + external_import_map.as_ref().map(|(_, v)| v), + )?; log::debug!("Finished config loading."); @@ -944,6 +973,7 @@ impl CliOptions { npmrc, Arc::new(start_dir), false, + external_import_map, ) } @@ -1064,7 +1094,7 @@ impl CliOptions { file_fetcher: &FileFetcher, pkg_json_dep_resolution: PackageJsonDepResolution, ) -> Result { - let overrode_no_import_map = self + let overrode_no_import_map: bool = self .overrides .import_map_specifier .as_ref() @@ -1092,7 +1122,19 @@ impl CliOptions { value, }) } - None => None, + None => { + if let Some((path, import_map)) = + self.maybe_external_import_map.as_ref() + { + let path_url = deno_path_util::url_from_file_path(path)?; + Some(deno_config::workspace::SpecifiedImportMap { + base_url: path_url, + value: import_map.clone(), + }) + } else { + None + } + } } }; Ok(self.workspace().create_resolver( diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 3c4cb0930e..839d28469e 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -3676,6 +3676,7 @@ impl Inner { .unwrap_or_else(create_default_npmrc), workspace, force_global_cache, + None, )?; let open_docs = self.documents.documents(DocumentsFilter::OpenDiagnosable); diff --git a/tests/specs/lockfile/external_import_map/__test__.jsonc b/tests/specs/lockfile/external_import_map/__test__.jsonc new file mode 100644 index 0000000000..2bdffed334 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/__test__.jsonc @@ -0,0 +1,10 @@ +{ + "tempDir": true, + "steps": [{ + "args": "run -A main.ts", + "output": "[WILDCARD]" + }, { + "args": ["eval", "console.log(Deno.readTextFileSync('deno.lock').trim())"], + "output": "deno.lock.out" + }] +} diff --git a/tests/specs/lockfile/external_import_map/deno.json b/tests/specs/lockfile/external_import_map/deno.json new file mode 100644 index 0000000000..ee44ba9472 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/deno.json @@ -0,0 +1,3 @@ +{ + "importMap": "import_map.json" +} diff --git a/tests/specs/lockfile/external_import_map/deno.lock.out b/tests/specs/lockfile/external_import_map/deno.lock.out new file mode 100644 index 0000000000..c811061125 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/deno.lock.out @@ -0,0 +1,17 @@ +{ + "version": "4", + "specifiers": { + "jsr:@denotest/add@1.0.0": "1.0.0" + }, + "jsr": { + "@denotest/add@1.0.0": { + "integrity": "[WILDLINE]" + } + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/add@1.0.0", + "npm:@denotest/esm-basic@1.0.0" + ] + } +} diff --git a/tests/specs/lockfile/external_import_map/import_map.json b/tests/specs/lockfile/external_import_map/import_map.json new file mode 100644 index 0000000000..069b294ce4 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/import_map.json @@ -0,0 +1,10 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@1.0.0" + }, + "scopes": { + "/foo/": { + "@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + } + } +} diff --git a/tests/specs/lockfile/external_import_map/main.ts b/tests/specs/lockfile/external_import_map/main.ts new file mode 100644 index 0000000000..b75bbc03ed --- /dev/null +++ b/tests/specs/lockfile/external_import_map/main.ts @@ -0,0 +1,2 @@ +import { add } from "@denotest/add"; +console.log(add(1, 2)); From 9d315f27ed4c583bdf7517c6846c08b4e22e1e8e Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:25:05 -0800 Subject: [PATCH 33/33] fix(outdated): support updating dependencies in external import maps (#27339) Fixes #27331. The support for it was already in `outdated`, but forgot to wire up the updating part Needs #27337 --- cli/tools/registry/pm/deps.rs | 61 +++++++++++-------- .../update/external_import_map/__test__.jsonc | 24 ++++++++ .../update/external_import_map/deno.json | 3 + .../update/external_import_map/deno.lock | 33 ++++++++++ .../external_import_map/import_map.json | 8 +++ .../external_import_map/import_map.json.out | 8 +++ .../specs/update/external_import_map/main.ts | 1 + .../update/external_import_map/outdated.out | 14 +++++ .../update/external_import_map/update.out | 11 ++++ 9 files changed, 137 insertions(+), 26 deletions(-) create mode 100644 tests/specs/update/external_import_map/__test__.jsonc create mode 100644 tests/specs/update/external_import_map/deno.json create mode 100644 tests/specs/update/external_import_map/deno.lock create mode 100644 tests/specs/update/external_import_map/import_map.json create mode 100644 tests/specs/update/external_import_map/import_map.json.out create mode 100644 tests/specs/update/external_import_map/main.ts create mode 100644 tests/specs/update/external_import_map/outdated.out create mode 100644 tests/specs/update/external_import_map/update.out diff --git a/cli/tools/registry/pm/deps.rs b/cli/tools/registry/pm/deps.rs index d82e9954cd..e4c38276f7 100644 --- a/cli/tools/registry/pm/deps.rs +++ b/cli/tools/registry/pm/deps.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::collections::HashMap; +use std::path::PathBuf; use std::sync::atomic::AtomicBool; use std::sync::Arc; @@ -11,6 +12,7 @@ use deno_config::deno_json::ConfigFileRc; use deno_config::workspace::Workspace; use deno_config::workspace::WorkspaceDirectory; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::future::try_join; use deno_core::futures::stream::FuturesOrdered; @@ -43,10 +45,10 @@ use crate::npm::NpmFetchResolver; use super::ConfigUpdater; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum ImportMapKind { Inline, - Outline, + Outline(PathBuf), } #[derive(Clone)] @@ -62,9 +64,12 @@ impl DepLocation { pub fn file_path(&self) -> Cow { match self { - DepLocation::DenoJson(arc, _, _) => { - Cow::Owned(arc.specifier.to_file_path().unwrap()) - } + DepLocation::DenoJson(arc, _, kind) => match kind { + ImportMapKind::Inline => { + Cow::Owned(arc.specifier.to_file_path().unwrap()) + } + ImportMapKind::Outline(path) => Cow::Borrowed(path.as_path()), + }, DepLocation::PackageJson(arc, _) => Cow::Borrowed(arc.path.as_ref()), } } @@ -238,22 +243,30 @@ fn to_import_map_value_from_imports( fn deno_json_import_map( deno_json: &ConfigFile, ) -> Result, AnyError> { - let (value, kind) = - if deno_json.json.imports.is_some() || deno_json.json.scopes.is_some() { - ( - to_import_map_value_from_imports(deno_json), - ImportMapKind::Inline, - ) - } else { - match deno_json.to_import_map_path()? { - Some(path) => { - let text = std::fs::read_to_string(&path)?; - let value = serde_json::from_str(&text)?; - (value, ImportMapKind::Outline) - } - None => return Ok(None), + let (value, kind) = if deno_json.json.imports.is_some() + || deno_json.json.scopes.is_some() + { + ( + to_import_map_value_from_imports(deno_json), + ImportMapKind::Inline, + ) + } else { + match deno_json.to_import_map_path()? { + Some(path) => { + let err_context = || { + format!( + "loading import map at '{}' (from \"importMap\" field in '{}')", + path.display(), + deno_json.specifier + ) + }; + let text = std::fs::read_to_string(&path).with_context(err_context)?; + let value = serde_json::from_str(&text).with_context(err_context)?; + (value, ImportMapKind::Outline(path)) } - }; + None => return Ok(None), + } + }; import_map::parse_from_value(deno_json.specifier.clone(), value) .map_err(Into::into) @@ -303,7 +316,7 @@ fn add_deps_from_deno_json( location: DepLocation::DenoJson( deno_json.clone(), key_path, - import_map_kind, + import_map_kind.clone(), ), kind, req, @@ -747,11 +760,7 @@ impl DepManager { let dep = &mut self.deps[dep_id.0]; dep.req.version_req = version_req.clone(); match &dep.location { - DepLocation::DenoJson(arc, key_path, import_map_kind) => { - if matches!(import_map_kind, ImportMapKind::Outline) { - // not supported - continue; - } + DepLocation::DenoJson(arc, key_path, _) => { let updater = get_or_create_updater(&mut config_updaters, &dep.location)?; diff --git a/tests/specs/update/external_import_map/__test__.jsonc b/tests/specs/update/external_import_map/__test__.jsonc new file mode 100644 index 0000000000..66dc55cc57 --- /dev/null +++ b/tests/specs/update/external_import_map/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "i", + "output": "[WILDCARD]" + }, + { + "args": "outdated", + "output": "outdated.out" + }, + { + "args": "outdated --update --latest", + "output": "update.out" + }, + { + "args": [ + "eval", + "console.log(Deno.readTextFileSync('import_map.json').trim())" + ], + "output": "import_map.json.out" + } + ] +} diff --git a/tests/specs/update/external_import_map/deno.json b/tests/specs/update/external_import_map/deno.json new file mode 100644 index 0000000000..ee44ba9472 --- /dev/null +++ b/tests/specs/update/external_import_map/deno.json @@ -0,0 +1,3 @@ +{ + "importMap": "import_map.json" +} diff --git a/tests/specs/update/external_import_map/deno.lock b/tests/specs/update/external_import_map/deno.lock new file mode 100644 index 0000000000..940f2527d8 --- /dev/null +++ b/tests/specs/update/external_import_map/deno.lock @@ -0,0 +1,33 @@ +{ + "version": "4", + "specifiers": { + "jsr:@denotest/add@0.2": "0.2.0", + "jsr:@denotest/subtract@0.2": "0.2.0", + "npm:@denotest/breaking-change-between-versions@1.0.0": "1.0.0", + "npm:@denotest/has-patch-versions@0.1": "0.1.0" + }, + "jsr": { + "@denotest/add@0.2.0": { + "integrity": "a9076d30ecb42b2fc6dd95e7055fbf4e6358b53f550741bd7f60089d19f68848" + }, + "@denotest/subtract@0.2.0": { + "integrity": "c9650fc559ab2430effc0c7fb1540e3aa89888fbdd926335ccfdeac57eb3a64d" + } + }, + "npm": { + "@denotest/breaking-change-between-versions@1.0.0": { + "integrity": "sha512-bzMGYx+DxxPlI74n/VsDAN7Db1BY7Sz2XqxXruMo9dEznsBZu7Ez3i8YQ8n0leTxAiiMk1RCG4zQHPG1aj3xRw==" + }, + "@denotest/has-patch-versions@0.1.0": { + "integrity": "sha512-H/MBo0jKDdMsX4AAGEGQbZj70nfNe3oUNZXbohYHhqf9EfpLnXp/7FC29ZdfV4+p6VjEcOGdCtXc6rilE6iYpg==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/add@0.2", + "jsr:@denotest/subtract@0.2", + "npm:@denotest/breaking-change-between-versions@1.0.0", + "npm:@denotest/has-patch-versions@0.1" + ] + } +} diff --git a/tests/specs/update/external_import_map/import_map.json b/tests/specs/update/external_import_map/import_map.json new file mode 100644 index 0000000000..9cc5c77e24 --- /dev/null +++ b/tests/specs/update/external_import_map/import_map.json @@ -0,0 +1,8 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@^0.2.0", + "@denotest/subtract": "jsr:@denotest/subtract@^0.2.0", + "@denotest/breaking-change-between-versions": "npm:@denotest/breaking-change-between-versions@1.0.0", + "@denotest/has-patch-versions": "npm:@denotest/has-patch-versions@^0.1.0" + } +} diff --git a/tests/specs/update/external_import_map/import_map.json.out b/tests/specs/update/external_import_map/import_map.json.out new file mode 100644 index 0000000000..b4e24decbc --- /dev/null +++ b/tests/specs/update/external_import_map/import_map.json.out @@ -0,0 +1,8 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@^1.0.0", + "@denotest/subtract": "jsr:@denotest/subtract@^1.0.0", + "@denotest/breaking-change-between-versions": "npm:@denotest/breaking-change-between-versions@^2.0.0", + "@denotest/has-patch-versions": "npm:@denotest/has-patch-versions@^0.2.0" + } +} diff --git a/tests/specs/update/external_import_map/main.ts b/tests/specs/update/external_import_map/main.ts new file mode 100644 index 0000000000..b008bb0d41 --- /dev/null +++ b/tests/specs/update/external_import_map/main.ts @@ -0,0 +1 @@ +import { add } from "@denotest/add"; diff --git a/tests/specs/update/external_import_map/outdated.out b/tests/specs/update/external_import_map/outdated.out new file mode 100644 index 0000000000..8752b132bf --- /dev/null +++ b/tests/specs/update/external_import_map/outdated.out @@ -0,0 +1,14 @@ +┌────────────────────────────────────────────────┬─────────┬────────┬────────┐ +│ Package │ Current │ Update │ Latest │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/add │ 0.2.0 │ 0.2.1 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ +└────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/external_import_map/update.out b/tests/specs/update/external_import_map/update.out new file mode 100644 index 0000000000..d075cff296 --- /dev/null +++ b/tests/specs/update/external_import_map/update.out @@ -0,0 +1,11 @@ +[UNORDERED_START] +Download http://127.0.0.1:4250/@denotest/subtract/1.0.0/mod.ts +Download http://127.0.0.1:4250/@denotest/add/1.0.0/mod.ts +Download http://localhost:4260/@denotest/has-patch-versions/0.2.0.tgz +Download http://localhost:4260/@denotest/breaking-change-between-versions/2.0.0.tgz +[UNORDERED_END] +Updated 4 dependencies: + - jsr:@denotest/add 0.2.0 -> 1.0.0 + - jsr:@denotest/subtract 0.2.0 -> 1.0.0 + - npm:@denotest/breaking-change-between-versions 1.0.0 -> 2.0.0 + - npm:@denotest/has-patch-versions 0.1.0 -> 0.2.0