From 5f8be055db1b276d2bb2d986ac0780997ddc68c4 Mon Sep 17 00:00:00 2001 From: snek Date: Thu, 12 Dec 2024 09:17:26 +0100 Subject: [PATCH 1/8] refactor(unstable): otel configuration (#27333) split up otel config into user configurable and runtime configurable parts. user configurable part is now set via env vars parsed according to the otel spec. otel is now enabled via `OTEL_DENO=true`, and `--unstable-otel` only acts as a guard. Fixes: https://github.com/denoland/deno/issues/27273 --- cli/args/flags.rs | 47 +++++++++++++++++++-------- cli/args/mod.rs | 10 +++++- cli/main.rs | 10 +++--- cli/mainrt.rs | 11 ++++--- cli/standalone/binary.rs | 2 +- cli/util/logger.rs | 51 +++++++++++++++++++++++------- cli/worker.rs | 4 +-- ext/telemetry/lib.rs | 32 ++++++++++++------- ext/telemetry/telemetry.ts | 10 +++--- runtime/worker_bootstrap.rs | 11 ++----- tests/specs/cli/otel_basic/main.ts | 1 + 11 files changed, 124 insertions(+), 65 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 9418739564..418edcf34b 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -37,6 +37,7 @@ use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsOptions; use deno_runtime::deno_permissions::SysDescriptor; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use log::debug; use log::Level; use serde::Deserialize; @@ -986,21 +987,41 @@ impl Flags { args } - pub fn otel_config(&self) -> Option { - if self + pub fn otel_config(&self) -> OtelConfig { + let has_unstable_flag = self .unstable_config .features - .contains(&String::from("otel")) - { - Some(OtelConfig { - runtime_name: Cow::Borrowed("deno"), - runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), - deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") - .is_ok(), - ..Default::default() - }) - } else { - None + .contains(&String::from("otel")); + + let otel_var = |name| match std::env::var(name) { + Ok(s) if s.to_lowercase() == "true" => Some(true), + Ok(s) if s.to_lowercase() == "false" => Some(false), + _ => None, + }; + + let disabled = + !has_unstable_flag || otel_var("OTEL_SDK_DISABLED").unwrap_or(false); + let default = !disabled && otel_var("OTEL_DENO").unwrap_or(false); + + OtelConfig { + tracing_enabled: !disabled + && otel_var("OTEL_DENO_TRACING").unwrap_or(default), + console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() { + Ok(_) if disabled => OtelConsoleConfig::Ignore, + Ok("ignore") => OtelConsoleConfig::Ignore, + Ok("capture") => OtelConsoleConfig::Capture, + Ok("replace") => OtelConsoleConfig::Replace, + _ => { + if default { + OtelConsoleConfig::Capture + } else { + OtelConsoleConfig::Ignore + } + } + }, + deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") + .as_deref() + == Ok("1"), } } diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 314c0ff17a..71f79e12e0 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -31,6 +31,7 @@ use deno_npm_cache::NpmCacheSetting; use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelRuntimeConfig; use import_map::resolve_import_map_value_from_specifier; pub use deno_config::deno_json::BenchConfig; @@ -1130,7 +1131,7 @@ impl CliOptions { } } - pub fn otel_config(&self) -> Option { + pub fn otel_config(&self) -> OtelConfig { self.flags.otel_config() } @@ -2000,6 +2001,13 @@ pub enum NpmCachingStrategy { Manual, } +pub(crate) fn otel_runtime_config() -> OtelRuntimeConfig { + OtelRuntimeConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + } +} + #[cfg(test)] mod test { use pretty_assertions::assert_eq; diff --git a/cli/main.rs b/cli/main.rs index d47f1e363c..0594739fd8 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -437,20 +437,18 @@ fn resolve_flags_and_init( if err.kind() == clap::error::ErrorKind::DisplayVersion => { // Ignore results to avoid BrokenPipe errors. - util::logger::init(None); + util::logger::init(None, None); let _ = err.print(); deno_runtime::exit(0); } Err(err) => { - util::logger::init(None); + util::logger::init(None, None); exit_for_error(AnyError::from(err)) } }; - if let Some(otel_config) = flags.otel_config() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(flags.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init(flags.log_level, Some(flags.otel_config())); // TODO(bartlomieju): remove in Deno v2.5 and hard error then. if flags.unstable_config.legacy_flag_enabled { diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 7ad3b3744b..18142bd0e7 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -87,17 +87,18 @@ fn main() { let future = async move { match standalone { Ok(Some(data)) => { - if let Some(otel_config) = data.metadata.otel_config.clone() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(data.metadata.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init( + data.metadata.log_level, + Some(data.metadata.otel_config.clone()), + ); load_env_vars(&data.metadata.env_vars_from_env_file); let exit_code = standalone::run(data).await?; deno_runtime::exit(exit_code); } Ok(None) => Ok(()), Err(err) => { - util::logger::init(None); + util::logger::init(None, None); Err(err) } } diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 7728728a26..1e7b0d3f70 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -192,7 +192,7 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, - pub otel_config: Option, // None means disabled. + pub otel_config: OtelConfig, } fn write_binary_bytes( diff --git a/cli/util/logger.rs b/cli/util/logger.rs index 2b8987c3e7..783f8a5f68 100644 --- a/cli/util/logger.rs +++ b/cli/util/logger.rs @@ -1,24 +1,34 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use super::draw_thread::DrawThread; +use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use std::io::Write; -use super::draw_thread::DrawThread; - -struct CliLogger(env_logger::Logger); +struct CliLogger { + otel_console_config: OtelConsoleConfig, + logger: env_logger::Logger, +} impl CliLogger { - pub fn new(logger: env_logger::Logger) -> Self { - Self(logger) + pub fn new( + logger: env_logger::Logger, + otel_console_config: OtelConsoleConfig, + ) -> Self { + Self { + logger, + otel_console_config, + } } pub fn filter(&self) -> log::LevelFilter { - self.0.filter() + self.logger.filter() } } impl log::Log for CliLogger { fn enabled(&self, metadata: &log::Metadata) -> bool { - self.0.enabled(metadata) + self.logger.enabled(metadata) } fn log(&self, record: &log::Record) { @@ -28,18 +38,30 @@ impl log::Log for CliLogger { // could potentially block other threads that access the draw // thread's state DrawThread::hide(); - self.0.log(record); - deno_telemetry::handle_log(record); + + match self.otel_console_config { + OtelConsoleConfig::Ignore => { + self.logger.log(record); + } + OtelConsoleConfig::Capture => { + self.logger.log(record); + deno_telemetry::handle_log(record); + } + OtelConsoleConfig::Replace => { + deno_telemetry::handle_log(record); + } + } + DrawThread::show(); } } fn flush(&self) { - self.0.flush(); + self.logger.flush(); } } -pub fn init(maybe_level: Option) { +pub fn init(maybe_level: Option, otel_config: Option) { let log_level = maybe_level.unwrap_or(log::Level::Info); let logger = env_logger::Builder::from_env( env_logger::Env::new() @@ -93,7 +115,12 @@ pub fn init(maybe_level: Option) { }) .build(); - let cli_logger = CliLogger::new(logger); + let cli_logger = CliLogger::new( + logger, + otel_config + .map(|c| c.console) + .unwrap_or(OtelConsoleConfig::Ignore), + ); let max_level = cli_logger.filter(); let r = log::set_boxed_logger(Box::new(cli_logger)); if r.is_ok() { diff --git a/cli/worker.rs b/cli/worker.rs index 81b8cd2f83..0bbc27b29f 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -154,7 +154,7 @@ struct SharedWorkerState { storage_key_resolver: StorageKeyResolver, options: CliMainWorkerOptions, subcommand: DenoSubcommand, - otel_config: Option, // `None` means OpenTelemetry is disabled. + otel_config: OtelConfig, default_npm_caching_strategy: NpmCachingStrategy, } @@ -426,7 +426,7 @@ impl CliMainWorkerFactory { storage_key_resolver: StorageKeyResolver, subcommand: DenoSubcommand, options: CliMainWorkerOptions, - otel_config: Option, + otel_config: OtelConfig, default_npm_caching_strategy: NpmCachingStrategy, ) -> Self { Self { diff --git a/ext/telemetry/lib.rs b/ext/telemetry/lib.rs index 06210a70e7..816e838743 100644 --- a/ext/telemetry/lib.rs +++ b/ext/telemetry/lib.rs @@ -97,13 +97,28 @@ deno_core::extension!( ); #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OtelConfig { +pub struct OtelRuntimeConfig { pub runtime_name: Cow<'static, str>, pub runtime_version: Cow<'static, str>, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct OtelConfig { + pub tracing_enabled: bool, pub console: OtelConsoleConfig, pub deterministic: bool, } +impl OtelConfig { + pub fn as_v8(&self) -> Box<[u8]> { + Box::new([ + self.tracing_enabled as u8, + self.console as u8, + self.deterministic as u8, + ]) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[repr(u8)] pub enum OtelConsoleConfig { @@ -112,14 +127,9 @@ pub enum OtelConsoleConfig { Replace = 2, } -impl Default for OtelConfig { +impl Default for OtelConsoleConfig { fn default() -> Self { - Self { - runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")), - runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), - console: OtelConsoleConfig::Capture, - deterministic: false, - } + Self::Ignore } } @@ -411,16 +421,14 @@ static BUILT_IN_INSTRUMENTATION_SCOPE: OnceCell< opentelemetry::InstrumentationScope, > = OnceCell::new(); -pub fn init(config: OtelConfig) -> anyhow::Result<()> { +pub fn init(config: OtelRuntimeConfig) -> anyhow::Result<()> { // Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_* // crates don't do this automatically. // TODO(piscisaureus): enable GRPC support. let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() { Ok("http/protobuf") => Protocol::HttpBinary, Ok("http/json") => Protocol::HttpJson, - Ok("") | Err(env::VarError::NotPresent) => { - return Ok(()); - } + Ok("") | Err(env::VarError::NotPresent) => Protocol::HttpBinary, Ok(protocol) => { return Err(anyhow!( "Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}", diff --git a/ext/telemetry/telemetry.ts b/ext/telemetry/telemetry.ts index acdfd4d715..d1335f65b5 100644 --- a/ext/telemetry/telemetry.ts +++ b/ext/telemetry/telemetry.ts @@ -950,15 +950,15 @@ const otelConsoleConfig = { }; export function bootstrap( - config: [] | [ + config: [ + 0 | 1, typeof otelConsoleConfig[keyof typeof otelConsoleConfig], - number, + 0 | 1, ], ): void { - if (config.length === 0) return; - const { 0: consoleConfig, 1: deterministic } = config; + const { 0: tracingEnabled, 1: consoleConfig, 2: deterministic } = config; - TRACING_ENABLED = true; + TRACING_ENABLED = tracingEnabled === 1; DETERMINISTIC = deterministic === 1; switch (consoleConfig) { diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index 4a8c5dba86..2020c2bc8d 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -119,8 +119,7 @@ pub struct BootstrapOptions { // Used by `deno serve` pub serve_port: Option, pub serve_host: Option, - // OpenTelemetry output options. If `None`, OpenTelemetry is disabled. - pub otel_config: Option, + pub otel_config: OtelConfig, } impl Default for BootstrapOptions { @@ -155,7 +154,7 @@ impl Default for BootstrapOptions { mode: WorkerExecutionMode::None, serve_port: Default::default(), serve_host: Default::default(), - otel_config: None, + otel_config: Default::default(), } } } @@ -225,11 +224,7 @@ impl BootstrapOptions { self.serve_host.as_deref(), serve_is_main, serve_worker_count, - if let Some(otel_config) = self.otel_config.as_ref() { - Box::new([otel_config.console as u8, otel_config.deterministic as u8]) - } else { - Box::new([]) - }, + self.otel_config.as_v8(), ); bootstrap.serialize(ser).unwrap() diff --git a/tests/specs/cli/otel_basic/main.ts b/tests/specs/cli/otel_basic/main.ts index ccba126cc1..634727cea7 100644 --- a/tests/specs/cli/otel_basic/main.ts +++ b/tests/specs/cli/otel_basic/main.ts @@ -13,6 +13,7 @@ const server = Deno.serve( const command = new Deno.Command(Deno.execPath(), { args: ["run", "-A", "-q", "--unstable-otel", Deno.args[0]], env: { + OTEL_DENO: "true", DENO_UNSTABLE_OTEL_DETERMINISTIC: "1", OTEL_EXPORTER_OTLP_PROTOCOL: "http/json", OTEL_EXPORTER_OTLP_ENDPOINT: `http://localhost:${port}`, From 4cfa34052d0d6650f4a2b6a6a0c3363e7f3232e3 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 13:07:35 -0500 Subject: [PATCH 2/8] fix(compile): analyze modules in directory specified in --include (#27296) I ended up changing the file system implementation to determine its root directory as the last step of building it instead of being the first step which makes it much more reliable. --- cli/standalone/binary.rs | 349 +++++---- cli/standalone/serialization.rs | 12 +- cli/standalone/virtual_fs.rs | 671 ++++++++++++------ cli/tools/compile.rs | 167 ++--- tests/integration/compile_tests.rs | 31 +- .../__test__.jsonc | 22 + .../compile.out | 47 ++ .../main.out | 0 .../main.ts | 0 .../include/data_files/non_existent.out | 3 +- .../include/folder_ts_file/__test__.jsonc | 25 + .../compile/include/folder_ts_file/main.js | 14 + .../include/folder_ts_file/math/add.ts | 3 + .../compile/include/folder_ts_file/output.out | 2 + .../include/folder_ts_file/src/main.ts | 2 + .../include/symlink_twice/__test__.jsonc | 4 +- .../compile/include/symlink_twice/compile.out | 9 + .../compile/include/symlink_twice/setup.js | 1 - tests/specs/compile/npm_fs/__test__.jsonc | 24 + tests/specs/compile/npm_fs/compile.out | 8 + tests/specs/compile/npm_fs/deno.json | 3 + .../compile/npm_fs/main.out | 0 .../compile/npm_fs/main.ts | 0 tests/specs/mod.rs | 16 + tests/specs/schema.json | 15 + .../main_compile_file.out | 1 - .../main_compile_folder.out | 9 +- tests/util/server/src/lib.rs | 10 +- 28 files changed, 907 insertions(+), 541 deletions(-) create mode 100644 tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc create mode 100644 tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out rename tests/{testdata/compile/vfs_implicit_read_permission => specs/compile/global_npm_cache_implicit_read_permission}/main.out (100%) rename tests/{testdata/compile/vfs_implicit_read_permission => specs/compile/global_npm_cache_implicit_read_permission}/main.ts (100%) create mode 100644 tests/specs/compile/include/folder_ts_file/__test__.jsonc create mode 100644 tests/specs/compile/include/folder_ts_file/main.js create mode 100644 tests/specs/compile/include/folder_ts_file/math/add.ts create mode 100644 tests/specs/compile/include/folder_ts_file/output.out create mode 100644 tests/specs/compile/include/folder_ts_file/src/main.ts create mode 100644 tests/specs/compile/include/symlink_twice/compile.out create mode 100644 tests/specs/compile/npm_fs/__test__.jsonc create mode 100644 tests/specs/compile/npm_fs/compile.out create mode 100644 tests/specs/compile/npm_fs/deno.json rename tests/{testdata => specs}/compile/npm_fs/main.out (100%) rename tests/{testdata => specs}/compile/npm_fs/main.ts (100%) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 1e7b0d3f70..85a22cf837 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; @@ -76,6 +79,7 @@ use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; +use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -88,31 +92,28 @@ use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::output_vfs; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; +use super::virtual_fs::WindowsSystemRootablePath; + +pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = + ".deno_compile_node_modules"; /// A URL that can be designated as the base for relative URLs. /// /// After creation, this URL may be used to get the key for a /// module in the binary. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); - -impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { - fn from(url: &'a Url) -> Self { - Self(url) - } +pub enum StandaloneRelativeFileBaseUrl<'a> { + WindowsSystemRoot, + Path(&'a Url), } impl<'a> StandaloneRelativeFileBaseUrl<'a> { - pub fn new(url: &'a Url) -> Self { - debug_assert_eq!(url.scheme(), "file"); - Self(url) - } - /// Gets the module map key of the provided specifier. /// /// * Descendant file specifiers will be made relative to the base. @@ -122,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> { if target.scheme() != "file" { return Cow::Borrowed(target.as_str()); } + let base = match self { + Self::Path(base) => base, + Self::WindowsSystemRoot => return Cow::Borrowed(target.path()), + }; - match self.0.make_relative(target) { + match base.make_relative(target) { Some(relative) => { - if relative.starts_with("../") { - Cow::Borrowed(target.as_str()) - } else { - Cow::Owned(relative) - } + // This is not a great scenario to have because it means that the + // specifier is outside the vfs and could cause the binary to act + // strangely. If you encounter this, the fix is to add more paths + // to the vfs builder by calling `add_possible_min_root_dir`. + debug_assert!( + !relative.starts_with("../"), + "{} -> {} ({})", + base.as_str(), + target.as_str(), + relative, + ); + Cow::Owned(relative) } None => Cow::Borrowed(target.as_str()), } } - - pub fn inner(&self) -> &Url { - self.0 - } } #[derive(Deserialize, Serialize)] @@ -201,7 +209,7 @@ fn write_binary_bytes( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = @@ -372,7 +380,6 @@ pub struct WriteBinOptions<'a> { pub writer: File, pub display_output_filename: &'a str, pub graph: &'a ModuleGraph, - pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>, pub entrypoint: &'a ModuleSpecifier, pub include_files: &'a [ModuleSpecifier], pub compile_flags: &'a CompileFlags, @@ -556,7 +563,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { writer, display_output_filename, graph, - root_dir_url, entrypoint, include_files, compile_flags, @@ -568,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; - let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self - .build_npm_vfs(&root_path) - .context("Building npm vfs.")?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self - .npm_resolver - .root_node_modules_path() - .map(|path| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), - ) - .into_owned() - }), - }), - Some(snapshot), - ) - } else { - (None, None, None) - } + let mut vfs = VfsBuilder::new(); + let npm_snapshot = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; + Some(snapshot) + } else { + None } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; - let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { - npm_vfs - } else { - VfsBuilder::new(root_path.clone())? + } + InnerCliNpmResolverRef::Byonm(_) => { + self.fill_npm_vfs(&mut vfs)?; + None + } }; for include_file in include_files { let path = deno_path_util::url_to_file_path(include_file)?; - if path.is_dir() { - // TODO(#26941): we should analyze if any of these are - // modules in order to include their dependencies - vfs - .add_dir_recursive(&path) - .with_context(|| format!("Including {}", path.display()))?; - } else { - vfs - .add_file_at_path(&path) - .with_context(|| format!("Including {}", path.display()))?; - } + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { @@ -707,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); + if let Some(import_map) = self.workspace_resolver.maybe_import_map() { + if let Ok(file_path) = url_to_file_path(import_map.base_url()) { + if let Some(import_map_parent_dir) = file_path.parent() { + // tell the vfs about the import map's parent directory in case it + // falls outside what the root of where the VFS will be based + vfs.add_possible_min_root_dir(import_map_parent_dir); + } + } + } + if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() { + // ensure the vfs doesn't go below the node_modules directory's parent + if let Some(parent) = node_modules_dir.parent() { + vfs.add_possible_min_root_dir(parent); + } + } + + let vfs = self.build_vfs_consolidating_global_npm_cache(vfs); + let root_dir_url = match &vfs.root_path { + WindowsSystemRootablePath::Path(dir) => { + Some(url_from_directory_path(dir)?) + } + WindowsSystemRootablePath::WindowSystemRoot => None, + }; + let root_dir_url = match &root_dir_url { + Some(url) => StandaloneRelativeFileBaseUrl::Path(url), + None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot, + }; + + let node_modules = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(_) => { + npm_snapshot.as_ref().map(|_| NodeModules::Managed { + node_modules_dir: self.npm_resolver.root_node_modules_path().map( + |path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }, + ), + }) + } + InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), + ) + .into_owned() + }, + ), + }), + }; + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); @@ -721,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => Default::default(), }; + output_vfs(&vfs, display_output_filename); + let metadata = Metadata { argv: compile_flags.args.clone(), seed: self.cli_options.seed(), @@ -785,21 +803,19 @@ impl<'a> DenoCompileBinaryWriter<'a> { otel_config: self.cli_options.otel_config(), }; - output_vfs(&vfs, display_output_filename); - write_binary_bytes( writer, original_bin, &metadata, npm_snapshot.map(|s| s.into_serialized()), &remote_modules_store, - vfs, + &vfs, compile_flags, ) .context("Writing binary bytes") } - fn build_npm_vfs(&self, root_path: &Path) -> Result { + fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -810,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Managed(npm_resolver) => { if let Some(node_modules_path) = npm_resolver.root_node_modules_path() { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; builder.add_dir_recursive(node_modules_path)?; - Ok(builder) + Ok(()) } else { - // DO NOT include the user's registry url as it may contain credentials, - // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_path(); - let mut builder = - VfsBuilder::new(global_cache_root_path.to_path_buf())?; + // we'll flatten to remove any custom registries later let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -827,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?; builder.add_dir_recursive(&folder)?; } - - // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder - // that will be used by denort when loading the npm cache. This avoids us exposing - // the user's private registry information and means we don't have to bother - // serializing all the different registry config into the binary. - builder.with_root_dir(|root_dir| { - root_dir.name = ".deno_compile_node_modules".to_string(); - let mut new_entries = Vec::with_capacity(root_dir.entries.len()); - let mut localhost_entries = IndexMap::new(); - for entry in std::mem::take(&mut root_dir.entries) { - match entry { - VfsEntry::Dir(dir) => { - for entry in dir.entries { - log::debug!( - "Flattening {} into node_modules", - entry.name() - ); - if let Some(existing) = - localhost_entries.insert(entry.name().to_string(), entry) - { - panic!( - "Unhandled scenario where a duplicate entry was found: {:?}", - existing - ); - } - } - } - VfsEntry::File(_) | VfsEntry::Symlink(_) => { - new_entries.push(entry); - } - } - } - new_entries.push(VfsEntry::Dir(VirtualDirectory { - name: "localhost".to_string(), - entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), - })); - // needs to be sorted by name - new_entries.sort_by(|a, b| a.name().cmp(b.name())); - root_dir.entries = new_entries; - }); - - builder.set_new_root_path(root_path.to_path_buf())?; - - Ok(builder) + Ok(()) } } InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } @@ -908,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } } - Ok(builder) + Ok(()) } } } + + fn build_vfs_consolidating_global_npm_cache( + &self, + mut vfs: VfsBuilder, + ) -> BuiltVfs { + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(npm_resolver) => { + if npm_resolver.root_node_modules_path().is_some() { + return vfs.build(); + } + + let global_cache_root_path = npm_resolver.global_cache_root_path(); + + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder + // that will be used by denort when loading the npm cache. This avoids us exposing + // the user's private registry information and means we don't have to bother + // serializing all the different registry config into the binary. + let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else { + return vfs.build(); + }; + + root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string(); + let mut new_entries = Vec::with_capacity(root_dir.entries.len()); + let mut localhost_entries = IndexMap::new(); + for entry in std::mem::take(&mut root_dir.entries) { + match entry { + VfsEntry::Dir(dir) => { + for entry in dir.entries { + log::debug!("Flattening {} into node_modules", entry.name()); + if let Some(existing) = + localhost_entries.insert(entry.name().to_string(), entry) + { + panic!( + "Unhandled scenario where a duplicate entry was found: {:?}", + existing + ); + } + } + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => { + new_entries.push(entry); + } + } + } + new_entries.push(VfsEntry::Dir(VirtualDirectory { + name: "localhost".to_string(), + entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), + })); + // needs to be sorted by name + new_entries.sort_by(|a, b| a.name().cmp(b.name())); + root_dir.entries = new_entries; + + // it's better to not expose the user's cache directory, so take it out + // of there + let parent = global_cache_root_path.parent().unwrap(); + let parent_dir = vfs.get_dir_mut(parent).unwrap(); + let index = parent_dir + .entries + .iter() + .position(|entry| { + entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME + }) + .unwrap(); + let npm_global_cache_dir_entry = parent_dir.entries.remove(index); + + // go up from the ancestors removing empty directories... + // this is not as optimized as it could be + let mut last_name = + Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME); + for ancestor in parent.ancestors() { + let dir = vfs.get_dir_mut(ancestor).unwrap(); + if let Some(index) = dir + .entries + .iter() + .position(|entry| entry.name() == last_name) + { + dir.entries.remove(index); + } + last_name = Cow::Owned(dir.name.clone()); + if !dir.entries.is_empty() { + break; + } + } + + // now build the vfs and add the global cache dir entry there + let mut built_vfs = vfs.build(); + built_vfs.root.insert_entry(npm_global_cache_dir_entry); + built_vfs + } + InnerCliNpmResolverRef::Byonm(_) => vfs.build(), + } + } } fn get_denort_path(deno_exe: PathBuf) -> Option { diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index a5eb649bfd..6062e21019 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -23,6 +23,7 @@ use deno_semver::package::PackageReq; use crate::standalone::virtual_fs::VirtualDirectory; use super::binary::Metadata; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::VfsBuilder; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; @@ -39,7 +40,7 @@ pub fn serialize_binary_data_section( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, ) -> Result, AnyError> { fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); @@ -73,12 +74,11 @@ pub fn serialize_binary_data_section( } // 4. VFS { - let (vfs, vfs_files) = vfs.into_dir_and_files(); - let vfs = serde_json::to_string(&vfs)?; - write_bytes_with_len(&mut bytes, vfs.as_bytes()); - let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + let serialized_vfs = serde_json::to_string(&vfs.root)?; + write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes()); + let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); - for file in &vfs_files { + for file in &vfs.files { bytes.extend_from_slice(file); } } diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index ce7c0bb625..8ddd179c7a 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -15,17 +15,21 @@ use std::rc::Rc; use std::sync::Arc; use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_path_util::normalize_path; +use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; use deno_runtime::deno_io; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; +use indexmap::IndexSet; use serde::Deserialize; use serde::Serialize; use thiserror::Error; @@ -34,6 +38,38 @@ use crate::util; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; +use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!(!name_component.contains('\\')); + debug_assert!(!name_component.contains('/')); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug)] +pub struct BuiltVfs { + pub root_path: WindowsSystemRootablePath, + pub root: VirtualDirectory, + pub files: Vec>, +} + #[derive(Debug, Copy, Clone)] pub enum VfsFileSubDataKind { /// Raw bytes of the file. @@ -43,84 +79,84 @@ pub enum VfsFileSubDataKind { ModuleGraph, } -#[derive(Error, Debug)] -#[error( - "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() -)] -pub struct StripRootError { - root_path: PathBuf, - target: PathBuf, -} - #[derive(Debug)] pub struct VfsBuilder { - root_path: PathBuf, - root_dir: VirtualDirectory, + executable_root: VirtualDirectory, files: Vec>, current_offset: u64, file_offsets: HashMap, + /// The minimum root directory that should be included in the VFS. + min_root_dir: Option, } impl VfsBuilder { - pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path) - .with_context(|| format!("Canonicalizing {}", root_path.display()))?; - log::debug!("Building vfs with root '{}'", root_path.display()); - Ok(Self { - root_dir: VirtualDirectory { - name: root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), + pub fn new() -> Self { + Self { + executable_root: VirtualDirectory { + name: "/".to_string(), entries: Vec::new(), }, - root_path, files: Vec::new(), current_offset: 0, file_offsets: Default::default(), - }) + min_root_dir: Default::default(), + } } - pub fn set_new_root_path( - &mut self, - root_path: PathBuf, - ) -> Result<(), AnyError> { - let root_path = canonicalize_path(&root_path)?; - self.root_path = root_path; - self.root_dir = VirtualDirectory { - name: self - .root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), - entries: vec![VfsEntry::Dir(VirtualDirectory { - name: std::mem::take(&mut self.root_dir.name), - entries: std::mem::take(&mut self.root_dir.entries), - })], - }; - Ok(()) - } + /// Add a directory that might be the minimum root directory + /// of the VFS. + /// + /// For example, say the user has a deno.json and specifies an + /// import map in a parent directory. The import map won't be + /// included in the VFS, but its base will meaning we need to + /// tell the VFS builder to include the base of the import map + /// by calling this method. + pub fn add_possible_min_root_dir(&mut self, path: &Path) { + self.add_dir_raw(path); - pub fn with_root_dir( - &mut self, - with_root: impl FnOnce(&mut VirtualDirectory) -> R, - ) -> R { - with_root(&mut self.root_dir) + match &self.min_root_dir { + Some(WindowsSystemRootablePath::WindowSystemRoot) => { + // already the root dir + } + Some(WindowsSystemRootablePath::Path(current_path)) => { + let mut common_components = Vec::new(); + for (a, b) in current_path.components().zip(path.components()) { + if a != b { + break; + } + common_components.push(a); + } + if common_components.is_empty() { + if cfg!(windows) { + self.min_root_dir = + Some(WindowsSystemRootablePath::WindowSystemRoot); + } else { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(PathBuf::from("/"))); + } + } else { + self.min_root_dir = Some(WindowsSystemRootablePath::Path( + common_components.iter().collect(), + )); + } + } + None => { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(path.to_path_buf())); + } + } } pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if path != target_path { - self.add_symlink(path, &target_path)?; - } - self.add_dir_recursive_internal(&target_path) + let target_path = self.resolve_target_path(path)?; + self.add_dir_recursive_not_symlink(&target_path) } - fn add_dir_recursive_internal( + fn add_dir_recursive_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { - self.add_dir(path)?; + self.add_dir_raw(path); let read_dir = std::fs::read_dir(path) .with_context(|| format!("Reading {}", path.display()))?; @@ -133,49 +169,26 @@ impl VfsBuilder { let path = entry.path(); if file_type.is_dir() { - self.add_dir_recursive_internal(&path)?; + self.add_dir_recursive_not_symlink(&path)?; } else if file_type.is_file() { self.add_file_at_path_not_symlink(&path)?; } else if file_type.is_symlink() { - match util::fs::canonicalize_path(&path) { - Ok(target) => { - if let Err(StripRootError { .. }) = self.add_symlink(&path, &target) - { - if target.is_file() { - // this may change behavior, so warn the user about it - log::warn!( - "{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - // inline the symlink and make the target file - let file_bytes = std::fs::read(&target) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner( - &path, - file_bytes, - VfsFileSubDataKind::Raw, - )?; - } else { - log::warn!( - "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - } + match self.add_symlink(&path) { + Ok(target) => match target { + SymlinkTarget::File(target) => { + self.add_file_at_path_not_symlink(&target)? } - } + SymlinkTarget::Dir(target) => { + self.add_dir_recursive_not_symlink(&target)?; + } + }, Err(err) => { log::warn!( - "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", - crate::colors::yellow("Warning"), - path.display(), - err - ); + "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", + crate::colors::yellow("Warning"), + path.display(), + err + ); } } } @@ -184,15 +197,15 @@ impl VfsBuilder { Ok(()) } - fn add_dir( - &mut self, - path: &Path, - ) -> Result<&mut VirtualDirectory, StripRootError> { + fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { log::debug!("Ensuring directory '{}'", path.display()); - let path = self.path_relative_root(path)?; - let mut current_dir = &mut self.root_dir; + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } let name = component.as_os_str().to_string_lossy(); let index = match current_dir .entries @@ -218,15 +231,44 @@ impl VfsBuilder { }; } - Ok(current_dir) + current_dir + } + + pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { + &mut self.executable_root + } + + pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let index = match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&name)) + { + Ok(index) => index, + Err(_) => return None, + }; + match &mut current_dir.entries[index] { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + Some(current_dir) } pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; - } - self.add_file_at_path_not_symlink(&target_path) + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw) } fn add_file_at_path_not_symlink( @@ -244,11 +286,15 @@ impl VfsBuilder { data: Vec, sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + let target = self.add_symlink(path)?.into_path_buf(); + self.add_file_with_data_inner(&target, data, sub_data_kind) + } else { + self.add_file_with_data_inner(path, data, sub_data_kind) } - self.add_file_with_data_inner(&target_path, data, sub_data_kind) } fn add_file_with_data_inner( @@ -267,7 +313,7 @@ impl VfsBuilder { self.current_offset }; - let dir = self.add_dir(path.parent().unwrap())?; + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); let offset_and_len = OffsetWithLength { offset, @@ -309,74 +355,162 @@ impl VfsBuilder { Ok(()) } - fn add_symlink( + fn resolve_target_path(&mut self, path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + Ok(self.add_symlink(path)?.into_path_buf()) + } else { + Ok(path.to_path_buf()) + } + } + + fn add_symlink(&mut self, path: &Path) -> Result { + self.add_symlink_inner(path, &mut IndexSet::new()) + } + + fn add_symlink_inner( &mut self, path: &Path, - target: &Path, - ) -> Result<(), StripRootError> { - log::debug!( - "Adding symlink '{}' to '{}'", - path.display(), - target.display() + visited: &mut IndexSet, + ) -> Result { + log::debug!("Adding symlink '{}'", path.display()); + let target = strip_unc_prefix( + std::fs::read_link(path) + .with_context(|| format!("Reading symlink '{}'", path.display()))?, ); - let relative_target = self.path_relative_root(target)?; - let relative_path = match self.path_relative_root(path) { - Ok(path) => path, - Err(StripRootError { .. }) => { - // ignore if the original path is outside the root directory - return Ok(()); - } - }; - if relative_target == relative_path { - // it's the same, ignore - return Ok(()); - } - let dir = self.add_dir(path.parent().unwrap())?; + let target = normalize_path(path.parent().unwrap().join(&target)); + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => Ok(()), // previously inserted + Ok(_) => {} // previously inserted Err(insert_index) => { dir.entries.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: relative_target - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(), + dest_parts: VirtualSymlinkParts::from_path(&target), }), ); - Ok(()) } } + let target_metadata = + std::fs::symlink_metadata(&target).with_context(|| { + format!("Reading symlink target '{}'", target.display()) + })?; + if target_metadata.is_symlink() { + if !visited.insert(target.clone()) { + // todo: probably don't error in this scenario + bail!( + "Circular symlink detected: {} -> {}", + visited + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> "), + target.display() + ); + } + self.add_symlink_inner(&target, visited) + } else if target_metadata.is_dir() { + Ok(SymlinkTarget::Dir(target)) + } else { + Ok(SymlinkTarget::File(target)) + } } - pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { - (self.root_dir, self.files) - } + pub fn build(self) -> BuiltVfs { + fn strip_prefix_from_symlinks( + dir: &mut VirtualDirectory, + parts: &[String], + ) { + for entry in &mut dir.entries { + match entry { + VfsEntry::Dir(dir) => { + strip_prefix_from_symlinks(dir, parts); + } + VfsEntry::File(_) => {} + VfsEntry::Symlink(symlink) => { + let old_parts = std::mem::take(&mut symlink.dest_parts.0); + symlink.dest_parts.0 = + old_parts.into_iter().skip(parts.len()).collect(); + } + } + } + } - fn path_relative_root(&self, path: &Path) -> Result { - match path.strip_prefix(&self.root_path) { - Ok(p) => Ok(p.to_path_buf()), - Err(_) => Err(StripRootError { - root_path: self.root_path.clone(), - target: path.to_path_buf(), - }), + let mut current_dir = self.executable_root; + let mut current_path = if cfg!(windows) { + WindowsSystemRootablePath::WindowSystemRoot + } else { + WindowsSystemRootablePath::Path(PathBuf::from("/")) + }; + loop { + if current_dir.entries.len() != 1 { + break; + } + if self.min_root_dir.as_ref() == Some(¤t_path) { + break; + } + match ¤t_dir.entries[0] { + VfsEntry::Dir(dir) => { + if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + // special directory we want to maintain + break; + } + match current_dir.entries.remove(0) { + VfsEntry::Dir(dir) => { + current_path = + WindowsSystemRootablePath::Path(current_path.join(&dir.name)); + current_dir = dir; + } + _ => unreachable!(), + }; + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => break, + } + } + if let WindowsSystemRootablePath::Path(path) = ¤t_path { + strip_prefix_from_symlinks( + &mut current_dir, + &VirtualSymlinkParts::from_path(path).0, + ); + } + BuiltVfs { + root_path: current_path, + root: current_dir, + files: self.files, } } } -pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { +#[derive(Debug)] +enum SymlinkTarget { + File(PathBuf), + Dir(PathBuf), +} + +impl SymlinkTarget { + pub fn into_path_buf(self) -> PathBuf { + match self { + Self::File(path) => path, + Self::Dir(path) => path, + } + } +} + +pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { if !log::log_enabled!(log::Level::Info) { return; // no need to compute if won't output } - if builder.root_dir.entries.is_empty() { + if vfs.root.entries.is_empty() { return; // nothing to output } let mut text = String::new(); - let display_tree = vfs_as_display_tree(builder, executable_name); + let display_tree = vfs_as_display_tree(vfs, executable_name); display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string log::info!( "\n{}\n", @@ -386,7 +520,7 @@ pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { } fn vfs_as_display_tree( - builder: &VfsBuilder, + vfs: &BuiltVfs, executable_name: &str, ) -> DisplayTreeNode { enum EntryOutput<'a> { @@ -398,20 +532,38 @@ fn vfs_as_display_tree( impl<'a> EntryOutput<'a> { pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + let mut children = match self { + EntryOutput::Subset(vec) => vec + .iter() + .map(|e| e.output.as_display_tree(e.name.to_string())) + .collect(), + EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => { + vec![] + } + }; + // we only want to collapse leafs so that nodes of the + // same depth have the same indentation + let collapse_single_child = + children.len() == 1 && children[0].children.is_empty(); DisplayTreeNode { text: match self { - EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name, + EntryOutput::All => format!("{}/*", name), + EntryOutput::Subset(_) => { + if collapse_single_child { + format!("{}/{}", name, children[0].text) + } else { + name + } + } + EntryOutput::File => name, EntryOutput::Symlink(parts) => { format!("{} --> {}", name, parts.join("/")) } }, - children: match self { - EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())], - EntryOutput::Subset(vec) => vec - .iter() - .map(|e| e.output.as_display_tree(e.name.to_string())) - .collect(), - EntryOutput::File | EntryOutput::Symlink(_) => vec![], + children: if collapse_single_child { + children.remove(0).children + } else { + children }, } } @@ -422,37 +574,81 @@ fn vfs_as_display_tree( output: EntryOutput<'a>, } - fn include_all_entries<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - EntryOutput::Subset( + fn show_global_node_modules_dir( + vfs_dir: &VirtualDirectory, + ) -> Vec { + fn show_subset_deep( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> EntryOutput { + if depth == 0 { + EntryOutput::All + } else { + EntryOutput::Subset(show_subset(vfs_dir, depth)) + } + } + + fn show_subset( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> Vec { vfs_dir .entries .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: match entry { + VfsEntry::Dir(virtual_directory) => { + show_subset_deep(virtual_directory, depth - 1) + } + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + } + }, }) - .collect(), - ) + .collect() + } + + // in this scenario, we want to show + // .deno_compile_node_modules/localhost///* + show_subset(vfs_dir, 3) } - fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> { + fn include_all_entries<'a>( + dir_path: &WindowsSystemRootablePath, + vfs_dir: &'a VirtualDirectory, + ) -> Vec> { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return show_global_node_modules_dir(vfs_dir); + } + + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(dir_path.join(entry.name()), entry), + }) + .collect() + } + + fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput { match entry { VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), VfsEntry::File(_) => EntryOutput::File, VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts) + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) } } } - fn analyze_dir<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - let real_entry_count = std::fs::read_dir(dir) + fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir)); + } + + let real_entry_count = std::fs::read_dir(&dir) .ok() .map(|entries| entries.flat_map(|e| e.ok()).count()) .unwrap_or(0); @@ -462,7 +658,7 @@ fn vfs_as_display_tree( .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: analyze_entry(dir.join(entry.name()), entry), }) .collect::>(); if children @@ -474,15 +670,23 @@ fn vfs_as_display_tree( EntryOutput::Subset(children) } } else { - include_all_entries(dir, vfs_dir) + EntryOutput::Subset(include_all_entries( + &WindowsSystemRootablePath::Path(dir), + vfs_dir, + )) } } // always include all the entries for the root directory, otherwise the // user might not have context about what's being shown - let output = include_all_entries(&builder.root_path, &builder.root_dir); - output - .as_display_tree(deno_terminal::colors::italic(executable_name).to_string()) + let child_entries = include_all_entries(&vfs.root_path, &vfs.root); + DisplayTreeNode { + text: deno_terminal::colors::italic(executable_name).to_string(), + children: child_entries + .iter() + .map(|entry| entry.output.as_display_tree(entry.name.to_string())) + .collect(), + } } #[derive(Debug)] @@ -603,6 +807,20 @@ pub struct VirtualDirectory { pub entries: Vec, } +impl VirtualDirectory { + pub fn insert_entry(&mut self, entry: VfsEntry) { + let name = entry.name(); + match self.entries.binary_search_by(|e| e.name().cmp(name)) { + Ok(index) => { + self.entries[index] = entry; + } + Err(insert_index) => { + self.entries.insert(insert_index, entry); + } + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct OffsetWithLength { #[serde(rename = "o")] @@ -626,18 +844,33 @@ pub struct VirtualFile { pub module_graph_offset: OffsetWithLength, } +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct VirtualSymlink { #[serde(rename = "n")] pub name: String, #[serde(rename = "p")] - pub dest_parts: Vec, + pub dest_parts: VirtualSymlinkParts, } impl VirtualSymlink { pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { let mut dest = root.to_path_buf(); - for part in &self.dest_parts { + for part in &self.dest_parts.0 { dest.push(part); } dest @@ -709,10 +942,10 @@ impl VfsRoot { let mut final_path = self.root_path.clone(); let mut current_entry = VfsEntryRef::Dir(&self.dir); for component in relative_path.components() { - let component = component.as_os_str().to_string_lossy(); + let component = component.as_os_str(); let current_dir = match current_entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } VfsEntryRef::Symlink(symlink) => { @@ -721,7 +954,7 @@ impl VfsRoot { final_path = resolved_path; // overwrite with the new resolved path match entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } _ => { @@ -739,6 +972,7 @@ impl VfsRoot { )); } }; + let component = component.to_string_lossy(); match current_dir .entries .binary_search_by(|e| e.name().cmp(&component)) @@ -1136,6 +1370,7 @@ impl FileBackedVfs { mod test { use console_static_text::ansi::strip_ansi_codes; use std::io::Write; + use test_util::assert_contains; use test_util::TempDir; use super::*; @@ -1159,8 +1394,11 @@ mod test { // will canonicalize the root path let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.join("sub_dir").create_dir_all(); + src_path.join("e.txt").write("e"); + src_path.symlink_file("e.txt", "sub_dir/e.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( &src_path.join("a.txt"), @@ -1190,18 +1428,9 @@ mod test { VfsFileSubDataKind::Raw, ) .unwrap(); + builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); builder - .add_file_with_data_inner( - &src_path.join("e.txt"), - "e".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_symlink( - &src_path.join("sub_dir").join("e.txt"), - &src_path.join("e.txt"), - ) + .add_symlink(&src_path.join("sub_dir").join("e.txt")) .unwrap(); // get the virtual fs @@ -1262,7 +1491,7 @@ mod test { // build and create the virtual fs let src_path = temp_dir_path.join("src").to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder.add_dir_recursive(&src_path).unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); @@ -1300,10 +1529,10 @@ mod test { temp_dir: &TempDir, ) -> (PathBuf, FileBackedVfs) { let virtual_fs_file = temp_dir.path().join("virtual_fs"); - let (root_dir, files) = builder.into_dir_and_files(); + let vfs = builder.build(); { let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); - for file_data in &files { + for file_data in &vfs.files { file.write_all(file_data).unwrap(); } } @@ -1314,7 +1543,7 @@ mod test { FileBackedVfs::new( Cow::Owned(data), VfsRoot { - dir: root_dir, + dir: vfs.root, root_path: dest_path.to_path_buf(), start_file_offset: 0, }, @@ -1327,41 +1556,22 @@ mod test { let temp_dir = TempDir::new(); let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.symlink_file("a.txt", "b.txt"); + src_path.symlink_file("b.txt", "c.txt"); + src_path.symlink_file("c.txt", "a.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); - builder - .add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt")) - .unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - assert_eq!( - virtual_fs - .file_entry(&dest_path.join("a.txt")) - .err() - .unwrap() - .to_string(), - "circular symlinks", - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(), - dest_path.join("b.txt") - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(), - dest_path.join("c.txt") - ); + let mut builder = VfsBuilder::new(); + let err = builder + .add_symlink(src_path.join("a.txt").as_path()) + .unwrap_err(); + assert_contains!(err.to_string(), "Circular symlink detected",); } #[tokio::test] async fn test_open_file() { let temp_dir = TempDir::new(); let temp_path = temp_dir.path().canonicalize(); - let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( temp_path.join("a.txt").as_path(), @@ -1436,8 +1646,7 @@ mod test { temp_dir.write("c/a.txt", "contents"); temp_dir.symlink_file("c/a.txt", "c/b.txt"); assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works - let mut vfs_builder = - VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap(); + let mut vfs_builder = VfsBuilder::new(); // full dir vfs_builder .add_dir_recursive(temp_dir.path().join("a").as_path()) @@ -1451,16 +1660,14 @@ mod test { .add_dir_recursive(temp_dir.path().join("c").as_path()) .unwrap(); temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory - let node = vfs_as_display_tree(&vfs_builder, "executable"); + let node = vfs_as_display_tree(&vfs_builder.build(), "executable"); let mut text = String::new(); node.print(&mut text).unwrap(); assert_eq!( strip_ansi_codes(&text), r#"executable -├─┬ a -│ └── * -├─┬ b -│ └── a.txt +├── a/* +├── b/a.txt └─┬ c ├── a.txt └── b.txt --> c/a.txt diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 4d0607ba71..7a463a7b09 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,7 +5,6 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; -use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::binary::WriteBinOptions; use crate::standalone::is_standalone_binary; use deno_ast::MediaType; @@ -17,8 +16,11 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_terminal::colors; use rand::Rng; +use std::collections::HashSet; +use std::collections::VecDeque; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -84,29 +86,6 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let root_dir_url = resolve_root_dir_from_specifiers( - cli_options.workspace().root_dir(), - graph - .specifiers() - .map(|(s, _)| s) - .chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ) - .chain(include_files.iter()) - .chain( - // sometimes the import map path is outside the root dir - cli_options - .workspace() - .to_import_map_path() - .ok() - .and_then(|p| p.and_then(|p| url_from_file_path(&p).ok())) - .iter(), - ), - ); - log::debug!("Binary root dir: {}", root_dir_url); log::info!( "{} {} to {}", colors::green("Compile"), @@ -138,7 +117,6 @@ pub async fn compile( .unwrap() .to_string_lossy(), graph: &graph, - root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url), entrypoint, include_files: &include_files, compile_flags: &compile_flags, @@ -261,15 +239,58 @@ fn get_module_roots_and_include_files( } } - let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); - let mut include_files = Vec::with_capacity(compile_flags.include.len()); + fn analyze_path( + url: &ModuleSpecifier, + module_roots: &mut Vec, + include_files: &mut Vec, + searched_paths: &mut HashSet, + ) -> Result<(), AnyError> { + let Ok(path) = url_to_file_path(url) else { + return Ok(()); + }; + let mut pending = VecDeque::from([path]); + while let Some(path) = pending.pop_front() { + if !searched_paths.insert(path.clone()) { + continue; + } + if !path.is_dir() { + let url = url_from_file_path(&path)?; + include_files.push(url.clone()); + if is_module_graph_module(&url) { + module_roots.push(url); + } + continue; + } + for entry in std::fs::read_dir(&path).with_context(|| { + format!("Failed reading directory '{}'", path.display()) + })? { + let entry = entry.with_context(|| { + format!("Failed reading entry in directory '{}'", path.display()) + })?; + pending.push_back(entry.path()); + } + } + Ok(()) + } + + let mut searched_paths = HashSet::new(); + let mut module_roots = Vec::new(); + let mut include_files = Vec::new(); module_roots.push(entrypoint.clone()); for side_module in &compile_flags.include { let url = resolve_url_or_path(side_module, initial_cwd)?; if is_module_graph_module(&url) { - module_roots.push(url); + module_roots.push(url.clone()); + if url.scheme() == "file" { + include_files.push(url); + } } else { - include_files.push(url); + analyze_path( + &url, + &mut module_roots, + &mut include_files, + &mut searched_paths, + )?; } } Ok((module_roots, include_files)) @@ -335,57 +356,6 @@ fn get_os_specific_filepath( } } -fn resolve_root_dir_from_specifiers<'a>( - starting_dir: &ModuleSpecifier, - specifiers: impl Iterator, -) -> ModuleSpecifier { - fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str { - let min_length = a.len().min(b.len()); - - let mut last_slash = 0; - for i in 0..min_length { - if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' { - last_slash = i; - } else if a.as_bytes()[i] != b.as_bytes()[i] { - break; - } - } - - // Return the common root path up to the last common slash. - // This returns a slice of the original string 'a', up to and including the last matching '/'. - let common = &a[..=last_slash]; - if cfg!(windows) && common == "file:///" { - a - } else { - common - } - } - - fn is_file_system_root(url: &str) -> bool { - let Some(path) = url.strip_prefix("file:///") else { - return false; - }; - if cfg!(windows) { - let Some((_drive, path)) = path.split_once('/') else { - return true; - }; - path.is_empty() - } else { - path.is_empty() - } - } - - let mut found_dir = starting_dir.as_str(); - if !is_file_system_root(found_dir) { - for specifier in specifiers { - if specifier.scheme() == "file" { - found_dir = select_common_root(found_dir, specifier.as_str()); - } - } - } - ModuleSpecifier::parse(found_dir).unwrap() -} - #[cfg(test)] mod test { pub use super::*; @@ -462,41 +432,4 @@ mod test { run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); } - - #[test] - fn test_resolve_root_dir_from_specifiers() { - fn resolve(start: &str, specifiers: &[&str]) -> String { - let specifiers = specifiers - .iter() - .map(|s| ModuleSpecifier::parse(s).unwrap()) - .collect::>(); - resolve_root_dir_from_specifiers( - &ModuleSpecifier::parse(start).unwrap(), - specifiers.iter(), - ) - .to_string() - } - - assert_eq!( - resolve("file:///a/b/e", &["file:///a/b/c/d"]), - "file:///a/b/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d"]), - "file:///a/b/c/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]), - "file:///a/b/c/" - ); - assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///"); - if cfg!(windows) { - assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/"); - // this will ignore the other one because it's on a separate drive - assert_eq!( - resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]), - "file:///c:/a/b/c/" - ); - } - } } diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index 62c5cf8fab..a34d2cdd1d 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -846,21 +846,6 @@ testing[WILDCARD]this .assert_matches_text("2\n"); } -#[test] -fn compile_npm_file_system() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/npm_fs/main.ts", - copy_temp_dir: Some("compile/npm_fs"), - compile_args: vec!["-A"], - run_args: vec![], - output_file: "compile/npm_fs/main.out", - node_modules_local: true, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_bin_esm() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() { }); } -#[test] -fn compile_npm_vfs_implicit_read_permissions() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/vfs_implicit_read_permission/main.ts", - copy_temp_dir: Some("compile/vfs_implicit_read_permission"), - compile_args: vec![], - run_args: vec![], - output_file: "compile/vfs_implicit_read_permission/main.out", - node_modules_local: false, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_no_permissions() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() { let symlink_target_dir = temp_dir.path().join("some_folder"); project_dir.join("node_modules").create_dir_all(); symlink_target_dir.create_dir_all(); + symlink_target_dir.join("file.txt").write("5"); let symlink_target_file = temp_dir.path().join("target.txt"); symlink_target_file.write("5"); let symlink_dir = project_dir.join("node_modules").join("symlink_dir"); diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc new file mode 100644 index 0000000000..d346c3ad20 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc @@ -0,0 +1,22 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out new file mode 100644 index 0000000000..c29c878593 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -0,0 +1,47 @@ +[WILDCARD] +Compile file:///[WILDLINE]/main.ts to [WILDLINE] + +Embedded File System + +main[WILDLINE] +├─┬ .deno_compile_node_modules +│ └─┬ localhost +│ ├─┬ ansi-regex +│ │ ├── 3.0.1/* +│ │ └── 5.0.1/* +│ ├── ansi-styles/4.3.0/* +│ ├── camelcase/5.3.1/* +│ ├── cliui/6.0.0/* +│ ├── color-convert/2.0.1/* +│ ├── color-name/1.1.4/* +│ ├── cowsay/1.5.0/* +│ ├── decamelize/1.2.0/* +│ ├── emoji-regex/8.0.0/* +│ ├── find-up/4.1.0/* +│ ├── get-caller-file/2.0.5/* +│ ├── get-stdin/8.0.0/* +│ ├─┬ is-fullwidth-code-point +│ │ ├── 2.0.0/* +│ │ └── 3.0.0/* +│ ├── locate-path/5.0.0/* +│ ├── p-limit/2.3.0/* +│ ├── p-locate/4.1.0/* +│ ├── p-try/2.2.0/* +│ ├── path-exists/4.0.0/* +│ ├── require-directory/2.1.1/* +│ ├── require-main-filename/2.0.0/* +│ ├── set-blocking/2.0.0/* +│ ├─┬ string-width +│ │ ├── 2.1.1/* +│ │ └── 4.2.3/* +│ ├─┬ strip-ansi +│ │ ├── 4.0.0/* +│ │ └── 6.0.1/* +│ ├── strip-final-newline/2.0.0/* +│ ├── which-module/2.0.0/* +│ ├── wrap-ansi/6.2.0/* +│ ├── y18n/4.0.3/* +│ ├── yargs/15.4.1/* +│ └── yargs-parser/18.1.3/* +└── main.ts + diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.out similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.out rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.out diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.ts b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.ts rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts diff --git a/tests/specs/compile/include/data_files/non_existent.out b/tests/specs/compile/include/data_files/non_existent.out index a88b441ba8..54bc69ef09 100644 --- a/tests/specs/compile/include/data_files/non_existent.out +++ b/tests/specs/compile/include/data_files/non_existent.out @@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]' Caused by: 0: Including [WILDLINE]does_not_exist.txt - 1: [WILDLINE] + 1: Reading [WILDLINE]does_not_exist.txt + 2: [WILDLINE] diff --git a/tests/specs/compile/include/folder_ts_file/__test__.jsonc b/tests/specs/compile/include/folder_ts_file/__test__.jsonc new file mode 100644 index 0000000000..f02ed1efc3 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/__test__.jsonc @@ -0,0 +1,25 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + // notice how the math folder is not included + "args": "compile --allow-read=data --include src --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=data --include src --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/include/folder_ts_file/main.js b/tests/specs/compile/include/folder_ts_file/main.js new file mode 100644 index 0000000000..23b490e390 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/main.js @@ -0,0 +1,14 @@ +const mathDir = import.meta.dirname + "/math"; +const files = Array.from( + Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name), +); +files.sort(); +for (const file of files) { + console.log(file); +} + +function nonAnalyzable() { + return "./src/main.ts"; +} + +await import(nonAnalyzable()); diff --git a/tests/specs/compile/include/folder_ts_file/math/add.ts b/tests/specs/compile/include/folder_ts_file/math/add.ts new file mode 100644 index 0000000000..3b399665dc --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/math/add.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a + b; +} diff --git a/tests/specs/compile/include/folder_ts_file/output.out b/tests/specs/compile/include/folder_ts_file/output.out new file mode 100644 index 0000000000..959e3d5c76 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/output.out @@ -0,0 +1,2 @@ +[WILDLINE]add.ts +3 diff --git a/tests/specs/compile/include/folder_ts_file/src/main.ts b/tests/specs/compile/include/folder_ts_file/src/main.ts new file mode 100644 index 0000000000..38868c3d82 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/src/main.ts @@ -0,0 +1,2 @@ +import { add } from "../math/add.ts"; +console.log(add(1, 2)); diff --git a/tests/specs/compile/include/symlink_twice/__test__.jsonc b/tests/specs/compile/include/symlink_twice/__test__.jsonc index ebdf824f43..f0f57292a6 100644 --- a/tests/specs/compile/include/symlink_twice/__test__.jsonc +++ b/tests/specs/compile/include/symlink_twice/__test__.jsonc @@ -6,7 +6,7 @@ }, { "if": "unix", "args": "compile --allow-read=data --include . --output main link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "unix", "commandName": "./main", @@ -16,7 +16,7 @@ }, { "if": "windows", "args": "compile --allow-read=data --include . --output main.exe link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "windows", "commandName": "./main.exe", diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out new file mode 100644 index 0000000000..c57eb9b2f1 --- /dev/null +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -0,0 +1,9 @@ +Compile [WILDLINE] + +Embedded File System + +main[WILDLINE] +├── index.js +├── link.js --> index.js +└── setup.js + diff --git a/tests/specs/compile/include/symlink_twice/setup.js b/tests/specs/compile/include/symlink_twice/setup.js index 3e713dd63e..4c7cebfaf5 100644 --- a/tests/specs/compile/include/symlink_twice/setup.js +++ b/tests/specs/compile/include/symlink_twice/setup.js @@ -1,3 +1,2 @@ -Deno.mkdirSync("data"); Deno.writeTextFileSync("index.js", "console.log(1);"); Deno.symlinkSync("index.js", "link.js"); diff --git a/tests/specs/compile/npm_fs/__test__.jsonc b/tests/specs/compile/npm_fs/__test__.jsonc new file mode 100644 index 0000000000..a8198bfb5d --- /dev/null +++ b/tests/specs/compile/npm_fs/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + // use this so the vfs output is all in the same folder + "canonicalizedTempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out new file mode 100644 index 0000000000..4944146788 --- /dev/null +++ b/tests/specs/compile/npm_fs/compile.out @@ -0,0 +1,8 @@ +[WILDCARD] + +Embedded File System + +main[WILDLINE] +├── main.ts +└── node_modules/* + diff --git a/tests/specs/compile/npm_fs/deno.json b/tests/specs/compile/npm_fs/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/compile/npm_fs/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/testdata/compile/npm_fs/main.out b/tests/specs/compile/npm_fs/main.out similarity index 100% rename from tests/testdata/compile/npm_fs/main.out rename to tests/specs/compile/npm_fs/main.out diff --git a/tests/testdata/compile/npm_fs/main.ts b/tests/specs/compile/npm_fs/main.ts similarity index 100% rename from tests/testdata/compile/npm_fs/main.ts rename to tests/specs/compile/npm_fs/main.ts diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs index f5820e4d88..985a6c7c40 100644 --- a/tests/specs/mod.rs +++ b/tests/specs/mod.rs @@ -118,6 +118,12 @@ struct MultiStepMetaData { /// steps. #[serde(default)] pub temp_dir: bool, + /// Whether the temporary directory should be canonicalized. + /// + /// This should be used sparingly, but is sometimes necessary + /// on the CI. + #[serde(default)] + pub canonicalized_temp_dir: bool, /// Whether the temporary directory should be symlinked to another path. #[serde(default)] pub symlinked_temp_dir: bool, @@ -144,6 +150,8 @@ struct SingleTestMetaData { #[serde(default)] pub temp_dir: bool, #[serde(default)] + pub canonicalized_temp_dir: bool, + #[serde(default)] pub symlinked_temp_dir: bool, #[serde(default)] pub repeat: Option, @@ -159,6 +167,7 @@ impl SingleTestMetaData { base: self.base, cwd: None, temp_dir: self.temp_dir, + canonicalized_temp_dir: self.canonicalized_temp_dir, symlinked_temp_dir: self.symlinked_temp_dir, repeat: self.repeat, envs: Default::default(), @@ -326,6 +335,13 @@ fn test_context_from_metadata( builder = builder.cwd(cwd.to_string_lossy()); } + if metadata.canonicalized_temp_dir { + // not actually deprecated, we just want to discourage its use + #[allow(deprecated)] + { + builder = builder.use_canonicalized_temp_dir(); + } + } if metadata.symlinked_temp_dir { // not actually deprecated, we just want to discourage its use // because it's mostly used for testing purposes locally diff --git a/tests/specs/schema.json b/tests/specs/schema.json index 2b35d9bd7d..77ffc59530 100644 --- a/tests/specs/schema.json +++ b/tests/specs/schema.json @@ -36,6 +36,9 @@ "flaky": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, "symlinkedTempDir": { "type": "boolean" }, @@ -66,6 +69,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, @@ -94,6 +103,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index 70de321361..633c2cca62 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,5 +1,4 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Warning Symlink target is outside '[WILDCARD]node_modules_symlink_outside'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. Embedded File System diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 205c6a9281..61f0a2456a 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -3,8 +3,13 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Warning Symlink target is outside '[WILDLINE]node_modules_symlink_outside'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. Embedded File System -[WILDCARD] +bin[WILDLINE] +├─┬ compile +│ └─┬ node_modules_symlink_outside +│ ├── main.ts +│ └── node_modules/* +└── some_folder/* + diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 953896cffd..531944bf6a 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -816,15 +816,17 @@ pub fn wildcard_match_detailed( } let actual_next_text = ¤t_text[max_current_text_found_index..]; - let max_next_text_len = 40; - let next_text_len = - std::cmp::min(max_next_text_len, actual_next_text.len()); + let next_text_len = actual_next_text + .chars() + .take(40) + .map(|c| c.len_utf8()) + .sum::(); output_lines.push(format!( "==== NEXT ACTUAL TEXT ====\n{}{}", colors::red(annotate_whitespace( &actual_next_text[..next_text_len] )), - if actual_next_text.len() > max_next_text_len { + if actual_next_text.len() > next_text_len { "[TRUNCATED]" } else { "" From b7564636b5ded00819cd1f89cf828f555d9085e2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 18:43:43 -0500 Subject: [PATCH 3/8] chore: add npm distribution at `deno` package (#27346) --- .github/workflows/cargo_publish.yml | 2 +- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/npm_publish.yml | 45 +++ .github/workflows/promote_to_release.yml | 2 +- .github/workflows/start_release.yml | 2 +- .github/workflows/version_bump.yml | 2 +- tools/deno.lock.json | 436 +++++++++++++++++++---- tools/release/npm/.gitignore | 1 + tools/release/npm/bin.cjs | 54 +++ tools/release/npm/build.ts | 237 ++++++++++++ tools/release/npm/install.cjs | 5 + tools/release/npm/install_api.cjs | 196 ++++++++++ 13 files changed, 904 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/npm_publish.yml create mode 100644 tools/release/npm/.gitignore create mode 100644 tools/release/npm/bin.cjs create mode 100644 tools/release/npm/build.ts create mode 100644 tools/release/npm/install.cjs create mode 100644 tools/release/npm/install_api.cjs diff --git a/.github/workflows/cargo_publish.yml b/.github/workflows/cargo_publish.yml index 3af97f4662..eb72e3739f 100644 --- a/.github/workflows/cargo_publish.yml +++ b/.github/workflows/cargo_publish.yml @@ -35,7 +35,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Publish env: diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 9dd1f3017b..d43db76414 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -196,7 +196,7 @@ const installNodeStep = { const installDenoStep = { name: "Install Deno", uses: "denoland/setup-deno@v2", - with: { "deno-version": "v1.x" }, + with: { "deno-version": "v2.x" }, }; const authenticateWithGoogleCloud = { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838373cb78..29dd694c6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: name: Install Deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/npm_publish.yml b/.github/workflows/npm_publish.yml new file mode 100644 index 0000000000..5e58005926 --- /dev/null +++ b/.github/workflows/npm_publish.yml @@ -0,0 +1,45 @@ +name: npm_publish + +on: + workflow_dispatch: + inputs: + version: + description: 'Version' + type: string + release: + types: [published] + +permissions: + id-token: write + +jobs: + build: + name: npm publish + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Configure git + run: | + git config --global core.symlinks true + git config --global fetch.parallel 32 + + - name: Clone repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: '22.x' + registry-url: 'https://registry.npmjs.org' + + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: ./tools/release/npm/build.ts ${{ github.event.inputs.version }} --publish diff --git a/.github/workflows/promote_to_release.yml b/.github/workflows/promote_to_release.yml index 79fefa6d6c..4079118d92 100644 --- a/.github/workflows/promote_to_release.yml +++ b/.github/workflows/promote_to_release.yml @@ -42,7 +42,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install rust-codesign run: |- diff --git a/.github/workflows/start_release.yml b/.github/workflows/start_release.yml index 40a44bb61a..35446c1adb 100644 --- a/.github/workflows/start_release.yml +++ b/.github/workflows/start_release.yml @@ -36,7 +36,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Create Gist URL env: diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 9038fe0d22..306a8642ad 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -41,7 +41,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Run version bump run: | diff --git a/tools/deno.lock.json b/tools/deno.lock.json index 46c09ce24b..3d21e5c9f6 100644 --- a/tools/deno.lock.json +++ b/tools/deno.lock.json @@ -1,81 +1,365 @@ { - "version": "3", - "packages": { - "specifiers": { - "jsr:@david/dax@0.41.0": "jsr:@david/dax@0.41.0", - "jsr:@david/which@^0.4.1": "jsr:@david/which@0.4.1", - "jsr:@deno/patchver@0.1.0": "jsr:@deno/patchver@0.1.0", - "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", - "jsr:@std/bytes@^0.221.0": "jsr:@std/bytes@0.221.0", - "jsr:@std/fmt@1": "jsr:@std/fmt@1.0.0", - "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", - "jsr:@std/fs@0.221.0": "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/io@^0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0": "jsr:@std/streams@0.221.0", - "jsr:@std/yaml@^0.221": "jsr:@std/yaml@0.221.0" + "version": "4", + "specifiers": { + "jsr:@david/dax@0.41.0": "0.41.0", + "jsr:@david/dax@0.42": "0.42.0", + "jsr:@david/path@0.2": "0.2.0", + "jsr:@david/which@~0.4.1": "0.4.1", + "jsr:@deno/patchver@0.1.0": "0.1.0", + "jsr:@std/assert@0.221": "0.221.0", + "jsr:@std/bytes@0.221": "0.221.0", + "jsr:@std/fmt@0.221": "0.221.0", + "jsr:@std/fmt@1": "1.0.0", + "jsr:@std/fs@0.221.0": "0.221.0", + "jsr:@std/fs@1": "1.0.5", + "jsr:@std/io@0.221": "0.221.0", + "jsr:@std/io@0.221.0": "0.221.0", + "jsr:@std/path@0.221": "0.221.0", + "jsr:@std/path@0.221.0": "0.221.0", + "jsr:@std/path@1": "1.0.8", + "jsr:@std/path@^1.0.7": "1.0.8", + "jsr:@std/streams@0.221": "0.221.0", + "jsr:@std/streams@0.221.0": "0.221.0", + "jsr:@std/yaml@0.221": "0.221.0", + "npm:decompress@4.2.1": "4.2.1" + }, + "jsr": { + "@david/dax@0.41.0": { + "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", + "dependencies": [ + "jsr:@david/which", + "jsr:@std/fmt@0.221", + "jsr:@std/fs@0.221.0", + "jsr:@std/io@0.221.0", + "jsr:@std/path@0.221.0", + "jsr:@std/streams@0.221.0" + ] }, - "jsr": { - "@david/dax@0.41.0": { - "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", - "dependencies": [ - "jsr:@david/which@^0.4.1", - "jsr:@std/fmt@^0.221.0", - "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0" - ] - }, - "@david/which@0.4.1": { - "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" - }, - "@deno/patchver@0.1.0": { - "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" - }, - "@std/assert@0.221.0": { - "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" - }, - "@std/bytes@0.221.0": { - "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" - }, - "@std/fmt@0.221.0": { - "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" - }, - "@std/fmt@1.0.0": { - "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" - }, - "@std/fs@0.221.0": { - "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, - "@std/io@0.221.0": { - "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/bytes@^0.221.0" - ] - }, - "@std/path@0.221.0": { - "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", - "dependencies": [ - "jsr:@std/assert@^0.221.0" - ] - }, - "@std/streams@0.221.0": { - "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", - "dependencies": [ - "jsr:@std/io@^0.221.0" - ] - }, - "@std/yaml@0.221.0": { - "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" - } + "@david/dax@0.42.0": { + "integrity": "0c547c9a20577a6072b90def194c159c9ddab82280285ebfd8268a4ebefbd80b", + "dependencies": [ + "jsr:@david/path", + "jsr:@david/which", + "jsr:@std/fmt@1", + "jsr:@std/fs@1", + "jsr:@std/io@0.221", + "jsr:@std/path@1", + "jsr:@std/streams@0.221" + ] + }, + "@david/path@0.2.0": { + "integrity": "f2d7aa7f02ce5a55e27c09f9f1381794acb09d328f8d3c8a2e3ab3ffc294dccd", + "dependencies": [ + "jsr:@std/fs@1", + "jsr:@std/path@1" + ] + }, + "@david/which@0.4.1": { + "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" + }, + "@deno/patchver@0.1.0": { + "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" + }, + "@std/assert@0.221.0": { + "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" + }, + "@std/bytes@0.221.0": { + "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" + }, + "@std/fmt@0.221.0": { + "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" + }, + "@std/fmt@1.0.0": { + "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" + }, + "@std/fs@0.221.0": { + "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/path@0.221" + ] + }, + "@std/fs@1.0.5": { + "integrity": "41806ad6823d0b5f275f9849a2640d87e4ef67c51ee1b8fb02426f55e02fd44e", + "dependencies": [ + "jsr:@std/path@^1.0.7" + ] + }, + "@std/io@0.221.0": { + "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/bytes" + ] + }, + "@std/path@0.221.0": { + "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", + "dependencies": [ + "jsr:@std/assert" + ] + }, + "@std/path@1.0.8": { + "integrity": "548fa456bb6a04d3c1a1e7477986b6cffbce95102d0bb447c67c4ee70e0364be" + }, + "@std/streams@0.221.0": { + "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", + "dependencies": [ + "jsr:@std/io@0.221" + ] + }, + "@std/yaml@0.221.0": { + "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" + } + }, + "npm": { + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl@1.2.3": { + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "dependencies": [ + "readable-stream", + "safe-buffer@5.2.1" + ] + }, + "buffer-alloc-unsafe@1.1.0": { + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, + "buffer-alloc@1.2.0": { + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dependencies": [ + "buffer-alloc-unsafe", + "buffer-fill" + ] + }, + "buffer-crc32@0.2.13": { + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" + }, + "buffer-fill@1.0.0": { + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==" + }, + "buffer@5.7.1": { + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "commander@2.20.3": { + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "decompress-tar@4.1.1": { + "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", + "dependencies": [ + "file-type@5.2.0", + "is-stream", + "tar-stream" + ] + }, + "decompress-tarbz2@4.1.1": { + "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", + "dependencies": [ + "decompress-tar", + "file-type@6.2.0", + "is-stream", + "seek-bzip", + "unbzip2-stream" + ] + }, + "decompress-targz@4.1.1": { + "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", + "dependencies": [ + "decompress-tar", + "file-type@5.2.0", + "is-stream" + ] + }, + "decompress-unzip@4.0.1": { + "integrity": "sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==", + "dependencies": [ + "file-type@3.9.0", + "get-stream", + "pify@2.3.0", + "yauzl" + ] + }, + "decompress@4.2.1": { + "integrity": "sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==", + "dependencies": [ + "decompress-tar", + "decompress-tarbz2", + "decompress-targz", + "decompress-unzip", + "graceful-fs", + "make-dir", + "pify@2.3.0", + "strip-dirs" + ] + }, + "end-of-stream@1.4.4": { + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": [ + "once" + ] + }, + "fd-slicer@1.1.0": { + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": [ + "pend" + ] + }, + "file-type@3.9.0": { + "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==" + }, + "file-type@5.2.0": { + "integrity": "sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==" + }, + "file-type@6.2.0": { + "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==" + }, + "fs-constants@1.0.0": { + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "get-stream@2.3.1": { + "integrity": "sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==", + "dependencies": [ + "object-assign", + "pinkie-promise" + ] + }, + "graceful-fs@4.2.11": { + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "ieee754@1.2.1": { + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-natural-number@4.0.1": { + "integrity": "sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==" + }, + "is-stream@1.1.0": { + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "make-dir@1.3.0": { + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dependencies": [ + "pify@3.0.0" + ] + }, + "object-assign@4.1.1": { + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "once@1.4.0": { + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": [ + "wrappy" + ] + }, + "pend@1.2.0": { + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "pify@2.3.0": { + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==" + }, + "pify@3.0.0": { + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==" + }, + "pinkie-promise@2.0.1": { + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": [ + "pinkie" + ] + }, + "pinkie@2.0.4": { + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray", + "process-nextick-args", + "safe-buffer@5.1.2", + "string_decoder", + "util-deprecate" + ] + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "seek-bzip@1.0.6": { + "integrity": "sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==", + "dependencies": [ + "commander" + ] + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer@5.1.2" + ] + }, + "strip-dirs@2.1.0": { + "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", + "dependencies": [ + "is-natural-number" + ] + }, + "tar-stream@1.6.2": { + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dependencies": [ + "bl", + "buffer-alloc", + "end-of-stream", + "fs-constants", + "readable-stream", + "to-buffer", + "xtend" + ] + }, + "through@2.3.8": { + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "to-buffer@1.1.1": { + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" + }, + "unbzip2-stream@1.4.3": { + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": [ + "buffer", + "through" + ] + }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "wrappy@1.0.2": { + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "xtend@4.0.2": { + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "yauzl@2.10.0": { + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": [ + "buffer-crc32", + "fd-slicer" + ] } }, "remote": { diff --git a/tools/release/npm/.gitignore b/tools/release/npm/.gitignore new file mode 100644 index 0000000000..1521c8b765 --- /dev/null +++ b/tools/release/npm/.gitignore @@ -0,0 +1 @@ +dist diff --git a/tools/release/npm/bin.cjs b/tools/release/npm/bin.cjs new file mode 100644 index 0000000000..984aa350f1 --- /dev/null +++ b/tools/release/npm/bin.cjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// @ts-check +const path = require("path"); +const child_process = require("child_process"); +const os = require("os"); +const fs = require("fs"); + +const exePath = path.join( + __dirname, + os.platform() === "win32" ? "deno.exe" : "deno", +); + +if (!fs.existsSync(exePath)) { + try { + const resolvedExePath = require("./install_api.cjs").runInstall(); + runDenoExe(resolvedExePath); + } catch (err) { + if (err !== undefined && typeof err.message === "string") { + console.error(err.message); + } else { + console.error(err); + } + process.exit(1); + } +} else { + runDenoExe(exePath); +} + +/** @param exePath {string} */ +function runDenoExe(exePath) { + const result = child_process.spawnSync( + exePath, + process.argv.slice(2), + { stdio: "inherit" }, + ); + if (result.error) { + throw result.error; + } + + throwIfNoExePath(); + + process.exitCode = result.status; + + function throwIfNoExePath() { + if (!fs.existsSync(exePath)) { + throw new Error( + "Could not find exe at path '" + exePath + + "'. Maybe try running deno again.", + ); + } + } +} diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts new file mode 100644 index 0000000000..b1f1c45cbf --- /dev/null +++ b/tools/release/npm/build.ts @@ -0,0 +1,237 @@ +#!/usr/bin/env -S deno run -A --lock=tools/deno.lock.json +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// NOTICE: This deployment/npm folder was lifted from https://github.com/dprint/dprint/blob/0ba79811cc96d2dee8e0cf766a8c8c0fc44879c2/deployment/npm/ +// with permission (Copyright 2019-2023 David Sherret) +import $ from "jsr:@david/dax@^0.42.0"; +// @ts-types="npm:@types/decompress@4.2.7" +import decompress from "npm:decompress@4.2.1"; +import { parseArgs } from "@std/cli/parse-args"; + +interface Package { + zipFileName: string; + os: "win32" | "darwin" | "linux"; + cpu: "x64" | "arm64"; + libc?: "glibc" | "musl"; +} + +const args = parseArgs(Deno.args, { + boolean: ["publish"], +}); +const packages: Package[] = [{ + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "x64", +}, { + // use x64_64 until there's an arm64 build + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-apple-darwin.zip", + os: "darwin", + cpu: "x64", +}, { + zipFileName: "deno-aarch64-apple-darwin.zip", + os: "darwin", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-unknown-linux-gnu.zip", + os: "linux", + cpu: "x64", + libc: "glibc", +}, { + zipFileName: "deno-aarch64-unknown-linux-gnu.zip", + os: "linux", + cpu: "arm64", + libc: "glibc", +}]; + +const markdownText = `# Deno + +[Deno](https://www.deno.com) +([/ˈdiːnoʊ/](http://ipa-reader.xyz/?text=%CB%88di%CB%90no%CA%8A), pronounced +\`dee-no\`) is a JavaScript, TypeScript, and WebAssembly runtime with secure +defaults and a great developer experience. It's built on [V8](https://v8.dev/), +[Rust](https://www.rust-lang.org/), and [Tokio](https://tokio.rs/). + +Learn more about the Deno runtime +[in the documentation](https://docs.deno.com/runtime/manual). +`; + +const currentDir = $.path(import.meta.url).parentOrThrow(); +const rootDir = currentDir.parentOrThrow().parentOrThrow().parentOrThrow(); +const outputDir = currentDir.join("./dist"); +const scopeDir = outputDir.join("@deno"); +const denoDir = outputDir.join("deno"); +const version = resolveVersion(); + +$.logStep(`Publishing ${version}...`); + +await $`rm -rf ${outputDir}`; +await $`mkdir -p ${denoDir} ${scopeDir}`; + +// setup Deno packages +{ + $.logStep(`Setting up deno ${version}...`); + const pkgJson = { + "name": "deno", + "version": version, + "description": "A modern runtime for JavaScript and TypeScript.", + "bin": "bin.cjs", + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + "keywords": [ + "runtime", + "typescript", + ], + "author": "the Deno authors", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.com", + // for yarn berry (https://github.com/dprint/dprint/issues/686) + "preferUnplugged": true, + "scripts": { + "postinstall": "node ./install.cjs", + }, + optionalDependencies: packages + .map((pkg) => `@deno/${getPackageNameNoScope(pkg)}`) + .reduce((obj, pkgName) => ({ ...obj, [pkgName]: version }), {}), + }; + currentDir.join("bin.cjs").copyFileToDirSync(denoDir); + currentDir.join("install_api.cjs").copyFileToDirSync(denoDir); + currentDir.join("install.cjs").copyFileToDirSync(denoDir); + denoDir.join("package.json").writeJsonPrettySync(pkgJson); + rootDir.join("LICENSE.md").copyFileSync(denoDir.join("LICENSE")); + denoDir.join("README.md").writeTextSync(markdownText); + // ensure the test files don't get published + denoDir.join(".npmignore").writeTextSync("deno\ndeno.exe\n"); + + // setup each binary package + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Setting up @deno/${pkgName}...`); + const pkgDir = scopeDir.join(pkgName); + const zipPath = pkgDir.join("output.zip"); + + await $`mkdir -p ${pkgDir}`; + + // download and extract the zip file + const zipUrl = + `https://github.com/denoland/deno/releases/download/v${version}/${pkg.zipFileName}`; + await $.request(zipUrl).showProgress().pipeToPath(zipPath); + await decompress(zipPath.toString(), pkgDir.toString()); + zipPath.removeSync(); + + // create the package.json and readme + pkgDir.join("README.md").writeTextSync( + `# @denoland/${pkgName}\n\n${pkgName} distribution of [Deno](https://deno.land).\n`, + ); + pkgDir.join("package.json").writeJsonPrettySync({ + "name": `@deno/${pkgName}`, + "version": version, + "description": `${pkgName} distribution of Deno`, + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + // force yarn to unpack + "preferUnplugged": true, + "author": "David Sherret", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.land", + "os": [pkg.os], + "cpu": [pkg.cpu], + libc: pkg.libc == null ? undefined : [pkg.libc], + }); + } +} + +// verify that the package is created correctly +{ + $.logStep("Verifying packages..."); + const testPlatform = Deno.build.os == "windows" + ? (Deno.build.arch === "x86_64" ? "@deno/win32-x64" : "@deno/win32-arm64") + : Deno.build.os === "darwin" + ? (Deno.build.arch === "x86_64" ? "@deno/darwin-x64" : "@deno/darwin-arm64") + : "@deno/linux-x64-glibc"; + outputDir.join("package.json").writeJsonPrettySync({ + workspaces: [ + "deno", + // There seems to be a bug with npm workspaces where this doesn't + // work, so for now make some assumptions and only include the package + // that works on the CI for the current operating system + // ...packages.map(p => `@deno/${getPackageNameNoScope(p)}`), + testPlatform, + ], + }); + + const denoExe = Deno.build.os === "windows" ? "deno.exe" : "deno"; + await $`npm install`.cwd(denoDir); + + // ensure the post-install script adds the executable to the deno package, + // which is necessary for faster caching and to ensure the vscode extension + // picks it up + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist after post install"); + } + + // run once after post install created deno, once with a simulated readonly file system, once creating the cache and once with + await $`node bin.cjs -v && rm ${denoExe} && DENO_SIMULATED_READONLY_FILE_SYSTEM=1 node bin.cjs -v && node bin.cjs -v && node bin.cjs -v` + .cwd(denoDir); + + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist when lazily initialized"); + } +} + +// publish if necessary +if (args.publish) { + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Publishing @deno/${pkgName}...`); + if (await checkPackagePublished(`@deno/${pkgName}`)) { + $.logLight(" Already published."); + continue; + } + const pkgDir = scopeDir.join(pkgName); + await $`cd ${pkgDir} && npm publish --provenance --access public`; + } + + $.logStep(`Publishing deno...`); + await $`cd ${denoDir} && npm publish --provenance --access public`; +} + +function getPackageNameNoScope(name: Package) { + const libc = name.libc == null ? "" : `-${name.libc}`; + return `${name.os}-${name.cpu}${libc}`; +} + +function resolveVersion() { + const firstArg = args._[0]; + if ( + firstArg != null && + typeof firstArg === "string" && + firstArg.trim().length > 0 + ) { + return firstArg; + } + const version = (rootDir.join("cli/Cargo.toml").readTextSync().match( + /version = "(.*?)"/, + ))?.[1]; + if (version == null) { + throw new Error("Could not resolve version."); + } + return version; +} + +async function checkPackagePublished(pkgName: string) { + const result = await $`npm info ${pkgName}@${version}`.quiet().noThrow(); + return result.code === 0; +} diff --git a/tools/release/npm/install.cjs b/tools/release/npm/install.cjs new file mode 100644 index 0000000000..8bf9aabe47 --- /dev/null +++ b/tools/release/npm/install.cjs @@ -0,0 +1,5 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +require("./install_api.cjs").runInstall(); diff --git a/tools/release/npm/install_api.cjs b/tools/release/npm/install_api.cjs new file mode 100644 index 0000000000..026d8ccc45 --- /dev/null +++ b/tools/release/npm/install_api.cjs @@ -0,0 +1,196 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +/** @type {string | undefined} */ +let cachedIsMusl = undefined; + +module.exports = { + runInstall() { + const denoFileName = os.platform() === "win32" ? "deno.exe" : "deno"; + const targetExecutablePath = path.join( + __dirname, + denoFileName, + ); + + if (fs.existsSync(targetExecutablePath)) { + return targetExecutablePath; + } + + const target = getTarget(); + const sourcePackagePath = path.dirname( + require.resolve("@deno/" + target + "/package.json"), + ); + const sourceExecutablePath = path.join(sourcePackagePath, denoFileName); + + if (!fs.existsSync(sourceExecutablePath)) { + throw new Error( + "Could not find executable for @deno/" + target + " at " + + sourceExecutablePath, + ); + } + + try { + if (process.env.DPRINT_SIMULATED_READONLY_FILE_SYSTEM === "1") { + console.warn("Simulating readonly file system for testing."); + throw new Error("Throwing for testing purposes."); + } + + // in order to make things faster the next time we run and to allow the + // deno vscode extension to easily pick this up, copy the executable + // into the deno package folder + hardLinkOrCopy(sourceExecutablePath, targetExecutablePath); + if (os.platform() !== "win32") { + // chomd +x + chmodX(targetExecutablePath); + } + return targetExecutablePath; + } catch (err) { + // this may fail on readonly file systems... in this case, fall + // back to using the resolved package path + if (process.env.DENO_DEBUG === "1") { + console.warn( + "Failed to copy executable from " + + sourceExecutablePath + " to " + targetExecutablePath + + ". Using resolved package path instead.", + err, + ); + } + // use the path found in the specific package + try { + chmodX(sourceExecutablePath); + } catch (_err) { + // ignore + } + return sourceExecutablePath; + } + }, +}; + +/** @filePath {string} */ +function chmodX(filePath) { + const perms = fs.statSync(filePath).mode; + fs.chmodSync(filePath, perms | 0o111); +} + +function getTarget() { + const platform = os.platform(); + if (platform === "linux") { + return platform + "-" + getArch() + "-" + getLinuxFamily(); + } else { + return platform + "-" + getArch(); + } +} + +function getArch() { + const arch = os.arch(); + if (arch !== "arm64" && arch !== "x64") { + throw new Error( + "Unsupported architecture " + os.arch() + + ". Only x64 and aarch64 binaries are available.", + ); + } + return arch; +} + +function getLinuxFamily() { + if (getIsMusl()) { + throw new Error( + "Musl is not supported. It's one of our priorities. Please upvote this issue: https://github.com/denoland/deno/issues/3711", + ); + // return "musl"; + } + return "glibc"; + + function getIsMusl() { + // code adapted from https://github.com/lovell/detect-libc + // Copyright Apache 2.0 license, the detect-libc maintainers + if (cachedIsMusl == null) { + cachedIsMusl = innerGet(); + } + return cachedIsMusl; + + function innerGet() { + try { + if (os.platform() !== "linux") { + return false; + } + return isProcessReportMusl() || isConfMusl(); + } catch (err) { + // just in case + console.warn("Error checking if musl.", err); + return false; + } + } + + function isProcessReportMusl() { + if (!process.report) { + return false; + } + const rawReport = process.report.getReport(); + const report = typeof rawReport === "string" + ? JSON.parse(rawReport) + : rawReport; + if (!report || !(report.sharedObjects instanceof Array)) { + return false; + } + return report.sharedObjects.some((o) => + o.includes("libc.musl-") || o.includes("ld-musl-") + ); + } + + function isConfMusl() { + const output = getCommandOutput(); + const [_, ldd1] = output.split(/[\r\n]+/); + return ldd1 && ldd1.includes("musl"); + } + + function getCommandOutput() { + try { + const command = + "getconf GNU_LIBC_VERSION 2>&1 || true; ldd --version 2>&1 || true"; + return require("child_process").execSync(command, { encoding: "utf8" }); + } catch (_err) { + return ""; + } + } + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function hardLinkOrCopy(sourcePath, destinationPath) { + try { + fs.linkSync(sourcePath, destinationPath); + } catch { + atomicCopyFile(sourcePath, destinationPath); + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function atomicCopyFile(sourcePath, destinationPath) { + const crypto = require("crypto"); + const rand = crypto.randomBytes(4).toString("hex"); + const tempFilePath = destinationPath + "." + rand; + fs.copyFileSync(sourcePath, tempFilePath); + try { + fs.renameSync(tempFilePath, destinationPath); + } catch (err) { + // will maybe throw when another process had already done this + // so just ignore and delete the created temporary file + try { + fs.unlinkSync(tempFilePath); + } catch (_err2) { + // ignore + } + throw err; + } +} From f4f64cbe2defd7fd4edf9b4ccd7e1ffc96f05917 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 18:58:14 -0500 Subject: [PATCH 4/8] fix(npm): search node_modules folder for package matching npm specifier (#27345) --- resolvers/deno/lib.rs | 18 +++--- resolvers/deno/npm/byonm.rs | 61 +++++++++++++++++-- .../__test__.jsonc | 18 ++++++ .../matches.ts | 3 + .../node_modules/aliased/index.js | 3 + .../node_modules/aliased/package.json | 4 ++ .../node_modules/package/index.js | 3 + .../node_modules/package/package.json | 4 ++ .../not_matches.out | 2 + .../not_matches.ts | 3 + .../not_matches_aliased.out | 2 + .../not_matches_aliased.ts | 3 + .../package.json | 2 + 13 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts create mode 100644 tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 661caf836d..a74ca614a3 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -355,16 +355,16 @@ impl< }) .map_err(|err| { match err.into_kind() { - ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { - err.into() + ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { + err.into() + } } - } }); } } diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index 771f23ce23..6e1be35ca0 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -205,9 +205,9 @@ impl ByonmNpmResolver { } // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = url_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { + let maybe_referrer_path = url_to_file_path(referrer).ok(); + if let Some(file_path) = maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { let package_json_path = dir_path.join("package.json"); if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { if let Some(alias) = @@ -216,11 +216,10 @@ impl ByonmNpmResolver { return Ok(Some((pkg_json, alias))); } } - current_path = dir_path; } } - // otherwise, fall fallback to the project's package.json + // fall fallback to the project's package.json if let Some(root_node_modules_dir) = &self.root_node_modules_dir { let root_pkg_json_path = root_node_modules_dir.parent().unwrap().join("package.json"); @@ -232,6 +231,58 @@ impl ByonmNpmResolver { } } + // now try to resolve based on the closest node_modules directory + let maybe_referrer_path = url_to_file_path(referrer).ok(); + let search_node_modules = |node_modules: &Path| { + if req.version_req.tag().is_some() { + return None; + } + + let pkg_folder = node_modules.join(&req.name); + if let Ok(Some(dep_pkg_json)) = + self.load_pkg_json(&pkg_folder.join("package.json")) + { + if dep_pkg_json.name.as_ref() == Some(&req.name) { + let matches_req = dep_pkg_json + .version + .as_ref() + .and_then(|v| Version::parse_from_npm(v).ok()) + .map(|version| req.version_req.matches(&version)) + .unwrap_or(true); + if matches_req { + return Some((dep_pkg_json, req.name.clone())); + } + } + } + None + }; + if let Some(file_path) = &maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { + if let Some(result) = + search_node_modules(&dir_path.join("node_modules")) + { + return Ok(Some(result)); + } + } + } + + // and finally check the root node_modules directory + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let already_searched = maybe_referrer_path + .as_ref() + .and_then(|referrer_path| { + root_node_modules_dir + .parent() + .map(|root_dir| referrer_path.starts_with(root_dir)) + }) + .unwrap_or(false); + if !already_searched { + if let Some(result) = search_node_modules(root_node_modules_dir) { + return Ok(Some(result)); + } + } + } + Ok(None) } diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc new file mode 100644 index 0000000000..e2c5495387 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc @@ -0,0 +1,18 @@ +{ + "tests": { + "matches": { + "args": "run -A matches.ts", + "output": "5\n" + }, + "not_matches": { + "args": "run -A not_matches.ts", + "output": "not_matches.out", + "exitCode": 1 + }, + "not_matches_aliased": { + "args": "run -A not_matches_aliased.ts", + "output": "not_matches_aliased.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts new file mode 100644 index 0000000000..986e7baf4c --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json new file mode 100644 index 0000000000..618960872f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json @@ -0,0 +1,4 @@ +{ + "name": "not-same-name", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json new file mode 100644 index 0000000000..5723987e9f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json @@ -0,0 +1,4 @@ +{ + "name": "package", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out new file mode 100644 index 0000000000..c549e13ff2 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:package@2' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts new file mode 100644 index 0000000000..a337bd7d86 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@2"; // won't match 2 + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out new file mode 100644 index 0000000000..93b52f64bc --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:aliased@1' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts new file mode 100644 index 0000000000..85fa11e319 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts @@ -0,0 +1,3 @@ +import { add } from "npm:aliased@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json @@ -0,0 +1,2 @@ +{ +} From a63f8452e99d1896703098799f246d07fa0f30c2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 19:30:21 -0500 Subject: [PATCH 5/8] chore: mark ./tools/release/npm/build.ts as executable (#27349) --- tools/release/npm/build.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/release/npm/build.ts diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts old mode 100644 new mode 100755 From 960776cd32331495034fcda21b194d897e780892 Mon Sep 17 00:00:00 2001 From: Satya Rohith Date: Fri, 13 Dec 2024 06:14:42 +0530 Subject: [PATCH 6/8] fix(ext/node): support createConnection option in node:http.request() (#25470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit changes "node:http" module to add support for the "createConnection" option when the "request()" API is called. Closes https://github.com/denoland/deno/issues/19507 --------- Signed-off-by: Yoshiya Hinosawa Signed-off-by: Satya Rohith Co-authored-by: Yoshiya Hinosawa Co-authored-by: Bartek Iwańczuk Co-authored-by: crowlkats --- ext/fetch/lib.rs | 3 - ext/node/lib.rs | 4 +- ext/node/ops/http.rs | 278 +++++++++++------- ext/node/polyfills/_http_outgoing.ts | 57 +++- ext/node/polyfills/_tls_wrap.ts | 7 + ext/node/polyfills/http.ts | 159 ++++++---- ext/node/polyfills/https.ts | 2 +- .../polyfills/internal_binding/cares_wrap.ts | 10 +- ext/node/polyfills/net.ts | 40 ++- runtime/errors.rs | 24 +- tests/node_compat/config.jsonc | 8 - tests/node_compat/runner/TODO.md | 10 +- .../test/parallel/test-http-agent-false.js | 53 ---- .../test-http-agent-keepalive-delay.js | 43 --- ...st-http-client-timeout-connect-listener.js | 49 --- .../test-http-dump-req-when-res-ends.js | 73 ----- .../test-http-hostname-typechecking.js | 49 --- .../test-net-better-error-messages-port.js | 24 -- .../test-net-connect-handle-econnrefused.js | 39 --- .../sequential/test-net-reconnect-error.js | 50 ---- tests/unit_node/http_test.ts | 80 ++++- 21 files changed, 465 insertions(+), 597 deletions(-) delete mode 100644 tests/node_compat/test/parallel/test-http-agent-false.js delete mode 100644 tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js delete mode 100644 tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js delete mode 100644 tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js delete mode 100644 tests/node_compat/test/parallel/test-http-hostname-typechecking.js delete mode 100644 tests/node_compat/test/sequential/test-net-better-error-messages-port.js delete mode 100644 tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js delete mode 100644 tests/node_compat/test/sequential/test-net-reconnect-error.js diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index a3f5d03e64..919c6d3044 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -206,9 +206,6 @@ pub enum FetchError { RequestBuilderHook(deno_core::error::AnyError), #[error(transparent)] Io(#[from] std::io::Error), - // Only used for node upgrade - #[error(transparent)] - Hyper(#[from] hyper::Error), } pub type CancelableResponseFuture = diff --git a/ext/node/lib.rs b/ext/node/lib.rs index bf593ad432..1e6c920c9e 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -364,9 +364,9 @@ deno_core::extension!(deno_node, ops::zlib::brotli::op_create_brotli_decompress, ops::zlib::brotli::op_brotli_decompress_stream, ops::zlib::brotli::op_brotli_decompress_stream_end, - ops::http::op_node_http_request

, ops::http::op_node_http_fetch_response_upgrade, - ops::http::op_node_http_fetch_send, + ops::http::op_node_http_request_with_conn

, + ops::http::op_node_http_await_response, ops::http2::op_http2_connect, ops::http2::op_http2_poll_client_connection, ops::http2::op_http2_client_request, diff --git a/ext/node/ops/http.rs b/ext/node/ops/http.rs index f4adb94060..eb28e68aee 100644 --- a/ext/node/ops/http.rs +++ b/ext/node/ops/http.rs @@ -2,18 +2,20 @@ use std::borrow::Cow; use std::cell::RefCell; +use std::fmt::Debug; use std::pin::Pin; use std::rc::Rc; use std::task::Context; use std::task::Poll; use bytes::Bytes; +use deno_core::error::bad_resource; +use deno_core::error::type_error; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::futures::Stream; use deno_core::futures::StreamExt; -use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::serde::Serialize; use deno_core::unsync::spawn; @@ -25,17 +27,17 @@ use deno_core::ByteString; use deno_core::CancelFuture; use deno_core::CancelHandle; use deno_core::CancelTryFuture; +use deno_core::Canceled; use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; -use deno_fetch::get_or_create_client_from_state; use deno_fetch::FetchCancelHandle; -use deno_fetch::FetchError; -use deno_fetch::FetchRequestResource; use deno_fetch::FetchReturn; -use deno_fetch::HttpClientResource; use deno_fetch::ResBody; +use deno_net::io::TcpStreamResource; +use deno_net::ops_tls::TlsStreamResource; +use deno_permissions::PermissionCheckError; use http::header::HeaderMap; use http::header::HeaderName; use http::header::HeaderValue; @@ -44,41 +46,140 @@ use http::header::CONTENT_LENGTH; use http::Method; use http_body_util::BodyExt; use hyper::body::Frame; +use hyper::body::Incoming; use hyper_util::rt::TokioIo; use std::cmp::min; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; -#[op2(stack_trace)] +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NodeHttpResponse { + pub status: u16, + pub status_text: String, + pub headers: Vec<(ByteString, ByteString)>, + pub url: String, + pub response_rid: ResourceId, + pub content_length: Option, + pub remote_addr_ip: Option, + pub remote_addr_port: Option, + pub error: Option, +} + +type CancelableResponseResult = + Result, hyper::Error>, Canceled>; + +pub struct NodeHttpClientResponse { + response: Pin>>, + url: String, +} + +impl Debug for NodeHttpClientResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeHttpClientResponse") + .field("url", &self.url) + .finish() + } +} + +impl deno_core::Resource for NodeHttpClientResponse { + fn name(&self) -> Cow { + "nodeHttpClientResponse".into() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ConnError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error("Invalid URL {0}")] + InvalidUrl(Url), + #[error(transparent)] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Method(#[from] http::method::InvalidMethod), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("TLS stream is currently in use")] + TlsStreamBusy, + #[error("TCP stream is currently in use")] + TcpStreamBusy, + #[error(transparent)] + ReuniteTcp(#[from] tokio::net::tcp::ReuniteError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error(transparent)] + Hyper(#[from] hyper::Error), +} + +#[op2(async, stack_trace)] #[serde] -pub fn op_node_http_request

( - state: &mut OpState, +pub async fn op_node_http_request_with_conn

( + state: Rc>, #[serde] method: ByteString, #[string] url: String, #[serde] headers: Vec<(ByteString, ByteString)>, - #[smi] client_rid: Option, #[smi] body: Option, -) -> Result + #[smi] conn_rid: ResourceId, + encrypted: bool, +) -> Result where P: crate::NodePermissions + 'static, { - let client = if let Some(rid) = client_rid { - let r = state + let (_handle, mut sender) = if encrypted { + let resource_rc = state + .borrow_mut() .resource_table - .get::(rid) - .map_err(FetchError::Resource)?; - r.client.clone() + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_e| ConnError::TlsStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.unsplit(write_half); + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + ( + tokio::task::spawn(async move { conn.with_upgrades().await }), + sender, + ) } else { - get_or_create_client_from_state(state)? + let resource_rc = state + .borrow_mut() + .resource_table + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_| ConnError::TcpStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.reunite(write_half)?; + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + + // Spawn a task to poll the connection, driving the HTTP state + ( + tokio::task::spawn(async move { + conn.with_upgrades().await?; + Ok::<_, _>(()) + }), + sender, + ) }; + // Create the request. let method = Method::from_bytes(&method)?; - let mut url = Url::parse(&url)?; - let maybe_authority = deno_fetch::extract_authority(&mut url); + let mut url_parsed = Url::parse(&url)?; + let maybe_authority = deno_fetch::extract_authority(&mut url_parsed); { - let permissions = state.borrow_mut::

(); - permissions.check_net_url(&url, "ClientRequest")?; + let mut state_ = state.borrow_mut(); + let permissions = state_.borrow_mut::

(); + permissions.check_net_url(&url_parsed, "ClientRequest")?; } let mut header_map = HeaderMap::new(); @@ -93,9 +194,10 @@ where ( BodyExt::boxed(NodeHttpResourceToBodyAdapter::new( state + .borrow_mut() .resource_table .take_any(body) - .map_err(FetchError::Resource)?, + .map_err(ConnError::Resource)?, )), None, ) @@ -117,10 +219,13 @@ where let mut request = http::Request::new(body); *request.method_mut() = method.clone(); - *request.uri_mut() = url - .as_str() + let path = url_parsed.path(); + let query = url_parsed.query(); + *request.uri_mut() = query + .map(|q| format!("{}?{}", path, q)) + .unwrap_or_else(|| path.to_string()) .parse() - .map_err(|_| FetchError::InvalidUrl(url.clone()))?; + .map_err(|_| ConnError::InvalidUrl(url_parsed.clone()))?; *request.headers_mut() = header_map; if let Some((username, password)) = maybe_authority { @@ -136,86 +241,44 @@ where let cancel_handle = CancelHandle::new_rc(); let cancel_handle_ = cancel_handle.clone(); - let fut = async move { - client - .send(request) - .map_err(Into::into) - .or_cancel(cancel_handle_) - .await - }; + let fut = + async move { sender.send_request(request).or_cancel(cancel_handle_).await }; - let request_rid = state.resource_table.add(FetchRequestResource { - future: Box::pin(fut), - url, - }); + let rid = state + .borrow_mut() + .resource_table + .add(NodeHttpClientResponse { + response: Box::pin(fut), + url: url.clone(), + }); - let cancel_handle_rid = - state.resource_table.add(FetchCancelHandle(cancel_handle)); + let cancel_handle_rid = state + .borrow_mut() + .resource_table + .add(FetchCancelHandle(cancel_handle)); Ok(FetchReturn { - request_rid, + request_rid: rid, cancel_handle_rid: Some(cancel_handle_rid), }) } -#[derive(Default, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct NodeHttpFetchResponse { - pub status: u16, - pub status_text: String, - pub headers: Vec<(ByteString, ByteString)>, - pub url: String, - pub response_rid: ResourceId, - pub content_length: Option, - pub remote_addr_ip: Option, - pub remote_addr_port: Option, - pub error: Option, -} - #[op2(async)] #[serde] -pub async fn op_node_http_fetch_send( +pub async fn op_node_http_await_response( state: Rc>, #[smi] rid: ResourceId, -) -> Result { - let request = state +) -> Result { + let resource = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; - - let request = Rc::try_unwrap(request) - .ok() - .expect("multiple op_node_http_fetch_send ongoing"); - - let res = match request.future.await { - Ok(Ok(res)) => res, - Ok(Err(err)) => { - // We're going to try and rescue the error cause from a stream and return it from this fetch. - // If any error in the chain is a hyper body error, return that as a special result we can use to - // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). - // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - - if let FetchError::ClientSend(err_src) = &err { - if let Some(client_err) = std::error::Error::source(&err_src.source) { - if let Some(err_src) = client_err.downcast_ref::() { - if let Some(err_src) = std::error::Error::source(err_src) { - return Ok(NodeHttpFetchResponse { - error: Some(err_src.to_string()), - ..Default::default() - }); - } - } - } - } - - return Err(err); - } - Err(_) => return Err(FetchError::RequestCanceled), - }; + .take::(rid) + .map_err(ConnError::Resource)?; + let resource = Rc::try_unwrap(resource) + .map_err(|_| ConnError::Resource(bad_resource("NodeHttpClientResponse")))?; + let res = resource.response.await??; let status = res.status(); - let url = request.url.into(); let mut res_headers = Vec::new(); for (key, val) in res.headers().iter() { res_headers.push((key.as_str().into(), val.as_bytes().into())); @@ -232,16 +295,22 @@ pub async fn op_node_http_fetch_send( (None, None) }; + let (parts, body) = res.into_parts(); + let body = body.map_err(deno_core::anyhow::Error::from); + let body = body.boxed(); + + let res = http::Response::from_parts(parts, body); + let response_rid = state .borrow_mut() .resource_table - .add(NodeHttpFetchResponseResource::new(res, content_length)); + .add(NodeHttpResponseResource::new(res, content_length)); - Ok(NodeHttpFetchResponse { + Ok(NodeHttpResponse { status: status.as_u16(), status_text: status.canonical_reason().unwrap_or("").to_string(), headers: res_headers, - url, + url: resource.url, response_rid, content_length, remote_addr_ip, @@ -255,12 +324,12 @@ pub async fn op_node_http_fetch_send( pub async fn op_node_http_fetch_response_upgrade( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let raw_response = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; + .take::(rid) + .map_err(ConnError::Resource)?; let raw_response = Rc::try_unwrap(raw_response) .expect("Someone is holding onto NodeHttpFetchResponseResource"); @@ -283,7 +352,7 @@ pub async fn op_node_http_fetch_response_upgrade( } read_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); spawn(async move { let mut buf = [0; 1024]; @@ -294,7 +363,7 @@ pub async fn op_node_http_fetch_response_upgrade( } upgraded_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); } @@ -379,13 +448,13 @@ impl Default for NodeHttpFetchResponseReader { } #[derive(Debug)] -pub struct NodeHttpFetchResponseResource { +pub struct NodeHttpResponseResource { pub response_reader: AsyncRefCell, pub cancel: CancelHandle, pub size: Option, } -impl NodeHttpFetchResponseResource { +impl NodeHttpResponseResource { pub fn new(response: http::Response, size: Option) -> Self { Self { response_reader: AsyncRefCell::new(NodeHttpFetchResponseReader::Start( @@ -400,14 +469,14 @@ impl NodeHttpFetchResponseResource { let reader = self.response_reader.into_inner(); match reader { NodeHttpFetchResponseReader::Start(resp) => { - Ok(hyper::upgrade::on(resp).await?) + hyper::upgrade::on(resp).await } _ => unreachable!(), } } } -impl Resource for NodeHttpFetchResponseResource { +impl Resource for NodeHttpResponseResource { fn name(&self) -> Cow { "fetchResponse".into() } @@ -454,9 +523,7 @@ impl Resource for NodeHttpFetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => { - break Err(deno_core::error::type_error(err.to_string())) - } + Err(err) => break Err(type_error(err.to_string())), }, None => break Ok(BufView::empty()), } @@ -464,7 +531,7 @@ impl Resource for NodeHttpFetchResponseResource { }; let cancel_handle = RcRef::map(self, |r| &r.cancel); - fut.try_or_cancel(cancel_handle).await.map_err(Into::into) + fut.try_or_cancel(cancel_handle).await }) } @@ -514,8 +581,9 @@ impl Stream for NodeHttpResourceToBodyAdapter { Poll::Ready(res) => match res { Ok(buf) if buf.is_empty() => Poll::Ready(None), Ok(buf) => { + let bytes: Bytes = buf.to_vec().into(); this.1 = Some(this.0.clone().read(64 * 1024)); - Poll::Ready(Some(Ok(buf.to_vec().into()))) + Poll::Ready(Some(Ok(bytes))) } Err(err) => Poll::Ready(Some(Err(err))), }, diff --git a/ext/node/polyfills/_http_outgoing.ts b/ext/node/polyfills/_http_outgoing.ts index 4da6b73e87..5a9a8ad7e6 100644 --- a/ext/node/polyfills/_http_outgoing.ts +++ b/ext/node/polyfills/_http_outgoing.ts @@ -491,19 +491,53 @@ Object.defineProperties( return ret; }, + /** Right after socket is ready, we need to writeHeader() to setup the request and + * client. This is invoked by onSocket(). */ + _flushHeaders() { + if (!this._headerSent) { + this._headerSent = true; + this._writeHeader(); + } + }, + // deno-lint-ignore no-explicit-any _send(data: any, encoding?: string | null, callback?: () => void) { - if (!this._headerSent && this._header !== null) { - this._writeHeader(); - this._headerSent = true; + // if socket is ready, write the data after headers are written. + // if socket is not ready, buffer data in outputbuffer. + if ( + this.socket && !this.socket.connecting && this.outputData.length === 0 + ) { + if (!this._headerSent) { + this._writeHeader(); + this._headerSent = true; + } + + return this._writeRaw(data, encoding, callback); + } else { + this.outputData.push({ data, encoding, callback }); } - return this._writeRaw(data, encoding, callback); + return false; }, _writeHeader() { throw new ERR_METHOD_NOT_IMPLEMENTED("_writeHeader()"); }, + _flushBuffer() { + const outputLength = this.outputData.length; + if (outputLength <= 0 || !this.socket || !this._bodyWriter) { + return undefined; + } + + const { data, encoding, callback } = this.outputData.shift(); + const ret = this._writeRaw(data, encoding, callback); + if (this.outputData.length > 0) { + this.once("drain", this._flushBuffer); + } + + return ret; + }, + _writeRaw( // deno-lint-ignore no-explicit-any data: any, @@ -517,11 +551,15 @@ Object.defineProperties( data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } if (data.buffer.byteLength > 0) { - this._bodyWriter.write(data).then(() => { - callback?.(); - this.emit("drain"); - }).catch((e) => { - this._requestSendError = e; + this._bodyWriter.ready.then(() => { + if (this._bodyWriter.desiredSize > 0) { + this._bodyWriter.write(data).then(() => { + callback?.(); + this.emit("drain"); + }).catch((e) => { + this._requestSendError = e; + }); + } }); } return false; @@ -658,7 +696,6 @@ Object.defineProperties( const { header } = state; this._header = header + "\r\n"; - this._headerSent = false; // Wait until the first body chunk, or close(), is sent to flush, // UNLESS we're sending Expect: 100-continue. diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index 4c7424a328..9e5def9f2b 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -154,6 +154,13 @@ export class TLSSocket extends net.Socket { const afterConnect = handle.afterConnect; handle.afterConnect = async (req: any, status: number) => { options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined + if (tlssock._isNpmAgent) { + // skips the TLS handshake for @npmcli/agent as it's handled by + // onSocket handler of ClientRequest object. + tlssock.emit("secure"); + tlssock.removeListener("end", onConnectEnd); + return afterConnect.call(handle, req, status); + } try { const conn = await Deno.startTls(handle[kStreamBaseField], options); diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index 948a3527bd..e911535be5 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -5,16 +5,17 @@ import { core, primordials } from "ext:core/mod.js"; import { + op_node_http_await_response, op_node_http_fetch_response_upgrade, - op_node_http_fetch_send, - op_node_http_request, + op_node_http_request_with_conn, + op_tls_start, } from "ext:core/ops"; import { TextEncoder } from "ext:deno_web/08_text_encoding.js"; import { setTimeout } from "ext:deno_web/02_timers.js"; import { _normalizeArgs, - // createConnection, + createConnection, ListenOptions, Socket, } from "node:net"; @@ -48,9 +49,10 @@ import { kOutHeaders } from "ext:deno_node/internal/http.ts"; import { _checkIsHttpToken as checkIsHttpToken } from "node:_http_common"; import { Agent, globalAgent } from "node:_http_agent"; import { urlToHttpOptions } from "ext:deno_node/internal/url.ts"; -import { kEmptyObject } from "ext:deno_node/internal/util.mjs"; +import { kEmptyObject, once } from "ext:deno_node/internal/util.mjs"; import { constants, TCP } from "ext:deno_node/internal_binding/tcp_wrap.ts"; -import { notImplemented, warnNotImplemented } from "ext:deno_node/_utils.ts"; +import { kStreamBaseField } from "ext:deno_node/internal_binding/stream_wrap.ts"; +import { notImplemented } from "ext:deno_node/_utils.ts"; import { connResetException, ERR_HTTP_HEADERS_SENT, @@ -62,7 +64,6 @@ import { } from "ext:deno_node/internal/errors.ts"; import { getTimerDuration } from "ext:deno_node/internal/timers.mjs"; import { serve, upgradeHttpRaw } from "ext:deno_http/00_serve.ts"; -import { createHttpClient } from "ext:deno_fetch/22_http_client.js"; import { headersEntries } from "ext:deno_fetch/20_headers.js"; import { timerId } from "ext:deno_web/03_abort_signal.js"; import { clearTimeout as webClearTimeout } from "ext:deno_web/02_timers.js"; @@ -148,6 +149,10 @@ class FakeSocket extends EventEmitter { } } +function emitErrorEvent(request, error) { + request.emit("error", error); +} + /** ClientRequest represents the http(s) request from the client */ class ClientRequest extends OutgoingMessage { defaultProtocol = "http:"; @@ -160,6 +165,8 @@ class ClientRequest extends OutgoingMessage { useChunkedEncodingByDefault: boolean; path: string; _req: { requestRid: number; cancelHandleRid: number | null } | undefined; + _encrypted = false; + socket: Socket; constructor( input: string | URL, @@ -382,17 +389,11 @@ class ClientRequest extends OutgoingMessage { delete optsWithoutSignal.signal; } - if (options!.createConnection) { - warnNotImplemented("ClientRequest.options.createConnection"); - } - if (options!.lookup) { notImplemented("ClientRequest.options.lookup"); } - // initiate connection - // TODO(crowlKats): finish this - /*if (this.agent) { + if (this.agent) { this.agent.addRequest(this, optsWithoutSignal); } else { // No agent, default to Connection:close. @@ -422,8 +423,7 @@ class ClientRequest extends OutgoingMessage { debug("CLIENT use net.createConnection", optsWithoutSignal); this.onSocket(createConnection(optsWithoutSignal)); } - }*/ - this.onSocket(new FakeSocket({ encrypted: this._encrypted })); + } } _writeHeader() { @@ -437,9 +437,6 @@ class ClientRequest extends OutgoingMessage { } } - const client = this._getClient() ?? createHttpClient({ http2: false }); - this._client = client; - if ( this.method === "POST" || this.method === "PATCH" || this.method === "PUT" ) { @@ -455,17 +452,29 @@ class ClientRequest extends OutgoingMessage { this._bodyWriteRid = resourceForReadableStream(readable); } - this._req = op_node_http_request( - this.method, - url, - headers, - client[internalRidSymbol], - this._bodyWriteRid, - ); - (async () => { try { - const res = await op_node_http_fetch_send(this._req.requestRid); + const parsedUrl = new URL(url); + let baseConnRid = + this.socket._handle[kStreamBaseField][internalRidSymbol]; + if (this._encrypted) { + [baseConnRid] = op_tls_start({ + rid: baseConnRid, + hostname: parsedUrl.hostname, + caCerts: [], + alpnProtocols: ["http/1.0", "http/1.1"], + }); + } + this._req = await op_node_http_request_with_conn( + this.method, + url, + headers, + this._bodyWriteRid, + baseConnRid, + this._encrypted, + ); + this._flushBuffer(); + const res = await op_node_http_await_response(this._req!.requestRid); if (this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -473,7 +482,6 @@ class ClientRequest extends OutgoingMessage { this._timeout.removeEventListener("abort", this._timeoutCb); webClearTimeout(this._timeout[timerId]); } - this._client.close(); const incoming = new IncomingMessageForClient(this.socket); incoming.req = this; this.res = incoming; @@ -512,12 +520,9 @@ class ClientRequest extends OutgoingMessage { if (this.method === "CONNECT") { throw new Error("not implemented CONNECT"); } - const upgradeRid = await op_node_http_fetch_response_upgrade( res.responseRid, ); - assert(typeof res.remoteAddrIp !== "undefined"); - assert(typeof res.remoteAddrIp !== "undefined"); const conn = new UpgradedConn( upgradeRid, { @@ -543,13 +548,11 @@ class ClientRequest extends OutgoingMessage { this._closed = true; this.emit("close"); } else { - { - incoming._bodyRid = res.responseRid; - } + incoming._bodyRid = res.responseRid; this.emit("response", incoming); } } catch (err) { - if (this._req.cancelHandleRid !== null) { + if (this._req && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -592,11 +595,54 @@ class ClientRequest extends OutgoingMessage { return undefined; } - // TODO(bartlomieju): handle error - onSocket(socket, _err) { + onSocket(socket, err) { nextTick(() => { - this.socket = socket; - this.emit("socket", socket); + // deno-lint-ignore no-this-alias + const req = this; + if (req.destroyed || err) { + req.destroyed = true; + + // deno-lint-ignore no-inner-declarations + function _destroy(req, err) { + if (!req.aborted && !err) { + err = new connResetException("socket hang up"); + } + if (err) { + emitErrorEvent(req, err); + } + req._closed = true; + req.emit("close"); + } + + if (socket) { + if (!err && req.agent && !socket.destroyed) { + socket.emit("free"); + } else { + finished(socket.destroy(err || req[kError]), (er) => { + if (er?.code === "ERR_STREAM_PREMATURE_CLOSE") { + er = null; + } + _destroy(req, er || err); + }); + return; + } + } + + _destroy(req, err || req[kError]); + } else { + // Note: this code is specific to deno to initiate a request. + const onConnect = () => { + // Flush the internal buffers once socket is ready. + this._flushHeaders(); + }; + this.socket = socket; + this.emit("socket", socket); + if (socket.readyState === "opening") { + socket.on("connect", onConnect); + } else { + onConnect(); + } + } }); } @@ -618,14 +664,20 @@ class ClientRequest extends OutgoingMessage { if (chunk) { this.write_(chunk, encoding, null, true); } else if (!this._headerSent) { - this._contentLength = 0; - this._implicitHeader(); - this._send("", "latin1"); + if ( + (this.socket && !this.socket.connecting) || // socket is not connecting, or + (!this.socket && this.outputData.length === 0) // no data to send + ) { + this._contentLength = 0; + this._implicitHeader(); + this._send("", "latin1"); + } } - (async () => { + const finish = async () => { try { + await this._bodyWriter.ready; await this._bodyWriter?.close(); - } catch (_) { + } catch { // The readable stream resource is dropped right after // read is complete closing the writable stream resource. // If we try to close the writer again, it will result in an @@ -633,10 +685,20 @@ class ClientRequest extends OutgoingMessage { } try { cb?.(); - } catch (_) { + } catch { // } - })(); + }; + + if (this.socket && this._bodyWriter) { + finish(); + } else { + this.on("drain", () => { + if (this.outputData.length === 0) { + finish(); + } + }); + } return this; } @@ -658,11 +720,6 @@ class ClientRequest extends OutgoingMessage { } this.destroyed = true; - const rid = this._client?.[internalRidSymbol]; - if (rid) { - core.tryClose(rid); - } - // Request might be closed before we actually made it if (this._req !== undefined && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); diff --git a/ext/node/polyfills/https.ts b/ext/node/polyfills/https.ts index f60c5e471a..fd700173eb 100644 --- a/ext/node/polyfills/https.ts +++ b/ext/node/polyfills/https.ts @@ -112,7 +112,7 @@ export const globalAgent = new Agent({ /** HttpsClientRequest class loosely follows http.ClientRequest class API. */ class HttpsClientRequest extends ClientRequest { - override _encrypted: true; + override _encrypted = true; override defaultProtocol = "https:"; override _getClient(): Deno.HttpClient | undefined { if (caCerts === null) { diff --git a/ext/node/polyfills/internal_binding/cares_wrap.ts b/ext/node/polyfills/internal_binding/cares_wrap.ts index 6feb7faf0d..cbd0bb8ef6 100644 --- a/ext/node/polyfills/internal_binding/cares_wrap.ts +++ b/ext/node/polyfills/internal_binding/cares_wrap.ts @@ -36,7 +36,6 @@ import { } from "ext:deno_node/internal_binding/async_wrap.ts"; import { ares_strerror } from "ext:deno_node/internal_binding/ares.ts"; import { notImplemented } from "ext:deno_node/_utils.ts"; -import { isWindows } from "ext:deno_node/_util/os.ts"; interface LookupAddress { address: string; @@ -68,7 +67,7 @@ export function getaddrinfo( _hints: number, verbatim: boolean, ): number { - let addresses: string[] = []; + const addresses: string[] = []; // TODO(cmorten): use hints // REF: https://nodejs.org/api/dns.html#dns_supported_getaddrinfo_flags @@ -107,13 +106,6 @@ export function getaddrinfo( }); } - // TODO(@bartlomieju): Forces IPv4 as a workaround for Deno not - // aligning with Node on implicit binding on Windows - // REF: https://github.com/denoland/deno/issues/10762 - if (isWindows && hostname === "localhost") { - addresses = addresses.filter((address) => isIPv4(address)); - } - req.oncomplete(error, addresses); })(); diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index 2b01125190..b2b0c9857c 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -986,16 +986,20 @@ function _lookupAndConnect( } else { self._unrefTimer(); - defaultTriggerAsyncIdScope( - self[asyncIdSymbol], - _internalConnect, - self, - ip, - port, - addressType, - localAddress, - localPort, - ); + defaultTriggerAsyncIdScope(self[asyncIdSymbol], nextTick, () => { + if (self.connecting) { + defaultTriggerAsyncIdScope( + self[asyncIdSymbol], + _internalConnect, + self, + ip, + port, + addressType, + localAddress, + localPort, + ); + } + }); } }, ); @@ -1197,6 +1201,9 @@ export class Socket extends Duplex { _host: string | null = null; // deno-lint-ignore no-explicit-any _parent: any = null; + // The flag for detecting if it's called in @npmcli/agent + // See discussions in https://github.com/denoland/deno/pull/25470 for more details. + _isNpmAgent = false; autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined; constructor(options: SocketOptions | number) { @@ -1217,6 +1224,19 @@ export class Socket extends Duplex { super(options); + // Note: If the socket is created from @npmcli/agent, the 'socket' event + // on ClientRequest object happens after 'connect' event on Socket object. + // That swaps the sequence of op_node_http_request_with_conn() call and + // initial socket read. That causes op_node_http_request_with_conn() not + // working. + // To avoid the above situation, we detect the socket created from + // @npmcli/agent and pause the socket (and also skips the startTls call + // if it's TLSSocket) + this._isNpmAgent = new Error().stack?.includes("@npmcli/agent") || false; + if (this._isNpmAgent) { + this.pause(); + } + if (options.handle) { this._handle = options.handle; this[asyncIdSymbol] = _getNewAsyncId(this._handle); diff --git a/runtime/errors.rs b/runtime/errors.rs index 22ba640bcf..3f8e900851 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -712,7 +712,6 @@ fn get_fetch_error(error: &FetchError) -> &'static str { FetchError::ClientSend(_) => "TypeError", FetchError::RequestBuilderHook(_) => "TypeError", FetchError::Io(e) => get_io_error_class(e), - FetchError::Hyper(e) => get_hyper_error_class(e), } } @@ -1083,6 +1082,7 @@ mod node { pub use deno_node::ops::crypto::SignEd25519Error; pub use deno_node::ops::crypto::VerifyEd25519Error; pub use deno_node::ops::fs::FsError; + pub use deno_node::ops::http::ConnError; pub use deno_node::ops::http2::Http2Error; pub use deno_node::ops::idna::IdnaError; pub use deno_node::ops::ipc::IpcError; @@ -1538,6 +1538,24 @@ mod node { pub fn get_verify_ed25519_error(_: &VerifyEd25519Error) -> &'static str { "TypeError" } + + pub fn get_conn_error(e: &ConnError) -> &'static str { + match e { + ConnError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + ConnError::Permission(e) => get_permission_check_error_class(e), + ConnError::InvalidUrl(_) => "TypeError", + ConnError::InvalidHeaderName(_) => "TypeError", + ConnError::InvalidHeaderValue(_) => "TypeError", + ConnError::Url(e) => get_url_parse_error_class(e), + ConnError::Method(_) => "TypeError", + ConnError::Io(e) => get_io_error_class(e), + ConnError::Hyper(e) => super::get_hyper_error_class(e), + ConnError::TlsStreamBusy => "Busy", + ConnError::TcpStreamBusy => "Busy", + ConnError::ReuniteTcp(_) => "Error", + ConnError::Canceled(_) => "Error", + } + } } fn get_os_error(error: &OsError) -> &'static str { @@ -1730,6 +1748,10 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { e.downcast_ref::() .map(node::get_verify_ed25519_error) }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_conn_error) + }) .or_else(|| e.downcast_ref::().map(get_napi_error_class)) .or_else(|| e.downcast_ref::().map(get_web_error_class)) .or_else(|| { diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 105341109c..cda2923789 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -565,9 +565,7 @@ "test-handle-wrap-close-abort.js", "test-http-abort-before-end.js", "test-http-addrequest-localaddress.js", - "test-http-agent-false.js", "test-http-agent-getname.js", - "test-http-agent-keepalive-delay.js", "test-http-agent-maxtotalsockets.js", "test-http-agent-no-protocol.js", "test-http-agent-null.js", @@ -590,7 +588,6 @@ "test-http-client-race.js", "test-http-client-read-in-error.js", "test-http-client-reject-unexpected-agent.js", - "test-http-client-timeout-connect-listener.js", "test-http-client-timeout-with-data.js", "test-http-client-unescaped-path.js", "test-http-client-upload-buf.js", @@ -604,7 +601,6 @@ "test-http-date-header.js", "test-http-decoded-auth.js", "test-http-default-encoding.js", - "test-http-dump-req-when-res-ends.js", "test-http-end-throw-socket-handling.js", "test-http-eof-on-connect.js", "test-http-extra-response.js", @@ -622,7 +618,6 @@ "test-http-hex-write.js", "test-http-highwatermark.js", "test-http-host-headers.js", - "test-http-hostname-typechecking.js", "test-http-incoming-message-destroy.js", "test-http-invalid-path-chars.js", "test-http-invalidheaderfield.js", @@ -1292,10 +1287,7 @@ "test-buffer-creation-regression.js", "test-child-process-exit.js", "test-http-server-keep-alive-timeout-slow-server.js", - "test-net-better-error-messages-port.js", - "test-net-connect-handle-econnrefused.js", "test-net-connect-local-error.js", - "test-net-reconnect-error.js", "test-net-response-size.js", "test-net-server-bind.js", "test-tls-lookup.js", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index 09d68aded7..8ad00c9bfd 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -1,7 +1,7 @@ # Remaining Node Tests -1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining). +1155 tests out of 3681 have been ported from Node 20.11.1 (31.38% ported, 69.14% remaining). NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead. @@ -792,6 +792,8 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-agent-destroyed-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-destroyed-socket.js) - [parallel/test-http-agent-domain-reused-gc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-domain-reused-gc.js) - [parallel/test-http-agent-error-on-idle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-error-on-idle.js) +- [parallel/test-http-agent-false.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-false.js) +- [parallel/test-http-agent-keepalive-delay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive-delay.js) - [parallel/test-http-agent-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive.js) - [parallel/test-http-agent-maxsockets-respected.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets-respected.js) - [parallel/test-http-agent-maxsockets.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets.js) @@ -848,6 +850,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-client-set-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-set-timeout.js) - [parallel/test-http-client-spurious-aborted.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-spurious-aborted.js) - [parallel/test-http-client-timeout-agent.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-agent.js) +- [parallel/test-http-client-timeout-connect-listener.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-connect-listener.js) - [parallel/test-http-client-timeout-event.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-event.js) - [parallel/test-http-client-timeout-on-connect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-on-connect.js) - [parallel/test-http-client-timeout-option-listeners.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-option-listeners.js) @@ -865,6 +868,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-destroyed-socket-write2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-destroyed-socket-write2.js) - [parallel/test-http-dns-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dns-error.js) - [parallel/test-http-double-content-length.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-double-content-length.js) +- [parallel/test-http-dump-req-when-res-ends.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dump-req-when-res-ends.js) - [parallel/test-http-early-hints-invalid-argument.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints-invalid-argument.js) - [parallel/test-http-early-hints.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints.js) - [parallel/test-http-exceptions.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-exceptions.js) @@ -876,6 +880,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-header-badrequest.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-badrequest.js) - [parallel/test-http-header-overflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-overflow.js) - [parallel/test-http-host-header-ipv6-fail.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-host-header-ipv6-fail.js) +- [parallel/test-http-hostname-typechecking.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-hostname-typechecking.js) - [parallel/test-http-incoming-matchKnownFields.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-matchKnownFields.js) - [parallel/test-http-incoming-message-connection-setter.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-connection-setter.js) - [parallel/test-http-incoming-message-options.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-options.js) @@ -2508,9 +2513,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [sequential/test-inspector-port-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-inspector-port-cluster.js) - [sequential/test-module-loading.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-module-loading.js) - [sequential/test-net-GH-5504.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-GH-5504.js) +- [sequential/test-net-better-error-messages-port.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-better-error-messages-port.js) - [sequential/test-net-connect-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-econnrefused.js) +- [sequential/test-net-connect-handle-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-handle-econnrefused.js) - [sequential/test-net-listen-shared-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-listen-shared-ports.js) - [sequential/test-net-localport.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-localport.js) +- [sequential/test-net-reconnect-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-reconnect-error.js) - [sequential/test-net-server-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-server-address.js) - [sequential/test-next-tick-error-spin.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-next-tick-error-spin.js) - [sequential/test-perf-hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-perf-hooks.js) diff --git a/tests/node_compat/test/parallel/test-http-agent-false.js b/tests/node_compat/test/parallel/test-http-agent-false.js deleted file mode 100644 index 60dc16d9b0..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-false.js +++ /dev/null @@ -1,53 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const http = require('http'); - -// Sending `agent: false` when `port: null` is also passed in (i.e. the result -// of a `url.parse()` call with the default port used, 80 or 443), should not -// result in an assertion error... -const opts = { - host: '127.0.0.1', - port: null, - path: '/', - method: 'GET', - agent: false -}; - -// We just want an "error" (no local HTTP server on port 80) or "response" -// to happen (user happens ot have HTTP server running on port 80). -// As long as the process doesn't crash from a C++ assertion then we're good. -const req = http.request(opts); - -// Will be called by either the response event or error event, not both -const oneResponse = common.mustCall(); -req.on('response', oneResponse); -req.on('error', oneResponse); -req.end(); diff --git a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js b/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js deleted file mode 100644 index 7cc6120d73..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js +++ /dev/null @@ -1,43 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); -const { Agent } = require('_http_agent'); - -const agent = new Agent({ - keepAlive: true, - keepAliveMsecs: 1000, -}); - -const server = http.createServer(common.mustCall((req, res) => { - res.end('ok'); -})); - -server.listen(0, common.mustCall(() => { - const createConnection = agent.createConnection; - agent.createConnection = (options, ...args) => { - assert.strictEqual(options.keepAlive, true); - assert.strictEqual(options.keepAliveInitialDelay, agent.keepAliveMsecs); - return createConnection.call(agent, options, ...args); - }; - http.get({ - host: 'localhost', - port: server.address().port, - agent: agent, - path: '/' - }, common.mustCall((res) => { - // for emit end event - res.on('data', () => {}); - res.on('end', () => { - server.close(); - }); - })); -})); diff --git a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js b/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js deleted file mode 100644 index c151d16556..0000000000 --- a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); - -// This test ensures that `ClientRequest.prototype.setTimeout()` does -// not add a listener for the `'connect'` event to the socket if the -// socket is already connected. - -const assert = require('assert'); -const http = require('http'); - -// Maximum allowed value for timeouts. -const timeout = 2 ** 31 - 1; - -const server = http.createServer((req, res) => { - res.end(); -}); - -server.listen(0, common.mustCall(() => { - const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }); - const options = { port: server.address().port, agent: agent }; - - doRequest(options, common.mustCall(() => { - const req = doRequest(options, common.mustCall(() => { - agent.destroy(); - server.close(); - })); - - req.on('socket', common.mustCall((socket) => { - assert.strictEqual(socket.listenerCount('connect'), 0); - })); - })); -})); - -function doRequest(options, callback) { - const req = http.get(options, (res) => { - res.on('end', callback); - res.resume(); - }); - - req.setTimeout(timeout); - return req; -} diff --git a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js b/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js deleted file mode 100644 index 3b94250f5a..0000000000 --- a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js +++ /dev/null @@ -1,73 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const { mustCall } = require('../common'); - -const fs = require('fs'); -const http = require('http'); -const { strictEqual } = require('assert'); - -const server = http.createServer(mustCall(function(req, res) { - strictEqual(req.socket.listenerCount('data'), 1); - req.socket.once('data', mustCall(function() { - // Ensure that a chunk of data is received before calling `res.end()`. - res.end('hello world'); - })); - // This checks if the request gets dumped - // resume will be triggered by res.end(). - req.on('resume', mustCall(function() { - // There is no 'data' event handler anymore - // it gets automatically removed when dumping the request. - strictEqual(req.listenerCount('data'), 0); - req.on('data', mustCall()); - })); - - // We explicitly pause the stream - // so that the following on('data') does not cause - // a resume. - req.pause(); - req.on('data', function() {}); - - // Start sending the response. - res.flushHeaders(); -})); - -server.listen(0, mustCall(function() { - const req = http.request({ - method: 'POST', - port: server.address().port - }); - - // Send the http request without waiting - // for the body. - req.flushHeaders(); - - req.on('response', mustCall(function(res) { - // Pipe the body as soon as we get the headers of the - // response back. - fs.createReadStream(__filename).pipe(req); - - res.resume(); - - // On some platforms the `'end'` event might not be emitted because the - // socket could be destroyed by the other peer while data is still being - // sent. In this case the 'aborted'` event is emitted instead of `'end'`. - // `'close'` is used here because it is always emitted and does not - // invalidate the test. - res.on('close', function() { - server.close(); - }); - })); - - req.on('error', function() { - // An error can happen if there is some data still - // being sent, as the other side is calling .destroy() - // this is safe to ignore. - }); -})); diff --git a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js b/tests/node_compat/test/parallel/test-http-hostname-typechecking.js deleted file mode 100644 index e42384504b..0000000000 --- a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); - -// All of these values should cause http.request() to throw synchronously -// when passed as the value of either options.hostname or options.host -const vals = [{}, [], NaN, Infinity, -Infinity, true, false, 1, 0, new Date()]; - -vals.forEach((v) => { - const received = common.invalidArgTypeHelper(v); - assert.throws( - () => http.request({ hostname: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.hostname" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); - - assert.throws( - () => http.request({ host: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.host" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); -}); - -// These values are OK and should not throw synchronously. -// Only testing for 'hostname' validation so ignore connection errors. -const dontCare = () => {}; -['', undefined, null].forEach((v) => { - http.request({ hostname: v }).on('error', dontCare).end(); - http.request({ host: v }).on('error', dontCare).end(); -}); diff --git a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js b/tests/node_compat/test/sequential/test-net-better-error-messages-port.js deleted file mode 100644 index f718ca3f84..0000000000 --- a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js +++ /dev/null @@ -1,24 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall()); - -c.on('error', common.mustCall(function(error) { - // Family autoselection might be skipped if only a single address is returned by DNS. - const failedAttempt = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(failedAttempt.code, 'ECONNREFUSED'); - assert.strictEqual(failedAttempt.port, common.PORT); - assert.match(failedAttempt.address, /^(127\.0\.0\.1|::1)$/); -})); diff --git a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js b/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js deleted file mode 100644 index 629705564b..0000000000 --- a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js +++ /dev/null @@ -1,39 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); -c.on('connect', common.mustNotCall()); -c.on('error', common.mustCall((e) => { - assert.strictEqual(c.connecting, false); - assert.strictEqual(e.code, 'ECONNREFUSED'); -})); diff --git a/tests/node_compat/test/sequential/test-net-reconnect-error.js b/tests/node_compat/test/sequential/test-net-reconnect-error.js deleted file mode 100644 index 450a0798bf..0000000000 --- a/tests/node_compat/test/sequential/test-net-reconnect-error.js +++ /dev/null @@ -1,50 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); -const N = 20; -let disconnectCount = 0; - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall('client should not have connected')); - -c.on('error', common.mustCall((error) => { - // Family autoselection might be skipped if only a single address is returned by DNS. - const actualError = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(actualError.code, 'ECONNREFUSED'); -}, N + 1)); - -c.on('close', common.mustCall(() => { - if (disconnectCount++ < N) - c.connect(common.PORT); // reconnect -}, N + 1)); diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index 048ddf30f5..e6c36eea19 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -499,7 +499,6 @@ Deno.test("[node/http] send request with non-chunked body", async () => { assert(socket.writable); assert(socket.readable); socket.setKeepAlive(); - socket.destroy(); socket.setTimeout(100); }); req.write("hello "); @@ -512,6 +511,11 @@ Deno.test("[node/http] send request with non-chunked body", async () => { // in order to not cause a flaky test sanitizer failure await new Promise((resolve) => setTimeout(resolve, 100)), ]); + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body", async () => { @@ -559,6 +563,11 @@ Deno.test("[node/http] send request with chunked body", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body as default", async () => { @@ -604,6 +613,11 @@ Deno.test("[node/http] send request with chunked body as default", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] ServerResponse _implicitHeader", async () => { @@ -689,7 +703,7 @@ Deno.test("[node/http] ClientRequest handle non-string headers", async () => { assertEquals(headers!["1"], "2"); }); -Deno.test("[node/http] ClientRequest uses HTTP/1.1", async () => { +Deno.test("[node/https] ClientRequest uses HTTP/1.1", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = https.request("https://localhost:5545/http_version", { @@ -800,8 +814,9 @@ Deno.test("[node/http] ClientRequest search params", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = http.request({ - host: "localhost:4545", - path: "search_params?foo=bar", + host: "localhost", + port: 4545, + path: "/search_params?foo=bar", }, (resp) => { resp.on("data", (chunk) => { body += chunk; @@ -1011,28 +1026,50 @@ Deno.test( Deno.test( "[node/http] client destroy before sending request should not error", - () => { + async () => { + const { resolve, promise } = Promise.withResolvers(); const request = http.request("http://localhost:5929/"); // Calling this would throw request.destroy(); + request.on("error", (e) => { + assertEquals(e.message, "socket hang up"); + }); + request.on("close", () => resolve()); + await promise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }, ); +const isWindows = Deno.build.os === "windows"; + Deno.test( "[node/http] destroyed requests should not be sent", + { sanitizeResources: !isWindows, sanitizeOps: !isWindows }, async () => { let receivedRequest = false; - const server = Deno.serve(() => { + const requestClosed = Promise.withResolvers(); + const ac = new AbortController(); + const server = Deno.serve({ port: 0, signal: ac.signal }, () => { receivedRequest = true; return new Response(null); }); const request = http.request(`http://localhost:${server.addr.port}/`); request.destroy(); request.end("hello"); - - await new Promise((r) => setTimeout(r, 500)); + request.on("error", (err) => { + assert(err.message.includes("socket hang up")); + ac.abort(); + }); + request.on("close", () => { + requestClosed.resolve(); + }); + await requestClosed.promise; assertEquals(receivedRequest, false); - await server.shutdown(); + await server.finished; }, ); @@ -1060,22 +1097,33 @@ Deno.test("[node/https] node:https exports globalAgent", async () => { ); }); -Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", () => { +Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", async () => { { - const req = http.request("http://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + await promise; } { - const req = https.request("https://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + + await promise; } }); From e46f42d052f05eb6dc9eabff3ec11483519ff1c4 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 12 Dec 2024 20:12:13 -0500 Subject: [PATCH 7/8] ci: try fix cache (#27348) --- .github/workflows/ci.generate.ts | 29 +++++++++++++---------------- .github/workflows/ci.yml | 4 ++-- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index d43db76414..5c2de96006 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -59,6 +59,15 @@ const Runners = { const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; +const prCacheKey = `${prCacheKeyPrefix}\${{ github.sha }}`; +const prCachePath = [ + // this must match for save and restore (https://github.com/actions/cache/issues/1444) + "./target", + "!./target/*/gn_out", + "!./target/*/gn_root", + "!./target/*/*.zip", + "!./target/*/*.tar.gz", +].join("\n"); // Note that you may need to add more version to the `apt-get remove` line below if you change this const llvmVersion = 19; @@ -612,7 +621,7 @@ const ci = { `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ hashFiles('Cargo.lock') }}`, // We will try to restore from the closest cargo-home we can find "restore-keys": - `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}`, + `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-`, }, }, { @@ -622,13 +631,7 @@ const ci = { if: "github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/gn_root", - "!./target/*/*.zip", - "!./target/*/*.tar.gz", - ].join("\n"), + path: prCachePath, key: "never_saved", "restore-keys": prCacheKeyPrefix, }, @@ -1080,14 +1083,8 @@ const ci = { if: "(matrix.job == 'test' || matrix.job == 'lint') && github.ref == 'refs/heads/main'", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/*.zip", - "!./target/*/*.sha256sum", - "!./target/*/*.tar.gz", - ].join("\n"), - key: prCacheKeyPrefix + "${{ github.sha }}", + path: prCachePath, + key: prCacheKey, }, }, ]), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29dd694c6c..44eb15cb95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -362,7 +362,7 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -682,8 +682,8 @@ jobs: path: |- ./target !./target/*/gn_out + !./target/*/gn_root !./target/*/*.zip - !./target/*/*.sha256sum !./target/*/*.tar.gz key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: From 3ddbea62c25891f832354134a2b24e738c45fe10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 13 Dec 2024 02:31:07 +0000 Subject: [PATCH 8/8] ci: use self-hosted mac arm runner only on main branch (#27347) We can have 2 concurrent runs now, so it's worth trying it out on `main`. --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 5c2de96006..1f9dc45340 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -42,7 +42,7 @@ const Runners = { os: "macos", arch: "aarch64", runner: - `\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, + `\${{ github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, windowsX86: { os: "windows", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44eb15cb95..91ba16c6ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,12 +68,12 @@ jobs: skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' job: test profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && github.ref == ''refs/heads/main'' && ''self-hosted'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}'