From 8590aa9cee9c63088ef8eac017d6325318aa9df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 18 Dec 2024 13:44:53 +0000 Subject: [PATCH 01/20] fix(ext/node): sort list of built-in modules alphabetically (#27410) --- Cargo.lock | 2 +- Cargo.toml | 2 +- ext/node/Cargo.toml | 2 +- ext/node/polyfill.rs | 9 ++++++++- tests/integration/lsp_tests.rs | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cfb3779e4..7203286773 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1925,7 +1925,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.120.0" +version = "0.121.0" dependencies = [ "aead-gcm-stream", "aes", diff --git a/Cargo.toml b/Cargo.toml index 0f52c28de6..b404114054 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ deno_io = { version = "0.93.0", path = "./ext/io" } deno_kv = { version = "0.91.0", path = "./ext/kv" } deno_napi = { version = "0.114.0", path = "./ext/napi" } deno_net = { version = "0.175.0", path = "./ext/net" } -deno_node = { version = "0.120.0", path = "./ext/node" } +deno_node = { version = "0.121.0", path = "./ext/node" } deno_telemetry = { version = "0.5.0", path = "./ext/telemetry" } deno_tls = { version = "0.170.0", path = "./ext/tls" } deno_url = { version = "0.183.0", path = "./ext/url" } diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 8ba1141ec1..127633a09b 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.120.0" +version = "0.121.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/polyfill.rs b/ext/node/polyfill.rs index 762e32f7ba..42cc794955 100644 --- a/ext/node/polyfill.rs +++ b/ext/node/polyfill.rs @@ -67,9 +67,9 @@ generate_builtin_node_module_lists! { "process", "punycode", "querystring", - "repl", "readline", "readline/promises", + "repl", "stream", "stream/consumers", "stream/promises", @@ -90,3 +90,10 @@ generate_builtin_node_module_lists! { "worker_threads", "zlib", } + +#[test] +fn test_builtins_are_sorted() { + let mut builtins_list = SUPPORTED_BUILTIN_NODE_MODULES.to_vec(); + builtins_list.sort(); + assert_eq!(SUPPORTED_BUILTIN_NODE_MODULES, builtins_list); +} diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 568cad44b0..56c060d958 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -8609,9 +8609,9 @@ fn lsp_completions_node_specifier() { "node:process", "node:punycode", "node:querystring", - "node:repl", "node:readline", "node:readline/promises", + "node:repl", "node:stream", "node:stream/consumers", "node:stream/promises", From ae744074128de129e6ceb3229cc69060968f0ebc Mon Sep 17 00:00:00 2001 From: snek Date: Wed, 18 Dec 2024 14:47:21 +0100 Subject: [PATCH 02/20] chore: upgrade libc (#27414) need to do this for quic and they deprecated this method in libc without actually providing an alternative so :/ --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- ext/node/ops/os/cpus.rs | 9 +++++++-- runtime/ops/os/mod.rs | 5 ++++- runtime/sys_info.rs | 6 +++++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7203286773..a74724e92d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4552,9 +4552,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libffi" diff --git a/Cargo.toml b/Cargo.toml index b404114054..0d045c9059 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,7 +149,7 @@ indexmap = { version = "2", features = ["serde"] } ipnet = "2.3" jsonc-parser = { version = "=0.26.2", features = ["serde"] } lazy-regex = "3" -libc = "0.2.126" +libc = "0.2.168" libz-sys = { version = "1.1.20", default-features = false } log = { version = "0.4.20", features = ["kv"] } lsp-types = "=0.97.0" # used by tower-lsp and "proposed" feature is unstable in patch releases diff --git a/ext/node/ops/os/cpus.rs b/ext/node/ops/os/cpus.rs index 3f5f430f65..2b931884c3 100644 --- a/ext/node/ops/os/cpus.rs +++ b/ext/node/ops/os/cpus.rs @@ -73,12 +73,17 @@ pub fn cpu_info() -> Option> { cpu_speed = 2_400_000_000; } + extern "C" { + fn mach_host_self() -> std::ffi::c_uint; + static mut mach_task_self_: std::ffi::c_uint; + } + let mut num_cpus: libc::natural_t = 0; let mut info: *mut libc::processor_cpu_load_info_data_t = std::ptr::null_mut(); let mut msg_type: libc::mach_msg_type_number_t = 0; if libc::host_processor_info( - libc::mach_host_self(), + mach_host_self(), libc::PROCESSOR_CPU_LOAD_INFO, &mut num_cpus, &mut info as *mut _ as *mut libc::processor_info_array_t, @@ -111,7 +116,7 @@ pub fn cpu_info() -> Option> { } libc::vm_deallocate( - libc::mach_task_self(), + mach_task_self_, info.as_ptr() as libc::vm_address_t, msg_type as _, ); diff --git a/runtime/ops/os/mod.rs b/runtime/ops/os/mod.rs index b8ebc88bed..71169217a7 100644 --- a/runtime/ops/os/mod.rs +++ b/runtime/ops/os/mod.rs @@ -424,8 +424,11 @@ fn rss() -> usize { let mut count = libc::MACH_TASK_BASIC_INFO_COUNT; // SAFETY: libc calls let r = unsafe { + extern "C" { + static mut mach_task_self_: std::ffi::c_uint; + } libc::task_info( - libc::mach_task_self(), + mach_task_self_, libc::MACH_TASK_BASIC_INFO, task_info.as_mut_ptr() as libc::task_info_t, &mut count as *mut libc::mach_msg_type_number_t, diff --git a/runtime/sys_info.rs b/runtime/sys_info.rs index cffc90e9da..f99cfc99f9 100644 --- a/runtime/sys_info.rs +++ b/runtime/sys_info.rs @@ -278,11 +278,15 @@ pub fn mem_info() -> Option { mem_info.swap_total = xs.xsu_total; mem_info.swap_free = xs.xsu_avail; + extern "C" { + fn mach_host_self() -> std::ffi::c_uint; + } + let mut count: u32 = libc::HOST_VM_INFO64_COUNT as _; let mut stat = std::mem::zeroed::(); if libc::host_statistics64( // TODO(@littledivy): Put this in a once_cell. - libc::mach_host_self(), + mach_host_self(), libc::HOST_VM_INFO64, &mut stat as *mut libc::vm_statistics64 as *mut _, &mut count, From b1c685f4b773abd1af8f543e86c73909cc41efd7 Mon Sep 17 00:00:00 2001 From: Sean McArthur Date: Wed, 18 Dec 2024 17:04:29 -0500 Subject: [PATCH 03/20] fix(ext/fetch): retry some http/2 errors (#27417) This brings some of the HTTP/2 retry behavior from reqwest to `ext/fetch`. It will retry very specific HTTP/2 errors once, if the body is able to be used again. Closes #27332 --- Cargo.lock | 28 +++- Cargo.toml | 2 +- cli/http_util.rs | 12 +- cli/tools/registry/mod.rs | 5 +- ext/fetch/Cargo.toml | 1 + ext/fetch/lib.rs | 190 +++++++++++++++++++++++++-- ext/fetch/tests.rs | 6 +- ext/kv/remote.rs | 4 +- runtime/ops/web_worker/sync_fetch.rs | 6 +- 9 files changed, 209 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a74724e92d..bb0617ceee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,7 +380,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -1651,6 +1651,7 @@ dependencies = [ "dyn-clone", "error_reporter", "fast-socks5", + "h2 0.4.4", "hickory-resolver", "http 1.1.0", "http-body-util", @@ -1667,7 +1668,7 @@ dependencies = [ "tokio-rustls", "tokio-socks", "tokio-util", - "tower", + "tower 0.5.2", "tower-http", "tower-service", ] @@ -2322,7 +2323,7 @@ dependencies = [ "serde_json", "tokio", "tokio-util", - "tower", + "tower 0.4.13", "tracing", ] @@ -7976,7 +7977,7 @@ dependencies = [ "socket2", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -8002,6 +8003,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.6.1" @@ -8030,9 +8046,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" diff --git a/Cargo.toml b/Cargo.toml index 0d045c9059..55a94b75a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,7 @@ tokio-metrics = { version = "0.3.0", features = ["rt"] } tokio-rustls = { version = "0.26.0", default-features = false, features = ["ring", "tls12"] } tokio-socks = "0.5.1" tokio-util = "0.7.4" -tower = { version = "0.4.13", default-features = false, features = ["util"] } +tower = { version = "0.5.2", default-features = false, features = ["retry", "util"] } tower-http = { version = "0.6.1", features = ["decompression-br", "decompression-gzip"] } tower-lsp = { package = "deno_tower_lsp", version = "0.1.0", features = ["proposed"] } tower-service = "0.3.2" diff --git a/cli/http_util.rs b/cli/http_util.rs index ce05d66b78..b24dd7bc0c 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -145,9 +145,7 @@ impl HttpClient { } pub fn get(&self, url: Url) -> Result { - let body = http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed(); + let body = deno_fetch::ReqBody::empty(); let mut req = http::Request::new(body); *req.uri_mut() = url.as_str().parse()?; Ok(RequestBuilder { @@ -179,9 +177,7 @@ impl HttpClient { S: serde::Serialize, { let json = deno_core::serde_json::to_vec(ser)?; - let body = http_body_util::Full::new(json.into()) - .map_err(|never| match never {}) - .boxed(); + let body = deno_fetch::ReqBody::full(json.into()); let builder = self.post(url, body)?; Ok(builder.header( http::header::CONTENT_TYPE, @@ -194,9 +190,7 @@ impl HttpClient { url: &Url, headers: HeaderMap, ) -> Result, SendError> { - let body = http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed(); + let body = deno_fetch::ReqBody::empty(); let mut request = http::Request::new(body); *request.uri_mut() = http::Uri::try_from(url.as_str())?; *request.headers_mut() = headers; diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index 001e401459..45a040d236 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -26,6 +26,7 @@ use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::url::Url; +use deno_runtime::deno_fetch; use deno_terminal::colors; use http_body_util::BodyExt; use serde::Deserialize; @@ -911,9 +912,7 @@ async fn publish_package( package.config ); - let body = http_body_util::Full::new(package.tarball.bytes.clone()) - .map_err(|never| match never {}) - .boxed(); + let body = deno_fetch::ReqBody::full(package.tarball.bytes.clone()); let response = http_client .post(url.parse()?, body)? .header( diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 98d2fdf5da..fee21808e7 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -23,6 +23,7 @@ deno_permissions.workspace = true deno_tls.workspace = true dyn-clone = "1" error_reporter = "1" +h2.workspace = true hickory-resolver.workspace = true http.workspace = true http-body-util.workspace = true diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 919c6d3044..103698b3bf 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; use std::cell::RefCell; use std::cmp::min; use std::convert::From; +use std::future; use std::path::Path; use std::path::PathBuf; use std::pin::Pin; @@ -66,6 +67,7 @@ use http::header::USER_AGENT; use http::Extensions; use http::Method; use http::Uri; +use http_body_util::combinators::BoxBody; use http_body_util::BodyExt; use hyper::body::Frame; use hyper_util::client::legacy::connect::HttpConnector; @@ -75,6 +77,7 @@ use hyper_util::rt::TokioExecutor; use hyper_util::rt::TokioTimer; use serde::Deserialize; use serde::Serialize; +use tower::retry; use tower::ServiceExt; use tower_http::decompression::Decompression; @@ -476,9 +479,7 @@ where // If a body is passed, we use it, and don't return a body for streaming. con_len = Some(data.len() as u64); - http_body_util::Full::new(data.to_vec().into()) - .map_err(|never| match never {}) - .boxed() + ReqBody::full(data.to_vec().into()) } (_, Some(resource)) => { let resource = state @@ -491,7 +492,7 @@ where } _ => {} } - ReqBody::new(ResourceToBodyAdapter::new(resource)) + ReqBody::streaming(ResourceToBodyAdapter::new(resource)) } (None, None) => unreachable!(), } @@ -501,9 +502,7 @@ where if matches!(method, Method::POST | Method::PUT) { con_len = Some(0); } - http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed() + ReqBody::empty() }; let mut request = http::Request::new(body); @@ -1066,7 +1065,8 @@ pub fn create_http_client( } let pooled_client = builder.build(connector); - let decompress = Decompression::new(pooled_client).gzip(true).br(true); + let retry_client = retry::Retry::new(FetchRetry, pooled_client); + let decompress = Decompression::new(retry_client).gzip(true).br(true); Ok(Client { inner: decompress, @@ -1083,7 +1083,12 @@ pub fn op_utf8_to_byte_string(#[string] input: String) -> ByteString { #[derive(Clone, Debug)] pub struct Client { - inner: Decompression>, + inner: Decompression< + retry::Retry< + FetchRetry, + hyper_util::client::legacy::Client, + >, + >, // Used to check whether to include a proxy-authorization header proxies: Arc, user_agent: HeaderValue, @@ -1174,10 +1179,70 @@ impl Client { } } -pub type ReqBody = - http_body_util::combinators::BoxBody; -pub type ResBody = - http_body_util::combinators::BoxBody; +// This is a custom enum to allow the retry policy to clone the variants that could be retried. +pub enum ReqBody { + Full(http_body_util::Full), + Empty(http_body_util::Empty), + Streaming(BoxBody), +} + +pub type ResBody = BoxBody; + +impl ReqBody { + pub fn full(bytes: Bytes) -> Self { + ReqBody::Full(http_body_util::Full::new(bytes)) + } + + pub fn empty() -> Self { + ReqBody::Empty(http_body_util::Empty::new()) + } + + pub fn streaming(body: B) -> Self + where + B: hyper::body::Body + + Send + + Sync + + 'static, + { + ReqBody::Streaming(BoxBody::new(body)) + } +} + +impl hyper::body::Body for ReqBody { + type Data = Bytes; + type Error = deno_core::error::AnyError; + + fn poll_frame( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + match &mut *self { + ReqBody::Full(ref mut b) => { + Pin::new(b).poll_frame(cx).map_err(|never| match never {}) + } + ReqBody::Empty(ref mut b) => { + Pin::new(b).poll_frame(cx).map_err(|never| match never {}) + } + ReqBody::Streaming(ref mut b) => Pin::new(b).poll_frame(cx), + } + } + + fn is_end_stream(&self) -> bool { + match self { + ReqBody::Full(ref b) => b.is_end_stream(), + ReqBody::Empty(ref b) => b.is_end_stream(), + ReqBody::Streaming(ref b) => b.is_end_stream(), + } + } + + fn size_hint(&self) -> hyper::body::SizeHint { + match self { + ReqBody::Full(ref b) => b.size_hint(), + ReqBody::Empty(ref b) => b.size_hint(), + ReqBody::Streaming(ref b) => b.size_hint(), + } + } +} /// Copied from https://github.com/seanmonstar/reqwest/blob/b9d62a0323d96f11672a61a17bf8849baec00275/src/async_impl/request.rs#L572 /// Check the request URL for a "username:password" type authority, and if @@ -1214,3 +1279,102 @@ pub fn extract_authority(url: &mut Url) -> Option<(String, Option)> { fn op_fetch_promise_is_settled(promise: v8::Local) -> bool { promise.state() != v8::PromiseState::Pending } + +/// Deno.fetch's retry policy. +#[derive(Clone, Debug)] +struct FetchRetry; + +/// Marker extension that a request has been retried once. +#[derive(Clone, Debug)] +struct Retried; + +impl + retry::Policy, http::Response, E> + for FetchRetry +where + E: std::error::Error + 'static, +{ + /// Don't delay retries. + type Future = future::Ready<()>; + + fn retry( + &mut self, + req: &mut http::Request, + result: &mut Result, E>, + ) -> Option { + if req.extensions().get::().is_some() { + // only retry once + return None; + } + + match result { + Ok(..) => { + // never retry a Response + None + } + Err(err) => { + if is_error_retryable(&*err) { + req.extensions_mut().insert(Retried); + Some(future::ready(())) + } else { + None + } + } + } + } + + fn clone_request( + &mut self, + req: &http::Request, + ) -> Option> { + let body = match req.body() { + ReqBody::Full(b) => ReqBody::Full(b.clone()), + ReqBody::Empty(b) => ReqBody::Empty(*b), + ReqBody::Streaming(..) => return None, + }; + + let mut clone = http::Request::new(body); + *clone.method_mut() = req.method().clone(); + *clone.uri_mut() = req.uri().clone(); + *clone.headers_mut() = req.headers().clone(); + *clone.extensions_mut() = req.extensions().clone(); + Some(clone) + } +} + +fn is_error_retryable(err: &(dyn std::error::Error + 'static)) -> bool { + // Note: hyper doesn't promise it will always be this h2 version. Keep up to date. + if let Some(err) = find_source::(err) { + // They sent us a graceful shutdown, try with a new connection! + if err.is_go_away() + && err.is_remote() + && err.reason() == Some(h2::Reason::NO_ERROR) + { + return true; + } + + // REFUSED_STREAM was sent from the server, which is safe to retry. + // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.7-3.2 + if err.is_reset() + && err.is_remote() + && err.reason() == Some(h2::Reason::REFUSED_STREAM) + { + return true; + } + } + + false +} + +fn find_source<'a, E: std::error::Error + 'static>( + err: &'a (dyn std::error::Error + 'static), +) -> Option<&'a E> { + let mut err = Some(err); + while let Some(src) = err { + if let Some(found) = src.downcast_ref::() { + return Some(found); + } + err = src.source(); + } + None +} diff --git a/ext/fetch/tests.rs b/ext/fetch/tests.rs index 3da29f8aa7..243b80bd90 100644 --- a/ext/fetch/tests.rs +++ b/ext/fetch/tests.rs @@ -133,11 +133,7 @@ async fn rust_test_client_with_resolver( let req = http::Request::builder() .uri(format!("https://{}/foo", src_addr)) - .body( - http_body_util::Empty::new() - .map_err(|err| match err {}) - .boxed(), - ) + .body(crate::ReqBody::empty()) .unwrap(); let resp = client.send(req).await.unwrap(); assert_eq!(resp.status(), http::StatusCode::OK); diff --git a/ext/kv/remote.rs b/ext/kv/remote.rs index 1830aa67ee..891786e319 100644 --- a/ext/kv/remote.rs +++ b/ext/kv/remote.rs @@ -122,9 +122,7 @@ impl RemoteTransport for FetchClient { headers: http::HeaderMap, body: Bytes, ) -> Result<(Url, http::StatusCode, Self::Response), anyhow::Error> { - let body = http_body_util::Full::new(body) - .map_err(|never| match never {}) - .boxed(); + let body = deno_fetch::ReqBody::full(body); let mut req = http::Request::new(body); *req.method_mut() = http::Method::POST; *req.uri_mut() = url.as_str().parse()?; diff --git a/runtime/ops/web_worker/sync_fetch.rs b/runtime/ops/web_worker/sync_fetch.rs index d1f133d3d2..508bcb7bb0 100644 --- a/runtime/ops/web_worker/sync_fetch.rs +++ b/runtime/ops/web_worker/sync_fetch.rs @@ -104,11 +104,7 @@ pub fn op_worker_sync_fetch( let (body, mime_type, res_url) = match script_url.scheme() { "http" | "https" => { - let mut req = http::Request::new( - http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed(), - ); + let mut req = http::Request::new(deno_fetch::ReqBody::empty()); *req.uri_mut() = script_url.as_str().parse()?; let resp = From 8fc4796ed5f765a8b55cf08b1802af6774f2d1a1 Mon Sep 17 00:00:00 2001 From: Filip Stevanovic <62512535+filipstev@users.noreply.github.com> Date: Wed, 18 Dec 2024 23:52:37 +0100 Subject: [PATCH 04/20] fix(ext/node): Fix `fs.access`/`fs.promises.access` with `X_OK` mode parameter on Windows (#27407) - Fixes an issue on Windows where the `fs.constants.X_OK` flag caused `fs.access` and `fs.promises.access` to incorrectly throw a "permission denied" error - Introduced formatting changes due to the formatting tool - Fixed the issue by always removing the `X_OK` bit from the mode variable m (not sure if it's necessary to check for the presence of it in the `mode` param first?) - Updated unit tests to handle the mentioned constant - `X_OK` bit is ignored in the Node implementation and should behave like `F_OK` fs constants Node documentation: https://nodejs.org/api/fs.html#fsconstants fixes https://github.com/denoland/deno/issues/27405 --- ext/node/polyfills/_fs/_fs_access.ts | 102 ++++++++++++++----------- tests/unit_node/_fs/_fs_access_test.ts | 4 + 2 files changed, 60 insertions(+), 46 deletions(-) diff --git a/ext/node/polyfills/_fs/_fs_access.ts b/ext/node/polyfills/_fs/_fs_access.ts index b501bcbcae..824386e64b 100644 --- a/ext/node/polyfills/_fs/_fs_access.ts +++ b/ext/node/polyfills/_fs/_fs_access.ts @@ -30,50 +30,58 @@ export function access( mode = getValidMode(mode, "access"); const cb = makeCallback(callback); - Deno.lstat(path).then((info) => { - if (info.mode === null) { - // If the file mode is unavailable, we pretend it has - // the permission - cb(null); - return; - } - const m = +mode || 0; - let fileMode = +info.mode || 0; - if (Deno.build.os !== "windows" && info.uid === Deno.uid()) { - // If the user is the owner of the file, then use the owner bits of - // the file permission - fileMode >>= 6; - } - // TODO(kt3k): Also check the case when the user belong to the group - // of the file - if ((m & fileMode) === m) { - // all required flags exist - cb(null); - } else { - // some required flags don't - // deno-lint-ignore no-explicit-any - const e: any = new Error(`EACCES: permission denied, access '${path}'`); - e.path = path; - e.syscall = "access"; - e.errno = codeMap.get("EACCES"); - e.code = "EACCES"; - cb(e); - } - }, (err) => { - if (err instanceof Deno.errors.NotFound) { - // deno-lint-ignore no-explicit-any - const e: any = new Error( - `ENOENT: no such file or directory, access '${path}'`, - ); - e.path = path; - e.syscall = "access"; - e.errno = codeMap.get("ENOENT"); - e.code = "ENOENT"; - cb(e); - } else { - cb(err); - } - }); + Deno.lstat(path).then( + (info) => { + if (info.mode === null) { + // If the file mode is unavailable, we pretend it has + // the permission + cb(null); + return; + } + let m = +mode || 0; + let fileMode = +info.mode || 0; + + if (Deno.build.os === "windows") { + m &= ~fs.X_OK; // Ignore the X_OK bit on Windows + } else if (info.uid === Deno.uid()) { + // If the user is the owner of the file, then use the owner bits of + // the file permission + fileMode >>= 6; + } + + // TODO(kt3k): Also check the case when the user belong to the group + // of the file + + if ((m & fileMode) === m) { + // all required flags exist + cb(null); + } else { + // some required flags don't + // deno-lint-ignore no-explicit-any + const e: any = new Error(`EACCES: permission denied, access '${path}'`); + e.path = path; + e.syscall = "access"; + e.errno = codeMap.get("EACCES"); + e.code = "EACCES"; + cb(e); + } + }, + (err) => { + if (err instanceof Deno.errors.NotFound) { + // deno-lint-ignore no-explicit-any + const e: any = new Error( + `ENOENT: no such file or directory, access '${path}'`, + ); + e.path = path; + e.syscall = "access"; + e.errno = codeMap.get("ENOENT"); + e.code = "ENOENT"; + cb(e); + } else { + cb(err); + } + }, + ); } export const accessPromise = promisify(access) as ( @@ -91,9 +99,11 @@ export function accessSync(path: string | Buffer | URL, mode?: number) { // the permission return; } - const m = +mode! || 0; + let m = +mode! || 0; let fileMode = +info.mode! || 0; - if (Deno.build.os !== "windows" && info.uid === Deno.uid()) { + if (Deno.build.os === "windows") { + m &= ~fs.X_OK; // Ignore the X_OK bit on Windows + } else if (info.uid === Deno.uid()) { // If the user is the owner of the file, then use the owner bits of // the file permission fileMode >>= 6; diff --git a/tests/unit_node/_fs/_fs_access_test.ts b/tests/unit_node/_fs/_fs_access_test.ts index f8010b0b8b..0881769f2c 100644 --- a/tests/unit_node/_fs/_fs_access_test.ts +++ b/tests/unit_node/_fs/_fs_access_test.ts @@ -28,6 +28,8 @@ Deno.test( try { await fs.promises.access(file, fs.constants.R_OK); await fs.promises.access(file, fs.constants.W_OK); + await fs.promises.access(file, fs.constants.X_OK); + await fs.promises.access(file, fs.constants.F_OK); } finally { await Deno.remove(file); } @@ -60,6 +62,8 @@ Deno.test( try { fs.accessSync(file, fs.constants.R_OK); fs.accessSync(file, fs.constants.W_OK); + fs.accessSync(file, fs.constants.X_OK); + fs.accessSync(file, fs.constants.F_OK); } finally { Deno.removeSync(file); } From 55d345baed709920cdf17c5662e8f3cb5c28be05 Mon Sep 17 00:00:00 2001 From: denobot <33910674+denobot@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:39:02 -0500 Subject: [PATCH 05/20] chore: release ext/ crates (#27419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: bartlomieju Co-authored-by: Bartek IwaƄczuk --- .github/workflows/ci.generate.ts | 2 +- .github/workflows/ci.yml | 8 ++--- Cargo.lock | 50 ++++++++++++++++---------------- Cargo.toml | 50 ++++++++++++++++---------------- bench_util/Cargo.toml | 2 +- ext/broadcast_channel/Cargo.toml | 2 +- ext/cache/Cargo.toml | 2 +- ext/canvas/Cargo.toml | 2 +- ext/console/Cargo.toml | 2 +- ext/cron/Cargo.toml | 2 +- ext/crypto/Cargo.toml | 2 +- ext/fetch/Cargo.toml | 2 +- ext/ffi/Cargo.toml | 2 +- ext/fs/Cargo.toml | 2 +- ext/http/Cargo.toml | 2 +- ext/io/Cargo.toml | 2 +- ext/kv/Cargo.toml | 2 +- ext/napi/Cargo.toml | 2 +- ext/napi/sym/Cargo.toml | 2 +- ext/net/Cargo.toml | 2 +- ext/node/Cargo.toml | 2 +- ext/telemetry/Cargo.toml | 2 +- ext/tls/Cargo.toml | 2 +- ext/url/Cargo.toml | 2 +- ext/web/Cargo.toml | 2 +- ext/webgpu/Cargo.toml | 2 +- ext/webidl/Cargo.toml | 2 +- ext/websocket/Cargo.toml | 2 +- ext/webstorage/Cargo.toml | 2 +- 29 files changed, 80 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 6fbcd8f242..bc3f15380b 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 31; +const cacheVersion = 32; const ubuntuX86Runner = "ubuntu-24.04"; const ubuntuX86XlRunner = "ubuntu-24.04-xl"; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f5d8f5b6d..cc1aa89669 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,8 +184,8 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - key: '31-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '31-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' + key: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - uses: dsherret/rust-toolchain-file@v1 if: '!(matrix.skip)' @@ -379,7 +379,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '31-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -689,7 +689,7 @@ jobs: !./target/*/gn_root !./target/*/*.zip !./target/*/*.tar.gz - key: '31-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-24.04 diff --git a/Cargo.lock b/Cargo.lock index bb0617ceee..4dddd53d75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1391,7 +1391,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.177.0" +version = "0.178.0" dependencies = [ "bencher", "deno_core", @@ -1400,7 +1400,7 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.177.0" +version = "0.178.0" dependencies = [ "async-trait", "deno_core", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "deno_cache" -version = "0.115.0" +version = "0.116.0" dependencies = [ "async-trait", "deno_core", @@ -1452,7 +1452,7 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.52.0" +version = "0.53.0" dependencies = [ "deno_core", "deno_webgpu", @@ -1487,7 +1487,7 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.183.0" +version = "0.184.0" dependencies = [ "deno_core", ] @@ -1536,7 +1536,7 @@ checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_cron" -version = "0.63.0" +version = "0.64.0" dependencies = [ "anyhow", "async-trait", @@ -1549,7 +1549,7 @@ dependencies = [ [[package]] name = "deno_crypto" -version = "0.197.0" +version = "0.198.0" dependencies = [ "aes", "aes-gcm", @@ -1639,7 +1639,7 @@ dependencies = [ [[package]] name = "deno_fetch" -version = "0.207.0" +version = "0.208.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1675,7 +1675,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.170.0" +version = "0.171.0" dependencies = [ "deno_core", "deno_permissions", @@ -1695,7 +1695,7 @@ dependencies = [ [[package]] name = "deno_fs" -version = "0.93.0" +version = "0.94.0" dependencies = [ "async-trait", "base32", @@ -1748,7 +1748,7 @@ dependencies = [ [[package]] name = "deno_http" -version = "0.181.0" +version = "0.182.0" dependencies = [ "async-compression", "async-trait", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.93.0" +version = "0.94.0" dependencies = [ "async-trait", "deno_core", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.91.0" +version = "0.92.0" dependencies = [ "anyhow", "async-trait", @@ -1881,7 +1881,7 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.114.0" +version = "0.115.0" dependencies = [ "deno_core", "deno_permissions", @@ -1909,7 +1909,7 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.175.0" +version = "0.176.0" dependencies = [ "deno_core", "deno_permissions", @@ -1926,7 +1926,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.121.0" +version = "0.122.0" dependencies = [ "aead-gcm-stream", "aes", @@ -2249,7 +2249,7 @@ dependencies = [ [[package]] name = "deno_telemetry" -version = "0.5.0" +version = "0.6.0" dependencies = [ "async-trait", "deno_core", @@ -2290,7 +2290,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.170.0" +version = "0.171.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2340,7 +2340,7 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.183.0" +version = "0.184.0" dependencies = [ "deno_bench_util", "deno_console", @@ -2352,7 +2352,7 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.214.0" +version = "0.215.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "deno_webgpu" -version = "0.150.0" +version = "0.151.0" dependencies = [ "deno_core", "raw-window-handle", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.183.0" +version = "0.184.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2395,7 +2395,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.188.0" +version = "0.189.0" dependencies = [ "bytes", "deno_core", @@ -2417,7 +2417,7 @@ dependencies = [ [[package]] name = "deno_webstorage" -version = "0.178.0" +version = "0.179.0" dependencies = [ "deno_core", "deno_web", @@ -4931,7 +4931,7 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.113.0" +version = "0.114.0" dependencies = [ "quote", "serde", diff --git a/Cargo.toml b/Cargo.toml index 55a94b75a0..63a6824e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ repository = "https://github.com/denoland/deno" deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.326.0" } -deno_bench_util = { version = "0.177.0", path = "./bench_util" } +deno_bench_util = { version = "0.178.0", path = "./bench_util" } deno_config = { version = "=0.39.3", features = ["workspace", "sync"] } deno_lockfile = "=0.23.2" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } @@ -60,7 +60,7 @@ deno_permissions = { version = "0.43.0", path = "./runtime/permissions" } deno_runtime = { version = "0.192.0", path = "./runtime" } deno_semver = "=0.6.1" deno_terminal = "0.2.0" -napi_sym = { version = "0.113.0", path = "./ext/napi/sym" } +napi_sym = { version = "0.114.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.4" @@ -69,29 +69,29 @@ denokv_remote = "0.8.4" denokv_sqlite = { default-features = false, version = "0.8.4" } # exts -deno_broadcast_channel = { version = "0.177.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.115.0", path = "./ext/cache" } -deno_canvas = { version = "0.52.0", path = "./ext/canvas" } -deno_console = { version = "0.183.0", path = "./ext/console" } -deno_cron = { version = "0.63.0", path = "./ext/cron" } -deno_crypto = { version = "0.197.0", path = "./ext/crypto" } -deno_fetch = { version = "0.207.0", path = "./ext/fetch" } -deno_ffi = { version = "0.170.0", path = "./ext/ffi" } -deno_fs = { version = "0.93.0", path = "./ext/fs" } -deno_http = { version = "0.181.0", path = "./ext/http" } -deno_io = { version = "0.93.0", path = "./ext/io" } -deno_kv = { version = "0.91.0", path = "./ext/kv" } -deno_napi = { version = "0.114.0", path = "./ext/napi" } -deno_net = { version = "0.175.0", path = "./ext/net" } -deno_node = { version = "0.121.0", path = "./ext/node" } -deno_telemetry = { version = "0.5.0", path = "./ext/telemetry" } -deno_tls = { version = "0.170.0", path = "./ext/tls" } -deno_url = { version = "0.183.0", path = "./ext/url" } -deno_web = { version = "0.214.0", path = "./ext/web" } -deno_webgpu = { version = "0.150.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.183.0", path = "./ext/webidl" } -deno_websocket = { version = "0.188.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.178.0", path = "./ext/webstorage" } +deno_broadcast_channel = { version = "0.178.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.116.0", path = "./ext/cache" } +deno_canvas = { version = "0.53.0", path = "./ext/canvas" } +deno_console = { version = "0.184.0", path = "./ext/console" } +deno_cron = { version = "0.64.0", path = "./ext/cron" } +deno_crypto = { version = "0.198.0", path = "./ext/crypto" } +deno_fetch = { version = "0.208.0", path = "./ext/fetch" } +deno_ffi = { version = "0.171.0", path = "./ext/ffi" } +deno_fs = { version = "0.94.0", path = "./ext/fs" } +deno_http = { version = "0.182.0", path = "./ext/http" } +deno_io = { version = "0.94.0", path = "./ext/io" } +deno_kv = { version = "0.92.0", path = "./ext/kv" } +deno_napi = { version = "0.115.0", path = "./ext/napi" } +deno_net = { version = "0.176.0", path = "./ext/net" } +deno_node = { version = "0.122.0", path = "./ext/node" } +deno_telemetry = { version = "0.6.0", path = "./ext/telemetry" } +deno_tls = { version = "0.171.0", path = "./ext/tls" } +deno_url = { version = "0.184.0", path = "./ext/url" } +deno_web = { version = "0.215.0", path = "./ext/web" } +deno_webgpu = { version = "0.151.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.184.0", path = "./ext/webidl" } +deno_websocket = { version = "0.189.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.179.0", path = "./ext/webstorage" } # resolvers deno_npm_cache = { version = "0.3.0", path = "./resolvers/npm_cache" } diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index 8a20f07638..014b74f264 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.177.0" +version = "0.178.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index 714f230cd2..4dea8f21e1 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.177.0" +version = "0.178.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index 7c05996498..96aec27576 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.115.0" +version = "0.116.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index ac9b236a95..7c7cc49b7c 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.52.0" +version = "0.53.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index df67b14a86..f68dd7d198 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.183.0" +version = "0.184.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index c5408e450b..022a8418cf 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.63.0" +version = "0.64.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index 86d984a421..c283cc9277 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.197.0" +version = "0.198.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index fee21808e7..e6e4ded4af 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.207.0" +version = "0.208.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index afcbf7b4e6..9cd5c77013 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.170.0" +version = "0.171.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index 608554607c..1d0b623718 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.93.0" +version = "0.94.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index dfb53559d6..e7aaad2fc0 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index 7a464ecde9..9298c654c1 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.93.0" +version = "0.94.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index e65880942b..c97aa75552 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.91.0" +version = "0.92.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index 5a9eb7441f..5d726b3e31 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.114.0" +version = "0.115.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml index b07dadd634..22228bd2f6 100644 --- a/ext/napi/sym/Cargo.toml +++ b/ext/napi/sym/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "napi_sym" -version = "0.113.0" +version = "0.114.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index 546152bd4b..8dbb0be391 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.175.0" +version = "0.176.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 127633a09b..60e7c96a08 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.121.0" +version = "0.122.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/telemetry/Cargo.toml b/ext/telemetry/Cargo.toml index d0bdc6be1b..fedaed6656 100644 --- a/ext/telemetry/Cargo.toml +++ b/ext/telemetry/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_telemetry" -version = "0.5.0" +version = "0.6.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index 690267b7e0..6bf1b8ea03 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.170.0" +version = "0.171.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index de4fc67df8..9ca3ce6752 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.183.0" +version = "0.184.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index b4cd69f970..44fb2e46bf 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.214.0" +version = "0.215.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index 858cdb2dab..3a491afcf8 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.150.0" +version = "0.151.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index 0ad7d8ac10..60cb9f29f8 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.183.0" +version = "0.184.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index 2cd48a3816..8b8359f074 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.188.0" +version = "0.189.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index ff76458f33..4f9795d098 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.178.0" +version = "0.179.0" authors.workspace = true edition.workspace = true license.workspace = true From 350d9dce41b78b124d23c70d78feef3c8ce39a1c Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Thu, 19 Dec 2024 17:39:20 +0900 Subject: [PATCH 06/20] fix(ext/node): do not exit worker thread when there is pending async op (#27378) This change fixes the premature exit of worker threads when there are still remaining pending ops. This change reuses the idea of #22647 (unref'ing `op_worker_recv_message` in worker threads if closeOnIdle specified) and uses `web_worker.has_message_event_listener` check in the opposite way as #22944. (Now we continue the worker when `has_message_event_listener` is true instead of stopping it when `has_message_event_listener` is false. closes #23061 closes #26154 --- cli/worker.rs | 2 ++ ext/node/polyfills/worker_threads.ts | 6 ++-- ext/web/13_message_port.js | 4 +-- runtime/js/99_main.js | 11 +++++-- runtime/web_worker.rs | 33 +++---------------- runtime/worker_bootstrap.rs | 5 +++ .../permission/allow_import_worker/denied.out | 3 +- tests/unit_node/worker_threads_test.ts | 23 +++++++++++++ 8 files changed, 49 insertions(+), 38 deletions(-) diff --git a/cli/worker.rs b/cli/worker.rs index 0bbc27b29f..c733f41321 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -612,6 +612,7 @@ impl CliMainWorkerFactory { serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), otel_config: shared.otel_config.clone(), + close_on_idle: true, }, extensions: custom_extensions, startup_snapshot: crate::js::deno_isolate_init(), @@ -812,6 +813,7 @@ fn create_web_worker_callback( serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), otel_config: shared.otel_config.clone(), + close_on_idle: args.close_on_idle, }, extensions: vec![], startup_snapshot: crate::js::deno_isolate_init(), diff --git a/ext/node/polyfills/worker_threads.ts b/ext/node/polyfills/worker_threads.ts index 1b175fb1dd..dc844169c5 100644 --- a/ext/node/polyfills/worker_threads.ts +++ b/ext/node/polyfills/worker_threads.ts @@ -21,7 +21,7 @@ import { nodeWorkerThreadCloseCb, refMessagePort, serializeJsMessageData, - unrefPollForMessages, + unrefParentPort, } from "ext:deno_web/13_message_port.js"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import { notImplemented } from "ext:deno_node/_utils.ts"; @@ -451,10 +451,10 @@ internals.__initWorkerThreads = ( parentPort.emit("close"); }); parentPort.unref = () => { - parentPort[unrefPollForMessages] = true; + parentPort[unrefParentPort] = true; }; parentPort.ref = () => { - parentPort[unrefPollForMessages] = false; + parentPort[unrefParentPort] = false; }; if (isWorkerThread) { diff --git a/ext/web/13_message_port.js b/ext/web/13_message_port.js index cf72c43e6f..79fec9de2f 100644 --- a/ext/web/13_message_port.js +++ b/ext/web/13_message_port.js @@ -102,8 +102,8 @@ const nodeWorkerThreadCloseCb = Symbol("nodeWorkerThreadCloseCb"); const nodeWorkerThreadCloseCbInvoked = Symbol("nodeWorkerThreadCloseCbInvoked"); export const refMessagePort = Symbol("refMessagePort"); /** It is used by 99_main.js and worker_threads to - * unref/ref on the global pollForMessages promise. */ -export const unrefPollForMessages = Symbol("unrefPollForMessages"); + * unref/ref on the global message event handler count. */ +export const unrefParentPort = Symbol("unrefParentPort"); /** * @param {number} id diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 19432745d4..bceb1f7ddb 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -170,12 +170,14 @@ function postMessage(message, transferOrOptions = { __proto__: null }) { let isClosing = false; let globalDispatchEvent; +let closeOnIdle; function hasMessageEventListener() { // the function name is kind of a misnomer, but we want to behave // as if we have message event listeners if a node message port is explicitly // refed (and the inverse as well) - return event.listenerCount(globalThis, "message") > 0 || + return (event.listenerCount(globalThis, "message") > 0 && + !globalThis[messagePort.unrefParentPort]) || messagePort.refedMessagePortsCount > 0; } @@ -188,7 +190,10 @@ async function pollForMessages() { } while (!isClosing) { const recvMessage = op_worker_recv_message(); - if (globalThis[messagePort.unrefPollForMessages] === true) { + // In a Node.js worker, unref() the op promise to prevent it from + // keeping the event loop alive. This avoids the need to explicitly + // call self.close() or worker.terminate(). + if (closeOnIdle) { core.unrefOpPromise(recvMessage); } const data = await recvMessage; @@ -915,6 +920,7 @@ function bootstrapWorkerRuntime( 6: argv0, 7: nodeDebug, 13: otelConfig, + 14: closeOnIdle_, } = runtimeOptions; performance.setTimeOrigin(); @@ -967,6 +973,7 @@ function bootstrapWorkerRuntime( globalThis.pollForMessages = pollForMessages; globalThis.hasMessageEventListener = hasMessageEventListener; + closeOnIdle = closeOnIdle_; for (let i = 0; i <= unstableFeatures.length; i++) { const id = unstableFeatures[i]; diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index e3a69b39c0..faf4f3fc52 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -58,7 +58,6 @@ use std::task::Poll; use crate::inspector_server::InspectorServer; use crate::ops; use crate::ops::process::NpmProcessStateProviderRc; -use crate::ops::worker_host::WorkersTable; use crate::shared::maybe_transpile_source; use crate::shared::runtime; use crate::tokio_util::create_and_run_current_thread; @@ -385,7 +384,6 @@ pub struct WebWorker { pub js_runtime: JsRuntime, pub name: String, close_on_idle: bool, - has_executed_main_module: bool, internal_handle: WebWorkerInternalHandle, pub worker_type: WebWorkerType, pub main_module: ModuleSpecifier, @@ -658,7 +656,6 @@ impl WebWorker { has_message_event_listener_fn: None, bootstrap_fn_global: Some(bootstrap_fn_global), close_on_idle: options.close_on_idle, - has_executed_main_module: false, maybe_worker_metadata: options.maybe_worker_metadata, }, external_handle, @@ -799,7 +796,6 @@ impl WebWorker { maybe_result = &mut receiver => { debug!("received worker module evaluate {:#?}", maybe_result); - self.has_executed_main_module = true; maybe_result } @@ -837,6 +833,9 @@ impl WebWorker { } if self.close_on_idle { + if self.has_message_event_listener() { + return Poll::Pending; + } return Poll::Ready(Ok(())); } @@ -851,22 +850,7 @@ impl WebWorker { Poll::Ready(Ok(())) } } - Poll::Pending => { - // This is special code path for workers created from `node:worker_threads` - // module that have different semantics than Web workers. - // We want the worker thread to terminate automatically if we've done executing - // Top-Level await, there are no child workers spawned by that workers - // and there's no "message" event listener. - if self.close_on_idle - && self.has_executed_main_module - && !self.has_child_workers() - && !self.has_message_event_listener() - { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - } + Poll::Pending => Poll::Pending, } } @@ -904,15 +888,6 @@ impl WebWorker { None => false, } } - - fn has_child_workers(&mut self) -> bool { - !self - .js_runtime - .op_state() - .borrow() - .borrow::() - .is_empty() - } } fn print_worker_error( diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index 2020c2bc8d..8364fe0d2b 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -120,6 +120,7 @@ pub struct BootstrapOptions { pub serve_port: Option, pub serve_host: Option, pub otel_config: OtelConfig, + pub close_on_idle: bool, } impl Default for BootstrapOptions { @@ -155,6 +156,7 @@ impl Default for BootstrapOptions { serve_port: Default::default(), serve_host: Default::default(), otel_config: Default::default(), + close_on_idle: false, } } } @@ -198,6 +200,8 @@ struct BootstrapV8<'a>( Option, // OTEL config Box<[u8]>, + // close on idle + bool, ); impl BootstrapOptions { @@ -225,6 +229,7 @@ impl BootstrapOptions { serve_is_main, serve_worker_count, self.otel_config.as_v8(), + self.close_on_idle, ); bootstrap.serialize(ser).unwrap() diff --git a/tests/specs/permission/allow_import_worker/denied.out b/tests/specs/permission/allow_import_worker/denied.out index 6e4dcaee09..af44ae21ee 100644 --- a/tests/specs/permission/allow_import_worker/denied.out +++ b/tests/specs/permission/allow_import_worker/denied.out @@ -3,5 +3,4 @@ await import(specifier); ^ at async file:///[WILDLINE] error: Uncaught (in promise) Error: Unhandled error in child worker. - at [WILDLINE] - at [WILDLINE] \ No newline at end of file + at [WILDCARD] \ No newline at end of file diff --git a/tests/unit_node/worker_threads_test.ts b/tests/unit_node/worker_threads_test.ts index 808fd6116e..5f38d51d4d 100644 --- a/tests/unit_node/worker_threads_test.ts +++ b/tests/unit_node/worker_threads_test.ts @@ -841,3 +841,26 @@ Deno.test({ assertEquals(result, true); }, }); + +Deno.test("[node/worker_threads] Worker runs async ops correctly", async () => { + const recvMessage = Promise.withResolvers(); + const timer = setTimeout(() => recvMessage.reject(), 1000); + const worker = new workerThreads.Worker( + ` + import { parentPort } from "node:worker_threads"; + setTimeout(() => { + parentPort.postMessage("Hello from worker"); + }, 10); + `, + { eval: true }, + ); + + worker.on("message", (msg) => { + assertEquals(msg, "Hello from worker"); + worker.terminate(); + recvMessage.resolve(); + clearTimeout(timer); + }); + + await recvMessage.promise; +}); From 074444ab6c0d431f9e61a3371db4bf426f973869 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 19 Dec 2024 12:53:52 -0500 Subject: [PATCH 07/20] fix(compile): be more deterministic when compiling the same code in different directories (#27395) Additionaly, this no longer unnecessarily stores the source twice for file specifiers and fixes some sourcemap issues. Closes https://github.com/denoland/deno/issues/27284 --- cli/args/mod.rs | 4 +- cli/emit.rs | 35 ++- cli/standalone/binary.rs | 193 ++++++++++------ cli/standalone/mod.rs | 50 ++++- cli/standalone/serialization.rs | 210 ++++++++++++++---- cli/standalone/virtual_fs.rs | 159 +++++++------ tests/integration/compile_tests.rs | 73 ------ tests/specs/compile/code_cache/__test__.jsonc | 3 + tests/specs/compile/code_cache/cleanup.ts | 11 + .../specs/compile/determinism/__test__.jsonc | 17 +- tests/specs/compile/determinism/setup.ts | 10 + .../specs/compile/error/local/__test__.jsonc | 24 ++ tests/specs/compile/error/local/output.out | 6 + .../compile/error/local}/standalone_error.ts | 0 .../specs/compile/error/remote/__test__.jsonc | 24 ++ tests/specs/compile/error/remote/main.ts | 1 + tests/specs/compile/error/remote/output.out | 5 + .../standalone_error_module_with_imports_2.ts | 7 +- 18 files changed, 563 insertions(+), 269 deletions(-) create mode 100644 tests/specs/compile/code_cache/cleanup.ts create mode 100644 tests/specs/compile/determinism/setup.ts create mode 100644 tests/specs/compile/error/local/__test__.jsonc create mode 100644 tests/specs/compile/error/local/output.out rename tests/{testdata/compile => specs/compile/error/local}/standalone_error.ts (100%) create mode 100644 tests/specs/compile/error/remote/__test__.jsonc create mode 100644 tests/specs/compile/error/remote/main.ts create mode 100644 tests/specs/compile/error/remote/output.out diff --git a/cli/args/mod.rs b/cli/args/mod.rs index f820afe78d..fd34b53c8c 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1363,9 +1363,9 @@ impl CliOptions { Ok(DenoLintConfig { default_jsx_factory: (!transpile_options.jsx_automatic) - .then(|| transpile_options.jsx_factory.clone()), + .then_some(transpile_options.jsx_factory), default_jsx_fragment_factory: (!transpile_options.jsx_automatic) - .then(|| transpile_options.jsx_fragment_factory.clone()), + .then_some(transpile_options.jsx_fragment_factory), }) } diff --git a/cli/emit.rs b/cli/emit.rs index 3cd23b7abb..733a89d832 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -5,6 +5,7 @@ use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; use crate::resolver::CjsTracker; +use deno_ast::EmittedSourceText; use deno_ast::ModuleKind; use deno_ast::SourceMapOption; use deno_ast::SourceRange; @@ -132,6 +133,7 @@ impl Emitter { &transpile_and_emit_options.0, &transpile_and_emit_options.1, ) + .map(|r| r.text) } }) .await @@ -166,7 +168,8 @@ impl Emitter { source.clone(), &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, - )?; + )? + .text; helper.post_emit_parsed_source( specifier, &transpiled_source, @@ -177,6 +180,31 @@ impl Emitter { } } + pub fn emit_parsed_source_for_deno_compile( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + module_kind: deno_ast::ModuleKind, + source: &Arc, + ) -> Result<(String, String), AnyError> { + let mut emit_options = self.transpile_and_emit_options.1.clone(); + emit_options.inline_sources = false; + emit_options.source_map = SourceMapOption::Separate; + // strip off the path to have more deterministic builds as we don't care + // about the source name because we manually provide the source map to v8 + emit_options.source_map_base = Some(deno_path_util::url_parent(specifier)); + let source = EmitParsedSourceHelper::transpile( + &self.parsed_source_cache, + specifier, + media_type, + module_kind, + source.clone(), + &self.transpile_and_emit_options.0, + &emit_options, + )?; + Ok((source.text, source.source_map.unwrap())) + } + /// Expects a file URL, panics otherwise. pub async fn load_and_emit_for_hmr( &self, @@ -282,7 +310,7 @@ impl<'a> EmitParsedSourceHelper<'a> { source: Arc, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, - ) -> Result { + ) -> Result { // nothing else needs the parsed source at this point, so remove from // the cache in order to not transpile owned let parsed_source = parsed_source_cache @@ -302,8 +330,7 @@ impl<'a> EmitParsedSourceHelper<'a> { source } }; - debug_assert!(transpiled_source.source_map.is_none()); - Ok(transpiled_source.text) + Ok(transpiled_source) } pub fn post_emit_parsed_source( diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 2ed52010fb..f7ffa46e4f 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -91,6 +91,7 @@ use super::serialization::DenoCompileModuleData; use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; +use super::serialization::SourceMapStore; use super::virtual_fs::output_vfs; use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; @@ -98,6 +99,7 @@ use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; +use super::virtual_fs::VirtualDirectoryEntries; use super::virtual_fs::WindowsSystemRootablePath; pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = @@ -203,18 +205,25 @@ pub struct Metadata { pub otel_config: OtelConfig, } +#[allow(clippy::too_many_arguments)] fn write_binary_bytes( mut file_writer: File, original_bin: Vec, metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, + source_map_store: &SourceMapStore, vfs: &BuiltVfs, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let data_section_bytes = - serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs) - .context("Serializing binary data section.")?; + let data_section_bytes = serialize_binary_data_section( + metadata, + npm_snapshot, + remote_modules, + source_map_store, + vfs, + ) + .context("Serializing binary data section.")?; let target = compile_flags.resolve_target(); if target.contains("linux") { @@ -256,6 +265,7 @@ pub struct StandaloneData { pub modules: StandaloneModules, pub npm_snapshot: Option, pub root_path: PathBuf, + pub source_maps: SourceMapStore, pub vfs: Arc, } @@ -283,13 +293,12 @@ impl StandaloneModules { pub fn read<'a>( &'a self, specifier: &'a ModuleSpecifier, + kind: VfsFileSubDataKind, ) -> Result>, AnyError> { if specifier.scheme() == "file" { let path = deno_path_util::url_to_file_path(specifier)?; let bytes = match self.vfs.file_entry(&path) { - Ok(entry) => self - .vfs - .read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?, + Ok(entry) => self.vfs.read_file_all(entry, kind)?, Err(err) if err.kind() == ErrorKind::NotFound => { match RealFs.read_file_sync(&path, None) { Ok(bytes) => bytes, @@ -307,7 +316,18 @@ impl StandaloneModules { data: bytes, })) } else { - self.remote_modules.read(specifier) + self.remote_modules.read(specifier).map(|maybe_entry| { + maybe_entry.map(|entry| DenoCompileModuleData { + media_type: entry.media_type, + specifier: entry.specifier, + data: match kind { + VfsFileSubDataKind::Raw => entry.data, + VfsFileSubDataKind::ModuleGraph => { + entry.transpiled_data.unwrap_or(entry.data) + } + }, + }) + }) } } } @@ -328,7 +348,8 @@ pub fn extract_standalone( mut metadata, npm_snapshot, remote_modules, - mut vfs_dir, + source_maps, + vfs_root_entries, vfs_files_data, } = match deserialize_binary_data_section(data)? { Some(data_section) => data_section, @@ -351,11 +372,12 @@ pub fn extract_standalone( metadata.argv.push(arg.into_string().unwrap()); } let vfs = { - // align the name of the directory with the root dir - vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string(); - let fs_root = VfsRoot { - dir: vfs_dir, + dir: VirtualDirectory { + // align the name of the directory with the root dir + name: root_path.file_name().unwrap().to_string_lossy().to_string(), + entries: vfs_root_entries, + }, root_path: root_path.clone(), start_file_offset: 0, }; @@ -372,6 +394,7 @@ pub fn extract_standalone( }, npm_snapshot, root_path, + source_maps, vfs, })) } @@ -451,7 +474,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self.write_standalone_binary(options, original_binary).await + self.write_standalone_binary(options, original_binary) } async fn get_base_binary( @@ -554,7 +577,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] - async fn write_standalone_binary( + fn write_standalone_binary( &self, options: WriteBinOptions<'_>, original_bin: Vec, @@ -598,71 +621,81 @@ impl<'a> DenoCompileBinaryWriter<'a> { .with_context(|| format!("Including {}", path.display()))?; } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); - let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { - Some(FastInsecureHasher::new_deno_versioned()) - } else { - None - }; + let mut source_maps = Vec::with_capacity(graph.specifiers_count()); + // todo(dsherret): transpile in parallel for module in graph.modules() { if module.specifier().scheme() == "data" { continue; // don't store data urls as an entry as they're in the code } - if let Some(hasher) = &mut code_cache_key_hasher { - if let Some(source) = module.source() { - hasher.write(module.specifier().as_str().as_bytes()); - hasher.write(source.as_bytes()); - } - } - let (maybe_source, media_type) = match module { + let (maybe_original_source, maybe_transpiled, media_type) = match module { deno_graph::Module::Js(m) => { - let source = if m.media_type.is_emittable() { + let original_bytes = m.source.as_bytes().to_vec(); + let maybe_transpiled = if m.media_type.is_emittable() { let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( &m.specifier, m.media_type, m.is_script, )?; let module_kind = ModuleKind::from_is_cjs(is_cjs); - let source = self - .emitter - .emit_parsed_source( + let (source, source_map) = + self.emitter.emit_parsed_source_for_deno_compile( &m.specifier, m.media_type, module_kind, &m.source, - ) - .await?; - source.into_bytes() + )?; + if source != m.source.as_ref() { + source_maps.push((&m.specifier, source_map)); + Some(source.into_bytes()) + } else { + None + } } else { - m.source.as_bytes().to_vec() + None }; - (Some(source), m.media_type) + (Some(original_bytes), maybe_transpiled, m.media_type) } deno_graph::Module::Json(m) => { - (Some(m.source.as_bytes().to_vec()), m.media_type) + (Some(m.source.as_bytes().to_vec()), None, m.media_type) } deno_graph::Module::Wasm(m) => { - (Some(m.source.to_vec()), MediaType::Wasm) + (Some(m.source.to_vec()), None, MediaType::Wasm) } deno_graph::Module::Npm(_) | deno_graph::Module::Node(_) - | deno_graph::Module::External(_) => (None, MediaType::Unknown), + | deno_graph::Module::External(_) => (None, None, MediaType::Unknown), }; - if module.specifier().scheme() == "file" { - let file_path = deno_path_util::url_to_file_path(module.specifier())?; - vfs - .add_file_with_data( - &file_path, - match maybe_source { - Some(source) => source, - None => RealFs.read_file_sync(&file_path, None)?.into_owned(), - }, - VfsFileSubDataKind::ModuleGraph, - ) - .with_context(|| { - format!("Failed adding '{}'", file_path.display()) - })?; - } else if let Some(source) = maybe_source { - remote_modules_store.add(module.specifier(), media_type, source); + if let Some(original_source) = maybe_original_source { + if module.specifier().scheme() == "file" { + let file_path = deno_path_util::url_to_file_path(module.specifier())?; + vfs + .add_file_with_data( + &file_path, + original_source, + VfsFileSubDataKind::Raw, + ) + .with_context(|| { + format!("Failed adding '{}'", file_path.display()) + })?; + if let Some(transpiled_source) = maybe_transpiled { + vfs + .add_file_with_data( + &file_path, + transpiled_source, + VfsFileSubDataKind::ModuleGraph, + ) + .with_context(|| { + format!("Failed adding '{}'", file_path.display()) + })?; + } + } else { + remote_modules_store.add( + module.specifier(), + media_type, + original_source, + maybe_transpiled, + ); + } } } remote_modules_store.add_redirects(&graph.redirects); @@ -695,6 +728,28 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot, }; + let code_cache_key = if self.cli_options.code_cache_enabled() { + let mut hasher = FastInsecureHasher::new_deno_versioned(); + for module in graph.modules() { + if let Some(source) = module.source() { + hasher + .write(root_dir_url.specifier_key(module.specifier()).as_bytes()); + hasher.write(source.as_bytes()); + } + } + Some(hasher.finish()) + } else { + None + }; + + let mut source_map_store = SourceMapStore::with_capacity(source_maps.len()); + for (specifier, source_map) in source_maps { + source_map_store.add( + Cow::Owned(root_dir_url.specifier_key(specifier).into_owned()), + Cow::Owned(source_map), + ); + } + let node_modules = match self.npm_resolver.as_inner() { InnerCliNpmResolverRef::Managed(_) => { npm_snapshot.as_ref().map(|_| NodeModules::Managed { @@ -742,7 +797,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { let metadata = Metadata { argv: compile_flags.args.clone(), seed: self.cli_options.seed(), - code_cache_key: code_cache_key_hasher.map(|h| h.finish()), + code_cache_key, location: self.cli_options.location_flag().clone(), permissions: self.cli_options.permission_flags().clone(), v8_flags: self.cli_options.v8_flags().clone(), @@ -809,6 +864,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { &metadata, npm_snapshot.map(|s| s.into_serialized()), &remote_modules_store, + &source_map_store, &vfs, compile_flags, ) @@ -903,10 +959,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string(); let mut new_entries = Vec::with_capacity(root_dir.entries.len()); let mut localhost_entries = IndexMap::new(); - for entry in std::mem::take(&mut root_dir.entries) { + for entry in root_dir.entries.take_inner() { match entry { - VfsEntry::Dir(dir) => { - for entry in dir.entries { + VfsEntry::Dir(mut dir) => { + for entry in dir.entries.take_inner() { log::debug!("Flattening {} into node_modules", entry.name()); if let Some(existing) = localhost_entries.insert(entry.name().to_string(), entry) @@ -925,11 +981,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { } new_entries.push(VfsEntry::Dir(VirtualDirectory { name: "localhost".to_string(), - entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), + entries: VirtualDirectoryEntries::new( + localhost_entries.into_iter().map(|(_, v)| v).collect(), + ), })); - // needs to be sorted by name - new_entries.sort_by(|a, b| a.name().cmp(b.name())); - root_dir.entries = new_entries; + root_dir.entries = VirtualDirectoryEntries::new(new_entries); // it's better to not expose the user's cache directory, so take it out // of there @@ -937,10 +993,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { let parent_dir = vfs.get_dir_mut(parent).unwrap(); let index = parent_dir .entries - .iter() - .position(|entry| { - entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME - }) + .binary_search(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME) .unwrap(); let npm_global_cache_dir_entry = parent_dir.entries.remove(index); @@ -950,11 +1003,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME); for ancestor in parent.ancestors() { let dir = vfs.get_dir_mut(ancestor).unwrap(); - if let Some(index) = dir - .entries - .iter() - .position(|entry| entry.name() == last_name) - { + if let Ok(index) = dir.entries.binary_search(&last_name) { dir.entries.remove(index); } last_name = Cow::Owned(dir.name.clone()); @@ -965,7 +1014,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { // now build the vfs and add the global cache dir entry there let mut built_vfs = vfs.build(); - built_vfs.root.insert_entry(npm_global_cache_dir_entry); + built_vfs.entries.insert(npm_global_cache_dir_entry); built_vfs } InnerCliNpmResolverRef::Byonm(_) => vfs.build(), diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 08ee5ba11a..d7227f2bd5 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -55,6 +55,7 @@ use node_resolver::errors::ClosestPkgJsonError; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; use serialization::DenoCompileModuleSource; +use serialization::SourceMapStore; use std::borrow::Cow; use std::rc::Rc; use std::sync::Arc; @@ -122,6 +123,7 @@ struct SharedModuleLoaderState { npm_module_loader: Arc, npm_req_resolver: Arc, npm_resolver: Arc, + source_maps: SourceMapStore, vfs: Arc, workspace_resolver: WorkspaceResolver, } @@ -396,7 +398,11 @@ impl ModuleLoader for EmbeddedModuleLoader { ); } - match self.shared.modules.read(original_specifier) { + match self + .shared + .modules + .read(original_specifier, VfsFileSubDataKind::ModuleGraph) + { Ok(Some(module)) => { let media_type = module.media_type; let (module_specifier, module_type, module_source) = @@ -495,6 +501,46 @@ impl ModuleLoader for EmbeddedModuleLoader { } std::future::ready(()).boxed_local() } + + fn get_source_map(&self, file_name: &str) -> Option> { + if file_name.starts_with("file:///") { + let url = + deno_path_util::url_from_directory_path(self.shared.vfs.root()).ok()?; + let file_url = ModuleSpecifier::parse(file_name).ok()?; + let relative_path = url.make_relative(&file_url)?; + self.shared.source_maps.get(&relative_path) + } else { + self.shared.source_maps.get(file_name) + } + // todo(https://github.com/denoland/deno_core/pull/1007): don't clone + .map(|s| s.as_bytes().to_vec()) + } + + fn get_source_mapped_source_line( + &self, + file_name: &str, + line_number: usize, + ) -> Option { + let specifier = ModuleSpecifier::parse(file_name).ok()?; + let data = self + .shared + .modules + .read(&specifier, VfsFileSubDataKind::Raw) + .ok()??; + + let source = String::from_utf8_lossy(&data.data); + // Do NOT use .lines(): it skips the terminating empty line. + // (due to internally using_terminator() instead of .split()) + let lines: Vec<&str> = source.split('\n').collect(); + if line_number >= lines.len() { + Some(format!( + "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)", + crate::colors::yellow("Warning"), line_number + 1, + )) + } else { + Some(lines[line_number].to_string()) + } + } } impl NodeRequireLoader for EmbeddedModuleLoader { @@ -590,6 +636,7 @@ pub async fn run(data: StandaloneData) -> Result { modules, npm_snapshot, root_path, + source_maps, vfs, } = data; let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); @@ -841,6 +888,7 @@ pub async fn run(data: StandaloneData) -> Result { )), npm_resolver: npm_resolver.clone(), npm_req_resolver, + source_maps, vfs, workspace_resolver, }), diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index ac76172e37..8fe593c005 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -6,6 +6,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::io::Write; +use deno_ast::swc::common::source_map; use deno_ast::MediaType; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -20,12 +21,14 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_semver::package::PackageReq; +use indexmap::IndexMap; use crate::standalone::virtual_fs::VirtualDirectory; use super::binary::Metadata; use super::virtual_fs::BuiltVfs; use super::virtual_fs::VfsBuilder; +use super::virtual_fs::VirtualDirectoryEntries; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; @@ -33,21 +36,22 @@ const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; /// * d3n0l4nd /// * /// * -/// * +/// * /// * /// * +/// * /// * d3n0l4nd pub fn serialize_binary_data_section( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, + source_map_store: &SourceMapStore, vfs: &BuiltVfs, ) -> Result, AnyError> { let metadata = serde_json::to_string(metadata)?; let npm_snapshot = npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); - let remote_modules_len = Cell::new(0_u64); - let serialized_vfs = serde_json::to_string(&vfs.root)?; + let serialized_vfs = serde_json::to_string(&vfs.entries)?; let bytes = capacity_builder::BytesBuilder::build(|builder| { builder.append(MAGIC_BYTES); @@ -63,10 +67,7 @@ pub fn serialize_binary_data_section( } // 3. Remote modules { - builder.append_le(remote_modules_len.get()); // this will be properly initialized on the second pass - let start_index = builder.len(); remote_modules.write(builder); - remote_modules_len.set((builder.len() - start_index) as u64); } // 4. VFS { @@ -78,6 +79,16 @@ pub fn serialize_binary_data_section( builder.append(file); } } + // 5. Source maps + { + builder.append_le(source_map_store.data.len() as u32); + for (specifier, source_map) in &source_map_store.data { + builder.append_le(specifier.len() as u32); + builder.append(specifier); + builder.append_le(source_map.len() as u32); + builder.append(source_map); + } + } // write the magic bytes at the end so we can use it // to make sure we've deserialized correctly @@ -91,19 +102,14 @@ pub struct DeserializedDataSection { pub metadata: Metadata, pub npm_snapshot: Option, pub remote_modules: RemoteModulesStore, - pub vfs_dir: VirtualDirectory, + pub source_maps: SourceMapStore, + pub vfs_root_entries: VirtualDirectoryEntries, pub vfs_files_data: &'static [u8], } pub fn deserialize_binary_data_section( data: &'static [u8], ) -> Result, AnyError> { - fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { - let (input, len) = read_u64(input)?; - let (input, data) = read_bytes(input, len as usize)?; - Ok((input, data)) - } - fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> { if input.len() < MAGIC_BYTES.len() { bail!("Unexpected end of data. Could not find magic bytes."); @@ -115,34 +121,51 @@ pub fn deserialize_binary_data_section( Ok((input, true)) } + #[allow(clippy::type_complexity)] + fn read_source_map_entry( + input: &[u8], + ) -> Result<(&[u8], (Cow, Cow)), AnyError> { + let (input, specifier) = read_string_lossy(input)?; + let (input, source_map) = read_string_lossy(input)?; + Ok((input, (specifier, source_map))) + } + let (input, found) = read_magic_bytes(data)?; if !found { return Ok(None); } // 1. Metadata - let (input, data) = read_bytes_with_len(input).context("reading metadata")?; + let (input, data) = + read_bytes_with_u64_len(input).context("reading metadata")?; let metadata: Metadata = serde_json::from_slice(data).context("deserializing metadata")?; // 2. Npm snapshot let (input, data) = - read_bytes_with_len(input).context("reading npm snapshot")?; + read_bytes_with_u64_len(input).context("reading npm snapshot")?; let npm_snapshot = if data.is_empty() { None } else { Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?) }; // 3. Remote modules - let (input, data) = - read_bytes_with_len(input).context("reading remote modules data")?; - let remote_modules = - RemoteModulesStore::build(data).context("deserializing remote modules")?; + let (input, remote_modules) = + RemoteModulesStore::build(input).context("deserializing remote modules")?; // 4. VFS - let (input, data) = read_bytes_with_len(input).context("vfs")?; - let vfs_dir: VirtualDirectory = + let (input, data) = read_bytes_with_u64_len(input).context("vfs")?; + let vfs_root_entries: VirtualDirectoryEntries = serde_json::from_slice(data).context("deserializing vfs data")?; let (input, vfs_files_data) = - read_bytes_with_len(input).context("reading vfs files data")?; + read_bytes_with_u64_len(input).context("reading vfs files data")?; + // 5. Source maps + let (mut input, source_map_data_len) = read_u32_as_usize(input)?; + let mut source_maps = SourceMapStore::with_capacity(source_map_data_len); + for _ in 0..source_map_data_len { + let (current_input, (specifier, source_map)) = + read_source_map_entry(input)?; + input = current_input; + source_maps.add(specifier, source_map); + } // finally ensure we read the magic bytes at the end let (_input, found) = read_magic_bytes(input)?; @@ -154,7 +177,8 @@ pub fn deserialize_binary_data_section( metadata, npm_snapshot, remote_modules, - vfs_dir, + source_maps, + vfs_root_entries, vfs_files_data, })) } @@ -162,19 +186,31 @@ pub fn deserialize_binary_data_section( #[derive(Default)] pub struct RemoteModulesStoreBuilder { specifiers: Vec<(String, u64)>, - data: Vec<(MediaType, Vec)>, + data: Vec<(MediaType, Vec, Option>)>, data_byte_len: u64, redirects: Vec<(String, String)>, redirects_len: u64, } impl RemoteModulesStoreBuilder { - pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec) { + pub fn add( + &mut self, + specifier: &Url, + media_type: MediaType, + data: Vec, + maybe_transpiled: Option>, + ) { log::debug!("Adding '{}' ({})", specifier, media_type); let specifier = specifier.to_string(); self.specifiers.push((specifier, self.data_byte_len)); - self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data - self.data.push((media_type, data)); + let maybe_transpiled_len = match &maybe_transpiled { + // data length (4 bytes), data + Some(data) => 4 + data.len() as u64, + None => 0, + }; + // media type (1 byte), data length (4 bytes), data, has transpiled (1 byte), transpiled length + self.data_byte_len += 1 + 4 + data.len() as u64 + 1 + maybe_transpiled_len; + self.data.push((media_type, data, maybe_transpiled)); } pub fn add_redirects(&mut self, redirects: &BTreeMap) { @@ -193,7 +229,7 @@ impl RemoteModulesStoreBuilder { builder.append_le(self.redirects.len() as u32); for (specifier, offset) in &self.specifiers { builder.append_le(specifier.len() as u32); - builder.append(specifier.as_bytes()); + builder.append(specifier); builder.append_le(*offset); } for (from, to) in &self.redirects { @@ -202,10 +238,32 @@ impl RemoteModulesStoreBuilder { builder.append_le(to.len() as u32); builder.append(to); } - for (media_type, data) in &self.data { + builder.append_le( + self + .data + .iter() + .map(|(_, data, maybe_transpiled)| { + 1 + 4 + + (data.len() as u64) + + 1 + + match maybe_transpiled { + Some(transpiled) => 4 + (transpiled.len() as u64), + None => 0, + } + }) + .sum::(), + ); + for (media_type, data, maybe_transpiled) in &self.data { builder.append(serialize_media_type(*media_type)); - builder.append_le(data.len() as u64); + builder.append_le(data.len() as u32); builder.append(data); + if let Some(transpiled) = maybe_transpiled { + builder.append(1); + builder.append_le(transpiled.len() as u32); + builder.append(transpiled); + } else { + builder.append(0); + } } } } @@ -234,6 +292,30 @@ impl DenoCompileModuleSource { } } +pub struct SourceMapStore { + data: IndexMap, Cow<'static, str>>, +} + +impl SourceMapStore { + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: IndexMap::with_capacity(capacity), + } + } + + pub fn add( + &mut self, + specifier: Cow<'static, str>, + source_map: Cow<'static, str>, + ) { + self.data.insert(specifier, source_map); + } + + pub fn get(&self, specifier: &str) -> Option<&Cow<'static, str>> { + self.data.get(specifier) + } +} + pub struct DenoCompileModuleData<'a> { pub specifier: &'a Url, pub media_type: MediaType, @@ -280,6 +362,13 @@ impl<'a> DenoCompileModuleData<'a> { } } +pub struct RemoteModuleEntry<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub data: Cow<'static, [u8]>, + pub transpiled_data: Option>, +} + enum RemoteModulesStoreSpecifierValue { Data(usize), Redirect(Url), @@ -291,7 +380,7 @@ pub struct RemoteModulesStore { } impl RemoteModulesStore { - fn build(data: &'static [u8]) -> Result { + fn build(input: &'static [u8]) -> Result<(&'static [u8], Self), AnyError> { fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> { let (input, specifier) = read_string_lossy(input)?; let specifier = Url::parse(&specifier)?; @@ -334,12 +423,16 @@ impl RemoteModulesStore { Ok((input, specifiers)) } - let (files_data, specifiers) = read_headers(data)?; + let (input, specifiers) = read_headers(input)?; + let (input, files_data) = read_bytes_with_u64_len(input)?; - Ok(Self { - specifiers, - files_data, - }) + Ok(( + input, + Self { + specifiers, + files_data, + }, + )) } pub fn resolve_specifier<'a>( @@ -370,7 +463,7 @@ impl RemoteModulesStore { pub fn read<'a>( &'a self, original_specifier: &'a Url, - ) -> Result>, AnyError> { + ) -> Result>, AnyError> { let mut count = 0; let mut specifier = original_specifier; loop { @@ -386,12 +479,25 @@ impl RemoteModulesStore { let input = &self.files_data[*offset..]; let (input, media_type_byte) = read_bytes(input, 1)?; let media_type = deserialize_media_type(media_type_byte[0])?; - let (input, len) = read_u64(input)?; - let (_input, data) = read_bytes(input, len as usize)?; - return Ok(Some(DenoCompileModuleData { + let (input, data) = read_bytes_with_u32_len(input)?; + check_has_len(input, 1)?; + let (input, has_transpiled) = (&input[1..], input[0]); + let (_, transpiled_data) = match has_transpiled { + 0 => (input, None), + 1 => { + let (input, data) = read_bytes_with_u32_len(input)?; + (input, Some(data)) + } + value => bail!( + "Invalid transpiled data flag: {}. Compiled data is corrupt.", + value + ), + }; + return Ok(Some(RemoteModuleEntry { specifier, media_type, data: Cow::Borrowed(data), + transpiled_data: transpiled_data.map(Cow::Borrowed), })); } None => { @@ -630,14 +736,32 @@ fn parse_vec_n_times_with_index( Ok((input, results)) } +fn read_bytes_with_u64_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { + let (input, len) = read_u64(input)?; + let (input, data) = read_bytes(input, len as usize)?; + Ok((input, data)) +} + +fn read_bytes_with_u32_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { + let (input, len) = read_u32_as_usize(input)?; + let (input, data) = read_bytes(input, len)?; + Ok((input, data)) +} + fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> { - if input.len() < len { - bail!("Unexpected end of data.",); - } + check_has_len(input, len)?; let (len_bytes, input) = input.split_at(len); Ok((input, len_bytes)) } +#[inline(always)] +fn check_has_len(input: &[u8], len: usize) -> Result<(), AnyError> { + if input.len() < len { + bail!("Unexpected end of data."); + } + Ok(()) +} + fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow), AnyError> { let (input, str_len) = read_u32_as_usize(input)?; let (input, data_bytes) = read_bytes(input, str_len)?; diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 04e66d680e..522fe47dd9 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -67,7 +67,7 @@ impl WindowsSystemRootablePath { #[derive(Debug)] pub struct BuiltVfs { pub root_path: WindowsSystemRootablePath, - pub root: VirtualDirectory, + pub entries: VirtualDirectoryEntries, pub files: Vec>, } @@ -95,7 +95,7 @@ impl VfsBuilder { Self { executable_root: VirtualDirectory { name: "/".to_string(), - entries: Vec::new(), + entries: Default::default(), }, files: Vec::new(), current_offset: 0, @@ -208,23 +208,20 @@ impl VfsBuilder { continue; } let name = component.as_os_str().to_string_lossy(); - let index = match current_dir - .entries - .binary_search_by(|e| e.name().cmp(&name)) - { + let index = match current_dir.entries.binary_search(&name) { Ok(index) => index, Err(insert_index) => { - current_dir.entries.insert( + current_dir.entries.0.insert( insert_index, VfsEntry::Dir(VirtualDirectory { name: name.to_string(), - entries: Vec::new(), + entries: Default::default(), }), ); insert_index } }; - match &mut current_dir.entries[index] { + match &mut current_dir.entries.0[index] { VfsEntry::Dir(dir) => { current_dir = dir; } @@ -248,14 +245,8 @@ impl VfsBuilder { continue; } let name = component.as_os_str().to_string_lossy(); - let index = match current_dir - .entries - .binary_search_by(|e| e.name().cmp(&name)) - { - Ok(index) => index, - Err(_) => return None, - }; - match &mut current_dir.entries[index] { + let entry = current_dir.entries.get_mut_by_name(&name)?; + match entry { VfsEntry::Dir(dir) => { current_dir = dir; } @@ -320,9 +311,9 @@ impl VfsBuilder { offset, len: data.len() as u64, }; - match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { + match dir.entries.binary_search(&name) { Ok(index) => { - let entry = &mut dir.entries[index]; + let entry = &mut dir.entries.0[index]; match entry { VfsEntry::File(virtual_file) => match sub_data_kind { VfsFileSubDataKind::Raw => { @@ -336,7 +327,7 @@ impl VfsBuilder { } } Err(insert_index) => { - dir.entries.insert( + dir.entries.0.insert( insert_index, VfsEntry::File(VirtualFile { name: name.to_string(), @@ -384,10 +375,10 @@ impl VfsBuilder { let target = normalize_path(path.parent().unwrap().join(&target)); let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); - match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { + match dir.entries.binary_search(&name) { Ok(_) => {} // previously inserted Err(insert_index) => { - dir.entries.insert( + dir.entries.0.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), @@ -426,7 +417,7 @@ impl VfsBuilder { dir: &mut VirtualDirectory, parts: &[String], ) { - for entry in &mut dir.entries { + for entry in &mut dir.entries.0 { match entry { VfsEntry::Dir(dir) => { strip_prefix_from_symlinks(dir, parts); @@ -454,13 +445,13 @@ impl VfsBuilder { if self.min_root_dir.as_ref() == Some(¤t_path) { break; } - match ¤t_dir.entries[0] { + match ¤t_dir.entries.0[0] { VfsEntry::Dir(dir) => { if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { // special directory we want to maintain break; } - match current_dir.entries.remove(0) { + match current_dir.entries.0.remove(0) { VfsEntry::Dir(dir) => { current_path = WindowsSystemRootablePath::Path(current_path.join(&dir.name)); @@ -480,7 +471,7 @@ impl VfsBuilder { } BuiltVfs { root_path: current_path, - root: current_dir, + entries: current_dir.entries, files: self.files, } } @@ -506,7 +497,7 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { return; // no need to compute if won't output } - if vfs.root.entries.is_empty() { + if vfs.entries.is_empty() { return; // nothing to output } @@ -696,7 +687,7 @@ fn vfs_as_display_tree( fn dir_size(dir: &VirtualDirectory, seen_offsets: &mut HashSet) -> Size { let mut size = Size::default(); - for entry in &dir.entries { + for entry in dir.entries.iter() { match entry { VfsEntry::Dir(virtual_directory) => { size = size + dir_size(virtual_directory, seen_offsets); @@ -760,15 +751,10 @@ fn vfs_as_display_tree( fn include_all_entries<'a>( dir_path: &WindowsSystemRootablePath, - vfs_dir: &'a VirtualDirectory, + entries: &'a VirtualDirectoryEntries, seen_offsets: &mut HashSet, ) -> Vec> { - if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { - return show_global_node_modules_dir(vfs_dir, seen_offsets); - } - - vfs_dir - .entries + entries .iter() .map(|entry| DirEntryOutput { name: Cow::Borrowed(entry.name()), @@ -826,10 +812,12 @@ fn vfs_as_display_tree( } else { EntryOutput::Subset(children) } + } else if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + EntryOutput::Subset(show_global_node_modules_dir(vfs_dir, seen_offsets)) } else { EntryOutput::Subset(include_all_entries( &WindowsSystemRootablePath::Path(dir), - vfs_dir, + &vfs_dir.entries, seen_offsets, )) } @@ -839,7 +827,7 @@ fn vfs_as_display_tree( // user might not have context about what's being shown let mut seen_offsets = HashSet::with_capacity(vfs.files.len()); let mut child_entries = - include_all_entries(&vfs.root_path, &vfs.root, &mut seen_offsets); + include_all_entries(&vfs.root_path, &vfs.entries, &mut seen_offsets); for child_entry in &mut child_entries { child_entry.collapse_leaf_nodes(); } @@ -961,27 +949,70 @@ impl VfsEntry { } } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct VirtualDirectoryEntries(Vec); + +impl VirtualDirectoryEntries { + pub fn new(mut entries: Vec) -> Self { + // needs to be sorted by name + entries.sort_by(|a, b| a.name().cmp(b.name())); + Self(entries) + } + + pub fn take_inner(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get_by_name(&self, name: &str) -> Option<&VfsEntry> { + self.binary_search(name).ok().map(|index| &self.0[index]) + } + + pub fn get_mut_by_name(&mut self, name: &str) -> Option<&mut VfsEntry> { + self + .binary_search(name) + .ok() + .map(|index| &mut self.0[index]) + } + + pub fn binary_search(&self, name: &str) -> Result { + self.0.binary_search_by(|e| e.name().cmp(name)) + } + + pub fn insert(&mut self, entry: VfsEntry) { + match self.binary_search(entry.name()) { + Ok(index) => { + self.0[index] = entry; + } + Err(insert_index) => { + self.0.insert(insert_index, entry); + } + } + } + + pub fn remove(&mut self, index: usize) -> VfsEntry { + self.0.remove(index) + } + + pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> { + self.0.iter() + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct VirtualDirectory { #[serde(rename = "n")] pub name: String, // should be sorted by name #[serde(rename = "e")] - pub entries: Vec, -} - -impl VirtualDirectory { - pub fn insert_entry(&mut self, entry: VfsEntry) { - let name = entry.name(); - match self.entries.binary_search_by(|e| e.name().cmp(name)) { - Ok(index) => { - self.entries[index] = entry; - } - Err(insert_index) => { - self.entries.insert(insert_index, entry); - } - } - } + pub entries: VirtualDirectoryEntries, } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -1136,20 +1167,13 @@ impl VfsRoot { } }; let component = component.to_string_lossy(); - match current_dir + current_entry = current_dir .entries - .binary_search_by(|e| e.name().cmp(&component)) - { - Ok(index) => { - current_entry = current_dir.entries[index].as_ref(); - } - Err(_) => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "path not found", - )); - } - } + .get_by_name(&component) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "path not found") + })? + .as_ref(); } Ok((final_path, current_entry)) @@ -1706,7 +1730,10 @@ mod test { FileBackedVfs::new( Cow::Owned(data), VfsRoot { - dir: vfs.root, + dir: VirtualDirectory { + name: "".to_string(), + entries: vfs.entries, + }, root_path: dest_path.to_path_buf(), start_file_offset: 0, }, diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index a69e873ab4..a715233933 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -2,7 +2,6 @@ use deno_core::serde_json; use test_util as util; -use util::assert_contains; use util::assert_not_contains; use util::testdata_path; use util::TestContext; @@ -90,78 +89,6 @@ fn standalone_args() { .assert_exit_code(0); } -#[test] -fn standalone_error() { - let context = TestContextBuilder::new().build(); - let dir = context.temp_dir(); - let exe = if cfg!(windows) { - dir.path().join("error.exe") - } else { - dir.path().join("error") - }; - context - .new_command() - .args_vec([ - "compile", - "--output", - &exe.to_string_lossy(), - "./compile/standalone_error.ts", - ]) - .run() - .skip_output_check() - .assert_exit_code(0); - - let output = context.new_command().name(&exe).split_output().run(); - output.assert_exit_code(1); - output.assert_stdout_matches_text(""); - let stderr = output.stderr(); - // On Windows, we cannot assert the file path (because '\'). - // Instead we just check for relevant output. - assert_contains!(stderr, "error: Uncaught (in promise) Error: boom!"); - assert_contains!(stderr, "\n at boom (file://"); - assert_contains!(stderr, "standalone_error.ts:2:9"); - assert_contains!(stderr, "at foo (file://"); - assert_contains!(stderr, "standalone_error.ts:5:3"); - assert_contains!(stderr, "standalone_error.ts:7:1"); -} - -#[test] -fn standalone_error_module_with_imports() { - let context = TestContextBuilder::new().build(); - let dir = context.temp_dir(); - let exe = if cfg!(windows) { - dir.path().join("error.exe") - } else { - dir.path().join("error") - }; - context - .new_command() - .args_vec([ - "compile", - "--output", - &exe.to_string_lossy(), - "./compile/standalone_error_module_with_imports_1.ts", - ]) - .run() - .skip_output_check() - .assert_exit_code(0); - - let output = context - .new_command() - .name(&exe) - .env("NO_COLOR", "1") - .split_output() - .run(); - output.assert_stdout_matches_text("hello\n"); - let stderr = output.stderr(); - // On Windows, we cannot assert the file path (because '\'). - // Instead we just check for relevant output. - assert_contains!(stderr, "error: Uncaught (in promise) Error: boom!"); - assert_contains!(stderr, "\n at file://"); - assert_contains!(stderr, "standalone_error_module_with_imports_2.ts:2:7"); - output.assert_exit_code(1); -} - #[test] fn standalone_load_datauri() { let context = TestContextBuilder::new().build(); diff --git a/tests/specs/compile/code_cache/__test__.jsonc b/tests/specs/compile/code_cache/__test__.jsonc index 72353e27da..f1c3461adc 100644 --- a/tests/specs/compile/code_cache/__test__.jsonc +++ b/tests/specs/compile/code_cache/__test__.jsonc @@ -1,6 +1,9 @@ { "tempDir": true, "steps": [{ + "args": "run -A cleanup.ts", + "output": "[WILDCARD]" + }, { "if": "unix", "args": "compile --output using_code_cache --log-level=debug main.ts", "output": "[WILDCARD]" diff --git a/tests/specs/compile/code_cache/cleanup.ts b/tests/specs/compile/code_cache/cleanup.ts new file mode 100644 index 0000000000..d9e7c805f8 --- /dev/null +++ b/tests/specs/compile/code_cache/cleanup.ts @@ -0,0 +1,11 @@ +import { tmpdir } from "node:os"; + +// cleanup the code cache file from a previous run +try { + if (Deno.build.os === "windows") { + Deno.removeSync(tmpdir() + "\\deno-compile-using_code_cache.exe.cache"); + } else { + Deno.removeSync(tmpdir() + "\\deno-compile-using_code_cache.cache"); + } +} catch { +} diff --git a/tests/specs/compile/determinism/__test__.jsonc b/tests/specs/compile/determinism/__test__.jsonc index 97045744f1..b84a1fdf18 100644 --- a/tests/specs/compile/determinism/__test__.jsonc +++ b/tests/specs/compile/determinism/__test__.jsonc @@ -1,28 +1,31 @@ { "tempDir": true, "steps": [{ - "if": "unix", - "args": "compile --output main1 main.ts", + "args": "run -A setup.ts", "output": "[WILDCARD]" }, { "if": "unix", - "args": "compile --output main2 main.ts", + "args": "compile --no-config --output a/main a/main.ts", "output": "[WILDCARD]" }, { "if": "unix", - "args": "run --allow-read=. assert_equal.ts main1 main2", + "args": "compile --no-config --output b/main b/main.ts", + "output": "[WILDCARD]" + }, { + "if": "unix", + "args": "run --allow-read=. assert_equal.ts a/main b/main", "output": "Same\n" }, { "if": "windows", - "args": "compile --output main1.exe main.ts", + "args": "compile --no-config --output a/main.exe a/main.ts", "output": "[WILDCARD]" }, { "if": "windows", - "args": "compile --output main2.exe main.ts", + "args": "compile --no-config --output b/main.exe b/main.ts", "output": "[WILDCARD]" }, { "if": "windows", - "args": "run --allow-read=. assert_equal.ts main1.exe main2.exe", + "args": "run --allow-read=. assert_equal.ts a/main.exe b/main.exe", "output": "Same\n" }] } diff --git a/tests/specs/compile/determinism/setup.ts b/tests/specs/compile/determinism/setup.ts new file mode 100644 index 0000000000..8bb5753079 --- /dev/null +++ b/tests/specs/compile/determinism/setup.ts @@ -0,0 +1,10 @@ +// for setup, we create two directories with the same file in each +// and then when compiling we ensure this directory name has no +// effect on the output +makeCopyDir("a"); +makeCopyDir("b"); + +function makeCopyDir(dirName) { + Deno.mkdirSync(dirName); + Deno.copyFileSync("main.ts", `${dirName}/main.ts`); +} diff --git a/tests/specs/compile/error/local/__test__.jsonc b/tests/specs/compile/error/local/__test__.jsonc new file mode 100644 index 0000000000..8d6a015a51 --- /dev/null +++ b/tests/specs/compile/error/local/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main standalone_error.ts", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 1 + }, { + "if": "windows", + "args": "compile --output main.exe standalone_error.ts", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 1 + }] +} diff --git a/tests/specs/compile/error/local/output.out b/tests/specs/compile/error/local/output.out new file mode 100644 index 0000000000..b346734ae6 --- /dev/null +++ b/tests/specs/compile/error/local/output.out @@ -0,0 +1,6 @@ +error: Uncaught (in promise) Error: boom! + throw new Error("boom!"); + ^ + at boom (file:///[WILDLINE]standalone_error.ts:2:9) + at foo (file:///[WILDLINE]standalone_error.ts:6:3) + at file:///[WILDLINE]standalone_error.ts:9:1 diff --git a/tests/testdata/compile/standalone_error.ts b/tests/specs/compile/error/local/standalone_error.ts similarity index 100% rename from tests/testdata/compile/standalone_error.ts rename to tests/specs/compile/error/local/standalone_error.ts diff --git a/tests/specs/compile/error/remote/__test__.jsonc b/tests/specs/compile/error/remote/__test__.jsonc new file mode 100644 index 0000000000..9ad9091ec6 --- /dev/null +++ b/tests/specs/compile/error/remote/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 1 + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 1 + }] +} diff --git a/tests/specs/compile/error/remote/main.ts b/tests/specs/compile/error/remote/main.ts new file mode 100644 index 0000000000..7a27276dd8 --- /dev/null +++ b/tests/specs/compile/error/remote/main.ts @@ -0,0 +1 @@ +import "http://localhost:4545/compile/standalone_error_module_with_imports_1.ts"; diff --git a/tests/specs/compile/error/remote/output.out b/tests/specs/compile/error/remote/output.out new file mode 100644 index 0000000000..3e23694c16 --- /dev/null +++ b/tests/specs/compile/error/remote/output.out @@ -0,0 +1,5 @@ +hello +error: Uncaught (in promise) Error: boom! +throw new Error(value); + ^ + at http://localhost:4545/compile/standalone_error_module_with_imports_2.ts:7:7 diff --git a/tests/testdata/compile/standalone_error_module_with_imports_2.ts b/tests/testdata/compile/standalone_error_module_with_imports_2.ts index ef052b512e..c83d7ceea6 100644 --- a/tests/testdata/compile/standalone_error_module_with_imports_2.ts +++ b/tests/testdata/compile/standalone_error_module_with_imports_2.ts @@ -1,2 +1,7 @@ +// file has blank lines to make the input line +// different than the output console.log("hello"); -throw new Error("boom!"); + +const value: string = "boom!"; + +throw new Error(value); From 351e79642a7cbceec5f5e97f41529207fc08796a Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 19 Dec 2024 19:10:58 +0100 Subject: [PATCH 08/20] fix(task): support tasks without commands (#27191) Support running tasks that have no command and only dependencies. This is useful for when you want to group tasks only. --- Cargo.lock | 4 +-- Cargo.toml | 2 +- cli/lsp/language_server.rs | 2 +- cli/lsp/lsp_custom.rs | 2 +- cli/schemas/config-file.v1.json | 1 - cli/tools/task.rs | 28 +++++++++++++------ tests/specs/task/dependencies/__test__.jsonc | 12 ++++++++ tests/specs/task/dependencies/no_command.out | 5 ++++ .../task/dependencies/no_command/deno.json | 13 +++++++++ .../task/dependencies/no_command_list.out | 7 +++++ 10 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 tests/specs/task/dependencies/no_command.out create mode 100644 tests/specs/task/dependencies/no_command/deno.json create mode 100644 tests/specs/task/dependencies/no_command_list.out diff --git a/Cargo.lock b/Cargo.lock index 4dddd53d75..05d3ef429b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1463,9 +1463,9 @@ dependencies = [ [[package]] name = "deno_config" -version = "0.39.3" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce717af3fe6788dae63965d58d5637fd62be8fe4f345f189137ffc06c51837d2" +checksum = "459b0bf193d2f9a177d18064a4888062ba0716312f56dbda8f1319444a8b2544" dependencies = [ "anyhow", "deno_package_json", diff --git a/Cargo.toml b/Cargo.toml index 63a6824e88..ccdf64986e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.326.0" } deno_bench_util = { version = "0.178.0", path = "./bench_util" } -deno_config = { version = "=0.39.3", features = ["workspace", "sync"] } +deno_config = { version = "=0.40.0", features = ["workspace", "sync"] } deno_lockfile = "=0.23.2" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.26.0" diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 3ffe4491e0..bd461833c2 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -3793,7 +3793,7 @@ impl Inner { for (name, command) in scripts { result.push(TaskDefinition { name: name.clone(), - command: command.clone(), + command: Some(command.clone()), source_uri: url_to_uri(&package_json.specifier()) .map_err(|_| LspError::internal_error())?, }); diff --git a/cli/lsp/lsp_custom.rs b/cli/lsp/lsp_custom.rs index 74c6aca88b..8df4ba1d07 100644 --- a/cli/lsp/lsp_custom.rs +++ b/cli/lsp/lsp_custom.rs @@ -14,7 +14,7 @@ pub const LATEST_DIAGNOSTIC_BATCH_INDEX: &str = #[serde(rename_all = "camelCase")] pub struct TaskDefinition { pub name: String, - pub command: String, + pub command: Option, pub source_uri: lsp::Uri, } diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index 1e3abb2c0d..d644072f4c 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -446,7 +446,6 @@ }, "command": { "type": "string", - "required": true, "description": "The task to execute" }, "dependencies": { diff --git a/cli/tools/task.rs b/cli/tools/task.rs index ec9b238847..740b0ce843 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -231,7 +231,7 @@ pub async fn execute_script( &Url::from_directory_path(cli_options.initial_cwd()).unwrap(), "", &TaskDefinition { - command: task_flags.task.as_ref().unwrap().to_string(), + command: Some(task_flags.task.as_ref().unwrap().to_string()), dependencies: vec![], description: None, }, @@ -448,6 +448,16 @@ impl<'a> TaskRunner<'a> { kill_signal: KillSignal, argv: &'a [String], ) -> Result { + let Some(command) = &definition.command else { + log::info!( + "{} {} {}", + colors::green("Task"), + colors::cyan(task_name), + colors::gray("(no command)") + ); + return Ok(0); + }; + if let Some(npm_resolver) = self.npm_resolver.as_managed() { npm_resolver.ensure_top_level_package_json_install().await?; npm_resolver @@ -469,7 +479,7 @@ impl<'a> TaskRunner<'a> { self .run_single(RunSingleOptions { task_name, - script: &definition.command, + script: command, cwd: &cwd, custom_commands, kill_signal, @@ -837,7 +847,7 @@ fn print_available_tasks( is_deno: false, name: name.to_string(), task: deno_config::deno_json::TaskDefinition { - command: script.to_string(), + command: Some(script.to_string()), dependencies: vec![], description: None, }, @@ -873,11 +883,13 @@ fn print_available_tasks( )?; } } - writeln!( - writer, - " {}", - strip_ansi_codes_and_escape_control_chars(&desc.task.command) - )?; + if let Some(command) = &desc.task.command { + writeln!( + writer, + " {}", + strip_ansi_codes_and_escape_control_chars(command) + )?; + }; if !desc.task.dependencies.is_empty() { let dependencies = desc .task diff --git a/tests/specs/task/dependencies/__test__.jsonc b/tests/specs/task/dependencies/__test__.jsonc index 84c98f11a4..c9032153b3 100644 --- a/tests/specs/task/dependencies/__test__.jsonc +++ b/tests/specs/task/dependencies/__test__.jsonc @@ -61,6 +61,18 @@ "cwd": "arg_task_with_deps", "args": "task a a", "output": "./arg_task_with_deps.out" + }, + "no_command": { + "cwd": "no_command", + "args": "task a", + "output": "./no_command.out", + "exitCode": 0 + }, + "no_command_list": { + "cwd": "no_command", + "args": "task", + "output": "./no_command_list.out", + "exitCode": 0 } } } diff --git a/tests/specs/task/dependencies/no_command.out b/tests/specs/task/dependencies/no_command.out new file mode 100644 index 0000000000..521b3541df --- /dev/null +++ b/tests/specs/task/dependencies/no_command.out @@ -0,0 +1,5 @@ +Task b echo 'b' +b +Task c echo 'c' +c +Task a (no command) diff --git a/tests/specs/task/dependencies/no_command/deno.json b/tests/specs/task/dependencies/no_command/deno.json new file mode 100644 index 0000000000..5588365a92 --- /dev/null +++ b/tests/specs/task/dependencies/no_command/deno.json @@ -0,0 +1,13 @@ +{ + "tasks": { + "a": { + "dependencies": ["b", "c"] + }, + "b": { + "command": "echo 'b'" + }, + "c": { + "command": "echo 'c'" + } + } +} diff --git a/tests/specs/task/dependencies/no_command_list.out b/tests/specs/task/dependencies/no_command_list.out new file mode 100644 index 0000000000..3d58c1cb06 --- /dev/null +++ b/tests/specs/task/dependencies/no_command_list.out @@ -0,0 +1,7 @@ +Available tasks: +- a + depends on: b, c +- b + echo 'b' +- c + echo 'c' From 3c147d6be1845743ef7b83386b3ef4f9c40938b1 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 19 Dec 2024 13:53:33 -0500 Subject: [PATCH 09/20] fix(publish): infer literal types in const contexts (#27425) * https://github.com/denoland/deno_graph/pull/555 --- Cargo.lock | 5 +++-- cli/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05d3ef429b..965b7a4150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1718,12 +1718,13 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.86.3" +version = "0.86.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc78ed0b4bbcb4197300f0d6e7d1edc2d2c5019cdb9dedba7ff229158441885b" +checksum = "f8502f5dd37f522c76e92961ec6e855f40bc351d50d62bc0752cc19517eed8ec" dependencies = [ "anyhow", "async-trait", + "capacity_builder", "data-url", "deno_ast", "deno_semver", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 70708002c7..4b0380717b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -74,7 +74,7 @@ deno_config.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] } deno_error.workspace = true -deno_graph = { version = "=0.86.3" } +deno_graph = { version = "=0.86.4" } deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true From feb94d09e714b345a41d77b72db61afc0f2d68d3 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Fri, 20 Dec 2024 02:33:35 +0000 Subject: [PATCH 10/20] fix(lsp): rewrite imports for 'Move to a new file' action (#27427) --- cli/lsp/analysis.rs | 16 +++-- cli/lsp/documents.rs | 7 ++ cli/lsp/language_server.rs | 44 +++++-------- tests/integration/lsp_tests.rs | 113 +++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 33 deletions(-) diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 9f9cf14864..4c6a20927b 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -603,18 +603,24 @@ fn try_reverse_map_package_json_exports( /// For a set of tsc changes, can them for any that contain something that looks /// like an import and rewrite the import specifier to include the extension pub fn fix_ts_import_changes( - referrer: &ModuleSpecifier, - resolution_mode: ResolutionMode, changes: &[tsc::FileTextChanges], language_server: &language_server::Inner, ) -> Result, AnyError> { - let import_mapper = language_server.get_ts_response_import_mapper(referrer); let mut r = Vec::new(); for change in changes { + let Ok(referrer) = ModuleSpecifier::parse(&change.file_name) else { + continue; + }; + let referrer_doc = language_server.get_asset_or_document(&referrer).ok(); + let resolution_mode = referrer_doc + .as_ref() + .map(|d| d.resolution_mode()) + .unwrap_or(ResolutionMode::Import); + let import_mapper = + language_server.get_ts_response_import_mapper(&referrer); let mut text_changes = Vec::new(); for text_change in &change.text_changes { let lines = text_change.new_text.split('\n'); - let new_lines: Vec = lines .map(|line| { // This assumes that there's only one import per line. @@ -622,7 +628,7 @@ pub fn fix_ts_import_changes( let specifier = captures.iter().skip(1).find_map(|s| s).unwrap().as_str(); if let Some(new_specifier) = import_mapper - .check_unresolved_specifier(specifier, referrer, resolution_mode) + .check_unresolved_specifier(specifier, &referrer, resolution_mode) { line.replace(specifier, &new_specifier) } else { diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index bdb64c9da3..d15cfe5a6c 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -251,6 +251,13 @@ impl AssetOrDocument { pub fn document_lsp_version(&self) -> Option { self.document().and_then(|d| d.maybe_lsp_version()) } + + pub fn resolution_mode(&self) -> ResolutionMode { + match self { + AssetOrDocument::Asset(_) => ResolutionMode::Import, + AssetOrDocument::Document(d) => d.resolution_mode(), + } + } } type ModuleResult = Result; diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index bd461833c2..a7a0a59743 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -1855,20 +1855,12 @@ impl Inner { } let changes = if code_action_data.fix_id == "fixMissingImport" { - fix_ts_import_changes( - &code_action_data.specifier, - maybe_asset_or_doc - .as_ref() - .and_then(|d| d.document()) - .map(|d| d.resolution_mode()) - .unwrap_or(ResolutionMode::Import), - &combined_code_actions.changes, - self, - ) - .map_err(|err| { - error!("Unable to remap changes: {:#}", err); - LspError::internal_error() - })? + fix_ts_import_changes(&combined_code_actions.changes, self).map_err( + |err| { + error!("Unable to remap changes: {:#}", err); + LspError::internal_error() + }, + )? } else { combined_code_actions.changes }; @@ -1912,20 +1904,16 @@ impl Inner { asset_or_doc.scope().cloned(), ) .await?; - if kind_suffix == ".rewrite.function.returnType" { - refactor_edit_info.edits = fix_ts_import_changes( - &action_data.specifier, - asset_or_doc - .document() - .map(|d| d.resolution_mode()) - .unwrap_or(ResolutionMode::Import), - &refactor_edit_info.edits, - self, - ) - .map_err(|err| { - error!("Unable to remap changes: {:#}", err); - LspError::internal_error() - })? + if kind_suffix == ".rewrite.function.returnType" + || kind_suffix == ".move.newFile" + { + refactor_edit_info.edits = + fix_ts_import_changes(&refactor_edit_info.edits, self).map_err( + |err| { + error!("Unable to remap changes: {:#}", err); + LspError::internal_error() + }, + )? } code_action.edit = refactor_edit_info.to_workspace_edit(self)?; code_action diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 56c060d958..825cef6247 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -6066,6 +6066,119 @@ fn lsp_jsr_code_action_missing_declaration() { ); } +#[test] +fn lsp_jsr_code_action_move_to_new_file() { + let context = TestContextBuilder::new() + .use_http_server() + .use_temp_cwd() + .build(); + let temp_dir = context.temp_dir(); + let file = source_file( + temp_dir.path().join("file.ts"), + r#" + import { someFunction } from "jsr:@denotest/types-file"; + export const someValue = someFunction(); + "#, + ); + let mut client = context.new_lsp_command().build(); + client.initialize_default(); + client.write_request( + "workspace/executeCommand", + json!({ + "command": "deno.cache", + "arguments": [[], file.url()], + }), + ); + client.did_open_file(&file); + let list = client + .write_request_with_res_as::>( + "textDocument/codeAction", + json!({ + "textDocument": { "uri": file.url() }, + "range": { + "start": { "line": 2, "character": 19 }, + "end": { "line": 2, "character": 28 }, + }, + "context": { "diagnostics": [] }, + }), + ) + .unwrap(); + let action = list + .iter() + .find_map(|c| match c { + lsp::CodeActionOrCommand::CodeAction(a) + if &a.title == "Move to a new file" => + { + Some(a) + } + _ => None, + }) + .unwrap(); + let res = client.write_request("codeAction/resolve", json!(action)); + assert_eq!( + res, + json!({ + "title": "Move to a new file", + "kind": "refactor.move.newFile", + "edit": { + "documentChanges": [ + { + "textDocument": { "uri": file.url(), "version": 1 }, + "edits": [ + { + "range": { + "start": { "line": 1, "character": 6 }, + "end": { "line": 2, "character": 0 }, + }, + "newText": "", + }, + { + "range": { + "start": { "line": 2, "character": 0 }, + "end": { "line": 3, "character": 4 }, + }, + "newText": "", + }, + ], + }, + { + "kind": "create", + "uri": file.url().join("someValue.ts").unwrap(), + "options": { + "ignoreIfExists": true, + }, + }, + { + "textDocument": { + "uri": file.url().join("someValue.ts").unwrap(), + "version": null, + }, + "edits": [ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 0 }, + }, + "newText": "import { someFunction } from \"jsr:@denotest/types-file\";\n\nexport const someValue = someFunction();\n", + }, + ], + }, + ], + }, + "isPreferred": false, + "data": { + "specifier": file.url(), + "range": { + "start": { "line": 2, "character": 19 }, + "end": { "line": 2, "character": 28 }, + }, + "refactorName": "Move to a new file", + "actionName": "Move to a new file", + }, + }), + ); +} + #[test] fn lsp_code_actions_deno_cache_npm() { let context = TestContextBuilder::new().use_temp_cwd().build(); From 23f7032d56630c3fcd8da9e226992fa61d3d9db6 Mon Sep 17 00:00:00 2001 From: Filip Stevanovic <62512535+filipstev@users.noreply.github.com> Date: Fri, 20 Dec 2024 05:23:51 +0100 Subject: [PATCH 11/20] fix(ext/node): add `truncate` method to the `FileHandle` class (#27389) --- ext/node/polyfills/_fs/_fs_ftruncate.ts | 18 ++++-- ext/node/polyfills/internal/fs/handle.ts | 25 +++----- tests/unit_node/_fs/_fs_handle_test.ts | 82 ++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/ext/node/polyfills/_fs/_fs_ftruncate.ts b/ext/node/polyfills/_fs/_fs_ftruncate.ts index 92af46f521..79320137f9 100644 --- a/ext/node/polyfills/_fs/_fs_ftruncate.ts +++ b/ext/node/polyfills/_fs/_fs_ftruncate.ts @@ -16,16 +16,24 @@ export function ftruncate( : undefined; const callback: CallbackWithError = typeof lenOrCallback === "function" ? lenOrCallback - : maybeCallback as CallbackWithError; + : (maybeCallback as CallbackWithError); if (!callback) throw new Error("No callback function supplied"); - new FsFile(fd, Symbol.for("Deno.internal.FsFile")).truncate(len).then( - () => callback(null), - callback, - ); + new FsFile(fd, Symbol.for("Deno.internal.FsFile")) + .truncate(len) + .then(() => callback(null), callback); } export function ftruncateSync(fd: number, len?: number) { new FsFile(fd, Symbol.for("Deno.internal.FsFile")).truncateSync(len); } + +export function ftruncatePromise(fd: number, len?: number): Promise { + return new Promise((resolve, reject) => { + ftruncate(fd, len, (err) => { + if (err) reject(err); + else resolve(); + }); + }); +} diff --git a/ext/node/polyfills/internal/fs/handle.ts b/ext/node/polyfills/internal/fs/handle.ts index 9ec0fc97e2..ee035f2f5c 100644 --- a/ext/node/polyfills/internal/fs/handle.ts +++ b/ext/node/polyfills/internal/fs/handle.ts @@ -13,6 +13,7 @@ import { ReadOptions, TextOptionsArgument, } from "ext:deno_node/_fs/_fs_common.ts"; +import { ftruncatePromise } from "ext:deno_node/_fs/_fs_ftruncate.ts"; import { core } from "ext:core/mod.js"; interface WriteResult { @@ -73,6 +74,10 @@ export class FileHandle extends EventEmitter { } } + truncate(len?: number): Promise { + return fsCall(ftruncatePromise, this, len); + } + readFile( opt?: TextOptionsArgument | BinaryOptionsArgument | FileOptionsArgument, ): Promise { @@ -85,11 +90,7 @@ export class FileHandle extends EventEmitter { length: number, position: number, ): Promise; - write( - str: string, - position: number, - encoding: string, - ): Promise; + write(str: string, position: number, encoding: string): Promise; write( bufferOrStr: Uint8Array | string, offsetOrPosition: number, @@ -120,16 +121,10 @@ export class FileHandle extends EventEmitter { const encoding = lengthOrEncoding; return new Promise((resolve, reject) => { - write( - this.fd, - str, - position, - encoding, - (err, bytesWritten, buffer) => { - if (err) reject(err); - else resolve({ buffer, bytesWritten }); - }, - ); + write(this.fd, str, position, encoding, (err, bytesWritten, buffer) => { + if (err) reject(err); + else resolve({ buffer, bytesWritten }); + }); }); } } diff --git a/tests/unit_node/_fs/_fs_handle_test.ts b/tests/unit_node/_fs/_fs_handle_test.ts index e26b82aa06..84d72c0745 100644 --- a/tests/unit_node/_fs/_fs_handle_test.ts +++ b/tests/unit_node/_fs/_fs_handle_test.ts @@ -117,3 +117,85 @@ Deno.test("[node/fs filehandle.writeFile] Write to file", async function () { assertEquals(decoder.decode(data), "hello world"); }); + +Deno.test( + "[node/fs filehandle.truncate] Truncate file with length", + async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "w+"); + + await fileHandle.writeFile("hello world"); + + await fileHandle.truncate(5); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(decoder.decode(data), "hello"); + }, +); + +Deno.test( + "[node/fs filehandle.truncate] Truncate file without length", + async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "w+"); + + await fileHandle.writeFile("hello world"); + + await fileHandle.truncate(); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(decoder.decode(data), ""); + }, +); + +Deno.test( + "[node/fs filehandle.truncate] Truncate file with extension", + async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "w+"); + + await fileHandle.writeFile("hi"); + + await fileHandle.truncate(5); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + const expected = new Uint8Array(5); + expected.set(new TextEncoder().encode("hi")); + + assertEquals(data, expected); + assertEquals(data.length, 5); + assertEquals(decoder.decode(data.subarray(0, 2)), "hi"); + // Verify null bytes + assertEquals(data[2], 0); + assertEquals(data[3], 0); + assertEquals(data[4], 0); + }, +); + +Deno.test( + "[node/fs filehandle.truncate] Truncate file with negative length", + async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "w+"); + + await fileHandle.writeFile("hello world"); + + await fileHandle.truncate(-1); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(decoder.decode(data), ""); + assertEquals(data.length, 0); + }, +); From 65b647909d75a586c087af3e10f54338e689e81c Mon Sep 17 00:00:00 2001 From: snek Date: Fri, 20 Dec 2024 13:48:48 +0100 Subject: [PATCH 12/20] feat(unstable): Implement QUIC (#21942) Implements a QUIC interface, loosely based on the WebTransport API (a future change could add the WebTransport API, built on top of this one). [quinn](https://docs.rs/quinn/latest/quinn/) is used for the underlying QUIC implementation, for a few reasons: - A cloneable "handle" api which fits quite nicely into deno resources. - Good collaboration with the rust ecosystem, especially rustls. - I like it. --- Cargo.lock | 53 ++- cli/tsc/99_main_compiler.js | 7 + ext/net/03_quic.js | 367 ++++++++++++++++ ext/net/Cargo.toml | 1 + ext/net/lib.deno_net.d.ts | 288 +++++++++++++ ext/net/lib.rs | 29 +- ext/net/quic.rs | 660 +++++++++++++++++++++++++++++ ext/web/06_streams.js | 8 +- runtime/fmt_errors.rs | 14 + runtime/js/90_deno_ns.js | 10 + tests/integration/js_unit_tests.rs | 1 + tests/unit/quic_test.ts | 172 ++++++++ 12 files changed, 1591 insertions(+), 19 deletions(-) create mode 100644 ext/net/03_quic.js create mode 100644 ext/net/quic.rs create mode 100644 tests/unit/quic_test.ts diff --git a/Cargo.lock b/Cargo.lock index 965b7a4150..6cd68e5ce1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -728,6 +728,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.37" @@ -1918,6 +1924,7 @@ dependencies = [ "hickory-proto", "hickory-resolver", "pin-project", + "quinn", "rustls-tokio-stream", "serde", "socket2", @@ -4025,9 +4032,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper 1.4.1", "hyper-util", @@ -5908,49 +5915,54 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 1.1.0", + "rustc-hash 2.0.0", "rustls", - "thiserror 1.0.64", + "socket2", + "thiserror 2.0.3", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash 2.0.0", "rustls", + "rustls-pki-types", "slab", - "thiserror 1.0.64", + "thiserror 2.0.3", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.2" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ + "cfg_aliases 0.2.1", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6427,6 +6439,9 @@ name = "rustls-pki-types" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +dependencies = [ + "web-time", +] [[package]] name = "rustls-tokio-stream" @@ -8523,6 +8538,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-root-certs" version = "0.26.6" @@ -8550,7 +8575,7 @@ dependencies = [ "arrayvec", "bit-vec", "bitflags 2.6.0", - "cfg_aliases", + "cfg_aliases 0.1.1", "codespan-reporting", "document-features", "indexmap 2.3.0", @@ -8582,7 +8607,7 @@ dependencies = [ "bit-set", "bitflags 2.6.0", "block", - "cfg_aliases", + "cfg_aliases 0.1.1", "core-graphics-types", "d3d12", "glow", diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 7e8a407cf9..f7862c95e4 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -41,6 +41,13 @@ delete Object.prototype.__proto__; "listen", "listenDatagram", "openKv", + "connectQuic", + "listenQuic", + "QuicBidirectionalStream", + "QuicConn", + "QuicListener", + "QuicReceiveStream", + "QuicSendStream", ]); const unstableMsgSuggestion = "If not, try changing the 'lib' compiler option to include 'deno.unstable' " + diff --git a/ext/net/03_quic.js b/ext/net/03_quic.js new file mode 100644 index 0000000000..e100e7bd64 --- /dev/null +++ b/ext/net/03_quic.js @@ -0,0 +1,367 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { core, primordials } from "ext:core/mod.js"; +import { + op_quic_accept, + op_quic_accept_bi, + op_quic_accept_incoming, + op_quic_accept_uni, + op_quic_close_connection, + op_quic_close_endpoint, + op_quic_connect, + op_quic_connection_closed, + op_quic_connection_get_protocol, + op_quic_connection_get_remote_addr, + op_quic_endpoint_get_addr, + op_quic_get_send_stream_priority, + op_quic_incoming_accept, + op_quic_incoming_ignore, + op_quic_incoming_local_ip, + op_quic_incoming_refuse, + op_quic_incoming_remote_addr, + op_quic_incoming_remote_addr_validated, + op_quic_listen, + op_quic_max_datagram_size, + op_quic_open_bi, + op_quic_open_uni, + op_quic_read_datagram, + op_quic_send_datagram, + op_quic_set_send_stream_priority, +} from "ext:core/ops"; +import { + getWritableStreamResourceBacking, + ReadableStream, + readableStreamForRid, + WritableStream, + writableStreamForRid, +} from "ext:deno_web/06_streams.js"; +import { loadTlsKeyPair } from "ext:deno_net/02_tls.js"; +const { + BadResourcePrototype, +} = core; +const { + Uint8Array, + TypedArrayPrototypeSubarray, + SymbolAsyncIterator, + SafePromisePrototypeFinally, + ObjectPrototypeIsPrototypeOf, +} = primordials; + +class QuicSendStream extends WritableStream { + get sendOrder() { + return op_quic_get_send_stream_priority( + getWritableStreamResourceBacking(this).rid, + ); + } + + set sendOrder(p) { + op_quic_set_send_stream_priority( + getWritableStreamResourceBacking(this).rid, + p, + ); + } +} + +class QuicReceiveStream extends ReadableStream {} + +function readableStream(rid, closed) { + // stream can be indirectly closed by closing connection. + SafePromisePrototypeFinally(closed, () => { + core.tryClose(rid); + }); + return readableStreamForRid(rid, true, QuicReceiveStream); +} + +function writableStream(rid, closed) { + // stream can be indirectly closed by closing connection. + SafePromisePrototypeFinally(closed, () => { + core.tryClose(rid); + }); + return writableStreamForRid(rid, true, QuicSendStream); +} + +class QuicBidirectionalStream { + #readable; + #writable; + + constructor(txRid, rxRid, closed) { + this.#readable = readableStream(rxRid, closed); + this.#writable = writableStream(txRid, closed); + } + + get readable() { + return this.#readable; + } + + get writable() { + return this.#writable; + } +} + +async function* bidiStream(conn, closed) { + try { + while (true) { + const r = await op_quic_accept_bi(conn); + yield new QuicBidirectionalStream(r[0], r[1], closed); + } + } catch (error) { + if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) { + return; + } + throw error; + } +} + +async function* uniStream(conn, closed) { + try { + while (true) { + const uniRid = await op_quic_accept_uni(conn); + yield readableStream(uniRid, closed); + } + } catch (error) { + if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) { + return; + } + throw error; + } +} + +class QuicConn { + #resource; + #bidiStream = null; + #uniStream = null; + #closed; + + constructor(resource) { + this.#resource = resource; + + this.#closed = op_quic_connection_closed(this.#resource); + core.unrefOpPromise(this.#closed); + } + + get protocol() { + return op_quic_connection_get_protocol(this.#resource); + } + + get remoteAddr() { + return op_quic_connection_get_remote_addr(this.#resource); + } + + async createBidirectionalStream( + { sendOrder, waitUntilAvailable } = { __proto__: null }, + ) { + const { 0: txRid, 1: rxRid } = await op_quic_open_bi( + this.#resource, + waitUntilAvailable ?? false, + ); + if (sendOrder !== null && sendOrder !== undefined) { + op_quic_set_send_stream_priority(txRid, sendOrder); + } + return new QuicBidirectionalStream(txRid, rxRid, this.#closed); + } + + async createUnidirectionalStream( + { sendOrder, waitUntilAvailable } = { __proto__: null }, + ) { + const rid = await op_quic_open_uni( + this.#resource, + waitUntilAvailable ?? false, + ); + if (sendOrder !== null && sendOrder !== undefined) { + op_quic_set_send_stream_priority(rid, sendOrder); + } + return writableStream(rid, this.#closed); + } + + get incomingBidirectionalStreams() { + if (this.#bidiStream === null) { + this.#bidiStream = ReadableStream.from( + bidiStream(this.#resource, this.#closed), + ); + } + return this.#bidiStream; + } + + get incomingUnidirectionalStreams() { + if (this.#uniStream === null) { + this.#uniStream = ReadableStream.from( + uniStream(this.#resource, this.#closed), + ); + } + return this.#uniStream; + } + + get maxDatagramSize() { + return op_quic_max_datagram_size(this.#resource); + } + + async readDatagram(p) { + const view = p || new Uint8Array(this.maxDatagramSize); + const nread = await op_quic_read_datagram(this.#resource, view); + return TypedArrayPrototypeSubarray(view, 0, nread); + } + + async sendDatagram(data) { + await op_quic_send_datagram(this.#resource, data); + } + + get closed() { + core.refOpPromise(this.#closed); + return this.#closed; + } + + close({ closeCode, reason }) { + op_quic_close_connection(this.#resource, closeCode, reason); + } +} + +class QuicIncoming { + #incoming; + + constructor(incoming) { + this.#incoming = incoming; + } + + get localIp() { + return op_quic_incoming_local_ip(this.#incoming); + } + + get remoteAddr() { + return op_quic_incoming_remote_addr(this.#incoming); + } + + get remoteAddressValidated() { + return op_quic_incoming_remote_addr_validated(this.#incoming); + } + + async accept() { + const conn = await op_quic_incoming_accept(this.#incoming); + return new QuicConn(conn); + } + + refuse() { + op_quic_incoming_refuse(this.#incoming); + } + + ignore() { + op_quic_incoming_ignore(this.#incoming); + } +} + +class QuicListener { + #endpoint; + + constructor(endpoint) { + this.#endpoint = endpoint; + } + + get addr() { + return op_quic_endpoint_get_addr(this.#endpoint); + } + + async accept() { + const conn = await op_quic_accept(this.#endpoint); + return new QuicConn(conn); + } + + async incoming() { + const incoming = await op_quic_accept_incoming(this.#endpoint); + return new QuicIncoming(incoming); + } + + async next() { + let conn; + try { + conn = await this.accept(); + } catch (error) { + if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) { + return { value: undefined, done: true }; + } + throw error; + } + return { value: conn, done: false }; + } + + [SymbolAsyncIterator]() { + return this; + } + + close({ closeCode, reason }) { + op_quic_close_endpoint(this.#endpoint, closeCode, reason); + } +} + +async function listenQuic( + { + hostname, + port, + cert, + key, + alpnProtocols, + keepAliveInterval, + maxIdleTimeout, + maxConcurrentBidirectionalStreams, + maxConcurrentUnidirectionalStreams, + }, +) { + hostname = hostname || "0.0.0.0"; + const keyPair = loadTlsKeyPair("Deno.listenQuic", { cert, key }); + const endpoint = await op_quic_listen( + { hostname, port }, + { alpnProtocols }, + { + keepAliveInterval, + maxIdleTimeout, + maxConcurrentBidirectionalStreams, + maxConcurrentUnidirectionalStreams, + }, + keyPair, + ); + return new QuicListener(endpoint); +} + +async function connectQuic( + { + hostname, + port, + serverName, + caCerts, + cert, + key, + alpnProtocols, + keepAliveInterval, + maxIdleTimeout, + maxConcurrentBidirectionalStreams, + maxConcurrentUnidirectionalStreams, + congestionControl, + }, +) { + const keyPair = loadTlsKeyPair("Deno.connectQuic", { cert, key }); + const conn = await op_quic_connect( + { hostname, port }, + { + caCerts, + alpnProtocols, + serverName, + }, + { + keepAliveInterval, + maxIdleTimeout, + maxConcurrentBidirectionalStreams, + maxConcurrentUnidirectionalStreams, + congestionControl, + }, + keyPair, + ); + return new QuicConn(conn); +} + +export { + connectQuic, + listenQuic, + QuicBidirectionalStream, + QuicConn, + QuicIncoming, + QuicListener, + QuicReceiveStream, + QuicSendStream, +}; diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index 8dbb0be391..eaee7bfb4b 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -20,6 +20,7 @@ deno_tls.workspace = true hickory-proto = "0.25.0-alpha.4" hickory-resolver.workspace = true pin-project.workspace = true +quinn = { version = "0.11.6", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } rustls-tokio-stream.workspace = true serde.workspace = true socket2.workspace = true diff --git a/ext/net/lib.deno_net.d.ts b/ext/net/lib.deno_net.d.ts index 827081f2a4..958474cbbd 100644 --- a/ext/net/lib.deno_net.d.ts +++ b/ext/net/lib.deno_net.d.ts @@ -450,5 +450,293 @@ declare namespace Deno { options?: StartTlsOptions, ): Promise; + /** + * **UNSTABLE**: New API, yet to be vetted. + * @experimental + * @category Network + */ + export interface QuicTransportOptions { + /** Period of inactivity before sending a keep-alive packet. Keep-alive + * packets prevent an inactive but otherwise healthy connection from timing + * out. Only one side of any given connection needs keep-alive enabled for + * the connection to be preserved. + * @default {undefined} + */ + keepAliveInterval?: number; + /** Maximum duration of inactivity to accept before timing out the + * connection. The true idle timeout is the minimum of this and the peer’s + * own max idle timeout. + * @default {undefined} + */ + maxIdleTimeout?: number; + /** Maximum number of incoming bidirectional streams that may be open + * concurrently. + * @default {100} + */ + maxConcurrentBidirectionalStreams?: number; + /** Maximum number of incoming unidirectional streams that may be open + * concurrently. + * @default {100} + */ + maxConcurrentUnidirectionalStreams?: number; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * @experimental + * @category Network + */ + export interface ListenQuicOptions extends QuicTransportOptions { + /** The port to connect to. */ + port: number; + /** + * A literal IP address or host name that can be resolved to an IP address. + * @default {"0.0.0.0"} + */ + hostname?: string; + /** Server private key in PEM format */ + key: string; + /** Cert chain in PEM format */ + cert: string; + /** Application-Layer Protocol Negotiation (ALPN) protocols to announce to + * the client. QUIC requires the use of ALPN. + */ + alpnProtocols: string[]; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * Listen announces on the local transport address over QUIC. + * + * ```ts + * const lstnr = await Deno.listenQuic({ port: 443, cert: "...", key: "...", alpnProtocols: ["h3"] }); + * ``` + * + * Requires `allow-net` permission. + * + * @experimental + * @tags allow-net + * @category Network + */ + export function listenQuic(options: ListenQuicOptions): Promise; + + /** + * **UNSTABLE**: New API, yet to be vetted. + * @experimental + * @category Network + */ + export interface ConnectQuicOptions extends QuicTransportOptions { + /** The port to connect to. */ + port: number; + /** A literal IP address or host name that can be resolved to an IP address. */ + hostname: string; + /** The name used for validating the certificate provided by the server. If + * not provided, defaults to `hostname`. */ + serverName?: string | undefined; + /** Application-Layer Protocol Negotiation (ALPN) protocols supported by + * the client. QUIC requires the use of ALPN. + */ + alpnProtocols: string[]; + /** A list of root certificates that will be used in addition to the + * default root certificates to verify the peer's certificate. + * + * Must be in PEM format. */ + caCerts?: string[]; + /** + * The congestion control algorithm used when sending data over this connection. + */ + congestionControl?: "throughput" | "low-latency"; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * Establishes a secure connection over QUIC using a hostname and port. The + * cert file is optional and if not included Mozilla's root certificates will + * be used. See also https://github.com/ctz/webpki-roots for specifics. + * + * ```ts + * const caCert = await Deno.readTextFile("./certs/my_custom_root_CA.pem"); + * const conn1 = await Deno.connectQuic({ hostname: "example.com", port: 443, alpnProtocols: ["h3"] }); + * const conn2 = await Deno.connectQuic({ caCerts: [caCert], hostname: "example.com", port: 443, alpnProtocols: ["h3"] }); + * ``` + * + * Requires `allow-net` permission. + * + * @experimental + * @tags allow-net + * @category Network + */ + export function connectQuic(options: ConnectQuicOptions): Promise; + + /** + * **UNSTABLE**: New API, yet to be vetted. + * @experimental + * @category Network + */ + export interface QuicCloseInfo { + /** A number representing the error code for the error. */ + closeCode: number; + /** A string representing the reason for closing the connection. */ + reason: string; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * An incoming connection for which the server has not yet begun its part of the handshake. + * + * @experimental + * @category Network + */ + export interface QuicIncoming { + /** + * The local IP address which was used when the peer established the connection. + */ + readonly localIp: string; + + /** + * The peer’s UDP address. + */ + readonly remoteAddr: NetAddr; + + /** + * Whether the socket address that is initiating this connection has proven that they can receive traffic. + */ + readonly remoteAddressValidated: boolean; + + /** + * Accept this incoming connection. + */ + accept(): Promise; + + /** + * Refuse this incoming connection. + */ + refuse(): void; + + /** + * Ignore this incoming connection attempt, not sending any packet in response. + */ + ignore(): void; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * Specialized listener that accepts QUIC connections. + * + * @experimental + * @category Network + */ + export interface QuicListener extends AsyncIterable { + /** Return the address of the `QuicListener`. */ + readonly addr: NetAddr; + + /** Waits for and resolves to the next connection to the `QuicListener`. */ + accept(): Promise; + + /** Waits for and resolves to the next incoming request to the `QuicListener`. */ + incoming(): Promise; + + /** Close closes the listener. Any pending accept promises will be rejected + * with errors. */ + close(info: QuicCloseInfo): void; + + [Symbol.asyncIterator](): AsyncIterableIterator; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * @experimental + * @category Network + */ + export interface QuicSendStreamOptions { + /** Indicates the send priority of this stream relative to other streams for + * which the value has been set. + * @default {undefined} + */ + sendOrder?: number; + /** Wait until there is sufficient flow credit to create the stream. + * @default {false} + */ + waitUntilAvailable?: boolean; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * @experimental + * @category Network + */ + export interface QuicConn { + /** Close closes the listener. Any pending accept promises will be rejected + * with errors. */ + close(info: QuicCloseInfo): void; + /** Opens and returns a bidirectional stream. */ + createBidirectionalStream( + options?: QuicSendStreamOptions, + ): Promise; + /** Opens and returns a unidirectional stream. */ + createUnidirectionalStream( + options?: QuicSendStreamOptions, + ): Promise; + /** Send a datagram. The provided data cannot be larger than + * `maxDatagramSize`. */ + sendDatagram(data: Uint8Array): Promise; + /** Receive a datagram. If no buffer is provider, one will be allocated. + * The size of the provided buffer should be at least `maxDatagramSize`. */ + readDatagram(buffer?: Uint8Array): Promise; + + /** Return the remote address for the connection. Clients may change + * addresses at will, for example when switching to a cellular internet + * connection. + */ + readonly remoteAddr: NetAddr; + /** The negotiated ALPN protocol, if provided. */ + readonly protocol: string | undefined; + /** Returns a promise that resolves when the connection is closed. */ + readonly closed: Promise; + /** A stream of bidirectional streams opened by the peer. */ + readonly incomingBidirectionalStreams: ReadableStream< + QuicBidirectionalStream + >; + /** A stream of unidirectional streams opened by the peer. */ + readonly incomingUnidirectionalStreams: ReadableStream; + /** Returns the datagram stream for sending and receiving datagrams. */ + readonly maxDatagramSize: number; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * @experimental + * @category Network + */ + export interface QuicBidirectionalStream { + /** Returns a QuicReceiveStream instance that can be used to read incoming data. */ + readonly readable: QuicReceiveStream; + /** Returns a QuicSendStream instance that can be used to write outgoing data. */ + readonly writable: QuicSendStream; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * @experimental + * @category Network + */ + export interface QuicSendStream extends WritableStream { + /** Indicates the send priority of this stream relative to other streams for + * which the value has been set. */ + sendOrder: number; + } + + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * @experimental + * @category Network + */ + export interface QuicReceiveStream extends ReadableStream {} + export {}; // only export exports } diff --git a/ext/net/lib.rs b/ext/net/lib.rs index f482750b38..04b3f80010 100644 --- a/ext/net/lib.rs +++ b/ext/net/lib.rs @@ -5,6 +5,7 @@ pub mod ops; pub mod ops_tls; #[cfg(unix)] pub mod ops_unix; +mod quic; pub mod raw; pub mod resolve_addr; pub mod tcp; @@ -158,8 +159,34 @@ deno_core::extension!(deno_net, ops_unix::op_node_unstable_net_listen_unixpacket

, ops_unix::op_net_recv_unixpacket, ops_unix::op_net_send_unixpacket

, + + quic::op_quic_accept, + quic::op_quic_accept_bi, + quic::op_quic_accept_incoming, + quic::op_quic_accept_uni, + quic::op_quic_close_connection, + quic::op_quic_close_endpoint, + quic::op_quic_connection_closed, + quic::op_quic_connection_get_protocol, + quic::op_quic_connection_get_remote_addr, + quic::op_quic_connect

, + quic::op_quic_endpoint_get_addr, + quic::op_quic_get_send_stream_priority, + quic::op_quic_incoming_accept, + quic::op_quic_incoming_refuse, + quic::op_quic_incoming_ignore, + quic::op_quic_incoming_local_ip, + quic::op_quic_incoming_remote_addr, + quic::op_quic_incoming_remote_addr_validated, + quic::op_quic_listen

, + quic::op_quic_max_datagram_size, + quic::op_quic_open_bi, + quic::op_quic_open_uni, + quic::op_quic_read_datagram, + quic::op_quic_send_datagram, + quic::op_quic_set_send_stream_priority, ], - esm = [ "01_net.js", "02_tls.js" ], + esm = [ "01_net.js", "02_tls.js", "03_quic.js" ], options = { root_cert_store_provider: Option>, unsafely_ignore_certificate_errors: Option>, diff --git a/ext/net/quic.rs b/ext/net/quic.rs new file mode 100644 index 0000000000..16f68364be --- /dev/null +++ b/ext/net/quic.rs @@ -0,0 +1,660 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use crate::resolve_addr::resolve_addr; +use crate::DefaultTlsOptions; +use crate::NetPermissions; +use crate::UnsafelyIgnoreCertificateErrors; +use deno_core::error::bad_resource; +use deno_core::error::generic_error; +use deno_core::error::AnyError; +use deno_core::futures::task::noop_waker_ref; +use deno_core::op2; +use deno_core::AsyncRefCell; +use deno_core::AsyncResult; +use deno_core::BufView; +use deno_core::GarbageCollected; +use deno_core::JsBuffer; +use deno_core::OpState; +use deno_core::RcRef; +use deno_core::Resource; +use deno_core::ResourceId; +use deno_core::WriteOutcome; +use deno_tls::create_client_config; +use deno_tls::SocketUse; +use deno_tls::TlsKeys; +use deno_tls::TlsKeysHolder; +use quinn::crypto::rustls::QuicClientConfig; +use quinn::crypto::rustls::QuicServerConfig; +use serde::Deserialize; +use serde::Serialize; +use std::borrow::Cow; +use std::cell::RefCell; +use std::future::Future; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; +use std::net::SocketAddrV4; +use std::net::SocketAddrV6; +use std::pin::pin; +use std::rc::Rc; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; +use std::time::Duration; + +#[derive(Debug, Deserialize, Serialize)] +struct Addr { + hostname: String, + port: u16, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ListenArgs { + alpn_protocols: Option>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct TransportConfig { + keep_alive_interval: Option, + max_idle_timeout: Option, + max_concurrent_bidirectional_streams: Option, + max_concurrent_unidirectional_streams: Option, + preferred_address_v4: Option, + preferred_address_v6: Option, + congestion_control: Option, +} + +impl TryInto for TransportConfig { + type Error = AnyError; + + fn try_into(self) -> Result { + let mut cfg = quinn::TransportConfig::default(); + + if let Some(interval) = self.keep_alive_interval { + cfg.keep_alive_interval(Some(Duration::from_millis(interval))); + } + + if let Some(timeout) = self.max_idle_timeout { + cfg.max_idle_timeout(Some(Duration::from_millis(timeout).try_into()?)); + } + + if let Some(max) = self.max_concurrent_bidirectional_streams { + cfg.max_concurrent_bidi_streams(max.into()); + } + + if let Some(max) = self.max_concurrent_unidirectional_streams { + cfg.max_concurrent_uni_streams(max.into()); + } + + if let Some(v) = self.congestion_control { + let controller: Option< + Arc, + > = match v.as_str() { + "low-latency" => { + Some(Arc::new(quinn::congestion::BbrConfig::default())) + } + "throughput" => { + Some(Arc::new(quinn::congestion::CubicConfig::default())) + } + _ => None, + }; + if let Some(controller) = controller { + cfg.congestion_controller_factory(controller); + } + } + + Ok(cfg) + } +} + +struct EndpointResource(quinn::Endpoint, Arc); + +impl GarbageCollected for EndpointResource {} + +#[op2(async)] +#[cppgc] +pub(crate) async fn op_quic_listen( + state: Rc>, + #[serde] addr: Addr, + #[serde] args: ListenArgs, + #[serde] transport_config: TransportConfig, + #[cppgc] keys: &TlsKeysHolder, +) -> Result +where + NP: NetPermissions + 'static, +{ + state + .borrow_mut() + .borrow_mut::() + .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listenQuic()")?; + + let addr = resolve_addr(&addr.hostname, addr.port) + .await? + .next() + .ok_or_else(|| generic_error("No resolved address found"))?; + + let TlsKeys::Static(deno_tls::TlsKey(cert, key)) = keys.take() else { + unreachable!() + }; + + let mut crypto = + quinn::rustls::ServerConfig::builder_with_protocol_versions(&[ + &quinn::rustls::version::TLS13, + ]) + .with_no_client_auth() + .with_single_cert(cert.clone(), key.clone_key())?; + + if let Some(alpn_protocols) = args.alpn_protocols { + crypto.alpn_protocols = alpn_protocols + .into_iter() + .map(|alpn| alpn.into_bytes()) + .collect(); + } + + let server_config = Arc::new(QuicServerConfig::try_from(crypto)?); + let mut config = quinn::ServerConfig::with_crypto(server_config.clone()); + config.preferred_address_v4(transport_config.preferred_address_v4); + config.preferred_address_v6(transport_config.preferred_address_v6); + config.transport_config(Arc::new(transport_config.try_into()?)); + let endpoint = quinn::Endpoint::server(config, addr)?; + + Ok(EndpointResource(endpoint, server_config)) +} + +#[op2] +#[serde] +pub(crate) fn op_quic_endpoint_get_addr( + #[cppgc] endpoint: &EndpointResource, +) -> Result { + let addr = endpoint.0.local_addr()?; + let addr = Addr { + hostname: format!("{}", addr.ip()), + port: addr.port(), + }; + Ok(addr) +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CloseInfo { + close_code: u64, + reason: String, +} + +#[op2(fast)] +pub(crate) fn op_quic_close_endpoint( + #[cppgc] endpoint: &EndpointResource, + #[bigint] close_code: u64, + #[string] reason: String, +) -> Result<(), AnyError> { + endpoint + .0 + .close(quinn::VarInt::from_u64(close_code)?, reason.as_bytes()); + Ok(()) +} + +struct ConnectionResource(quinn::Connection); + +impl GarbageCollected for ConnectionResource {} + +#[op2(async)] +#[cppgc] +pub(crate) async fn op_quic_accept( + #[cppgc] endpoint: &EndpointResource, +) -> Result { + match endpoint.0.accept().await { + Some(incoming) => { + let conn = incoming.accept()?.await?; + Ok(ConnectionResource(conn)) + } + None => Err(bad_resource("QuicListener is closed")), + } +} + +struct IncomingResource( + RefCell>, + Arc, +); + +impl GarbageCollected for IncomingResource {} + +#[op2(async)] +#[cppgc] +pub(crate) async fn op_quic_accept_incoming( + #[cppgc] endpoint: &EndpointResource, +) -> Result { + match endpoint.0.accept().await { + Some(incoming) => Ok(IncomingResource( + RefCell::new(Some(incoming)), + endpoint.1.clone(), + )), + None => Err(bad_resource("QuicListener is closed")), + } +} + +#[op2] +#[string] +pub(crate) fn op_quic_incoming_local_ip( + #[cppgc] incoming_resource: &IncomingResource, +) -> Result, AnyError> { + let Some(incoming) = incoming_resource.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + Ok(incoming.local_ip().map(|ip| ip.to_string())) +} + +#[op2] +#[serde] +pub(crate) fn op_quic_incoming_remote_addr( + #[cppgc] incoming_resource: &IncomingResource, +) -> Result { + let Some(incoming) = incoming_resource.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + let addr = incoming.remote_address(); + Ok(Addr { + hostname: format!("{}", addr.ip()), + port: addr.port(), + }) +} + +#[op2(fast)] +pub(crate) fn op_quic_incoming_remote_addr_validated( + #[cppgc] incoming_resource: &IncomingResource, +) -> Result { + let Some(incoming) = incoming_resource.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + Ok(incoming.remote_address_validated()) +} + +#[op2(async)] +#[cppgc] +pub(crate) async fn op_quic_incoming_accept( + #[cppgc] incoming_resource: &IncomingResource, + #[serde] transport_config: Option, +) -> Result { + let Some(incoming) = incoming_resource.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + let conn = match transport_config { + Some(transport_config) => { + let mut config = + quinn::ServerConfig::with_crypto(incoming_resource.1.clone()); + config.preferred_address_v4(transport_config.preferred_address_v4); + config.preferred_address_v6(transport_config.preferred_address_v6); + config.transport_config(Arc::new(transport_config.try_into()?)); + incoming.accept_with(Arc::new(config))?.await? + } + None => incoming.accept()?.await?, + }; + Ok(ConnectionResource(conn)) +} + +#[op2] +#[serde] +pub(crate) fn op_quic_incoming_refuse( + #[cppgc] incoming: &IncomingResource, +) -> Result<(), AnyError> { + let Some(incoming) = incoming.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + incoming.refuse(); + Ok(()) +} + +#[op2] +#[serde] +pub(crate) fn op_quic_incoming_ignore( + #[cppgc] incoming: &IncomingResource, +) -> Result<(), AnyError> { + let Some(incoming) = incoming.0.borrow_mut().take() else { + return Err(bad_resource("QuicIncoming already used")); + }; + incoming.ignore(); + Ok(()) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ConnectArgs { + ca_certs: Option>, + alpn_protocols: Option>, + server_name: Option, +} + +#[op2(async)] +#[cppgc] +pub(crate) async fn op_quic_connect( + state: Rc>, + #[serde] addr: Addr, + #[serde] args: ConnectArgs, + #[serde] transport_config: TransportConfig, + #[cppgc] key_pair: &TlsKeysHolder, +) -> Result +where + NP: NetPermissions + 'static, +{ + state + .borrow_mut() + .borrow_mut::() + .check_net(&(&addr.hostname, Some(addr.port)), "Deno.connectQuic()")?; + + let sock_addr = resolve_addr(&addr.hostname, addr.port) + .await? + .next() + .ok_or_else(|| generic_error("No resolved address found"))?; + + let root_cert_store = state + .borrow() + .borrow::() + .root_cert_store()?; + + let unsafely_ignore_certificate_errors = state + .borrow() + .try_borrow::() + .and_then(|it| it.0.clone()); + + let ca_certs = args + .ca_certs + .unwrap_or_default() + .into_iter() + .map(|s| s.into_bytes()) + .collect::>(); + + let mut tls_config = create_client_config( + root_cert_store, + ca_certs, + unsafely_ignore_certificate_errors, + key_pair.take(), + SocketUse::GeneralSsl, + )?; + + if let Some(alpn_protocols) = args.alpn_protocols { + tls_config.alpn_protocols = + alpn_protocols.into_iter().map(|s| s.into_bytes()).collect(); + } + + let client_config = QuicClientConfig::try_from(tls_config)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(client_config)); + client_config.transport_config(Arc::new(transport_config.try_into()?)); + + let local_addr = match sock_addr.ip() { + IpAddr::V4(_) => IpAddr::from(Ipv4Addr::new(0, 0, 0, 0)), + IpAddr::V6(_) => IpAddr::from(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), + }; + + let conn = quinn::Endpoint::client((local_addr, 0).into())? + .connect_with( + client_config, + sock_addr, + &args.server_name.unwrap_or(addr.hostname), + )? + .await?; + + Ok(ConnectionResource(conn)) +} + +#[op2] +#[string] +pub(crate) fn op_quic_connection_get_protocol( + #[cppgc] connection: &ConnectionResource, +) -> Option { + connection + .0 + .handshake_data() + .and_then(|h| h.downcast::().ok()) + .and_then(|h| h.protocol) + .map(|p| String::from_utf8_lossy(&p).into_owned()) +} + +#[op2] +#[serde] +pub(crate) fn op_quic_connection_get_remote_addr( + #[cppgc] connection: &ConnectionResource, +) -> Result { + let addr = connection.0.remote_address(); + Ok(Addr { + hostname: format!("{}", addr.ip()), + port: addr.port(), + }) +} + +#[op2(fast)] +pub(crate) fn op_quic_close_connection( + #[cppgc] connection: &ConnectionResource, + #[bigint] close_code: u64, + #[string] reason: String, +) -> Result<(), AnyError> { + connection + .0 + .close(quinn::VarInt::from_u64(close_code)?, reason.as_bytes()); + Ok(()) +} + +#[op2(async)] +#[serde] +pub(crate) async fn op_quic_connection_closed( + #[cppgc] connection: &ConnectionResource, +) -> Result { + let e = connection.0.closed().await; + match e { + quinn::ConnectionError::LocallyClosed => Ok(CloseInfo { + close_code: 0, + reason: "".into(), + }), + quinn::ConnectionError::ApplicationClosed(i) => Ok(CloseInfo { + close_code: i.error_code.into(), + reason: String::from_utf8_lossy(&i.reason).into_owned(), + }), + e => Err(e.into()), + } +} + +struct SendStreamResource(AsyncRefCell); + +impl SendStreamResource { + fn new(stream: quinn::SendStream) -> Self { + Self(AsyncRefCell::new(stream)) + } +} + +impl Resource for SendStreamResource { + fn name(&self) -> Cow { + "quicSendStream".into() + } + + fn write(self: Rc, view: BufView) -> AsyncResult { + Box::pin(async move { + let mut r = RcRef::map(self, |r| &r.0).borrow_mut().await; + let nwritten = r.write(&view).await?; + Ok(WriteOutcome::Partial { nwritten, view }) + }) + } +} + +struct RecvStreamResource(AsyncRefCell); + +impl RecvStreamResource { + fn new(stream: quinn::RecvStream) -> Self { + Self(AsyncRefCell::new(stream)) + } +} + +impl Resource for RecvStreamResource { + fn name(&self) -> Cow { + "quicReceiveStream".into() + } + + fn read(self: Rc, limit: usize) -> AsyncResult { + Box::pin(async move { + let mut r = RcRef::map(self, |r| &r.0).borrow_mut().await; + let mut data = vec![0; limit]; + let nread = r.read(&mut data).await?.unwrap_or(0); + data.truncate(nread); + Ok(BufView::from(data)) + }) + } +} + +#[op2(async)] +#[serde] +pub(crate) async fn op_quic_accept_bi( + #[cppgc] connection: &ConnectionResource, + state: Rc>, +) -> Result<(ResourceId, ResourceId), AnyError> { + match connection.0.accept_bi().await { + Ok((tx, rx)) => { + let mut state = state.borrow_mut(); + let tx_rid = state.resource_table.add(SendStreamResource::new(tx)); + let rx_rid = state.resource_table.add(RecvStreamResource::new(rx)); + Ok((tx_rid, rx_rid)) + } + Err(e) => match e { + quinn::ConnectionError::LocallyClosed + | quinn::ConnectionError::ApplicationClosed(..) => { + Err(bad_resource("QuicConn is closed")) + } + _ => Err(e.into()), + }, + } +} + +#[op2(async)] +#[serde] +pub(crate) async fn op_quic_open_bi( + #[cppgc] connection: &ConnectionResource, + state: Rc>, + wait_for_available: bool, +) -> Result<(ResourceId, ResourceId), AnyError> { + let (tx, rx) = if wait_for_available { + connection.0.open_bi().await? + } else { + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + match pin!(connection.0.open_bi()).poll(&mut cx) { + Poll::Ready(r) => r?, + Poll::Pending => { + return Err(generic_error("Connection has reached the maximum number of outgoing concurrent bidirectional streams")); + } + } + }; + let mut state = state.borrow_mut(); + let tx_rid = state.resource_table.add(SendStreamResource::new(tx)); + let rx_rid = state.resource_table.add(RecvStreamResource::new(rx)); + Ok((tx_rid, rx_rid)) +} + +#[op2(async)] +#[serde] +pub(crate) async fn op_quic_accept_uni( + #[cppgc] connection: &ConnectionResource, + state: Rc>, +) -> Result { + match connection.0.accept_uni().await { + Ok(rx) => { + let rid = state + .borrow_mut() + .resource_table + .add(RecvStreamResource::new(rx)); + Ok(rid) + } + Err(e) => match e { + quinn::ConnectionError::LocallyClosed + | quinn::ConnectionError::ApplicationClosed(..) => { + Err(bad_resource("QuicConn is closed")) + } + _ => Err(e.into()), + }, + } +} + +#[op2(async)] +#[serde] +pub(crate) async fn op_quic_open_uni( + #[cppgc] connection: &ConnectionResource, + state: Rc>, + wait_for_available: bool, +) -> Result { + let tx = if wait_for_available { + connection.0.open_uni().await? + } else { + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + match pin!(connection.0.open_uni()).poll(&mut cx) { + Poll::Ready(r) => r?, + Poll::Pending => { + return Err(generic_error("Connection has reached the maximum number of outgoing concurrent unidirectional streams")); + } + } + }; + let rid = state + .borrow_mut() + .resource_table + .add(SendStreamResource::new(tx)); + Ok(rid) +} + +#[op2(async)] +pub(crate) async fn op_quic_send_datagram( + #[cppgc] connection: &ConnectionResource, + #[buffer] buf: JsBuffer, +) -> Result<(), AnyError> { + connection.0.send_datagram_wait(buf.to_vec().into()).await?; + Ok(()) +} + +#[op2(async)] +pub(crate) async fn op_quic_read_datagram( + #[cppgc] connection: &ConnectionResource, + #[buffer] mut buf: JsBuffer, +) -> Result { + let data = connection.0.read_datagram().await?; + buf[0..data.len()].copy_from_slice(&data); + Ok(data.len() as _) +} + +#[op2(fast)] +pub(crate) fn op_quic_max_datagram_size( + #[cppgc] connection: &ConnectionResource, +) -> Result { + Ok(connection.0.max_datagram_size().unwrap_or(0) as _) +} + +#[op2(fast)] +pub(crate) fn op_quic_get_send_stream_priority( + state: Rc>, + #[smi] rid: ResourceId, +) -> Result { + let resource = state + .borrow() + .resource_table + .get::(rid)?; + let r = RcRef::map(resource, |r| &r.0).try_borrow(); + match r { + Some(s) => Ok(s.priority()?), + None => Err(generic_error("Unable to get priority")), + } +} + +#[op2(fast)] +pub(crate) fn op_quic_set_send_stream_priority( + state: Rc>, + #[smi] rid: ResourceId, + priority: i32, +) -> Result<(), AnyError> { + let resource = state + .borrow() + .resource_table + .get::(rid)?; + let r = RcRef::map(resource, |r| &r.0).try_borrow(); + match r { + Some(s) => { + s.set_priority(priority)?; + Ok(()) + } + None => Err(generic_error("Unable to set priority")), + } +} diff --git a/ext/web/06_streams.js b/ext/web/06_streams.js index e673ee2bb4..f3ac711fc7 100644 --- a/ext/web/06_streams.js +++ b/ext/web/06_streams.js @@ -908,8 +908,8 @@ const _original = Symbol("[[original]]"); * @param {boolean=} autoClose If the resource should be auto-closed when the stream closes. Defaults to true. * @returns {ReadableStream} */ -function readableStreamForRid(rid, autoClose = true) { - const stream = new ReadableStream(_brand); +function readableStreamForRid(rid, autoClose = true, Super) { + const stream = new (Super ?? ReadableStream)(_brand); stream[_resourceBacking] = { rid, autoClose }; const tryClose = () => { @@ -1130,8 +1130,8 @@ async function readableStreamCollectIntoUint8Array(stream) { * @param {boolean=} autoClose If the resource should be auto-closed when the stream closes. Defaults to true. * @returns {ReadableStream} */ -function writableStreamForRid(rid, autoClose = true) { - const stream = new WritableStream(_brand); +function writableStreamForRid(rid, autoClose = true, Super) { + const stream = new (Super ?? WritableStream)(_brand); stream[_resourceBacking] = { rid, autoClose }; const tryClose = () => { diff --git a/runtime/fmt_errors.rs b/runtime/fmt_errors.rs index 6f120b5d46..3c60c3a3d7 100644 --- a/runtime/fmt_errors.rs +++ b/runtime/fmt_errors.rs @@ -422,6 +422,20 @@ fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { "Run again with `--unstable-webgpu` flag to enable this API.", ), ]; + } else if msg.contains("listenQuic is not a function") { + return vec![ + FixSuggestion::info("listenQuic is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-net` flag to enable this API.", + ), + ]; + } else if msg.contains("connectQuic is not a function") { + return vec![ + FixSuggestion::info("connectQuic is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-net` flag to enable this API.", + ), + ]; // Try to capture errors like: // ``` // Uncaught Error: Cannot find module '../build/Release/canvas.node' diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index a510ee33c4..5511649279 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -13,6 +13,7 @@ import * as console from "ext:deno_console/01_console.js"; import * as ffi from "ext:deno_ffi/00_ffi.js"; import * as net from "ext:deno_net/01_net.js"; import * as tls from "ext:deno_net/02_tls.js"; +import * as quic from "ext:deno_net/03_quic.js"; import * as serve from "ext:deno_http/00_serve.ts"; import * as http from "ext:deno_http/01_http.js"; import * as websocket from "ext:deno_http/02_websocket.ts"; @@ -174,6 +175,15 @@ denoNsUnstableById[unstableIds.net] = { op_net_listen_udp, op_net_listen_unixpacket, ), + + connectQuic: quic.connectQuic, + listenQuic: quic.listenQuic, + QuicBidirectionalStream: quic.QuicBidirectionalStream, + QuicConn: quic.QuicConn, + QuicListener: quic.QuicListener, + QuicReceiveStream: quic.QuicReceiveStream, + QuicSendStream: quic.QuicSendStream, + QuicIncoming: quic.QuicIncoming, }; // denoNsUnstableById[unstableIds.unsafeProto] = { __proto__: null } diff --git a/tests/integration/js_unit_tests.rs b/tests/integration/js_unit_tests.rs index 577ca043ca..717a8d8e7c 100644 --- a/tests/integration/js_unit_tests.rs +++ b/tests/integration/js_unit_tests.rs @@ -66,6 +66,7 @@ util::unit_test_factory!( process_test, progressevent_test, promise_hooks_test, + quic_test, read_dir_test, read_file_test, read_link_test, diff --git a/tests/unit/quic_test.ts b/tests/unit/quic_test.ts new file mode 100644 index 0000000000..f5423327de --- /dev/null +++ b/tests/unit/quic_test.ts @@ -0,0 +1,172 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "./test_util.ts"; + +const cert = Deno.readTextFileSync("tests/testdata/tls/localhost.crt"); +const key = Deno.readTextFileSync("tests/testdata/tls/localhost.key"); +const caCerts = [Deno.readTextFileSync("tests/testdata/tls/RootCA.pem")]; + +async function pair(opt?: Deno.QuicTransportOptions): Promise< + [Deno.QuicConn, Deno.QuicConn, Deno.QuicListener] +> { + const listener = await Deno.listenQuic({ + hostname: "localhost", + port: 0, + cert, + key, + alpnProtocols: ["deno-test"], + ...opt, + }); + + const [server, client] = await Promise.all([ + listener.accept(), + Deno.connectQuic({ + hostname: "localhost", + port: listener.addr.port, + caCerts, + alpnProtocols: ["deno-test"], + ...opt, + }), + ]); + + assertEquals(server.protocol, "deno-test"); + assertEquals(client.protocol, "deno-test"); + assertEquals(client.remoteAddr, listener.addr); + + return [server, client, listener]; +} + +Deno.test("bidirectional stream", async () => { + const [server, client, listener] = await pair(); + + const encoded = (new TextEncoder()).encode("hi!"); + + { + const bi = await server.createBidirectionalStream({ sendOrder: 42 }); + assertEquals(bi.writable.sendOrder, 42); + bi.writable.sendOrder = 0; + assertEquals(bi.writable.sendOrder, 0); + await bi.writable.getWriter().write(encoded); + } + + { + const { value: bi } = await client.incomingBidirectionalStreams + .getReader() + .read(); + const { value: data } = await bi!.readable.getReader().read(); + assertEquals(data, encoded); + } + + listener.close({ closeCode: 0, reason: "" }); + client.close({ closeCode: 0, reason: "" }); +}); + +Deno.test("unidirectional stream", async () => { + const [server, client, listener] = await pair(); + + const encoded = (new TextEncoder()).encode("hi!"); + + { + const uni = await server.createUnidirectionalStream({ sendOrder: 42 }); + assertEquals(uni.sendOrder, 42); + uni.sendOrder = 0; + assertEquals(uni.sendOrder, 0); + await uni.getWriter().write(encoded); + } + + { + const { value: uni } = await client.incomingUnidirectionalStreams + .getReader() + .read(); + const { value: data } = await uni!.getReader().read(); + assertEquals(data, encoded); + } + + listener.close({ closeCode: 0, reason: "" }); + client.close({ closeCode: 0, reason: "" }); +}); + +Deno.test("datagrams", async () => { + const [server, client, listener] = await pair(); + + const encoded = (new TextEncoder()).encode("hi!"); + + await server.sendDatagram(encoded); + + const data = await client.readDatagram(); + assertEquals(data, encoded); + + listener.close({ closeCode: 0, reason: "" }); + client.close({ closeCode: 0, reason: "" }); +}); + +Deno.test("closing", async () => { + const [server, client, listener] = await pair(); + + server.close({ closeCode: 42, reason: "hi!" }); + + assertEquals(await client.closed, { closeCode: 42, reason: "hi!" }); + + listener.close({ closeCode: 0, reason: "" }); +}); + +Deno.test("max concurrent streams", async () => { + const [server, client, listener] = await pair({ + maxConcurrentBidirectionalStreams: 1, + maxConcurrentUnidirectionalStreams: 1, + }); + + { + await server.createBidirectionalStream(); + await server.createBidirectionalStream() + .then(() => { + throw new Error("expected failure"); + }, () => { + // success! + }); + } + + { + await server.createUnidirectionalStream(); + await server.createUnidirectionalStream() + .then(() => { + throw new Error("expected failure"); + }, () => { + // success! + }); + } + + listener.close({ closeCode: 0, reason: "" }); + server.close({ closeCode: 0, reason: "" }); + client.close({ closeCode: 0, reason: "" }); +}); + +Deno.test("incoming", async () => { + const listener = await Deno.listenQuic({ + hostname: "localhost", + port: 0, + cert, + key, + alpnProtocols: ["deno-test"], + }); + + const connect = () => + Deno.connectQuic({ + hostname: "localhost", + port: listener.addr.port, + caCerts, + alpnProtocols: ["deno-test"], + }); + + const c1p = connect(); + const i1 = await listener.incoming(); + const server = await i1.accept(); + const client = await c1p; + + assertEquals(server.protocol, "deno-test"); + assertEquals(client.protocol, "deno-test"); + assertEquals(client.remoteAddr, listener.addr); + + listener.close({ closeCode: 0, reason: "" }); + client.close({ closeCode: 0, reason: "" }); +}); From c30f3450c64d6db07b2f8ec40a29f50835fa2108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 20 Dec 2024 17:43:03 +0000 Subject: [PATCH 13/20] perf: don't store duplicate info for ops in the snapshot (#27430) Mostly for changes from https://github.com/denoland/deno_core/pull/1010 --------- Co-authored-by: David Sherret --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- cli/module_loader.rs | 4 ++-- cli/standalone/binary.rs | 2 +- cli/standalone/mod.rs | 5 ++--- cli/standalone/serialization.rs | 19 +++++++++---------- cli/tools/coverage/mod.rs | 4 ++-- cli/util/text_encoding.rs | 12 ++++++------ 8 files changed, 29 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cd68e5ce1..90e857dc57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1500,9 +1500,9 @@ dependencies = [ [[package]] name = "deno_core" -version = "0.326.0" +version = "0.327.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed157162dc5320a2b46ffeeaec24788339df0f2437cfaea78a8d82696715ad7f" +checksum = "eaf8dff204b9c2415deb47b9f30d4d38b0925d0d88f1f9074e8e76f59e6d7ded" dependencies = [ "anyhow", "az", @@ -2074,9 +2074,9 @@ dependencies = [ [[package]] name = "deno_ops" -version = "0.202.0" +version = "0.203.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd8ac1af251e292388e516dd339b9a3b982a6d1e7f8644c08e34671ca39003c" +checksum = "b146ca74cac431843486ade58e2accc16c11315fb2c6934590a52a73c56b7ec3" dependencies = [ "proc-macro-rules", "proc-macro2", @@ -6725,9 +6725,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.235.0" +version = "0.236.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07afd8b67b4a442ecc2823038473ac0e9e5682de93c213323b60661afdd7eb4" +checksum = "e23b3abce64010612f88f4ff689a959736f99eb3dc0dbf1c7903434b8bd8cda5" dependencies = [ "num-bigint", "serde", diff --git a/Cargo.toml b/Cargo.toml index ccdf64986e..f290d1480b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ repository = "https://github.com/denoland/deno" [workspace.dependencies] deno_ast = { version = "=0.44.0", features = ["transpiling"] } -deno_core = { version = "0.326.0" } +deno_core = { version = "0.327.0" } deno_bench_util = { version = "0.178.0", path = "./bench_util" } deno_config = { version = "=0.40.0", features = ["workspace", "sync"] } diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 5e4ff875dc..3ba8753335 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -996,7 +996,7 @@ impl ModuleLoader std::future::ready(()).boxed_local() } - fn get_source_map(&self, file_name: &str) -> Option> { + fn get_source_map(&self, file_name: &str) -> Option> { let specifier = resolve_url(file_name).ok()?; match specifier.scheme() { // we should only be looking for emits for schemes that denote external @@ -1008,7 +1008,7 @@ impl ModuleLoader .0 .load_prepared_module_for_source_map_sync(&specifier) .ok()??; - source_map_from_code(source.code.as_bytes()) + source_map_from_code(source.code.as_bytes()).map(Cow::Owned) } fn get_source_mapped_source_line( diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index f7ffa46e4f..3707543eb0 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -746,7 +746,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { for (specifier, source_map) in source_maps { source_map_store.add( Cow::Owned(root_dir_url.specifier_key(specifier).into_owned()), - Cow::Owned(source_map), + Cow::Owned(source_map.into_bytes()), ); } diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index d7227f2bd5..75247a98c0 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -502,7 +502,7 @@ impl ModuleLoader for EmbeddedModuleLoader { std::future::ready(()).boxed_local() } - fn get_source_map(&self, file_name: &str) -> Option> { + fn get_source_map(&self, file_name: &str) -> Option> { if file_name.starts_with("file:///") { let url = deno_path_util::url_from_directory_path(self.shared.vfs.root()).ok()?; @@ -512,8 +512,7 @@ impl ModuleLoader for EmbeddedModuleLoader { } else { self.shared.source_maps.get(file_name) } - // todo(https://github.com/denoland/deno_core/pull/1007): don't clone - .map(|s| s.as_bytes().to_vec()) + .map(Cow::Borrowed) } fn get_source_mapped_source_line( diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index 8fe593c005..4c18e0eb53 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -86,7 +86,7 @@ pub fn serialize_binary_data_section( builder.append_le(specifier.len() as u32); builder.append(specifier); builder.append_le(source_map.len() as u32); - builder.append(source_map); + builder.append(source_map.as_ref()); } } @@ -124,9 +124,9 @@ pub fn deserialize_binary_data_section( #[allow(clippy::type_complexity)] fn read_source_map_entry( input: &[u8], - ) -> Result<(&[u8], (Cow, Cow)), AnyError> { + ) -> Result<(&[u8], (Cow, &[u8])), AnyError> { let (input, specifier) = read_string_lossy(input)?; - let (input, source_map) = read_string_lossy(input)?; + let (input, source_map) = read_bytes_with_u32_len(input)?; Ok((input, (specifier, source_map))) } @@ -164,7 +164,7 @@ pub fn deserialize_binary_data_section( let (current_input, (specifier, source_map)) = read_source_map_entry(input)?; input = current_input; - source_maps.add(specifier, source_map); + source_maps.add(specifier, Cow::Borrowed(source_map)); } // finally ensure we read the magic bytes at the end @@ -293,7 +293,7 @@ impl DenoCompileModuleSource { } pub struct SourceMapStore { - data: IndexMap, Cow<'static, str>>, + data: IndexMap, Cow<'static, [u8]>>, } impl SourceMapStore { @@ -306,13 +306,13 @@ impl SourceMapStore { pub fn add( &mut self, specifier: Cow<'static, str>, - source_map: Cow<'static, str>, + source_map: Cow<'static, [u8]>, ) { self.data.insert(specifier, source_map); } - pub fn get(&self, specifier: &str) -> Option<&Cow<'static, str>> { - self.data.get(specifier) + pub fn get(&self, specifier: &str) -> Option<&[u8]> { + self.data.get(specifier).map(|v| v.as_ref()) } } @@ -763,8 +763,7 @@ fn check_has_len(input: &[u8], len: usize) -> Result<(), AnyError> { } fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow), AnyError> { - let (input, str_len) = read_u32_as_usize(input)?; - let (input, data_bytes) = read_bytes(input, str_len)?; + let (input, data_bytes) = read_bytes_with_u32_len(input)?; Ok((input, String::from_utf8_lossy(data_bytes))) } diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 624fa76bf6..2b36a8ddd8 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -198,7 +198,7 @@ pub struct CoverageReport { fn generate_coverage_report( script_coverage: &cdp::ScriptCoverage, script_source: String, - maybe_source_map: &Option>, + maybe_source_map: Option<&[u8]>, output: &Option, ) -> CoverageReport { let maybe_source_map = maybe_source_map @@ -625,7 +625,7 @@ pub fn cover_files( let coverage_report = generate_coverage_report( &script_coverage, runtime_code.as_str().to_owned(), - &source_map, + source_map.as_deref(), &out_mode, ); diff --git a/cli/util/text_encoding.rs b/cli/util/text_encoding.rs index 06b311e150..107b78a213 100644 --- a/cli/util/text_encoding.rs +++ b/cli/util/text_encoding.rs @@ -140,23 +140,23 @@ mod tests { #[test] fn test_source_map_from_code() { let to_string = - |bytes: Vec| -> String { String::from_utf8(bytes).unwrap() }; + |bytes: Vec| -> String { String::from_utf8(bytes.to_vec()).unwrap() }; assert_eq!( source_map_from_code( - b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=", + b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=" ).map(to_string), Some("testingtesting".to_string()) ); assert_eq!( source_map_from_code( - b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n", + b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n" ).map(to_string), Some("testingtesting".to_string()) ); assert_eq!( source_map_from_code( - b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n", - ), + b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n" + ).map(to_string), None ); assert_eq!( @@ -164,7 +164,7 @@ mod tests { b"\"use strict\"; throw new Error(\"Hello world!\"); -//# sourceMappingURL=data:application/json;base64,{", +//# sourceMappingURL=data:application/json;base64,{" ), None ); From ece718eb3e77917b67a63c56c63271a4a1fb9d8e Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 20 Dec 2024 16:14:37 -0500 Subject: [PATCH 14/20] perf: upgrade to deno_semver 0.7 (#27426) --- Cargo.lock | 88 +++++++++++++++---- Cargo.toml | 12 +-- cli/Cargo.toml | 2 +- cli/args/mod.rs | 31 ++++--- cli/args/package_json.rs | 31 ++++--- cli/graph_util.rs | 3 +- cli/lsp/analysis.rs | 17 +++- cli/lsp/jsr.rs | 23 +++-- cli/lsp/search.rs | 4 +- cli/npm/managed/mod.rs | 6 +- cli/npm/managed/resolution.rs | 10 ++- .../managed/resolvers/common/bin_entries.rs | 4 +- cli/npm/managed/resolvers/local.rs | 15 ++-- cli/standalone/serialization.rs | 12 ++- cli/tools/doc.rs | 4 +- cli/tools/info.rs | 19 ++-- cli/tools/installer.rs | 4 +- cli/tools/registry/pm.rs | 33 ++++--- cli/tools/registry/pm/deps.rs | 30 +++---- cli/tools/registry/pm/outdated.rs | 3 +- cli/tools/upgrade.rs | 5 +- resolvers/deno/npm/byonm.rs | 23 ++--- resolvers/node/resolution.rs | 2 + resolvers/npm_cache/Cargo.toml | 1 + resolvers/npm_cache/lib.rs | 3 +- resolvers/npm_cache/registry_info.rs | 32 ++++++- resolvers/npm_cache/tarball_extract.rs | 2 +- runtime/permissions/lib.rs | 4 +- tests/integration/check_tests.rs | 4 +- tests/integration/jsr_tests.rs | 2 +- 30 files changed, 282 insertions(+), 147 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90e857dc57..4bb0cfb18e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -677,6 +677,28 @@ dependencies = [ "itoa", ] +[[package]] +name = "capacity_builder" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2d24a6dcf0cd402a21b65d35340f3a49ff3475dc5fdac91d22d2733e6641c6" +dependencies = [ + "capacity_builder_macros", + "ecow", + "hipstr", + "itoa", +] + +[[package]] +name = "capacity_builder_macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" +dependencies = [ + "quote", + "syn 2.0.87", +] + [[package]] name = "caseless" version = "0.2.1" @@ -1230,7 +1252,7 @@ dependencies = [ "boxed_error", "bytes", "cache_control", - "capacity_builder", + "capacity_builder 0.5.0", "chrono", "clap", "clap_complete", @@ -1469,9 +1491,9 @@ dependencies = [ [[package]] name = "deno_config" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459b0bf193d2f9a177d18064a4888062ba0716312f56dbda8f1319444a8b2544" +checksum = "8afa3beb6b9e0604cfe0380d30f88c5b758d44e228d5a5fc42ae637ccfb7d089" dependencies = [ "anyhow", "deno_package_json", @@ -1510,7 +1532,7 @@ dependencies = [ "bit-set", "bit-vec", "bytes", - "capacity_builder", + "capacity_builder 0.1.3", "cooked-waker", "deno_core_icudata", "deno_ops", @@ -1724,13 +1746,13 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.86.4" +version = "0.86.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8502f5dd37f522c76e92961ec6e855f40bc351d50d62bc0752cc19517eed8ec" +checksum = "f669d96d63841d9ba10f86b161d898678ce05bc1e3c9ee1c1f7449a68eed2b64" dependencies = [ "anyhow", "async-trait", - "capacity_builder", + "capacity_builder 0.5.0", "data-url", "deno_ast", "deno_semver", @@ -1865,9 +1887,9 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.23.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559c19feb00af0c34f0bd4a20e56e12463fafd5c5069d6005f3ce33008027eea" +checksum = "632e835a53ed667d62fdd766c5780fe8361c831d3e3fbf1a760a0b7896657587" dependencies = [ "deno_semver", "serde", @@ -2026,12 +2048,13 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f125a5dba7839c46394a0a9c835da9fe60f5f412587ab4956a76492a1cc6a8" +checksum = "5f818ad5dc4c206b50b5cfa6f10b4b94b127e15c8342c152768eba40c225ca23" dependencies = [ - "anyhow", "async-trait", + "capacity_builder 0.5.0", + "deno_error", "deno_lockfile", "deno_semver", "futures", @@ -2053,6 +2076,7 @@ dependencies = [ "boxed_error", "deno_cache_dir", "deno_core", + "deno_error", "deno_npm", "deno_semver", "deno_unsync", @@ -2090,10 +2114,11 @@ dependencies = [ [[package]] name = "deno_package_json" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b0a3d81c592624a1ae15332a04b4dc2b7c163ef1dfc7c60171f736d1babdf5" +checksum = "81d72db99fdebfc371d7be16972c18a47daa7a29cb5fbb3900ab2114b1f42d96" dependencies = [ + "boxed_error", "deno_error", "deno_path_util", "deno_semver", @@ -2120,7 +2145,7 @@ dependencies = [ name = "deno_permissions" version = "0.43.0" dependencies = [ - "capacity_builder", + "capacity_builder 0.5.0", "deno_core", "deno_path_util", "deno_terminal 0.2.0", @@ -2225,11 +2250,14 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d1259270d66a5e6d29bb75c9289656541874f79ae9ff6c9f1c790846d5c07ba" +checksum = "4775271f9b5602482698f76d24ea9ed8ba27af7f587a7e9a876916300c542435" dependencies = [ + "capacity_builder 0.5.0", "deno_error", + "ecow", + "hipstr", "monch", "once_cell", "serde", @@ -2896,6 +2924,15 @@ dependencies = [ "spki", ] +[[package]] +name = "ecow" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42fc0a93992b20c58b99e59d61eaf1635a25bfbe49e4275c34ba0aee98119ba" +dependencies = [ + "serde", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -3832,6 +3869,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "hipstr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97971ffc85d4c98de12e2608e992a43f5294ebb625fdb045b27c731b64c4c6d6" +dependencies = [ + "serde", + "serde_bytes", + "sptr", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -7003,6 +7051,12 @@ dependencies = [ "der", ] +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + [[package]] name = "sqlformat" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index f290d1480b..81c750f0af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,14 +51,14 @@ deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.327.0" } deno_bench_util = { version = "0.178.0", path = "./bench_util" } -deno_config = { version = "=0.40.0", features = ["workspace", "sync"] } -deno_lockfile = "=0.23.2" +deno_config = { version = "=0.41.0", features = ["workspace", "sync"] } +deno_lockfile = "=0.24.0" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } -deno_npm = "=0.26.0" +deno_npm = "=0.27.0" deno_path_util = "=0.2.2" deno_permissions = { version = "0.43.0", path = "./runtime/permissions" } deno_runtime = { version = "0.192.0", path = "./runtime" } -deno_semver = "=0.6.1" +deno_semver = "=0.7.1" deno_terminal = "0.2.0" napi_sym = { version = "0.114.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } @@ -108,7 +108,7 @@ boxed_error = "0.2.3" brotli = "6.0.0" bytes = "1.4.0" cache_control = "=0.2.0" -capacity_builder = "0.1.3" +capacity_builder = "0.5.0" cbc = { version = "=0.1.2", features = ["alloc"] } # Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS. # Instead use util::time::utc_now() @@ -120,7 +120,7 @@ data-encoding = "2.3.3" data-url = "=0.3.1" deno_cache_dir = "=0.15.0" deno_error = "=0.5.2" -deno_package_json = { version = "0.2.1", default-features = false } +deno_package_json = { version = "0.3.0", default-features = false } deno_unsync = "0.4.2" dlopen2 = "0.6.1" ecb = "=0.1.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4b0380717b..ad0e4fa95d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -74,7 +74,7 @@ deno_config.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] } deno_error.workspace = true -deno_graph = { version = "=0.86.4" } +deno_graph = { version = "=0.86.5" } deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true diff --git a/cli/args/mod.rs b/cli/args/mod.rs index fd34b53c8c..3f42a6b9d4 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -31,6 +31,7 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; +use deno_semver::StackString; use deno_telemetry::OtelConfig; use deno_telemetry::OtelRuntimeConfig; use import_map::resolve_import_map_value_from_specifier; @@ -992,24 +993,24 @@ impl CliOptions { // https://nodejs.org/api/process.html match target.as_str() { "aarch64-apple-darwin" => NpmSystemInfo { - os: "darwin".to_string(), - cpu: "arm64".to_string(), + os: "darwin".into(), + cpu: "arm64".into(), }, "aarch64-unknown-linux-gnu" => NpmSystemInfo { - os: "linux".to_string(), - cpu: "arm64".to_string(), + os: "linux".into(), + cpu: "arm64".into(), }, "x86_64-apple-darwin" => NpmSystemInfo { - os: "darwin".to_string(), - cpu: "x64".to_string(), + os: "darwin".into(), + cpu: "x64".into(), }, "x86_64-unknown-linux-gnu" => NpmSystemInfo { - os: "linux".to_string(), - cpu: "x64".to_string(), + os: "linux".into(), + cpu: "x64".into(), }, "x86_64-pc-windows-msvc" => NpmSystemInfo { - os: "win32".to_string(), - cpu: "x64".to_string(), + os: "win32".into(), + cpu: "x64".into(), }, value => { log::warn!( @@ -1946,15 +1947,17 @@ pub fn has_flag_env_var(name: &str) -> bool { pub fn npm_pkg_req_ref_to_binary_command( req_ref: &NpmPackageReqReference, ) -> String { - let binary_name = req_ref.sub_path().unwrap_or(req_ref.req().name.as_str()); - binary_name.to_string() + req_ref + .sub_path() + .map(|s| s.to_string()) + .unwrap_or_else(|| req_ref.req().name.to_string()) } pub fn config_to_deno_graph_workspace_member( config: &ConfigFile, ) -> Result { - let name = match &config.json.name { - Some(name) => name.clone(), + let name: StackString = match &config.json.name { + Some(name) => name.as_str().into(), None => bail!("Missing 'name' field in config file."), }; let version = match &config.json.version { diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index b0f0a2f9ba..50d1c04799 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -11,19 +11,20 @@ use deno_package_json::PackageJsonDepValueParseError; use deno_package_json::PackageJsonDepWorkspaceReq; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; +use deno_semver::StackString; use deno_semver::VersionReq; use thiserror::Error; #[derive(Debug)] pub struct InstallNpmRemotePkg { - pub alias: Option, + pub alias: Option, pub base_dir: PathBuf, pub req: PackageReq, } #[derive(Debug)] pub struct InstallNpmWorkspacePkg { - pub alias: Option, + pub alias: Option, pub target_dir: PathBuf, } @@ -31,7 +32,7 @@ pub struct InstallNpmWorkspacePkg { #[error("Failed to install '{}'\n at {}", alias, location)] pub struct PackageJsonDepValueParseWithLocationError { pub location: Url, - pub alias: String, + pub alias: StackString, #[source] pub source: PackageJsonDepValueParseError, } @@ -100,10 +101,8 @@ impl NpmInstallDepsProvider { let mut pkg_pkgs = Vec::with_capacity( deps.dependencies.len() + deps.dev_dependencies.len(), ); - for (alias, dep) in deps - .dependencies - .into_iter() - .chain(deps.dev_dependencies.into_iter()) + for (alias, dep) in + deps.dependencies.iter().chain(deps.dev_dependencies.iter()) { let dep = match dep { Ok(dep) => dep, @@ -111,8 +110,8 @@ impl NpmInstallDepsProvider { pkg_json_dep_errors.push( PackageJsonDepValueParseWithLocationError { location: pkg_json.specifier(), - alias, - source: err, + alias: alias.clone(), + source: err.clone(), }, ); continue; @@ -121,28 +120,28 @@ impl NpmInstallDepsProvider { match dep { PackageJsonDepValue::Req(pkg_req) => { let workspace_pkg = workspace_npm_pkgs.iter().find(|pkg| { - pkg.matches_req(&pkg_req) + pkg.matches_req(pkg_req) // do not resolve to the current package && pkg.pkg_json.path != pkg_json.path }); if let Some(pkg) = workspace_pkg { workspace_pkgs.push(InstallNpmWorkspacePkg { - alias: Some(alias), + alias: Some(alias.clone()), target_dir: pkg.pkg_json.dir_path().to_path_buf(), }); } else { pkg_pkgs.push(InstallNpmRemotePkg { - alias: Some(alias), + alias: Some(alias.clone()), base_dir: pkg_json.dir_path().to_path_buf(), - req: pkg_req, + req: pkg_req.clone(), }); } } PackageJsonDepValue::Workspace(workspace_version_req) => { let version_req = match workspace_version_req { PackageJsonDepWorkspaceReq::VersionReq(version_req) => { - version_req + version_req.clone() } PackageJsonDepWorkspaceReq::Tilde | PackageJsonDepWorkspaceReq::Caret => { @@ -150,10 +149,10 @@ impl NpmInstallDepsProvider { } }; if let Some(pkg) = workspace_npm_pkgs.iter().find(|pkg| { - pkg.matches_name_and_version_req(&alias, &version_req) + pkg.matches_name_and_version_req(alias, &version_req) }) { workspace_pkgs.push(InstallNpmWorkspacePkg { - alias: Some(alias), + alias: Some(alias.clone()), target_dir: pkg.pkg_json.dir_path().to_path_buf(), }); } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 6abdbe247a..88c412b65a 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -52,6 +52,7 @@ use deno_runtime::deno_node; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; +use deno_semver::SmallStackString; use import_map::ImportMapError; use node_resolver::InNpmPackageChecker; use std::collections::HashSet; @@ -680,7 +681,7 @@ impl ModuleGraphBuilder { for (from, to) in graph.packages.mappings() { lockfile.insert_package_specifier( JsrDepPackageReq::jsr(from.clone()), - to.version.to_string(), + to.version.to_custom_string::(), ); } } diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 4c6a20927b..8fb3454bc8 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -36,6 +36,8 @@ use deno_semver::package::PackageNv; use deno_semver::package::PackageNvReference; use deno_semver::package::PackageReq; use deno_semver::package::PackageReqReference; +use deno_semver::SmallStackString; +use deno_semver::StackString; use deno_semver::Version; use import_map::ImportMap; use node_resolver::NodeResolutionKind; @@ -278,9 +280,16 @@ impl<'a> TsResponseImportMapper<'a> { { let mut segments = jsr_path.split('/'); let name = if jsr_path.starts_with('@') { - format!("{}/{}", segments.next()?, segments.next()?) + let scope = segments.next()?; + let name = segments.next()?; + capacity_builder::StringBuilder::::build(|builder| { + builder.append(scope); + builder.append("/"); + builder.append(name); + }) + .unwrap() } else { - segments.next()?.to_string() + StackString::from(segments.next()?) }; let version = Version::parse_standard(segments.next()?).ok()?; let nv = PackageNv { name, version }; @@ -290,7 +299,9 @@ impl<'a> TsResponseImportMapper<'a> { &path, Some(&self.file_referrer), )?; - let sub_path = (export != ".").then_some(export); + let sub_path = (export != ".") + .then_some(export) + .map(SmallStackString::from_string); let mut req = None; req = req.or_else(|| { let import_map = self.maybe_import_map?; diff --git a/cli/lsp/jsr.rs b/cli/lsp/jsr.rs index 1d012b42f0..fc30de2ae0 100644 --- a/cli/lsp/jsr.rs +++ b/cli/lsp/jsr.rs @@ -18,6 +18,7 @@ use deno_graph::ModuleSpecifier; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use deno_semver::StackString; use deno_semver::Version; use serde::Deserialize; use std::collections::HashMap; @@ -33,8 +34,8 @@ pub struct JsrCacheResolver { /// The `module_graph` fields of the version infos should be forcibly absent. /// It can be large and we don't want to store it. info_by_nv: DashMap>>, - info_by_name: DashMap>>, - workspace_scope_by_name: HashMap, + info_by_name: DashMap>>, + workspace_scope_by_name: HashMap, cache: Arc, } @@ -59,7 +60,7 @@ impl JsrCacheResolver { continue; }; let nv = PackageNv { - name: jsr_pkg_config.name.clone(), + name: jsr_pkg_config.name.as_str().into(), version: version.clone(), }; info_by_name.insert( @@ -125,8 +126,8 @@ impl JsrCacheResolver { return nv.value().clone(); } let maybe_get_nv = || { - let name = req.name.clone(); - let package_info = self.package_info(&name)?; + let name = &req.name; + let package_info = self.package_info(name)?; // Find the first matching version of the package which is cached. let mut versions = package_info.versions.keys().collect::>(); versions.sort(); @@ -144,7 +145,10 @@ impl JsrCacheResolver { self.package_version_info(&nv).is_some() }) .cloned()?; - Some(PackageNv { name, version }) + Some(PackageNv { + name: name.clone(), + version, + }) }; let nv = maybe_get_nv(); self.nv_by_req.insert(req.clone(), nv.clone()); @@ -216,7 +220,10 @@ impl JsrCacheResolver { None } - pub fn package_info(&self, name: &str) -> Option> { + pub fn package_info( + &self, + name: &StackString, + ) -> Option> { if let Some(info) = self.info_by_name.get(name) { return info.value().clone(); } @@ -226,7 +233,7 @@ impl JsrCacheResolver { serde_json::from_slice::(&meta_bytes).ok() }; let info = read_cached_package_info().map(Arc::new); - self.info_by_name.insert(name.to_string(), info.clone()); + self.info_by_name.insert(name.clone(), info.clone()); info } diff --git a/cli/lsp/search.rs b/cli/lsp/search.rs index 8933eeb186..c98acde6f1 100644 --- a/cli/lsp/search.rs +++ b/cli/lsp/search.rs @@ -67,7 +67,9 @@ pub mod tests { &self, nv: &PackageNv, ) -> Result>, AnyError> { - let Some(exports_by_version) = self.package_versions.get(&nv.name) else { + let Some(exports_by_version) = + self.package_versions.get(nv.name.as_str()) + else { return Err(anyhow!("Package not found.")); }; let Some(exports) = exports_by_version.get(&nv.version) else { diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 4545800e99..5006902aa1 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -560,11 +560,11 @@ impl ManagedCliNpmResolver { &self, ) -> Result<(), Box> { for err in self.npm_install_deps_provider.pkg_json_dep_errors() { - match &err.source { - deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => { + match err.source.as_kind() { + deno_package_json::PackageJsonDepValueParseErrorKind::VersionReq(_) => { return Err(Box::new(err.clone())); } - deno_package_json::PackageJsonDepValueParseError::Unsupported { + deno_package_json::PackageJsonDepValueParseErrorKind::Unsupported { .. } => { // only warn for this one diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 73c5c31caf..5d9fcf4646 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use capacity_builder::StringBuilder; use deno_core::error::AnyError; use deno_lockfile::NpmPackageDependencyLockfileInfo; use deno_lockfile::NpmPackageLockfileInfo; @@ -24,6 +25,7 @@ use deno_npm::NpmSystemInfo; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use deno_semver::SmallStackString; use deno_semver::VersionReq; use crate::args::CliLockfile; @@ -336,7 +338,13 @@ fn populate_lockfile_from_snapshot( let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id; lockfile.insert_package_specifier( JsrDepPackageReq::npm(package_req.clone()), - format!("{}{}", id.nv.version, id.peer_deps_serialized()), + { + StringBuilder::::build(|builder| { + builder.append(&id.nv.version); + builder.append(&id.peer_dependencies); + }) + .unwrap() + }, ); } for package in snapshot.all_packages_for_every_system() { diff --git a/cli/npm/managed/resolvers/common/bin_entries.rs b/cli/npm/managed/resolvers/common/bin_entries.rs index e4a1845689..ca47b9a086 100644 --- a/cli/npm/managed/resolvers/common/bin_entries.rs +++ b/cli/npm/managed/resolvers/common/bin_entries.rs @@ -28,8 +28,10 @@ fn default_bin_name(package: &NpmResolutionPackage) -> &str { .id .nv .name + .as_str() .rsplit_once('/') - .map_or(package.id.nv.name.as_str(), |(_, name)| name) + .map(|(_, name)| name) + .unwrap_or(package.id.nv.name.as_str()) } pub fn warn_missing_entrypoint( diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 1e83717f15..788d6569ae 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -38,6 +38,7 @@ use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodePermissions; use deno_semver::package::PackageNv; +use deno_semver::StackString; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageNotFoundError; @@ -355,8 +356,10 @@ async fn sync_resolution_with_fs( let package_partitions = snapshot.all_system_packages_partitioned(system_info); let mut cache_futures = FuturesUnordered::new(); - let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> = - HashMap::with_capacity(package_partitions.packages.len()); + let mut newest_packages_by_name: HashMap< + &StackString, + &NpmResolutionPackage, + > = HashMap::with_capacity(package_partitions.packages.len()); let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new())); let mut lifecycle_scripts = super::common::lifecycle_scripts::LifecycleScripts::new( @@ -536,7 +539,7 @@ async fn sync_resolution_with_fs( } } - let mut found_names: HashMap<&String, &PackageNv> = HashMap::new(); + let mut found_names: HashMap<&StackString, &PackageNv> = HashMap::new(); // set of node_modules in workspace packages that we've already ensured exist let mut existing_child_node_modules_dirs: HashSet = HashSet::new(); @@ -1012,10 +1015,10 @@ fn get_package_folder_id_from_folder_name( ) -> Option { let folder_name = folder_name.replace('+', "/"); let (name, ending) = folder_name.rsplit_once('@')?; - let name = if let Some(encoded_name) = name.strip_prefix('_') { - mixed_case_package_name_decode(encoded_name)? + let name: StackString = if let Some(encoded_name) = name.strip_prefix('_') { + StackString::from_string(mixed_case_package_name_decode(encoded_name)?) } else { - name.to_string() + name.into() }; let (raw_version, copy_index) = match ending.split_once('_') { Some((raw_version, copy_index)) => { diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index 4c18e0eb53..30802aa081 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -6,6 +6,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::io::Write; +use capacity_builder::BytesAppendable; use deno_ast::swc::common::source_map; use deno_ast::MediaType; use deno_core::anyhow::bail; @@ -21,6 +22,7 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_semver::package::PackageReq; +use deno_semver::StackString; use indexmap::IndexMap; use crate::standalone::virtual_fs::VirtualDirectory; @@ -224,7 +226,10 @@ impl RemoteModulesStoreBuilder { } } - fn write<'a>(&'a self, builder: &mut capacity_builder::BytesBuilder<'a>) { + fn write<'a, TBytes: capacity_builder::BytesType>( + &'a self, + builder: &mut capacity_builder::BytesBuilder<'a, TBytes>, + ) { builder.append_le(self.specifiers.len() as u32); builder.append_le(self.redirects.len() as u32); for (specifier, offset) in &self.specifiers { @@ -581,12 +586,13 @@ fn deserialize_npm_snapshot( #[allow(clippy::needless_lifetimes)] // clippy bug fn parse_package_dep<'a>( id_to_npm_id: &'a impl Fn(usize) -> Result, - ) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a + ) -> impl Fn(&[u8]) -> Result<(&[u8], (StackString, NpmPackageId)), AnyError> + 'a { |input| { let (input, req) = read_string_lossy(input)?; let (input, id) = read_u32_as_usize(input)?; - Ok((input, (req.into_owned(), id_to_npm_id(id)?))) + let req = StackString::from_cow(req); + Ok((input, (req, id_to_npm_id(id)?))) } } diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 647a36dc48..90d54f6cd9 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -343,14 +343,14 @@ impl deno_doc::html::HrefResolver for DocResolver { let name = &res.req().name; Some(( format!("https://www.npmjs.com/package/{name}"), - name.to_owned(), + name.to_string(), )) } "jsr" => { let res = deno_semver::jsr::JsrPackageReqReference::from_str(module).ok()?; let name = &res.req().name; - Some((format!("https://jsr.io/{name}"), name.to_owned())) + Some((format!("https://jsr.io/{name}"), name.to_string())) } _ => None, } diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 7a35f597c3..39a7a912bf 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -278,8 +278,10 @@ fn add_npm_packages_to_json( }); if let Some(pkg) = maybe_package { if let Some(module) = module.as_object_mut() { - module - .insert("npmPackage".to_string(), pkg.id.as_serialized().into()); + module.insert( + "npmPackage".to_string(), + pkg.id.as_serialized().into_string().into(), + ); } } } @@ -296,7 +298,7 @@ fn add_npm_packages_to_json( { dep.insert( "npmPackage".to_string(), - pkg.id.as_serialized().into(), + pkg.id.as_serialized().into_string().into(), ); } } @@ -324,19 +326,19 @@ fn add_npm_packages_to_json( let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len()); for pkg in sorted_packages { let mut kv = serde_json::Map::new(); - kv.insert("name".to_string(), pkg.id.nv.name.clone().into()); + kv.insert("name".to_string(), pkg.id.nv.name.to_string().into()); kv.insert("version".to_string(), pkg.id.nv.version.to_string().into()); let mut deps = pkg.dependencies.values().collect::>(); deps.sort(); let deps = deps .into_iter() - .map(|id| serde_json::Value::String(id.as_serialized())) + .map(|id| serde_json::Value::String(id.as_serialized().into_string())) .collect::>(); kv.insert("dependencies".to_string(), deps.into()); let registry_url = npmrc.get_registry_url(&pkg.id.nv.name); kv.insert("registryUrl".to_string(), registry_url.to_string().into()); - json_packages.insert(pkg.id.as_serialized(), kv.into()); + json_packages.insert(pkg.id.as_serialized().into_string(), kv.into()); } json.insert("npmPackages".to_string(), json_packages.into()); @@ -549,7 +551,7 @@ impl<'a> GraphDisplayContext<'a> { None => Specifier(module.specifier().clone()), }; let was_seen = !self.seen.insert(match &package_or_specifier { - Package(package) => package.id.as_serialized(), + Package(package) => package.id.as_serialized().into_string(), Specifier(specifier) => specifier.to_string(), }); let header_text = if was_seen { @@ -631,7 +633,8 @@ impl<'a> GraphDisplayContext<'a> { )); if let Some(package) = self.npm_info.packages.get(dep_id) { if !package.dependencies.is_empty() { - let was_seen = !self.seen.insert(package.id.as_serialized()); + let was_seen = + !self.seen.insert(package.id.as_serialized().into_string()); if was_seen { child.text = format!("{} {}", child.text, colors::gray("*")); } else { diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index dac7340d40..ec538ecb0a 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -161,11 +161,11 @@ pub async fn infer_name_from_url( let npm_ref = npm_ref.into_inner(); if let Some(sub_path) = npm_ref.sub_path { if !sub_path.contains('/') { - return Some(sub_path); + return Some(sub_path.to_string()); } } if !npm_ref.req.name.contains('/') { - return Some(npm_ref.req.name); + return Some(npm_ref.req.name.into_string()); } return None; } diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index 791e54c67c..afa9b0222c 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -15,6 +15,7 @@ use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use deno_semver::StackString; use deno_semver::Version; use deno_semver::VersionReq; use deps::KeyPath; @@ -283,7 +284,7 @@ fn package_json_dependency_entry( (npm_package.into(), selected.version_req) } else { ( - selected.import_name, + selected.import_name.into_string(), format!("npm:{}@{}", npm_package, selected.version_req), ) } @@ -292,7 +293,7 @@ fn package_json_dependency_entry( let scope_replaced = jsr_package.replace('/', "__"); let version_req = format!("npm:@jsr/{scope_replaced}@{}", selected.version_req); - (selected.import_name, version_req) + (selected.import_name.into_string(), version_req) } else { (selected.package_name, selected.version_req) } @@ -549,10 +550,10 @@ pub async fn add( } struct SelectedPackage { - import_name: String, + import_name: StackString, package_name: String, version_req: String, - selected_version: String, + selected_version: StackString, } enum NotFoundHelp { @@ -683,7 +684,7 @@ async fn find_package_and_select_version_for_req( import_name: add_package_req.alias, package_name: prefixed_name, version_req: format!("{}{}", range_symbol, &nv.version), - selected_version: nv.version.to_string(), + selected_version: nv.version.to_custom_string::(), })) } @@ -705,7 +706,7 @@ enum AddRmPackageReqValue { #[derive(Debug, PartialEq, Eq)] pub struct AddRmPackageReq { - alias: String, + alias: StackString, value: AddRmPackageReqValue, } @@ -753,7 +754,11 @@ impl AddRmPackageReq { return Ok(Err(PackageReq::from_str(entry_text)?)); } - (maybe_prefix.unwrap(), Some(alias.to_string()), entry_text) + ( + maybe_prefix.unwrap(), + Some(StackString::from(alias)), + entry_text, + ) } None => return Ok(Err(PackageReq::from_str(entry_text)?)), }, @@ -765,7 +770,7 @@ impl AddRmPackageReq { JsrPackageReqReference::from_str(&format!("jsr:{}", entry_text))?; let package_req = req_ref.into_inner().req; Ok(Ok(AddRmPackageReq { - alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()), + alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()), value: AddRmPackageReqValue::Jsr(package_req), })) } @@ -785,7 +790,7 @@ impl AddRmPackageReq { ); } Ok(Ok(AddRmPackageReq { - alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()), + alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()), value: AddRmPackageReqValue::Npm(package_req), })) } @@ -878,14 +883,14 @@ mod test { assert_eq!( AddRmPackageReq::parse("jsr:foo").unwrap().unwrap(), AddRmPackageReq { - alias: "foo".to_string(), + alias: "foo".into(), value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); assert_eq!( AddRmPackageReq::parse("alias@jsr:foo").unwrap().unwrap(), AddRmPackageReq { - alias: "alias".to_string(), + alias: "alias".into(), value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); @@ -894,7 +899,7 @@ mod test { .unwrap() .unwrap(), AddRmPackageReq { - alias: "@alias/pkg".to_string(), + alias: "@alias/pkg".into(), value: AddRmPackageReqValue::Npm( PackageReq::from_str("foo@latest").unwrap() ) @@ -905,7 +910,7 @@ mod test { .unwrap() .unwrap(), AddRmPackageReq { - alias: "@alias/pkg".to_string(), + alias: "@alias/pkg".into(), value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); @@ -914,7 +919,7 @@ mod test { .unwrap() .unwrap(), AddRmPackageReq { - alias: "alias".to_string(), + alias: "alias".into(), value: AddRmPackageReqValue::Jsr( PackageReq::from_str("foo@^1.5.0").unwrap() ) diff --git a/cli/tools/registry/pm/deps.rs b/cli/tools/registry/pm/deps.rs index bb03e97f2d..ffa53417e9 100644 --- a/cli/tools/registry/pm/deps.rs +++ b/cli/tools/registry/pm/deps.rs @@ -27,6 +27,7 @@ use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use deno_semver::package::PackageReqReference; +use deno_semver::StackString; use deno_semver::Version; use deno_semver::VersionReq; use import_map::ImportMap; @@ -139,13 +140,7 @@ pub enum KeyPart { Scopes, Dependencies, DevDependencies, - String(String), -} - -impl From for KeyPart { - fn from(value: String) -> Self { - KeyPart::String(value) - } + String(StackString), } impl From for KeyPart { @@ -164,7 +159,7 @@ impl KeyPart { KeyPart::Scopes => "scopes", KeyPart::Dependencies => "dependencies", KeyPart::DevDependencies => "devDependencies", - KeyPart::String(s) => s, + KeyPart::String(s) => s.as_str(), } } } @@ -217,12 +212,12 @@ fn import_map_entries( .chain(import_map.scopes().flat_map(|scope| { let path = KeyPath::from_parts([ KeyPart::Scopes, - scope.raw_key.to_string().into(), + KeyPart::String(scope.raw_key.into()), ]); scope.imports.entries().map(move |entry| { let mut full_path = path.clone(); - full_path.push(KeyPart::String(entry.raw_key.to_string())); + full_path.push(KeyPart::String(entry.raw_key.into())); (full_path, entry) }) })) @@ -338,7 +333,7 @@ fn add_deps_from_package_json( package_json: &PackageJsonRc, mut filter: impl DepFilter, package_dep_kind: PackageJsonDepKind, - package_json_deps: PackageJsonDepsMap, + package_json_deps: &PackageJsonDepsMap, deps: &mut Vec, ) { for (k, v) in package_json_deps { @@ -353,7 +348,7 @@ fn add_deps_from_package_json( deno_package_json::PackageJsonDepValue::Req(req) => { let alias = k.as_str(); let alias = (alias != req.name).then(|| alias.to_string()); - if !filter.should_include(alias.as_deref(), &req, DepKind::Npm) { + if !filter.should_include(alias.as_deref(), req, DepKind::Npm) { continue; } let id = DepId(deps.len()); @@ -362,9 +357,12 @@ fn add_deps_from_package_json( kind: DepKind::Npm, location: DepLocation::PackageJson( package_json.clone(), - KeyPath::from_parts([package_dep_kind.into(), k.into()]), + KeyPath::from_parts([ + package_dep_kind.into(), + KeyPart::String(k.clone()), + ]), ), - req, + req: req.clone(), alias, }) } @@ -377,14 +375,14 @@ fn add_deps_from_package_json( package_json, filter, PackageJsonDepKind::Normal, - package_json_deps.dependencies, + &package_json_deps.dependencies, deps, ); iterate( package_json, filter, PackageJsonDepKind::Dev, - package_json_deps.dev_dependencies, + &package_json_deps.dev_dependencies, deps, ); } diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index f767eb1522..20a6043f26 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -8,6 +8,7 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; +use deno_semver::StackString; use deno_semver::VersionReq; use deno_terminal::colors; @@ -31,7 +32,7 @@ struct OutdatedPackage { latest: String, semver_compatible: String, current: String, - name: String, + name: StackString, } #[allow(clippy::print_stdout)] diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index cb85859f7a..b3d7618be9 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -21,6 +21,7 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::unsync::spawn; use deno_core::url::Url; +use deno_semver::SmallStackString; use deno_semver::Version; use once_cell::sync::Lazy; use std::borrow::Cow; @@ -255,7 +256,7 @@ async fn print_release_notes( let is_deno_2_rc = new_semver.major == 2 && new_semver.minor == 0 && new_semver.patch == 0 - && new_semver.pre.first() == Some(&"rc".to_string()); + && new_semver.pre.first().map(|s| s.as_str()) == Some("rc"); if is_deno_2_rc || is_switching_from_deno1_to_deno2 { log::info!( @@ -674,7 +675,7 @@ impl RequestedVersion { ); }; - if semver.pre.contains(&"rc".to_string()) { + if semver.pre.contains(&SmallStackString::from_static("rc")) { (ReleaseChannel::Rc, passed_version) } else { (ReleaseChannel::Stable, passed_version) diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index 08d06f9cac..c6f5d1cd04 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -9,6 +9,7 @@ use deno_package_json::PackageJsonDepValue; use deno_package_json::PackageJsonRc; use deno_path_util::url_to_file_path; use deno_semver::package::PackageReq; +use deno_semver::StackString; use deno_semver::Version; use node_resolver::env::NodeResolverEnv; use node_resolver::errors::PackageFolderResolveError; @@ -30,7 +31,7 @@ use super::ResolvePkgFolderFromDenoReqError; #[derive(Debug, Error)] pub enum ByonmResolvePkgFolderFromDenoReqError { #[error("Could not find \"{}\" in a node_modules folder. Deno expects the node_modules/ directory to be up to date. Did you forget to run `deno install`?", .0)] - MissingAlias(String), + MissingAlias(StackString), #[error(transparent)] PackageJson(#[from] PackageJsonLoadError), #[error("Could not find a matching package for 'npm:{}' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `\"nodeModulesDir\": \"auto\"` in your deno.json file.", .0)] @@ -177,16 +178,14 @@ impl ByonmNpmResolver { &self, req: &PackageReq, referrer: &Url, - ) -> Result, PackageJsonLoadError> { + ) -> Result, PackageJsonLoadError> { fn resolve_alias_from_pkg_json( req: &PackageReq, pkg_json: &PackageJson, - ) -> Option { + ) -> Option { let deps = pkg_json.resolve_local_package_json_deps(); - for (key, value) in deps - .dependencies - .into_iter() - .chain(deps.dev_dependencies.into_iter()) + for (key, value) in + deps.dependencies.iter().chain(deps.dev_dependencies.iter()) { if let Ok(value) = value { match value { @@ -194,12 +193,14 @@ impl ByonmNpmResolver { if dep_req.name == req.name && dep_req.version_req.intersects(&req.version_req) { - return Some(key); + return Some(key.clone()); } } PackageJsonDepValue::Workspace(_workspace) => { - if key == req.name && req.version_req.tag() == Some("workspace") { - return Some(key); + if key.as_str() == req.name + && req.version_req.tag() == Some("workspace") + { + return Some(key.clone()); } } } @@ -246,7 +247,7 @@ impl ByonmNpmResolver { if let Ok(Some(dep_pkg_json)) = self.load_pkg_json(&pkg_folder.join("package.json")) { - if dep_pkg_json.name.as_ref() == Some(&req.name) { + if dep_pkg_json.name.as_deref() == Some(req.name.as_str()) { let matches_req = dep_pkg_json .version .as_ref() diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index 5f87698cd6..6b0bda57d7 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -318,6 +318,8 @@ impl NodeResolver { resolution_mode: ResolutionMode, resolution_kind: NodeResolutionKind, ) -> Result { + // todo(dsherret): don't allocate a string here (maybe use an + // enum that says the subpath is not prefixed with a ./) let package_subpath = package_subpath .map(|s| format!("./{s}")) .unwrap_or_else(|| ".".to_string()); diff --git a/resolvers/npm_cache/Cargo.toml b/resolvers/npm_cache/Cargo.toml index a0a106c89b..010f8a436b 100644 --- a/resolvers/npm_cache/Cargo.toml +++ b/resolvers/npm_cache/Cargo.toml @@ -23,6 +23,7 @@ async-trait.workspace = true base64.workspace = true boxed_error.workspace = true deno_cache_dir.workspace = true +deno_error.workspace = true deno_npm.workspace = true deno_semver.workspace = true deno_unsync = { workspace = true, features = ["tokio"] } diff --git a/resolvers/npm_cache/lib.rs b/resolvers/npm_cache/lib.rs index c16c29aaf2..dbd33d29eb 100644 --- a/resolvers/npm_cache/lib.rs +++ b/resolvers/npm_cache/lib.rs @@ -15,6 +15,7 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::NpmPackageCacheFolderId; use deno_semver::package::PackageNv; +use deno_semver::StackString; use deno_semver::Version; use http::HeaderName; use http::HeaderValue; @@ -260,7 +261,7 @@ impl NpmCache { .and_then(|cache_id| { Some(NpmPackageCacheFolderId { nv: PackageNv { - name: cache_id.name, + name: StackString::from_string(cache_id.name), version: Version::parse_from_npm(&cache_id.version).ok()?, }, copy_index: cache_id.copy_index, diff --git a/resolvers/npm_cache/registry_info.rs b/resolvers/npm_cache/registry_info.rs index 543ddadc5a..dd8a639591 100644 --- a/resolvers/npm_cache/registry_info.rs +++ b/resolvers/npm_cache/registry_info.rs @@ -18,6 +18,7 @@ use deno_unsync::sync::MultiRuntimeAsyncValueCreator; use futures::future::LocalBoxFuture; use futures::FutureExt; use parking_lot::Mutex; +use thiserror::Error; use url::Url; use crate::remote::maybe_auth_header_for_npm_registry; @@ -28,6 +29,31 @@ use crate::NpmCacheSetting; type LoadResult = Result>; type LoadFuture = LocalBoxFuture<'static, LoadResult>; +#[derive(Debug, Error)] +#[error(transparent)] +pub struct AnyhowJsError(pub AnyError); + +impl deno_error::JsErrorClass for AnyhowJsError { + fn get_class(&self) -> &'static str { + "generic" + } + + fn get_message(&self) -> std::borrow::Cow<'static, str> { + self.0.to_string().into() + } + + fn get_additional_properties( + &self, + ) -> Option< + Vec<( + std::borrow::Cow<'static, str>, + std::borrow::Cow<'static, str>, + )>, + > { + None + } +} + #[derive(Debug, Clone)] enum FutureResult { PackageNotExists, @@ -157,9 +183,9 @@ impl RegistryInfoProvider { Ok(None) => Err(NpmRegistryPackageInfoLoadError::PackageNotExists { package_name: name.to_string(), }), - Err(err) => { - Err(NpmRegistryPackageInfoLoadError::LoadError(Arc::new(err))) - } + Err(err) => Err(NpmRegistryPackageInfoLoadError::LoadError(Arc::new( + AnyhowJsError(err), + ))), } } diff --git a/resolvers/npm_cache/tarball_extract.rs b/resolvers/npm_cache/tarball_extract.rs index c4c614b35f..affe93eaa4 100644 --- a/resolvers/npm_cache/tarball_extract.rs +++ b/resolvers/npm_cache/tarball_extract.rs @@ -236,7 +236,7 @@ mod test { #[test] pub fn test_verify_tarball() { let package = PackageNv { - name: "package".to_string(), + name: "package".into(), version: Version::parse_from_npm("1.0.0").unwrap(), }; let actual_checksum = diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index bbd0301db4..1c5fb36f93 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -183,7 +183,7 @@ impl PermissionState { PermissionState::Prompt if prompt => { let msg = { let info = info(); - StringBuilder::build(|builder| { + StringBuilder::::build(|builder| { builder.append(name); builder.append(" access"); if let Some(info) = &info { @@ -498,7 +498,7 @@ impl UnaryPermission { } let maybe_formatted_display_name = desc.map(|d| format_display_name(d.display_name())); - let message = StringBuilder::build(|builder| { + let message = StringBuilder::::build(|builder| { builder.append(TQuery::flag_name()); builder.append(" access"); if let Some(display_name) = &maybe_formatted_display_name { diff --git a/tests/integration/check_tests.rs b/tests/integration/check_tests.rs index b98d719fca..a1fdf83403 100644 --- a/tests/integration/check_tests.rs +++ b/tests/integration/check_tests.rs @@ -218,7 +218,7 @@ fn npm_module_check_then_error() { "npm:@denotest/breaking-change-between-versions", ) .unwrap(), - "1.0.0".to_string(), + "1.0.0".into(), ); lockfile_path.write(lockfile.as_json_string()); temp_dir.write( @@ -236,7 +236,7 @@ fn npm_module_check_then_error() { "npm:@denotest/breaking-change-between-versions", ) .unwrap(), - "2.0.0".to_string(), + "2.0.0".into(), ); lockfile_path.write(lockfile.as_json_string()); diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index c4812e6bfb..f438ebfd50 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -159,7 +159,7 @@ console.log(version);"#, .get_mut( &JsrDepPackageReq::from_str("jsr:@denotest/no-module-graph@0.1").unwrap(), ) - .unwrap() = "0.1.0".to_string(); + .unwrap() = "0.1.0".into(); lockfile_path.write(lockfile.as_json_string()); test_context From 77e1af79bd47f4c134cfa08d33fce8b2753bb4fa Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 20 Dec 2024 17:35:02 -0500 Subject: [PATCH 15/20] perf: remove now needless canonicalization getting closest package.json (#27437) This is no longer required because we now store everything in the file system for deno compile instead of in an eszip. --- resolvers/node/errors.rs | 17 ----------------- resolvers/node/package_json.rs | 33 +-------------------------------- 2 files changed, 1 insertion(+), 49 deletions(-) diff --git a/resolvers/node/errors.rs b/resolvers/node/errors.rs index 600a365a8f..26b1a1d84a 100644 --- a/resolvers/node/errors.rs +++ b/resolvers/node/errors.rs @@ -320,7 +320,6 @@ impl NodeJsErrorCoded for PackageJsonLoadError { impl NodeJsErrorCoded for ClosestPkgJsonError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { - ClosestPkgJsonErrorKind::CanonicalizingDir(e) => e.code(), ClosestPkgJsonErrorKind::Load(e) => e.code(), } } @@ -331,26 +330,10 @@ pub struct ClosestPkgJsonError(pub Box); #[derive(Debug, Error)] pub enum ClosestPkgJsonErrorKind { - #[error(transparent)] - CanonicalizingDir(#[from] CanonicalizingPkgJsonDirError), #[error(transparent)] Load(#[from] PackageJsonLoadError), } -#[derive(Debug, Error)] -#[error("[{}] Failed canonicalizing package.json directory '{}'.", self.code(), dir_path.display())] -pub struct CanonicalizingPkgJsonDirError { - pub dir_path: PathBuf, - #[source] - pub source: std::io::Error, -} - -impl NodeJsErrorCoded for CanonicalizingPkgJsonDirError { - fn code(&self) -> NodeJsErrorCode { - NodeJsErrorCode::ERR_MODULE_NOT_FOUND - } -} - // todo(https://github.com/denoland/deno_core/issues/810): make this a TypeError #[derive(Debug, Error)] #[error( diff --git a/resolvers/node/package_json.rs b/resolvers/node/package_json.rs index e3793af84a..cb99e5a0aa 100644 --- a/resolvers/node/package_json.rs +++ b/resolvers/node/package_json.rs @@ -2,7 +2,6 @@ use deno_package_json::PackageJson; use deno_package_json::PackageJsonRc; -use deno_path_util::strip_unc_prefix; use std::cell::RefCell; use std::collections::HashMap; use std::io::ErrorKind; @@ -11,7 +10,6 @@ use std::path::PathBuf; use url::Url; use crate::env::NodeResolverEnv; -use crate::errors::CanonicalizingPkgJsonDirError; use crate::errors::ClosestPkgJsonError; use crate::errors::PackageJsonLoadError; @@ -67,37 +65,8 @@ impl PackageJsonResolver { &self, file_path: &Path, ) -> Result, ClosestPkgJsonError> { - // we use this for deno compile using byonm because the script paths - // won't be in virtual file system, but the package.json paths will be - fn canonicalize_first_ancestor_exists( - dir_path: &Path, - env: &TEnv, - ) -> Result, std::io::Error> { - for ancestor in dir_path.ancestors() { - match env.realpath_sync(ancestor) { - Ok(dir_path) => return Ok(Some(dir_path)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - // keep searching - } - Err(err) => return Err(err), - } - } - Ok(None) - } - let parent_dir = file_path.parent().unwrap(); - let Some(start_dir) = canonicalize_first_ancestor_exists( - parent_dir, &self.env, - ) - .map_err(|source| CanonicalizingPkgJsonDirError { - dir_path: parent_dir.to_path_buf(), - source, - })? - else { - return Ok(None); - }; - let start_dir = strip_unc_prefix(start_dir); - for current_dir in start_dir.ancestors() { + for current_dir in parent_dir.ancestors() { let package_json_path = current_dir.join("package.json"); if let Some(pkg_json) = self.load_package_json(&package_json_path)? { return Ok(Some(pkg_json)); From 26425a137b7489fe675d106c3943cdcea6fce0cb Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Sat, 21 Dec 2024 00:58:03 +0100 Subject: [PATCH 16/20] feat(unstable): add JS linting plugin infrastructure (#27416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR extracts the core part of https://github.com/denoland/deno/pull/27203 to make it easier to review and land in parts. It contains: - The JS plugin code the deserializes and walks the buffer - The Rust portion to serialize SWC to the buffer format (a bunch of nodes are still todos, but imo these can land anytime later) - Basic lint plugin types, without the AST node types to make this PR easier to review - Added more code comments to explain the format etc. More fixes and changes will be done in follow-up PRs. --------- Co-authored-by: Bartek IwaƄczuk --- cli/js/40_lint.js | 783 ++++++ cli/js/40_lint_types.d.ts | 50 + cli/ops/lint.rs | 34 + cli/ops/mod.rs | 1 + cli/tools/lint/ast_buffer/buffer.rs | 516 ++++ cli/tools/lint/ast_buffer/mod.rs | 13 + cli/tools/lint/ast_buffer/swc.rs | 3018 ++++++++++++++++++++++++ cli/tools/lint/ast_buffer/ts_estree.rs | 513 ++++ cli/tools/lint/mod.rs | 3 + cli/tools/test/mod.rs | 5 +- cli/worker.rs | 3 +- runtime/js/99_main.js | 3 + tests/integration/js_unit_tests.rs | 1 + tests/unit/lint_plugin_test.ts | 557 +++++ tests/unit/ops_test.ts | 2 +- 15 files changed, 5499 insertions(+), 3 deletions(-) create mode 100644 cli/js/40_lint.js create mode 100644 cli/js/40_lint_types.d.ts create mode 100644 cli/ops/lint.rs create mode 100644 cli/tools/lint/ast_buffer/buffer.rs create mode 100644 cli/tools/lint/ast_buffer/mod.rs create mode 100644 cli/tools/lint/ast_buffer/swc.rs create mode 100644 cli/tools/lint/ast_buffer/ts_estree.rs create mode 100644 tests/unit/lint_plugin_test.ts diff --git a/cli/js/40_lint.js b/cli/js/40_lint.js new file mode 100644 index 0000000000..9606f787b3 --- /dev/null +++ b/cli/js/40_lint.js @@ -0,0 +1,783 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// @ts-check + +import { core, internals } from "ext:core/mod.js"; +const { + op_lint_create_serialized_ast, +} = core.ops; + +// Keep in sync with Rust +// These types are expected to be present on every node. Note that this +// isn't set in stone. We could revise this at a future point. +const AST_PROP_TYPE = 0; +const AST_PROP_PARENT = 1; +const AST_PROP_RANGE = 2; + +// Keep in sync with Rust +// Each node property is tagged with this enum to denote +// what kind of value it holds. +/** @enum {number} */ +const PropFlags = { + /** This is an offset to another node */ + Ref: 0, + /** This is an array of offsets to other nodes (like children of a BlockStatement) */ + RefArr: 1, + /** + * This is a string id. The actual string needs to be looked up in + * the string table that was included in the message. + */ + String: 2, + /** This value is either 0 = false, or 1 = true */ + Bool: 3, + /** No value, it's null */ + Null: 4, + /** No value, it's undefined */ + Undefined: 5, +}; + +/** @typedef {import("./40_lint_types.d.ts").AstContext} AstContext */ +/** @typedef {import("./40_lint_types.d.ts").VisitorFn} VisitorFn */ +/** @typedef {import("./40_lint_types.d.ts").CompiledVisitor} CompiledVisitor */ +/** @typedef {import("./40_lint_types.d.ts").LintState} LintState */ +/** @typedef {import("./40_lint_types.d.ts").RuleContext} RuleContext */ +/** @typedef {import("./40_lint_types.d.ts").NodeFacade} NodeFacade */ +/** @typedef {import("./40_lint_types.d.ts").LintPlugin} LintPlugin */ +/** @typedef {import("./40_lint_types.d.ts").LintReportData} LintReportData */ +/** @typedef {import("./40_lint_types.d.ts").TestReportData} TestReportData */ + +/** @type {LintState} */ +const state = { + plugins: [], + installedPlugins: new Set(), +}; + +/** + * Every rule gets their own instance of this class. This is the main + * API lint rules interact with. + * @implements {RuleContext} + */ +export class Context { + id; + + fileName; + + /** + * @param {string} id + * @param {string} fileName + */ + constructor(id, fileName) { + this.id = id; + this.fileName = fileName; + } +} + +/** + * @param {LintPlugin} plugin + */ +export function installPlugin(plugin) { + if (typeof plugin !== "object") { + throw new Error("Linter plugin must be an object"); + } + if (typeof plugin.name !== "string") { + throw new Error("Linter plugin name must be a string"); + } + if (typeof plugin.rules !== "object") { + throw new Error("Linter plugin rules must be an object"); + } + if (state.installedPlugins.has(plugin.name)) { + throw new Error(`Linter plugin ${plugin.name} has already been registered`); + } + state.plugins.push(plugin); + state.installedPlugins.add(plugin.name); +} + +/** + * @param {AstContext} ctx + * @param {number} offset + * @returns + */ +function getNode(ctx, offset) { + if (offset === 0) return null; + + const cached = ctx.nodes.get(offset); + if (cached !== undefined) return cached; + + const node = new Node(ctx, offset); + ctx.nodes.set(offset, /** @type {*} */ (cached)); + return node; +} + +/** + * Find the offset of a specific property of a specific node. This will + * be used later a lot more for selectors. + * @param {Uint8Array} buf + * @param {number} search + * @param {number} offset + * @returns {number} + */ +function findPropOffset(buf, offset, search) { + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + const propCount = buf[offset]; + offset += 1; + + for (let i = 0; i < propCount; i++) { + const maybe = offset; + const prop = buf[offset++]; + const kind = buf[offset++]; + if (prop === search) return maybe; + + if (kind === PropFlags.Ref) { + offset += 4; + } else if (kind === PropFlags.RefArr) { + const len = readU32(buf, offset); + offset += 4 + (len * 4); + } else if (kind === PropFlags.String) { + offset += 4; + } else if (kind === PropFlags.Bool) { + offset++; + } else if (kind === PropFlags.Null || kind === PropFlags.Undefined) { + // No value + } else { + offset++; + } + } + + return -1; +} + +const INTERNAL_CTX = Symbol("ctx"); +const INTERNAL_OFFSET = Symbol("offset"); + +// This class is a facade for all materialized nodes. Instead of creating a +// unique class per AST node, we have one class with getters for every +// possible node property. This allows us to lazily materialize child node +// only when they are needed. +class Node { + [INTERNAL_CTX]; + [INTERNAL_OFFSET]; + + /** + * @param {AstContext} ctx + * @param {number} offset + */ + constructor(ctx, offset) { + this[INTERNAL_CTX] = ctx; + this[INTERNAL_OFFSET] = offset; + } + + /** + * Logging a class with only getters prints just the class name. This + * makes debugging difficult because you don't see any of the properties. + * For that reason we'll intercept inspection and serialize the node to + * a plain JSON structure which can be logged and allows users to see all + * properties and their values. + * + * This is only expected to be used during development of a rule. + * @param {*} _ + * @param {Deno.InspectOptions} options + * @returns {string} + */ + [Symbol.for("Deno.customInspect")](_, options) { + const json = toJsValue(this[INTERNAL_CTX], this[INTERNAL_OFFSET]); + return Deno.inspect(json, options); + } + + [Symbol.for("Deno.lint.toJsValue")]() { + return toJsValue(this[INTERNAL_CTX], this[INTERNAL_OFFSET]); + } +} + +/** @type {Set} */ +const appliedGetters = new Set(); + +/** + * Add getters for all potential properties found in the message. + * @param {AstContext} ctx + */ +function setNodeGetters(ctx) { + if (appliedGetters.size === ctx.strByProp.length) return; + + for (let i = 0; i < ctx.strByProp.length; i++) { + const id = ctx.strByProp[i]; + if (id === 0 || appliedGetters.has(i)) continue; + appliedGetters.add(i); + + const name = getString(ctx.strTable, id); + + Object.defineProperty(Node.prototype, name, { + get() { + return readValue(this[INTERNAL_CTX], this[INTERNAL_OFFSET], i); + }, + }); + } +} + +/** + * Serialize a node recursively to plain JSON + * @param {AstContext} ctx + * @param {number} offset + * @returns {*} + */ +function toJsValue(ctx, offset) { + const { buf } = ctx; + + /** @type {Record} */ + const node = { + type: readValue(ctx, offset, AST_PROP_TYPE), + range: readValue(ctx, offset, AST_PROP_RANGE), + }; + + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + const count = buf[offset++]; + for (let i = 0; i < count; i++) { + const prop = buf[offset++]; + const kind = buf[offset++]; + const name = getString(ctx.strTable, ctx.strByProp[prop]); + + if (kind === PropFlags.Ref) { + const v = readU32(buf, offset); + offset += 4; + node[name] = v === 0 ? null : toJsValue(ctx, v); + } else if (kind === PropFlags.RefArr) { + const len = readU32(buf, offset); + offset += 4; + const nodes = new Array(len); + for (let i = 0; i < len; i++) { + const v = readU32(buf, offset); + if (v === 0) continue; + nodes[i] = toJsValue(ctx, v); + offset += 4; + } + node[name] = nodes; + } else if (kind === PropFlags.Bool) { + const v = buf[offset++]; + node[name] = v === 1; + } else if (kind === PropFlags.String) { + const v = readU32(buf, offset); + offset += 4; + node[name] = getString(ctx.strTable, v); + } else if (kind === PropFlags.Null) { + node[name] = null; + } else if (kind === PropFlags.Undefined) { + node[name] = undefined; + } + } + + return node; +} + +/** + * Read a specific property from a node + * @param {AstContext} ctx + * @param {number} offset + * @param {number} search + * @returns {*} + */ +function readValue(ctx, offset, search) { + const { buf } = ctx; + const type = buf[offset]; + + if (search === AST_PROP_TYPE) { + return getString(ctx.strTable, ctx.strByType[type]); + } else if (search === AST_PROP_RANGE) { + const start = readU32(buf, offset + 1 + 4); + const end = readU32(buf, offset + 1 + 4 + 4); + return [start, end]; + } else if (search === AST_PROP_PARENT) { + const pos = readU32(buf, offset + 1); + return getNode(ctx, pos); + } + + offset = findPropOffset(ctx.buf, offset, search); + if (offset === -1) return undefined; + + const kind = buf[offset + 1]; + + if (kind === PropFlags.Ref) { + const value = readU32(buf, offset + 2); + return getNode(ctx, value); + } else if (kind === PropFlags.RefArr) { + const len = readU32(buf, offset); + offset += 4; + + const nodes = new Array(len); + for (let i = 0; i < len; i++) { + nodes[i] = getNode(ctx, readU32(buf, offset)); + offset += 4; + } + return nodes; + } else if (kind === PropFlags.Bool) { + return buf[offset] === 1; + } else if (kind === PropFlags.String) { + const v = readU32(buf, offset); + return getString(ctx.strTable, v); + } else if (kind === PropFlags.Null) { + return null; + } else if (kind === PropFlags.Undefined) { + return undefined; + } + + throw new Error(`Unknown prop kind: ${kind}`); +} + +const DECODER = new TextDecoder(); + +/** + * TODO: Check if it's faster to use the `ArrayView` API instead. + * @param {Uint8Array} buf + * @param {number} i + * @returns {number} + */ +function readU32(buf, i) { + return (buf[i] << 24) + (buf[i + 1] << 16) + (buf[i + 2] << 8) + + buf[i + 3]; +} + +/** + * Get a string by id and error if it wasn't found + * @param {AstContext["strTable"]} strTable + * @param {number} id + * @returns {string} + */ +function getString(strTable, id) { + const name = strTable.get(id); + if (name === undefined) { + throw new Error(`Missing string id: ${id}`); + } + + return name; +} + +/** + * @param {Uint8Array} buf + * @param {AstContext} buf + */ +function createAstContext(buf) { + /** @type {Map} */ + const strTable = new Map(); + + // The buffer has a few offsets at the end which allows us to easily + // jump to the relevant sections of the message. + const typeMapOffset = readU32(buf, buf.length - 16); + const propMapOffset = readU32(buf, buf.length - 12); + const strTableOffset = readU32(buf, buf.length - 8); + + // Offset of the topmost node in the AST Tree. + const rootOffset = readU32(buf, buf.length - 4); + + let offset = strTableOffset; + const stringCount = readU32(buf, offset); + offset += 4; + + // TODO(@marvinhagemeister): We could lazily decode the strings on an as needed basis. + // Not sure if this matters much in practice though. + let id = 0; + for (let i = 0; i < stringCount; i++) { + const len = readU32(buf, offset); + offset += 4; + + const strBytes = buf.slice(offset, offset + len); + offset += len; + const s = DECODER.decode(strBytes); + strTable.set(id, s); + id++; + } + + if (strTable.size !== stringCount) { + throw new Error( + `Could not deserialize string table. Expected ${stringCount} items, but got ${strTable.size}`, + ); + } + + offset = typeMapOffset; + const typeCount = readU32(buf, offset); + offset += 4; + + const typeByStr = new Map(); + const strByType = new Array(typeCount).fill(0); + for (let i = 0; i < typeCount; i++) { + const v = readU32(buf, offset); + offset += 4; + + strByType[i] = v; + typeByStr.set(strTable.get(v), i); + } + + offset = propMapOffset; + const propCount = readU32(buf, offset); + offset += 4; + + const propByStr = new Map(); + const strByProp = new Array(propCount).fill(0); + for (let i = 0; i < propCount; i++) { + const v = readU32(buf, offset); + offset += 4; + + strByProp[i] = v; + propByStr.set(strTable.get(v), i); + } + + /** @type {AstContext} */ + const ctx = { + buf, + strTable, + rootOffset, + nodes: new Map(), + strTableOffset, + strByProp, + strByType, + typeByStr, + propByStr, + }; + + setNodeGetters(ctx); + + // DEV ONLY: Enable this to inspect the buffer message + // _dump(ctx); + + return ctx; +} + +/** + * @param {*} _node + */ +const NOOP = (_node) => {}; + +/** + * Kick off the actual linting process of JS plugins. + * @param {string} fileName + * @param {Uint8Array} serializedAst + */ +export function runPluginsForFile(fileName, serializedAst) { + const ctx = createAstContext(serializedAst); + + /** @type {Map} */ + const bySelector = new Map(); + + const destroyFns = []; + + // Instantiate and merge visitors. This allows us to only traverse + // the AST once instead of per plugin. When ever we enter or exit a + // node we'll call all visitors that match. + for (let i = 0; i < state.plugins.length; i++) { + const plugin = state.plugins[i]; + + for (const name of Object.keys(plugin.rules)) { + const rule = plugin.rules[name]; + const id = `${plugin.name}/${name}`; + const ctx = new Context(id, fileName); + const visitor = rule.create(ctx); + + // deno-lint-ignore guard-for-in + for (let key in visitor) { + const fn = visitor[key]; + if (fn === undefined) continue; + + // Support enter and exit callbacks on a visitor. + // Exit callbacks are marked by having `:exit` at the end. + let isExit = false; + if (key.endsWith(":exit")) { + isExit = true; + key = key.slice(0, -":exit".length); + } + + let info = bySelector.get(key); + if (info === undefined) { + info = { enter: NOOP, exit: NOOP }; + bySelector.set(key, info); + } + const prevFn = isExit ? info.exit : info.enter; + + /** + * @param {*} node + */ + const wrapped = (node) => { + prevFn(node); + + try { + fn(node); + } catch (err) { + throw new Error(`Visitor "${name}" of plugin "${id}" errored`, { + cause: err, + }); + } + }; + + if (isExit) { + info.exit = wrapped; + } else { + info.enter = wrapped; + } + } + + if (typeof rule.destroy === "function") { + const destroyFn = rule.destroy.bind(rule); + destroyFns.push(() => { + try { + destroyFn(ctx); + } catch (err) { + throw new Error(`Destroy hook of "${id}" errored`, { cause: err }); + } + }); + } + } + } + + /** @type {CompiledVisitor[]} */ + const visitors = []; + for (const [sel, info] of bySelector.entries()) { + // This will make more sense once selectors land as it's faster + // to precompile them once upfront. + + // Convert the visiting element name to a number. This number + // is part of the serialized buffer and comparing a single number + // is quicker than strings. + const elemId = ctx.typeByStr.get(sel) ?? -1; + + visitors.push({ + info, + // Check if we should call this visitor + matcher: (offset) => { + const type = ctx.buf[offset]; + return type === elemId; + }, + }); + } + + // Traverse ast with all visitors at the same time to avoid traversing + // multiple times. + try { + traverse(ctx, visitors, ctx.rootOffset); + } finally { + ctx.nodes.clear(); + + // Optional: Destroy rules + for (let i = 0; i < destroyFns.length; i++) { + destroyFns[i](); + } + } +} + +/** + * @param {AstContext} ctx + * @param {CompiledVisitor[]} visitors + * @param {number} offset + */ +function traverse(ctx, visitors, offset) { + // The 0 offset is used to denote an empty/placeholder node + if (offset === 0) return; + + const { buf } = ctx; + + /** @type {VisitorFn[] | null} */ + let exits = null; + + for (let i = 0; i < visitors.length; i++) { + const v = visitors[i]; + + if (v.matcher(offset)) { + if (v.info.exit !== NOOP) { + if (exits === null) { + exits = [v.info.exit]; + } else { + exits.push(v.info.exit); + } + } + + if (v.info.enter !== NOOP) { + const node = /** @type {*} */ (getNode(ctx, offset)); + v.info.enter(node); + } + } + } + + // Search for node references in the properties of the current node. All + // other properties can be ignored. + try { + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + const propCount = buf[offset]; + offset += 1; + + for (let i = 0; i < propCount; i++) { + const kind = buf[offset + 1]; + offset += 2; // propId + propFlags + + if (kind === PropFlags.Ref) { + const next = readU32(buf, offset); + offset += 4; + traverse(ctx, visitors, next); + } else if (kind === PropFlags.RefArr) { + const len = readU32(buf, offset); + offset += 4; + + for (let j = 0; j < len; j++) { + const child = readU32(buf, offset); + offset += 4; + traverse(ctx, visitors, child); + } + } else if (kind === PropFlags.String) { + offset += 4; + } else if (kind === PropFlags.Bool) { + offset += 1; + } else if (kind === PropFlags.Null || kind === PropFlags.Undefined) { + // No value + } + } + } finally { + if (exits !== null) { + for (let i = 0; i < exits.length; i++) { + const node = /** @type {*} */ (getNode(ctx, offset)); + exits[i](node); + } + } + } +} + +/** + * This is useful debugging helper to display the buffer's contents. + * @param {AstContext} ctx + */ +function _dump(ctx) { + const { buf, strTableOffset, strTable, strByType, strByProp } = ctx; + + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(strTable); + + for (let i = 0; i < strByType.length; i++) { + const v = strByType[i]; + // @ts-ignore dump fn + // deno-lint-ignore no-console + if (v > 0) console.log(" > type:", i, getString(ctx.strTable, v), v); + } + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(); + for (let i = 0; i < strByProp.length; i++) { + const v = strByProp[i]; + // @ts-ignore dump fn + // deno-lint-ignore no-console + if (v > 0) console.log(" > prop:", i, getString(ctx.strTable, v), v); + } + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(); + + let offset = 0; + + while (offset < strTableOffset) { + const type = buf[offset]; + const name = getString(ctx.strTable, ctx.strByType[type]); + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(`${name}, offset: ${offset}, type: ${type}`); + offset += 1; + + const parent = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` parent: ${parent}`); + + const start = readU32(buf, offset); + offset += 4; + const end = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` range: ${start} -> ${end}`); + + const count = buf[offset++]; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` prop count: ${count}`); + + for (let i = 0; i < count; i++) { + const prop = buf[offset++]; + const kind = buf[offset++]; + const name = getString(ctx.strTable, ctx.strByProp[prop]); + + let kindName = "unknown"; + for (const k in PropFlags) { + // @ts-ignore dump fn + if (kind === PropFlags[k]) { + kindName = k; + } + } + + if (kind === PropFlags.Ref) { + const v = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` ${name}: ${v} (${kindName}, ${prop})`); + } else if (kind === PropFlags.RefArr) { + const len = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` ${name}: Array(${len}) (${kindName}, ${prop})`); + + for (let j = 0; j < len; j++) { + const v = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` - ${v} (${prop})`); + } + } else if (kind === PropFlags.Bool) { + const v = buf[offset]; + offset += 1; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` ${name}: ${v} (${kindName}, ${prop})`); + } else if (kind === PropFlags.String) { + const v = readU32(buf, offset); + offset += 4; + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log( + ` ${name}: ${getString(ctx.strTable, v)} (${kindName}, ${prop})`, + ); + } else if (kind === PropFlags.Null) { + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` ${name}: null (${kindName}, ${prop})`); + } else if (kind === PropFlags.Undefined) { + // @ts-ignore dump fn + // deno-lint-ignore no-console + console.log(` ${name}: undefined (${kindName}, ${prop})`); + } + } + } +} + +// TODO(bartlomieju): this is temporary, until we get plugins plumbed through +// the CLI linter +/** + * @param {LintPlugin} plugin + * @param {string} fileName + * @param {string} sourceText + */ +function runLintPlugin(plugin, fileName, sourceText) { + installPlugin(plugin); + const serializedAst = op_lint_create_serialized_ast(fileName, sourceText); + + try { + runPluginsForFile(fileName, serializedAst); + } finally { + // During testing we don't want to keep plugins around + state.installedPlugins.clear(); + } +} + +// TODO(bartlomieju): this is temporary, until we get plugins plumbed through +// the CLI linter +internals.runLintPlugin = runLintPlugin; diff --git a/cli/js/40_lint_types.d.ts b/cli/js/40_lint_types.d.ts new file mode 100644 index 0000000000..8c252f10ad --- /dev/null +++ b/cli/js/40_lint_types.d.ts @@ -0,0 +1,50 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +export interface NodeFacade { + type: string; + range: [number, number]; + [key: string]: unknown; +} + +export interface AstContext { + buf: Uint8Array; + strTable: Map; + strTableOffset: number; + rootOffset: number; + nodes: Map; + strByType: number[]; + strByProp: number[]; + typeByStr: Map; + propByStr: Map; +} + +// TODO(@marvinhagemeister) Remove once we land "official" types +export interface RuleContext { + id: string; +} + +// TODO(@marvinhagemeister) Remove once we land "official" types +export interface LintRule { + create(ctx: RuleContext): Record void>; + destroy?(ctx: RuleContext): void; +} + +// TODO(@marvinhagemeister) Remove once we land "official" types +export interface LintPlugin { + name: string; + rules: Record; +} + +export interface LintState { + plugins: LintPlugin[]; + installedPlugins: Set; +} + +export type VisitorFn = (node: unknown) => void; + +export interface CompiledVisitor { + matcher: (offset: number) => boolean; + info: { enter: VisitorFn; exit: VisitorFn }; +} + +export {}; diff --git a/cli/ops/lint.rs b/cli/ops/lint.rs new file mode 100644 index 0000000000..c38ac0c8a2 --- /dev/null +++ b/cli/ops/lint.rs @@ -0,0 +1,34 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_ast::MediaType; +use deno_ast::ModuleSpecifier; +use deno_core::error::generic_error; +use deno_core::error::AnyError; +use deno_core::op2; + +use crate::tools::lint; + +deno_core::extension!(deno_lint, ops = [op_lint_create_serialized_ast,],); + +#[op2] +#[buffer] +fn op_lint_create_serialized_ast( + #[string] file_name: &str, + #[string] source: String, +) -> Result, AnyError> { + let file_text = deno_ast::strip_bom(source); + let path = std::env::current_dir()?.join(file_name); + let specifier = ModuleSpecifier::from_file_path(&path).map_err(|_| { + generic_error(format!("Failed to parse path as URL: {}", path.display())) + })?; + let media_type = MediaType::from_specifier(&specifier); + let parsed_source = deno_ast::parse_program(deno_ast::ParseParams { + specifier, + text: file_text.into(), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + })?; + Ok(lint::serialize_ast_to_buffer(&parsed_source)) +} diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs index 230d268ab4..4ac1618816 100644 --- a/cli/ops/mod.rs +++ b/cli/ops/mod.rs @@ -2,4 +2,5 @@ pub mod bench; pub mod jupyter; +pub mod lint; pub mod testing; diff --git a/cli/tools/lint/ast_buffer/buffer.rs b/cli/tools/lint/ast_buffer/buffer.rs new file mode 100644 index 0000000000..c440b73ccd --- /dev/null +++ b/cli/tools/lint/ast_buffer/buffer.rs @@ -0,0 +1,516 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::fmt::Display; + +use deno_ast::swc::common::Span; +use deno_ast::swc::common::DUMMY_SP; +use indexmap::IndexMap; + +/// Each property has this flag to mark what kind of value it holds- +/// Plain objects and arrays are not supported yet, but could be easily +/// added if needed. +#[derive(Debug, PartialEq)] +pub enum PropFlags { + Ref, + RefArr, + String, + Bool, + Null, + Undefined, +} + +impl From for u8 { + fn from(m: PropFlags) -> u8 { + m as u8 + } +} + +impl TryFrom for PropFlags { + type Error = &'static str; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(PropFlags::Ref), + 1 => Ok(PropFlags::RefArr), + 2 => Ok(PropFlags::String), + 3 => Ok(PropFlags::Bool), + 4 => Ok(PropFlags::Null), + 5 => Ok(PropFlags::Undefined), + _ => Err("Unknown Prop flag"), + } + } +} + +const MASK_U32_1: u32 = 0b11111111_00000000_00000000_00000000; +const MASK_U32_2: u32 = 0b00000000_11111111_00000000_00000000; +const MASK_U32_3: u32 = 0b00000000_00000000_11111111_00000000; +const MASK_U32_4: u32 = 0b00000000_00000000_00000000_11111111; + +// TODO: There is probably a native Rust function to do this. +pub fn append_u32(result: &mut Vec, value: u32) { + let v1: u8 = ((value & MASK_U32_1) >> 24) as u8; + let v2: u8 = ((value & MASK_U32_2) >> 16) as u8; + let v3: u8 = ((value & MASK_U32_3) >> 8) as u8; + let v4: u8 = (value & MASK_U32_4) as u8; + + result.push(v1); + result.push(v2); + result.push(v3); + result.push(v4); +} + +pub fn append_usize(result: &mut Vec, value: usize) { + let raw = u32::try_from(value).unwrap(); + append_u32(result, raw); +} + +pub fn write_usize(result: &mut [u8], value: usize, idx: usize) { + let raw = u32::try_from(value).unwrap(); + + let v1: u8 = ((raw & MASK_U32_1) >> 24) as u8; + let v2: u8 = ((raw & MASK_U32_2) >> 16) as u8; + let v3: u8 = ((raw & MASK_U32_3) >> 8) as u8; + let v4: u8 = (raw & MASK_U32_4) as u8; + + result[idx] = v1; + result[idx + 1] = v2; + result[idx + 2] = v3; + result[idx + 3] = v4; +} + +#[derive(Debug)] +pub struct StringTable { + id: usize, + table: IndexMap, +} + +impl StringTable { + pub fn new() -> Self { + Self { + id: 0, + table: IndexMap::new(), + } + } + + pub fn insert(&mut self, s: &str) -> usize { + if let Some(id) = self.table.get(s) { + return *id; + } + + let id = self.id; + self.id += 1; + self.table.insert(s.to_string(), id); + id + } + + pub fn serialize(&mut self) -> Vec { + let mut result: Vec = vec![]; + append_u32(&mut result, self.table.len() as u32); + + // Assume that it's sorted by id + for (s, _id) in &self.table { + let bytes = s.as_bytes(); + append_u32(&mut result, bytes.len() as u32); + result.append(&mut bytes.to_vec()); + } + + result + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct NodeRef(pub usize); + +#[derive(Debug)] +pub struct BoolPos(pub usize); +#[derive(Debug)] +pub struct FieldPos(pub usize); +#[derive(Debug)] +pub struct FieldArrPos(pub usize); +#[derive(Debug)] +pub struct StrPos(pub usize); +#[derive(Debug)] +pub struct UndefPos(pub usize); +#[derive(Debug)] +pub struct NullPos(pub usize); + +#[derive(Debug)] +pub enum NodePos { + Bool(BoolPos), + #[allow(dead_code)] + Field(FieldPos), + #[allow(dead_code)] + FieldArr(FieldArrPos), + Str(StrPos), + Undef(UndefPos), + #[allow(dead_code)] + Null(NullPos), +} + +pub trait AstBufSerializer +where + K: Into + Display, + P: Into + Display, +{ + fn header( + &mut self, + kind: K, + parent: NodeRef, + span: &Span, + prop_count: usize, + ) -> NodeRef; + fn ref_field(&mut self, prop: P) -> FieldPos; + fn ref_vec_field(&mut self, prop: P, len: usize) -> FieldArrPos; + fn str_field(&mut self, prop: P) -> StrPos; + fn bool_field(&mut self, prop: P) -> BoolPos; + fn undefined_field(&mut self, prop: P) -> UndefPos; + #[allow(dead_code)] + fn null_field(&mut self, prop: P) -> NullPos; + + fn write_ref(&mut self, pos: FieldPos, value: NodeRef); + fn write_maybe_ref(&mut self, pos: FieldPos, value: Option); + fn write_refs(&mut self, pos: FieldArrPos, value: Vec); + fn write_str(&mut self, pos: StrPos, value: &str); + fn write_bool(&mut self, pos: BoolPos, value: bool); + + fn serialize(&mut self) -> Vec; +} + +#[derive(Debug)] +pub struct SerializeCtx { + buf: Vec, + start_buf: NodeRef, + str_table: StringTable, + kind_map: Vec, + prop_map: Vec, +} + +/// This is the internal context used to allocate and fill the buffer. The point +/// is to be able to write absolute offsets directly in place. +/// +/// The typical workflow is to reserve all necessary space for the currrent +/// node with placeholders for the offsets of the child nodes. Once child +/// nodes have been traversed, we know their offsets and can replace the +/// placeholder values with the actual ones. +impl SerializeCtx { + pub fn new(kind_len: u8, prop_len: u8) -> Self { + let kind_size = kind_len as usize; + let prop_size = prop_len as usize; + let mut ctx = Self { + start_buf: NodeRef(0), + buf: vec![], + str_table: StringTable::new(), + kind_map: vec![0; kind_size + 1], + prop_map: vec![0; prop_size + 1], + }; + + ctx.str_table.insert(""); + + // Placeholder node is always 0 + ctx.append_node(0, NodeRef(0), &DUMMY_SP, 0); + ctx.kind_map[0] = 0; + ctx.start_buf = NodeRef(ctx.buf.len()); + + // Insert default props that are always present + let type_str = ctx.str_table.insert("type"); + let parent_str = ctx.str_table.insert("parent"); + let range_str = ctx.str_table.insert("range"); + + // These values are expected to be in this order on the JS side + ctx.prop_map[0] = type_str; + ctx.prop_map[1] = parent_str; + ctx.prop_map[2] = range_str; + + ctx + } + + /// Allocate a node's header + fn field_header

(&mut self, prop: P, prop_flags: PropFlags) -> usize + where + P: Into + Display + Clone, + { + let offset = self.buf.len(); + + let n: u8 = prop.clone().into(); + self.buf.push(n); + + if let Some(v) = self.prop_map.get::(n.into()) { + if *v == 0 { + let id = self.str_table.insert(&format!("{prop}")); + self.prop_map[n as usize] = id; + } + } + + let flags: u8 = prop_flags.into(); + self.buf.push(flags); + + offset + } + + /// Allocate a property pointing to another node. + fn field

(&mut self, prop: P, prop_flags: PropFlags) -> usize + where + P: Into + Display + Clone, + { + let offset = self.field_header(prop, prop_flags); + + append_usize(&mut self.buf, 0); + + offset + } + + fn append_node( + &mut self, + kind: u8, + parent: NodeRef, + span: &Span, + prop_count: usize, + ) -> NodeRef { + let offset = self.buf.len(); + + // Node type fits in a u8 + self.buf.push(kind); + + // Offset to the parent node. Will be 0 if none exists + append_usize(&mut self.buf, parent.0); + + // Span, the start and end location of this node + append_u32(&mut self.buf, span.lo.0); + append_u32(&mut self.buf, span.hi.0); + + // No node has more than <10 properties + debug_assert!(prop_count < 10); + self.buf.push(prop_count as u8); + + NodeRef(offset) + } + + /// Allocate the node header. It's always the same for every node. + /// + /// + /// + /// + /// (There is no node with more than 10 properties) + pub fn header( + &mut self, + kind: N, + parent: NodeRef, + span: &Span, + prop_count: usize, + ) -> NodeRef + where + N: Into + Display + Clone, + { + let n: u8 = kind.clone().into(); + + if let Some(v) = self.kind_map.get::(n.into()) { + if *v == 0 { + let id = self.str_table.insert(&format!("{kind}")); + self.kind_map[n as usize] = id; + } + } + + self.append_node(n, parent, span, prop_count) + } + + /// Allocate a reference property that will hold the offset of + /// another node. + pub fn ref_field

(&mut self, prop: P) -> usize + where + P: Into + Display + Clone, + { + self.field(prop, PropFlags::Ref) + } + + /// Allocate a property that is a vec of node offsets pointing to other + /// nodes. + pub fn ref_vec_field

(&mut self, prop: P, len: usize) -> usize + where + P: Into + Display + Clone, + { + let offset = self.field(prop, PropFlags::RefArr); + + for _ in 0..len { + append_u32(&mut self.buf, 0); + } + + offset + } + + // Allocate a property representing a string. Strings are deduplicated + // in the message and the property will only contain the string id. + pub fn str_field

(&mut self, prop: P) -> usize + where + P: Into + Display + Clone, + { + self.field(prop, PropFlags::String) + } + + /// Allocate a bool field + pub fn bool_field

(&mut self, prop: P) -> usize + where + P: Into + Display + Clone, + { + let offset = self.field_header(prop, PropFlags::Bool); + self.buf.push(0); + offset + } + + /// Allocate an undefined field + pub fn undefined_field

(&mut self, prop: P) -> usize + where + P: Into + Display + Clone, + { + self.field_header(prop, PropFlags::Undefined) + } + + /// Allocate an undefined field + #[allow(dead_code)] + pub fn null_field

(&mut self, prop: P) -> usize + where + P: Into + Display + Clone, + { + self.field_header(prop, PropFlags::Null) + } + + /// Replace the placeholder of a reference field with the actual offset + /// to the node we want to point to. + pub fn write_ref(&mut self, field_offset: usize, value: NodeRef) { + #[cfg(debug_assertions)] + { + let value_kind = self.buf[field_offset + 1]; + if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref { + panic!("Trying to write a ref into a non-ref field") + } + } + + write_usize(&mut self.buf, value.0, field_offset + 2); + } + + /// Helper for writing optional node offsets + pub fn write_maybe_ref( + &mut self, + field_offset: usize, + value: Option, + ) { + #[cfg(debug_assertions)] + { + let value_kind = self.buf[field_offset + 1]; + if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref { + panic!("Trying to write a ref into a non-ref field") + } + } + + let ref_value = if let Some(v) = value { v } else { NodeRef(0) }; + write_usize(&mut self.buf, ref_value.0, field_offset + 2); + } + + /// Write a vec of node offsets into the property. The necessary space + /// has been reserved earlier. + pub fn write_refs(&mut self, field_offset: usize, value: Vec) { + #[cfg(debug_assertions)] + { + let value_kind = self.buf[field_offset + 1]; + if PropFlags::try_from(value_kind).unwrap() != PropFlags::RefArr { + panic!("Trying to write a ref into a non-ref array field") + } + } + + let mut offset = field_offset + 2; + write_usize(&mut self.buf, value.len(), offset); + offset += 4; + + for item in value { + write_usize(&mut self.buf, item.0, offset); + offset += 4; + } + } + + /// Store the string in our string table and save the id of the string + /// in the current field. + pub fn write_str(&mut self, field_offset: usize, value: &str) { + #[cfg(debug_assertions)] + { + let value_kind = self.buf[field_offset + 1]; + if PropFlags::try_from(value_kind).unwrap() != PropFlags::String { + panic!("Trying to write a ref into a non-string field") + } + } + + let id = self.str_table.insert(value); + write_usize(&mut self.buf, id, field_offset + 2); + } + + /// Write a bool to a field. + pub fn write_bool(&mut self, field_offset: usize, value: bool) { + #[cfg(debug_assertions)] + { + let value_kind = self.buf[field_offset + 1]; + if PropFlags::try_from(value_kind).unwrap() != PropFlags::Bool { + panic!("Trying to write a ref into a non-bool field") + } + } + + self.buf[field_offset + 2] = if value { 1 } else { 0 }; + } + + /// Serialize all information we have into a buffer that can be sent to JS. + /// It has the following structure: + /// + /// <...ast> + /// + /// <- node kind id maps to string id + /// <- node property id maps to string id + /// + /// + /// + pub fn serialize(&mut self) -> Vec { + let mut buf: Vec = vec![]; + + // The buffer starts with the serialized AST first, because that + // contains absolute offsets. By butting this at the start of the + // message we don't have to waste time updating any offsets. + buf.append(&mut self.buf); + + // Next follows the string table. We'll keep track of the offset + // in the message of where the string table begins + let offset_str_table = buf.len(); + + // Serialize string table + buf.append(&mut self.str_table.serialize()); + + // Next, serialize the mappings of kind -> string of encountered + // nodes in the AST. We use this additional lookup table to compress + // the message so that we can save space by using a u8 . All nodes of + // JS, TS and JSX together are <200 + let offset_kind_map = buf.len(); + + // Write the total number of entries in the kind -> str mapping table + // TODO: make this a u8 + append_usize(&mut buf, self.kind_map.len()); + for v in &self.kind_map { + append_usize(&mut buf, *v); + } + + // Store offset to prop -> string map. It's the same as with node kind + // as the total number of properties is <120 which allows us to store it + // as u8. + let offset_prop_map = buf.len(); + // Write the total number of entries in the kind -> str mapping table + append_usize(&mut buf, self.prop_map.len()); + for v in &self.prop_map { + append_usize(&mut buf, *v); + } + + // Putting offsets of relevant parts of the buffer at the end. This + // allows us to hop to the relevant part by merely looking at the last + // for values in the message. Each value represents an offset into the + // buffer. + append_usize(&mut buf, offset_kind_map); + append_usize(&mut buf, offset_prop_map); + append_usize(&mut buf, offset_str_table); + append_usize(&mut buf, self.start_buf.0); + + buf + } +} diff --git a/cli/tools/lint/ast_buffer/mod.rs b/cli/tools/lint/ast_buffer/mod.rs new file mode 100644 index 0000000000..8838bcc5f2 --- /dev/null +++ b/cli/tools/lint/ast_buffer/mod.rs @@ -0,0 +1,13 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_ast::ParsedSource; +use swc::serialize_swc_to_buffer; + +mod buffer; +mod swc; +mod ts_estree; + +pub fn serialize_ast_to_buffer(parsed_source: &ParsedSource) -> Vec { + // TODO: We could support multiple languages here + serialize_swc_to_buffer(parsed_source) +} diff --git a/cli/tools/lint/ast_buffer/swc.rs b/cli/tools/lint/ast_buffer/swc.rs new file mode 100644 index 0000000000..785a38a7d8 --- /dev/null +++ b/cli/tools/lint/ast_buffer/swc.rs @@ -0,0 +1,3018 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_ast::swc::ast::AssignTarget; +use deno_ast::swc::ast::AssignTargetPat; +use deno_ast::swc::ast::BlockStmtOrExpr; +use deno_ast::swc::ast::Callee; +use deno_ast::swc::ast::ClassMember; +use deno_ast::swc::ast::Decl; +use deno_ast::swc::ast::ExportSpecifier; +use deno_ast::swc::ast::Expr; +use deno_ast::swc::ast::ExprOrSpread; +use deno_ast::swc::ast::FnExpr; +use deno_ast::swc::ast::ForHead; +use deno_ast::swc::ast::Function; +use deno_ast::swc::ast::Ident; +use deno_ast::swc::ast::IdentName; +use deno_ast::swc::ast::JSXAttrName; +use deno_ast::swc::ast::JSXAttrOrSpread; +use deno_ast::swc::ast::JSXAttrValue; +use deno_ast::swc::ast::JSXElement; +use deno_ast::swc::ast::JSXElementChild; +use deno_ast::swc::ast::JSXElementName; +use deno_ast::swc::ast::JSXEmptyExpr; +use deno_ast::swc::ast::JSXExpr; +use deno_ast::swc::ast::JSXExprContainer; +use deno_ast::swc::ast::JSXFragment; +use deno_ast::swc::ast::JSXMemberExpr; +use deno_ast::swc::ast::JSXNamespacedName; +use deno_ast::swc::ast::JSXObject; +use deno_ast::swc::ast::JSXOpeningElement; +use deno_ast::swc::ast::Lit; +use deno_ast::swc::ast::MemberExpr; +use deno_ast::swc::ast::MemberProp; +use deno_ast::swc::ast::ModuleDecl; +use deno_ast::swc::ast::ModuleExportName; +use deno_ast::swc::ast::ModuleItem; +use deno_ast::swc::ast::ObjectPatProp; +use deno_ast::swc::ast::OptChainBase; +use deno_ast::swc::ast::Param; +use deno_ast::swc::ast::ParamOrTsParamProp; +use deno_ast::swc::ast::Pat; +use deno_ast::swc::ast::PrivateName; +use deno_ast::swc::ast::Program; +use deno_ast::swc::ast::Prop; +use deno_ast::swc::ast::PropName; +use deno_ast::swc::ast::PropOrSpread; +use deno_ast::swc::ast::SimpleAssignTarget; +use deno_ast::swc::ast::Stmt; +use deno_ast::swc::ast::SuperProp; +use deno_ast::swc::ast::Tpl; +use deno_ast::swc::ast::TsEntityName; +use deno_ast::swc::ast::TsEnumMemberId; +use deno_ast::swc::ast::TsFnOrConstructorType; +use deno_ast::swc::ast::TsFnParam; +use deno_ast::swc::ast::TsIndexSignature; +use deno_ast::swc::ast::TsLit; +use deno_ast::swc::ast::TsLitType; +use deno_ast::swc::ast::TsThisTypeOrIdent; +use deno_ast::swc::ast::TsType; +use deno_ast::swc::ast::TsTypeAnn; +use deno_ast::swc::ast::TsTypeElement; +use deno_ast::swc::ast::TsTypeParam; +use deno_ast::swc::ast::TsTypeParamDecl; +use deno_ast::swc::ast::TsTypeParamInstantiation; +use deno_ast::swc::ast::TsTypeQueryExpr; +use deno_ast::swc::ast::TsUnionOrIntersectionType; +use deno_ast::swc::ast::VarDeclOrExpr; +use deno_ast::swc::common::Span; +use deno_ast::swc::common::Spanned; +use deno_ast::swc::common::SyntaxContext; +use deno_ast::view::Accessibility; +use deno_ast::view::AssignOp; +use deno_ast::view::BinaryOp; +use deno_ast::view::TruePlusMinus; +use deno_ast::view::TsKeywordTypeKind; +use deno_ast::view::TsTypeOperatorOp; +use deno_ast::view::UnaryOp; +use deno_ast::view::UpdateOp; +use deno_ast::view::VarDeclKind; +use deno_ast::ParsedSource; + +use super::buffer::AstBufSerializer; +use super::buffer::BoolPos; +use super::buffer::NodePos; +use super::buffer::NodeRef; +use super::buffer::StrPos; +use super::ts_estree::AstNode; +use super::ts_estree::AstProp; +use super::ts_estree::TsEsTreeBuilder; + +pub fn serialize_swc_to_buffer(parsed_source: &ParsedSource) -> Vec { + let mut ctx = TsEsTreeBuilder::new(); + + let program = &parsed_source.program(); + + let pos = ctx.header(AstNode::Program, NodeRef(0), &program.span(), 2); + let source_type_pos = ctx.str_field(AstProp::SourceType); + + match program.as_ref() { + Program::Module(module) => { + let body_pos = ctx.ref_vec_field(AstProp::Body, module.body.len()); + + let children = module + .body + .iter() + .map(|item| match item { + ModuleItem::ModuleDecl(module_decl) => { + serialize_module_decl(&mut ctx, module_decl, pos) + } + ModuleItem::Stmt(stmt) => serialize_stmt(&mut ctx, stmt, pos), + }) + .collect::>(); + + ctx.write_str(source_type_pos, "module"); + ctx.write_refs(body_pos, children); + } + Program::Script(script) => { + let body_pos = ctx.ref_vec_field(AstProp::Body, script.body.len()); + let children = script + .body + .iter() + .map(|stmt| serialize_stmt(&mut ctx, stmt, pos)) + .collect::>(); + + ctx.write_str(source_type_pos, "script"); + ctx.write_refs(body_pos, children); + } + } + + ctx.serialize() +} + +fn serialize_module_decl( + ctx: &mut TsEsTreeBuilder, + module_decl: &ModuleDecl, + parent: NodeRef, +) -> NodeRef { + match module_decl { + ModuleDecl::Import(node) => { + ctx.header(AstNode::ImportExpression, parent, &node.span, 0) + } + ModuleDecl::ExportDecl(node) => { + let pos = + ctx.header(AstNode::ExportNamedDeclaration, parent, &node.span, 1); + let decl_pos = ctx.ref_field(AstProp::Declarations); + + let decl = serialize_decl(ctx, &node.decl, pos); + + ctx.write_ref(decl_pos, decl); + + pos + } + ModuleDecl::ExportNamed(node) => { + let id = + ctx.header(AstNode::ExportNamedDeclaration, parent, &node.span, 2); + let src_pos = ctx.ref_field(AstProp::Source); + let spec_pos = + ctx.ref_vec_field(AstProp::Specifiers, node.specifiers.len()); + + // FIXME: Flags + // let mut flags = FlagValue::new(); + // flags.set(Flag::ExportType); + + let src_id = node + .src + .as_ref() + .map(|src| serialize_lit(ctx, &Lit::Str(*src.clone()), id)); + + let spec_ids = node + .specifiers + .iter() + .map(|spec| { + match spec { + ExportSpecifier::Named(child) => { + let spec_pos = + ctx.header(AstNode::ExportSpecifier, id, &child.span, 2); + let local_pos = ctx.ref_field(AstProp::Local); + let exp_pos = ctx.ref_field(AstProp::Exported); + + // let mut flags = FlagValue::new(); + // flags.set(Flag::ExportType); + + let local = + serialize_module_exported_name(ctx, &child.orig, spec_pos); + + let exported = child.exported.as_ref().map(|exported| { + serialize_module_exported_name(ctx, exported, spec_pos) + }); + + // ctx.write_flags(&flags); + ctx.write_ref(local_pos, local); + ctx.write_maybe_ref(exp_pos, exported); + + spec_pos + } + + // These two aren't syntactically valid + ExportSpecifier::Namespace(_) => todo!(), + ExportSpecifier::Default(_) => todo!(), + } + }) + .collect::>(); + + // ctx.write_flags(&flags); + ctx.write_maybe_ref(src_pos, src_id); + ctx.write_refs(spec_pos, spec_ids); + + id + } + ModuleDecl::ExportDefaultDecl(node) => { + ctx.header(AstNode::ExportDefaultDeclaration, parent, &node.span, 0) + } + ModuleDecl::ExportDefaultExpr(node) => { + ctx.header(AstNode::ExportDefaultDeclaration, parent, &node.span, 0) + } + ModuleDecl::ExportAll(node) => { + ctx.header(AstNode::ExportAllDeclaration, parent, &node.span, 0) + } + ModuleDecl::TsImportEquals(node) => { + ctx.header(AstNode::TsImportEquals, parent, &node.span, 0) + } + ModuleDecl::TsExportAssignment(node) => { + ctx.header(AstNode::TsExportAssignment, parent, &node.span, 0) + } + ModuleDecl::TsNamespaceExport(node) => { + ctx.header(AstNode::TsNamespaceExport, parent, &node.span, 0) + } + } +} + +fn serialize_stmt( + ctx: &mut TsEsTreeBuilder, + stmt: &Stmt, + parent: NodeRef, +) -> NodeRef { + match stmt { + Stmt::Block(node) => { + let pos = ctx.header(AstNode::BlockStatement, parent, &node.span, 1); + let body_pos = ctx.ref_vec_field(AstProp::Body, node.stmts.len()); + + let children = node + .stmts + .iter() + .map(|stmt| serialize_stmt(ctx, stmt, pos)) + .collect::>(); + + ctx.write_refs(body_pos, children); + + pos + } + Stmt::Empty(_) => NodeRef(0), + Stmt::Debugger(node) => { + ctx.header(AstNode::DebuggerStatement, parent, &node.span, 0) + } + Stmt::With(_) => todo!(), + Stmt::Return(node) => { + let pos = ctx.header(AstNode::ReturnStatement, parent, &node.span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = node.arg.as_ref().map(|arg| serialize_expr(ctx, arg, pos)); + ctx.write_maybe_ref(arg_pos, arg); + + pos + } + Stmt::Labeled(node) => { + let pos = ctx.header(AstNode::LabeledStatement, parent, &node.span, 2); + let label_pos = ctx.ref_field(AstProp::Label); + let body_pos = ctx.ref_field(AstProp::Body); + + let ident = serialize_ident(ctx, &node.label, pos); + let stmt = serialize_stmt(ctx, &node.body, pos); + + ctx.write_ref(label_pos, ident); + ctx.write_ref(body_pos, stmt); + + pos + } + Stmt::Break(node) => { + let pos = ctx.header(AstNode::BreakStatement, parent, &node.span, 1); + let label_pos = ctx.ref_field(AstProp::Label); + + let arg = node + .label + .as_ref() + .map(|label| serialize_ident(ctx, label, pos)); + + ctx.write_maybe_ref(label_pos, arg); + + pos + } + Stmt::Continue(node) => { + let pos = ctx.header(AstNode::ContinueStatement, parent, &node.span, 1); + let label_pos = ctx.ref_field(AstProp::Label); + + let arg = node + .label + .as_ref() + .map(|label| serialize_ident(ctx, label, pos)); + + ctx.write_maybe_ref(label_pos, arg); + + pos + } + Stmt::If(node) => { + let pos = ctx.header(AstNode::IfStatement, parent, &node.span, 3); + let test_pos = ctx.ref_field(AstProp::Test); + let cons_pos = ctx.ref_field(AstProp::Consequent); + let alt_pos = ctx.ref_field(AstProp::Alternate); + + let test = serialize_expr(ctx, node.test.as_ref(), pos); + let cons = serialize_stmt(ctx, node.cons.as_ref(), pos); + let alt = node.alt.as_ref().map(|alt| serialize_stmt(ctx, alt, pos)); + + ctx.write_ref(test_pos, test); + ctx.write_ref(cons_pos, cons); + ctx.write_maybe_ref(alt_pos, alt); + + pos + } + Stmt::Switch(node) => { + let id = ctx.header(AstNode::SwitchStatement, parent, &node.span, 2); + let disc_pos = ctx.ref_field(AstProp::Discriminant); + let cases_pos = ctx.ref_vec_field(AstProp::Cases, node.cases.len()); + + let disc = serialize_expr(ctx, &node.discriminant, id); + + let cases = node + .cases + .iter() + .map(|case| { + let case_pos = ctx.header(AstNode::SwitchCase, id, &case.span, 2); + let test_pos = ctx.ref_field(AstProp::Test); + let cons_pos = + ctx.ref_vec_field(AstProp::Consequent, case.cons.len()); + + let test = case + .test + .as_ref() + .map(|test| serialize_expr(ctx, test, case_pos)); + + let cons = case + .cons + .iter() + .map(|cons| serialize_stmt(ctx, cons, case_pos)) + .collect::>(); + + ctx.write_maybe_ref(test_pos, test); + ctx.write_refs(cons_pos, cons); + + case_pos + }) + .collect::>(); + + ctx.write_ref(disc_pos, disc); + ctx.write_refs(cases_pos, cases); + + id + } + Stmt::Throw(node) => { + let pos = ctx.header(AstNode::ThrowStatement, parent, &node.span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = serialize_expr(ctx, &node.arg, pos); + ctx.write_ref(arg_pos, arg); + + pos + } + Stmt::Try(node) => { + let pos = ctx.header(AstNode::TryStatement, parent, &node.span, 3); + let block_pos = ctx.ref_field(AstProp::Block); + let handler_pos = ctx.ref_field(AstProp::Handler); + let finalizer_pos = ctx.ref_field(AstProp::Finalizer); + + let block = serialize_stmt(ctx, &Stmt::Block(node.block.clone()), pos); + + let handler = node.handler.as_ref().map(|catch| { + let clause_pos = ctx.header(AstNode::CatchClause, pos, &catch.span, 2); + let param_pos = ctx.ref_field(AstProp::Param); + let body_pos = ctx.ref_field(AstProp::Body); + + let param = catch + .param + .as_ref() + .map(|param| serialize_pat(ctx, param, clause_pos)); + + let body = + serialize_stmt(ctx, &Stmt::Block(catch.body.clone()), clause_pos); + + ctx.write_maybe_ref(param_pos, param); + ctx.write_ref(body_pos, body); + + clause_pos + }); + + let finalizer = node.finalizer.as_ref().map(|finalizer| { + serialize_stmt(ctx, &Stmt::Block(finalizer.clone()), pos) + }); + + ctx.write_ref(block_pos, block); + ctx.write_maybe_ref(handler_pos, handler); + ctx.write_maybe_ref(finalizer_pos, finalizer); + + pos + } + Stmt::While(node) => { + let pos = ctx.header(AstNode::WhileStatement, parent, &node.span, 2); + let test_pos = ctx.ref_field(AstProp::Test); + let body_pos = ctx.ref_field(AstProp::Body); + + let test = serialize_expr(ctx, node.test.as_ref(), pos); + let stmt = serialize_stmt(ctx, node.body.as_ref(), pos); + + ctx.write_ref(test_pos, test); + ctx.write_ref(body_pos, stmt); + + pos + } + Stmt::DoWhile(node) => { + let pos = ctx.header(AstNode::DoWhileStatement, parent, &node.span, 2); + let test_pos = ctx.ref_field(AstProp::Test); + let body_pos = ctx.ref_field(AstProp::Body); + + let expr = serialize_expr(ctx, node.test.as_ref(), pos); + let stmt = serialize_stmt(ctx, node.body.as_ref(), pos); + + ctx.write_ref(test_pos, expr); + ctx.write_ref(body_pos, stmt); + + pos + } + Stmt::For(node) => { + let pos = ctx.header(AstNode::ForStatement, parent, &node.span, 4); + let init_pos = ctx.ref_field(AstProp::Init); + let test_pos = ctx.ref_field(AstProp::Test); + let update_pos = ctx.ref_field(AstProp::Update); + let body_pos = ctx.ref_field(AstProp::Body); + + let init = node.init.as_ref().map(|init| match init { + VarDeclOrExpr::VarDecl(var_decl) => { + serialize_stmt(ctx, &Stmt::Decl(Decl::Var(var_decl.clone())), pos) + } + VarDeclOrExpr::Expr(expr) => serialize_expr(ctx, expr, pos), + }); + + let test = node + .test + .as_ref() + .map(|expr| serialize_expr(ctx, expr, pos)); + let update = node + .update + .as_ref() + .map(|expr| serialize_expr(ctx, expr, pos)); + let body = serialize_stmt(ctx, node.body.as_ref(), pos); + + ctx.write_maybe_ref(init_pos, init); + ctx.write_maybe_ref(test_pos, test); + ctx.write_maybe_ref(update_pos, update); + ctx.write_ref(body_pos, body); + + pos + } + Stmt::ForIn(node) => { + let pos = ctx.header(AstNode::ForInStatement, parent, &node.span, 3); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + let body_pos = ctx.ref_field(AstProp::Body); + + let left = serialize_for_head(ctx, &node.left, pos); + let right = serialize_expr(ctx, node.right.as_ref(), pos); + let body = serialize_stmt(ctx, node.body.as_ref(), pos); + + ctx.write_ref(left_pos, left); + ctx.write_ref(right_pos, right); + ctx.write_ref(body_pos, body); + + pos + } + Stmt::ForOf(node) => { + let pos = ctx.header(AstNode::ForOfStatement, parent, &node.span, 4); + let await_pos = ctx.bool_field(AstProp::Await); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + let body_pos = ctx.ref_field(AstProp::Body); + + let left = serialize_for_head(ctx, &node.left, pos); + let right = serialize_expr(ctx, node.right.as_ref(), pos); + let body = serialize_stmt(ctx, node.body.as_ref(), pos); + + ctx.write_bool(await_pos, node.is_await); + ctx.write_ref(left_pos, left); + ctx.write_ref(right_pos, right); + ctx.write_ref(body_pos, body); + + pos + } + Stmt::Decl(node) => serialize_decl(ctx, node, parent), + Stmt::Expr(node) => { + let pos = ctx.header(AstNode::ExpressionStatement, parent, &node.span, 1); + let expr_pos = ctx.ref_field(AstProp::Expression); + + let expr = serialize_expr(ctx, node.expr.as_ref(), pos); + ctx.write_ref(expr_pos, expr); + + pos + } + } +} + +fn serialize_expr( + ctx: &mut TsEsTreeBuilder, + expr: &Expr, + parent: NodeRef, +) -> NodeRef { + match expr { + Expr::This(node) => { + ctx.header(AstNode::ThisExpression, parent, &node.span, 0) + } + Expr::Array(node) => { + let pos = ctx.header(AstNode::ArrayExpression, parent, &node.span, 1); + let elems_pos = ctx.ref_vec_field(AstProp::Elements, node.elems.len()); + + let elems = node + .elems + .iter() + .map(|item| { + item + .as_ref() + .map_or(NodeRef(0), |item| serialize_expr_or_spread(ctx, item, pos)) + }) + .collect::>(); + + ctx.write_refs(elems_pos, elems); + + pos + } + Expr::Object(node) => { + let pos = ctx.header(AstNode::ObjectExpression, parent, &node.span, 1); + let props_pos = ctx.ref_vec_field(AstProp::Properties, node.props.len()); + + let prop_ids = node + .props + .iter() + .map(|prop| serialize_prop_or_spread(ctx, prop, pos)) + .collect::>(); + + ctx.write_refs(props_pos, prop_ids); + + pos + } + Expr::Fn(node) => { + let fn_obj = node.function.as_ref(); + + let pos = + ctx.header(AstNode::FunctionExpression, parent, &fn_obj.span, 7); + + let async_pos = ctx.bool_field(AstProp::Async); + let gen_pos = ctx.bool_field(AstProp::Generator); + let id_pos = ctx.ref_field(AstProp::Id); + let tparams_pos = ctx.ref_field(AstProp::TypeParameters); + let params_pos = ctx.ref_vec_field(AstProp::Params, fn_obj.params.len()); + let return_pos = ctx.ref_field(AstProp::ReturnType); + let body_pos = ctx.ref_field(AstProp::Body); + + let ident = node + .ident + .as_ref() + .map(|ident| serialize_ident(ctx, ident, pos)); + + let type_params = + maybe_serialize_ts_type_param(ctx, &fn_obj.type_params, pos); + + let params = fn_obj + .params + .iter() + .map(|param| serialize_pat(ctx, ¶m.pat, pos)) + .collect::>(); + + let return_id = + maybe_serialize_ts_type_ann(ctx, &fn_obj.return_type, pos); + let body = fn_obj + .body + .as_ref() + .map(|block| serialize_stmt(ctx, &Stmt::Block(block.clone()), pos)); + + ctx.write_bool(async_pos, fn_obj.is_async); + ctx.write_bool(gen_pos, fn_obj.is_generator); + ctx.write_maybe_ref(id_pos, ident); + ctx.write_maybe_ref(tparams_pos, type_params); + ctx.write_refs(params_pos, params); + ctx.write_maybe_ref(return_pos, return_id); + ctx.write_maybe_ref(body_pos, body); + + pos + } + Expr::Unary(node) => { + let pos = ctx.header(AstNode::UnaryExpression, parent, &node.span, 2); + let flag_pos = ctx.str_field(AstProp::Operator); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = serialize_expr(ctx, &node.arg, pos); + + ctx.write_str( + flag_pos, + match node.op { + UnaryOp::Minus => "-", + UnaryOp::Plus => "+", + UnaryOp::Bang => "!", + UnaryOp::Tilde => "~", + UnaryOp::TypeOf => "typeof", + UnaryOp::Void => "void", + UnaryOp::Delete => "delete", + }, + ); + ctx.write_ref(arg_pos, arg); + + pos + } + Expr::Update(node) => { + let pos = ctx.header(AstNode::UpdateExpression, parent, &node.span, 3); + let prefix_pos = ctx.bool_field(AstProp::Prefix); + let arg_pos = ctx.ref_field(AstProp::Argument); + let op_ops = ctx.str_field(AstProp::Operator); + + let arg = serialize_expr(ctx, node.arg.as_ref(), pos); + + ctx.write_bool(prefix_pos, node.prefix); + ctx.write_ref(arg_pos, arg); + ctx.write_str( + op_ops, + match node.op { + UpdateOp::PlusPlus => "++", + UpdateOp::MinusMinus => "--", + }, + ); + + pos + } + Expr::Bin(node) => { + let (node_type, flag_str) = match node.op { + BinaryOp::LogicalAnd => (AstNode::LogicalExpression, "&&"), + BinaryOp::LogicalOr => (AstNode::LogicalExpression, "||"), + BinaryOp::NullishCoalescing => (AstNode::LogicalExpression, "??"), + BinaryOp::EqEq => (AstNode::BinaryExpression, "=="), + BinaryOp::NotEq => (AstNode::BinaryExpression, "!="), + BinaryOp::EqEqEq => (AstNode::BinaryExpression, "==="), + BinaryOp::NotEqEq => (AstNode::BinaryExpression, "!="), + BinaryOp::Lt => (AstNode::BinaryExpression, "<"), + BinaryOp::LtEq => (AstNode::BinaryExpression, "<="), + BinaryOp::Gt => (AstNode::BinaryExpression, ">"), + BinaryOp::GtEq => (AstNode::BinaryExpression, ">="), + BinaryOp::LShift => (AstNode::BinaryExpression, "<<"), + BinaryOp::RShift => (AstNode::BinaryExpression, ">>"), + BinaryOp::ZeroFillRShift => (AstNode::BinaryExpression, ">>>"), + BinaryOp::Add => (AstNode::BinaryExpression, "+"), + BinaryOp::Sub => (AstNode::BinaryExpression, "-"), + BinaryOp::Mul => (AstNode::BinaryExpression, "*"), + BinaryOp::Div => (AstNode::BinaryExpression, "/"), + BinaryOp::Mod => (AstNode::BinaryExpression, "%"), + BinaryOp::BitOr => (AstNode::BinaryExpression, "|"), + BinaryOp::BitXor => (AstNode::BinaryExpression, "^"), + BinaryOp::BitAnd => (AstNode::BinaryExpression, "&"), + BinaryOp::In => (AstNode::BinaryExpression, "in"), + BinaryOp::InstanceOf => (AstNode::BinaryExpression, "instanceof"), + BinaryOp::Exp => (AstNode::BinaryExpression, "**"), + }; + + let pos = ctx.header(node_type, parent, &node.span, 3); + let op_pos = ctx.str_field(AstProp::Operator); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + + let left_id = serialize_expr(ctx, node.left.as_ref(), pos); + let right_id = serialize_expr(ctx, node.right.as_ref(), pos); + + ctx.write_str(op_pos, flag_str); + ctx.write_ref(left_pos, left_id); + ctx.write_ref(right_pos, right_id); + + pos + } + Expr::Assign(node) => { + let pos = + ctx.header(AstNode::AssignmentExpression, parent, &node.span, 3); + let op_pos = ctx.str_field(AstProp::Operator); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + + let left = match &node.left { + AssignTarget::Simple(simple_assign_target) => { + match simple_assign_target { + SimpleAssignTarget::Ident(target) => { + serialize_ident(ctx, &target.id, pos) + } + SimpleAssignTarget::Member(target) => { + serialize_expr(ctx, &Expr::Member(target.clone()), pos) + } + SimpleAssignTarget::SuperProp(target) => { + serialize_expr(ctx, &Expr::SuperProp(target.clone()), pos) + } + SimpleAssignTarget::Paren(target) => { + serialize_expr(ctx, &target.expr, pos) + } + SimpleAssignTarget::OptChain(target) => { + serialize_expr(ctx, &Expr::OptChain(target.clone()), pos) + } + SimpleAssignTarget::TsAs(target) => { + serialize_expr(ctx, &Expr::TsAs(target.clone()), pos) + } + SimpleAssignTarget::TsSatisfies(target) => { + serialize_expr(ctx, &Expr::TsSatisfies(target.clone()), pos) + } + SimpleAssignTarget::TsNonNull(target) => { + serialize_expr(ctx, &Expr::TsNonNull(target.clone()), pos) + } + SimpleAssignTarget::TsTypeAssertion(target) => { + serialize_expr(ctx, &Expr::TsTypeAssertion(target.clone()), pos) + } + SimpleAssignTarget::TsInstantiation(target) => { + serialize_expr(ctx, &Expr::TsInstantiation(target.clone()), pos) + } + SimpleAssignTarget::Invalid(_) => unreachable!(), + } + } + AssignTarget::Pat(target) => match target { + AssignTargetPat::Array(array_pat) => { + serialize_pat(ctx, &Pat::Array(array_pat.clone()), pos) + } + AssignTargetPat::Object(object_pat) => { + serialize_pat(ctx, &Pat::Object(object_pat.clone()), pos) + } + AssignTargetPat::Invalid(_) => unreachable!(), + }, + }; + + let right = serialize_expr(ctx, node.right.as_ref(), pos); + + ctx.write_str( + op_pos, + match node.op { + AssignOp::Assign => "=", + AssignOp::AddAssign => "+=", + AssignOp::SubAssign => "-=", + AssignOp::MulAssign => "*=", + AssignOp::DivAssign => "/=", + AssignOp::ModAssign => "%=", + AssignOp::LShiftAssign => "<<=", + AssignOp::RShiftAssign => ">>=", + AssignOp::ZeroFillRShiftAssign => ">>>=", + AssignOp::BitOrAssign => "|=", + AssignOp::BitXorAssign => "^=", + AssignOp::BitAndAssign => "&=", + AssignOp::ExpAssign => "**=", + AssignOp::AndAssign => "&&=", + AssignOp::OrAssign => "||=", + AssignOp::NullishAssign => "??=", + }, + ); + ctx.write_ref(left_pos, left); + ctx.write_ref(right_pos, right); + + pos + } + Expr::Member(node) => serialize_member_expr(ctx, node, parent, false), + Expr::SuperProp(node) => { + let pos = ctx.header(AstNode::MemberExpression, parent, &node.span, 3); + let computed_pos = ctx.bool_field(AstProp::Computed); + let obj_pos = ctx.ref_field(AstProp::Object); + let prop_pos = ctx.ref_field(AstProp::Property); + + let obj = ctx.header(AstNode::Super, pos, &node.obj.span, 0); + + let mut computed = false; + let prop = match &node.prop { + SuperProp::Ident(ident_name) => { + serialize_ident_name(ctx, ident_name, pos) + } + SuperProp::Computed(prop) => { + computed = true; + serialize_expr(ctx, &prop.expr, pos) + } + }; + + ctx.write_bool(computed_pos, computed); + ctx.write_ref(obj_pos, obj); + ctx.write_ref(prop_pos, prop); + + pos + } + Expr::Cond(node) => { + let pos = + ctx.header(AstNode::ConditionalExpression, parent, &node.span, 3); + let test_pos = ctx.ref_field(AstProp::Test); + let cons_pos = ctx.ref_field(AstProp::Consequent); + let alt_pos = ctx.ref_field(AstProp::Alternate); + + let test = serialize_expr(ctx, node.test.as_ref(), pos); + let cons = serialize_expr(ctx, node.cons.as_ref(), pos); + let alt = serialize_expr(ctx, node.alt.as_ref(), pos); + + ctx.write_ref(test_pos, test); + ctx.write_ref(cons_pos, cons); + ctx.write_ref(alt_pos, alt); + + pos + } + Expr::Call(node) => { + let pos = ctx.header(AstNode::CallExpression, parent, &node.span, 4); + let opt_pos = ctx.bool_field(AstProp::Optional); + let callee_pos = ctx.ref_field(AstProp::Callee); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + let args_pos = ctx.ref_vec_field(AstProp::Arguments, node.args.len()); + + let callee = match &node.callee { + Callee::Super(super_node) => { + ctx.header(AstNode::Super, pos, &super_node.span, 0) + } + Callee::Import(_) => todo!(), + Callee::Expr(expr) => serialize_expr(ctx, expr, pos), + }; + + let type_arg = node.type_args.clone().map(|param_node| { + serialize_ts_param_inst(ctx, param_node.as_ref(), pos) + }); + + let args = node + .args + .iter() + .map(|arg| serialize_expr_or_spread(ctx, arg, pos)) + .collect::>(); + + ctx.write_bool(opt_pos, false); + ctx.write_ref(callee_pos, callee); + ctx.write_maybe_ref(type_args_pos, type_arg); + ctx.write_refs(args_pos, args); + + pos + } + Expr::New(node) => { + let pos = ctx.header(AstNode::NewExpression, parent, &node.span, 3); + let callee_pos = ctx.ref_field(AstProp::Callee); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + let args_pos = ctx.ref_vec_field( + AstProp::Arguments, + node.args.as_ref().map_or(0, |v| v.len()), + ); + + let callee = serialize_expr(ctx, node.callee.as_ref(), pos); + + let args: Vec = node.args.as_ref().map_or(vec![], |args| { + args + .iter() + .map(|arg| serialize_expr_or_spread(ctx, arg, pos)) + .collect::>() + }); + + let type_args = node.type_args.clone().map(|param_node| { + serialize_ts_param_inst(ctx, param_node.as_ref(), pos) + }); + + ctx.write_ref(callee_pos, callee); + ctx.write_maybe_ref(type_args_pos, type_args); + ctx.write_refs(args_pos, args); + + pos + } + Expr::Seq(node) => { + let pos = ctx.header(AstNode::SequenceExpression, parent, &node.span, 1); + let exprs_pos = ctx.ref_vec_field(AstProp::Expressions, node.exprs.len()); + + let children = node + .exprs + .iter() + .map(|expr| serialize_expr(ctx, expr, pos)) + .collect::>(); + + ctx.write_refs(exprs_pos, children); + + pos + } + Expr::Ident(node) => serialize_ident(ctx, node, parent), + Expr::Lit(node) => serialize_lit(ctx, node, parent), + Expr::Tpl(node) => { + let pos = ctx.header(AstNode::TemplateLiteral, parent, &node.span, 2); + let quasis_pos = ctx.ref_vec_field(AstProp::Quasis, node.quasis.len()); + let exprs_pos = ctx.ref_vec_field(AstProp::Expressions, node.exprs.len()); + + let quasis = node + .quasis + .iter() + .map(|quasi| { + let tpl_pos = + ctx.header(AstNode::TemplateElement, pos, &quasi.span, 3); + let tail_pos = ctx.bool_field(AstProp::Tail); + let raw_pos = ctx.str_field(AstProp::Raw); + let cooked_pos = ctx.str_field(AstProp::Cooked); + + ctx.write_bool(tail_pos, quasi.tail); + ctx.write_str(raw_pos, &quasi.raw); + ctx.write_str( + cooked_pos, + &quasi + .cooked + .as_ref() + .map_or("".to_string(), |v| v.to_string()), + ); + + tpl_pos + }) + .collect::>(); + + let exprs = node + .exprs + .iter() + .map(|expr| serialize_expr(ctx, expr, pos)) + .collect::>(); + + ctx.write_refs(quasis_pos, quasis); + ctx.write_refs(exprs_pos, exprs); + + pos + } + Expr::TaggedTpl(node) => { + let pos = + ctx.header(AstNode::TaggedTemplateExpression, parent, &node.span, 3); + let tag_pos = ctx.ref_field(AstProp::Tag); + let type_arg_pos = ctx.ref_field(AstProp::TypeArguments); + let quasi_pos = ctx.ref_field(AstProp::Quasi); + + let tag = serialize_expr(ctx, &node.tag, pos); + + let type_param_id = node + .type_params + .clone() + .map(|params| serialize_ts_param_inst(ctx, params.as_ref(), pos)); + let quasi = serialize_expr(ctx, &Expr::Tpl(*node.tpl.clone()), pos); + + ctx.write_ref(tag_pos, tag); + ctx.write_maybe_ref(type_arg_pos, type_param_id); + ctx.write_ref(quasi_pos, quasi); + + pos + } + Expr::Arrow(node) => { + let pos = + ctx.header(AstNode::ArrowFunctionExpression, parent, &node.span, 6); + let async_pos = ctx.bool_field(AstProp::Async); + let gen_pos = ctx.bool_field(AstProp::Generator); + let type_param_pos = ctx.ref_field(AstProp::TypeParameters); + let params_pos = ctx.ref_vec_field(AstProp::Params, node.params.len()); + let body_pos = ctx.ref_field(AstProp::Body); + let return_type_pos = ctx.ref_field(AstProp::ReturnType); + + let type_param = + maybe_serialize_ts_type_param(ctx, &node.type_params, pos); + + let params = node + .params + .iter() + .map(|param| serialize_pat(ctx, param, pos)) + .collect::>(); + + let body = match node.body.as_ref() { + BlockStmtOrExpr::BlockStmt(block_stmt) => { + serialize_stmt(ctx, &Stmt::Block(block_stmt.clone()), pos) + } + BlockStmtOrExpr::Expr(expr) => serialize_expr(ctx, expr.as_ref(), pos), + }; + + let return_type = + maybe_serialize_ts_type_ann(ctx, &node.return_type, pos); + + ctx.write_bool(async_pos, node.is_async); + ctx.write_bool(gen_pos, node.is_generator); + ctx.write_maybe_ref(type_param_pos, type_param); + ctx.write_refs(params_pos, params); + ctx.write_ref(body_pos, body); + ctx.write_maybe_ref(return_type_pos, return_type); + + pos + } + Expr::Class(node) => { + // FIXME + ctx.header(AstNode::ClassExpression, parent, &node.class.span, 0) + } + Expr::Yield(node) => { + let pos = ctx.header(AstNode::YieldExpression, parent, &node.span, 2); + let delegate_pos = ctx.bool_field(AstProp::Delegate); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = node + .arg + .as_ref() + .map(|arg| serialize_expr(ctx, arg.as_ref(), pos)); + + ctx.write_bool(delegate_pos, node.delegate); + ctx.write_maybe_ref(arg_pos, arg); + + pos + } + Expr::MetaProp(node) => { + ctx.header(AstNode::MetaProp, parent, &node.span, 0) + } + Expr::Await(node) => { + let pos = ctx.header(AstNode::AwaitExpression, parent, &node.span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = serialize_expr(ctx, node.arg.as_ref(), pos); + + ctx.write_ref(arg_pos, arg); + + pos + } + Expr::Paren(node) => { + // Paren nodes are treated as a syntax only thing in TSEStree + // and are never materialized to actual AST nodes. + serialize_expr(ctx, &node.expr, parent) + } + Expr::JSXMember(node) => serialize_jsx_member_expr(ctx, node, parent), + Expr::JSXNamespacedName(node) => { + serialize_jsx_namespaced_name(ctx, node, parent) + } + Expr::JSXEmpty(node) => serialize_jsx_empty_expr(ctx, node, parent), + Expr::JSXElement(node) => serialize_jsx_element(ctx, node, parent), + Expr::JSXFragment(node) => serialize_jsx_fragment(ctx, node, parent), + Expr::TsTypeAssertion(node) => { + let pos = ctx.header(AstNode::TSTypeAssertion, parent, &node.span, 2); + let expr_pos = ctx.ref_field(AstProp::Expression); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let expr = serialize_expr(ctx, &node.expr, parent); + let type_ann = serialize_ts_type(ctx, &node.type_ann, pos); + + ctx.write_ref(expr_pos, expr); + ctx.write_ref(type_ann_pos, type_ann); + + pos + } + Expr::TsConstAssertion(node) => { + let pos = ctx.header(AstNode::TsConstAssertion, parent, &node.span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + let arg = serialize_expr(ctx, node.expr.as_ref(), pos); + + // FIXME + ctx.write_ref(arg_pos, arg); + + pos + } + Expr::TsNonNull(node) => { + let pos = ctx.header(AstNode::TSNonNullExpression, parent, &node.span, 1); + let expr_pos = ctx.ref_field(AstProp::Expression); + + let expr_id = serialize_expr(ctx, node.expr.as_ref(), pos); + + ctx.write_ref(expr_pos, expr_id); + + pos + } + Expr::TsAs(node) => { + let id = ctx.header(AstNode::TSAsExpression, parent, &node.span, 2); + let expr_pos = ctx.ref_field(AstProp::Expression); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let expr = serialize_expr(ctx, node.expr.as_ref(), id); + let type_ann = serialize_ts_type(ctx, node.type_ann.as_ref(), id); + + ctx.write_ref(expr_pos, expr); + ctx.write_ref(type_ann_pos, type_ann); + + id + } + Expr::TsInstantiation(node) => { + let pos = ctx.header(AstNode::TsInstantiation, parent, &node.span, 1); + let expr_pos = ctx.ref_field(AstProp::Expression); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + + let expr = serialize_expr(ctx, node.expr.as_ref(), pos); + + let type_arg = serialize_ts_param_inst(ctx, node.type_args.as_ref(), pos); + + ctx.write_ref(expr_pos, expr); + ctx.write_ref(type_args_pos, type_arg); + + pos + } + Expr::TsSatisfies(node) => { + let pos = + ctx.header(AstNode::TSSatisfiesExpression, parent, &node.span, 2); + let expr_pos = ctx.ref_field(AstProp::Expression); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let epxr = serialize_expr(ctx, node.expr.as_ref(), pos); + let type_ann = serialize_ts_type(ctx, node.type_ann.as_ref(), pos); + + ctx.write_ref(expr_pos, epxr); + ctx.write_ref(type_ann_pos, type_ann); + + pos + } + Expr::PrivateName(node) => serialize_private_name(ctx, node, parent), + Expr::OptChain(node) => { + let pos = ctx.header(AstNode::ChainExpression, parent, &node.span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = match node.base.as_ref() { + OptChainBase::Member(member_expr) => { + serialize_member_expr(ctx, member_expr, pos, true) + } + OptChainBase::Call(opt_call) => { + let call_pos = + ctx.header(AstNode::CallExpression, pos, &opt_call.span, 4); + let opt_pos = ctx.bool_field(AstProp::Optional); + let callee_pos = ctx.ref_field(AstProp::Callee); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + let args_pos = + ctx.ref_vec_field(AstProp::Arguments, opt_call.args.len()); + + let callee = serialize_expr(ctx, &opt_call.callee, pos); + + let type_param_id = opt_call.type_args.clone().map(|params| { + serialize_ts_param_inst(ctx, params.as_ref(), call_pos) + }); + + let args = opt_call + .args + .iter() + .map(|arg| serialize_expr_or_spread(ctx, arg, pos)) + .collect::>(); + + ctx.write_bool(opt_pos, true); + ctx.write_ref(callee_pos, callee); + ctx.write_maybe_ref(type_args_pos, type_param_id); + ctx.write_refs(args_pos, args); + + call_pos + } + }; + + ctx.write_ref(arg_pos, arg); + + pos + } + Expr::Invalid(_) => { + unreachable!() + } + } +} + +fn serialize_prop_or_spread( + ctx: &mut TsEsTreeBuilder, + prop: &PropOrSpread, + parent: NodeRef, +) -> NodeRef { + match prop { + PropOrSpread::Spread(spread_element) => serialize_spread( + ctx, + spread_element.expr.as_ref(), + &spread_element.dot3_token, + parent, + ), + PropOrSpread::Prop(prop) => { + let pos = ctx.header(AstNode::Property, parent, &prop.span(), 6); + + let shorthand_pos = ctx.bool_field(AstProp::Shorthand); + let computed_pos = ctx.bool_field(AstProp::Computed); + let method_pos = ctx.bool_field(AstProp::Method); + let kind_pos = ctx.str_field(AstProp::Kind); + let key_pos = ctx.ref_field(AstProp::Key); + let value_pos = ctx.ref_field(AstProp::Value); + + let mut shorthand = false; + let mut computed = false; + let mut method = false; + let mut kind = "init"; + + // FIXME: optional + let (key_id, value_id) = match prop.as_ref() { + Prop::Shorthand(ident) => { + shorthand = true; + + let value = serialize_ident(ctx, ident, pos); + (value, value) + } + Prop::KeyValue(key_value_prop) => { + if let PropName::Computed(_) = key_value_prop.key { + computed = true; + } + + let key = serialize_prop_name(ctx, &key_value_prop.key, pos); + let value = serialize_expr(ctx, key_value_prop.value.as_ref(), pos); + + (key, value) + } + Prop::Assign(assign_prop) => { + let child_id = + ctx.header(AstNode::AssignmentPattern, pos, &assign_prop.span, 2); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + + let left = serialize_ident(ctx, &assign_prop.key, child_id); + let right = serialize_expr(ctx, assign_prop.value.as_ref(), child_id); + + ctx.write_ref(left_pos, left); + ctx.write_ref(right_pos, right); + + (left, child_id) + } + Prop::Getter(getter_prop) => { + kind = "get"; + + let key = serialize_prop_name(ctx, &getter_prop.key, pos); + + let value = serialize_expr( + ctx, + &Expr::Fn(FnExpr { + ident: None, + function: Box::new(Function { + params: vec![], + decorators: vec![], + span: getter_prop.span, + ctxt: SyntaxContext::empty(), + body: getter_prop.body.clone(), + is_generator: false, + is_async: false, + type_params: None, // FIXME + return_type: None, + }), + }), + pos, + ); + + (key, value) + } + Prop::Setter(setter_prop) => { + kind = "set"; + + let key_id = serialize_prop_name(ctx, &setter_prop.key, pos); + + let param = Param::from(*setter_prop.param.clone()); + + let value_id = serialize_expr( + ctx, + &Expr::Fn(FnExpr { + ident: None, + function: Box::new(Function { + params: vec![param], + decorators: vec![], + span: setter_prop.span, + ctxt: SyntaxContext::empty(), + body: setter_prop.body.clone(), + is_generator: false, + is_async: false, + type_params: None, + return_type: None, + }), + }), + pos, + ); + + (key_id, value_id) + } + Prop::Method(method_prop) => { + method = true; + + let key_id = serialize_prop_name(ctx, &method_prop.key, pos); + + let value_id = serialize_expr( + ctx, + &Expr::Fn(FnExpr { + ident: None, + function: method_prop.function.clone(), + }), + pos, + ); + + (key_id, value_id) + } + }; + + ctx.write_bool(shorthand_pos, shorthand); + ctx.write_bool(computed_pos, computed); + ctx.write_bool(method_pos, method); + ctx.write_str(kind_pos, kind); + ctx.write_ref(key_pos, key_id); + ctx.write_ref(value_pos, value_id); + + pos + } + } +} + +fn serialize_member_expr( + ctx: &mut TsEsTreeBuilder, + node: &MemberExpr, + parent: NodeRef, + optional: bool, +) -> NodeRef { + let pos = ctx.header(AstNode::MemberExpression, parent, &node.span, 4); + let opt_pos = ctx.bool_field(AstProp::Optional); + let computed_pos = ctx.bool_field(AstProp::Computed); + let obj_pos = ctx.ref_field(AstProp::Object); + let prop_pos = ctx.ref_field(AstProp::Property); + + let obj = serialize_expr(ctx, node.obj.as_ref(), pos); + + let mut computed = false; + + let prop = match &node.prop { + MemberProp::Ident(ident_name) => serialize_ident_name(ctx, ident_name, pos), + MemberProp::PrivateName(private_name) => { + serialize_private_name(ctx, private_name, pos) + } + MemberProp::Computed(computed_prop_name) => { + computed = true; + serialize_expr(ctx, computed_prop_name.expr.as_ref(), pos) + } + }; + + ctx.write_bool(opt_pos, optional); + ctx.write_bool(computed_pos, computed); + ctx.write_ref(obj_pos, obj); + ctx.write_ref(prop_pos, prop); + + pos +} + +fn serialize_class_member( + ctx: &mut TsEsTreeBuilder, + member: &ClassMember, + parent: NodeRef, +) -> NodeRef { + match member { + ClassMember::Constructor(constructor) => { + let member_id = + ctx.header(AstNode::MethodDefinition, parent, &constructor.span, 3); + let key_pos = ctx.ref_field(AstProp::Key); + let body_pos = ctx.ref_field(AstProp::Body); + let args_pos = + ctx.ref_vec_field(AstProp::Arguments, constructor.params.len()); + let acc_pos = if constructor.accessibility.is_some() { + NodePos::Str(ctx.str_field(AstProp::Accessibility)) + } else { + NodePos::Undef(ctx.undefined_field(AstProp::Accessibility)) + }; + + // FIXME flags + + let key = serialize_prop_name(ctx, &constructor.key, member_id); + let body = constructor + .body + .as_ref() + .map(|body| serialize_stmt(ctx, &Stmt::Block(body.clone()), member_id)); + + let params = constructor + .params + .iter() + .map(|param| match param { + ParamOrTsParamProp::TsParamProp(_) => { + todo!() + } + ParamOrTsParamProp::Param(param) => { + serialize_pat(ctx, ¶m.pat, member_id) + } + }) + .collect::>(); + + if let Some(acc) = constructor.accessibility { + if let NodePos::Str(str_pos) = acc_pos { + ctx.write_str(str_pos, &accessibility_to_str(acc)); + } + } + + ctx.write_ref(key_pos, key); + ctx.write_maybe_ref(body_pos, body); + // FIXME + ctx.write_refs(args_pos, params); + + member_id + } + ClassMember::Method(method) => { + let member_id = + ctx.header(AstNode::MethodDefinition, parent, &method.span, 0); + + // let mut flags = FlagValue::new(); + // flags.set(Flag::ClassMethod); + if method.function.is_async { + // FIXME + } + + // accessibility_to_flag(&mut flags, method.accessibility); + + let _key_id = serialize_prop_name(ctx, &method.key, member_id); + + let _body_id = + method.function.body.as_ref().map(|body| { + serialize_stmt(ctx, &Stmt::Block(body.clone()), member_id) + }); + + let _params = method + .function + .params + .iter() + .map(|param| serialize_pat(ctx, ¶m.pat, member_id)) + .collect::>(); + + // ctx.write_node(member_id, ); + // ctx.write_flags(&flags); + // ctx.write_id(key_id); + // ctx.write_id(body_id); + // ctx.write_ids(AstProp::Params, params); + + member_id + } + ClassMember::PrivateMethod(_) => todo!(), + ClassMember::ClassProp(_) => todo!(), + ClassMember::PrivateProp(_) => todo!(), + ClassMember::TsIndexSignature(member) => { + serialize_ts_index_sig(ctx, member, parent) + } + ClassMember::Empty(_) => unreachable!(), + ClassMember::StaticBlock(_) => todo!(), + ClassMember::AutoAccessor(_) => todo!(), + } +} + +fn serialize_expr_or_spread( + ctx: &mut TsEsTreeBuilder, + arg: &ExprOrSpread, + parent: NodeRef, +) -> NodeRef { + if let Some(spread) = &arg.spread { + serialize_spread(ctx, &arg.expr, spread, parent) + } else { + serialize_expr(ctx, arg.expr.as_ref(), parent) + } +} + +fn serialize_ident( + ctx: &mut TsEsTreeBuilder, + ident: &Ident, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::Identifier, parent, &ident.span, 1); + let name_pos = ctx.str_field(AstProp::Name); + ctx.write_str(name_pos, ident.sym.as_str()); + + pos +} + +fn serialize_module_exported_name( + ctx: &mut TsEsTreeBuilder, + name: &ModuleExportName, + parent: NodeRef, +) -> NodeRef { + match &name { + ModuleExportName::Ident(ident) => serialize_ident(ctx, ident, parent), + ModuleExportName::Str(lit) => { + serialize_lit(ctx, &Lit::Str(lit.clone()), parent) + } + } +} + +fn serialize_decl( + ctx: &mut TsEsTreeBuilder, + decl: &Decl, + parent: NodeRef, +) -> NodeRef { + match decl { + Decl::Class(node) => { + let id = + ctx.header(AstNode::ClassDeclaration, parent, &node.class.span, 8); + let declare_pos = ctx.bool_field(AstProp::Declare); + let abstract_pos = ctx.bool_field(AstProp::Abstract); + let id_pos = ctx.ref_field(AstProp::Id); + let body_pos = ctx.ref_field(AstProp::Body); + let type_params_pos = ctx.ref_field(AstProp::TypeParameters); + let super_pos = ctx.ref_field(AstProp::SuperClass); + let super_type_pos = ctx.ref_field(AstProp::SuperTypeArguments); + let impl_pos = + ctx.ref_vec_field(AstProp::Implements, node.class.implements.len()); + + let body_id = ctx.header(AstNode::ClassBody, id, &node.class.span, 1); + let body_body_pos = + ctx.ref_vec_field(AstProp::Body, node.class.body.len()); + + let ident = serialize_ident(ctx, &node.ident, id); + let type_params = + maybe_serialize_ts_type_param(ctx, &node.class.type_params, id); + + let super_class = node + .class + .super_class + .as_ref() + .map(|super_class| serialize_expr(ctx, super_class, id)); + + let super_type_params = node + .class + .super_type_params + .as_ref() + .map(|super_params| serialize_ts_param_inst(ctx, super_params, id)); + + let implement_ids = node + .class + .implements + .iter() + .map(|implements| { + let child_pos = + ctx.header(AstNode::TSClassImplements, id, &implements.span, 2); + + let expr_pos = ctx.ref_field(AstProp::Expression); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + + let type_args = implements + .type_args + .clone() + .map(|args| serialize_ts_param_inst(ctx, &args, child_pos)); + + let expr = serialize_expr(ctx, &implements.expr, child_pos); + + ctx.write_ref(expr_pos, expr); + ctx.write_maybe_ref(type_args_pos, type_args); + + child_pos + }) + .collect::>(); + + let member_ids = node + .class + .body + .iter() + .map(|member| serialize_class_member(ctx, member, parent)) + .collect::>(); + + ctx.write_ref(body_pos, body_id); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_bool(abstract_pos, node.class.is_abstract); + ctx.write_ref(id_pos, ident); + ctx.write_maybe_ref(type_params_pos, type_params); + ctx.write_maybe_ref(super_pos, super_class); + ctx.write_maybe_ref(super_type_pos, super_type_params); + ctx.write_refs(impl_pos, implement_ids); + + // body + ctx.write_refs(body_body_pos, member_ids); + + id + } + Decl::Fn(node) => { + let pos = ctx.header( + AstNode::FunctionDeclaration, + parent, + &node.function.span, + 8, + ); + let declare_pos = ctx.bool_field(AstProp::Declare); + let async_pos = ctx.bool_field(AstProp::Async); + let gen_pos = ctx.bool_field(AstProp::Generator); + let id_pos = ctx.ref_field(AstProp::Id); + let type_params_pos = ctx.ref_field(AstProp::TypeParameters); + let return_pos = ctx.ref_field(AstProp::ReturnType); + let body_pos = ctx.ref_field(AstProp::Body); + let params_pos = + ctx.ref_vec_field(AstProp::Params, node.function.params.len()); + + let ident_id = serialize_ident(ctx, &node.ident, parent); + let type_param_id = + maybe_serialize_ts_type_param(ctx, &node.function.type_params, pos); + let return_type = + maybe_serialize_ts_type_ann(ctx, &node.function.return_type, pos); + + let body = node + .function + .body + .as_ref() + .map(|body| serialize_stmt(ctx, &Stmt::Block(body.clone()), pos)); + + let params = node + .function + .params + .iter() + .map(|param| serialize_pat(ctx, ¶m.pat, pos)) + .collect::>(); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_bool(async_pos, node.function.is_async); + ctx.write_bool(gen_pos, node.function.is_generator); + ctx.write_ref(id_pos, ident_id); + ctx.write_maybe_ref(type_params_pos, type_param_id); + ctx.write_maybe_ref(return_pos, return_type); + ctx.write_maybe_ref(body_pos, body); + ctx.write_refs(params_pos, params); + + pos + } + Decl::Var(node) => { + let id = ctx.header(AstNode::VariableDeclaration, parent, &node.span, 3); + let declare_pos = ctx.bool_field(AstProp::Declare); + let kind_pos = ctx.str_field(AstProp::Kind); + let decls_pos = + ctx.ref_vec_field(AstProp::Declarations, node.decls.len()); + + let children = node + .decls + .iter() + .map(|decl| { + let child_id = + ctx.header(AstNode::VariableDeclarator, id, &decl.span, 2); + let id_pos = ctx.ref_field(AstProp::Id); + let init_pos = ctx.ref_field(AstProp::Init); + + // FIXME: Definite? + + let ident = serialize_pat(ctx, &decl.name, child_id); + + let init = decl + .init + .as_ref() + .map(|init| serialize_expr(ctx, init.as_ref(), child_id)); + + ctx.write_ref(id_pos, ident); + ctx.write_maybe_ref(init_pos, init); + + child_id + }) + .collect::>(); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_str( + kind_pos, + match node.kind { + VarDeclKind::Var => "var", + VarDeclKind::Let => "let", + VarDeclKind::Const => "const", + }, + ); + ctx.write_refs(decls_pos, children); + + id + } + Decl::Using(_) => { + todo!(); + } + Decl::TsInterface(node) => { + let pos = ctx.header(AstNode::TSInterface, parent, &node.span, 0); + let declare_pos = ctx.bool_field(AstProp::Declare); + let id_pos = ctx.ref_field(AstProp::Id); + let extends_pos = ctx.ref_vec_field(AstProp::Extends, node.extends.len()); + let type_param_pos = ctx.ref_field(AstProp::TypeParameters); + let body_pos = ctx.ref_field(AstProp::Body); + + let body_id = + ctx.header(AstNode::TSInterfaceBody, pos, &node.body.span, 0); + let body_body_pos = + ctx.ref_vec_field(AstProp::Body, node.body.body.len()); + + let ident_id = serialize_ident(ctx, &node.id, pos); + let type_param = + maybe_serialize_ts_type_param(ctx, &node.type_params, pos); + + let extend_ids = node + .extends + .iter() + .map(|item| { + let child_pos = + ctx.header(AstNode::TSInterfaceHeritage, pos, &item.span, 1); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + let expr_pos = ctx.ref_field(AstProp::Expression); + + let expr = serialize_expr(ctx, &item.expr, child_pos); + let type_args = item.type_args.clone().map(|params| { + serialize_ts_param_inst(ctx, params.as_ref(), child_pos) + }); + + ctx.write_ref(expr_pos, expr); + ctx.write_maybe_ref(type_args_pos, type_args); + + child_pos + }) + .collect::>(); + + let body_elem_ids = node + .body + .body + .iter() + .map(|item| match item { + TsTypeElement::TsCallSignatureDecl(ts_call) => { + let item_id = ctx.header( + AstNode::TsCallSignatureDeclaration, + pos, + &ts_call.span, + 3, + ); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + let params_pos = + ctx.ref_vec_field(AstProp::Params, ts_call.params.len()); + let return_pos = ctx.ref_field(AstProp::ReturnType); + + let type_param = + maybe_serialize_ts_type_param(ctx, &ts_call.type_params, pos); + let return_type = + maybe_serialize_ts_type_ann(ctx, &ts_call.type_ann, pos); + let params = ts_call + .params + .iter() + .map(|param| serialize_ts_fn_param(ctx, param, pos)) + .collect::>(); + + ctx.write_maybe_ref(type_ann_pos, type_param); + ctx.write_refs(params_pos, params); + ctx.write_maybe_ref(return_pos, return_type); + + item_id + } + TsTypeElement::TsConstructSignatureDecl(_) => todo!(), + TsTypeElement::TsPropertySignature(sig) => { + let item_pos = + ctx.header(AstNode::TSPropertySignature, pos, &sig.span, 6); + + let computed_pos = ctx.bool_field(AstProp::Computed); + let optional_pos = ctx.bool_field(AstProp::Optional); + let readonly_pos = ctx.bool_field(AstProp::Readonly); + // TODO: where is this coming from? + let _static_bos = ctx.bool_field(AstProp::Static); + let key_pos = ctx.ref_field(AstProp::Key); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let key = serialize_expr(ctx, &sig.key, item_pos); + let type_ann = + maybe_serialize_ts_type_ann(ctx, &sig.type_ann, item_pos); + + ctx.write_bool(computed_pos, sig.computed); + ctx.write_bool(optional_pos, sig.optional); + ctx.write_bool(readonly_pos, sig.readonly); + ctx.write_ref(key_pos, key); + ctx.write_maybe_ref(type_ann_pos, type_ann); + + item_pos + } + TsTypeElement::TsGetterSignature(sig) => { + let item_pos = + ctx.header(AstNode::TSMethodSignature, pos, &sig.span, 6); + let computed_pos = ctx.bool_field(AstProp::Computed); + let optional_pos = ctx.bool_field(AstProp::Optional); + let readonly_pos = ctx.bool_field(AstProp::Readonly); + // TODO: where is this coming from? + let _static_bos = ctx.bool_field(AstProp::Static); + let kind_pos = ctx.str_field(AstProp::Kind); + let key_pos = ctx.ref_field(AstProp::Key); + let return_type_pos = ctx.ref_field(AstProp::ReturnType); + + let key = serialize_expr(ctx, sig.key.as_ref(), item_pos); + let return_type = + maybe_serialize_ts_type_ann(ctx, &sig.type_ann, item_pos); + + ctx.write_bool(computed_pos, false); + ctx.write_bool(optional_pos, false); + ctx.write_bool(readonly_pos, false); + ctx.write_str(kind_pos, "getter"); + ctx.write_maybe_ref(return_type_pos, return_type); + ctx.write_ref(key_pos, key); + + item_pos + } + TsTypeElement::TsSetterSignature(sig) => { + let item_pos = + ctx.header(AstNode::TSMethodSignature, pos, &sig.span, 6); + let computed_pos = ctx.bool_field(AstProp::Computed); + let optional_pos = ctx.bool_field(AstProp::Optional); + let readonly_pos = ctx.bool_field(AstProp::Readonly); + // TODO: where is this coming from? + let _static_bos = ctx.bool_field(AstProp::Static); + let kind_pos = ctx.str_field(AstProp::Kind); + let key_pos = ctx.ref_field(AstProp::Key); + let params_pos = ctx.ref_vec_field(AstProp::Params, 1); + + let key = serialize_expr(ctx, sig.key.as_ref(), item_pos); + let params = serialize_ts_fn_param(ctx, &sig.param, item_pos); + + ctx.write_bool(computed_pos, false); + ctx.write_bool(optional_pos, false); + ctx.write_bool(readonly_pos, false); + ctx.write_str(kind_pos, "setter"); + ctx.write_ref(key_pos, key); + ctx.write_refs(params_pos, vec![params]); + + item_pos + } + TsTypeElement::TsMethodSignature(sig) => { + let item_pos = + ctx.header(AstNode::TSMethodSignature, pos, &sig.span, 8); + let computed_pos = ctx.bool_field(AstProp::Computed); + let optional_pos = ctx.bool_field(AstProp::Optional); + let readonly_pos = ctx.bool_field(AstProp::Readonly); + // TODO: where is this coming from? + let _static_bos = ctx.bool_field(AstProp::Static); + let kind_pos = ctx.str_field(AstProp::Kind); + let key_pos = ctx.ref_field(AstProp::Key); + let params_pos = + ctx.ref_vec_field(AstProp::Params, sig.params.len()); + let return_type_pos = ctx.ref_field(AstProp::ReturnType); + + let key = serialize_expr(ctx, sig.key.as_ref(), item_pos); + let params = sig + .params + .iter() + .map(|param| serialize_ts_fn_param(ctx, param, item_pos)) + .collect::>(); + let return_type = + maybe_serialize_ts_type_ann(ctx, &sig.type_ann, item_pos); + + ctx.write_bool(computed_pos, false); + ctx.write_bool(optional_pos, false); + ctx.write_bool(readonly_pos, false); + ctx.write_str(kind_pos, "method"); + ctx.write_ref(key_pos, key); + ctx.write_refs(params_pos, params); + ctx.write_maybe_ref(return_type_pos, return_type); + + item_pos + } + TsTypeElement::TsIndexSignature(sig) => { + serialize_ts_index_sig(ctx, sig, pos) + } + }) + .collect::>(); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_ref(id_pos, ident_id); + ctx.write_maybe_ref(type_param_pos, type_param); + ctx.write_refs(extends_pos, extend_ids); + ctx.write_ref(body_pos, body_id); + + // Body + ctx.write_refs(body_body_pos, body_elem_ids); + + pos + } + Decl::TsTypeAlias(node) => { + let pos = ctx.header(AstNode::TsTypeAlias, parent, &node.span, 4); + let declare_pos = ctx.bool_field(AstProp::Declare); + let id_pos = ctx.ref_field(AstProp::Id); + let type_params_pos = ctx.ref_field(AstProp::TypeParameters); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let ident = serialize_ident(ctx, &node.id, pos); + let type_ann = serialize_ts_type(ctx, &node.type_ann, pos); + let type_param = + maybe_serialize_ts_type_param(ctx, &node.type_params, pos); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_ref(id_pos, ident); + ctx.write_maybe_ref(type_params_pos, type_param); + ctx.write_ref(type_ann_pos, type_ann); + + pos + } + Decl::TsEnum(node) => { + let pos = ctx.header(AstNode::TSEnumDeclaration, parent, &node.span, 3); + let declare_pos = ctx.bool_field(AstProp::Declare); + let const_pos = ctx.bool_field(AstProp::Const); + let id_pos = ctx.ref_field(AstProp::Id); + let body_pos = ctx.ref_field(AstProp::Body); + + let body = ctx.header(AstNode::TSEnumBody, pos, &node.span, 1); + let members_pos = ctx.ref_vec_field(AstProp::Members, node.members.len()); + + let ident_id = serialize_ident(ctx, &node.id, parent); + + let members = node + .members + .iter() + .map(|member| { + let member_id = + ctx.header(AstNode::TSEnumMember, body, &member.span, 2); + let id_pos = ctx.ref_field(AstProp::Id); + let init_pos = ctx.ref_field(AstProp::Initializer); + + let ident = match &member.id { + TsEnumMemberId::Ident(ident) => { + serialize_ident(ctx, ident, member_id) + } + TsEnumMemberId::Str(lit_str) => { + serialize_lit(ctx, &Lit::Str(lit_str.clone()), member_id) + } + }; + + let init = member + .init + .as_ref() + .map(|init| serialize_expr(ctx, init, member_id)); + + ctx.write_ref(id_pos, ident); + ctx.write_maybe_ref(init_pos, init); + + member_id + }) + .collect::>(); + + ctx.write_refs(members_pos, members); + + ctx.write_bool(declare_pos, node.declare); + ctx.write_bool(const_pos, node.is_const); + ctx.write_ref(id_pos, ident_id); + ctx.write_ref(body_pos, body); + + pos + } + Decl::TsModule(ts_module_decl) => { + ctx.header(AstNode::TsModule, parent, &ts_module_decl.span, 0) + } + } +} + +fn serialize_ts_index_sig( + ctx: &mut TsEsTreeBuilder, + node: &TsIndexSignature, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::TSMethodSignature, parent, &node.span, 4); + let readonly_pos = ctx.bool_field(AstProp::Readonly); + // TODO: where is this coming from? + let static_pos = ctx.bool_field(AstProp::Static); + let params_pos = ctx.ref_vec_field(AstProp::Params, node.params.len()); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let type_ann = maybe_serialize_ts_type_ann(ctx, &node.type_ann, pos); + + let params = node + .params + .iter() + .map(|param| serialize_ts_fn_param(ctx, param, pos)) + .collect::>(); + + ctx.write_bool(readonly_pos, false); + ctx.write_bool(static_pos, node.is_static); + ctx.write_refs(params_pos, params); + ctx.write_maybe_ref(type_ann_pos, type_ann); + + pos +} + +fn accessibility_to_str(accessibility: Accessibility) -> String { + match accessibility { + Accessibility::Public => "public".to_string(), + Accessibility::Protected => "protected".to_string(), + Accessibility::Private => "private".to_string(), + } +} + +fn serialize_private_name( + ctx: &mut TsEsTreeBuilder, + node: &PrivateName, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::PrivateIdentifier, parent, &node.span, 1); + let name_pos = ctx.str_field(AstProp::Name); + + ctx.write_str(name_pos, node.name.as_str()); + + pos +} + +fn serialize_jsx_element( + ctx: &mut TsEsTreeBuilder, + node: &JSXElement, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXElement, parent, &node.span, 3); + let open_pos = ctx.ref_field(AstProp::OpeningElement); + let close_pos = ctx.ref_field(AstProp::ClosingElement); + let children_pos = ctx.ref_vec_field(AstProp::Children, node.children.len()); + + let open = serialize_jsx_opening_element(ctx, &node.opening, pos); + + let close = node.closing.as_ref().map(|closing| { + let closing_pos = + ctx.header(AstNode::JSXClosingElement, pos, &closing.span, 1); + let name_pos = ctx.ref_field(AstProp::Name); + + let name = serialize_jsx_element_name(ctx, &closing.name, closing_pos); + ctx.write_ref(name_pos, name); + + closing_pos + }); + + let children = serialize_jsx_children(ctx, &node.children, pos); + + ctx.write_ref(open_pos, open); + ctx.write_maybe_ref(close_pos, close); + ctx.write_refs(children_pos, children); + + pos +} + +fn serialize_jsx_fragment( + ctx: &mut TsEsTreeBuilder, + node: &JSXFragment, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXFragment, parent, &node.span, 3); + + let opening_pos = ctx.ref_field(AstProp::OpeningFragment); + let closing_pos = ctx.ref_field(AstProp::ClosingFragment); + let children_pos = ctx.ref_vec_field(AstProp::Children, node.children.len()); + + let opening_id = + ctx.header(AstNode::JSXOpeningFragment, pos, &node.opening.span, 0); + let closing_id = + ctx.header(AstNode::JSXClosingFragment, pos, &node.closing.span, 0); + + let children = serialize_jsx_children(ctx, &node.children, pos); + + ctx.write_ref(opening_pos, opening_id); + ctx.write_ref(closing_pos, closing_id); + ctx.write_refs(children_pos, children); + + pos +} + +fn serialize_jsx_children( + ctx: &mut TsEsTreeBuilder, + children: &[JSXElementChild], + parent: NodeRef, +) -> Vec { + children + .iter() + .map(|child| { + match child { + JSXElementChild::JSXText(text) => { + let pos = ctx.header(AstNode::JSXText, parent, &text.span, 2); + let raw_pos = ctx.str_field(AstProp::Raw); + let value_pos = ctx.str_field(AstProp::Value); + + ctx.write_str(raw_pos, &text.raw); + ctx.write_str(value_pos, &text.value); + + pos + } + JSXElementChild::JSXExprContainer(container) => { + serialize_jsx_container_expr(ctx, container, parent) + } + JSXElementChild::JSXElement(el) => { + serialize_jsx_element(ctx, el, parent) + } + JSXElementChild::JSXFragment(frag) => { + serialize_jsx_fragment(ctx, frag, parent) + } + // No parser supports this + JSXElementChild::JSXSpreadChild(_) => unreachable!(), + } + }) + .collect::>() +} + +fn serialize_jsx_member_expr( + ctx: &mut TsEsTreeBuilder, + node: &JSXMemberExpr, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXMemberExpression, parent, &node.span, 2); + let obj_ref = ctx.ref_field(AstProp::Object); + let prop_ref = ctx.ref_field(AstProp::Property); + + let obj = match &node.obj { + JSXObject::JSXMemberExpr(member) => { + serialize_jsx_member_expr(ctx, member, pos) + } + JSXObject::Ident(ident) => serialize_jsx_identifier(ctx, ident, parent), + }; + + let prop = serialize_ident_name_as_jsx_identifier(ctx, &node.prop, pos); + + ctx.write_ref(obj_ref, obj); + ctx.write_ref(prop_ref, prop); + + pos +} + +fn serialize_jsx_element_name( + ctx: &mut TsEsTreeBuilder, + node: &JSXElementName, + parent: NodeRef, +) -> NodeRef { + match &node { + JSXElementName::Ident(ident) => { + serialize_jsx_identifier(ctx, ident, parent) + } + JSXElementName::JSXMemberExpr(member) => { + serialize_jsx_member_expr(ctx, member, parent) + } + JSXElementName::JSXNamespacedName(ns) => { + serialize_jsx_namespaced_name(ctx, ns, parent) + } + } +} + +fn serialize_jsx_opening_element( + ctx: &mut TsEsTreeBuilder, + node: &JSXOpeningElement, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXOpeningElement, parent, &node.span, 3); + let sclose_pos = ctx.bool_field(AstProp::SelfClosing); + let name_pos = ctx.ref_field(AstProp::Name); + let attrs_pos = ctx.ref_vec_field(AstProp::Attributes, node.attrs.len()); + + let name = serialize_jsx_element_name(ctx, &node.name, pos); + + // FIXME: type args + + let attrs = node + .attrs + .iter() + .map(|attr| match attr { + JSXAttrOrSpread::JSXAttr(attr) => { + let attr_pos = ctx.header(AstNode::JSXAttribute, pos, &attr.span, 2); + let name_pos = ctx.ref_field(AstProp::Name); + let value_pos = ctx.ref_field(AstProp::Value); + + let name = match &attr.name { + JSXAttrName::Ident(name) => { + serialize_ident_name_as_jsx_identifier(ctx, name, attr_pos) + } + JSXAttrName::JSXNamespacedName(node) => { + serialize_jsx_namespaced_name(ctx, node, attr_pos) + } + }; + + let value = attr.value.as_ref().map(|value| match value { + JSXAttrValue::Lit(lit) => serialize_lit(ctx, lit, attr_pos), + JSXAttrValue::JSXExprContainer(container) => { + serialize_jsx_container_expr(ctx, container, attr_pos) + } + JSXAttrValue::JSXElement(el) => { + serialize_jsx_element(ctx, el, attr_pos) + } + JSXAttrValue::JSXFragment(frag) => { + serialize_jsx_fragment(ctx, frag, attr_pos) + } + }); + + ctx.write_ref(name_pos, name); + ctx.write_maybe_ref(value_pos, value); + + attr_pos + } + JSXAttrOrSpread::SpreadElement(spread) => { + let attr_pos = + ctx.header(AstNode::JSXAttribute, pos, &spread.dot3_token, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let arg = serialize_expr(ctx, &spread.expr, attr_pos); + + ctx.write_ref(arg_pos, arg); + + attr_pos + } + }) + .collect::>(); + + ctx.write_bool(sclose_pos, node.self_closing); + ctx.write_ref(name_pos, name); + ctx.write_refs(attrs_pos, attrs); + + pos +} + +fn serialize_jsx_container_expr( + ctx: &mut TsEsTreeBuilder, + node: &JSXExprContainer, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXExpressionContainer, parent, &node.span, 1); + let expr_pos = ctx.ref_field(AstProp::Expression); + + let expr = match &node.expr { + JSXExpr::JSXEmptyExpr(expr) => serialize_jsx_empty_expr(ctx, expr, pos), + JSXExpr::Expr(expr) => serialize_expr(ctx, expr, pos), + }; + + ctx.write_ref(expr_pos, expr); + + pos +} + +fn serialize_jsx_empty_expr( + ctx: &mut TsEsTreeBuilder, + node: &JSXEmptyExpr, + parent: NodeRef, +) -> NodeRef { + ctx.header(AstNode::JSXEmptyExpression, parent, &node.span, 0) +} + +fn serialize_jsx_namespaced_name( + ctx: &mut TsEsTreeBuilder, + node: &JSXNamespacedName, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXNamespacedName, parent, &node.span, 2); + let ns_pos = ctx.ref_field(AstProp::Namespace); + let name_pos = ctx.ref_field(AstProp::Name); + + let ns_id = serialize_ident_name_as_jsx_identifier(ctx, &node.ns, pos); + let name_id = serialize_ident_name_as_jsx_identifier(ctx, &node.name, pos); + + ctx.write_ref(ns_pos, ns_id); + ctx.write_ref(name_pos, name_id); + + pos +} + +fn serialize_ident_name_as_jsx_identifier( + ctx: &mut TsEsTreeBuilder, + node: &IdentName, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXIdentifier, parent, &node.span, 1); + let name_pos = ctx.str_field(AstProp::Name); + + ctx.write_str(name_pos, &node.sym); + + pos +} + +fn serialize_jsx_identifier( + ctx: &mut TsEsTreeBuilder, + node: &Ident, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::JSXIdentifier, parent, &node.span, 1); + let name_pos = ctx.str_field(AstProp::Name); + + ctx.write_str(name_pos, &node.sym); + + pos +} + +fn serialize_pat( + ctx: &mut TsEsTreeBuilder, + pat: &Pat, + parent: NodeRef, +) -> NodeRef { + match pat { + Pat::Ident(node) => serialize_ident(ctx, &node.id, parent), + Pat::Array(node) => { + let pos = ctx.header(AstNode::ArrayPattern, parent, &node.span, 3); + let opt_pos = ctx.bool_field(AstProp::Optional); + let type_pos = ctx.ref_field(AstProp::TypeAnnotation); + let elems_pos = ctx.ref_vec_field(AstProp::Elements, node.elems.len()); + + let type_ann = maybe_serialize_ts_type_ann(ctx, &node.type_ann, pos); + + let children = node + .elems + .iter() + .map(|pat| { + pat + .as_ref() + .map_or(NodeRef(0), |v| serialize_pat(ctx, v, pos)) + }) + .collect::>(); + + ctx.write_bool(opt_pos, node.optional); + ctx.write_maybe_ref(type_pos, type_ann); + ctx.write_refs(elems_pos, children); + + pos + } + Pat::Rest(node) => { + let pos = ctx.header(AstNode::RestElement, parent, &node.span, 2); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let type_ann = maybe_serialize_ts_type_ann(ctx, &node.type_ann, pos); + let arg = serialize_pat(ctx, &node.arg, parent); + + ctx.write_maybe_ref(type_ann_pos, type_ann); + ctx.write_ref(arg_pos, arg); + + pos + } + Pat::Object(node) => { + let pos = ctx.header(AstNode::ObjectPattern, parent, &node.span, 3); + let opt_pos = ctx.bool_field(AstProp::Optional); + let props_pos = ctx.ref_vec_field(AstProp::Properties, node.props.len()); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let type_ann = maybe_serialize_ts_type_ann(ctx, &node.type_ann, pos); + + let children = node + .props + .iter() + .map(|prop| match prop { + ObjectPatProp::KeyValue(key_value_prop) => { + let child_pos = + ctx.header(AstNode::Property, pos, &key_value_prop.span(), 3); + let computed_pos = ctx.bool_field(AstProp::Computed); + let key_pos = ctx.ref_field(AstProp::Key); + let value_pos = ctx.ref_field(AstProp::Value); + + let computed = matches!(key_value_prop.key, PropName::Computed(_)); + + let key = serialize_prop_name(ctx, &key_value_prop.key, child_pos); + let value = + serialize_pat(ctx, key_value_prop.value.as_ref(), child_pos); + + ctx.write_bool(computed_pos, computed); + ctx.write_ref(key_pos, key); + ctx.write_ref(value_pos, value); + + child_pos + } + ObjectPatProp::Assign(assign_pat_prop) => { + let child_pos = + ctx.header(AstNode::Property, pos, &assign_pat_prop.span, 3); + // TOOD: Doesn't seem to be present in SWC ast + let _computed_pos = ctx.bool_field(AstProp::Computed); + let key_pos = ctx.ref_field(AstProp::Key); + let value_pos = ctx.ref_field(AstProp::Value); + + let ident = serialize_ident(ctx, &assign_pat_prop.key.id, parent); + + let value = assign_pat_prop + .value + .as_ref() + .map(|value| serialize_expr(ctx, value, child_pos)); + + ctx.write_ref(key_pos, ident); + ctx.write_maybe_ref(value_pos, value); + + child_pos + } + ObjectPatProp::Rest(rest_pat) => { + serialize_pat(ctx, &Pat::Rest(rest_pat.clone()), parent) + } + }) + .collect::>(); + + ctx.write_bool(opt_pos, node.optional); + ctx.write_maybe_ref(type_ann_pos, type_ann); + ctx.write_refs(props_pos, children); + + pos + } + Pat::Assign(node) => { + let pos = ctx.header(AstNode::AssignmentPattern, parent, &node.span, 2); + let left_pos = ctx.ref_field(AstProp::Left); + let right_pos = ctx.ref_field(AstProp::Right); + + let left = serialize_pat(ctx, &node.left, pos); + let right = serialize_expr(ctx, &node.right, pos); + + ctx.write_ref(left_pos, left); + ctx.write_ref(right_pos, right); + + pos + } + Pat::Invalid(_) => unreachable!(), + Pat::Expr(node) => serialize_expr(ctx, node, parent), + } +} + +fn serialize_for_head( + ctx: &mut TsEsTreeBuilder, + for_head: &ForHead, + parent: NodeRef, +) -> NodeRef { + match for_head { + ForHead::VarDecl(var_decl) => { + serialize_decl(ctx, &Decl::Var(var_decl.clone()), parent) + } + ForHead::UsingDecl(using_decl) => { + serialize_decl(ctx, &Decl::Using(using_decl.clone()), parent) + } + ForHead::Pat(pat) => serialize_pat(ctx, pat, parent), + } +} + +fn serialize_spread( + ctx: &mut TsEsTreeBuilder, + expr: &Expr, + span: &Span, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::SpreadElement, parent, span, 1); + let arg_pos = ctx.ref_field(AstProp::Argument); + + let expr_pos = serialize_expr(ctx, expr, parent); + ctx.write_ref(arg_pos, expr_pos); + + pos +} + +fn serialize_ident_name( + ctx: &mut TsEsTreeBuilder, + ident_name: &IdentName, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::Identifier, parent, &ident_name.span, 1); + let name_pos = ctx.str_field(AstProp::Name); + ctx.write_str(name_pos, ident_name.sym.as_str()); + + pos +} + +fn serialize_prop_name( + ctx: &mut TsEsTreeBuilder, + prop_name: &PropName, + parent: NodeRef, +) -> NodeRef { + match prop_name { + PropName::Ident(ident_name) => { + serialize_ident_name(ctx, ident_name, parent) + } + PropName::Str(str_prop) => { + let child_pos = + ctx.header(AstNode::StringLiteral, parent, &str_prop.span, 1); + let value_pos = ctx.str_field(AstProp::Value); + ctx.write_str(value_pos, &str_prop.value); + + child_pos + } + PropName::Num(number) => { + serialize_lit(ctx, &Lit::Num(number.clone()), parent) + } + PropName::Computed(node) => serialize_expr(ctx, &node.expr, parent), + PropName::BigInt(big_int) => { + serialize_lit(ctx, &Lit::BigInt(big_int.clone()), parent) + } + } +} + +fn serialize_lit( + ctx: &mut TsEsTreeBuilder, + lit: &Lit, + parent: NodeRef, +) -> NodeRef { + match lit { + Lit::Str(node) => { + let pos = ctx.header(AstNode::StringLiteral, parent, &node.span, 1); + let value_pos = ctx.str_field(AstProp::Value); + + ctx.write_str(value_pos, &node.value); + + pos + } + Lit::Bool(lit_bool) => { + let pos = ctx.header(AstNode::Bool, parent, &lit_bool.span, 1); + let value_pos = ctx.bool_field(AstProp::Value); + + ctx.write_bool(value_pos, lit_bool.value); + + pos + } + Lit::Null(node) => ctx.header(AstNode::Null, parent, &node.span, 0), + Lit::Num(node) => { + let pos = ctx.header(AstNode::NumericLiteral, parent, &node.span, 1); + let value_pos = ctx.str_field(AstProp::Value); + + let value = node.raw.as_ref().unwrap(); + ctx.write_str(value_pos, value); + + pos + } + Lit::BigInt(node) => { + let pos = ctx.header(AstNode::BigIntLiteral, parent, &node.span, 1); + let value_pos = ctx.str_field(AstProp::Value); + + ctx.write_str(value_pos, &node.value.to_string()); + + pos + } + Lit::Regex(node) => { + let pos = ctx.header(AstNode::RegExpLiteral, parent, &node.span, 2); + let pattern_pos = ctx.str_field(AstProp::Pattern); + let flags_pos = ctx.str_field(AstProp::Flags); + + ctx.write_str(pattern_pos, node.exp.as_str()); + ctx.write_str(flags_pos, node.flags.as_str()); + + pos + } + Lit::JSXText(jsxtext) => { + ctx.header(AstNode::JSXText, parent, &jsxtext.span, 0) + } + } +} + +fn serialize_ts_param_inst( + ctx: &mut TsEsTreeBuilder, + node: &TsTypeParamInstantiation, + parent: NodeRef, +) -> NodeRef { + let pos = + ctx.header(AstNode::TSTypeParameterInstantiation, parent, &node.span, 1); + let params_pos = ctx.ref_vec_field(AstProp::Params, node.params.len()); + + let params = node + .params + .iter() + .map(|param| serialize_ts_type(ctx, param, pos)) + .collect::>(); + + ctx.write_refs(params_pos, params); + + pos +} + +fn serialize_ts_type( + ctx: &mut TsEsTreeBuilder, + node: &TsType, + parent: NodeRef, +) -> NodeRef { + match node { + TsType::TsKeywordType(node) => { + let kind = match node.kind { + TsKeywordTypeKind::TsAnyKeyword => AstNode::TSAnyKeyword, + TsKeywordTypeKind::TsUnknownKeyword => AstNode::TSUnknownKeyword, + TsKeywordTypeKind::TsNumberKeyword => AstNode::TSNumberKeyword, + TsKeywordTypeKind::TsObjectKeyword => AstNode::TSObjectKeyword, + TsKeywordTypeKind::TsBooleanKeyword => AstNode::TSBooleanKeyword, + TsKeywordTypeKind::TsBigIntKeyword => AstNode::TSBigIntKeyword, + TsKeywordTypeKind::TsStringKeyword => AstNode::TSStringKeyword, + TsKeywordTypeKind::TsSymbolKeyword => AstNode::TSSymbolKeyword, + TsKeywordTypeKind::TsVoidKeyword => AstNode::TSVoidKeyword, + TsKeywordTypeKind::TsUndefinedKeyword => AstNode::TSUndefinedKeyword, + TsKeywordTypeKind::TsNullKeyword => AstNode::TSNullKeyword, + TsKeywordTypeKind::TsNeverKeyword => AstNode::TSNeverKeyword, + TsKeywordTypeKind::TsIntrinsicKeyword => AstNode::TSIntrinsicKeyword, + }; + + ctx.header(kind, parent, &node.span, 0) + } + TsType::TsThisType(node) => { + ctx.header(AstNode::TSThisType, parent, &node.span, 0) + } + TsType::TsFnOrConstructorType(node) => match node { + TsFnOrConstructorType::TsFnType(node) => { + let pos = ctx.header(AstNode::TSFunctionType, parent, &node.span, 1); + let params_pos = ctx.ref_vec_field(AstProp::Params, node.params.len()); + + let param_ids = node + .params + .iter() + .map(|param| serialize_ts_fn_param(ctx, param, pos)) + .collect::>(); + + ctx.write_refs(params_pos, param_ids); + + pos + } + TsFnOrConstructorType::TsConstructorType(_) => { + todo!() + } + }, + TsType::TsTypeRef(node) => { + let pos = ctx.header(AstNode::TSTypeReference, parent, &node.span, 2); + let name_pos = ctx.ref_field(AstProp::TypeName); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + + let name = serialize_ts_entity_name(ctx, &node.type_name, pos); + + let type_args = node + .type_params + .clone() + .map(|param| serialize_ts_param_inst(ctx, ¶m, pos)); + + ctx.write_ref(name_pos, name); + ctx.write_maybe_ref(type_args_pos, type_args); + + pos + } + TsType::TsTypeQuery(node) => { + let pos = ctx.header(AstNode::TSTypeQuery, parent, &node.span, 2); + let name_pos = ctx.ref_field(AstProp::ExprName); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + + let expr_name = match &node.expr_name { + TsTypeQueryExpr::TsEntityName(entity) => { + serialize_ts_entity_name(ctx, entity, pos) + } + TsTypeQueryExpr::Import(child) => { + serialize_ts_type(ctx, &TsType::TsImportType(child.clone()), pos) + } + }; + + let type_args = node + .type_args + .clone() + .map(|param| serialize_ts_param_inst(ctx, ¶m, pos)); + + ctx.write_ref(name_pos, expr_name); + ctx.write_maybe_ref(type_args_pos, type_args); + + pos + } + TsType::TsTypeLit(_) => { + // TODO: Not sure what this is + todo!() + } + TsType::TsArrayType(node) => { + let pos = ctx.header(AstNode::TSArrayType, parent, &node.span, 1); + let elem_pos = ctx.ref_field(AstProp::ElementType); + + let elem = serialize_ts_type(ctx, &node.elem_type, pos); + + ctx.write_ref(elem_pos, elem); + + pos + } + TsType::TsTupleType(node) => { + let pos = ctx.header(AstNode::TSTupleType, parent, &node.span, 1); + let children_pos = + ctx.ref_vec_field(AstProp::ElementTypes, node.elem_types.len()); + + let children = node + .elem_types + .iter() + .map(|elem| { + if let Some(label) = &elem.label { + let child_pos = + ctx.header(AstNode::TSNamedTupleMember, pos, &elem.span, 1); + let label_pos = ctx.ref_field(AstProp::Label); + let type_pos = ctx.ref_field(AstProp::ElementType); + + let label_id = serialize_pat(ctx, label, child_pos); + let type_id = serialize_ts_type(ctx, elem.ty.as_ref(), child_pos); + + ctx.write_ref(label_pos, label_id); + ctx.write_ref(type_pos, type_id); + + child_pos + } else { + serialize_ts_type(ctx, elem.ty.as_ref(), pos) + } + }) + .collect::>(); + + ctx.write_refs(children_pos, children); + + pos + } + TsType::TsOptionalType(_) => todo!(), + TsType::TsRestType(node) => { + let pos = ctx.header(AstNode::TSRestType, parent, &node.span, 1); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let type_ann = serialize_ts_type(ctx, &node.type_ann, pos); + + ctx.write_ref(type_ann_pos, type_ann); + + pos + } + TsType::TsUnionOrIntersectionType(node) => match node { + TsUnionOrIntersectionType::TsUnionType(node) => { + let pos = ctx.header(AstNode::TSUnionType, parent, &node.span, 1); + let types_pos = ctx.ref_vec_field(AstProp::Types, node.types.len()); + + let children = node + .types + .iter() + .map(|item| serialize_ts_type(ctx, item, pos)) + .collect::>(); + + ctx.write_refs(types_pos, children); + + pos + } + TsUnionOrIntersectionType::TsIntersectionType(node) => { + let pos = + ctx.header(AstNode::TSIntersectionType, parent, &node.span, 1); + let types_pos = ctx.ref_vec_field(AstProp::Types, node.types.len()); + + let children = node + .types + .iter() + .map(|item| serialize_ts_type(ctx, item, pos)) + .collect::>(); + + ctx.write_refs(types_pos, children); + + pos + } + }, + TsType::TsConditionalType(node) => { + let pos = ctx.header(AstNode::TSConditionalType, parent, &node.span, 4); + let check_pos = ctx.ref_field(AstProp::CheckType); + let extends_pos = ctx.ref_field(AstProp::ExtendsType); + let true_pos = ctx.ref_field(AstProp::TrueType); + let false_pos = ctx.ref_field(AstProp::FalseType); + + let check = serialize_ts_type(ctx, &node.check_type, pos); + let extends = serialize_ts_type(ctx, &node.extends_type, pos); + let v_true = serialize_ts_type(ctx, &node.true_type, pos); + let v_false = serialize_ts_type(ctx, &node.false_type, pos); + + ctx.write_ref(check_pos, check); + ctx.write_ref(extends_pos, extends); + ctx.write_ref(true_pos, v_true); + ctx.write_ref(false_pos, v_false); + + pos + } + TsType::TsInferType(node) => { + let pos = ctx.header(AstNode::TSInferType, parent, &node.span, 1); + let param_pos = ctx.ref_field(AstProp::TypeParameter); + + let param = serialize_ts_type_param(ctx, &node.type_param, parent); + + ctx.write_ref(param_pos, param); + + pos + } + TsType::TsParenthesizedType(_) => todo!(), + TsType::TsTypeOperator(node) => { + let pos = ctx.header(AstNode::TSTypeOperator, parent, &node.span, 2); + + let operator_pos = ctx.str_field(AstProp::Operator); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let type_ann = serialize_ts_type(ctx, &node.type_ann, pos); + + ctx.write_str( + operator_pos, + match node.op { + TsTypeOperatorOp::KeyOf => "keyof", + TsTypeOperatorOp::Unique => "unique", + TsTypeOperatorOp::ReadOnly => "readonly", + }, + ); + ctx.write_ref(type_ann_pos, type_ann); + + pos + } + TsType::TsIndexedAccessType(node) => { + let pos = ctx.header(AstNode::TSIndexedAccessType, parent, &node.span, 2); + + let index_type_pos = ctx.ref_field(AstProp::IndexType); + let obj_type_pos = ctx.ref_field(AstProp::ObjectType); + + let index = serialize_ts_type(ctx, &node.index_type, pos); + let obj = serialize_ts_type(ctx, &node.obj_type, pos); + + ctx.write_ref(index_type_pos, index); + ctx.write_ref(obj_type_pos, obj); + + pos + } + TsType::TsMappedType(node) => { + let pos = ctx.header(AstNode::TSMappedType, parent, &node.span, 5); + + let name_pos = ctx.ref_field(AstProp::NameType); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + let type_param_pos = ctx.ref_field(AstProp::TypeParameter); + + let opt_pos = + create_true_plus_minus_field(ctx, AstProp::Optional, node.optional); + let readonly_pos = + create_true_plus_minus_field(ctx, AstProp::Readonly, node.readonly); + + let name_id = maybe_serialize_ts_type(ctx, &node.name_type, pos); + let type_ann = maybe_serialize_ts_type(ctx, &node.type_ann, pos); + let type_param = serialize_ts_type_param(ctx, &node.type_param, pos); + + write_true_plus_minus(ctx, opt_pos, node.optional); + write_true_plus_minus(ctx, readonly_pos, node.readonly); + ctx.write_maybe_ref(name_pos, name_id); + ctx.write_maybe_ref(type_ann_pos, type_ann); + ctx.write_ref(type_param_pos, type_param); + + pos + } + TsType::TsLitType(node) => serialize_ts_lit_type(ctx, node, parent), + TsType::TsTypePredicate(node) => { + let pos = ctx.header(AstNode::TSTypePredicate, parent, &node.span, 3); + + let asserts_pos = ctx.bool_field(AstProp::Asserts); + let param_name_pos = ctx.ref_field(AstProp::ParameterName); + let type_ann_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let param_name = match &node.param_name { + TsThisTypeOrIdent::TsThisType(ts_this_type) => { + ctx.header(AstNode::TSThisType, pos, &ts_this_type.span, 0) + } + TsThisTypeOrIdent::Ident(ident) => serialize_ident(ctx, ident, pos), + }; + + let type_ann = maybe_serialize_ts_type_ann(ctx, &node.type_ann, pos); + + ctx.write_bool(asserts_pos, node.asserts); + ctx.write_ref(param_name_pos, param_name); + ctx.write_maybe_ref(type_ann_pos, type_ann); + + pos + } + TsType::TsImportType(node) => { + let pos = ctx.header(AstNode::TSTypePredicate, parent, &node.span, 3); + let arg_pos = ctx.ref_field(AstProp::Argument); + let type_args_pos = ctx.ref_field(AstProp::TypeArguments); + let qualifier_pos = ctx.ref_field(AstProp::Qualifier); + + let arg = serialize_ts_lit_type( + ctx, + &TsLitType { + lit: TsLit::Str(node.arg.clone()), + span: node.arg.span, + }, + pos, + ); + + let type_arg = node.type_args.clone().map(|param_node| { + serialize_ts_param_inst(ctx, param_node.as_ref(), pos) + }); + + let qualifier = node.qualifier.clone().map_or(NodeRef(0), |quali| { + serialize_ts_entity_name(ctx, &quali, pos) + }); + + ctx.write_ref(arg_pos, arg); + ctx.write_ref(qualifier_pos, qualifier); + ctx.write_maybe_ref(type_args_pos, type_arg); + + pos + } + } +} + +fn serialize_ts_lit_type( + ctx: &mut TsEsTreeBuilder, + node: &TsLitType, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::TSLiteralType, parent, &node.span, 1); + let lit_pos = ctx.ref_field(AstProp::Literal); + + let lit = match &node.lit { + TsLit::Number(lit) => serialize_lit(ctx, &Lit::Num(lit.clone()), pos), + TsLit::Str(lit) => serialize_lit(ctx, &Lit::Str(lit.clone()), pos), + TsLit::Bool(lit) => serialize_lit(ctx, &Lit::Bool(*lit), pos), + TsLit::BigInt(lit) => serialize_lit(ctx, &Lit::BigInt(lit.clone()), pos), + TsLit::Tpl(lit) => serialize_expr( + ctx, + &Expr::Tpl(Tpl { + span: lit.span, + exprs: vec![], + quasis: lit.quasis.clone(), + }), + pos, + ), + }; + + ctx.write_ref(lit_pos, lit); + + pos +} + +fn create_true_plus_minus_field( + ctx: &mut TsEsTreeBuilder, + prop: AstProp, + value: Option, +) -> NodePos { + if let Some(v) = value { + match v { + TruePlusMinus::True => NodePos::Bool(ctx.bool_field(prop)), + TruePlusMinus::Plus | TruePlusMinus::Minus => { + NodePos::Str(ctx.str_field(prop)) + } + } + } else { + NodePos::Undef(ctx.undefined_field(prop)) + } +} + +fn extract_pos(pos: NodePos) -> usize { + match pos { + NodePos::Bool(bool_pos) => bool_pos.0, + NodePos::Field(field_pos) => field_pos.0, + NodePos::FieldArr(field_arr_pos) => field_arr_pos.0, + NodePos::Str(str_pos) => str_pos.0, + NodePos::Undef(undef_pos) => undef_pos.0, + NodePos::Null(null_pos) => null_pos.0, + } +} + +fn write_true_plus_minus( + ctx: &mut TsEsTreeBuilder, + pos: NodePos, + value: Option, +) { + if let Some(v) = value { + match v { + TruePlusMinus::True => { + let bool_pos = BoolPos(extract_pos(pos)); + ctx.write_bool(bool_pos, true); + } + TruePlusMinus::Plus => { + let str_pos = StrPos(extract_pos(pos)); + ctx.write_str(str_pos, "+") + } + TruePlusMinus::Minus => { + let str_pos = StrPos(extract_pos(pos)); + ctx.write_str(str_pos, "-") + } + } + } +} + +fn serialize_ts_entity_name( + ctx: &mut TsEsTreeBuilder, + node: &TsEntityName, + parent: NodeRef, +) -> NodeRef { + match &node { + TsEntityName::TsQualifiedName(_) => todo!(), + TsEntityName::Ident(ident) => serialize_ident(ctx, ident, parent), + } +} + +fn maybe_serialize_ts_type_ann( + ctx: &mut TsEsTreeBuilder, + node: &Option>, + parent: NodeRef, +) -> Option { + node + .as_ref() + .map(|type_ann| serialize_ts_type_ann(ctx, type_ann, parent)) +} + +fn serialize_ts_type_ann( + ctx: &mut TsEsTreeBuilder, + node: &TsTypeAnn, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::TSTypeAnnotation, parent, &node.span, 1); + let type_pos = ctx.ref_field(AstProp::TypeAnnotation); + + let v_type = serialize_ts_type(ctx, &node.type_ann, pos); + + ctx.write_ref(type_pos, v_type); + + pos +} + +fn maybe_serialize_ts_type( + ctx: &mut TsEsTreeBuilder, + node: &Option>, + parent: NodeRef, +) -> Option { + node + .as_ref() + .map(|item| serialize_ts_type(ctx, item, parent)) +} + +fn serialize_ts_type_param( + ctx: &mut TsEsTreeBuilder, + node: &TsTypeParam, + parent: NodeRef, +) -> NodeRef { + let pos = ctx.header(AstNode::TSTypeParameter, parent, &node.span, 6); + let name_pos = ctx.ref_field(AstProp::Name); + let constraint_pos = ctx.ref_field(AstProp::Constraint); + let default_pos = ctx.ref_field(AstProp::Default); + let const_pos = ctx.bool_field(AstProp::Const); + let in_pos = ctx.bool_field(AstProp::In); + let out_pos = ctx.bool_field(AstProp::Out); + + let name = serialize_ident(ctx, &node.name, pos); + let constraint = maybe_serialize_ts_type(ctx, &node.constraint, pos); + let default = maybe_serialize_ts_type(ctx, &node.default, pos); + + ctx.write_bool(const_pos, node.is_const); + ctx.write_bool(in_pos, node.is_in); + ctx.write_bool(out_pos, node.is_out); + ctx.write_ref(name_pos, name); + ctx.write_maybe_ref(constraint_pos, constraint); + ctx.write_maybe_ref(default_pos, default); + + pos +} + +fn maybe_serialize_ts_type_param( + ctx: &mut TsEsTreeBuilder, + node: &Option>, + parent: NodeRef, +) -> Option { + node.as_ref().map(|node| { + let pos = + ctx.header(AstNode::TSTypeParameterDeclaration, parent, &node.span, 1); + let params_pos = ctx.ref_vec_field(AstProp::Params, node.params.len()); + + let params = node + .params + .iter() + .map(|param| serialize_ts_type_param(ctx, param, pos)) + .collect::>(); + + ctx.write_refs(params_pos, params); + + pos + }) +} + +fn serialize_ts_fn_param( + ctx: &mut TsEsTreeBuilder, + node: &TsFnParam, + parent: NodeRef, +) -> NodeRef { + match node { + TsFnParam::Ident(ident) => serialize_ident(ctx, ident, parent), + TsFnParam::Array(pat) => { + serialize_pat(ctx, &Pat::Array(pat.clone()), parent) + } + TsFnParam::Rest(pat) => serialize_pat(ctx, &Pat::Rest(pat.clone()), parent), + TsFnParam::Object(pat) => { + serialize_pat(ctx, &Pat::Object(pat.clone()), parent) + } + } +} diff --git a/cli/tools/lint/ast_buffer/ts_estree.rs b/cli/tools/lint/ast_buffer/ts_estree.rs new file mode 100644 index 0000000000..af5fea4b46 --- /dev/null +++ b/cli/tools/lint/ast_buffer/ts_estree.rs @@ -0,0 +1,513 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::fmt; +use std::fmt::Debug; +use std::fmt::Display; + +use deno_ast::swc::common::Span; + +use super::buffer::AstBufSerializer; +use super::buffer::BoolPos; +use super::buffer::FieldArrPos; +use super::buffer::FieldPos; +use super::buffer::NodeRef; +use super::buffer::NullPos; +use super::buffer::SerializeCtx; +use super::buffer::StrPos; +use super::buffer::UndefPos; + +#[derive(Debug, Clone, PartialEq)] +pub enum AstNode { + // First node must always be the empty/invalid node + Invalid, + // Typically the + Program, + + // Module declarations + ExportAllDeclaration, + ExportDefaultDeclaration, + ExportNamedDeclaration, + ImportDeclaration, + TsExportAssignment, + TsImportEquals, + TsNamespaceExport, + + // Decls + ClassDeclaration, + FunctionDeclaration, + TSEnumDeclaration, + TSInterface, + TsModule, + TsTypeAlias, + Using, + VariableDeclaration, + + // Statements + BlockStatement, + BreakStatement, + ContinueStatement, + DebuggerStatement, + DoWhileStatement, + EmptyStatement, + ExpressionStatement, + ForInStatement, + ForOfStatement, + ForStatement, + IfStatement, + LabeledStatement, + ReturnStatement, + SwitchCase, + SwitchStatement, + ThrowStatement, + TryStatement, + WhileStatement, + WithStatement, + + // Expressions + ArrayExpression, + ArrowFunctionExpression, + AssignmentExpression, + AwaitExpression, + BinaryExpression, + CallExpression, + ChainExpression, + ClassExpression, + ConditionalExpression, + FunctionExpression, + Identifier, + ImportExpression, + LogicalExpression, + MemberExpression, + MetaProp, + NewExpression, + ObjectExpression, + PrivateIdentifier, + SequenceExpression, + Super, + TaggedTemplateExpression, + TemplateLiteral, + ThisExpression, + TSAsExpression, + TsConstAssertion, + TsInstantiation, + TSNonNullExpression, + TSSatisfiesExpression, + TSTypeAssertion, + UnaryExpression, + UpdateExpression, + YieldExpression, + + // TODO: TSEsTree uses a single literal node + // Literals + StringLiteral, + Bool, + Null, + NumericLiteral, + BigIntLiteral, + RegExpLiteral, + + EmptyExpr, + SpreadElement, + Property, + VariableDeclarator, + CatchClause, + RestElement, + ExportSpecifier, + TemplateElement, + MethodDefinition, + ClassBody, + + // Patterns + ArrayPattern, + AssignmentPattern, + ObjectPattern, + + // JSX + JSXAttribute, + JSXClosingElement, + JSXClosingFragment, + JSXElement, + JSXEmptyExpression, + JSXExpressionContainer, + JSXFragment, + JSXIdentifier, + JSXMemberExpression, + JSXNamespacedName, + JSXOpeningElement, + JSXOpeningFragment, + JSXSpreadAttribute, + JSXSpreadChild, + JSXText, + + TSTypeAnnotation, + TSTypeParameterDeclaration, + TSTypeParameter, + TSTypeParameterInstantiation, + TSEnumMember, + TSInterfaceBody, + TSInterfaceHeritage, + TSTypeReference, + TSThisType, + TSLiteralType, + TSInferType, + TSConditionalType, + TSUnionType, + TSIntersectionType, + TSMappedType, + TSTypeQuery, + TSTupleType, + TSNamedTupleMember, + TSFunctionType, + TsCallSignatureDeclaration, + TSPropertySignature, + TSMethodSignature, + TSIndexSignature, + TSIndexedAccessType, + TSTypeOperator, + TSTypePredicate, + TSImportType, + TSRestType, + TSArrayType, + TSClassImplements, + + TSAnyKeyword, + TSBigIntKeyword, + TSBooleanKeyword, + TSIntrinsicKeyword, + TSNeverKeyword, + TSNullKeyword, + TSNumberKeyword, + TSObjectKeyword, + TSStringKeyword, + TSSymbolKeyword, + TSUndefinedKeyword, + TSUnknownKeyword, + TSVoidKeyword, + TSEnumBody, // Last value is used for max value +} + +impl Display for AstNode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +impl From for u8 { + fn from(m: AstNode) -> u8 { + m as u8 + } +} + +#[derive(Debug, Clone)] +pub enum AstProp { + // Base, these three must be in sync with JS. The + // order here for these 3 fields is important. + Type, + Parent, + Range, + + // Starting from here the order doesn't matter. + // Following are all possible AST node properties. + Abstract, + Accessibility, + Alternate, + Argument, + Arguments, + Asserts, + Async, + Attributes, + Await, + Block, + Body, + Callee, + Cases, + Children, + CheckType, + ClosingElement, + ClosingFragment, + Computed, + Consequent, + Const, + Constraint, + Cooked, + Declaration, + Declarations, + Declare, + Default, + Definite, + Delegate, + Discriminant, + Elements, + ElementType, + ElementTypes, + ExprName, + Expression, + Expressions, + Exported, + Extends, + ExtendsType, + FalseType, + Finalizer, + Flags, + Generator, + Handler, + Id, + In, + IndexType, + Init, + Initializer, + Implements, + Key, + Kind, + Label, + Left, + Literal, + Local, + Members, + Meta, + Method, + Name, + Namespace, + NameType, + Object, + ObjectType, + OpeningElement, + OpeningFragment, + Operator, + Optional, + Out, + Param, + ParameterName, + Params, + Pattern, + Prefix, + Properties, + Property, + Qualifier, + Quasi, + Quasis, + Raw, + Readonly, + ReturnType, + Right, + SelfClosing, + Shorthand, + Source, + SourceType, + Specifiers, + Static, + SuperClass, + SuperTypeArguments, + Tag, + Tail, + Test, + TrueType, + TypeAnnotation, + TypeArguments, + TypeName, + TypeParameter, + TypeParameters, + Types, + Update, + Value, // Last value is used for max value +} + +// TODO: Feels like there should be an easier way to iterater over an +// enum in Rust and lowercase the first letter. +impl Display for AstProp { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let s = match self { + AstProp::Parent => "parent", + AstProp::Range => "range", + AstProp::Type => "type", + AstProp::Abstract => "abstract", + AstProp::Accessibility => "accessibility", + AstProp::Alternate => "alternate", + AstProp::Argument => "argument", + AstProp::Arguments => "arguments", + AstProp::Asserts => "asserts", + AstProp::Async => "async", + AstProp::Attributes => "attributes", + AstProp::Await => "await", + AstProp::Block => "block", + AstProp::Body => "body", + AstProp::Callee => "callee", + AstProp::Cases => "cases", + AstProp::Children => "children", + AstProp::CheckType => "checkType", + AstProp::ClosingElement => "closingElement", + AstProp::ClosingFragment => "closingFragment", + AstProp::Computed => "computed", + AstProp::Consequent => "consequent", + AstProp::Const => "const", + AstProp::Constraint => "constraint", + AstProp::Cooked => "cooked", + AstProp::Declaration => "declaration", + AstProp::Declarations => "declarations", + AstProp::Declare => "declare", + AstProp::Default => "default", + AstProp::Definite => "definite", + AstProp::Delegate => "delegate", + AstProp::Discriminant => "discriminant", + AstProp::Elements => "elements", + AstProp::ElementType => "elementType", + AstProp::ElementTypes => "elementTypes", + AstProp::ExprName => "exprName", + AstProp::Expression => "expression", + AstProp::Expressions => "expressions", + AstProp::Exported => "exported", + AstProp::Extends => "extends", + AstProp::ExtendsType => "extendsType", + AstProp::FalseType => "falseType", + AstProp::Finalizer => "finalizer", + AstProp::Flags => "flags", + AstProp::Generator => "generator", + AstProp::Handler => "handler", + AstProp::Id => "id", + AstProp::In => "in", + AstProp::IndexType => "indexType", + AstProp::Init => "init", + AstProp::Initializer => "initializer", + AstProp::Implements => "implements", + AstProp::Key => "key", + AstProp::Kind => "kind", + AstProp::Label => "label", + AstProp::Left => "left", + AstProp::Literal => "literal", + AstProp::Local => "local", + AstProp::Members => "members", + AstProp::Meta => "meta", + AstProp::Method => "method", + AstProp::Name => "name", + AstProp::Namespace => "namespace", + AstProp::NameType => "nameType", + AstProp::Object => "object", + AstProp::ObjectType => "objectType", + AstProp::OpeningElement => "openingElement", + AstProp::OpeningFragment => "openingFragment", + AstProp::Operator => "operator", + AstProp::Optional => "optional", + AstProp::Out => "out", + AstProp::Param => "param", + AstProp::ParameterName => "parameterName", + AstProp::Params => "params", + AstProp::Pattern => "pattern", + AstProp::Prefix => "prefix", + AstProp::Properties => "properties", + AstProp::Property => "property", + AstProp::Qualifier => "qualifier", + AstProp::Quasi => "quasi", + AstProp::Quasis => "quasis", + AstProp::Raw => "raw", + AstProp::Readonly => "readonly", + AstProp::ReturnType => "returnType", + AstProp::Right => "right", + AstProp::SelfClosing => "selfClosing", + AstProp::Shorthand => "shorthand", + AstProp::Source => "source", + AstProp::SourceType => "sourceType", + AstProp::Specifiers => "specifiers", + AstProp::Static => "static", + AstProp::SuperClass => "superClass", + AstProp::SuperTypeArguments => "superTypeArguments", + AstProp::Tag => "tag", + AstProp::Tail => "tail", + AstProp::Test => "test", + AstProp::TrueType => "trueType", + AstProp::TypeAnnotation => "typeAnnotation", + AstProp::TypeArguments => "typeArguments", + AstProp::TypeName => "typeName", + AstProp::TypeParameter => "typeParameter", + AstProp::TypeParameters => "typeParameters", + AstProp::Types => "types", + AstProp::Update => "update", + AstProp::Value => "value", + }; + + write!(f, "{}", s) + } +} + +impl From for u8 { + fn from(m: AstProp) -> u8 { + m as u8 + } +} + +pub struct TsEsTreeBuilder { + ctx: SerializeCtx, +} + +// TODO: Add a builder API to make it easier to convert from different source +// ast formats. +impl TsEsTreeBuilder { + pub fn new() -> Self { + // Max values + // TODO: Maybe there is a rust macro to grab the last enum value? + let kind_count: u8 = AstNode::TSEnumBody.into(); + let prop_count: u8 = AstProp::Value.into(); + Self { + ctx: SerializeCtx::new(kind_count, prop_count), + } + } +} + +impl AstBufSerializer for TsEsTreeBuilder { + fn header( + &mut self, + kind: AstNode, + parent: NodeRef, + span: &Span, + prop_count: usize, + ) -> NodeRef { + self.ctx.header(kind, parent, span, prop_count) + } + + fn ref_field(&mut self, prop: AstProp) -> FieldPos { + FieldPos(self.ctx.ref_field(prop)) + } + + fn ref_vec_field(&mut self, prop: AstProp, len: usize) -> FieldArrPos { + FieldArrPos(self.ctx.ref_vec_field(prop, len)) + } + + fn str_field(&mut self, prop: AstProp) -> StrPos { + StrPos(self.ctx.str_field(prop)) + } + + fn bool_field(&mut self, prop: AstProp) -> BoolPos { + BoolPos(self.ctx.bool_field(prop)) + } + + fn undefined_field(&mut self, prop: AstProp) -> UndefPos { + UndefPos(self.ctx.undefined_field(prop)) + } + + fn null_field(&mut self, prop: AstProp) -> NullPos { + NullPos(self.ctx.null_field(prop)) + } + + fn write_ref(&mut self, pos: FieldPos, value: NodeRef) { + self.ctx.write_ref(pos.0, value); + } + + fn write_maybe_ref(&mut self, pos: FieldPos, value: Option) { + self.ctx.write_maybe_ref(pos.0, value); + } + + fn write_refs(&mut self, pos: FieldArrPos, value: Vec) { + self.ctx.write_refs(pos.0, value); + } + + fn write_str(&mut self, pos: StrPos, value: &str) { + self.ctx.write_str(pos.0, value); + } + + fn write_bool(&mut self, pos: BoolPos, value: bool) { + self.ctx.write_bool(pos.0, value); + } + + fn serialize(&mut self) -> Vec { + self.ctx.serialize() + } +} diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index e49197bbad..50fc16799a 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -51,10 +51,13 @@ use crate::util::fs::canonicalize_path; use crate::util::path::is_script_ext; use crate::util::sync::AtomicFlag; +mod ast_buffer; mod linter; mod reporters; mod rules; +// TODO(bartlomieju): remove once we wire plugins through the CLI linter +pub use ast_buffer::serialize_ast_to_buffer; pub use linter::CliLinter; pub use linter::CliLinterOptions; pub use rules::collect_no_slow_type_diagnostics; diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index 48bf42c9c7..3164b8ae59 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -616,7 +616,10 @@ async fn configure_main_worker( WorkerExecutionMode::Test, specifier.clone(), permissions_container, - vec![ops::testing::deno_test::init_ops(worker_sender.sender)], + vec![ + ops::testing::deno_test::init_ops(worker_sender.sender), + ops::lint::deno_lint::init_ops(), + ], Stdio { stdin: StdioPipe::inherit(), stdout: StdioPipe::file(worker_sender.stdout), diff --git a/cli/worker.rs b/cli/worker.rs index c733f41321..6b87b5966a 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -656,7 +656,8 @@ impl CliMainWorkerFactory { "40_test_common.js", "40_test.js", "40_bench.js", - "40_jupyter.js" + "40_jupyter.js", + "40_lint.js" ); } diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index bceb1f7ddb..a11444bc36 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -526,6 +526,9 @@ const NOT_IMPORTED_OPS = [ // Used in jupyter API "op_base64_encode", + // Used in the lint API + "op_lint_create_serialized_ast", + // Related to `Deno.test()` API "op_test_event_step_result_failed", "op_test_event_step_result_ignored", diff --git a/tests/integration/js_unit_tests.rs b/tests/integration/js_unit_tests.rs index 717a8d8e7c..899329b319 100644 --- a/tests/integration/js_unit_tests.rs +++ b/tests/integration/js_unit_tests.rs @@ -52,6 +52,7 @@ util::unit_test_factory!( kv_queue_test, kv_queue_undelivered_test, link_test, + lint_plugin_test, make_temp_test, message_channel_test, mkdir_test, diff --git a/tests/unit/lint_plugin_test.ts b/tests/unit/lint_plugin_test.ts new file mode 100644 index 0000000000..649c8bde9e --- /dev/null +++ b/tests/unit/lint_plugin_test.ts @@ -0,0 +1,557 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "./test_util.ts"; + +// TODO(@marvinhagemeister) Remove once we land "official" types +export interface LintReportData { + // deno-lint-ignore no-explicit-any + node: any; + message: string; +} +// TODO(@marvinhagemeister) Remove once we land "official" types +interface LintContext { + id: string; +} +// TODO(@marvinhagemeister) Remove once we land "official" types +// deno-lint-ignore no-explicit-any +type LintVisitor = Record void>; + +// TODO(@marvinhagemeister) Remove once we land "official" types +interface LintRule { + create(ctx: LintContext): LintVisitor; + destroy?(): void; +} + +// TODO(@marvinhagemeister) Remove once we land "official" types +interface LintPlugin { + name: string; + rules: Record; +} + +function runLintPlugin(plugin: LintPlugin, fileName: string, source: string) { + // deno-lint-ignore no-explicit-any + return (Deno as any)[(Deno as any).internal].runLintPlugin( + plugin, + fileName, + source, + ); +} + +function testPlugin( + source: string, + rule: LintRule, +) { + const plugin = { + name: "test-plugin", + rules: { + testRule: rule, + }, + }; + + return runLintPlugin(plugin, "source.tsx", source); +} + +function testVisit(source: string, ...selectors: string[]): string[] { + const log: string[] = []; + + testPlugin(source, { + create() { + const visitor: LintVisitor = {}; + + for (const s of selectors) { + visitor[s] = () => log.push(s); + } + + return visitor; + }, + }); + + return log; +} + +function testLintNode(source: string, ...selectors: string[]) { + // deno-lint-ignore no-explicit-any + const log: any[] = []; + + testPlugin(source, { + create() { + const visitor: LintVisitor = {}; + + for (const s of selectors) { + visitor[s] = (node) => { + log.push(node[Symbol.for("Deno.lint.toJsValue")]()); + }; + } + + return visitor; + }, + }); + + return log; +} + +Deno.test("Plugin - visitor enter/exit", () => { + const enter = testVisit("foo", "Identifier"); + assertEquals(enter, ["Identifier"]); + + const exit = testVisit("foo", "Identifier:exit"); + assertEquals(exit, ["Identifier:exit"]); + + const both = testVisit("foo", "Identifier", "Identifier:exit"); + assertEquals(both, ["Identifier", "Identifier:exit"]); +}); + +Deno.test("Plugin - Program", () => { + const node = testLintNode("", "Program"); + assertEquals(node[0], { + type: "Program", + sourceType: "script", + range: [1, 1], + body: [], + }); +}); + +Deno.test("Plugin - BlockStatement", () => { + const node = testLintNode("{ foo; }", "BlockStatement"); + assertEquals(node[0], { + type: "BlockStatement", + range: [1, 9], + body: [{ + type: "ExpressionStatement", + range: [3, 7], + expression: { + type: "Identifier", + name: "foo", + range: [3, 6], + }, + }], + }); +}); + +Deno.test("Plugin - BreakStatement", () => { + let node = testLintNode("break;", "BreakStatement"); + assertEquals(node[0], { + type: "BreakStatement", + range: [1, 7], + label: null, + }); + + node = testLintNode("break foo;", "BreakStatement"); + assertEquals(node[0], { + type: "BreakStatement", + range: [1, 11], + label: { + type: "Identifier", + range: [7, 10], + name: "foo", + }, + }); +}); + +Deno.test("Plugin - ContinueStatement", () => { + let node = testLintNode("continue;", "ContinueStatement"); + assertEquals(node[0], { + type: "ContinueStatement", + range: [1, 10], + label: null, + }); + + node = testLintNode("continue foo;", "ContinueStatement"); + assertEquals(node[0], { + type: "ContinueStatement", + range: [1, 14], + label: { + type: "Identifier", + range: [10, 13], + name: "foo", + }, + }); +}); + +Deno.test("Plugin - DebuggerStatement", () => { + const node = testLintNode("debugger;", "DebuggerStatement"); + assertEquals(node[0], { + type: "DebuggerStatement", + range: [1, 10], + }); +}); + +Deno.test("Plugin - DoWhileStatement", () => { + const node = testLintNode("do {} while (foo);", "DoWhileStatement"); + assertEquals(node[0], { + type: "DoWhileStatement", + range: [1, 19], + test: { + type: "Identifier", + range: [14, 17], + name: "foo", + }, + body: { + type: "BlockStatement", + range: [4, 6], + body: [], + }, + }); +}); + +Deno.test("Plugin - ExpressionStatement", () => { + const node = testLintNode("foo;", "ExpressionStatement"); + assertEquals(node[0], { + type: "ExpressionStatement", + range: [1, 5], + expression: { + type: "Identifier", + range: [1, 4], + name: "foo", + }, + }); +}); + +Deno.test("Plugin - ForInStatement", () => { + const node = testLintNode("for (a in b) {}", "ForInStatement"); + assertEquals(node[0], { + type: "ForInStatement", + range: [1, 16], + left: { + type: "Identifier", + range: [6, 7], + name: "a", + }, + right: { + type: "Identifier", + range: [11, 12], + name: "b", + }, + body: { + type: "BlockStatement", + range: [14, 16], + body: [], + }, + }); +}); + +Deno.test("Plugin - ForOfStatement", () => { + let node = testLintNode("for (a of b) {}", "ForOfStatement"); + assertEquals(node[0], { + type: "ForOfStatement", + range: [1, 16], + await: false, + left: { + type: "Identifier", + range: [6, 7], + name: "a", + }, + right: { + type: "Identifier", + range: [11, 12], + name: "b", + }, + body: { + type: "BlockStatement", + range: [14, 16], + body: [], + }, + }); + + node = testLintNode("for await (a of b) {}", "ForOfStatement"); + assertEquals(node[0], { + type: "ForOfStatement", + range: [1, 22], + await: true, + left: { + type: "Identifier", + range: [12, 13], + name: "a", + }, + right: { + type: "Identifier", + range: [17, 18], + name: "b", + }, + body: { + type: "BlockStatement", + range: [20, 22], + body: [], + }, + }); +}); + +Deno.test("Plugin - ForStatement", () => { + let node = testLintNode("for (;;) {}", "ForStatement"); + assertEquals(node[0], { + type: "ForStatement", + range: [1, 12], + init: null, + test: null, + update: null, + body: { + type: "BlockStatement", + range: [10, 12], + body: [], + }, + }); + + node = testLintNode("for (a; b; c) {}", "ForStatement"); + assertEquals(node[0], { + type: "ForStatement", + range: [1, 17], + init: { + type: "Identifier", + range: [6, 7], + name: "a", + }, + test: { + type: "Identifier", + range: [9, 10], + name: "b", + }, + update: { + type: "Identifier", + range: [12, 13], + name: "c", + }, + body: { + type: "BlockStatement", + range: [15, 17], + body: [], + }, + }); +}); + +Deno.test("Plugin - IfStatement", () => { + let node = testLintNode("if (foo) {}", "IfStatement"); + assertEquals(node[0], { + type: "IfStatement", + range: [1, 12], + test: { + type: "Identifier", + name: "foo", + range: [5, 8], + }, + consequent: { + type: "BlockStatement", + range: [10, 12], + body: [], + }, + alternate: null, + }); + + node = testLintNode("if (foo) {} else {}", "IfStatement"); + assertEquals(node[0], { + type: "IfStatement", + range: [1, 20], + test: { + type: "Identifier", + name: "foo", + range: [5, 8], + }, + consequent: { + type: "BlockStatement", + range: [10, 12], + body: [], + }, + alternate: { + type: "BlockStatement", + range: [18, 20], + body: [], + }, + }); +}); + +Deno.test("Plugin - LabeledStatement", () => { + const node = testLintNode("foo: {};", "LabeledStatement"); + assertEquals(node[0], { + type: "LabeledStatement", + range: [1, 8], + label: { + type: "Identifier", + name: "foo", + range: [1, 4], + }, + body: { + type: "BlockStatement", + range: [6, 8], + body: [], + }, + }); +}); + +Deno.test("Plugin - ReturnStatement", () => { + let node = testLintNode("return", "ReturnStatement"); + assertEquals(node[0], { + type: "ReturnStatement", + range: [1, 7], + argument: null, + }); + + node = testLintNode("return foo;", "ReturnStatement"); + assertEquals(node[0], { + type: "ReturnStatement", + range: [1, 12], + argument: { + type: "Identifier", + name: "foo", + range: [8, 11], + }, + }); +}); + +Deno.test("Plugin - SwitchStatement", () => { + const node = testLintNode( + `switch (foo) { + case foo: + case bar: + break; + default: + {} + }`, + "SwitchStatement", + ); + assertEquals(node[0], { + type: "SwitchStatement", + range: [1, 94], + discriminant: { + type: "Identifier", + range: [9, 12], + name: "foo", + }, + cases: [ + { + type: "SwitchCase", + range: [22, 31], + test: { + type: "Identifier", + range: [27, 30], + name: "foo", + }, + consequent: [], + }, + { + type: "SwitchCase", + range: [38, 62], + test: { + type: "Identifier", + range: [43, 46], + name: "bar", + }, + consequent: [ + { + type: "BreakStatement", + label: null, + range: [56, 62], + }, + ], + }, + { + type: "SwitchCase", + range: [69, 88], + test: null, + consequent: [ + { + type: "BlockStatement", + range: [86, 88], + body: [], + }, + ], + }, + ], + }); +}); + +Deno.test("Plugin - ThrowStatement", () => { + const node = testLintNode("throw foo;", "ThrowStatement"); + assertEquals(node[0], { + type: "ThrowStatement", + range: [1, 11], + argument: { + type: "Identifier", + range: [7, 10], + name: "foo", + }, + }); +}); + +Deno.test("Plugin - TryStatement", () => { + let node = testLintNode("try {} catch {};", "TryStatement"); + assertEquals(node[0], { + type: "TryStatement", + range: [1, 16], + block: { + type: "BlockStatement", + range: [5, 7], + body: [], + }, + handler: { + type: "CatchClause", + range: [8, 16], + param: null, + body: { + type: "BlockStatement", + range: [14, 16], + body: [], + }, + }, + finalizer: null, + }); + + node = testLintNode("try {} catch (e) {};", "TryStatement"); + assertEquals(node[0], { + type: "TryStatement", + range: [1, 20], + block: { + type: "BlockStatement", + range: [5, 7], + body: [], + }, + handler: { + type: "CatchClause", + range: [8, 20], + param: { + type: "Identifier", + range: [15, 16], + name: "e", + }, + body: { + type: "BlockStatement", + range: [18, 20], + body: [], + }, + }, + finalizer: null, + }); + + node = testLintNode("try {} finally {};", "TryStatement"); + assertEquals(node[0], { + type: "TryStatement", + range: [1, 18], + block: { + type: "BlockStatement", + range: [5, 7], + body: [], + }, + handler: null, + finalizer: { + type: "BlockStatement", + range: [16, 18], + body: [], + }, + }); +}); + +Deno.test("Plugin - WhileStatement", () => { + const node = testLintNode("while (foo) {}", "WhileStatement"); + assertEquals(node[0], { + type: "WhileStatement", + range: [1, 15], + test: { + type: "Identifier", + range: [8, 11], + name: "foo", + }, + body: { + type: "BlockStatement", + range: [13, 15], + body: [], + }, + }); +}); diff --git a/tests/unit/ops_test.ts b/tests/unit/ops_test.ts index 6de55f8b66..631e5c5736 100644 --- a/tests/unit/ops_test.ts +++ b/tests/unit/ops_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -const EXPECTED_OP_COUNT = 12; +const EXPECTED_OP_COUNT = 13; Deno.test(function checkExposedOps() { // @ts-ignore TS doesn't allow to index with symbol From e6869d7fa668017bacf23ad80a52a4168f562e7b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 21 Dec 2024 10:00:18 -0500 Subject: [PATCH 17/20] fix(node): handle cjs exports with escaped chars (#27438) Closes https://github.com/denoland/deno/issues/27422 --- resolvers/node/analyze.rs | 26 +++++++++++-------- .../cjs_key_escaped_whitespace/__test__.jsonc | 4 +++ .../node/cjs_key_escaped_whitespace/main.js | 2 ++ .../cjs_key_escaped_whitespace/module.cjs | 6 +++++ .../cjs_key_escaped_whitespace/output.out | 7 +++++ 5 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 tests/specs/node/cjs_key_escaped_whitespace/__test__.jsonc create mode 100644 tests/specs/node/cjs_key_escaped_whitespace/main.js create mode 100644 tests/specs/node/cjs_key_escaped_whitespace/module.cjs create mode 100644 tests/specs/node/cjs_key_escaped_whitespace/output.out diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index a444f4d923..9e6219b082 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -162,7 +162,7 @@ impl add_export( &mut source, export, - &format!("mod[\"{}\"]", escape_for_double_quote_string(export)), + &format!("mod[{}]", to_double_quote_string(export)), &mut temp_var_count, ); } @@ -561,8 +561,8 @@ fn add_export( "const __deno_export_{temp_var_count}__ = {initializer};" )); source.push(format!( - "export {{ __deno_export_{temp_var_count}__ as \"{}\" }};", - escape_for_double_quote_string(name) + "export {{ __deno_export_{temp_var_count}__ as {} }};", + to_double_quote_string(name) )); } else { source.push(format!("export const {name} = {initializer};")); @@ -620,14 +620,9 @@ fn not_found(path: &str, referrer: &Path) -> AnyError { std::io::Error::new(std::io::ErrorKind::NotFound, msg).into() } -fn escape_for_double_quote_string(text: &str) -> Cow { - // this should be rare, so doing a scan first before allocating is ok - if text.chars().any(|c| matches!(c, '"' | '\\')) { - // don't bother making this more complex for perf because it's rare - Cow::Owned(text.replace('\\', "\\\\").replace('"', "\\\"")) - } else { - Cow::Borrowed(text) - } +fn to_double_quote_string(text: &str) -> String { + // serde can handle this for us + serde_json::to_string(text).unwrap() } #[cfg(test)] @@ -665,4 +660,13 @@ mod tests { Some(("@some-package/core".to_string(), "./actions".to_string())) ); } + + #[test] + fn test_to_double_quote_string() { + assert_eq!(to_double_quote_string("test"), "\"test\""); + assert_eq!( + to_double_quote_string("\r\n\t\"test"), + "\"\\r\\n\\t\\\"test\"" + ); + } } diff --git a/tests/specs/node/cjs_key_escaped_whitespace/__test__.jsonc b/tests/specs/node/cjs_key_escaped_whitespace/__test__.jsonc new file mode 100644 index 0000000000..ebaae5bfd6 --- /dev/null +++ b/tests/specs/node/cjs_key_escaped_whitespace/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run -A main.js", + "output": "output.out" +} diff --git a/tests/specs/node/cjs_key_escaped_whitespace/main.js b/tests/specs/node/cjs_key_escaped_whitespace/main.js new file mode 100644 index 0000000000..9d4f2ee26c --- /dev/null +++ b/tests/specs/node/cjs_key_escaped_whitespace/main.js @@ -0,0 +1,2 @@ +const bang = await import("./module.cjs"); +console.log("imported:", bang); diff --git a/tests/specs/node/cjs_key_escaped_whitespace/module.cjs b/tests/specs/node/cjs_key_escaped_whitespace/module.cjs new file mode 100644 index 0000000000..5accc6196a --- /dev/null +++ b/tests/specs/node/cjs_key_escaped_whitespace/module.cjs @@ -0,0 +1,6 @@ +module.exports = { + "\nx": "test", + "\ty": "test", + "\rz": "test", + '"a': "test", +}; diff --git a/tests/specs/node/cjs_key_escaped_whitespace/output.out b/tests/specs/node/cjs_key_escaped_whitespace/output.out new file mode 100644 index 0000000000..49e92abdec --- /dev/null +++ b/tests/specs/node/cjs_key_escaped_whitespace/output.out @@ -0,0 +1,7 @@ +imported: [Module: null prototype] { + "\ty": "test", + "\nx": "test", + "\rz": "test", + '"a': "test", + default: { "\nx": "test", "\ty": "test", "\rz": "test", '"a': "test" } +} From 2e58e088b09642acf9e4b7f08f89cfc2218985a8 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 23 Dec 2024 13:56:37 +0900 Subject: [PATCH 18/20] fix(ext/fs): do not throw for bigint ctime/mtime/atime (#27453) --- ext/fs/30_fs.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/fs/30_fs.js b/ext/fs/30_fs.js index fc2b18be13..4e71acb1b2 100644 --- a/ext/fs/30_fs.js +++ b/ext/fs/30_fs.js @@ -77,6 +77,7 @@ const { Error, Function, MathTrunc, + Number, ObjectEntries, ObjectDefineProperty, ObjectPrototypeIsPrototypeOf, @@ -373,12 +374,12 @@ function parseFileInfo(response) { isDirectory: response.isDirectory, isSymlink: response.isSymlink, size: response.size, - mtime: response.mtimeSet === true ? new Date(response.mtime) : null, - atime: response.atimeSet === true ? new Date(response.atime) : null, + mtime: response.mtimeSet === true ? new Date(Number(response.mtime)) : null, + atime: response.atimeSet === true ? new Date(Number(response.atime)) : null, birthtime: response.birthtimeSet === true ? new Date(response.birthtime) : null, - ctime: response.ctimeSet === true ? new Date(response.ctime) : null, + ctime: response.ctimeSet === true ? new Date(Number(response.ctime)) : null, dev: response.dev, mode: response.mode, ino: unix ? response.ino : null, From 3cc861cdca4fb204fbcf40c9542396965039f31f Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 23 Dec 2024 13:59:04 +0900 Subject: [PATCH 19/20] fix(ext/fetch): better error message when body resource is unavailable (#27429) fixes #27132 When the body resource is unavailable when start reading it, the error message is `Bad Resource ID` and that doesn't tell what's wrong very well. This PR changes that error message to `Cannot read body as underlying resource unavailable` --- ext/fetch/22_body.js | 15 ++++++++++++++- tests/unit/body_test.ts | 13 ++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ext/fetch/22_body.js b/ext/fetch/22_body.js index a34758d19a..bb2bee77e2 100644 --- a/ext/fetch/22_body.js +++ b/ext/fetch/22_body.js @@ -13,6 +13,7 @@ import { core, primordials } from "ext:core/mod.js"; const { + BadResourcePrototype, isAnyArrayBuffer, isArrayBuffer, isStringObject, @@ -26,6 +27,7 @@ const { JSONParse, ObjectDefineProperties, ObjectPrototypeIsPrototypeOf, + PromisePrototypeCatch, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetByteLength, TypedArrayPrototypeGetByteOffset, @@ -160,7 +162,18 @@ class InnerBody { ) ) { readableStreamThrowIfErrored(this.stream); - return readableStreamCollectIntoUint8Array(this.stream); + return PromisePrototypeCatch( + readableStreamCollectIntoUint8Array(this.stream), + (e) => { + if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, e)) { + // TODO(kt3k): We probably like to pass e as `cause` if BadResource supports it. + throw new e.constructor( + "Cannot read body as underlying resource unavailable", + ); + } + throw e; + }, + ); } else { this.streamOrStatic.consumed = true; return this.streamOrStatic.body; diff --git a/tests/unit/body_test.ts b/tests/unit/body_test.ts index 18cdb22be0..fb51fd0076 100644 --- a/tests/unit/body_test.ts +++ b/tests/unit/body_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assert, assertEquals } from "./test_util.ts"; +import { assert, assertEquals, assertRejects } from "./test_util.ts"; // just a hack to get a body object // deno-lint-ignore no-explicit-any @@ -187,3 +187,14 @@ Deno.test( assertEquals(file.size, 1); }, ); + +Deno.test(async function bodyBadResourceError() { + const file = await Deno.open("README.md"); + file.close(); + const body = buildBody(file.readable); + await assertRejects( + () => body.arrayBuffer(), + Deno.errors.BadResource, + "Cannot read body as underlying resource unavailable", + ); +}); From 1a809b811559b5d77039eb7b270482816766ab74 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Mon, 23 Dec 2024 08:45:47 +0100 Subject: [PATCH 20/20] feat(unstable): support selectors in JS lint plugins (#27452) This PR adds support for using selectors in the JS linting plugin API. Supported at the moment are: - `Foo Bar` (descendant) - `Foo > Bar` (child combinator) - `Foo + Foo` (next sibling) - `Foo ~ Foo` (subsequent sibling) - `[attr]`, `[attr=value]` (attribute selectors, supported operators: `=`, `!=`, `<`, `>`, `<=`, `>=`) - `:first-child` - `:last-child` - `:nth-child(2)`, `:nth-child(2n + 1)` --- cli/js/40_lint.js | 402 +++++++++- cli/js/40_lint_selector.js | 1014 ++++++++++++++++++++++++ cli/js/40_lint_types.d.ts | 84 +- cli/tools/lint/ast_buffer/buffer.rs | 2 + cli/tools/lint/ast_buffer/ts_estree.rs | 2 + cli/worker.rs | 2 + tests/integration/js_unit_tests.rs | 1 + tests/unit/lint_plugin_test.ts | 208 ++++- tests/unit/lint_selectors_test.ts | 610 ++++++++++++++ tools/core_import_map.json | 1 + 10 files changed, 2272 insertions(+), 54 deletions(-) create mode 100644 cli/js/40_lint_selector.js create mode 100644 tests/unit/lint_selectors_test.ts diff --git a/cli/js/40_lint.js b/cli/js/40_lint.js index 9606f787b3..30b1f86884 100644 --- a/cli/js/40_lint.js +++ b/cli/js/40_lint.js @@ -2,6 +2,11 @@ // @ts-check +import { + compileSelector, + parseSelector, + splitSelectors, +} from "ext:cli/40_lint_selector.js"; import { core, internals } from "ext:core/mod.js"; const { op_lint_create_serialized_ast, @@ -13,6 +18,7 @@ const { const AST_PROP_TYPE = 0; const AST_PROP_PARENT = 1; const AST_PROP_RANGE = 2; +const AST_PROP_LENGTH = 3; // Keep in sync with Rust // Each node property is tagged with this enum to denote @@ -43,8 +49,8 @@ const PropFlags = { /** @typedef {import("./40_lint_types.d.ts").RuleContext} RuleContext */ /** @typedef {import("./40_lint_types.d.ts").NodeFacade} NodeFacade */ /** @typedef {import("./40_lint_types.d.ts").LintPlugin} LintPlugin */ -/** @typedef {import("./40_lint_types.d.ts").LintReportData} LintReportData */ -/** @typedef {import("./40_lint_types.d.ts").TestReportData} TestReportData */ +/** @typedef {import("./40_lint_types.d.ts").TransformFn} TransformFn */ +/** @typedef {import("./40_lint_types.d.ts").MatchContext} MatchContext */ /** @type {LintState} */ const state = { @@ -99,7 +105,6 @@ export function installPlugin(plugin) { */ function getNode(ctx, offset) { if (offset === 0) return null; - const cached = ctx.nodes.get(offset); if (cached !== undefined) return cached; @@ -297,9 +302,10 @@ function readValue(ctx, offset, search) { if (offset === -1) return undefined; const kind = buf[offset + 1]; + offset += 2; if (kind === PropFlags.Ref) { - const value = readU32(buf, offset + 2); + const value = readU32(buf, offset); return getNode(ctx, value); } else if (kind === PropFlags.RefArr) { const len = readU32(buf, offset); @@ -353,6 +359,303 @@ function getString(strTable, id) { return name; } +/** + * @param {AstContext["buf"]} buf + * @param {number} child + * @returns {null | [number, number]} + */ +function findChildOffset(buf, child) { + let offset = readU32(buf, child + 1); + + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + const propCount = buf[offset++]; + for (let i = 0; i < propCount; i++) { + const _prop = buf[offset++]; + const kind = buf[offset++]; + + switch (kind) { + case PropFlags.Ref: { + const start = offset; + const value = readU32(buf, offset); + offset += 4; + if (value === child) { + return [start, -1]; + } + break; + } + case PropFlags.RefArr: { + const start = offset; + + const len = readU32(buf, offset); + offset += 4; + + for (let j = 0; j < len; j++) { + const value = readU32(buf, offset); + offset += 4; + if (value === child) { + return [start, j]; + } + } + + break; + } + case PropFlags.String: + offset += 4; + break; + case PropFlags.Bool: + offset++; + break; + case PropFlags.Null: + case PropFlags.Undefined: + break; + } + } + + return null; +} + +/** @implements {MatchContext} */ +class MatchCtx { + /** + * @param {AstContext["buf"]} buf + * @param {AstContext["strTable"]} strTable + */ + constructor(buf, strTable) { + this.buf = buf; + this.strTable = strTable; + } + + /** + * @param {number} offset + * @returns {number} + */ + getParent(offset) { + return readU32(this.buf, offset + 1); + } + + /** + * @param {number} offset + * @returns {number} + */ + getType(offset) { + return this.buf[offset]; + } + + /** + * @param {number} offset + * @param {number[]} propIds + * @param {number} idx + * @returns {unknown} + */ + getAttrPathValue(offset, propIds, idx) { + const { buf } = this; + + offset = findPropOffset(buf, offset, propIds[idx]); + if (offset === -1) return undefined; + const _prop = buf[offset++]; + const kind = buf[offset++]; + + if (kind === PropFlags.Ref) { + const value = readU32(buf, offset); + // Checks need to end with a value, not a node + if (idx === propIds.length - 1) return undefined; + return this.getAttrPathValue(value, propIds, idx + 1); + } else if (kind === PropFlags.RefArr) { + const count = readU32(buf, offset); + offset += 4; + + if (idx < propIds.length - 1 && propIds[idx + 1] === AST_PROP_LENGTH) { + return count; + } + + // TODO(@marvinhagemeister): Allow traversing into array children? + } + + // Cannot traverse into primitives further + if (idx < propIds.length - 1) return undefined; + + if (kind === PropFlags.String) { + const s = readU32(buf, offset); + return getString(this.strTable, s); + } else if (kind === PropFlags.Bool) { + return buf[offset] === 1; + } else if (kind === PropFlags.Null) { + return null; + } else if (kind === PropFlags.Undefined) { + return undefined; + } + + return undefined; + } + + /** + * @param {number} offset + * @param {number[]} propIds + * @param {number} idx + * @returns {boolean} + */ + hasAttrPath(offset, propIds, idx) { + const { buf } = this; + + offset = findPropOffset(buf, offset, propIds[idx]); + if (offset === -1) return false; + if (idx === propIds.length - 1) return true; + + const _prop = buf[offset++]; + const kind = buf[offset++]; + if (kind === PropFlags.Ref) { + const value = readU32(buf, offset); + return this.hasAttrPath(value, propIds, idx + 1); + } else if (kind === PropFlags.RefArr) { + const _count = readU32(buf, offset); + offset += 4; + + if (idx < propIds.length - 1 && propIds[idx + 1] === AST_PROP_LENGTH) { + return true; + } + + // TODO(@marvinhagemeister): Allow traversing into array children? + } + + // Primitives cannot be traversed further. This means we + // didn't found the attribute. + if (idx < propIds.length - 1) return false; + + return true; + } + + /** + * @param {number} offset + * @returns {number} + */ + getFirstChild(offset) { + const { buf } = this; + + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + const count = buf[offset++]; + for (let i = 0; i < count; i++) { + const _prop = buf[offset++]; + const kind = buf[offset++]; + + switch (kind) { + case PropFlags.Ref: { + const v = readU32(buf, offset); + offset += 4; + return v; + } + case PropFlags.RefArr: { + const len = readU32(buf, offset); + offset += 4; + for (let j = 0; j < len; j++) { + const v = readU32(buf, offset); + offset += 4; + return v; + } + + return len; + } + + case PropFlags.String: + offset += 4; + break; + case PropFlags.Bool: + offset++; + break; + case PropFlags.Null: + case PropFlags.Undefined: + break; + } + } + + return -1; + } + + /** + * @param {number} offset + * @returns {number} + */ + getLastChild(offset) { + const { buf } = this; + + // type + parentId + SpanLo + SpanHi + offset += 1 + 4 + 4 + 4; + + let last = -1; + + const count = buf[offset++]; + for (let i = 0; i < count; i++) { + const _prop = buf[offset++]; + const kind = buf[offset++]; + + switch (kind) { + case PropFlags.Ref: { + const v = readU32(buf, offset); + offset += 4; + last = v; + break; + } + case PropFlags.RefArr: { + const len = readU32(buf, offset); + offset += 4; + for (let j = 0; j < len; j++) { + const v = readU32(buf, offset); + last = v; + offset += 4; + } + + break; + } + + case PropFlags.String: + offset += 4; + break; + case PropFlags.Bool: + offset++; + break; + case PropFlags.Null: + case PropFlags.Undefined: + break; + } + } + + return last; + } + + /** + * @param {number} id + * @returns {number[]} + */ + getSiblings(id) { + const { buf } = this; + + const result = findChildOffset(buf, id); + // Happens for program nodes + if (result === null) return []; + + if (result[1] === -1) { + return [id]; + } + + let offset = result[0]; + const count = readU32(buf, offset); + offset += 4; + + /** @type {number[]} */ + const out = []; + for (let i = 0; i < count; i++) { + const v = readU32(buf, offset); + offset += 4; + out.push(v); + } + + return out; + } +} + /** * @param {Uint8Array} buf * @param {AstContext} buf @@ -433,6 +736,7 @@ function createAstContext(buf) { strByType, typeByStr, propByStr, + matcher: new MatchCtx(buf, strTable), }; setNodeGetters(ctx); @@ -456,7 +760,7 @@ const NOOP = (_node) => {}; export function runPluginsForFile(fileName, serializedAst) { const ctx = createAstContext(serializedAst); - /** @type {Map} */ + /** @type {Map}>} */ const bySelector = new Map(); const destroyFns = []; @@ -486,32 +790,38 @@ export function runPluginsForFile(fileName, serializedAst) { key = key.slice(0, -":exit".length); } - let info = bySelector.get(key); - if (info === undefined) { - info = { enter: NOOP, exit: NOOP }; - bySelector.set(key, info); - } - const prevFn = isExit ? info.exit : info.enter; + const selectors = splitSelectors(key); - /** - * @param {*} node - */ - const wrapped = (node) => { - prevFn(node); + for (let j = 0; j < selectors.length; j++) { + const key = selectors[j]; - try { - fn(node); - } catch (err) { - throw new Error(`Visitor "${name}" of plugin "${id}" errored`, { - cause: err, - }); + let info = bySelector.get(key); + if (info === undefined) { + info = { enter: NOOP, exit: NOOP }; + bySelector.set(key, info); } - }; + const prevFn = isExit ? info.exit : info.enter; - if (isExit) { - info.exit = wrapped; - } else { - info.enter = wrapped; + /** + * @param {*} node + */ + const wrapped = (node) => { + prevFn(node); + + try { + fn(node); + } catch (err) { + throw new Error(`Visitor "${name}" of plugin "${id}" errored`, { + cause: err, + }); + } + }; + + if (isExit) { + info.exit = wrapped; + } else { + info.enter = wrapped; + } } } @@ -528,25 +838,27 @@ export function runPluginsForFile(fileName, serializedAst) { } } + // Create selectors + /** @type {TransformFn} */ + const toElem = (str) => { + const id = ctx.typeByStr.get(str); + return id === undefined ? 0 : id; + }; + /** @type {TransformFn} */ + const toAttr = (str) => { + const id = ctx.propByStr.get(str); + return id === undefined ? 0 : id; + }; + /** @type {CompiledVisitor[]} */ const visitors = []; for (const [sel, info] of bySelector.entries()) { - // This will make more sense once selectors land as it's faster - // to precompile them once upfront. + // Selectors are already split here. + // TODO(@marvinhagemeister): Avoid array allocation (not sure if that matters) + const parsed = parseSelector(sel, toElem, toAttr)[0]; + const matcher = compileSelector(parsed); - // Convert the visiting element name to a number. This number - // is part of the serialized buffer and comparing a single number - // is quicker than strings. - const elemId = ctx.typeByStr.get(sel) ?? -1; - - visitors.push({ - info, - // Check if we should call this visitor - matcher: (offset) => { - const type = ctx.buf[offset]; - return type === elemId; - }, - }); + visitors.push({ info, matcher }); } // Traverse ast with all visitors at the same time to avoid traversing @@ -572,6 +884,8 @@ function traverse(ctx, visitors, offset) { // The 0 offset is used to denote an empty/placeholder node if (offset === 0) return; + const originalOffset = offset; + const { buf } = ctx; /** @type {VisitorFn[] | null} */ @@ -580,7 +894,7 @@ function traverse(ctx, visitors, offset) { for (let i = 0; i < visitors.length; i++) { const v = visitors[i]; - if (v.matcher(offset)) { + if (v.matcher(ctx.matcher, offset)) { if (v.info.exit !== NOOP) { if (exits === null) { exits = [v.info.exit]; @@ -633,7 +947,7 @@ function traverse(ctx, visitors, offset) { } finally { if (exits !== null) { for (let i = 0; i < exits.length; i++) { - const node = /** @type {*} */ (getNode(ctx, offset)); + const node = /** @type {*} */ (getNode(ctx, originalOffset)); exits[i](node); } } diff --git a/cli/js/40_lint_selector.js b/cli/js/40_lint_selector.js new file mode 100644 index 0000000000..b78f7a5d0e --- /dev/null +++ b/cli/js/40_lint_selector.js @@ -0,0 +1,1014 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// @ts-check + +/** @typedef {import("./40_lint_types.d.ts").LintState} LintState */ +/** @typedef {import("./40_lint_types.d.ts").AstContext} AstContext */ +/** @typedef {import("./40_lint_types.d.ts").MatchContext} MatchCtx */ +/** @typedef {import("./40_lint_types.d.ts").AttrExists} AttrExists */ +/** @typedef {import("./40_lint_types.d.ts").AttrBin} AttrBin */ +/** @typedef {import("./40_lint_types.d.ts").AttrSelector} AttrSelector */ +/** @typedef {import("./40_lint_types.d.ts").ElemSelector} ElemSelector */ +/** @typedef {import("./40_lint_types.d.ts").PseudoNthChild} PseudoNthChild */ +/** @typedef {import("./40_lint_types.d.ts").PseudoHas} PseudoHas */ +/** @typedef {import("./40_lint_types.d.ts").PseudoNot} PseudoNot */ +/** @typedef {import("./40_lint_types.d.ts").Relation} SRelation */ +/** @typedef {import("./40_lint_types.d.ts").Selector} Selector */ +/** @typedef {import("./40_lint_types.d.ts").SelectorParseCtx} SelectorParseCtx */ +/** @typedef {import("./40_lint_types.d.ts").NextFn} NextFn */ +/** @typedef {import("./40_lint_types.d.ts").MatcherFn} MatcherFn */ +/** @typedef {import("./40_lint_types.d.ts").TransformFn} Transformer */ + +const Char = { + Tab: 9, + Space: 32, + Bang: 33, + DoubleQuote: 34, + Quote: 39, + BraceOpen: 40, + BraceClose: 41, + Plus: 43, + Comma: 44, + Minus: 45, + Dot: 46, + Slash: 47, + n0: 49, + n9: 57, + Colon: 58, + Less: 60, + Equal: 61, + Greater: 62, + A: 65, + Z: 90, + BracketOpen: 91, + BackSlash: 92, + BracketClose: 93, + Underscore: 95, + a: 97, + z: 122, + Tilde: 126, +}; + +export const Token = { + EOF: 0, + Word: 1, + Space: 2, + Op: 3, + Colon: 4, + Comma: 7, + BraceOpen: 8, + BraceClose: 9, + BracketOpen: 10, + BracketClose: 11, + String: 12, + Number: 13, + Bool: 14, + Null: 15, + Undefined: 16, + Dot: 17, + Minus: 17, +}; + +export const BinOp = { + /** [attr="value"] or [attr=value] */ + Equal: 1, + /** [attr!="value"] or [attr!=value] */ + NotEqual: 2, + /** [attr>1] */ + Greater: 3, + /** [attr>=1] */ + GreaterThan: 4, + /** [attr<1] */ + Less: 5, + /** [attr<=1] */ + LessThan: 6, + Tilde: 7, + Plus: 8, + Space: 9, +}; + +/** + * @param {string} s + * @returns {number} + */ +function getAttrOp(s) { + switch (s) { + case "=": + return BinOp.Equal; + case "!=": + return BinOp.NotEqual; + case ">": + return BinOp.Greater; + case ">=": + return BinOp.GreaterThan; + case "<": + return BinOp.Less; + case "<=": + return BinOp.LessThan; + case "~": + return BinOp.Tilde; + case "+": + return BinOp.Plus; + default: + throw new Error(`Unknown attribute operator: '${s}'`); + } +} + +export class Lexer { + token = Token.Word; + start = 0; + end = 0; + ch = 0; + i = -1; + + value = ""; + + /** + * @param {string} input + */ + constructor(input) { + this.input = input; + this.step(); + this.next(); + } + + /** + * @param {number} token + */ + expect(token) { + if (this.token !== token) { + throw new Error( + `Expected token '${token}', but got '${this.token}'.\n\n${this.input}\n${ + " ".repeat(this.i) + }^`, + ); + } + } + + /** + * @param {number} token + */ + readAsWordUntil(token) { + const s = this.i; + while (this.token !== Token.EOF && this.token !== token) { + this.next(); + } + + this.start = s; + this.end = this.i - 1; + this.value = this.getSlice(); + } + + getSlice() { + return this.input.slice(this.start, this.end); + } + + step() { + this.i++; + if (this.i >= this.input.length) { + this.ch = -1; + } else { + this.ch = this.input.charCodeAt(this.i); + } + } + + next() { + this.value = ""; + + if (this.i >= this.input.length) { + this.token = Token.EOF; + return; + } + + // console.log( + // "NEXT", + // this.input, + // this.i, + // JSON.stringify(String.fromCharCode(this.ch)), + // ); + + while (true) { + switch (this.ch) { + case Char.Space: + while (this.isWhiteSpace()) { + this.step(); + } + + // Check if space preceeded operator + if (this.isOpContinue()) { + continue; + } + + this.token = Token.Space; + return; + case Char.BracketOpen: + this.token = Token.BracketOpen; + this.step(); + return; + case Char.BracketClose: + this.token = Token.BracketClose; + this.step(); + return; + case Char.BraceOpen: + this.token = Token.BraceOpen; + this.step(); + return; + case Char.BraceClose: + this.token = Token.BraceClose; + this.step(); + return; + case Char.Colon: + this.token = Token.Colon; + this.step(); + return; + case Char.Comma: + this.token = Token.Comma; + this.step(); + return; + case Char.Dot: + this.token = Token.Dot; + this.step(); + return; + case Char.Minus: + this.token = Token.Minus; + this.step(); + return; + + case Char.Plus: + case Char.Tilde: + case Char.Greater: + case Char.Equal: + case Char.Less: + case Char.Bang: { + this.token = Token.Op; + this.start = this.i; + this.step(); + + while (this.isOpContinue()) { + this.step(); + } + + this.end = this.i; + this.value = this.getSlice(); + + // Consume remaining space + while (this.isWhiteSpace()) { + this.step(); + } + + return; + } + + case Char.Quote: + case Char.DoubleQuote: { + this.token = Token.String; + const ch = this.ch; + + this.step(); + this.start = this.i; + + while (this.ch > 0 && this.ch !== ch) { + this.step(); + } + + this.end = this.i; + this.value = this.getSlice(); + this.step(); + + return; + } + + default: + this.start = this.i; + this.step(); + + while (this.isWordContinue()) { + this.step(); + } + + this.end = this.i; + this.value = this.getSlice(); + this.token = Token.Word; + return; + } + } + } + + isWordContinue() { + const ch = this.ch; + switch (ch) { + case Char.Minus: + case Char.Underscore: + return true; + default: + return (ch >= Char.a && ch <= Char.z) || + (ch >= Char.A && ch <= Char.Z) || + (ch >= Char.n0 && ch <= Char.n9); + } + } + + isOpContinue() { + const ch = this.ch; + switch (ch) { + case Char.Equal: + case Char.Bang: + case Char.Greater: + case Char.Less: + case Char.Tilde: + case Char.Plus: + return true; + default: + return false; + } + } + + isWhiteSpace() { + return this.ch === Char.Space || this.ch === Char.Tab; + } +} + +const NUMBER_REG = /^(\d+\.)?\d+$/; +const BIGINT_REG = /^\d+n$/; + +/** + * @param {string} raw + * @returns {any} + */ +function getFromRawValue(raw) { + switch (raw) { + case "true": + return true; + case "false": + return false; + case "null": + return null; + case "undefined": + return undefined; + default: + if (raw.startsWith("'") && raw.endsWith("'")) { + if (raw.length === 2) return ""; + return raw.slice(1, -1); + } else if (raw.startsWith('"') && raw.endsWith('"')) { + if (raw.length === 2) return ""; + return raw.slice(1, -1); + } else if (raw.startsWith("/")) { + const end = raw.lastIndexOf("/"); + if (end === -1) throw new Error(`Invalid RegExp pattern: ${raw}`); + const pattern = raw.slice(1, end); + const flags = end < raw.length - 1 ? raw.slice(end + 1) : undefined; + return new RegExp(pattern, flags); + } else if (NUMBER_REG.test(raw)) { + return Number(raw); + } else if (BIGINT_REG.test(raw)) { + return BigInt(raw.slice(0, -1)); + } + + return raw; + } +} + +export const ELEM_NODE = 1; +export const RELATION_NODE = 2; +export const ATTR_EXISTS_NODE = 3; +export const ATTR_BIN_NODE = 4; +export const PSEUDO_NTH_CHILD = 5; +export const PSEUDO_HAS = 6; +export const PSEUDO_NOT = 7; +export const PSEUDO_FIRST_CHILD = 8; +export const PSEUDO_LAST_CHILD = 9; + +/** + * Parse out all unique selectors of a selector list. + * @param {string} input + * @returns {string[]} + */ +export function splitSelectors(input) { + /** @type {string[]} */ + const out = []; + + let last = 0; + let depth = 0; + for (let i = 0; i < input.length; i++) { + const ch = input.charCodeAt(i); + switch (ch) { + case Char.BraceOpen: + depth++; + break; + case Char.BraceClose: + depth--; + break; + case Char.Comma: + if (depth === 0) { + out.push(input.slice(last, i).trim()); + last = i + 1; + } + break; + } + } + + if (last < input.length - 1) { + out.push(input.slice(last).trim()); + } + + return out; +} + +/** + * @param {string} input + * @param {Transformer} toElem + * @param {Transformer} toAttr + * @returns {Selector[]} + */ +export function parseSelector(input, toElem, toAttr) { + /** @type {Selector[]} */ + const result = []; + + /** @type {Selector[]} */ + const stack = [[]]; + + const lex = new Lexer(input); + + // Some subselectors like `:nth-child(.. of )` must have + // a single selector instead of selector list. + let throwOnComma = false; + + while (lex.token !== Token.EOF) { + const current = /** @type {Selector} */ (stack.at(-1)); + + if (lex.token === Token.Word) { + const value = lex.value; + const wildcard = value === "*"; + + const elem = !wildcard ? toElem(value) : 0; + current.push({ + type: ELEM_NODE, + elem, + wildcard, + }); + lex.next(); + + continue; + } else if (lex.token === Token.Space) { + lex.next(); + + if (lex.token === Token.Word) { + current.push({ + type: RELATION_NODE, + op: BinOp.Space, + }); + } + + continue; + } else if (lex.token === Token.BracketOpen) { + lex.next(); + lex.expect(Token.Word); + + // Check for value comparison + const prop = [toAttr(lex.value)]; + lex.next(); + + while (lex.token === Token.Dot) { + lex.next(); + lex.expect(Token.Word); + + prop.push(toAttr(lex.value)); + lex.next(); + } + + if (lex.token === Token.Op) { + const op = getAttrOp(lex.value); + lex.readAsWordUntil(Token.BracketClose); + + const value = getFromRawValue(lex.value); + current.push({ type: ATTR_BIN_NODE, prop, op, value }); + } else { + current.push({ + type: ATTR_EXISTS_NODE, + prop, + }); + } + + lex.expect(Token.BracketClose); + lex.next(); + continue; + } else if (lex.token === Token.Colon) { + lex.next(); + lex.expect(Token.Word); + + switch (lex.value) { + case "first-child": + current.push({ + type: PSEUDO_FIRST_CHILD, + }); + break; + case "last-child": + current.push({ + type: PSEUDO_LAST_CHILD, + }); + break; + case "nth-child": { + lex.next(); + lex.expect(Token.BraceOpen); + lex.next(); + + let mul = 1; + let repeat = false; + let step = 0; + if (lex.token === Token.Minus) { + mul = -1; + lex.next(); + } + + lex.expect(Token.Word); + const value = lex.getSlice(); + + if (value.endsWith("n")) { + repeat = true; + step = +value.slice(0, -1) * mul; + } else { + step = +value * mul; + } + + lex.next(); + + /** @type {PseudoNthChild} */ + const node = { + type: PSEUDO_NTH_CHILD, + of: null, + op: null, + step, + stepOffset: 0, + repeat, + }; + current.push(node); + + if (lex.token === Token.Space) lex.next(); + + if (lex.token !== Token.BraceClose) { + if (lex.token === Token.Op) { + node.op = lex.value; + lex.next(); + + if (lex.token === Token.Space) lex.next(); + } else if (lex.token === Token.Minus) { + node.op = "-"; + lex.next(); + + if (lex.token === Token.Space) { + lex.next(); + } + } + + lex.expect(Token.Word); + node.stepOffset = +lex.value; + lex.next(); + + if (lex.token !== Token.BraceClose) { + lex.next(); // Space + + if (lex.token === Token.Word) { + if (/** @type {string} */ (lex.value) !== "of") { + throw new Error( + `Expected 'of' keyword in ':nth-child' but got: ${lex.value}`, + ); + } + + lex.next(); + lex.expect(Token.Space); + lex.next(); + throwOnComma = true; + stack.push([]); + } + + continue; + } + + lex.expect(Token.BraceClose); + } else if (!node.repeat) { + // :nth-child(2) -> step is actually stepOffset + node.stepOffset = node.step - 1; + node.step = 0; + } + + lex.next(); + + continue; + } + + case "has": + case "where": + case "is": { + lex.next(); + lex.expect(Token.BraceOpen); + lex.next(); + + current.push({ + type: PSEUDO_HAS, + selectors: [], + }); + stack.push([]); + + continue; + } + case "not": { + lex.next(); + lex.expect(Token.BraceOpen); + lex.next(); + + current.push({ + type: PSEUDO_NOT, + selectors: [], + }); + stack.push([]); + + continue; + } + default: + throw new Error(`Unknown pseudo selector: '${lex.value}'`); + } + } else if (lex.token === Token.Comma) { + if (throwOnComma) { + throw new Error(`Multiple selector arguments not supported here`); + } + + lex.next(); + if (lex.token === Token.Space) { + lex.next(); + } + + popSelector(result, stack); + stack.push([]); + continue; + } else if (lex.token === Token.BraceClose) { + throwOnComma = false; + popSelector(result, stack); + } else if (lex.token === Token.Op) { + current.push({ + type: RELATION_NODE, + op: getAttrOp(lex.value), + }); + } + + lex.next(); + } + + if (stack.length > 0) { + result.push(stack[0]); + } + + return result; +} + +/** + * @param {Selector[]} result + * @param {Selector[]} stack + */ +function popSelector(result, stack) { + const sel = /** @type {Selector} */ (stack.pop()); + + if (stack.length === 0) { + result.push(sel); + stack.push([]); + } else { + const prev = /** @type {Selector} */ (stack.at(-1)); + if (prev.length === 0) { + throw new Error(`Empty selector`); + } + + const node = prev.at(-1); + if (node === undefined) { + throw new Error(`Empty node`); + } + + if (node.type === PSEUDO_NTH_CHILD) { + node.of = sel; + } else if (node.type === PSEUDO_HAS || node.type === PSEUDO_NOT) { + node.selectors.push(sel); + } else { + throw new Error(`Multiple selectors not allowed here`); + } + } +} + +const TRUE_FN = () => { + return true; +}; + +/** + * @param {Selector} selector + * @returns {MatcherFn} + */ +export function compileSelector(selector) { + /** @type {MatcherFn} */ + let fn = TRUE_FN; + + for (let i = 0; i < selector.length; i++) { + const node = selector[i]; + + switch (node.type) { + case ELEM_NODE: + fn = matchElem(node, fn); + break; + case RELATION_NODE: + switch (node.op) { + case BinOp.Space: + fn = matchDescendant(fn); + break; + case BinOp.Greater: + fn = matchChild(fn); + break; + case BinOp.Plus: + fn = matchAdjacent(fn); + break; + case BinOp.Tilde: + fn = matchFollowing(fn); + break; + default: + throw new Error(`Unknown relation op ${node.op}`); + } + break; + case ATTR_EXISTS_NODE: + fn = matchAttrExists(node, fn); + break; + case ATTR_BIN_NODE: + fn = matchAttrBin(node, fn); + break; + case PSEUDO_FIRST_CHILD: + fn = matchFirstChild(fn); + break; + case PSEUDO_LAST_CHILD: + fn = matchLastChild(fn); + break; + case PSEUDO_NTH_CHILD: + fn = matchNthChild(node, fn); + break; + case PSEUDO_HAS: + // FIXME + // fn = matchIs(part, fn); + throw new Error("TODO: :has"); + case PSEUDO_NOT: + fn = matchNot(node.selectors, fn); + break; + default: + // @ts-ignore error handling + // deno-lint-ignore no-console + console.log(node); + throw new Error(`Unknown selector node`); + } + } + + return fn; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchFirstChild(next) { + return (ctx, id) => { + const parent = ctx.getParent(id); + const first = ctx.getFirstChild(parent); + return first === id && next(ctx, first); + }; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchLastChild(next) { + return (ctx, id) => { + const parent = ctx.getParent(id); + const last = ctx.getLastChild(parent); + return last === id && next(ctx, id); + }; +} + +/** + * @param {PseudoNthChild} node + * @param {number} i + * @returns {number} + */ +function getNthAnB(node, i) { + const n = node.step * i; + + if (node.op === null) return n; + + switch (node.op) { + case "+": + return n + node.stepOffset; + case "-": + return n - node.stepOffset; + default: + throw new Error("Not supported nth-child operator: " + node.op); + } +} + +/** + * @param {PseudoNthChild} node + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchNthChild(node, next) { + const ofSelector = node.of !== null ? compileSelector(node.of) : TRUE_FN; + + // TODO(@marvinhagemeister): we should probably cache results here + + return (ctx, id) => { + const siblings = ctx.getSiblings(id); + const idx = siblings.indexOf(id); + + if (!node.repeat) { + return idx === node.stepOffset && next(ctx, id); + } + + for (let i = 0; i < siblings.length; i++) { + const n = getNthAnB(node, i); + + if (n > siblings.length - 1) return false; + + const search = siblings[n]; + if (id === search) { + if (node.of !== null && !ofSelector(ctx, id)) { + continue; + } else if (next(ctx, id)) { + return true; + } + } else if (n > idx) { + return false; + } + } + + return false; + }; +} + +/** + * @param {Selector[]} selectors + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchNot(selectors, next) { + /** @type {MatcherFn[]} */ + const compiled = []; + + for (let i = 0; i < selectors.length; i++) { + const sel = selectors[i]; + compiled.push(compileSelector(sel)); + } + + return (ctx, id) => { + for (let i = 0; i < compiled.length; i++) { + const fn = compiled[i]; + if (fn(ctx, id)) { + return false; + } + } + + return next(ctx, id); + }; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchDescendant(next) { + // TODO(@marvinhagemeister): we should probably cache results here + return (ctx, id) => { + let current = ctx.getParent(id); + while (current > 0) { + if (next(ctx, current)) { + return true; + } + + current = ctx.getParent(current); + } + + return false; + }; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchChild(next) { + return (ctx, id) => { + const parent = ctx.getParent(id); + if (parent < 0) return false; + + return next(ctx, parent); + }; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchAdjacent(next) { + return (ctx, id) => { + const siblings = ctx.getSiblings(id); + const idx = siblings.indexOf(id) - 1; + + if (idx < 0) return false; + + const prev = siblings[idx]; + return next(ctx, prev); + }; +} + +/** + * @param {NextFn} next + * @returns {MatcherFn} + */ +function matchFollowing(next) { + return (ctx, id) => { + const siblings = ctx.getSiblings(id); + const idx = siblings.indexOf(id) - 1; + + if (idx < 0) return false; + + for (let i = idx; i >= 0; i--) { + const sib = siblings[i]; + if (next(ctx, sib)) return true; + } + + return false; + }; +} + +/** + * @param {ElemSelector} part + * @param {MatcherFn} next + * @returns {MatcherFn} + */ +function matchElem(part, next) { + return (ctx, id) => { + // Placeholder node cannot be matched + if (id === 0) return false; + // Wildcard always matches + else if (part.wildcard) return next(ctx, id); + // 0 means it's the placeholder node which + // can never be matched. + else if (part.elem === 0) return false; + + const type = ctx.getType(id); + if (type > 0 && type === part.elem) return next(ctx, id); + + return false; + }; +} + +/** + * @param {AttrExists} attr + * @param {MatcherFn} next + * @returns {MatcherFn} + */ +function matchAttrExists(attr, next) { + return (ctx, id) => { + return ctx.hasAttrPath(id, attr.prop, 0) ? next(ctx, id) : false; + }; +} + +/** + * @param {AttrBin} attr + * @param {MatcherFn} next + * @returns {MatcherFn} + */ +function matchAttrBin(attr, next) { + return (ctx, id) => { + if (!ctx.hasAttrPath(id, attr.prop, 0)) return false; + const value = ctx.getAttrPathValue(id, attr.prop, 0); + if (!matchAttrValue(attr, value)) return false; + return next(ctx, id); + }; +} + +/** + * @param {AttrBin} attr + * @param {*} value + * @returns {boolean} + */ +function matchAttrValue(attr, value) { + switch (attr.op) { + case BinOp.Equal: + return value === attr.value; + case BinOp.NotEqual: + return value !== attr.value; + case BinOp.Greater: + return typeof value === "number" && typeof attr.value === "number" && + value > attr.value; + case BinOp.GreaterThan: + return typeof value === "number" && typeof attr.value === "number" && + value >= attr.value; + case BinOp.Less: + return typeof value === "number" && typeof attr.value === "number" && + value < attr.value; + case BinOp.LessThan: + return typeof value === "number" && typeof attr.value === "number" && + value <= attr.value; + default: + return false; + } +} diff --git a/cli/js/40_lint_types.d.ts b/cli/js/40_lint_types.d.ts index 8c252f10ad..7b06e36098 100644 --- a/cli/js/40_lint_types.d.ts +++ b/cli/js/40_lint_types.d.ts @@ -16,6 +16,7 @@ export interface AstContext { strByProp: number[]; typeByStr: Map; propByStr: Map; + matcher: MatchContext; } // TODO(@marvinhagemeister) Remove once we land "official" types @@ -43,8 +44,89 @@ export interface LintState { export type VisitorFn = (node: unknown) => void; export interface CompiledVisitor { - matcher: (offset: number) => boolean; + matcher: (ctx: MatchContext, offset: number) => boolean; info: { enter: VisitorFn; exit: VisitorFn }; } +export interface AttrExists { + type: 3; + prop: number[]; +} + +export interface AttrBin { + type: 4; + prop: number[]; + op: number; + // deno-lint-ignore no-explicit-any + value: any; +} + +export type AttrSelector = AttrExists | AttrBin; + +export interface ElemSelector { + type: 1; + wildcard: boolean; + elem: number; +} + +export interface PseudoNthChild { + type: 5; + op: string | null; + step: number; + stepOffset: number; + of: Selector | null; + repeat: boolean; +} + +export interface PseudoHas { + type: 6; + selectors: Selector[]; +} +export interface PseudoNot { + type: 7; + selectors: Selector[]; +} +export interface PseudoFirstChild { + type: 8; +} +export interface PseudoLastChild { + type: 9; +} + +export interface Relation { + type: 2; + op: number; +} + +export type Selector = Array< + | ElemSelector + | Relation + | AttrExists + | AttrBin + | PseudoNthChild + | PseudoNot + | PseudoHas + | PseudoFirstChild + | PseudoLastChild +>; + +export interface SelectorParseCtx { + root: Selector; + current: Selector; +} + +export interface MatchContext { + getFirstChild(id: number): number; + getLastChild(id: number): number; + getSiblings(id: number): number[]; + getParent(id: number): number; + getType(id: number): number; + hasAttrPath(id: number, propIds: number[], idx: number): boolean; + getAttrPathValue(id: number, propIds: number[], idx: number): unknown; +} + +export type NextFn = (ctx: MatchContext, id: number) => boolean; +export type MatcherFn = (ctx: MatchContext, id: number) => boolean; +export type TransformFn = (value: string) => number; + export {}; diff --git a/cli/tools/lint/ast_buffer/buffer.rs b/cli/tools/lint/ast_buffer/buffer.rs index c440b73ccd..f37041eff2 100644 --- a/cli/tools/lint/ast_buffer/buffer.rs +++ b/cli/tools/lint/ast_buffer/buffer.rs @@ -215,11 +215,13 @@ impl SerializeCtx { let type_str = ctx.str_table.insert("type"); let parent_str = ctx.str_table.insert("parent"); let range_str = ctx.str_table.insert("range"); + let length_str = ctx.str_table.insert("length"); // These values are expected to be in this order on the JS side ctx.prop_map[0] = type_str; ctx.prop_map[1] = parent_str; ctx.prop_map[2] = range_str; + ctx.prop_map[3] = length_str; ctx } diff --git a/cli/tools/lint/ast_buffer/ts_estree.rs b/cli/tools/lint/ast_buffer/ts_estree.rs index af5fea4b46..599499aa8d 100644 --- a/cli/tools/lint/ast_buffer/ts_estree.rs +++ b/cli/tools/lint/ast_buffer/ts_estree.rs @@ -205,6 +205,7 @@ pub enum AstProp { Type, Parent, Range, + Length, // Not used in AST, but can be used in attr selectors // Starting from here the order doesn't matter. // Following are all possible AST node properties. @@ -320,6 +321,7 @@ impl Display for AstProp { AstProp::Parent => "parent", AstProp::Range => "range", AstProp::Type => "type", + AstProp::Length => "length", AstProp::Abstract => "abstract", AstProp::Accessibility => "accessibility", AstProp::Alternate => "alternate", diff --git a/cli/worker.rs b/cli/worker.rs index 6b87b5966a..7653e72b75 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -657,6 +657,8 @@ impl CliMainWorkerFactory { "40_test.js", "40_bench.js", "40_jupyter.js", + // TODO(bartlomieju): probably shouldn't include these files here? + "40_lint_selector.js", "40_lint.js" ); } diff --git a/tests/integration/js_unit_tests.rs b/tests/integration/js_unit_tests.rs index 899329b319..afb97a3458 100644 --- a/tests/integration/js_unit_tests.rs +++ b/tests/integration/js_unit_tests.rs @@ -52,6 +52,7 @@ util::unit_test_factory!( kv_queue_test, kv_queue_undelivered_test, link_test, + lint_selectors_test, lint_plugin_test, make_temp_test, message_channel_test, diff --git a/tests/unit/lint_plugin_test.ts b/tests/unit/lint_plugin_test.ts index 649c8bde9e..9506c3e0a8 100644 --- a/tests/unit/lint_plugin_test.ts +++ b/tests/unit/lint_plugin_test.ts @@ -51,22 +51,38 @@ function testPlugin( return runLintPlugin(plugin, "source.tsx", source); } -function testVisit(source: string, ...selectors: string[]): string[] { - const log: string[] = []; +interface VisitResult { + selector: string; + kind: "enter" | "exit"; + // deno-lint-ignore no-explicit-any + node: any; +} + +function testVisit( + source: string, + ...selectors: string[] +): VisitResult[] { + const result: VisitResult[] = []; testPlugin(source, { create() { const visitor: LintVisitor = {}; for (const s of selectors) { - visitor[s] = () => log.push(s); + visitor[s] = (node) => { + result.push({ + kind: s.endsWith(":exit") ? "exit" : "enter", + selector: s, + node, + }); + }; } return visitor; }, }); - return log; + return result; } function testLintNode(source: string, ...selectors: string[]) { @@ -91,14 +107,188 @@ function testLintNode(source: string, ...selectors: string[]) { } Deno.test("Plugin - visitor enter/exit", () => { - const enter = testVisit("foo", "Identifier"); - assertEquals(enter, ["Identifier"]); + const enter = testVisit( + "foo", + "Identifier", + ); + assertEquals(enter[0].node.type, "Identifier"); - const exit = testVisit("foo", "Identifier:exit"); - assertEquals(exit, ["Identifier:exit"]); + const exit = testVisit( + "foo", + "Identifier:exit", + ); + assertEquals(exit[0].node.type, "Identifier"); const both = testVisit("foo", "Identifier", "Identifier:exit"); - assertEquals(both, ["Identifier", "Identifier:exit"]); + assertEquals(both.map((t) => t.selector), ["Identifier", "Identifier:exit"]); +}); + +Deno.test("Plugin - visitor descendant", () => { + let result = testVisit( + "if (false) foo; if (false) bar()", + "IfStatement CallExpression", + ); + assertEquals(result[0].node.type, "CallExpression"); + assertEquals(result[0].node.callee.name, "bar"); + + result = testVisit( + "if (false) foo; foo()", + "IfStatement IfStatement", + ); + assertEquals(result, []); + + result = testVisit( + "if (false) foo; foo()", + "* CallExpression", + ); + assertEquals(result[0].node.type, "CallExpression"); +}); + +Deno.test("Plugin - visitor child combinator", () => { + let result = testVisit( + "if (false) foo; if (false) { bar; }", + "IfStatement > ExpressionStatement > Identifier", + ); + assertEquals(result[0].node.name, "foo"); + + result = testVisit( + "if (false) foo; foo()", + "IfStatement IfStatement", + ); + assertEquals(result, []); +}); + +Deno.test("Plugin - visitor next sibling", () => { + const result = testVisit( + "if (false) foo; if (false) bar;", + "IfStatement + IfStatement Identifier", + ); + assertEquals(result[0].node.name, "bar"); +}); + +Deno.test("Plugin - visitor subsequent sibling", () => { + const result = testVisit( + "if (false) foo; if (false) bar; if (false) baz;", + "IfStatement ~ IfStatement Identifier", + ); + assertEquals(result.map((r) => r.node.name), ["bar", "baz"]); +}); + +Deno.test("Plugin - visitor attr", () => { + let result = testVisit( + "for (const a of b) {}", + "[await]", + ); + assertEquals(result[0].node.await, false); + + result = testVisit( + "for await (const a of b) {}", + "[await=true]", + ); + assertEquals(result[0].node.await, true); + + result = testVisit( + "for await (const a of b) {}", + "ForOfStatement[await=true]", + ); + assertEquals(result[0].node.await, true); + + result = testVisit( + "for (const a of b) {}", + "ForOfStatement[await != true]", + ); + assertEquals(result[0].node.await, false); + + result = testVisit( + "async function *foo() {}", + "FunctionDeclaration[async=true][generator=true]", + ); + assertEquals(result[0].node.type, "FunctionDeclaration"); + + result = testVisit( + "foo", + "[name='foo']", + ); + assertEquals(result[0].node.name, "foo"); +}); + +Deno.test("Plugin - visitor attr length special case", () => { + let result = testVisit( + "foo(1); foo(1, 2);", + "CallExpression[arguments.length=2]", + ); + assertEquals(result[0].node.arguments.length, 2); + + result = testVisit( + "foo(1); foo(1, 2);", + "CallExpression[arguments.length>1]", + ); + assertEquals(result[0].node.arguments.length, 2); + + result = testVisit( + "foo(1); foo(1, 2);", + "CallExpression[arguments.length<2]", + ); + assertEquals(result[0].node.arguments.length, 1); + + result = testVisit( + "foo(1); foo(1, 2);", + "CallExpression[arguments.length<=3]", + ); + assertEquals(result[0].node.arguments.length, 1); + assertEquals(result[1].node.arguments.length, 2); + + result = testVisit( + "foo(1); foo(1, 2);", + "CallExpression[arguments.length>=1]", + ); + assertEquals(result[0].node.arguments.length, 1); + assertEquals(result[1].node.arguments.length, 2); +}); + +Deno.test("Plugin - visitor :first-child", () => { + const result = testVisit( + "{ foo; bar }", + "BlockStatement ExpressionStatement:first-child Identifier", + ); + assertEquals(result[0].node.name, "foo"); +}); + +Deno.test("Plugin - visitor :last-child", () => { + const result = testVisit( + "{ foo; bar }", + "BlockStatement ExpressionStatement:last-child Identifier", + ); + assertEquals(result[0].node.name, "bar"); +}); + +Deno.test("Plugin - visitor :nth-child", () => { + let result = testVisit( + "{ foo; bar; baz; foobar; }", + "BlockStatement ExpressionStatement:nth-child(2) Identifier", + ); + assertEquals(result[0].node.name, "bar"); + + result = testVisit( + "{ foo; bar; baz; foobar; }", + "BlockStatement ExpressionStatement:nth-child(2n) Identifier", + ); + assertEquals(result[0].node.name, "foo"); + assertEquals(result[1].node.name, "baz"); + + result = testVisit( + "{ foo; bar; baz; foobar; }", + "BlockStatement ExpressionStatement:nth-child(2n + 1) Identifier", + ); + assertEquals(result[0].node.name, "bar"); + assertEquals(result[1].node.name, "foobar"); + + result = testVisit( + "{ foo; bar; baz; foobar; }", + "BlockStatement *:nth-child(2n + 1 of ExpressionStatement) Identifier", + ); + assertEquals(result[0].node.name, "bar"); + assertEquals(result[1].node.name, "foobar"); }); Deno.test("Plugin - Program", () => { diff --git a/tests/unit/lint_selectors_test.ts b/tests/unit/lint_selectors_test.ts new file mode 100644 index 0000000000..0909a4907a --- /dev/null +++ b/tests/unit/lint_selectors_test.ts @@ -0,0 +1,610 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "@std/assert/equals"; +import { + ATTR_BIN_NODE, + ATTR_EXISTS_NODE, + BinOp, + ELEM_NODE, + Lexer, + parseSelector, + PSEUDO_FIRST_CHILD, + PSEUDO_HAS, + PSEUDO_LAST_CHILD, + PSEUDO_NOT, + PSEUDO_NTH_CHILD, + RELATION_NODE, + splitSelectors, + Token, +} from "../../cli/js/40_lint_selector.js"; +import { assertThrows } from "@std/assert"; + +Deno.test("splitSelectors", () => { + assertEquals(splitSelectors("foo"), ["foo"]); + assertEquals(splitSelectors("foo, bar"), ["foo", "bar"]); + assertEquals(splitSelectors("foo:f(bar, baz)"), ["foo:f(bar, baz)"]); + assertEquals(splitSelectors("foo:f(bar, baz), foobar"), [ + "foo:f(bar, baz)", + "foobar", + ]); +}); + +interface LexState { + token: number; + value: string; +} + +function testLexer(input: string): LexState[] { + const out: LexState[] = []; + const l = new Lexer(input); + + while (l.token !== Token.EOF) { + out.push({ token: l.token, value: l.value }); + l.next(); + } + + return out; +} + +const Tags: Record = { Foo: 1, Bar: 2, FooBar: 3 }; +const Attrs: Record = { foo: 1, bar: 2, foobar: 3, attr: 4 }; +const toTag = (name: string): number => Tags[name]; +const toAttr = (name: string): number => Attrs[name]; + +const testParse = (input: string) => parseSelector(input, toTag, toAttr); + +Deno.test("Lexer - Elem", () => { + assertEquals(testLexer("Foo"), [ + { token: Token.Word, value: "Foo" }, + ]); + assertEquals(testLexer("foo-bar"), [ + { token: Token.Word, value: "foo-bar" }, + ]); + assertEquals(testLexer("foo_bar"), [ + { token: Token.Word, value: "foo_bar" }, + ]); + assertEquals(testLexer("Foo Bar Baz"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Space, value: "" }, + { token: Token.Word, value: "Bar" }, + { token: Token.Space, value: "" }, + { token: Token.Word, value: "Baz" }, + ]); + assertEquals(testLexer("Foo Bar Baz"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Space, value: "" }, + { token: Token.Word, value: "Bar" }, + { token: Token.Space, value: "" }, + { token: Token.Word, value: "Baz" }, + ]); +}); + +Deno.test("Lexer - Relation >", () => { + assertEquals(testLexer("Foo > Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: ">" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer("Foo>Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: ">" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer(">Bar"), [ + { token: Token.Op, value: ">" }, + { token: Token.Word, value: "Bar" }, + ]); +}); + +Deno.test("Lexer - Relation +", () => { + assertEquals(testLexer("Foo + Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: "+" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer("Foo+Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: "+" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer("+Bar"), [ + { token: Token.Op, value: "+" }, + { token: Token.Word, value: "Bar" }, + ]); +}); + +Deno.test("Lexer - Relation ~", () => { + assertEquals(testLexer("Foo ~ Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: "~" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer("Foo~Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Op, value: "~" }, + { token: Token.Word, value: "Bar" }, + ]); + assertEquals(testLexer("~Bar"), [ + { token: Token.Op, value: "~" }, + { token: Token.Word, value: "Bar" }, + ]); + + assertEquals(testLexer("Foo Bar ~ Bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Space, value: "" }, + { token: Token.Word, value: "Bar" }, + { token: Token.Op, value: "~" }, + { token: Token.Word, value: "Bar" }, + ]); +}); + +Deno.test("Lexer - Attr", () => { + assertEquals(testLexer("[attr]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr=1]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: "=" }, + { token: Token.Word, value: "1" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr='foo']"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: "=" }, + { token: Token.String, value: "foo" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr>=2]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: ">=" }, + { token: Token.Word, value: "2" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr<=2]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: "<=" }, + { token: Token.Word, value: "2" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr>2]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: ">" }, + { token: Token.Word, value: "2" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr<2]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: "<" }, + { token: Token.Word, value: "2" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr!=2]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Op, value: "!=" }, + { token: Token.Word, value: "2" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr.foo=1]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.Dot, value: "" }, + { token: Token.Word, value: "foo" }, + { token: Token.Op, value: "=" }, + { token: Token.Word, value: "1" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("[attr] [attr]"), [ + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.BracketClose, value: "" }, + { token: Token.Space, value: "" }, + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.BracketClose, value: "" }, + ]); + assertEquals(testLexer("Foo[attr][attr2=1]"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr" }, + { token: Token.BracketClose, value: "" }, + { token: Token.BracketOpen, value: "" }, + { token: Token.Word, value: "attr2" }, + { token: Token.Op, value: "=" }, + { token: Token.Word, value: "1" }, + { token: Token.BracketClose, value: "" }, + ]); +}); + +Deno.test("Lexer - Pseudo", () => { + assertEquals(testLexer(":foo-bar"), [ + { token: Token.Colon, value: "" }, + { token: Token.Word, value: "foo-bar" }, + ]); + assertEquals(testLexer("Foo:foo-bar"), [ + { token: Token.Word, value: "Foo" }, + { token: Token.Colon, value: "" }, + { token: Token.Word, value: "foo-bar" }, + ]); + assertEquals(testLexer(":foo-bar(baz)"), [ + { token: Token.Colon, value: "" }, + { token: Token.Word, value: "foo-bar" }, + { token: Token.BraceOpen, value: "" }, + { token: Token.Word, value: "baz" }, + { token: Token.BraceClose, value: "" }, + ]); + assertEquals(testLexer(":foo-bar(2n + 1)"), [ + { token: Token.Colon, value: "" }, + { token: Token.Word, value: "foo-bar" }, + { token: Token.BraceOpen, value: "" }, + { token: Token.Word, value: "2n" }, + { token: Token.Op, value: "+" }, + { token: Token.Word, value: "1" }, + { token: Token.BraceClose, value: "" }, + ]); +}); + +Deno.test("Parser - Elem", () => { + assertEquals(testParse("Foo"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + ]]); +}); + +Deno.test("Parser - Relation (descendant)", () => { + assertEquals(testParse("Foo Bar"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + { + type: RELATION_NODE, + op: BinOp.Space, + }, + { + type: ELEM_NODE, + elem: 2, + wildcard: false, + }, + ]]); +}); + +Deno.test("Parser - Relation", () => { + assertEquals(testParse("Foo > Bar"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + { + type: RELATION_NODE, + op: BinOp.Greater, + }, + { + type: ELEM_NODE, + elem: 2, + wildcard: false, + }, + ]]); + + assertEquals(testParse("Foo ~ Bar"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + { + type: RELATION_NODE, + op: BinOp.Tilde, + }, + { + type: ELEM_NODE, + elem: 2, + wildcard: false, + }, + ]]); + + assertEquals(testParse("Foo + Bar"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + { + type: RELATION_NODE, + op: BinOp.Plus, + }, + { + type: ELEM_NODE, + elem: 2, + wildcard: false, + }, + ]]); +}); + +Deno.test("Parser - Attr", () => { + assertEquals(testParse("[foo]"), [[ + { + type: ATTR_EXISTS_NODE, + prop: [1], + }, + ]]); + + assertEquals(testParse("[foo][bar]"), [[ + { + type: ATTR_EXISTS_NODE, + prop: [1], + }, + { + type: ATTR_EXISTS_NODE, + prop: [2], + }, + ]]); + + assertEquals(testParse("[foo=1]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: 1, + }, + ]]); + assertEquals(testParse("[foo=true]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: true, + }, + ]]); + assertEquals(testParse("[foo=false]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: false, + }, + ]]); + assertEquals(testParse("[foo=null]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: null, + }, + ]]); + assertEquals(testParse("[foo='str']"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: "str", + }, + ]]); + assertEquals(testParse('[foo="str"]'), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: "str", + }, + ]]); + assertEquals(testParse("[foo=/str/]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: /str/, + }, + ]]); + assertEquals(testParse("[foo=/str/g]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1], + value: /str/g, + }, + ]]); +}); + +Deno.test("Parser - Attr nested", () => { + assertEquals(testParse("[foo.bar]"), [[ + { + type: ATTR_EXISTS_NODE, + prop: [1, 2], + }, + ]]); + + assertEquals(testParse("[foo.bar = 2]"), [[ + { + type: ATTR_BIN_NODE, + op: BinOp.Equal, + prop: [1, 2], + value: 2, + }, + ]]); +}); + +Deno.test("Parser - Pseudo no value", () => { + assertEquals(testParse(":first-child"), [[ + { + type: PSEUDO_FIRST_CHILD, + }, + ]]); + assertEquals(testParse(":last-child"), [[ + { + type: PSEUDO_LAST_CHILD, + }, + ]]); +}); + +Deno.test("Parser - Pseudo nth-child", () => { + assertEquals(testParse(":nth-child(2)"), [[ + { + type: PSEUDO_NTH_CHILD, + of: null, + op: null, + step: 0, + stepOffset: 1, + repeat: false, + }, + ]]); + assertEquals(testParse(":nth-child(2n)"), [[ + { + type: PSEUDO_NTH_CHILD, + of: null, + op: null, + step: 2, + stepOffset: 0, + repeat: true, + }, + ]]); + assertEquals(testParse(":nth-child(-2n)"), [[ + { + type: PSEUDO_NTH_CHILD, + of: null, + op: null, + step: -2, + stepOffset: 0, + repeat: true, + }, + ]]); + assertEquals(testParse(":nth-child(2n + 1)"), [[ + { + type: PSEUDO_NTH_CHILD, + of: null, + op: "+", + step: 2, + stepOffset: 1, + repeat: true, + }, + ]]); + assertEquals(testParse(":nth-child(2n + 1 of Foo[attr])"), [[ + { + type: PSEUDO_NTH_CHILD, + of: [ + { type: ELEM_NODE, elem: 1, wildcard: false }, + { type: ATTR_EXISTS_NODE, prop: [4] }, + ], + op: "+", + step: 2, + stepOffset: 1, + repeat: true, + }, + ]]); + + // Invalid selectors + assertThrows(() => testParse(":nth-child(2n + 1 of Foo[attr], Bar)")); + assertThrows(() => testParse(":nth-child(2n - 1 foo)")); +}); + +Deno.test("Parser - Pseudo has/is/where", () => { + assertEquals(testParse(":has(Foo:has(Foo), Bar)"), [[ + { + type: PSEUDO_HAS, + selectors: [ + [ + { type: ELEM_NODE, elem: 1, wildcard: false }, + { + type: PSEUDO_HAS, + selectors: [ + [{ type: ELEM_NODE, elem: 1, wildcard: false }], + ], + }, + ], + [ + { type: ELEM_NODE, elem: 2, wildcard: false }, + ], + ], + }, + ]]); + assertEquals(testParse(":where(Foo:where(Foo), Bar)"), [[ + { + type: PSEUDO_HAS, + selectors: [ + [ + { type: ELEM_NODE, elem: 1, wildcard: false }, + { + type: PSEUDO_HAS, + selectors: [ + [{ type: ELEM_NODE, elem: 1, wildcard: false }], + ], + }, + ], + [ + { type: ELEM_NODE, elem: 2, wildcard: false }, + ], + ], + }, + ]]); + assertEquals(testParse(":is(Foo:is(Foo), Bar)"), [[ + { + type: PSEUDO_HAS, + selectors: [ + [ + { type: ELEM_NODE, elem: 1, wildcard: false }, + { + type: PSEUDO_HAS, + selectors: [ + [{ type: ELEM_NODE, elem: 1, wildcard: false }], + ], + }, + ], + [ + { type: ELEM_NODE, elem: 2, wildcard: false }, + ], + ], + }, + ]]); +}); + +Deno.test("Parser - Pseudo not", () => { + assertEquals(testParse(":not(Foo:not(Foo), Bar)"), [[ + { + type: PSEUDO_NOT, + selectors: [ + [ + { type: ELEM_NODE, elem: 1, wildcard: false }, + { + type: PSEUDO_NOT, + selectors: [ + [{ type: ELEM_NODE, elem: 1, wildcard: false }], + ], + }, + ], + [ + { type: ELEM_NODE, elem: 2, wildcard: false }, + ], + ], + }, + ]]); +}); + +Deno.test("Parser - mixed", () => { + assertEquals(testParse("Foo[foo=true] Bar"), [[ + { + type: ELEM_NODE, + elem: 1, + wildcard: false, + }, + { type: ATTR_BIN_NODE, op: BinOp.Equal, prop: [1], value: true }, + { type: RELATION_NODE, op: BinOp.Space }, + { + type: ELEM_NODE, + elem: 2, + wildcard: false, + }, + ]]); +}); diff --git a/tools/core_import_map.json b/tools/core_import_map.json index bc0674277e..d38221eb4c 100644 --- a/tools/core_import_map.json +++ b/tools/core_import_map.json @@ -250,6 +250,7 @@ "ext:deno_node/_util/std_fmt_colors.ts": "../ext/node/polyfills/_util/std_fmt_colors.ts", "ext:deno_telemetry/telemetry.ts": "../ext/deno_telemetry/telemetry.ts", "ext:deno_telemetry/util.ts": "../ext/deno_telemetry/util.ts", + "ext:cli/40_lint_selector.js": "../cli/js/40_lint_selector.js", "@std/archive": "../tests/util/std/archive/mod.ts", "@std/archive/tar": "../tests/util/std/archive/tar.ts", "@std/archive/untar": "../tests/util/std/archive/untar.ts",