From 95928c46eb83024551e4ede49ea00c4f4b21afef Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 16 Dec 2024 18:39:40 -0500 Subject: [PATCH] refactor: extract out `FileFetcher` to `deno_cache_dir` (#27263) --- Cargo.lock | 34 +- Cargo.toml | 9 +- cli/Cargo.toml | 2 + cli/args/import_map.rs | 12 +- cli/args/mod.rs | 52 +- cli/auth_tokens.rs | 369 ----- cli/cache/mod.rs | 93 +- cli/factory.rs | 13 +- cli/file_fetcher.rs | 1387 ++++++++--------- cli/graph_util.rs | 6 +- cli/http_util.rs | 895 ++--------- cli/jsr.rs | 6 +- cli/lsp/config.rs | 8 +- cli/lsp/jsr.rs | 13 +- cli/lsp/language_server.rs | 12 +- cli/lsp/npm.rs | 13 +- cli/lsp/registries.rs | 61 +- cli/lsp/resolver.rs | 2 +- cli/lsp/tsc.rs | 5 +- cli/main.rs | 1 - cli/mainrt.rs | 1 - cli/npm/managed/mod.rs | 5 +- cli/npm/mod.rs | 12 +- cli/standalone/binary.rs | 6 +- cli/standalone/mod.rs | 2 +- cli/tools/check.rs | 2 +- cli/tools/coverage/mod.rs | 28 +- cli/tools/installer.rs | 12 +- cli/tools/registry/pm.rs | 12 +- cli/tools/registry/pm/outdated.rs | 12 +- cli/tools/repl/mod.rs | 7 +- cli/tools/run/mod.rs | 6 +- cli/tools/test/mod.rs | 10 +- cli/util/extract.rs | 19 +- resolvers/npm_cache/lib.rs | 22 + tests/integration/bench_tests.rs | 2 +- tests/integration/cache_tests.rs | 2 +- tests/integration/run_tests.rs | 2 +- tests/integration/test_tests.rs | 2 +- .../localhost_unsafe_ssl.ts.out | 7 +- .../run/jsx_import_source/__test__.jsonc | 1 + tests/util/server/src/servers/mod.rs | 5 - 42 files changed, 1018 insertions(+), 2152 deletions(-) delete mode 100644 cli/auth_tokens.rs diff --git a/Cargo.lock b/Cargo.lock index 0a8c28ceb5..850ab038f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,9 +596,9 @@ dependencies = [ [[package]] name = "boxed_error" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69aae56aaf59d1994b902ed5c0c79024012bdc2426741def75a635999a030e7e" +checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", "syn 2.0.87", @@ -1193,9 +1193,9 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "data-url" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f" +checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "debug-ignore" @@ -1221,6 +1221,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "bincode", + "boxed_error", "bytes", "cache_control", "chrono", @@ -1237,6 +1238,7 @@ dependencies = [ "deno_config", "deno_core", "deno_doc", + "deno_error", "deno_graph", "deno_lint", "deno_lockfile", @@ -1421,13 +1423,21 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cca43605c8cbce6c6787e0daf227864487c07c2b31d438c0bf43d1b38da94b7f" +checksum = "54df1c5177ace01d92b872584ab9af8290681bb150fd9b423c37a494ad5ddbdc" dependencies = [ + "async-trait", "base32", + "base64 0.21.7", + "boxed_error", + "cache_control", + "chrono", + "data-url", + "deno_error", "deno_media_type", "deno_path_util", + "http 1.1.0", "indexmap 2.3.0", "log", "once_cell", @@ -1612,6 +1622,7 @@ dependencies = [ "libc", "serde", "serde_json", + "url", ] [[package]] @@ -1857,9 +1868,9 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf552fbdedbe81c89705349d7d2485c7051382b000dfddbdbf7fc25931cf83" +checksum = "eaa135b8a9febc9a51c16258e294e268a1276750780d69e46edb31cced2826e4" dependencies = [ "data-url", "serde", @@ -2085,12 +2096,13 @@ dependencies = [ [[package]] name = "deno_path_util" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff25f6e08e7a0214bbacdd6f7195c7f1ebcd850c87a624e4ff06326b68b42d99" +checksum = "b02c7d341e1b2cf089daff0f4fb2b4be8f3b5511b1d96040b3f7ed63a66c737b" dependencies = [ + "deno_error", "percent-encoding", - "thiserror 1.0.64", + "thiserror 2.0.3", "url", ] diff --git a/Cargo.toml b/Cargo.toml index 984cb187ef..0ab9c93376 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ deno_config = { version = "=0.39.3", features = ["workspace", "sync"] } deno_lockfile = "=0.23.2" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.26.0" -deno_path_util = "=0.2.1" +deno_path_util = "=0.2.2" deno_permissions = { version = "0.42.0", path = "./runtime/permissions" } deno_runtime = { version = "0.191.0", path = "./runtime" } deno_semver = "=0.6.1" @@ -104,7 +104,7 @@ async-trait = "0.1.73" base32 = "=0.5.1" base64 = "0.21.7" bencher = "0.1" -boxed_error = "0.2.2" +boxed_error = "0.2.3" brotli = "6.0.0" bytes = "1.4.0" cache_control = "=0.2.0" @@ -117,8 +117,9 @@ color-print = "0.3.5" console_static_text = "=0.8.1" dashmap = "5.5.3" data-encoding = "2.3.3" -data-url = "=0.3.0" -deno_cache_dir = "=0.14.0" +data-url = "=0.3.1" +deno_cache_dir = "=0.15.0" +deno_error = "=0.5.2" deno_package_json = { version = "0.2.1", default-features = false } deno_unsync = "0.4.2" dlopen2 = "0.6.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 253ea80f19..84464bb01f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -73,6 +73,7 @@ deno_cache_dir.workspace = true deno_config.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] } +deno_error.workspace = true deno_graph = { version = "=0.86.3" } deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true @@ -93,6 +94,7 @@ anstream = "0.6.14" async-trait.workspace = true base64.workspace = true bincode = "=1.3.3" +boxed_error.workspace = true bytes.workspace = true cache_control.workspace = true chrono = { workspace = true, features = ["now"] } diff --git a/cli/args/import_map.rs b/cli/args/import_map.rs index ff2f158715..d6434ed46a 100644 --- a/cli/args/import_map.rs +++ b/cli/args/import_map.rs @@ -4,21 +4,21 @@ use deno_core::error::AnyError; use deno_core::serde_json; use deno_core::url::Url; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; +use crate::file_fetcher::TextDecodedFile; pub async fn resolve_import_map_value_from_specifier( specifier: &Url, - file_fetcher: &FileFetcher, + file_fetcher: &CliFileFetcher, ) -> Result { if specifier.scheme() == "data" { let data_url_text = deno_graph::source::RawDataUrl::parse(specifier)?.decode()?; Ok(serde_json::from_str(&data_url_text)?) } else { - let file = file_fetcher - .fetch_bypass_permissions(specifier) - .await? - .into_text_decoded()?; + let file = TextDecodedFile::decode( + file_fetcher.fetch_bypass_permissions(specifier).await?, + )?; Ok(serde_json::from_str(&file.source)?) } } diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 450aa11652..ddf990fcab 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -9,6 +9,7 @@ mod package_json; use deno_ast::MediaType; use deno_ast::SourceMapOption; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_config::deno_json::NodeModulesDirMode; use deno_config::workspace::CreateResolverOptions; use deno_config::workspace::FolderConfigs; @@ -27,7 +28,6 @@ use deno_npm::npm_rc::NpmRc; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; -use deno_npm_cache::NpmCacheSetting; use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; use deno_telemetry::OtelConfig; @@ -85,7 +85,7 @@ use thiserror::Error; use crate::cache; use crate::cache::DenoDirProvider; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::version; @@ -217,52 +217,6 @@ pub fn ts_config_to_transpile_and_emit_options( )) } -/// Indicates how cached source files should be handled. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum CacheSetting { - /// Only the cached files should be used. Any files not in the cache will - /// error. This is the equivalent of `--cached-only` in the CLI. - Only, - /// No cached source files should be used, and all files should be reloaded. - /// This is the equivalent of `--reload` in the CLI. - ReloadAll, - /// Only some cached resources should be used. This is the equivalent of - /// `--reload=jsr:@std/http/file-server` or - /// `--reload=jsr:@std/http/file-server,jsr:@std/assert/assert-equals`. - ReloadSome(Vec), - /// The usability of a cached value is determined by analyzing the cached - /// headers and other metadata associated with a cached response, reloading - /// any cached "non-fresh" cached responses. - RespectHeaders, - /// The cached source files should be used for local modules. This is the - /// default behavior of the CLI. - Use, -} - -impl CacheSetting { - pub fn as_npm_cache_setting(&self) -> NpmCacheSetting { - match self { - CacheSetting::Only => NpmCacheSetting::Only, - CacheSetting::ReloadAll => NpmCacheSetting::ReloadAll, - CacheSetting::ReloadSome(values) => { - if values.iter().any(|v| v == "npm:") { - NpmCacheSetting::ReloadAll - } else { - NpmCacheSetting::ReloadSome { - npm_package_names: values - .iter() - .filter_map(|v| v.strip_prefix("npm:")) - .map(|n| n.to_string()) - .collect(), - } - } - } - CacheSetting::RespectHeaders => unreachable!(), // not supported - CacheSetting::Use => NpmCacheSetting::Use, - } - } -} - pub struct WorkspaceBenchOptions { pub filter: Option, pub json: bool, @@ -1091,7 +1045,7 @@ impl CliOptions { pub async fn create_workspace_resolver( &self, - file_fetcher: &FileFetcher, + file_fetcher: &CliFileFetcher, pkg_json_dep_resolution: PackageJsonDepResolution, ) -> Result { let overrode_no_import_map: bool = self diff --git a/cli/auth_tokens.rs b/cli/auth_tokens.rs deleted file mode 100644 index ef9f9d0746..0000000000 --- a/cli/auth_tokens.rs +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use base64::prelude::BASE64_STANDARD; -use base64::Engine; -use deno_core::ModuleSpecifier; -use log::debug; -use log::error; -use std::borrow::Cow; -use std::fmt; -use std::net::IpAddr; -use std::net::Ipv4Addr; -use std::net::Ipv6Addr; -use std::net::SocketAddr; -use std::str::FromStr; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AuthTokenData { - Bearer(String), - Basic { username: String, password: String }, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AuthToken { - host: AuthDomain, - token: AuthTokenData, -} - -impl fmt::Display for AuthToken { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match &self.token { - AuthTokenData::Bearer(token) => write!(f, "Bearer {token}"), - AuthTokenData::Basic { username, password } => { - let credentials = format!("{username}:{password}"); - write!(f, "Basic {}", BASE64_STANDARD.encode(credentials)) - } - } - } -} - -/// A structure which contains bearer tokens that can be used when sending -/// requests to websites, intended to authorize access to private resources -/// such as remote modules. -#[derive(Debug, Clone)] -pub struct AuthTokens(Vec); - -/// An authorization domain, either an exact or suffix match. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AuthDomain { - Ip(IpAddr), - IpPort(SocketAddr), - /// Suffix match, no dot. May include a port. - Suffix(Cow<'static, str>), -} - -impl From for AuthDomain { - fn from(value: T) -> Self { - let s = value.to_string().to_lowercase(); - if let Ok(ip) = SocketAddr::from_str(&s) { - return AuthDomain::IpPort(ip); - }; - if s.starts_with('[') && s.ends_with(']') { - if let Ok(ip) = Ipv6Addr::from_str(&s[1..s.len() - 1]) { - return AuthDomain::Ip(ip.into()); - } - } else if let Ok(ip) = Ipv4Addr::from_str(&s) { - return AuthDomain::Ip(ip.into()); - } - if let Some(s) = s.strip_prefix('.') { - AuthDomain::Suffix(Cow::Owned(s.to_owned())) - } else { - AuthDomain::Suffix(Cow::Owned(s)) - } - } -} - -impl AuthDomain { - pub fn matches(&self, specifier: &ModuleSpecifier) -> bool { - let Some(host) = specifier.host_str() else { - return false; - }; - match *self { - Self::Ip(ip) => { - let AuthDomain::Ip(parsed) = AuthDomain::from(host) else { - return false; - }; - ip == parsed && specifier.port().is_none() - } - Self::IpPort(ip) => { - let AuthDomain::Ip(parsed) = AuthDomain::from(host) else { - return false; - }; - ip.ip() == parsed && specifier.port() == Some(ip.port()) - } - Self::Suffix(ref suffix) => { - let hostname = if let Some(port) = specifier.port() { - Cow::Owned(format!("{}:{}", host, port)) - } else { - Cow::Borrowed(host) - }; - - if suffix.len() == hostname.len() { - return suffix == &hostname; - } - - // If it's a suffix match, ensure a dot - if hostname.ends_with(suffix.as_ref()) - && hostname.ends_with(&format!(".{suffix}")) - { - return true; - } - - false - } - } - } -} - -impl AuthTokens { - /// Create a new set of tokens based on the provided string. It is intended - /// that the string be the value of an environment variable and the string is - /// parsed for token values. The string is expected to be a semi-colon - /// separated string, where each value is `{token}@{hostname}`. - pub fn new(maybe_tokens_str: Option) -> Self { - let mut tokens = Vec::new(); - if let Some(tokens_str) = maybe_tokens_str { - for token_str in tokens_str.trim().split(';') { - if token_str.contains('@') { - let mut iter = token_str.rsplitn(2, '@'); - let host = AuthDomain::from(iter.next().unwrap()); - let token = iter.next().unwrap(); - if token.contains(':') { - let mut iter = token.rsplitn(2, ':'); - let password = iter.next().unwrap().to_owned(); - let username = iter.next().unwrap().to_owned(); - tokens.push(AuthToken { - host, - token: AuthTokenData::Basic { username, password }, - }); - } else { - tokens.push(AuthToken { - host, - token: AuthTokenData::Bearer(token.to_string()), - }); - } - } else { - error!("Badly formed auth token discarded."); - } - } - debug!("Parsed {} auth token(s).", tokens.len()); - } - - Self(tokens) - } - - /// Attempt to match the provided specifier to the tokens in the set. The - /// matching occurs from the right of the hostname plus port, irrespective of - /// scheme. For example `https://www.deno.land:8080/` would match a token - /// with a host value of `deno.land:8080` but not match `www.deno.land`. The - /// matching is case insensitive. - pub fn get(&self, specifier: &ModuleSpecifier) -> Option { - self.0.iter().find_map(|t| { - if t.host.matches(specifier) { - Some(t.clone()) - } else { - None - } - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use deno_core::resolve_url; - - #[test] - fn test_auth_token() { - let auth_tokens = AuthTokens::new(Some("abc123@deno.land".to_string())); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123" - ); - let fixture = resolve_url("https://www.deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123".to_string() - ); - let fixture = resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - let fixture = - resolve_url("https://deno.land.example.com/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - let fixture = resolve_url("https://deno.land:8080/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - } - - #[test] - fn test_auth_tokens_multiple() { - let auth_tokens = - AuthTokens::new(Some("abc123@deno.land;def456@example.com".to_string())); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123".to_string() - ); - let fixture = resolve_url("http://example.com/a/file.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer def456".to_string() - ); - } - - #[test] - fn test_auth_tokens_space() { - let auth_tokens = AuthTokens::new(Some( - " abc123@deno.land;def456@example.com\t".to_string(), - )); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123".to_string() - ); - let fixture = resolve_url("http://example.com/a/file.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer def456".to_string() - ); - } - - #[test] - fn test_auth_tokens_newline() { - let auth_tokens = AuthTokens::new(Some( - "\nabc123@deno.land;def456@example.com\n".to_string(), - )); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123".to_string() - ); - let fixture = resolve_url("http://example.com/a/file.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer def456".to_string() - ); - } - - #[test] - fn test_auth_tokens_port() { - let auth_tokens = - AuthTokens::new(Some("abc123@deno.land:8080".to_string())); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - let fixture = resolve_url("http://deno.land:8080/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc123".to_string() - ); - } - - #[test] - fn test_auth_tokens_contain_at() { - let auth_tokens = AuthTokens::new(Some("abc@123@deno.land".to_string())); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Bearer abc@123".to_string() - ); - } - - #[test] - fn test_auth_token_basic() { - let auth_tokens = AuthTokens::new(Some("abc:123@deno.land".to_string())); - let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Basic YWJjOjEyMw==" - ); - let fixture = resolve_url("https://www.deno.land/x/mod.ts").unwrap(); - assert_eq!( - auth_tokens.get(&fixture).unwrap().to_string(), - "Basic YWJjOjEyMw==".to_string() - ); - let fixture = resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - let fixture = - resolve_url("https://deno.land.example.com/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - let fixture = resolve_url("https://deno.land:8080/x/mod.ts").unwrap(); - assert_eq!(auth_tokens.get(&fixture), None); - } - - #[test] - fn test_parse_ip() { - let ip = AuthDomain::from("[2001:db8:a::123]"); - assert_eq!("Ip(2001:db8:a::123)", format!("{ip:?}")); - let ip = AuthDomain::from("[2001:db8:a::123]:8080"); - assert_eq!("IpPort([2001:db8:a::123]:8080)", format!("{ip:?}")); - let ip = AuthDomain::from("1.1.1.1"); - assert_eq!("Ip(1.1.1.1)", format!("{ip:?}")); - } - - #[test] - fn test_case_insensitive() { - let domain = AuthDomain::from("EXAMPLE.com"); - assert!( - domain.matches(&ModuleSpecifier::parse("http://example.com").unwrap()) - ); - assert!( - domain.matches(&ModuleSpecifier::parse("http://example.COM").unwrap()) - ); - } - - #[test] - fn test_matches() { - let candidates = [ - "example.com", - "www.example.com", - "1.1.1.1", - "[2001:db8:a::123]", - // These will never match - "example.com.evil.com", - "1.1.1.1.evil.com", - "notexample.com", - "www.notexample.com", - ]; - let domains = [ - ("example.com", vec!["example.com", "www.example.com"]), - (".example.com", vec!["example.com", "www.example.com"]), - ("www.example.com", vec!["www.example.com"]), - ("1.1.1.1", vec!["1.1.1.1"]), - ("[2001:db8:a::123]", vec!["[2001:db8:a::123]"]), - ]; - let url = |c: &str| ModuleSpecifier::parse(&format!("http://{c}")).unwrap(); - let url_port = - |c: &str| ModuleSpecifier::parse(&format!("http://{c}:8080")).unwrap(); - - // Generate each candidate with and without a port - let candidates = candidates - .into_iter() - .flat_map(|c| [url(c), url_port(c)]) - .collect::>(); - - for (domain, expected_domain) in domains { - // Test without a port -- all candidates return without a port - let auth_domain = AuthDomain::from(domain); - let actual = candidates - .iter() - .filter(|c| auth_domain.matches(c)) - .cloned() - .collect::>(); - let expected = expected_domain.iter().map(|u| url(u)).collect::>(); - assert_eq!(actual, expected); - - // Test with a port, all candidates return with a port - let auth_domain = AuthDomain::from(&format!("{domain}:8080")); - let actual = candidates - .iter() - .filter(|c| auth_domain.matches(c)) - .cloned() - .collect::>(); - let expected = expected_domain - .iter() - .map(|u| url_port(u)) - .collect::>(); - assert_eq!(actual, expected); - } - } -} diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index e3e242e975..31968be0c2 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -1,18 +1,19 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::args::jsr_url; -use crate::args::CacheSetting; -use crate::errors::get_error_class_name; +use crate::file_fetcher::CliFetchNoFollowErrorKind; +use crate::file_fetcher::CliFileFetcher; use crate::file_fetcher::FetchNoFollowOptions; -use crate::file_fetcher::FetchOptions; use crate::file_fetcher::FetchPermissionsOptionRef; -use crate::file_fetcher::FileFetcher; -use crate::file_fetcher::FileOrRedirect; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::atomic_write_file_with_retries_and_fs; use crate::util::fs::AtomicWriteFileFsAdapter; use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::CacheSetting; +use deno_cache_dir::file_fetcher::FetchNoFollowErrorKind; +use deno_cache_dir::file_fetcher::FileOrRedirect; +use deno_core::error::AnyError; use deno_core::futures; use deno_core::futures::FutureExt; use deno_core::ModuleSpecifier; @@ -190,7 +191,7 @@ pub struct FetchCacherOptions { /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { pub file_header_overrides: HashMap>, - file_fetcher: Arc, + file_fetcher: Arc, fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, @@ -202,7 +203,7 @@ pub struct FetchCacher { impl FetchCacher { pub fn new( - file_fetcher: Arc, + file_fetcher: Arc, fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, @@ -320,18 +321,18 @@ impl Loader for FetchCacher { LoaderCacheSetting::Only => Some(CacheSetting::Only), }; file_fetcher - .fetch_no_follow_with_options(FetchNoFollowOptions { - fetch_options: FetchOptions { - specifier: &specifier, - permissions: if is_statically_analyzable { - FetchPermissionsOptionRef::StaticContainer(&permissions) - } else { - FetchPermissionsOptionRef::DynamicContainer(&permissions) - }, - maybe_auth: None, - maybe_accept: None, - maybe_cache_setting: maybe_cache_setting.as_ref(), - }, + .fetch_no_follow( + &specifier, + FetchPermissionsOptionRef::Restricted(&permissions, + if is_statically_analyzable { + deno_runtime::deno_permissions::CheckSpecifierKind::Static + } else { + deno_runtime::deno_permissions::CheckSpecifierKind::Dynamic + }), + FetchNoFollowOptions { + maybe_auth: None, + maybe_accept: None, + maybe_cache_setting: maybe_cache_setting.as_ref(), maybe_checksum: options.maybe_checksum.as_ref(), }) .await @@ -348,7 +349,7 @@ impl Loader for FetchCacher { (None, None) => None, }; Ok(Some(LoadResponse::Module { - specifier: file.specifier, + specifier: file.url, maybe_headers, content: file.source, })) @@ -361,18 +362,46 @@ impl Loader for FetchCacher { } }) .unwrap_or_else(|err| { - if let Some(io_err) = err.downcast_ref::() { - if io_err.kind() == std::io::ErrorKind::NotFound { - return Ok(None); - } else { - return Err(err); - } - } - let error_class_name = get_error_class_name(&err); - match error_class_name { - "NotFound" => Ok(None), - "NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None), - _ => Err(err), + let err = err.into_kind(); + match err { + CliFetchNoFollowErrorKind::FetchNoFollow(err) => { + let err = err.into_kind(); + match err { + FetchNoFollowErrorKind::NotFound(_) => Ok(None), + FetchNoFollowErrorKind::UrlToFilePath { .. } | + FetchNoFollowErrorKind::ReadingBlobUrl { .. } | + FetchNoFollowErrorKind::ReadingFile { .. } | + FetchNoFollowErrorKind::FetchingRemote { .. } | + FetchNoFollowErrorKind::ClientError { .. } | + FetchNoFollowErrorKind::NoRemote { .. } | + FetchNoFollowErrorKind::DataUrlDecode { .. } | + FetchNoFollowErrorKind::RedirectResolution { .. } | + FetchNoFollowErrorKind::CacheRead { .. } | + FetchNoFollowErrorKind::CacheSave { .. } | + FetchNoFollowErrorKind::UnsupportedScheme { .. } | + FetchNoFollowErrorKind::RedirectHeaderParse { .. } | + FetchNoFollowErrorKind::InvalidHeader { .. } => Err(AnyError::from(err)), + FetchNoFollowErrorKind::NotCached { .. } => { + if options.cache_setting == LoaderCacheSetting::Only { + Ok(None) + } else { + Err(AnyError::from(err)) + } + }, + FetchNoFollowErrorKind::ChecksumIntegrity(err) => { + // convert to the equivalent deno_graph error so that it + // enhances it if this is passed to deno_graph + Err( + deno_graph::source::ChecksumIntegrityError { + actual: err.actual, + expected: err.expected, + } + .into(), + ) + } + } + }, + CliFetchNoFollowErrorKind::PermissionCheck(permission_check_error) => Err(AnyError::from(permission_check_error)), } }) } diff --git a/cli/factory.rs b/cli/factory.rs index f08bf7e4b1..d6940d6df1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -22,7 +22,7 @@ use crate::cache::ModuleInfoCache; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::graph_container::MainModuleGraphContainer; use crate::graph_util::FileWatcherReporter; use crate::graph_util::ModuleGraphBuilder; @@ -185,7 +185,7 @@ struct CliFactoryServices { emit_cache: Deferred>, emitter: Deferred>, feature_checker: Deferred>, - file_fetcher: Deferred>, + file_fetcher: Deferred>, fs: Deferred>, global_http_cache: Deferred>, http_cache: Deferred>, @@ -350,16 +350,17 @@ impl CliFactory { }) } - pub fn file_fetcher(&self) -> Result<&Arc, AnyError> { + pub fn file_fetcher(&self) -> Result<&Arc, AnyError> { self.services.file_fetcher.get_or_try_init(|| { let cli_options = self.cli_options()?; - Ok(Arc::new(FileFetcher::new( + Ok(Arc::new(CliFileFetcher::new( self.http_cache()?.clone(), - cli_options.cache_setting(), - !cli_options.no_remote(), self.http_client_provider().clone(), self.blob_store().clone(), Some(self.text_only_progress_bar().clone()), + !cli_options.no_remote(), + cli_options.cache_setting(), + log::Level::Info, ))) }) } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 29f9c6ba3f..1b286c76b7 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -1,41 +1,45 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::args::CacheSetting; -use crate::auth_tokens::AuthTokens; use crate::cache::HttpCache; +use crate::cache::RealDenoCacheEnv; use crate::colors; -use crate::http_util::CacheSemantics; -use crate::http_util::FetchOnceArgs; -use crate::http_util::FetchOnceResult; +use crate::http_util::get_response_body_with_progress; use crate::http_util::HttpClientProvider; use crate::util::progress_bar::ProgressBar; +use boxed_error::Boxed; use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::AuthTokens; +use deno_cache_dir::file_fetcher::BlobData; +use deno_cache_dir::file_fetcher::CacheSetting; +use deno_cache_dir::file_fetcher::FetchNoFollowError; +use deno_cache_dir::file_fetcher::File; +use deno_cache_dir::file_fetcher::FileFetcherOptions; +use deno_cache_dir::file_fetcher::FileOrRedirect; +use deno_cache_dir::file_fetcher::SendError; +use deno_cache_dir::file_fetcher::SendResponse; +use deno_cache_dir::file_fetcher::TooManyRedirectsError; +use deno_cache_dir::file_fetcher::UnsupportedSchemeError; use deno_core::anyhow::Context; -use deno_core::error::custom_error; -use deno_core::error::generic_error; -use deno_core::error::uri_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_error::JsError; use deno_graph::source::LoaderChecksum; -use deno_path_util::url_to_file_path; +use deno_runtime::deno_permissions::CheckSpecifierKind; +use deno_runtime::deno_permissions::PermissionCheckError; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_web::BlobStore; use http::header; -use log::debug; +use http::HeaderMap; +use http::StatusCode; use std::borrow::Cow; use std::collections::HashMap; use std::env; -use std::fs; -use std::path::PathBuf; use std::sync::Arc; -use std::time::SystemTime; - -pub const SUPPORTED_SCHEMES: [&str; 5] = - ["data", "blob", "file", "http", "https"]; +use thiserror::Error; #[derive(Debug, Clone, Eq, PartialEq)] pub struct TextDecodedFile { @@ -47,62 +51,19 @@ pub struct TextDecodedFile { pub source: Arc, } -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum FileOrRedirect { - File(File), - Redirect(ModuleSpecifier), -} - -impl FileOrRedirect { - fn from_deno_cache_entry( - specifier: &ModuleSpecifier, - cache_entry: deno_cache_dir::CacheEntry, - ) -> Result { - if let Some(redirect_to) = cache_entry.metadata.headers.get("location") { - let redirect = specifier.join(redirect_to)?; - Ok(FileOrRedirect::Redirect(redirect)) - } else { - Ok(FileOrRedirect::File(File { - specifier: specifier.clone(), - maybe_headers: Some(cache_entry.metadata.headers), - source: Arc::from(cache_entry.content), - })) - } - } -} - -/// A structure representing a source file. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct File { - /// The _final_ specifier for the file. The requested specifier and the final - /// specifier maybe different for remote files that have been redirected. - pub specifier: ModuleSpecifier, - pub maybe_headers: Option>, - /// The source of the file. - pub source: Arc<[u8]>, -} - -impl File { - pub fn resolve_media_type_and_charset(&self) -> (MediaType, Option<&str>) { - deno_graph::source::resolve_media_type_and_charset_from_headers( - &self.specifier, - self.maybe_headers.as_ref(), - ) - } - +impl TextDecodedFile { /// Decodes the source bytes into a string handling any encoding rules /// for local vs remote files and dealing with the charset. - pub fn into_text_decoded(self) -> Result { - // lots of borrow checker fighting here + pub fn decode(file: File) -> Result { let (media_type, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( - &self.specifier, - self.maybe_headers.as_ref(), + &file.url, + file.maybe_headers.as_ref(), ); - let specifier = self.specifier; + let specifier = file.url; match deno_graph::source::decode_source( &specifier, - self.source, + file.source, maybe_charset, ) { Ok(source) => Ok(TextDecodedFile { @@ -117,14 +78,146 @@ impl File { } } -#[derive(Debug, Clone, Default)] -struct MemoryFiles(Arc>>); +#[derive(Debug)] +struct BlobStoreAdapter(Arc); + +#[async_trait::async_trait(?Send)] +impl deno_cache_dir::file_fetcher::BlobStore for BlobStoreAdapter { + async fn get(&self, specifier: &Url) -> std::io::Result> { + let Some(blob) = self.0.get_object_url(specifier.clone()) else { + return Ok(None); + }; + Ok(Some(BlobData { + media_type: blob.media_type.clone(), + bytes: blob.read_all().await, + })) + } +} + +#[derive(Debug)] +struct HttpClientAdapter { + http_client_provider: Arc, + download_log_level: log::Level, + progress_bar: Option, +} + +#[async_trait::async_trait(?Send)] +impl deno_cache_dir::file_fetcher::HttpClient for HttpClientAdapter { + async fn send_no_follow( + &self, + url: &Url, + headers: HeaderMap, + ) -> Result { + async fn handle_request_or_server_error( + retried: &mut bool, + specifier: &Url, + err_str: String, + ) -> Result<(), ()> { + // Retry once, and bail otherwise. + if !*retried { + *retried = true; + log::debug!("Import '{}' failed: {}. Retrying...", specifier, err_str); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + Ok(()) + } else { + Err(()) + } + } + + let mut maybe_progress_guard = None; + if let Some(pb) = self.progress_bar.as_ref() { + maybe_progress_guard = Some(pb.update(url.as_str())); + } else { + log::log!( + self.download_log_level, + "{} {}", + colors::green("Download"), + url + ); + } + + let mut retried = false; // retry intermittent failures + loop { + let response = match self + .http_client_provider + .get_or_create() + .map_err(|err| SendError::Failed(err.into()))? + .send(url, headers.clone()) + .await + { + Ok(response) => response, + Err(crate::http_util::SendError::Send(err)) => { + if err.is_connect_error() { + handle_request_or_server_error(&mut retried, url, err.to_string()) + .await + .map_err(|()| SendError::Failed(err.into()))?; + continue; + } else { + return Err(SendError::Failed(err.into())); + } + } + Err(crate::http_util::SendError::InvalidUri(err)) => { + return Err(SendError::Failed(err.into())); + } + }; + if response.status() == StatusCode::NOT_MODIFIED { + return Ok(SendResponse::NotModified); + } + + if let Some(warning) = response.headers().get("X-Deno-Warning") { + log::warn!( + "{} {}", + crate::colors::yellow("Warning"), + warning.to_str().unwrap() + ); + } + + if response.status().is_redirection() { + return Ok(SendResponse::Redirect(response.into_parts().0.headers)); + } + + if response.status().is_server_error() { + handle_request_or_server_error( + &mut retried, + url, + response.status().to_string(), + ) + .await + .map_err(|()| SendError::StatusCode(response.status()))?; + } else if response.status().is_client_error() { + let err = if response.status() == StatusCode::NOT_FOUND { + SendError::NotFound + } else { + SendError::StatusCode(response.status()) + }; + return Err(err); + } else { + let body_result = get_response_body_with_progress( + response, + maybe_progress_guard.as_ref(), + ) + .await; + + match body_result { + Ok((headers, body)) => { + return Ok(SendResponse::Success(headers, body)); + } + Err(err) => { + handle_request_or_server_error(&mut retried, url, err.to_string()) + .await + .map_err(|()| SendError::Failed(err.into()))?; + continue; + } + } + } + } + } +} + +#[derive(Debug, Default)] +struct MemoryFiles(Mutex>); impl MemoryFiles { - pub fn get(&self, specifier: &ModuleSpecifier) -> Option { - self.0.lock().get(specifier).cloned() - } - pub fn insert(&self, specifier: ModuleSpecifier, file: File) -> Option { self.0.lock().insert(specifier, file) } @@ -134,416 +227,93 @@ impl MemoryFiles { } } -/// Fetch a source file from the local file system. -fn fetch_local(specifier: &ModuleSpecifier) -> Result { - let local = url_to_file_path(specifier).map_err(|_| { - uri_error(format!("Invalid file path.\n Specifier: {specifier}")) - })?; - // If it doesnt have a extension, we want to treat it as typescript by default - let headers = if local.extension().is_none() { - Some(HashMap::from([( - "content-type".to_string(), - "application/typescript".to_string(), - )])) - } else { - None - }; - let bytes = fs::read(local)?; - - Ok(File { - specifier: specifier.clone(), - maybe_headers: headers, - source: bytes.into(), - }) +impl deno_cache_dir::file_fetcher::MemoryFiles for MemoryFiles { + fn get(&self, specifier: &ModuleSpecifier) -> Option { + self.0.lock().get(specifier).cloned() + } } -/// Return a validated scheme for a given module specifier. -fn get_validated_scheme( - specifier: &ModuleSpecifier, -) -> Result { - let scheme = specifier.scheme(); - if !SUPPORTED_SCHEMES.contains(&scheme) { - // NOTE(bartlomieju): this message list additional `npm` and `jsr` schemes, but they should actually be handled - // before `file_fetcher.rs` APIs are even hit. - let mut all_supported_schemes = SUPPORTED_SCHEMES.to_vec(); - all_supported_schemes.extend_from_slice(&["npm", "jsr"]); - all_supported_schemes.sort(); - let scheme_list = all_supported_schemes - .iter() - .map(|scheme| format!(" - \"{}\"", scheme)) - .collect::>() - .join("\n"); - Err(generic_error(format!( - "Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes:\n{}", - scheme_list - ))) - } else { - Ok(scheme.to_string()) - } +#[derive(Debug, Boxed, JsError)] +pub struct CliFetchNoFollowError(pub Box); + +#[derive(Debug, Error, JsError)] +pub enum CliFetchNoFollowErrorKind { + #[error(transparent)] + #[class(inherit)] + FetchNoFollow(#[from] FetchNoFollowError), + #[error(transparent)] + #[class(generic)] + PermissionCheck(#[from] PermissionCheckError), } #[derive(Debug, Copy, Clone)] pub enum FetchPermissionsOptionRef<'a> { AllowAll, - DynamicContainer(&'a PermissionsContainer), - StaticContainer(&'a PermissionsContainer), + Restricted(&'a PermissionsContainer, CheckSpecifierKind), } +#[derive(Debug, Default)] pub struct FetchOptions<'a> { - pub specifier: &'a ModuleSpecifier, - pub permissions: FetchPermissionsOptionRef<'a>, pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, } pub struct FetchNoFollowOptions<'a> { - pub fetch_options: FetchOptions<'a>, - /// This setting doesn't make sense to provide for `FetchOptions` - /// since the required checksum may change for a redirect. + pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, + pub maybe_accept: Option<&'a str>, + pub maybe_cache_setting: Option<&'a CacheSetting>, pub maybe_checksum: Option<&'a LoaderChecksum>, } +type DenoCacheDirFileFetcher = deno_cache_dir::file_fetcher::FileFetcher< + BlobStoreAdapter, + RealDenoCacheEnv, + HttpClientAdapter, +>; + /// A structure for resolving, fetching and caching source files. #[derive(Debug)] -pub struct FileFetcher { - auth_tokens: AuthTokens, - allow_remote: bool, - memory_files: MemoryFiles, - cache_setting: CacheSetting, - http_cache: Arc, - http_client_provider: Arc, - blob_store: Arc, - download_log_level: log::Level, - progress_bar: Option, +pub struct CliFileFetcher { + file_fetcher: DenoCacheDirFileFetcher, + memory_files: Arc, } -impl FileFetcher { +impl CliFileFetcher { pub fn new( http_cache: Arc, - cache_setting: CacheSetting, - allow_remote: bool, http_client_provider: Arc, blob_store: Arc, progress_bar: Option, + allow_remote: bool, + cache_setting: CacheSetting, + download_log_level: log::Level, ) -> Self { - Self { - auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), - allow_remote, - memory_files: Default::default(), - cache_setting, + let memory_files = Arc::new(MemoryFiles::default()); + let file_fetcher = DenoCacheDirFileFetcher::new( + BlobStoreAdapter(blob_store), + RealDenoCacheEnv, http_cache, - http_client_provider, - blob_store, - download_log_level: log::Level::Info, - progress_bar, + HttpClientAdapter { + http_client_provider: http_client_provider.clone(), + download_log_level, + progress_bar, + }, + memory_files.clone(), + FileFetcherOptions { + allow_remote, + cache_setting, + auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), + }, + ); + Self { + file_fetcher, + memory_files, } } pub fn cache_setting(&self) -> &CacheSetting { - &self.cache_setting - } - - /// Sets the log level to use when outputting the download message. - pub fn set_download_log_level(&mut self, level: log::Level) { - self.download_log_level = level; - } - - /// Fetch cached remote file. - /// - /// This is a recursive operation if source file has redirections. - pub fn fetch_cached( - &self, - specifier: &ModuleSpecifier, - redirect_limit: i64, - ) -> Result, AnyError> { - let mut specifier = Cow::Borrowed(specifier); - for _ in 0..=redirect_limit { - match self.fetch_cached_no_follow(&specifier, None)? { - Some(FileOrRedirect::File(file)) => { - return Ok(Some(file)); - } - Some(FileOrRedirect::Redirect(redirect_specifier)) => { - specifier = Cow::Owned(redirect_specifier); - } - None => { - return Ok(None); - } - } - } - Err(custom_error("Http", "Too many redirects.")) - } - - fn fetch_cached_no_follow( - &self, - specifier: &ModuleSpecifier, - maybe_checksum: Option<&LoaderChecksum>, - ) -> Result, AnyError> { - debug!( - "FileFetcher::fetch_cached_no_follow - specifier: {}", - specifier - ); - - let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once - let result = self.http_cache.get( - &cache_key, - maybe_checksum - .as_ref() - .map(|c| deno_cache_dir::Checksum::new(c.as_str())), - ); - match result { - Ok(Some(cache_data)) => Ok(Some(FileOrRedirect::from_deno_cache_entry( - specifier, cache_data, - )?)), - Ok(None) => Ok(None), - Err(err) => match err { - deno_cache_dir::CacheReadFileError::Io(err) => Err(err.into()), - deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => { - // convert to the equivalent deno_graph error so that it - // enhances it if this is passed to deno_graph - Err( - deno_graph::source::ChecksumIntegrityError { - actual: err.actual, - expected: err.expected, - } - .into(), - ) - } - }, - } - } - - /// Convert a data URL into a file, resulting in an error if the URL is - /// invalid. - fn fetch_data_url( - &self, - specifier: &ModuleSpecifier, - ) -> Result { - debug!("FileFetcher::fetch_data_url() - specifier: {}", specifier); - let data_url = deno_graph::source::RawDataUrl::parse(specifier)?; - let (bytes, headers) = data_url.into_bytes_and_headers(); - Ok(File { - specifier: specifier.clone(), - maybe_headers: Some(headers), - source: Arc::from(bytes), - }) - } - - /// Get a blob URL. - async fn fetch_blob_url( - &self, - specifier: &ModuleSpecifier, - ) -> Result { - debug!("FileFetcher::fetch_blob_url() - specifier: {}", specifier); - let blob = self - .blob_store - .get_object_url(specifier.clone()) - .ok_or_else(|| { - custom_error( - "NotFound", - format!("Blob URL not found: \"{specifier}\"."), - ) - })?; - - let bytes = blob.read_all().await; - let headers = - HashMap::from([("content-type".to_string(), blob.media_type.clone())]); - - Ok(File { - specifier: specifier.clone(), - maybe_headers: Some(headers), - source: Arc::from(bytes), - }) - } - - async fn fetch_remote_no_follow( - &self, - specifier: &ModuleSpecifier, - maybe_accept: Option<&str>, - cache_setting: &CacheSetting, - maybe_checksum: Option<&LoaderChecksum>, - maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, - ) -> Result { - debug!( - "FileFetcher::fetch_remote_no_follow - specifier: {}", - specifier - ); - - if self.should_use_cache(specifier, cache_setting) { - if let Some(file_or_redirect) = - self.fetch_cached_no_follow(specifier, maybe_checksum)? - { - return Ok(file_or_redirect); - } - } - - if *cache_setting == CacheSetting::Only { - return Err(custom_error( - "NotCached", - format!( - "Specifier not found in cache: \"{specifier}\", --cached-only is specified." - ), - )); - } - - let mut maybe_progress_guard = None; - if let Some(pb) = self.progress_bar.as_ref() { - maybe_progress_guard = Some(pb.update(specifier.as_str())); - } else { - log::log!( - self.download_log_level, - "{} {}", - colors::green("Download"), - specifier - ); - } - - let maybe_etag_cache_entry = self - .http_cache - .cache_item_key(specifier) - .ok() - .and_then(|key| { - self - .http_cache - .get( - &key, - maybe_checksum - .as_ref() - .map(|c| deno_cache_dir::Checksum::new(c.as_str())), - ) - .ok() - .flatten() - }) - .and_then(|cache_entry| { - cache_entry - .metadata - .headers - .get("etag") - .cloned() - .map(|etag| (cache_entry, etag)) - }); - let maybe_auth_token = self.auth_tokens.get(specifier); - - async fn handle_request_or_server_error( - retried: &mut bool, - specifier: &Url, - err_str: String, - ) -> Result<(), AnyError> { - // Retry once, and bail otherwise. - if !*retried { - *retried = true; - log::debug!("Import '{}' failed: {}. Retrying...", specifier, err_str); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - Ok(()) - } else { - Err(generic_error(format!( - "Import '{}' failed: {}", - specifier, err_str - ))) - } - } - - let mut retried = false; // retry intermittent failures - let result = loop { - let result = match self - .http_client_provider - .get_or_create()? - .fetch_no_follow(FetchOnceArgs { - url: specifier.clone(), - maybe_accept: maybe_accept.map(ToOwned::to_owned), - maybe_etag: maybe_etag_cache_entry - .as_ref() - .map(|(_, etag)| etag.clone()), - maybe_auth_token: maybe_auth_token.clone(), - maybe_auth: maybe_auth.clone(), - maybe_progress_guard: maybe_progress_guard.as_ref(), - }) - .await? - { - FetchOnceResult::NotModified => { - let (cache_entry, _) = maybe_etag_cache_entry.unwrap(); - FileOrRedirect::from_deno_cache_entry(specifier, cache_entry) - } - FetchOnceResult::Redirect(redirect_url, headers) => { - self.http_cache.set(specifier, headers, &[])?; - Ok(FileOrRedirect::Redirect(redirect_url)) - } - FetchOnceResult::Code(bytes, headers) => { - self.http_cache.set(specifier, headers.clone(), &bytes)?; - if let Some(checksum) = &maybe_checksum { - checksum.check_source(&bytes)?; - } - Ok(FileOrRedirect::File(File { - specifier: specifier.clone(), - maybe_headers: Some(headers), - source: Arc::from(bytes), - })) - } - FetchOnceResult::RequestError(err) => { - handle_request_or_server_error(&mut retried, specifier, err).await?; - continue; - } - FetchOnceResult::ServerError(status) => { - handle_request_or_server_error( - &mut retried, - specifier, - status.to_string(), - ) - .await?; - continue; - } - }; - break result; - }; - - drop(maybe_progress_guard); - result - } - - /// Returns if the cache should be used for a given specifier. - fn should_use_cache( - &self, - specifier: &ModuleSpecifier, - cache_setting: &CacheSetting, - ) -> bool { - match cache_setting { - CacheSetting::ReloadAll => false, - CacheSetting::Use | CacheSetting::Only => true, - CacheSetting::RespectHeaders => { - let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { - return false; - }; - let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { - return false; - }; - let Ok(Some(download_time)) = - self.http_cache.read_download_time(&cache_key) - else { - return false; - }; - let cache_semantics = - CacheSemantics::new(headers, download_time, SystemTime::now()); - cache_semantics.should_use() - } - CacheSetting::ReloadSome(list) => { - let mut url = specifier.clone(); - url.set_fragment(None); - if list.iter().any(|x| x == url.as_str()) { - return false; - } - url.set_query(None); - let mut path = PathBuf::from(url.as_str()); - loop { - if list.contains(&path.to_str().unwrap().to_string()) { - return false; - } - if !path.pop() { - break; - } - } - true - } - } + self.file_fetcher.cache_setting() } #[inline(always)] @@ -578,7 +348,10 @@ impl FileFetcher { .fetch_inner( specifier, None, - FetchPermissionsOptionRef::StaticContainer(permissions), + FetchPermissionsOptionRef::Restricted( + permissions, + CheckSpecifierKind::Static, + ), ) .await } @@ -590,42 +363,50 @@ impl FileFetcher { permissions: FetchPermissionsOptionRef<'_>, ) -> Result { self - .fetch_with_options(FetchOptions { + .fetch_with_options( specifier, permissions, - maybe_auth, - maybe_accept: None, - maybe_cache_setting: None, - }) + FetchOptions { + maybe_auth, + maybe_accept: None, + maybe_cache_setting: None, + }, + ) .await } pub async fn fetch_with_options( &self, + specifier: &ModuleSpecifier, + permissions: FetchPermissionsOptionRef<'_>, options: FetchOptions<'_>, ) -> Result { - self.fetch_with_options_and_max_redirect(options, 10).await + self + .fetch_with_options_and_max_redirect(specifier, permissions, options, 10) + .await } async fn fetch_with_options_and_max_redirect( &self, + specifier: &ModuleSpecifier, + permissions: FetchPermissionsOptionRef<'_>, options: FetchOptions<'_>, max_redirect: usize, ) -> Result { - let mut specifier = Cow::Borrowed(options.specifier); - let mut maybe_auth = options.maybe_auth.clone(); + let mut specifier = Cow::Borrowed(specifier); + let mut maybe_auth = options.maybe_auth; for _ in 0..=max_redirect { match self - .fetch_no_follow_with_options(FetchNoFollowOptions { - fetch_options: FetchOptions { - specifier: &specifier, - permissions: options.permissions, + .fetch_no_follow( + &specifier, + permissions, + FetchNoFollowOptions { maybe_auth: maybe_auth.clone(), maybe_accept: options.maybe_accept, maybe_cache_setting: options.maybe_cache_setting, + maybe_checksum: None, }, - maybe_checksum: None, - }) + ) .await? { FileOrRedirect::File(file) => { @@ -641,92 +422,61 @@ impl FileFetcher { } } - Err(custom_error("Http", "Too many redirects.")) + Err(TooManyRedirectsError(specifier.into_owned()).into()) } /// Fetches without following redirects. - pub async fn fetch_no_follow_with_options( + pub async fn fetch_no_follow( &self, + specifier: &ModuleSpecifier, + permissions: FetchPermissionsOptionRef<'_>, options: FetchNoFollowOptions<'_>, - ) -> Result { - let maybe_checksum = options.maybe_checksum; - let options = options.fetch_options; - let specifier = options.specifier; - // note: this debug output is used by the tests - debug!( - "FileFetcher::fetch_no_follow_with_options - specifier: {}", - specifier - ); - let scheme = get_validated_scheme(specifier)?; - match options.permissions { + ) -> Result { + validate_scheme(specifier).map_err(|err| { + CliFetchNoFollowErrorKind::FetchNoFollow(err.into()).into_box() + })?; + match permissions { FetchPermissionsOptionRef::AllowAll => { // allow } - FetchPermissionsOptionRef::StaticContainer(permissions) => { - permissions.check_specifier( - specifier, - deno_runtime::deno_permissions::CheckSpecifierKind::Static, - )?; - } - FetchPermissionsOptionRef::DynamicContainer(permissions) => { - permissions.check_specifier( - specifier, - deno_runtime::deno_permissions::CheckSpecifierKind::Dynamic, - )?; + FetchPermissionsOptionRef::Restricted(permissions, kind) => { + permissions.check_specifier(specifier, kind)?; } } - if let Some(file) = self.memory_files.get(specifier) { - Ok(FileOrRedirect::File(file)) - } else if scheme == "file" { - // we do not in memory cache files, as this would prevent files on the - // disk changing effecting things like workers and dynamic imports. - fetch_local(specifier).map(FileOrRedirect::File) - } else if scheme == "data" { - self.fetch_data_url(specifier).map(FileOrRedirect::File) - } else if scheme == "blob" { - self - .fetch_blob_url(specifier) - .await - .map(FileOrRedirect::File) - } else if !self.allow_remote { - Err(custom_error( - "NoRemote", - format!("A remote specifier was requested: \"{specifier}\", but --no-remote is specified."), - )) - } else { - self - .fetch_remote_no_follow( - specifier, - options.maybe_accept, - options.maybe_cache_setting.unwrap_or(&self.cache_setting), - maybe_checksum, - options.maybe_auth, - ) - .await - } + self + .file_fetcher + .fetch_no_follow( + specifier, + deno_cache_dir::file_fetcher::FetchNoFollowOptions { + maybe_auth: options.maybe_auth, + maybe_checksum: options + .maybe_checksum + .map(|c| deno_cache_dir::Checksum::new(c.as_str())), + maybe_accept: options.maybe_accept, + maybe_cache_setting: options.maybe_cache_setting, + }, + ) + .await + .map_err(|err| CliFetchNoFollowErrorKind::FetchNoFollow(err).into_box()) } /// A synchronous way to retrieve a source file, where if the file has already /// been cached in memory it will be returned, otherwise for local files will /// be read from disk. - pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option { - let maybe_file = self.memory_files.get(specifier); - if maybe_file.is_none() { - let is_local = specifier.scheme() == "file"; - if is_local { - if let Ok(file) = fetch_local(specifier) { - return Some(file); - } - } - None + pub fn get_cached_source_or_local( + &self, + specifier: &ModuleSpecifier, + ) -> Result, AnyError> { + if specifier.scheme() == "file" { + Ok(self.file_fetcher.fetch_local(specifier)?) } else { - maybe_file + Ok(self.file_fetcher.fetch_cached(specifier, 10)?) } } /// Insert a temporary module for the file fetcher. pub fn insert_memory_files(&self, file: File) -> Option { - self.memory_files.insert(file.specifier.clone(), file) + self.memory_files.insert(file.url.clone(), file) } pub fn clear_memory_files(&self) { @@ -734,6 +484,16 @@ impl FileFetcher { } } +fn validate_scheme(specifier: &Url) -> Result<(), UnsupportedSchemeError> { + match deno_cache_dir::file_fetcher::is_valid_scheme(specifier.scheme()) { + true => Ok(()), + false => Err(UnsupportedSchemeError { + scheme: specifier.scheme().to_string(), + url: specifier.clone(), + }), + } +} + #[cfg(test)] mod tests { use crate::cache::GlobalHttpCache; @@ -741,7 +501,8 @@ mod tests { use crate::http_util::HttpClientProvider; use super::*; - use deno_core::error::get_custom_error_class; + use deno_cache_dir::file_fetcher::FetchNoFollowErrorKind; + use deno_cache_dir::file_fetcher::HttpClient; use deno_core::resolve_url; use deno_runtime::deno_web::Blob; use deno_runtime::deno_web::InMemoryBlobPart; @@ -750,7 +511,7 @@ mod tests { fn setup( cache_setting: CacheSetting, maybe_temp_dir: Option, - ) -> (FileFetcher, TempDir) { + ) -> (CliFileFetcher, TempDir) { let (file_fetcher, temp_dir, _) = setup_with_blob_store(cache_setting, maybe_temp_dir); (file_fetcher, temp_dir) @@ -759,22 +520,38 @@ mod tests { fn setup_with_blob_store( cache_setting: CacheSetting, maybe_temp_dir: Option, - ) -> (FileFetcher, TempDir, Arc) { - let temp_dir = maybe_temp_dir.unwrap_or_default(); - let location = temp_dir.path().join("remote").to_path_buf(); - let blob_store: Arc = Default::default(); - let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), - cache_setting, - true, - Arc::new(HttpClientProvider::new(None, None)), - blob_store.clone(), - None, - ); + ) -> (CliFileFetcher, TempDir, Arc) { + let (file_fetcher, temp_dir, blob_store, _) = + setup_with_blob_store_and_cache(cache_setting, maybe_temp_dir); (file_fetcher, temp_dir, blob_store) } - async fn test_fetch(specifier: &ModuleSpecifier) -> (File, FileFetcher) { + fn setup_with_blob_store_and_cache( + cache_setting: CacheSetting, + maybe_temp_dir: Option, + ) -> ( + CliFileFetcher, + TempDir, + Arc, + Arc, + ) { + let temp_dir = maybe_temp_dir.unwrap_or_default(); + let location = temp_dir.path().join("remote").to_path_buf(); + let blob_store: Arc = Default::default(); + let cache = Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)); + let file_fetcher = CliFileFetcher::new( + cache.clone(), + Arc::new(HttpClientProvider::new(None, None)), + blob_store.clone(), + None, + true, + cache_setting, + log::Level::Info, + ); + (file_fetcher, temp_dir, blob_store, cache) + } + + async fn test_fetch(specifier: &ModuleSpecifier) -> (File, CliFileFetcher) { let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); let result = file_fetcher.fetch_bypass_permissions(specifier).await; assert!(result.is_ok()); @@ -785,27 +562,20 @@ mod tests { specifier: &ModuleSpecifier, ) -> (File, HashMap) { let _http_server_guard = test_util::http_server(); - let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); + let (file_fetcher, _, _, http_cache) = + setup_with_blob_store_and_cache(CacheSetting::ReloadAll, None); let result: Result = file_fetcher .fetch_with_options_and_max_redirect( - FetchOptions { - specifier, - permissions: FetchPermissionsOptionRef::AllowAll, - maybe_auth: None, - maybe_accept: None, - maybe_cache_setting: Some(&file_fetcher.cache_setting), - }, + specifier, + FetchPermissionsOptionRef::AllowAll, + Default::default(), 1, ) .await; - let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); + let cache_key = http_cache.cache_item_key(specifier).unwrap(); ( result.unwrap(), - file_fetcher - .http_cache - .read_headers(&cache_key) - .unwrap() - .unwrap(), + http_cache.read_headers(&cache_key).unwrap().unwrap(), ) } @@ -850,28 +620,6 @@ mod tests { ); } - #[test] - fn test_get_validated_scheme() { - let fixtures = vec![ - ("https://deno.land/x/mod.ts", true, "https"), - ("http://deno.land/x/mod.ts", true, "http"), - ("file:///a/b/c.ts", true, "file"), - ("file:///C:/a/b/c.ts", true, "file"), - ("data:,some%20text", true, "data"), - ("ftp://a/b/c.ts", false, ""), - ("mailto:dino@deno.land", false, ""), - ]; - - for (specifier, is_ok, expected) in fixtures { - let specifier = ModuleSpecifier::parse(specifier).unwrap(); - let actual = get_validated_scheme(&specifier); - assert_eq!(actual.is_ok(), is_ok); - if is_ok { - assert_eq!(actual.unwrap(), expected); - } - } - } - #[tokio::test] async fn test_insert_cached() { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); @@ -879,7 +627,7 @@ mod tests { let specifier = ModuleSpecifier::from_file_path(&local).unwrap(); let file = File { source: Arc::from("some source code".as_bytes()), - specifier: specifier.clone(), + url: specifier.clone(), maybe_headers: Some(HashMap::from([( "content-type".to_string(), "application/javascript".to_string(), @@ -900,7 +648,7 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" @@ -929,7 +677,7 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" @@ -941,33 +689,36 @@ mod tests { #[tokio::test] async fn test_fetch_complex() { let _http_server_guard = test_util::http_server(); - let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); + let (file_fetcher, temp_dir, _, http_cache) = + setup_with_blob_store_and_cache(CacheSetting::Use, None); let (file_fetcher_01, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); - let (file_fetcher_02, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); + let (file_fetcher_02, _, _, http_cache_02) = + setup_with_blob_store_and_cache( + CacheSetting::Use, + Some(temp_dir.clone()), + ); let specifier = ModuleSpecifier::parse("http://localhost:4545/subdir/mod2.ts").unwrap(); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::TypeScript); - let cache_item_key = - file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); + let cache_item_key = http_cache.cache_item_key(&specifier).unwrap(); let mut headers = HashMap::new(); headers.insert("content-type".to_string(), "text/javascript".to_string()); - file_fetcher - .http_cache + http_cache .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_01.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" @@ -976,22 +727,20 @@ mod tests { // the value above. assert_eq!(file.media_type, MediaType::JavaScript); - let headers2 = file_fetcher_02 - .http_cache + let headers2 = http_cache_02 .read_headers(&cache_item_key) .unwrap() .unwrap(); assert_eq!(headers2.get("content-type").unwrap(), "text/javascript"); headers = HashMap::new(); headers.insert("content-type".to_string(), "application/json".to_string()); - file_fetcher_02 - .http_cache + http_cache_02 .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_02.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" @@ -1001,20 +750,21 @@ mod tests { // This creates a totally new instance, simulating another Deno process // invocation and indicates to "cache bust". let location = temp_dir.path().join("remote").to_path_buf(); - let file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), - CacheSetting::ReloadAll, - true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::ReloadAll, + log::Level::Info, ); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" @@ -1030,73 +780,52 @@ mod tests { let specifier = resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap(); + let http_cache = Arc::new(GlobalHttpCache::new( + location.clone(), + crate::cache::RealDenoCacheEnv, + )); let file_modified_01 = { - let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new( - location.clone(), - crate::cache::RealDenoCacheEnv, - )), - CacheSetting::Use, - true, + let file_fetcher = CliFileFetcher::new( + http_cache.clone(), Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::Use, + log::Level::Info, ); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let cache_key = - file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); + let cache_key = http_cache.cache_item_key(&specifier).unwrap(); ( - file_fetcher - .http_cache - .read_modified_time(&cache_key) - .unwrap(), - file_fetcher - .http_cache - .read_headers(&cache_key) - .unwrap() - .unwrap(), - file_fetcher - .http_cache - .read_download_time(&cache_key) - .unwrap() - .unwrap(), + http_cache.read_modified_time(&cache_key).unwrap(), + http_cache.read_headers(&cache_key).unwrap().unwrap(), + http_cache.read_download_time(&cache_key).unwrap().unwrap(), ) }; let file_modified_02 = { - let file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), - CacheSetting::Use, - true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::Use, + log::Level::Info, ); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let cache_key = - file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); + let cache_key = http_cache.cache_item_key(&specifier).unwrap(); ( - file_fetcher - .http_cache - .read_modified_time(&cache_key) - .unwrap(), - file_fetcher - .http_cache - .read_headers(&cache_key) - .unwrap() - .unwrap(), - file_fetcher - .http_cache - .read_download_time(&cache_key) - .unwrap() - .unwrap(), + http_cache.read_modified_time(&cache_key).unwrap(), + http_cache.read_headers(&cache_key).unwrap().unwrap(), + http_cache.read_download_time(&cache_key).unwrap().unwrap(), ) }; @@ -1106,7 +835,8 @@ mod tests { #[tokio::test] async fn test_fetch_redirected() { let _http_server_guard = test_util::http_server(); - let (file_fetcher, _) = setup(CacheSetting::Use, None); + let (file_fetcher, _, _, http_cache) = + setup_with_blob_store_and_cache(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4546/subdir/redirects/redirect1.js") .unwrap(); @@ -1117,24 +847,27 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); let file = result.unwrap(); - assert_eq!(file.specifier, redirected_specifier); + assert_eq!(file.url, redirected_specifier); assert_eq!( - get_text_from_cache(&file_fetcher, &specifier), + get_text_from_cache(http_cache.as_ref(), &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &specifier), + get_location_header_from_cache(http_cache.as_ref(), &specifier), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( - get_text_from_cache(&file_fetcher, &redirected_specifier), + get_text_from_cache(http_cache.as_ref(), &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &redirected_specifier), + get_location_header_from_cache( + http_cache.as_ref(), + &redirected_specifier + ), None, ); } @@ -1142,7 +875,8 @@ mod tests { #[tokio::test] async fn test_fetch_multiple_redirects() { let _http_server_guard = test_util::http_server(); - let (file_fetcher, _) = setup(CacheSetting::Use, None); + let (file_fetcher, _, _, http_cache) = + setup_with_blob_store_and_cache(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4548/subdir/redirects/redirect1.js") .unwrap(); @@ -1156,34 +890,40 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); let file = result.unwrap(); - assert_eq!(file.specifier, redirected_02_specifier); + assert_eq!(file.url, redirected_02_specifier); assert_eq!( - get_text_from_cache(&file_fetcher, &specifier), + get_text_from_cache(http_cache.as_ref(), &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &specifier), + get_location_header_from_cache(http_cache.as_ref(), &specifier), Some("http://localhost:4546/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( - get_text_from_cache(&file_fetcher, &redirected_01_specifier), + get_text_from_cache(http_cache.as_ref(), &redirected_01_specifier), "", "redirected files should have empty cached contents" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &redirected_01_specifier), + get_location_header_from_cache( + http_cache.as_ref(), + &redirected_01_specifier + ), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( - get_text_from_cache(&file_fetcher, &redirected_02_specifier), + get_text_from_cache(http_cache.as_ref(), &redirected_02_specifier), "export const redirect = 1;\n" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &redirected_02_specifier), + get_location_header_from_cache( + http_cache.as_ref(), + &redirected_02_specifier + ), None, ); } @@ -1197,81 +937,53 @@ mod tests { resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap(); let redirected_specifier = resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap(); + let http_cache = Arc::new(GlobalHttpCache::new( + location.clone(), + crate::cache::RealDenoCacheEnv, + )); let metadata_file_modified_01 = { - let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new( - location.clone(), - crate::cache::RealDenoCacheEnv, - )), - CacheSetting::Use, - true, + let file_fetcher = CliFileFetcher::new( + http_cache.clone(), Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::Use, + log::Level::Info, ); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let cache_key = file_fetcher - .http_cache - .cache_item_key(&redirected_specifier) - .unwrap(); + let cache_key = http_cache.cache_item_key(&redirected_specifier).unwrap(); ( - file_fetcher - .http_cache - .read_modified_time(&cache_key) - .unwrap(), - file_fetcher - .http_cache - .read_headers(&cache_key) - .unwrap() - .unwrap(), - file_fetcher - .http_cache - .read_download_time(&cache_key) - .unwrap() - .unwrap(), + http_cache.read_modified_time(&cache_key).unwrap(), + http_cache.read_headers(&cache_key).unwrap().unwrap(), + http_cache.read_download_time(&cache_key).unwrap().unwrap(), ) }; let metadata_file_modified_02 = { - let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new( - location, - crate::cache::RealDenoCacheEnv, - )), - CacheSetting::Use, - true, + let file_fetcher = CliFileFetcher::new( + http_cache.clone(), Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::Use, + log::Level::Info, ); let result = file_fetcher .fetch_bypass_permissions(&redirected_specifier) .await; assert!(result.is_ok()); - let cache_key = file_fetcher - .http_cache - .cache_item_key(&redirected_specifier) - .unwrap(); + let cache_key = http_cache.cache_item_key(&redirected_specifier).unwrap(); ( - file_fetcher - .http_cache - .read_modified_time(&cache_key) - .unwrap(), - file_fetcher - .http_cache - .read_headers(&cache_key) - .unwrap() - .unwrap(), - file_fetcher - .http_cache - .read_download_time(&cache_key) - .unwrap() - .unwrap(), + http_cache.read_modified_time(&cache_key).unwrap(), + http_cache.read_headers(&cache_key).unwrap().unwrap(), + http_cache.read_download_time(&cache_key).unwrap().unwrap(), ) }; @@ -1288,13 +1000,9 @@ mod tests { let result = file_fetcher .fetch_with_options_and_max_redirect( - FetchOptions { - specifier: &specifier, - permissions: FetchPermissionsOptionRef::AllowAll, - maybe_auth: None, - maybe_accept: None, - maybe_cache_setting: Some(&file_fetcher.cache_setting), - }, + &specifier, + FetchPermissionsOptionRef::AllowAll, + Default::default(), 2, ) .await; @@ -1302,29 +1010,26 @@ mod tests { let result = file_fetcher .fetch_with_options_and_max_redirect( - FetchOptions { - specifier: &specifier, - permissions: FetchPermissionsOptionRef::AllowAll, - maybe_auth: None, - maybe_accept: None, - maybe_cache_setting: Some(&file_fetcher.cache_setting), - }, + &specifier, + FetchPermissionsOptionRef::AllowAll, + Default::default(), 1, ) .await; assert!(result.is_err()); - let result = file_fetcher.fetch_cached(&specifier, 2); + let result = file_fetcher.file_fetcher.fetch_cached(&specifier, 2); assert!(result.is_ok()); - let result = file_fetcher.fetch_cached(&specifier, 1); + let result = file_fetcher.file_fetcher.fetch_cached(&specifier, 1); assert!(result.is_err()); } #[tokio::test] async fn test_fetch_same_host_redirect() { let _http_server_guard = test_util::http_server(); - let (file_fetcher, _) = setup(CacheSetting::Use, None); + let (file_fetcher, _, _, http_cache) = + setup_with_blob_store_and_cache(CacheSetting::Use, None); let specifier = resolve_url( "http://localhost:4550/REDIRECT/subdir/redirects/redirect1.js", ) @@ -1336,24 +1041,27 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); let file = result.unwrap(); - assert_eq!(file.specifier, redirected_specifier); + assert_eq!(file.url, redirected_specifier); assert_eq!( - get_text_from_cache(&file_fetcher, &specifier), + get_text_from_cache(http_cache.as_ref(), &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &specifier), + get_location_header_from_cache(http_cache.as_ref(), &specifier), Some("/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( - get_text_from_cache(&file_fetcher, &redirected_specifier), + get_text_from_cache(http_cache.as_ref(), &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( - get_location_header_from_cache(&file_fetcher, &redirected_specifier), + get_location_header_from_cache( + http_cache.as_ref(), + &redirected_specifier + ), None ); } @@ -1363,16 +1071,17 @@ mod tests { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("remote").to_path_buf(); - let file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), - CacheSetting::Use, - false, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + false, + CacheSetting::Use, + log::Level::Info, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap(); @@ -1380,8 +1089,19 @@ mod tests { let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_err()); let err = result.unwrap_err(); - assert_eq!(get_custom_error_class(&err), Some("NoRemote")); - assert_eq!(err.to_string(), "A remote specifier was requested: \"http://localhost:4545/run/002_hello.ts\", but --no-remote is specified."); + let err = err.downcast::().unwrap().into_kind(); + match err { + CliFetchNoFollowErrorKind::FetchNoFollow(err) => { + let err = err.into_kind(); + match &err { + FetchNoFollowErrorKind::NoRemote { .. } => { + assert_eq!(err.to_string(), "A remote specifier was requested: \"http://localhost:4545/run/002_hello.ts\", but --no-remote is specified."); + } + _ => unreachable!(), + } + } + _ => unreachable!(), + } } #[tokio::test] @@ -1389,21 +1109,23 @@ mod tests { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("remote").to_path_buf(); - let file_fetcher_01 = FileFetcher::new( + let file_fetcher_01 = CliFileFetcher::new( Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), + Arc::new(HttpClientProvider::new(None, None)), + Default::default(), + None, + true, CacheSetting::Only, - true, - Arc::new(HttpClientProvider::new(None, None)), - Default::default(), - None, + log::Level::Info, ); - let file_fetcher_02 = FileFetcher::new( + let file_fetcher_02 = CliFileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), - CacheSetting::Use, - true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, + true, + CacheSetting::Use, + log::Level::Info, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap(); @@ -1411,8 +1133,19 @@ mod tests { let result = file_fetcher_01.fetch_bypass_permissions(&specifier).await; assert!(result.is_err()); let err = result.unwrap_err(); - assert_eq!(err.to_string(), "Specifier not found in cache: \"http://localhost:4545/run/002_hello.ts\", --cached-only is specified."); - assert_eq!(get_custom_error_class(&err), Some("NotCached")); + let err = err.downcast::().unwrap().into_kind(); + match err { + CliFetchNoFollowErrorKind::FetchNoFollow(err) => { + let err = err.into_kind(); + match &err { + FetchNoFollowErrorKind::NotCached { .. } => { + assert_eq!(err.to_string(), "Specifier not found in cache: \"http://localhost:4545/run/002_hello.ts\", --cached-only is specified."); + } + _ => unreachable!(), + } + } + _ => unreachable!(), + } let result = file_fetcher_02.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); @@ -1426,16 +1159,16 @@ mod tests { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let fixture_path = temp_dir.path().join("mod.ts"); let specifier = ModuleSpecifier::from_file_path(&fixture_path).unwrap(); - fs::write(fixture_path.clone(), r#"console.log("hello deno");"#).unwrap(); + fixture_path.write(r#"console.log("hello deno");"#); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!(&*file.source, r#"console.log("hello deno");"#); - fs::write(fixture_path, r#"console.log("goodbye deno");"#).unwrap(); + fixture_path.write(r#"console.log("goodbye deno");"#); let result = file_fetcher.fetch_bypass_permissions(&specifier).await; assert!(result.is_ok()); - let file = result.unwrap().into_text_decoded().unwrap(); + let file = TextDecodedFile::decode(result.unwrap()).unwrap(); assert_eq!(&*file.source, r#"console.log("goodbye deno");"#); } @@ -1527,29 +1260,169 @@ mod tests { test_fetch_remote_encoded("windows-1255", "windows-1255", expected).await; } + fn create_http_client_adapter() -> HttpClientAdapter { + HttpClientAdapter { + http_client_provider: Arc::new(HttpClientProvider::new(None, None)), + download_log_level: log::Level::Info, + progress_bar: None, + } + } + + #[tokio::test] + async fn test_fetch_string() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, HeaderMap::new()).await; + if let Ok(SendResponse::Success(headers, body)) = result { + assert!(!body.is_empty()); + assert_eq!(headers.get("content-type").unwrap(), "application/json"); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); + } else { + panic!(); + } + } + + #[tokio::test] + async fn test_fetch_gzip() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped") + .unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, HeaderMap::new()).await; + if let Ok(SendResponse::Success(headers, body)) = result { + assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/javascript" + ); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); + } else { + panic!(); + } + } + + #[tokio::test] + async fn test_fetch_with_etag() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, HeaderMap::new()).await; + if let Ok(SendResponse::Success(headers, body)) = result { + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/typescript" + ); + assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); + } else { + panic!(); + } + + let mut headers = HeaderMap::new(); + headers.insert("if-none-match", "33a64df551425fcc55e".parse().unwrap()); + let res = client.send_no_follow(&url, headers).await; + assert_eq!(res.unwrap(), SendResponse::NotModified); + } + + #[tokio::test] + async fn test_fetch_brotli() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli") + .unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, HeaderMap::new()).await; + if let Ok(SendResponse::Success(headers, body)) = result { + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/javascript" + ); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); + } else { + panic!(); + } + } + + #[tokio::test] + async fn test_fetch_accept() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap(); + let client = create_http_client_adapter(); + let mut headers = HeaderMap::new(); + headers.insert("accept", "application/json".parse().unwrap()); + let result = client.send_no_follow(&url, headers).await; + if let Ok(SendResponse::Success(_, body)) = result { + assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes()); + } else { + panic!(); + } + } + + #[tokio::test] + async fn test_fetch_no_follow_with_redirect() { + let _http_server_guard = test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4546/assets/fixture.json").unwrap(); + // Dns resolver substitutes `127.0.0.1` with `localhost` + let target_url = + Url::parse("http://localhost:4545/assets/fixture.json").unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, Default::default()).await; + if let Ok(SendResponse::Redirect(headers)) = result { + assert_eq!(headers.get("location").unwrap(), target_url.as_str()); + } else { + panic!(); + } + } + + #[tokio::test] + async fn server_error() { + let _g = test_util::http_server(); + let url_str = "http://127.0.0.1:4545/server_error"; + let url = Url::parse(url_str).unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, Default::default()).await; + + if let Err(SendError::StatusCode(status)) = result { + assert_eq!(status, 500); + } else { + panic!("{:?}", result); + } + } + + #[tokio::test] + async fn request_error() { + let _g = test_util::http_server(); + let url_str = "http://127.0.0.1:9999/"; + let url = Url::parse(url_str).unwrap(); + let client = create_http_client_adapter(); + let result = client.send_no_follow(&url, Default::default()).await; + + assert!(matches!(result, Err(SendError::Failed(_)))); + } + #[track_caller] fn get_text_from_cache( - file_fetcher: &FileFetcher, + http_cache: &dyn HttpCache, url: &ModuleSpecifier, ) -> String { - let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); - let bytes = file_fetcher - .http_cache - .get(&cache_key, None) - .unwrap() - .unwrap() - .content; + let cache_key = http_cache.cache_item_key(url).unwrap(); + let bytes = http_cache.get(&cache_key, None).unwrap().unwrap().content; String::from_utf8(bytes.into_owned()).unwrap() } #[track_caller] fn get_location_header_from_cache( - file_fetcher: &FileFetcher, + http_cache: &dyn HttpCache, url: &ModuleSpecifier, ) -> Option { - let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); - file_fetcher - .http_cache + let cache_key = http_cache.cache_item_key(url).unwrap(); + http_cache .read_headers(&cache_key) .unwrap() .unwrap() diff --git a/cli/graph_util.rs b/cli/graph_util.rs index b655dda0f6..6abdbe247a 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -13,7 +13,7 @@ use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; use crate::colors; use crate::errors::get_error_class_name; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::npm::CliNpmResolver; use crate::resolver::CjsTracker; use crate::resolver::CliResolver; @@ -431,7 +431,7 @@ pub struct ModuleGraphBuilder { caches: Arc, cjs_tracker: Arc, cli_options: Arc, - file_fetcher: Arc, + file_fetcher: Arc, fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, @@ -450,7 +450,7 @@ impl ModuleGraphBuilder { caches: Arc, cjs_tracker: Arc, cli_options: Arc, - file_fetcher: Arc, + file_fetcher: Arc, fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, diff --git a/cli/http_util.rs b/cli/http_util.rs index 4b17936d68..ce05d66b78 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -1,14 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::auth_tokens::AuthToken; use crate::util::progress_bar::UpdateGuard; use crate::version; -use cache_control::Cachability; -use cache_control::CacheControl; -use chrono::DateTime; +use boxed_error::Boxed; +use deno_cache_dir::file_fetcher::RedirectHeaderParseError; use deno_core::error::custom_error; -use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; @@ -18,195 +15,26 @@ use deno_core::url::Url; use deno_runtime::deno_fetch; use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::CreateHttpClientOptions; +use deno_runtime::deno_fetch::ResBody; use deno_runtime::deno_tls::RootCertStoreProvider; -use http::header; use http::header::HeaderName; use http::header::HeaderValue; -use http::header::ACCEPT; -use http::header::AUTHORIZATION; use http::header::CONTENT_LENGTH; -use http::header::IF_NONE_MATCH; -use http::header::LOCATION; +use http::HeaderMap; use http::StatusCode; use http_body_util::BodyExt; use std::collections::HashMap; use std::sync::Arc; use std::thread::ThreadId; -use std::time::Duration; -use std::time::SystemTime; use thiserror::Error; -// TODO(ry) HTTP headers are not unique key, value pairs. There may be more than -// one header line with the same key. This should be changed to something like -// Vec<(String, String)> -pub type HeadersMap = HashMap; - -/// A structure used to determine if a entity in the http cache can be used. -/// -/// This is heavily influenced by -/// which is BSD -/// 2-Clause Licensed and copyright Kornel Lesiński -pub struct CacheSemantics { - cache_control: CacheControl, - cached: SystemTime, - headers: HashMap, - now: SystemTime, -} - -impl CacheSemantics { - pub fn new( - headers: HashMap, - cached: SystemTime, - now: SystemTime, - ) -> Self { - let cache_control = headers - .get("cache-control") - .map(|v| CacheControl::from_value(v).unwrap_or_default()) - .unwrap_or_default(); - Self { - cache_control, - cached, - headers, - now, - } - } - - fn age(&self) -> Duration { - let mut age = self.age_header_value(); - - if let Ok(resident_time) = self.now.duration_since(self.cached) { - age += resident_time; - } - - age - } - - fn age_header_value(&self) -> Duration { - Duration::from_secs( - self - .headers - .get("age") - .and_then(|v| v.parse().ok()) - .unwrap_or(0), - ) - } - - fn is_stale(&self) -> bool { - self.max_age() <= self.age() - } - - fn max_age(&self) -> Duration { - if self.cache_control.cachability == Some(Cachability::NoCache) { - return Duration::from_secs(0); - } - - if self.headers.get("vary").map(|s| s.trim()) == Some("*") { - return Duration::from_secs(0); - } - - if let Some(max_age) = self.cache_control.max_age { - return max_age; - } - - let default_min_ttl = Duration::from_secs(0); - - let server_date = self.raw_server_date(); - if let Some(expires) = self.headers.get("expires") { - return match DateTime::parse_from_rfc2822(expires) { - Err(_) => Duration::from_secs(0), - Ok(expires) => { - let expires = SystemTime::UNIX_EPOCH - + Duration::from_secs(expires.timestamp().max(0) as _); - return default_min_ttl - .max(expires.duration_since(server_date).unwrap_or_default()); - } - }; - } - - if let Some(last_modified) = self.headers.get("last-modified") { - if let Ok(last_modified) = DateTime::parse_from_rfc2822(last_modified) { - let last_modified = SystemTime::UNIX_EPOCH - + Duration::from_secs(last_modified.timestamp().max(0) as _); - if let Ok(diff) = server_date.duration_since(last_modified) { - let secs_left = diff.as_secs() as f64 * 0.1; - return default_min_ttl.max(Duration::from_secs(secs_left as _)); - } - } - } - - default_min_ttl - } - - fn raw_server_date(&self) -> SystemTime { - self - .headers - .get("date") - .and_then(|d| DateTime::parse_from_rfc2822(d).ok()) - .and_then(|d| { - SystemTime::UNIX_EPOCH - .checked_add(Duration::from_secs(d.timestamp() as _)) - }) - .unwrap_or(self.cached) - } - - /// Returns true if the cached value is "fresh" respecting cached headers, - /// otherwise returns false. - pub fn should_use(&self) -> bool { - if self.cache_control.cachability == Some(Cachability::NoCache) { - return false; - } - - if let Some(max_age) = self.cache_control.max_age { - if self.age() > max_age { - return false; - } - } - - if let Some(min_fresh) = self.cache_control.min_fresh { - if self.time_to_live() < min_fresh { - return false; - } - } - - if self.is_stale() { - let has_max_stale = self.cache_control.max_stale.is_some(); - let allows_stale = has_max_stale - && self - .cache_control - .max_stale - .map(|val| val > self.age() - self.max_age()) - .unwrap_or(true); - if !allows_stale { - return false; - } - } - - true - } - - fn time_to_live(&self) -> Duration { - self.max_age().checked_sub(self.age()).unwrap_or_default() - } -} - -#[derive(Debug, Eq, PartialEq)] -pub enum FetchOnceResult { - Code(Vec, HeadersMap), - NotModified, - Redirect(Url, HeadersMap), - RequestError(String), - ServerError(StatusCode), -} - -#[derive(Debug)] -pub struct FetchOnceArgs<'a> { - pub url: Url, - pub maybe_accept: Option, - pub maybe_etag: Option, - pub maybe_auth_token: Option, - pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, - pub maybe_progress_guard: Option<&'a UpdateGuard>, +#[derive(Debug, Error)] +pub enum SendError { + #[error(transparent)] + Send(#[from] deno_fetch::ClientSendError), + #[error(transparent)] + InvalidUri(#[from] http::uri::InvalidUri), } pub struct HttpClientProvider { @@ -273,8 +101,11 @@ pub struct BadResponseError { pub response_text: Option, } +#[derive(Debug, Boxed)] +pub struct DownloadError(pub Box); + #[derive(Debug, Error)] -pub enum DownloadError { +pub enum DownloadErrorKind { #[error(transparent)] Fetch(AnyError), #[error(transparent)] @@ -285,8 +116,8 @@ pub enum DownloadError { Json(#[from] serde_json::Error), #[error(transparent)] ToStr(#[from] http::header::ToStrError), - #[error("Redirection from '{}' did not provide location header", .request_url)] - NoRedirectHeader { request_url: Url }, + #[error(transparent)] + RedirectHeaderParse(RedirectHeaderParseError), #[error("Too many redirects.")] TooManyRedirects, #[error(transparent)] @@ -358,107 +189,24 @@ impl HttpClient { )) } - /// Asynchronously fetches the given HTTP URL one pass only. - /// If no redirect is present and no error occurs, - /// yields Code(ResultPayload). - /// If redirect occurs, does not follow and - /// yields Redirect(url). - pub async fn fetch_no_follow<'a>( + pub async fn send( &self, - args: FetchOnceArgs<'a>, - ) -> Result { + url: &Url, + headers: HeaderMap, + ) -> Result, SendError> { let body = http_body_util::Empty::new() .map_err(|never| match never {}) .boxed(); let mut request = http::Request::new(body); - *request.uri_mut() = args.url.as_str().parse()?; + *request.uri_mut() = http::Uri::try_from(url.as_str())?; + *request.headers_mut() = headers; - if let Some(etag) = args.maybe_etag { - let if_none_match_val = HeaderValue::from_str(&etag)?; - request - .headers_mut() - .insert(IF_NONE_MATCH, if_none_match_val); - } - if let Some(auth_token) = args.maybe_auth_token { - let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; - request - .headers_mut() - .insert(AUTHORIZATION, authorization_val); - } else if let Some((header, value)) = args.maybe_auth { - request.headers_mut().insert(header, value); - } - if let Some(accept) = args.maybe_accept { - let accepts_val = HeaderValue::from_str(&accept)?; - request.headers_mut().insert(ACCEPT, accepts_val); - } - let response = match self.client.clone().send(request).await { - Ok(resp) => resp, - Err(err) => { - if err.is_connect_error() { - return Ok(FetchOnceResult::RequestError(err.to_string())); - } - return Err(err.into()); - } - }; - - if response.status() == StatusCode::NOT_MODIFIED { - return Ok(FetchOnceResult::NotModified); - } - - let mut result_headers = HashMap::new(); - let response_headers = response.headers(); - - if let Some(warning) = response_headers.get("X-Deno-Warning") { - log::warn!( - "{} {}", - crate::colors::yellow("Warning"), - warning.to_str().unwrap() - ); - } - - for key in response_headers.keys() { - let key_str = key.to_string(); - let values = response_headers.get_all(key); - let values_str = values - .iter() - .map(|e| e.to_str().unwrap().to_string()) - .collect::>() - .join(","); - result_headers.insert(key_str, values_str); - } - - if response.status().is_redirection() { - let new_url = resolve_redirect_from_response(&args.url, &response)?; - return Ok(FetchOnceResult::Redirect(new_url, result_headers)); - } - - let status = response.status(); - - if status.is_server_error() { - return Ok(FetchOnceResult::ServerError(status)); - } - - if status.is_client_error() { - let err = if response.status() == StatusCode::NOT_FOUND { - custom_error( - "NotFound", - format!("Import '{}' failed, not found.", args.url), - ) - } else { - generic_error(format!( - "Import '{}' failed: {}", - args.url, - response.status() - )) - }; - return Err(err); - } - - let body = - get_response_body_with_progress(response, args.maybe_progress_guard) - .await?; - - Ok(FetchOnceResult::Code(body, result_headers)) + self + .client + .clone() + .send(request) + .await + .map_err(SendError::Send) } pub async fn download_text(&self, url: Url) -> Result { @@ -488,7 +236,12 @@ impl HttpClient { Some(progress_guard), ) }, - |e| matches!(e, DownloadError::BadResponse(_) | DownloadError::Fetch(_)), + |e| { + matches!( + e.as_kind(), + DownloadErrorKind::BadResponse(_) | DownloadErrorKind::Fetch(_) + ) + }, ) .await } @@ -515,18 +268,21 @@ impl HttpClient { } else if !response.status().is_success() { let status = response.status(); let maybe_response_text = body_to_string(response).await.ok(); - return Err(DownloadError::BadResponse(BadResponseError { - status_code: status, - response_text: maybe_response_text - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()), - })); + return Err( + DownloadErrorKind::BadResponse(BadResponseError { + status_code: status, + response_text: maybe_response_text + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()), + }) + .into_box(), + ); } get_response_body_with_progress(response, progress_guard) .await - .map(Some) - .map_err(DownloadError::Fetch) + .map(|(_, body)| Some(body)) + .map_err(|err| DownloadErrorKind::Fetch(err).into_box()) } async fn get_redirected_response( @@ -543,7 +299,7 @@ impl HttpClient { .clone() .send(req) .await - .map_err(|e| DownloadError::Fetch(e.into()))?; + .map_err(|e| DownloadErrorKind::Fetch(e.into()).into_box())?; let status = response.status(); if status.is_redirection() { for _ in 0..5 { @@ -563,7 +319,7 @@ impl HttpClient { .clone() .send(req) .await - .map_err(|e| DownloadError::Fetch(e.into()))?; + .map_err(|e| DownloadErrorKind::Fetch(e.into()).into_box())?; let status = new_response.status(); if status.is_redirection() { response = new_response; @@ -572,17 +328,17 @@ impl HttpClient { return Ok((new_response, new_url)); } } - Err(DownloadError::TooManyRedirects) + Err(DownloadErrorKind::TooManyRedirects.into_box()) } else { Ok((response, url)) } } } -async fn get_response_body_with_progress( +pub async fn get_response_body_with_progress( response: http::Response, progress_guard: Option<&UpdateGuard>, -) -> Result, AnyError> { +) -> Result<(HeaderMap, Vec), AnyError> { use http_body::Body as _; if let Some(progress_guard) = progress_guard { let mut total_size = response.body().size_hint().exact(); @@ -597,45 +353,21 @@ async fn get_response_body_with_progress( progress_guard.set_total_size(total_size); let mut current_size = 0; let mut data = Vec::with_capacity(total_size as usize); - let mut stream = response.into_body().into_data_stream(); + let (parts, body) = response.into_parts(); + let mut stream = body.into_data_stream(); while let Some(item) = stream.next().await { let bytes = item?; current_size += bytes.len() as u64; progress_guard.set_position(current_size); data.extend(bytes.into_iter()); } - return Ok(data); + return Ok((parts.headers, data)); } } - let bytes = response.collect().await?.to_bytes(); - Ok(bytes.into()) -} -/// Construct the next uri based on base uri and location header fragment -/// See -fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { - if location.starts_with("http://") || location.starts_with("https://") { - // absolute uri - Url::parse(location).expect("provided redirect url should be a valid url") - } else if location.starts_with("//") { - // "//" authority path-abempty - Url::parse(&format!("{}:{}", base_url.scheme(), location)) - .expect("provided redirect url should be a valid url") - } else if location.starts_with('/') { - // path-absolute - base_url - .join(location) - .expect("provided redirect url should be a valid url") - } else { - // assuming path-noscheme | path-empty - let base_url_path_str = base_url.path().to_owned(); - // Pop last part or url (after last slash) - let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect(); - let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location); - base_url - .join(&new_path) - .expect("provided redirect url should be a valid url") - } + let (parts, body) = response.into_parts(); + let bytes = body.collect().await?.to_bytes(); + Ok((parts.headers, bytes.into())) } fn resolve_redirect_from_response( @@ -643,16 +375,11 @@ fn resolve_redirect_from_response( response: &http::Response, ) -> Result { debug_assert!(response.status().is_redirection()); - if let Some(location) = response.headers().get(LOCATION) { - let location_string = location.to_str()?; - log::debug!("Redirecting to {:?}...", &location_string); - let new_url = resolve_url_from_location(request_url, location_string); - Ok(new_url) - } else { - Err(DownloadError::NoRedirectHeader { - request_url: request_url.clone(), - }) - } + deno_cache_dir::file_fetcher::resolve_redirect_from_headers( + request_url, + response.headers(), + ) + .map_err(|err| DownloadErrorKind::RedirectHeaderParse(*err).into_box()) } pub async fn body_to_string(body: B) -> Result @@ -707,8 +434,6 @@ mod test { use deno_runtime::deno_tls::rustls::RootCertStore; - use crate::version; - use super::*; #[tokio::test] @@ -738,231 +463,9 @@ mod test { assert_eq!(err.to_string(), "Too many redirects."); } - #[test] - fn test_resolve_url_from_location_full_1() { - let url = "http://deno.land".parse::().unwrap(); - let new_uri = resolve_url_from_location(&url, "http://golang.org"); - assert_eq!(new_uri.host_str().unwrap(), "golang.org"); - } - - #[test] - fn test_resolve_url_from_location_full_2() { - let url = "https://deno.land".parse::().unwrap(); - let new_uri = resolve_url_from_location(&url, "https://golang.org"); - assert_eq!(new_uri.host_str().unwrap(), "golang.org"); - } - - #[test] - fn test_resolve_url_from_location_relative_1() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US"); - assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org"); - assert_eq!(new_uri.path(), "/en-US"); - } - - #[test] - fn test_resolve_url_from_location_relative_2() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_url_from_location(&url, "/y"); - assert_eq!(new_uri.host_str().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/y"); - } - - #[test] - fn test_resolve_url_from_location_relative_3() { - let url = "http://deno.land/x".parse::().unwrap(); - let new_uri = resolve_url_from_location(&url, "z"); - assert_eq!(new_uri.host_str().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/z"); - } - - fn create_test_client() -> HttpClient { - HttpClient::new( - create_http_client("test_client", CreateHttpClientOptions::default()) - .unwrap(), - ) - } - - #[tokio::test] - async fn test_fetch_string() { - let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server - let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(headers.get("content-type").unwrap(), "application/json"); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } - } - - #[tokio::test] - async fn test_fetch_gzip() { - let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server - let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped") - .unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/javascript" - ); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } - } - - #[tokio::test] - async fn test_fetch_with_etag() { - let _http_server_guard = test_util::http_server(); - let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url: url.clone(), - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/typescript" - ); - assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); - } else { - panic!(); - } - - let res = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: Some("33a64df551425fcc55e".to_string()), - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - assert_eq!(res.unwrap(), FetchOnceResult::NotModified); - } - - #[tokio::test] - async fn test_fetch_brotli() { - let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server - let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli") - .unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/javascript" - ); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } - } - - #[tokio::test] - async fn test_fetch_accept() { - let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server - let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: Some("application/json".to_string()), - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, _)) = result { - assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes()); - } else { - panic!(); - } - } - - #[tokio::test] - async fn test_fetch_no_follow_with_redirect() { - let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server - let url = Url::parse("http://127.0.0.1:4546/assets/fixture.json").unwrap(); - // Dns resolver substitutes `127.0.0.1` with `localhost` - let target_url = - Url::parse("http://localhost:4545/assets/fixture.json").unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Redirect(url, _)) = result { - assert_eq!(url, target_url); - } else { - panic!(); - } - } - #[tokio::test] async fn test_fetch_with_cafile_string() { let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap(); let client = HttpClient::new( @@ -978,24 +481,15 @@ mod test { ) .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(headers.get("content-type").unwrap(), "application/json"); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } + let response = client.send(&url, Default::default()).await.unwrap(); + assert!(response.status().is_success()); + let (parts, body) = response.into_parts(); + let headers = parts.headers; + let body = body.collect().await.unwrap().to_bytes(); + assert!(!body.is_empty()); + assert_eq!(headers.get("content-type").unwrap(), "application/json"); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } static PUBLIC_HTTPS_URLS: &[&str] = &[ @@ -1026,34 +520,15 @@ mod test { .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - + let result = client.send(&url, Default::default()).await; match result { - Err(_) => { - eprintln!("Fetch error: {result:?}"); - continue; + Ok(response) if response.status().is_success() => { + return; // success } - Ok( - FetchOnceResult::Code(..) - | FetchOnceResult::NotModified - | FetchOnceResult::Redirect(..), - ) => return, - Ok( - FetchOnceResult::RequestError(_) | FetchOnceResult::ServerError(_), - ) => { - eprintln!("HTTP error: {result:?}"); - continue; + _ => { + // keep going } - }; + } } // Use 1.1.1.1 and 8.8.8.8 as our last-ditch internet check @@ -1089,42 +564,13 @@ mod test { .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - - match result { - Err(_) => { - eprintln!("Fetch error (expected): {result:?}"); - return; - } - Ok( - FetchOnceResult::Code(..) - | FetchOnceResult::NotModified - | FetchOnceResult::Redirect(..), - ) => { - panic!("Should not have successfully fetched a URL"); - } - Ok( - FetchOnceResult::RequestError(_) | FetchOnceResult::ServerError(_), - ) => { - eprintln!("HTTP error (expected): {result:?}"); - return; - } - }; + let result = client.send(&url, HeaderMap::new()).await; + assert!(result.is_err() || !result.unwrap().status().is_success()); } #[tokio::test] async fn test_fetch_with_cafile_gzip() { let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/run/import_compression/gziped") .unwrap(); @@ -1143,27 +589,18 @@ mod test { ) .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/javascript" - ); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } + let response = client.send(&url, Default::default()).await.unwrap(); + assert!(response.status().is_success()); + let (parts, body) = response.into_parts(); + let headers = parts.headers; + let body = body.collect().await.unwrap().to_bytes().to_vec(); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/javascript" + ); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } #[tokio::test] @@ -1185,46 +622,29 @@ mod test { ) .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url: url.clone(), - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/typescript" - ); - assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } + let response = client.send(&url, Default::default()).await.unwrap(); + assert!(response.status().is_success()); + let (parts, body) = response.into_parts(); + let headers = parts.headers; + let body = body.collect().await.unwrap().to_bytes().to_vec(); + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/typescript" + ); + assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); + assert_eq!(headers.get("x-typescript-types"), None); - let res = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: Some("33a64df551425fcc55e".to_string()), - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - assert_eq!(res.unwrap(), FetchOnceResult::NotModified); + let mut headers = HeaderMap::new(); + headers.insert("If-None-Match", "33a64df551425fcc55e".parse().unwrap()); + let res = client.send(&url, headers).await.unwrap(); + assert_eq!(res.status(), StatusCode::NOT_MODIFIED); } #[tokio::test] async fn test_fetch_with_cafile_brotli() { let _http_server_guard = test_util::http_server(); - // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/run/import_compression/brotli") .unwrap(); @@ -1243,93 +663,18 @@ mod test { ) .unwrap(), ); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - if let Ok(FetchOnceResult::Code(body, headers)) = result { - assert!(!body.is_empty()); - assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/javascript" - ); - assert_eq!(headers.get("etag"), None); - assert_eq!(headers.get("x-typescript-types"), None); - } else { - panic!(); - } - } - - #[tokio::test] - async fn bad_redirect() { - let _g = test_util::http_server(); - let url_str = "http://127.0.0.1:4545/bad_redirect"; - let url = Url::parse(url_str).unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - assert!(result.is_err()); - let err = result.unwrap_err(); - // Check that the error message contains the original URL - assert!(err.to_string().contains(url_str)); - } - - #[tokio::test] - async fn server_error() { - let _g = test_util::http_server(); - let url_str = "http://127.0.0.1:4545/server_error"; - let url = Url::parse(url_str).unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - - if let Ok(FetchOnceResult::ServerError(status)) = result { - assert_eq!(status, 500); - } else { - panic!(); - } - } - - #[tokio::test] - async fn request_error() { - let _g = test_util::http_server(); - let url_str = "http://127.0.0.1:9999/"; - let url = Url::parse(url_str).unwrap(); - let client = create_test_client(); - let result = client - .fetch_no_follow(FetchOnceArgs { - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - maybe_progress_guard: None, - maybe_auth: None, - }) - .await; - - assert!(matches!(result, Ok(FetchOnceResult::RequestError(_)))); + let response = client.send(&url, Default::default()).await.unwrap(); + assert!(response.status().is_success()); + let (parts, body) = response.into_parts(); + let headers = parts.headers; + let body = body.collect().await.unwrap().to_bytes().to_vec(); + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/javascript" + ); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } } diff --git a/cli/jsr.rs b/cli/jsr.rs index 767d304d60..acfbb1c8e2 100644 --- a/cli/jsr.rs +++ b/cli/jsr.rs @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::args::jsr_url; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use dashmap::DashMap; use deno_core::serde_json; use deno_graph::packages::JsrPackageInfo; @@ -19,11 +19,11 @@ pub struct JsrFetchResolver { /// It can be large and we don't want to store it. info_by_nv: DashMap>>, info_by_name: DashMap>>, - file_fetcher: Arc, + file_fetcher: Arc, } impl JsrFetchResolver { - pub fn new(file_fetcher: Arc) -> Self { + pub fn new(file_fetcher: Arc) -> Self { Self { nv_by_req: Default::default(), info_by_nv: Default::default(), diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 47e36d1328..3efebe63b1 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -63,7 +63,7 @@ use crate::args::ConfigFile; use crate::args::LintFlags; use crate::args::LintOptions; use crate::cache::FastInsecureHasher; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::lsp::logging::lsp_warn; use crate::resolver::CliSloppyImportsResolver; use crate::resolver::SloppyImportsCachedFs; @@ -1218,7 +1218,7 @@ impl ConfigData { specified_config: Option<&Path>, scope: &ModuleSpecifier, settings: &Settings, - file_fetcher: &Arc, + file_fetcher: &Arc, // sync requirement is because the lsp requires sync cached_deno_config_fs: &(dyn DenoConfigFs + Sync), deno_json_cache: &(dyn DenoJsonCache + Sync), @@ -1313,7 +1313,7 @@ impl ConfigData { member_dir: Arc, scope: Arc, settings: &Settings, - file_fetcher: Option<&Arc>, + file_fetcher: Option<&Arc>, ) -> Self { let (settings, workspace_folder) = settings.get_for_specifier(&scope); let mut watched_files = HashMap::with_capacity(10); @@ -1834,7 +1834,7 @@ impl ConfigTree { &mut self, settings: &Settings, workspace_files: &IndexSet, - file_fetcher: &Arc, + file_fetcher: &Arc, ) { lsp_log!("Refreshing configuration tree..."); // since we're resolving a workspace multiple times in different diff --git a/cli/lsp/jsr.rs b/cli/lsp/jsr.rs index ab570f6348..1d012b42f0 100644 --- a/cli/lsp/jsr.rs +++ b/cli/lsp/jsr.rs @@ -2,7 +2,8 @@ use crate::args::jsr_api_url; use crate::args::jsr_url; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; +use crate::file_fetcher::TextDecodedFile; use crate::jsr::partial_jsr_package_version_info_from_slice; use crate::jsr::JsrFetchResolver; use dashmap::DashMap; @@ -267,7 +268,7 @@ fn read_cached_url( #[derive(Debug)] pub struct CliJsrSearchApi { - file_fetcher: Arc, + file_fetcher: Arc, resolver: JsrFetchResolver, search_cache: DashMap>>, versions_cache: DashMap>>, @@ -275,7 +276,7 @@ pub struct CliJsrSearchApi { } impl CliJsrSearchApi { - pub fn new(file_fetcher: Arc) -> Self { + pub fn new(file_fetcher: Arc) -> Self { let resolver = JsrFetchResolver::new(file_fetcher.clone()); Self { file_fetcher, @@ -309,10 +310,8 @@ impl PackageSearchApi for CliJsrSearchApi { let file_fetcher = self.file_fetcher.clone(); // spawn due to the lsp's `Send` requirement let file = deno_core::unsync::spawn(async move { - file_fetcher - .fetch_bypass_permissions(&search_url) - .await? - .into_text_decoded() + let file = file_fetcher.fetch_bypass_permissions(&search_url).await?; + TextDecodedFile::decode(file) }) .await??; let names = Arc::new(parse_jsr_search_response(&file.source)?); diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 839d28469e..3ffe4491e0 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_config::workspace::WorkspaceDirectory; use deno_config::workspace::WorkspaceDiscoverOptions; use deno_core::anyhow::anyhow; @@ -95,13 +96,12 @@ use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; use crate::args::has_flag_env_var; use crate::args::CaData; -use crate::args::CacheSetting; use crate::args::CliOptions; use crate::args::Flags; use crate::args::InternalFlags; use crate::args::UnstableFmtOptions; use crate::factory::CliFactory; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::graph_util; use crate::http_util::HttpClientProvider; use crate::lsp::config::ConfigWatchedFileType; @@ -958,15 +958,15 @@ impl Inner { } async fn refresh_config_tree(&mut self) { - let mut file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( self.cache.global().clone(), - CacheSetting::RespectHeaders, - true, self.http_client_provider.clone(), Default::default(), None, + true, + CacheSetting::RespectHeaders, + super::logging::lsp_log_level(), ); - file_fetcher.set_download_log_level(super::logging::lsp_log_level()); let file_fetcher = Arc::new(file_fetcher); self .config diff --git a/cli/lsp/npm.rs b/cli/lsp/npm.rs index 2decfc3429..18c7e2fccf 100644 --- a/cli/lsp/npm.rs +++ b/cli/lsp/npm.rs @@ -11,21 +11,22 @@ use serde::Deserialize; use std::sync::Arc; use crate::args::npm_registry_url; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; +use crate::file_fetcher::TextDecodedFile; use crate::npm::NpmFetchResolver; use super::search::PackageSearchApi; #[derive(Debug)] pub struct CliNpmSearchApi { - file_fetcher: Arc, + file_fetcher: Arc, resolver: NpmFetchResolver, search_cache: DashMap>>, versions_cache: DashMap>>, } impl CliNpmSearchApi { - pub fn new(file_fetcher: Arc) -> Self { + pub fn new(file_fetcher: Arc) -> Self { let resolver = NpmFetchResolver::new( file_fetcher.clone(), Arc::new(NpmRc::default().as_resolved(npm_registry_url()).unwrap()), @@ -57,10 +58,8 @@ impl PackageSearchApi for CliNpmSearchApi { .append_pair("text", &format!("{} boost-exact:false", query)); let file_fetcher = self.file_fetcher.clone(); let file = deno_core::unsync::spawn(async move { - file_fetcher - .fetch_bypass_permissions(&search_url) - .await? - .into_text_decoded() + let file = file_fetcher.fetch_bypass_permissions(&search_url).await?; + TextDecodedFile::decode(file) }) .await??; let names = Arc::new(parse_npm_search_response(&file.source)?); diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index ade353e683..067f201829 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -12,14 +12,15 @@ use super::path_to_regex::StringOrNumber; use super::path_to_regex::StringOrVec; use super::path_to_regex::Token; -use crate::args::CacheSetting; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; +use crate::file_fetcher::CliFileFetcher; use crate::file_fetcher::FetchOptions; use crate::file_fetcher::FetchPermissionsOptionRef; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::TextDecodedFile; use crate::http_util::HttpClientProvider; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::serde::Deserialize; @@ -418,7 +419,7 @@ enum VariableItems { pub struct ModuleRegistry { origins: HashMap>, pub location: PathBuf, - pub file_fetcher: Arc, + pub file_fetcher: Arc, http_cache: Arc, } @@ -432,15 +433,15 @@ impl ModuleRegistry { location.clone(), crate::cache::RealDenoCacheEnv, )); - let mut file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( http_cache.clone(), - CacheSetting::RespectHeaders, - true, http_client_provider, Default::default(), None, + true, + CacheSetting::RespectHeaders, + super::logging::lsp_log_level(), ); - file_fetcher.set_download_log_level(super::logging::lsp_log_level()); Self { origins: HashMap::new(), @@ -479,13 +480,15 @@ impl ModuleRegistry { let specifier = specifier.clone(); async move { file_fetcher - .fetch_with_options(FetchOptions { - specifier: &specifier, - permissions: FetchPermissionsOptionRef::AllowAll, - maybe_auth: None, - maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), - maybe_cache_setting: None, - }) + .fetch_with_options( + &specifier, +FetchPermissionsOptionRef::AllowAll, + FetchOptions { + maybe_auth: None, + maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), + maybe_cache_setting: None, + } + ) .await } }).await?; @@ -500,7 +503,7 @@ impl ModuleRegistry { ); self.http_cache.set(specifier, headers_map, &[])?; } - let file = fetch_result?.into_text_decoded()?; + let file = TextDecodedFile::decode(fetch_result?)?; let config: RegistryConfigurationJson = serde_json::from_str(&file.source)?; validate_config(&config)?; Ok(config.registries) @@ -584,12 +587,11 @@ impl ModuleRegistry { // spawn due to the lsp's `Send` requirement let file = deno_core::unsync::spawn({ async move { - file_fetcher + let file = file_fetcher .fetch_bypass_permissions(&endpoint) .await - .ok()? - .into_text_decoded() - .ok() + .ok()?; + TextDecodedFile::decode(file).ok() } }) .await @@ -983,12 +985,11 @@ impl ModuleRegistry { let file_fetcher = self.file_fetcher.clone(); // spawn due to the lsp's `Send` requirement let file = deno_core::unsync::spawn(async move { - file_fetcher + let file = file_fetcher .fetch_bypass_permissions(&specifier) .await - .ok()? - .into_text_decoded() - .ok() + .ok()?; + TextDecodedFile::decode(file).ok() }) .await .ok()??; @@ -1049,7 +1050,7 @@ impl ModuleRegistry { let file_fetcher = self.file_fetcher.clone(); let specifier = specifier.clone(); async move { - file_fetcher + let file = file_fetcher .fetch_bypass_permissions(&specifier) .await .map_err(|err| { @@ -1058,9 +1059,8 @@ impl ModuleRegistry { specifier, err ); }) - .ok()? - .into_text_decoded() - .ok() + .ok()?; + TextDecodedFile::decode(file).ok() } }) .await @@ -1095,7 +1095,7 @@ impl ModuleRegistry { let file_fetcher = self.file_fetcher.clone(); let specifier = specifier.clone(); async move { - file_fetcher + let file = file_fetcher .fetch_bypass_permissions(&specifier) .await .map_err(|err| { @@ -1104,9 +1104,8 @@ impl ModuleRegistry { specifier, err ); }) - .ok()? - .into_text_decoded() - .ok() + .ok()?; + TextDecodedFile::decode(file).ok() } }) .await diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 28c7b04fc9..482f2ddb40 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -2,6 +2,7 @@ use dashmap::DashMap; use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_cache_dir::npm::NpmCacheDir; use deno_cache_dir::HttpCache; use deno_config::deno_json::JsxImportSourceConfig; @@ -39,7 +40,6 @@ use std::sync::Arc; use super::cache::LspCache; use super::jsr::JsrCacheResolver; use crate::args::create_default_npmrc; -use crate::args::CacheSetting; use crate::args::CliLockfile; use crate::args::NpmInstallDepsProvider; use crate::cache::DenoCacheEnvFsAdapter; diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index fddcd6e738..6ae6265688 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -5516,7 +5516,6 @@ impl TscRequest { mod tests { use super::*; use crate::cache::HttpCache; - use crate::http_util::HeadersMap; use crate::lsp::cache::LspCache; use crate::lsp::config::Config; use crate::lsp::config::WorkspaceSettings; @@ -5953,7 +5952,7 @@ mod tests { .global() .set( &specifier_dep, - HeadersMap::default(), + Default::default(), b"export const b = \"b\";\n", ) .unwrap(); @@ -5992,7 +5991,7 @@ mod tests { .global() .set( &specifier_dep, - HeadersMap::default(), + Default::default(), b"export const b = \"b\";\n\nexport const a = \"b\";\n", ) .unwrap(); diff --git a/cli/main.rs b/cli/main.rs index 0594739fd8..d68f27146b 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. mod args; -mod auth_tokens; mod cache; mod cdp; mod emit; diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 18142bd0e7..cba54b044c 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -8,7 +8,6 @@ mod standalone; mod args; -mod auth_tokens; mod cache; mod emit; mod errors; diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 2c6e6d318a..4545800e99 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -20,6 +20,7 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_npm_cache::NpmCacheSetting; use deno_resolver::npm::CliNpmReqResolver; use deno_runtime::colors; use deno_runtime::deno_fs::FileSystem; @@ -70,7 +71,7 @@ pub struct CliManagedNpmResolverCreateOptions { pub fs: Arc, pub http_client_provider: Arc, pub npm_cache_dir: Arc, - pub cache_setting: crate::args::CacheSetting, + pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting, pub text_only_progress_bar: crate::util::progress_bar::ProgressBar, pub maybe_node_modules_path: Option, pub npm_system_info: NpmSystemInfo, @@ -203,7 +204,7 @@ fn create_cache( ) -> Arc { Arc::new(CliNpmCache::new( options.npm_cache_dir.clone(), - options.cache_setting.as_npm_cache_setting(), + NpmCacheSetting::from_cache_setting(&options.cache_setting), env, options.npmrc.clone(), )) diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index b39e0a340d..312ea2055b 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -28,7 +28,7 @@ use managed::create_managed_in_npm_pkg_checker; use node_resolver::InNpmPackageChecker; use node_resolver::NpmPackageFolderResolver; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; use crate::util::fs::atomic_write_file_with_retries_and_fs; use crate::util::fs::hard_link_dir_recursive; @@ -115,14 +115,14 @@ impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv { .download_with_progress_and_retries(url, maybe_auth_header, &guard) .await .map_err(|err| { - use crate::http_util::DownloadError::*; - let status_code = match &err { + use crate::http_util::DownloadErrorKind::*; + let status_code = match err.as_kind() { Fetch { .. } | UrlParse { .. } | HttpParse { .. } | Json { .. } | ToStr { .. } - | NoRedirectHeader { .. } + | RedirectHeaderParse { .. } | TooManyRedirects => None, BadResponse(bad_response_error) => { Some(bad_response_error.status_code) @@ -232,13 +232,13 @@ pub trait CliNpmResolver: NpmPackageFolderResolver + CliNpmReqResolver { pub struct NpmFetchResolver { nv_by_req: DashMap>, info_by_name: DashMap>>, - file_fetcher: Arc, + file_fetcher: Arc, npmrc: Arc, } impl NpmFetchResolver { pub fn new( - file_fetcher: Arc, + file_fetcher: Arc, npmrc: Arc, ) -> Self { Self { diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 85a22cf837..2ed52010fb 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -71,7 +71,7 @@ use crate::args::UnstableConfig; use crate::cache::DenoDir; use crate::cache::FastInsecureHasher; use crate::emit::Emitter; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; @@ -390,7 +390,7 @@ pub struct DenoCompileBinaryWriter<'a> { cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, - file_fetcher: &'a FileFetcher, + file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, @@ -404,7 +404,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, - file_fetcher: &'a FileFetcher, + file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 22e0b6d115..08ee5ba11a 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -9,6 +9,7 @@ use binary::StandaloneData; use binary::StandaloneModules; use code_cache::DenoCompileCodeCache; use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::MappedResolution; use deno_config::workspace::MappedResolutionError; @@ -64,7 +65,6 @@ use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; use crate::args::npm_pkg_req_ref_to_binary_command; use crate::args::CaData; -use crate::args::CacheSetting; use crate::args::NpmInstallDepsProvider; use crate::args::StorageKeyResolver; use crate::cache::Caches; diff --git a/cli/tools/check.rs b/cli/tools/check.rs index ad5c7c3ab1..9af084806f 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -64,7 +64,7 @@ pub async fn check( let file = file_fetcher.fetch(&s, root_permissions).await?; let snippet_files = extract::extract_snippet_files(file)?; for snippet_file in snippet_files { - specifiers_for_typecheck.push(snippet_file.specifier.clone()); + specifiers_for_typecheck.push(snippet_file.url.clone()); file_fetcher.insert_memory_files(snippet_file); } } diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 2a554c1335..624fa76bf6 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -6,6 +6,7 @@ use crate::args::FileFlags; use crate::args::Flags; use crate::cdp; use crate::factory::CliFactory; +use crate::file_fetcher::TextDecodedFile; use crate::tools::fmt::format_json; use crate::tools::test::is_supported_test_path; use crate::util::text_encoding::source_map_from_code; @@ -559,6 +560,12 @@ pub fn cover_files( }, None => None, }; + let get_message = |specifier: &ModuleSpecifier| -> String { + format!( + "Failed to fetch \"{}\" from cache. Before generating coverage report, run `deno test --coverage` to ensure consistent state.", + specifier, + ) + }; for script_coverage in script_coverages { let module_specifier = deno_core::resolve_url_or_path( @@ -566,21 +573,14 @@ pub fn cover_files( cli_options.initial_cwd(), )?; - let maybe_file = if module_specifier.scheme() == "file" { - file_fetcher.get_source(&module_specifier) - } else { - file_fetcher - .fetch_cached(&module_specifier, 10) - .with_context(|| { - format!("Failed to fetch \"{module_specifier}\" from cache.") - })? + let maybe_file_result = file_fetcher + .get_cached_source_or_local(&module_specifier) + .map_err(AnyError::from); + let file = match maybe_file_result { + Ok(Some(file)) => TextDecodedFile::decode(file)?, + Ok(None) => return Err(anyhow!("{}", get_message(&module_specifier))), + Err(err) => return Err(err).context(get_message(&module_specifier)), }; - let file = maybe_file.ok_or_else(|| { - anyhow!("Failed to fetch \"{}\" from cache. - Before generating coverage report, run `deno test --coverage` to ensure consistent state.", - module_specifier - ) - })?.into_text_decoded()?; let original_source = file.source.clone(); // Check if file was transpiled diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index d7c484beba..dac7340d40 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -3,7 +3,6 @@ use crate::args::resolve_no_prompt; use crate::args::AddFlags; use crate::args::CaData; -use crate::args::CacheSetting; use crate::args::ConfigFlag; use crate::args::Flags; use crate::args::InstallFlags; @@ -13,13 +12,14 @@ use crate::args::TypeCheckMode; use crate::args::UninstallFlags; use crate::args::UninstallKind; use crate::factory::CliFactory; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::graph_container::ModuleGraphContainer; use crate::http_util::HttpClientProvider; use crate::jsr::JsrFetchResolver; use crate::npm::NpmFetchResolver; use crate::util::fs::canonicalize_path_maybe_not_exists; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::generic_error; @@ -361,18 +361,18 @@ async fn install_global( let cli_options = factory.cli_options()?; let http_client = factory.http_client_provider(); let deps_http_cache = factory.global_http_cache()?; - let mut deps_file_fetcher = FileFetcher::new( + let deps_file_fetcher = CliFileFetcher::new( deps_http_cache.clone(), - CacheSetting::ReloadAll, - true, http_client.clone(), Default::default(), None, + true, + CacheSetting::ReloadAll, + log::Level::Trace, ); let npmrc = factory.cli_options().unwrap().npmrc(); - deps_file_fetcher.set_download_log_level(log::Level::Trace); let deps_file_fetcher = Arc::new(deps_file_fetcher); let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone())); let npm_resolver = Arc::new(NpmFetchResolver::new( diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index 6f89ec7aae..791e54c67c 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -4,6 +4,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -23,12 +24,11 @@ use jsonc_parser::cst::CstRootNode; use jsonc_parser::json; use crate::args::AddFlags; -use crate::args::CacheSetting; use crate::args::CliOptions; use crate::args::Flags; use crate::args::RemoveFlags; use crate::factory::CliFactory; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::jsr::JsrFetchResolver; use crate::npm::NpmFetchResolver; @@ -411,18 +411,18 @@ pub async fn add( let http_client = cli_factory.http_client_provider(); let deps_http_cache = cli_factory.global_http_cache()?; - let mut deps_file_fetcher = FileFetcher::new( + let deps_file_fetcher = CliFileFetcher::new( deps_http_cache.clone(), - CacheSetting::ReloadAll, - true, http_client.clone(), Default::default(), None, + true, + CacheSetting::ReloadAll, + log::Level::Trace, ); let npmrc = cli_factory.cli_options().unwrap().npmrc(); - deps_file_fetcher.set_download_log_level(log::Level::Trace); let deps_file_fetcher = Arc::new(deps_file_fetcher); let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone())); let npm_resolver = diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index aef65a5de0..f767eb1522 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::sync::Arc; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_semver::package::PackageNv; @@ -10,12 +11,11 @@ use deno_semver::package::PackageReq; use deno_semver::VersionReq; use deno_terminal::colors; -use crate::args::CacheSetting; use crate::args::CliOptions; use crate::args::Flags; use crate::args::OutdatedFlags; use crate::factory::CliFactory; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::jsr::JsrFetchResolver; use crate::npm::NpmFetchResolver; use crate::tools::registry::pm::deps::DepKind; @@ -181,15 +181,15 @@ pub async fn outdated( let workspace = cli_options.workspace(); let http_client = factory.http_client_provider(); let deps_http_cache = factory.global_http_cache()?; - let mut file_fetcher = FileFetcher::new( + let file_fetcher = CliFileFetcher::new( deps_http_cache.clone(), - CacheSetting::RespectHeaders, - true, http_client.clone(), Default::default(), None, + true, + CacheSetting::RespectHeaders, + log::Level::Trace, ); - file_fetcher.set_download_log_level(log::Level::Trace); let file_fetcher = Arc::new(file_fetcher); let npm_fetch_resolver = Arc::new(NpmFetchResolver::new( file_fetcher.clone(), diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index a303046879..9fb4624fa4 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -11,7 +11,8 @@ use crate::args::ReplFlags; use crate::cdp; use crate::colors; use crate::factory::CliFactory; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; +use crate::file_fetcher::TextDecodedFile; use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::serde_json; @@ -143,7 +144,7 @@ async fn read_line_and_poll( async fn read_eval_file( cli_options: &CliOptions, - file_fetcher: &FileFetcher, + file_fetcher: &CliFileFetcher, eval_file: &str, ) -> Result, AnyError> { let specifier = @@ -151,7 +152,7 @@ async fn read_eval_file( let file = file_fetcher.fetch_bypass_permissions(&specifier).await?; - Ok(file.into_text_decoded()?.source) + Ok(TextDecodedFile::decode(file)?.source) } #[allow(clippy::print_stdout)] diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index d3f7b093d4..cd7d1dd6c4 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -3,6 +3,7 @@ use std::io::Read; use std::sync::Arc; +use deno_cache_dir::file_fetcher::File; use deno_config::deno_json::NodeModulesDirMode; use deno_core::error::AnyError; use deno_runtime::WorkerExecutionMode; @@ -11,7 +12,6 @@ use crate::args::EvalFlags; use crate::args::Flags; use crate::args::WatchFlagsWithPaths; use crate::factory::CliFactory; -use crate::file_fetcher::File; use crate::util; use crate::util::file_watcher::WatcherRestartMode; @@ -97,7 +97,7 @@ pub async fn run_from_stdin(flags: Arc) -> Result { // Save a fake file into file fetcher cache // to allow module access by TS compiler file_fetcher.insert_memory_files(File { - specifier: main_module.clone(), + url: main_module.clone(), maybe_headers: None, source: source.into(), }); @@ -184,7 +184,7 @@ pub async fn eval_command( // Save a fake file into file fetcher cache // to allow module access by TS compiler. file_fetcher.insert_memory_files(File { - specifier: main_module.clone(), + url: main_module.clone(), maybe_headers: None, source: source_code.into_bytes().into(), }); diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index 2e46bdd4da..48bf42c9c7 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -7,8 +7,7 @@ use crate::args::TestReporterConfig; use crate::colors; use crate::display; use crate::factory::CliFactory; -use crate::file_fetcher::File; -use crate::file_fetcher::FileFetcher; +use crate::file_fetcher::CliFileFetcher; use crate::graph_util::has_graph_root_local_dependent_changed; use crate::ops; use crate::util::extract::extract_doc_tests; @@ -21,6 +20,7 @@ use crate::worker::CliMainWorkerFactory; use crate::worker::CoverageCollector; use deno_ast::MediaType; +use deno_cache_dir::file_fetcher::File; use deno_config::glob::FilePatterns; use deno_config::glob::WalkEntry; use deno_core::anyhow; @@ -1514,7 +1514,7 @@ fn collect_specifiers_with_test_mode( /// as well. async fn fetch_specifiers_with_test_mode( cli_options: &CliOptions, - file_fetcher: &FileFetcher, + file_fetcher: &CliFileFetcher, member_patterns: impl Iterator, doc: &bool, ) -> Result, AnyError> { @@ -1822,7 +1822,7 @@ pub async fn run_tests_with_watch( /// Extracts doc tests from files specified by the given specifiers. async fn get_doc_tests( specifiers_with_mode: &[(Url, TestMode)], - file_fetcher: &FileFetcher, + file_fetcher: &CliFileFetcher, ) -> Result, AnyError> { let specifiers_needing_extraction = specifiers_with_mode .iter() @@ -1847,7 +1847,7 @@ fn get_target_specifiers( specifiers_with_mode .into_iter() .filter_map(|(s, mode)| mode.needs_test_run().then_some(s)) - .chain(doc_tests.iter().map(|d| d.specifier.clone())) + .chain(doc_tests.iter().map(|d| d.url.clone())) .collect() } diff --git a/cli/util/extract.rs b/cli/util/extract.rs index be68202aa1..c4562060d8 100644 --- a/cli/util/extract.rs +++ b/cli/util/extract.rs @@ -13,6 +13,7 @@ use deno_ast::swc::visit::VisitMut; use deno_ast::swc::visit::VisitWith as _; use deno_ast::MediaType; use deno_ast::SourceRangedForSpanned as _; +use deno_cache_dir::file_fetcher::File; use deno_core::error::AnyError; use deno_core::ModuleSpecifier; use regex::Regex; @@ -20,7 +21,7 @@ use std::collections::BTreeSet; use std::fmt::Write as _; use std::sync::Arc; -use crate::file_fetcher::File; +use crate::file_fetcher::TextDecodedFile; use crate::util::path::mapped_specifier_for_tsc; /// Extracts doc tests from a given file, transforms them into pseudo test @@ -52,7 +53,7 @@ fn extract_inner( file: File, wrap_kind: WrapKind, ) -> Result, AnyError> { - let file = file.into_text_decoded()?; + let file = TextDecodedFile::decode(file)?; let exports = match deno_ast::parse_program(deno_ast::ParseParams { specifier: file.specifier.clone(), @@ -230,7 +231,7 @@ fn extract_files_from_regex_blocks( .unwrap_or(file_specifier); Some(File { - specifier: file_specifier, + url: file_specifier, maybe_headers: None, source: file_source.into_bytes().into(), }) @@ -558,7 +559,7 @@ fn generate_pseudo_file( exports: &ExportCollector, wrap_kind: WrapKind, ) -> Result { - let file = file.into_text_decoded()?; + let file = TextDecodedFile::decode(file)?; let parsed = deno_ast::parse_program(deno_ast::ParseParams { specifier: file.specifier.clone(), @@ -594,7 +595,7 @@ fn generate_pseudo_file( log::debug!("{}:\n{}", file.specifier, source); Ok(File { - specifier: file.specifier, + url: file.specifier, maybe_headers: None, source: source.into_bytes().into(), }) @@ -1199,14 +1200,14 @@ Deno.test("file:///main.ts$3-7.ts", async ()=>{ for test in tests { let file = File { - specifier: ModuleSpecifier::parse(test.input.specifier).unwrap(), + url: ModuleSpecifier::parse(test.input.specifier).unwrap(), maybe_headers: None, source: test.input.source.as_bytes().into(), }; let got_decoded = extract_doc_tests(file) .unwrap() .into_iter() - .map(|f| f.into_text_decoded().unwrap()) + .map(|f| TextDecodedFile::decode(f).unwrap()) .collect::>(); let expected = test .expected @@ -1435,14 +1436,14 @@ add('1', '2'); for test in tests { let file = File { - specifier: ModuleSpecifier::parse(test.input.specifier).unwrap(), + url: ModuleSpecifier::parse(test.input.specifier).unwrap(), maybe_headers: None, source: test.input.source.as_bytes().into(), }; let got_decoded = extract_snippet_files(file) .unwrap() .into_iter() - .map(|f| f.into_text_decoded().unwrap()) + .map(|f| TextDecodedFile::decode(f).unwrap()) .collect::>(); let expected = test .expected diff --git a/resolvers/npm_cache/lib.rs b/resolvers/npm_cache/lib.rs index 9f5424dc46..c16c29aaf2 100644 --- a/resolvers/npm_cache/lib.rs +++ b/resolvers/npm_cache/lib.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use anyhow::bail; use anyhow::Context; use anyhow::Error as AnyError; +use deno_cache_dir::file_fetcher::CacheSetting; use deno_cache_dir::npm::NpmCacheDir; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; @@ -90,6 +91,27 @@ pub enum NpmCacheSetting { } impl NpmCacheSetting { + pub fn from_cache_setting(cache_setting: &CacheSetting) -> NpmCacheSetting { + match cache_setting { + CacheSetting::Only => NpmCacheSetting::Only, + CacheSetting::ReloadAll => NpmCacheSetting::ReloadAll, + CacheSetting::ReloadSome(values) => { + if values.iter().any(|v| v == "npm:") { + NpmCacheSetting::ReloadAll + } else { + NpmCacheSetting::ReloadSome { + npm_package_names: values + .iter() + .filter_map(|v| v.strip_prefix("npm:")) + .map(|n| n.to_string()) + .collect(), + } + } + } + CacheSetting::RespectHeaders => panic!("not supported"), + CacheSetting::Use => NpmCacheSetting::Use, + } + } pub fn should_use_for_npm_package(&self, package_name: &str) -> bool { match self { NpmCacheSetting::ReloadAll => false, diff --git a/tests/integration/bench_tests.rs b/tests/integration/bench_tests.rs index d588f5b437..4ee029d648 100644 --- a/tests/integration/bench_tests.rs +++ b/tests/integration/bench_tests.rs @@ -43,7 +43,7 @@ fn conditionally_loads_type_graph() { .new_command() .args("bench --reload -L debug run/type_directives_js_main.js") .run(); - output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); + output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); let output = context .new_command() .args("bench --reload -L debug --no-check run/type_directives_js_main.js") diff --git a/tests/integration/cache_tests.rs b/tests/integration/cache_tests.rs index d9fb8e38e5..4cddae1af1 100644 --- a/tests/integration/cache_tests.rs +++ b/tests/integration/cache_tests.rs @@ -107,5 +107,5 @@ fn loads_type_graph() { .new_command() .args("cache --reload -L debug run/type_directives_js_main.js") .run(); - output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); + output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); } diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index f0b536aa22..77c0a46c5f 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -922,7 +922,7 @@ fn type_directives_js_main() { .new_command() .args("run --reload -L debug --check run/type_directives_js_main.js") .run(); - output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); + output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); let output = context .new_command() .args("run --reload -L debug run/type_directives_js_main.js") diff --git a/tests/integration/test_tests.rs b/tests/integration/test_tests.rs index 64857ae110..ca83682833 100644 --- a/tests/integration/test_tests.rs +++ b/tests/integration/test_tests.rs @@ -111,7 +111,7 @@ fn conditionally_loads_type_graph() { .new_command() .args("test --reload -L debug run/type_directives_js_main.js") .run(); - output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow_with_options - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); + output.assert_matches_text("[WILDCARD] - FileFetcher::fetch_no_follow - specifier: file:///[WILDCARD]/subdir/type_reference.d.ts[WILDCARD]"); let output = context .new_command() .args("test --reload -L debug --no-check run/type_directives_js_main.js") diff --git a/tests/specs/cert/localhost_unsafe_ssl/localhost_unsafe_ssl.ts.out b/tests/specs/cert/localhost_unsafe_ssl/localhost_unsafe_ssl.ts.out index c7bdfde0ed..ffb84ebfde 100644 --- a/tests/specs/cert/localhost_unsafe_ssl/localhost_unsafe_ssl.ts.out +++ b/tests/specs/cert/localhost_unsafe_ssl/localhost_unsafe_ssl.ts.out @@ -1,3 +1,6 @@ DANGER: TLS certificate validation is disabled for: deno.land -error: Import 'https://localhost:5545/subdir/mod2.ts' failed: error sending request for url (https://localhost:5545/subdir/mod2.ts): client error[WILDCARD] - at file:///[WILDCARD]/cafile_url_imports.ts:[WILDCARD] +error: Import 'https://localhost:5545/subdir/mod2.ts' failed. + 0: error sending request for url (https://localhost:5545/subdir/mod2.ts): client error (Connect): invalid peer certificate: UnknownIssuer + 1: client error (Connect) + 2: invalid peer certificate: UnknownIssuer + at file:///[WILDLINE]/cafile_url_imports.ts:[WILDLINE] diff --git a/tests/specs/run/jsx_import_source/__test__.jsonc b/tests/specs/run/jsx_import_source/__test__.jsonc index cbda2dd32e..0350df7f17 100644 --- a/tests/specs/run/jsx_import_source/__test__.jsonc +++ b/tests/specs/run/jsx_import_source/__test__.jsonc @@ -19,6 +19,7 @@ "output": "jsx_import_source_dev.out" }, "jsx_import_source_pragma_with_config_vendor_dir": { + "tempDir": true, "args": "run --allow-import --reload --config jsx/deno-jsx.jsonc --no-lock --vendor jsx_import_source_pragma.tsx", "output": "jsx_import_source.out" }, diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 0b1d99aeb9..4345c27cde 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -577,11 +577,6 @@ async fn main_server( ); Ok(res) } - (_, "/bad_redirect") => { - let mut res = Response::new(empty_body()); - *res.status_mut() = StatusCode::FOUND; - Ok(res) - } (_, "/server_error") => { let mut res = Response::new(empty_body()); *res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;