diff --git a/Cargo.lock b/Cargo.lock index b83f3b4425..4be228f921 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,18 +853,16 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" name = "cli_tests" version = "0.0.0" dependencies = [ + "anyhow", "bytes", "chrono", - "deno_ast", "deno_bench_util", "deno_cache_dir", - "deno_core", - "deno_fetch", "deno_lockfile", "deno_semver", "deno_terminal 0.2.0", - "deno_tls", "deno_tower_lsp", + "deno_unsync", "fastwebsockets", "file_test_runner", "flaky_test", @@ -881,7 +879,11 @@ dependencies = [ "pretty_assertions", "regex", "reqwest", + "rustls", + "rustls-pemfile", + "rustls-tokio-stream", "serde", + "serde_json", "sys_traits", "test_server", "tokio", @@ -1284,6 +1286,7 @@ dependencies = [ "deno_resolver", "deno_runtime", "deno_semver", + "deno_snapshots", "deno_task_shell", "deno_telemetry", "deno_terminal 0.2.0", @@ -1295,7 +1298,6 @@ dependencies = [ "dprint-plugin-jupyter", "dprint-plugin-markdown", "dprint-plugin-typescript", - "env_logger", "fancy-regex", "faster-hex", "flate2", @@ -1898,24 +1900,33 @@ dependencies = [ name = "deno_lib" version = "0.2.0" dependencies = [ - "deno_cache_dir", + "capacity_builder 0.5.0", + "deno_config", "deno_error", "deno_fs", + "deno_media_type", "deno_node", + "deno_npm", "deno_path_util", "deno_resolver", "deno_runtime", + "deno_semver", "deno_terminal 0.2.0", + "env_logger", "faster-hex", + "indexmap 2.3.0", + "libsui", "log", "node_resolver", "parking_lot", "ring", "serde", + "serde_json", "sys_traits", "test_server", "thiserror 2.0.3", "tokio", + "twox-hash", "url", ] @@ -1951,11 +1962,12 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a417f8bd3f1074185c4c8ccb6ea6261ae173781596cc358e68ad07aaac11009d" +checksum = "577fe2bbe04f3e9b1b7c6fac6a75101a9fbd611c50a6b68789e69f4d63dcb2b4" dependencies = [ "data-url", + "encoding_rs", "serde", "url", ] @@ -2028,7 +2040,6 @@ dependencies = [ "deno_fetch", "deno_fs", "deno_io", - "deno_media_type", "deno_net", "deno_package_json", "deno_path_util", @@ -2042,7 +2053,7 @@ dependencies = [ "ecdsa", "ed25519-dalek", "elliptic-curve", - "errno 0.2.8", + "errno", "faster-hex", "h2 0.4.4", "hkdf", @@ -2331,7 +2342,6 @@ dependencies = [ "dlopen2", "encoding_rs", "fastwebsockets", - "flate2", "http 1.1.0", "http-body-util", "hyper 0.14.28", @@ -2379,6 +2389,13 @@ dependencies = [ "url", ] +[[package]] +name = "deno_snapshots" +version = "0.1.0" +dependencies = [ + "deno_runtime", +] + [[package]] name = "deno_task_shell" version = "0.20.2" @@ -2662,6 +2679,43 @@ dependencies = [ "v8_valueserializer", ] +[[package]] +name = "denort" +version = "2.1.5" +dependencies = [ + "async-trait", + "bincode", + "deno_cache_dir", + "deno_config", + "deno_core", + "deno_error", + "deno_lib", + "deno_media_type", + "deno_npm", + "deno_package_json", + "deno_path_util", + "deno_resolver", + "deno_runtime", + "deno_semver", + "deno_snapshots", + "deno_terminal 0.2.0", + "import_map", + "indexmap 2.3.0", + "libsui", + "log", + "node_resolver", + "pretty_assertions", + "serde", + "serde_json", + "sys_traits", + "test_server", + "thiserror 2.0.3", + "tokio", + "tokio-util", + "twox-hash", + "url", +] + [[package]] name = "der" version = "0.7.9" @@ -3191,17 +3245,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - [[package]] name = "errno" version = "0.3.8" @@ -3212,16 +3255,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "error-code" version = "3.2.0" @@ -5222,7 +5255,6 @@ dependencies = [ "async-trait", "boxed_error", "deno_error", - "deno_media_type", "deno_package_json", "deno_path_util", "futures", @@ -5230,6 +5262,7 @@ dependencies = [ "once_cell", "path-clean", "regex", + "serde", "serde_json", "sys_traits", "thiserror 2.0.3", @@ -6633,7 +6666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ "bitflags 2.6.0", - "errno 0.3.8", + "errno", "libc", "linux-raw-sys", "windows-sys 0.52.0", diff --git a/Cargo.toml b/Cargo.toml index 4f3563ce47..6137505aa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ members = [ "bench_util", "cli", "cli/lib", + "cli/rt", + "cli/snapshot", "ext/broadcast_channel", "ext/cache", "ext/canvas", @@ -54,7 +56,7 @@ deno_core = { version = "0.331.0" } deno_bench_util = { version = "0.180.0", path = "./bench_util" } deno_config = { version = "=0.45.0", features = ["workspace", "sync"] } deno_lockfile = "=0.24.0" -deno_media_type = { version = "0.2.3", features = ["module_specifier"] } +deno_media_type = { version = "0.2.4", features = ["module_specifier"] } deno_npm = "=0.27.2" deno_path_util = "=0.3.0" deno_permissions = { version = "0.45.0", path = "./runtime/permissions" } @@ -100,6 +102,7 @@ deno_webstorage = { version = "0.181.0", path = "./ext/webstorage" } deno_lib = { version = "0.2.0", path = "./cli/lib" } deno_npm_cache = { version = "0.5.0", path = "./resolvers/npm_cache" } deno_resolver = { version = "0.17.0", path = "./resolvers/deno" } +deno_snapshots = { version = "0.1.0", path = "./cli/snapshot" } node_resolver = { version = "0.24.0", path = "./resolvers/node" } aes = "=0.8.3" @@ -154,6 +157,7 @@ ipnet = "2.3" jsonc-parser = { version = "=0.26.2", features = ["serde"] } lazy-regex = "3" libc = "0.2.168" +libsui = "0.5.0" libz-sys = { version = "1.1.20", default-features = false } log = { version = "0.4.20", features = ["kv"] } lsp-types = "=0.97.0" # used by tower-lsp and "proposed" feature is unstable in patch releases diff --git a/README.md b/README.md index 19d4fa8a12..ca71529e28 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ the deno mascot dinosaur standing in the rain -[Deno](https://www.deno.com) -([/ˈdiːnoʊ/](http://ipa-reader.xyz/?text=%CB%88di%CB%90no%CA%8A), pronounced +[Deno](https://deno.com) +([/ˈdiːnoʊ/](https://ipa-reader.com/?text=%CB%88di%CB%90no%CA%8A), pronounced `dee-no`) is a JavaScript, TypeScript, and WebAssembly runtime with secure defaults and a great developer experience. It's built on [V8](https://v8.dev/), [Rust](https://www.rust-lang.org/), and [Tokio](https://tokio.rs/). diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d71047cc63..29515b67b6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -16,11 +16,6 @@ name = "deno" path = "main.rs" doc = false -[[bin]] -name = "denort" -path = "mainrt.rs" -doc = false - [[test]] name = "integration" path = "integration_tests_runner.rs" @@ -49,7 +44,7 @@ dhat-heap = ["dhat"] upgrade = [] # A dev feature to disable creations and loading of snapshots in favor of # loading JS sources at runtime. -hmr = ["deno_runtime/hmr"] +hmr = ["deno_runtime/hmr", "deno_snapshots/disable"] # Vendor zlib as zlib-ng __vendored_zlib_ng = ["flate2/zlib-ng-compat", "libz-sys/zlib-ng"] @@ -60,10 +55,12 @@ lazy-regex.workspace = true serde.workspace = true serde_json.workspace = true zstd.workspace = true -glibc_version = "0.1.2" flate2 = { workspace = true, features = ["default"] } deno_error.workspace = true +[target.'cfg(unix)'.build-dependencies] +glibc_version = "0.1.2" + [target.'cfg(windows)'.build-dependencies] winapi.workspace = true winres.workspace = true @@ -86,10 +83,11 @@ deno_path_util.workspace = true deno_resolver = { workspace = true, features = ["sync"] } deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver.workspace = true +deno_snapshots = { workspace = true } deno_task_shell = "=0.20.2" deno_telemetry.workspace = true deno_terminal.workspace = true -libsui = "0.5.0" +libsui.workspace = true node_resolver.workspace = true anstream = "0.6.14" @@ -115,7 +113,6 @@ dprint-plugin-json = "=0.19.4" dprint-plugin-jupyter = "=0.1.5" dprint-plugin-markdown = "=0.17.8" dprint-plugin-typescript = "=0.93.3" -env_logger = "=0.10.0" fancy-regex = "=0.10.0" faster-hex.workspace = true # If you disable the default __vendored_zlib_ng feature above, you _must_ be able to link against `-lz`. @@ -156,7 +153,6 @@ rustyline-derive = "=0.7.0" serde.workspace = true serde_repr.workspace = true sha2.workspace = true -shell-escape = "=0.1.5" spki = { version = "0.7", features = ["pem"] } sqlformat = "=0.3.2" strsim = "0.11.1" @@ -185,6 +181,7 @@ winapi = { workspace = true, features = ["knownfolders", "mswsock", "objbase", " [target.'cfg(unix)'.dependencies] nix.workspace = true +shell-escape = "=0.1.5" [dev-dependencies] deno_bench_util.workspace = true diff --git a/cli/args/flags.rs b/cli/args/flags.rs index fb64b4eeaa..f86aa50186 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -31,6 +31,9 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_core::url::Url; use deno_graph::GraphKind; +use deno_lib::args::CaData; +use deno_lib::args::UnstableConfig; +use deno_lib::version::DENO_VERSION_INFO; use deno_path_util::normalize_path; use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::SysDescriptor; @@ -546,15 +549,6 @@ impl Default for TypeCheckMode { } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum CaData { - /// The string is a file path - File(String), - /// This variant is not exposed as an option in the CLI, it is used internally - /// for standalone binaries. - Bytes(Vec), -} - // Info needed to run NPM lifecycle scripts #[derive(Clone, Debug, Eq, PartialEq, Default)] pub struct LifecycleScriptsConfig { @@ -582,19 +576,6 @@ fn parse_packages_allowed_scripts(s: &str) -> Result { } } -#[derive( - Clone, Default, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize, -)] -pub struct UnstableConfig { - // TODO(bartlomieju): remove in Deno 2.5 - pub legacy_flag_enabled: bool, // --unstable - pub bare_node_builtins: bool, - pub detect_cjs: bool, - pub sloppy_imports: bool, - pub npm_lazy_caching: bool, - pub features: Vec, // --unstabe-kv --unstable-cron -} - #[derive(Clone, Debug, Eq, PartialEq, Default)] pub struct InternalFlags { /// Used when the language server is configured with an @@ -1484,14 +1465,15 @@ fn handle_repl_flags(flags: &mut Flags, repl_flags: ReplFlags) { } pub fn clap_root() -> Command { + debug_assert_eq!(DENO_VERSION_INFO.typescript, deno_snapshots::TS_VERSION); let long_version = format!( "{} ({}, {}, {})\nv8 {}\ntypescript {}", - crate::version::DENO_VERSION_INFO.deno, - crate::version::DENO_VERSION_INFO.release_channel.name(), + DENO_VERSION_INFO.deno, + DENO_VERSION_INFO.release_channel.name(), env!("PROFILE"), env!("TARGET"), deno_core::v8::VERSION_STRING, - crate::version::DENO_VERSION_INFO.typescript + DENO_VERSION_INFO.typescript ); run_args(Command::new("deno"), true) @@ -1507,7 +1489,7 @@ pub fn clap_root() -> Command { ) .color(ColorChoice::Auto) .term_width(800) - .version(crate::version::DENO_VERSION_INFO.deno) + .version(DENO_VERSION_INFO.deno) .long_version(long_version) .disable_version_flag(true) .disable_help_flag(true) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 46ada5440f..e4f332a8bc 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -10,10 +10,6 @@ mod package_json; use std::borrow::Cow; use std::collections::HashMap; use std::env; -use std::io::BufReader; -use std::io::Cursor; -use std::io::Read; -use std::io::Seek; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::path::Path; @@ -58,8 +54,12 @@ use deno_core::serde_json; use deno_core::url::Url; use deno_graph::GraphKind; pub use deno_json::check_warn_tsconfig; -use deno_lib::cache::DenoDirProvider; -use deno_lib::env::has_flag_env_var; +use deno_lib::args::has_flag_env_var; +use deno_lib::args::npm_pkg_req_ref_to_binary_command; +use deno_lib::args::CaData; +use deno_lib::args::NpmProcessStateKind; +use deno_lib::args::NPM_PROCESS_STATE; +use deno_lib::version::DENO_VERSION_INFO; use deno_lib::worker::StorageKeyResolver; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::NpmRc; @@ -68,16 +68,10 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_path_util::normalize_path; use deno_runtime::deno_permissions::PermissionsOptions; -use deno_runtime::deno_tls::deno_native_certs::load_native_certs; -use deno_runtime::deno_tls::rustls; -use deno_runtime::deno_tls::rustls::RootCertStore; -use deno_runtime::deno_tls::rustls_pemfile; -use deno_runtime::deno_tls::webpki_roots; use deno_runtime::inspector_server::InspectorServer; use deno_semver::npm::NpmPackageReqReference; use deno_semver::StackString; use deno_telemetry::OtelConfig; -use deno_telemetry::OtelRuntimeConfig; use deno_terminal::colors; use dotenvy::from_filename; pub use flags::*; @@ -88,15 +82,13 @@ pub use lockfile::CliLockfileReadFromPathOptions; use once_cell::sync::Lazy; pub use package_json::NpmInstallDepsProvider; pub use package_json::PackageJsonDepValueParseWithLocationError; -use serde::Deserialize; -use serde::Serialize; use sys_traits::EnvHomeDir; use thiserror::Error; +use crate::cache::DenoDirProvider; use crate::file_fetcher::CliFileFetcher; use crate::sys::CliSys; use crate::util::fs::canonicalize_path_maybe_not_exists; -use crate::version; pub fn npm_registry_url() -> &'static Url { static NPM_REGISTRY_DEFAULT_URL: Lazy = Lazy::new(|| { @@ -608,147 +600,6 @@ pub fn create_default_npmrc() -> Arc { }) } -#[derive(Error, Debug, Clone, deno_error::JsError)] -#[class(generic)] -pub enum RootCertStoreLoadError { - #[error( - "Unknown certificate store \"{0}\" specified (allowed: \"system,mozilla\")" - )] - UnknownStore(String), - #[error("Unable to add pem file to certificate store: {0}")] - FailedAddPemFile(String), - #[error("Failed opening CA file: {0}")] - CaFileOpenError(String), -} - -/// Create and populate a root cert store based on the passed options and -/// environment. -pub fn get_root_cert_store( - maybe_root_path: Option, - maybe_ca_stores: Option>, - maybe_ca_data: Option, -) -> Result { - let mut root_cert_store = RootCertStore::empty(); - let ca_stores: Vec = maybe_ca_stores - .or_else(|| { - let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?; - Some( - env_ca_store - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(), - ) - }) - .unwrap_or_else(|| vec!["mozilla".to_string()]); - - for store in ca_stores.iter() { - match store.as_str() { - "mozilla" => { - root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.to_vec()); - } - "system" => { - let roots = load_native_certs().expect("could not load platform certs"); - for root in roots { - if let Err(err) = root_cert_store - .add(rustls::pki_types::CertificateDer::from(root.0.clone())) - { - log::error!( - "{}", - colors::yellow(&format!( - "Unable to add system certificate to certificate store: {:?}", - err - )) - ); - let hex_encoded_root = faster_hex::hex_string(&root.0); - log::error!("{}", colors::gray(&hex_encoded_root)); - } - } - } - _ => { - return Err(RootCertStoreLoadError::UnknownStore(store.clone())); - } - } - } - - let ca_data = - maybe_ca_data.or_else(|| env::var("DENO_CERT").ok().map(CaData::File)); - if let Some(ca_data) = ca_data { - let result = match ca_data { - CaData::File(ca_file) => { - let ca_file = if let Some(root) = &maybe_root_path { - root.join(&ca_file) - } else { - PathBuf::from(ca_file) - }; - let certfile = std::fs::File::open(ca_file).map_err(|err| { - RootCertStoreLoadError::CaFileOpenError(err.to_string()) - })?; - let mut reader = BufReader::new(certfile); - rustls_pemfile::certs(&mut reader).collect::, _>>() - } - CaData::Bytes(data) => { - let mut reader = BufReader::new(Cursor::new(data)); - rustls_pemfile::certs(&mut reader).collect::, _>>() - } - }; - - match result { - Ok(certs) => { - root_cert_store.add_parsable_certificates(certs); - } - Err(e) => { - return Err(RootCertStoreLoadError::FailedAddPemFile(e.to_string())); - } - } - } - - Ok(root_cert_store) -} - -/// State provided to the process via an environment variable. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct NpmProcessState { - pub kind: NpmProcessStateKind, - pub local_node_modules_path: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub enum NpmProcessStateKind { - Snapshot(deno_npm::resolution::SerializedNpmResolutionSnapshot), - Byonm, -} - -static NPM_PROCESS_STATE: Lazy> = Lazy::new(|| { - use deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME; - let fd = std::env::var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME).ok()?; - std::env::remove_var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME); - let fd = fd.parse::().ok()?; - let mut file = { - use deno_runtime::deno_io::FromRawIoHandle; - unsafe { std::fs::File::from_raw_io_handle(fd as _) } - }; - let mut buf = Vec::new(); - // seek to beginning. after the file is written the position will be inherited by this subprocess, - // and also this file might have been read before - file.seek(std::io::SeekFrom::Start(0)).unwrap(); - file - .read_to_end(&mut buf) - .inspect_err(|e| { - log::error!("failed to read npm process state from fd {fd}: {e}"); - }) - .ok()?; - let state: NpmProcessState = serde_json::from_slice(&buf) - .inspect_err(|e| { - log::error!( - "failed to deserialize npm process state: {e} {}", - String::from_utf8_lossy(&buf) - ) - }) - .ok()?; - Some(state) -}); - /// Overrides for the options below that when set will /// use these values over the values derived from the /// CLI flags or config file. @@ -771,7 +622,7 @@ pub struct CliOptions { maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, overrides: CliOptionOverrides, pub start_dir: Arc, - pub deno_dir_provider: Arc>, + pub deno_dir_provider: Arc, } impl CliOptions { @@ -1294,7 +1145,7 @@ impl CliOptions { Ok(Some(InspectorServer::new( host, - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, )?)) } @@ -1884,7 +1735,7 @@ fn resolve_node_modules_folder( cwd: &Path, flags: &Flags, workspace: &Workspace, - deno_dir_provider: &Arc>, + deno_dir_provider: &Arc, ) -> Result, AnyError> { fn resolve_from_root(root_folder: &FolderConfigs, cwd: &Path) -> PathBuf { root_folder @@ -1993,15 +1844,6 @@ pub fn resolve_no_prompt(flags: &PermissionFlags) -> bool { flags.no_prompt || has_flag_env_var("DENO_NO_PROMPT") } -pub fn npm_pkg_req_ref_to_binary_command( - req_ref: &NpmPackageReqReference, -) -> String { - req_ref - .sub_path() - .map(|s| s.to_string()) - .unwrap_or_else(|| req_ref.req().name.to_string()) -} - pub fn config_to_deno_graph_workspace_member( config: &ConfigFile, ) -> Result { @@ -2062,13 +1904,6 @@ pub enum NpmCachingStrategy { Manual, } -pub fn otel_runtime_config() -> OtelRuntimeConfig { - OtelRuntimeConfig { - runtime_name: Cow::Borrowed("deno"), - runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), - } -} - #[cfg(test)] mod test { use pretty_assertions::assert_eq; diff --git a/cli/build.rs b/cli/build.rs index 590fee795d..62f7101942 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -5,7 +5,6 @@ use std::path::PathBuf; use deno_core::snapshot::*; use deno_runtime::*; -mod shared; mod ts { use std::collections::HashMap; @@ -310,57 +309,6 @@ mod ts { println!("cargo:rerun-if-changed={}", path.display()); } } - - pub(crate) fn version() -> String { - let file_text = std::fs::read_to_string("tsc/00_typescript.js").unwrap(); - let version_text = " version = \""; - for line in file_text.lines() { - if let Some(index) = line.find(version_text) { - let remaining_line = &line[index + version_text.len()..]; - return remaining_line[..remaining_line.find('"').unwrap()].to_string(); - } - } - panic!("Could not find ts version.") - } -} - -#[cfg(not(feature = "hmr"))] -fn create_cli_snapshot(snapshot_path: PathBuf) { - use deno_runtime::ops::bootstrap::SnapshotOptions; - - let snapshot_options = SnapshotOptions { - ts_version: ts::version(), - v8_version: deno_core::v8::VERSION_STRING, - target: std::env::var("TARGET").unwrap(), - }; - - deno_runtime::snapshot::create_runtime_snapshot( - snapshot_path, - snapshot_options, - vec![], - ); -} - -fn git_commit_hash() -> String { - if let Ok(output) = std::process::Command::new("git") - .arg("rev-list") - .arg("-1") - .arg("HEAD") - .output() - { - if output.status.success() { - std::str::from_utf8(&output.stdout[..40]) - .unwrap() - .to_string() - } else { - // When not in git repository - // (e.g. when the user install by `cargo install deno`) - "UNKNOWN".to_string() - } - } else { - // When there is no git command for some reason - "UNKNOWN".to_string() - } } fn main() { @@ -370,7 +318,7 @@ fn main() { } deno_napi::print_linker_flags("deno"); - deno_napi::print_linker_flags("denort"); + deno_webgpu::print_linker_flags("deno"); // Host snapshots won't work when cross compiling. let target = env::var("TARGET").unwrap(); @@ -389,51 +337,15 @@ fn main() { } println!("cargo:rerun-if-env-changed=DENO_CANARY"); - println!("cargo:rustc-env=GIT_COMMIT_HASH={}", git_commit_hash()); - println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH"); - println!( - "cargo:rustc-env=GIT_COMMIT_HASH_SHORT={}", - &git_commit_hash()[..7] - ); - - let ts_version = ts::version(); - debug_assert_eq!(ts_version, "5.6.2"); // bump this assertion when it changes - println!("cargo:rustc-env=TS_VERSION={}", ts_version); - println!("cargo:rerun-if-env-changed=TS_VERSION"); - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - if cfg!(windows) { - // these dls load slowly, so delay loading them - let dlls = [ - // webgpu - "d3dcompiler_47", - "OPENGL32", - // network related functions - "iphlpapi", - ]; - for dll in dlls { - println!("cargo:rustc-link-arg-bin=deno=/delayload:{dll}.dll"); - println!("cargo:rustc-link-arg-bin=denort=/delayload:{dll}.dll"); - } - // enable delay loading - println!("cargo:rustc-link-arg-bin=deno=delayimp.lib"); - println!("cargo:rustc-link-arg-bin=denort=delayimp.lib"); - } - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin"); ts::create_compiler_snapshot(compiler_snapshot_path, &c); - #[cfg(not(feature = "hmr"))] - { - let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin"); - create_cli_snapshot(cli_snapshot_path); - } - #[cfg(target_os = "windows")] { let mut res = winres::WindowsResource::new(); diff --git a/cli/cache/cache_db.rs b/cli/cache/cache_db.rs index 7fd66e9333..6d6d967304 100644 --- a/cli/cache/cache_db.rs +++ b/cli/cache/cache_db.rs @@ -9,14 +9,13 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::MutexGuard; use deno_core::unsync::spawn_blocking; +use deno_lib::util::hash::FastInsecureHasher; use deno_runtime::deno_webstorage::rusqlite; use deno_runtime::deno_webstorage::rusqlite::Connection; use deno_runtime::deno_webstorage::rusqlite::OptionalExtension; use deno_runtime::deno_webstorage::rusqlite::Params; use once_cell::sync::OnceCell; -use super::FastInsecureHasher; - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct CacheDBHash(u64); diff --git a/cli/cache/caches.rs b/cli/cache/caches.rs index dd4a974814..fad61f1dc3 100644 --- a/cli/cache/caches.rs +++ b/cli/cache/caches.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::sync::Arc; -use deno_lib::cache::DenoDirProvider; +use deno_lib::version::DENO_VERSION_INFO; use once_cell::sync::OnceCell; use super::cache_db::CacheDB; @@ -14,10 +14,10 @@ use super::fast_check::FAST_CHECK_CACHE_DB; use super::incremental::INCREMENTAL_CACHE_DB; use super::module_info::MODULE_INFO_CACHE_DB; use super::node::NODE_ANALYSIS_CACHE_DB; -use crate::sys::CliSys; +use crate::cache::DenoDirProvider; pub struct Caches { - dir_provider: Arc>, + dir_provider: Arc, fmt_incremental_cache_db: OnceCell, lint_incremental_cache_db: OnceCell, dep_analysis_db: OnceCell, @@ -28,7 +28,7 @@ pub struct Caches { } impl Caches { - pub fn new(dir: Arc>) -> Self { + pub fn new(dir: Arc) -> Self { Self { dir_provider: dir, fmt_incremental_cache_db: Default::default(), @@ -49,13 +49,9 @@ impl Caches { cell .get_or_init(|| { if let Some(path) = path { - CacheDB::from_path( - config, - path, - crate::version::DENO_VERSION_INFO.deno, - ) + CacheDB::from_path(config, path, DENO_VERSION_INFO.deno) } else { - CacheDB::in_memory(config, crate::version::DENO_VERSION_INFO.deno) + CacheDB::in_memory(config, DENO_VERSION_INFO.deno) } }) .clone() diff --git a/cli/cache/code_cache.rs b/cli/cache/code_cache.rs index 27ec544b5f..d938732635 100644 --- a/cli/cache/code_cache.rs +++ b/cli/cache/code_cache.rs @@ -1,7 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::sync::Arc; - use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_runtime::code_cache; @@ -11,7 +9,6 @@ use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::cache_db::CacheDBHash; use super::cache_db::CacheFailure; -use crate::worker::CliCodeCache; pub static CODE_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration { table_initializer: concat!( @@ -85,12 +82,6 @@ impl CodeCache { } } -impl CliCodeCache for CodeCache { - fn as_code_cache(self: Arc) -> Arc { - self - } -} - impl code_cache::CodeCache for CodeCache { fn get_sync( &self, diff --git a/cli/lib/cache/deno_dir.rs b/cli/cache/deno_dir.rs similarity index 90% rename from cli/lib/cache/deno_dir.rs rename to cli/cache/deno_dir.rs index 00bc83ff9b..5be08446cf 100644 --- a/cli/lib/cache/deno_dir.rs +++ b/cli/cache/deno_dir.rs @@ -6,18 +6,18 @@ use std::path::PathBuf; use deno_cache_dir::DenoDirResolutionError; use super::DiskCache; -use crate::sys::DenoLibSys; +use crate::sys::CliSys; /// Lazily creates the deno dir which might be useful in scenarios /// where functionality wants to continue if the DENO_DIR can't be created. -pub struct DenoDirProvider { - sys: TSys, +pub struct DenoDirProvider { + sys: CliSys, maybe_custom_root: Option, - deno_dir: std::sync::OnceLock, DenoDirResolutionError>>, + deno_dir: std::sync::OnceLock>, } -impl DenoDirProvider { - pub fn new(sys: TSys, maybe_custom_root: Option) -> Self { +impl DenoDirProvider { + pub fn new(sys: CliSys, maybe_custom_root: Option) -> Self { Self { sys, maybe_custom_root, @@ -25,9 +25,7 @@ impl DenoDirProvider { } } - pub fn get_or_create( - &self, - ) -> Result<&DenoDir, DenoDirResolutionError> { + pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> { self .deno_dir .get_or_init(|| { @@ -50,16 +48,16 @@ impl DenoDirProvider { /// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them /// in single directory that can be controlled with `$DENO_DIR` env variable. #[derive(Debug, Clone)] -pub struct DenoDir { +pub struct DenoDir { /// Example: /Users/rld/.deno/ pub root: PathBuf, /// Used by TsCompiler to cache compiler output. - pub gen_cache: DiskCache, + pub gen_cache: DiskCache, } -impl DenoDir { +impl DenoDir { pub fn new( - sys: TSys, + sys: CliSys, maybe_custom_root: Option, ) -> Result { let root = deno_cache_dir::resolve_deno_dir( diff --git a/cli/lib/cache/disk_cache.rs b/cli/cache/disk_cache.rs similarity index 97% rename from cli/lib/cache/disk_cache.rs rename to cli/cache/disk_cache.rs index 2c735a34b2..cb28c02922 100644 --- a/cli/lib/cache/disk_cache.rs +++ b/cli/cache/disk_cache.rs @@ -10,21 +10,21 @@ use std::str; use deno_cache_dir::url_to_filename; use deno_cache_dir::CACHE_PERM; +use deno_core::url::Host; +use deno_core::url::Url; use deno_path_util::fs::atomic_write_file_with_retries; -use url::Host; -use url::Url; -use crate::sys::DenoLibSys; +use crate::sys::CliSys; #[derive(Debug, Clone)] -pub struct DiskCache { - sys: TSys, +pub struct DiskCache { + sys: CliSys, pub location: PathBuf, } -impl DiskCache { +impl DiskCache { /// `location` must be an absolute path. - pub fn new(sys: TSys, location: &Path) -> Self { + pub fn new(sys: CliSys, location: &Path) -> Self { assert!(location.is_absolute()); Self { sys, diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index e8a940b3be..116454dd9a 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -6,25 +6,25 @@ use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::unsync::sync::AtomicFlag; -use deno_lib::cache::DiskCache; +use deno_lib::version::DENO_VERSION_INFO; -use crate::sys::CliSys; +use super::DiskCache; /// The cache that stores previously emitted files. #[derive(Debug)] pub struct EmitCache { - disk_cache: DiskCache, + disk_cache: DiskCache, emit_failed_flag: AtomicFlag, file_serializer: EmitFileSerializer, } impl EmitCache { - pub fn new(disk_cache: DiskCache) -> Self { + pub fn new(disk_cache: DiskCache) -> Self { Self { disk_cache, emit_failed_flag: Default::default(), file_serializer: EmitFileSerializer { - cli_version: crate::version::DENO_VERSION_INFO.deno, + cli_version: DENO_VERSION_INFO.deno, }, } } @@ -148,7 +148,7 @@ impl EmitFileSerializer { // it's ok to use an insecure hash here because // if someone can change the emit source then they // can also change the version hash - crate::cache::FastInsecureHasher::new_without_deno_version() // use cli_version property instead + deno_lib::util::hash::FastInsecureHasher::new_without_deno_version() // use cli_version property instead .write(bytes) // emit should not be re-used between cli versions .write_str(self.cli_version) diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index e16f95e56f..695d21f3d1 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -30,7 +30,8 @@ mod cache_db; mod caches; mod check; mod code_cache; -mod common; +mod deno_dir; +mod disk_cache; mod emit; mod fast_check; mod incremental; @@ -42,9 +43,11 @@ pub use cache_db::CacheDBHash; pub use caches::Caches; pub use check::TypeCheckCache; pub use code_cache::CodeCache; -pub use common::FastInsecureHasher; /// Permissions used to save a file in the disk caches. pub use deno_cache_dir::CACHE_PERM; +pub use deno_dir::DenoDir; +pub use deno_dir::DenoDirProvider; +pub use disk_cache::DiskCache; pub use emit::EmitCache; pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; diff --git a/cli/emit.rs b/cli/emit.rs index f928591eaf..5e39df2874 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -20,9 +20,9 @@ use deno_error::JsErrorBox; use deno_graph::MediaType; use deno_graph::Module; use deno_graph::ModuleGraph; +use deno_lib::util::hash::FastInsecureHasher; use crate::cache::EmitCache; -use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; use crate::resolver::CliCjsTracker; diff --git a/cli/factory.rs b/cli/factory.rs index bfe6d05570..91c5d07b75 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,8 +11,10 @@ use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::FeatureChecker; use deno_error::JsErrorBox; -use deno_lib::cache::DenoDir; -use deno_lib::cache::DenoDirProvider; +use deno_lib::args::get_root_cert_store; +use deno_lib::args::CaData; +use deno_lib::loader::NpmModuleLoader; +use deno_lib::npm::create_npm_process_state_provider; use deno_lib::npm::NpmRegistryReadPermissionChecker; use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; use deno_lib::worker::LibMainWorkerFactory; @@ -42,8 +44,6 @@ use node_resolver::analyze::NodeCodeTranslator; use once_cell::sync::OnceCell; use crate::args::check_warn_tsconfig; -use crate::args::get_root_cert_store; -use crate::args::CaData; use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::Flags; @@ -51,6 +51,8 @@ use crate::args::NpmInstallDepsProvider; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; +use crate::cache::DenoDir; +use crate::cache::DenoDirProvider; use crate::cache::EmitCache; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; @@ -71,7 +73,6 @@ use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::node::CliPackageJsonResolver; -use crate::npm::create_npm_process_state_provider; use crate::npm::installer::NpmInstaller; use crate::npm::installer::NpmResolutionInstaller; use crate::npm::CliByonmNpmResolverCreateOptions; @@ -91,7 +92,6 @@ use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; use crate::resolver::CliSloppyImportsResolver; use crate::resolver::FoundPackageJsonDepFlag; -use crate::resolver::NpmModuleLoader; use crate::standalone::binary::DenoCompileBinaryWriter; use crate::sys::CliSys; use crate::tools::check::TypeChecker; @@ -283,13 +283,11 @@ impl CliFactory { }) } - pub fn deno_dir_provider( - &self, - ) -> Result<&Arc>, AnyError> { + pub fn deno_dir_provider(&self) -> Result<&Arc, AnyError> { Ok(&self.cli_options()?.deno_dir_provider) } - pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { + pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { Ok(self.deno_dir_provider()?.get_or_create()?) } @@ -815,19 +813,11 @@ impl CliFactory { .services .node_code_translator .get_or_try_init_async(async { - let caches = self.caches()?; - let node_analysis_cache = - NodeAnalysisCache::new(caches.node_analysis_db()); let node_resolver = self.node_resolver().await?.clone(); - let cjs_esm_analyzer = CliCjsCodeAnalyzer::new( - node_analysis_cache, - self.cjs_tracker()?.clone(), - self.fs().clone(), - Some(self.parsed_source_cache().clone()), - ); + let cjs_code_analyzer = self.create_cjs_code_analyzer()?; Ok(Arc::new(NodeCodeTranslator::new( - cjs_esm_analyzer, + cjs_code_analyzer, self.in_npm_pkg_checker()?.clone(), node_resolver, self.npm_resolver().await?.clone(), @@ -838,6 +828,17 @@ impl CliFactory { .await } + fn create_cjs_code_analyzer(&self) -> Result { + let caches = self.caches()?; + let node_analysis_cache = NodeAnalysisCache::new(caches.node_analysis_db()); + Ok(CliCjsCodeAnalyzer::new( + node_analysis_cache, + self.cjs_tracker()?.clone(), + self.fs().clone(), + Some(self.parsed_source_cache().clone()), + )) + } + pub async fn npm_req_resolver( &self, ) -> Result<&Arc, AnyError> { @@ -1027,11 +1028,11 @@ impl CliFactory { ) -> Result { let cli_options = self.cli_options()?; Ok(DenoCompileBinaryWriter::new( + self.create_cjs_code_analyzer()?, self.cjs_tracker()?, self.cli_options()?, self.deno_dir()?, self.emitter()?, - self.file_fetcher()?, self.http_client_provider(), self.npm_resolver().await?, self.workspace_resolver().await?.as_ref(), @@ -1103,8 +1104,8 @@ impl CliFactory { node_resolver.clone(), NpmModuleLoader::new( self.cjs_tracker()?.clone(), - fs.clone(), node_code_translator.clone(), + self.sys(), ), npm_registry_permission_checker, npm_req_resolver.clone(), @@ -1138,7 +1139,6 @@ impl CliFactory { lib_main_worker_factory, maybe_file_watcher_communicator, cli_options.maybe_lockfile().cloned(), - node_resolver.clone(), self.npm_installer_if_managed()?.cloned(), npm_resolver.clone(), self.sys(), @@ -1180,8 +1180,6 @@ impl CliFactory { node_ipc: cli_options.node_ipc_fd(), serve_port: cli_options.serve_port(), serve_host: cli_options.serve_host(), - deno_version: crate::version::DENO_VERSION_INFO.deno, - deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, otel_config: self.cli_options()?.otel_config(), startup_snapshot: crate::js::deno_isolate_init(), }) diff --git a/cli/http_util.rs b/cli/http_util.rs index 5e63ab0a4a..a12fde937e 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -14,6 +14,7 @@ use deno_core::serde_json; use deno_core::url::Url; use deno_error::JsError; use deno_error::JsErrorBox; +use deno_lib::version::DENO_VERSION_INFO; use deno_runtime::deno_fetch; use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::CreateHttpClientOptions; @@ -28,7 +29,6 @@ use http_body_util::BodyExt; use thiserror::Error; use crate::util::progress_bar::UpdateGuard; -use crate::version; #[derive(Debug, Error)] pub enum SendError { @@ -79,7 +79,7 @@ impl HttpClientProvider { Entry::Occupied(entry) => Ok(HttpClient::new(entry.get().clone())), Entry::Vacant(entry) => { let client = create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { root_cert_store: match &self.root_cert_store_provider { Some(provider) => Some(provider.get_or_try_init()?.clone()), @@ -481,7 +481,7 @@ mod test { let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { ca_certs: vec![std::fs::read( test_util::testdata_path().join("tls/RootCA.pem"), @@ -525,7 +525,7 @@ mod test { let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions::default(), ) .unwrap(), @@ -566,7 +566,7 @@ mod test { let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { root_cert_store: Some(root_cert_store), ..Default::default() @@ -587,7 +587,7 @@ mod test { .unwrap(); let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { ca_certs: vec![std::fs::read( test_util::testdata_path() @@ -620,7 +620,7 @@ mod test { let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { ca_certs: vec![std::fs::read( test_util::testdata_path() @@ -661,7 +661,7 @@ mod test { .unwrap(); let client = HttpClient::new( create_http_client( - version::DENO_VERSION_INFO.user_agent, + DENO_VERSION_INFO.user_agent, CreateHttpClientOptions { ca_certs: vec![std::fs::read( test_util::testdata_path() diff --git a/cli/integration_tests_runner.rs b/cli/integration_tests_runner.rs index 7342e62fa0..63f2abe460 100644 --- a/cli/integration_tests_runner.rs +++ b/cli/integration_tests_runner.rs @@ -1,18 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. + pub fn main() { - let mut args = vec!["cargo", "test", "-p", "cli_tests", "--features", "run"]; - - if !cfg!(debug_assertions) { - args.push("--release"); - } - - args.push("--"); - - // If any args were passed to this process, pass them through to the child - let orig_args = std::env::args().skip(1).collect::>(); - let orig_args: Vec<&str> = - orig_args.iter().map(|x| x.as_ref()).collect::>(); - args.extend(orig_args); - - test_util::spawn::exec_replace("cargo", &args).unwrap(); + // this file exists to cause the executable to be built when running cargo test } diff --git a/cli/js.rs b/cli/js.rs index 5337c53f76..37004ad444 100644 --- a/cli/js.rs +++ b/cli/js.rs @@ -2,18 +2,7 @@ use log::debug; -#[cfg(not(feature = "hmr"))] -static CLI_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); - pub fn deno_isolate_init() -> Option<&'static [u8]> { debug!("Deno isolate init with snapshots."); - #[cfg(not(feature = "hmr"))] - { - Some(CLI_SNAPSHOT) - } - #[cfg(feature = "hmr")] - { - None - } + deno_snapshots::CLI_SNAPSHOT } diff --git a/cli/lib/Cargo.toml b/cli/lib/Cargo.toml index 67caf6e944..ab051de995 100644 --- a/cli/lib/Cargo.toml +++ b/cli/lib/Cargo.toml @@ -14,23 +14,32 @@ description = "Shared code between the Deno CLI and denort" path = "lib.rs" [dependencies] -deno_cache_dir.workspace = true +capacity_builder.workspace = true +deno_config.workspace = true deno_error.workspace = true deno_fs = { workspace = true, features = ["sync_fs"] } +deno_media_type.workspace = true deno_node = { workspace = true, features = ["sync_fs"] } +deno_npm.workspace = true deno_path_util.workspace = true deno_resolver = { workspace = true, features = ["sync"] } deno_runtime.workspace = true +deno_semver.workspace = true deno_terminal.workspace = true +env_logger = "=0.10.0" faster-hex.workspace = true +indexmap.workspace = true +libsui.workspace = true log.workspace = true node_resolver = { workspace = true, features = ["sync"] } parking_lot.workspace = true ring.workspace = true serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true sys_traits = { workspace = true, features = ["getrandom"] } thiserror.workspace = true tokio.workspace = true +twox-hash.workspace = true url.workspace = true [dev-dependencies] diff --git a/cli/lib/args.rs b/cli/lib/args.rs new file mode 100644 index 0000000000..3e64f5ab4c --- /dev/null +++ b/cli/lib/args.rs @@ -0,0 +1,199 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::io::BufReader; +use std::io::Cursor; +use std::io::Read; +use std::io::Seek; +use std::path::PathBuf; +use std::sync::LazyLock; + +use deno_runtime::colors; +use deno_runtime::deno_tls::deno_native_certs::load_native_certs; +use deno_runtime::deno_tls::rustls; +use deno_runtime::deno_tls::rustls::RootCertStore; +use deno_runtime::deno_tls::rustls_pemfile; +use deno_runtime::deno_tls::webpki_roots; +use deno_semver::npm::NpmPackageReqReference; +use serde::Deserialize; +use serde::Serialize; +use thiserror::Error; + +pub fn npm_pkg_req_ref_to_binary_command( + req_ref: &NpmPackageReqReference, +) -> String { + req_ref + .sub_path() + .map(|s| s.to_string()) + .unwrap_or_else(|| req_ref.req().name.to_string()) +} + +pub fn has_trace_permissions_enabled() -> bool { + has_flag_env_var("DENO_TRACE_PERMISSIONS") +} + +pub fn has_flag_env_var(name: &str) -> bool { + let value = std::env::var(name); + matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CaData { + /// The string is a file path + File(String), + /// This variant is not exposed as an option in the CLI, it is used internally + /// for standalone binaries. + Bytes(Vec), +} + +#[derive(Error, Debug, Clone, deno_error::JsError)] +#[class(generic)] +pub enum RootCertStoreLoadError { + #[error( + "Unknown certificate store \"{0}\" specified (allowed: \"system,mozilla\")" + )] + UnknownStore(String), + #[error("Unable to add pem file to certificate store: {0}")] + FailedAddPemFile(String), + #[error("Failed opening CA file: {0}")] + CaFileOpenError(String), +} + +/// Create and populate a root cert store based on the passed options and +/// environment. +pub fn get_root_cert_store( + maybe_root_path: Option, + maybe_ca_stores: Option>, + maybe_ca_data: Option, +) -> Result { + let mut root_cert_store = RootCertStore::empty(); + let ca_stores: Vec = maybe_ca_stores + .or_else(|| { + let env_ca_store = std::env::var("DENO_TLS_CA_STORE").ok()?; + Some( + env_ca_store + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(), + ) + }) + .unwrap_or_else(|| vec!["mozilla".to_string()]); + + for store in ca_stores.iter() { + match store.as_str() { + "mozilla" => { + root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.to_vec()); + } + "system" => { + let roots = load_native_certs().expect("could not load platform certs"); + for root in roots { + if let Err(err) = root_cert_store + .add(rustls::pki_types::CertificateDer::from(root.0.clone())) + { + log::error!( + "{}", + colors::yellow(&format!( + "Unable to add system certificate to certificate store: {:?}", + err + )) + ); + let hex_encoded_root = faster_hex::hex_string(&root.0); + log::error!("{}", colors::gray(&hex_encoded_root)); + } + } + } + _ => { + return Err(RootCertStoreLoadError::UnknownStore(store.clone())); + } + } + } + + let ca_data = + maybe_ca_data.or_else(|| std::env::var("DENO_CERT").ok().map(CaData::File)); + if let Some(ca_data) = ca_data { + let result = match ca_data { + CaData::File(ca_file) => { + let ca_file = if let Some(root) = &maybe_root_path { + root.join(&ca_file) + } else { + PathBuf::from(ca_file) + }; + let certfile = std::fs::File::open(ca_file).map_err(|err| { + RootCertStoreLoadError::CaFileOpenError(err.to_string()) + })?; + let mut reader = BufReader::new(certfile); + rustls_pemfile::certs(&mut reader).collect::, _>>() + } + CaData::Bytes(data) => { + let mut reader = BufReader::new(Cursor::new(data)); + rustls_pemfile::certs(&mut reader).collect::, _>>() + } + }; + + match result { + Ok(certs) => { + root_cert_store.add_parsable_certificates(certs); + } + Err(e) => { + return Err(RootCertStoreLoadError::FailedAddPemFile(e.to_string())); + } + } + } + + Ok(root_cert_store) +} + +/// State provided to the process via an environment variable. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NpmProcessState { + pub kind: NpmProcessStateKind, + pub local_node_modules_path: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum NpmProcessStateKind { + Snapshot(deno_npm::resolution::SerializedNpmResolutionSnapshot), + Byonm, +} + +pub static NPM_PROCESS_STATE: LazyLock> = + LazyLock::new(|| { + use deno_runtime::deno_process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME; + let fd = std::env::var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME).ok()?; + std::env::remove_var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME); + let fd = fd.parse::().ok()?; + let mut file = { + use deno_runtime::deno_io::FromRawIoHandle; + unsafe { std::fs::File::from_raw_io_handle(fd as _) } + }; + let mut buf = Vec::new(); + // seek to beginning. after the file is written the position will be inherited by this subprocess, + // and also this file might have been read before + file.seek(std::io::SeekFrom::Start(0)).unwrap(); + file + .read_to_end(&mut buf) + .inspect_err(|e| { + log::error!("failed to read npm process state from fd {fd}: {e}"); + }) + .ok()?; + let state: NpmProcessState = serde_json::from_slice(&buf) + .inspect_err(|e| { + log::error!( + "failed to deserialize npm process state: {e} {}", + String::from_utf8_lossy(&buf) + ) + }) + .ok()?; + Some(state) + }); + +#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct UnstableConfig { + // TODO(bartlomieju): remove in Deno 2.5 + pub legacy_flag_enabled: bool, // --unstable + pub bare_node_builtins: bool, + pub detect_cjs: bool, + pub sloppy_imports: bool, + pub npm_lazy_caching: bool, + pub features: Vec, // --unstabe-kv --unstable-cron +} diff --git a/cli/lib/build.rs b/cli/lib/build.rs new file mode 100644 index 0000000000..1f52e0c02a --- /dev/null +++ b/cli/lib/build.rs @@ -0,0 +1,42 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +fn main() { + // todo(dsherret): remove this after Deno 2.2.0 is published and then + // align the version of this crate with Deno then. We need to wait because + // there was previously a deno_lib 2.2.0 published (https://crates.io/crates/deno_lib/versions) + let version_path = std::path::Path::new(".").join("version.txt"); + println!("cargo:rerun-if-changed={}", version_path.display()); + #[allow(clippy::disallowed_methods)] + let text = std::fs::read_to_string(version_path).unwrap(); + println!("cargo:rustc-env=DENO_VERSION={}", text); + + let commit_hash = git_commit_hash(); + println!("cargo:rustc-env=GIT_COMMIT_HASH={}", commit_hash); + println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH"); + println!( + "cargo:rustc-env=GIT_COMMIT_HASH_SHORT={}", + &commit_hash[..7] + ); +} + +fn git_commit_hash() -> String { + if let Ok(output) = std::process::Command::new("git") + .arg("rev-list") + .arg("-1") + .arg("HEAD") + .output() + { + if output.status.success() { + std::str::from_utf8(&output.stdout[..40]) + .unwrap() + .to_string() + } else { + // When not in git repository + // (e.g. when the user install by `cargo install deno`) + "UNKNOWN".to_string() + } + } else { + // When there is no git command for some reason + "UNKNOWN".to_string() + } +} diff --git a/cli/lib/cache/mod.rs b/cli/lib/cache/mod.rs deleted file mode 100644 index c4395df3e1..0000000000 --- a/cli/lib/cache/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -pub use deno_dir::DenoDir; -pub use deno_dir::DenoDirProvider; -pub use disk_cache::DiskCache; - -mod deno_dir; -mod disk_cache; diff --git a/cli/lib/clippy.toml b/cli/lib/clippy.toml new file mode 100644 index 0000000000..0060289cf2 --- /dev/null +++ b/cli/lib/clippy.toml @@ -0,0 +1,48 @@ +disallowed-methods = [ + { path = "std::env::current_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::is_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::is_file", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::is_symlink", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::read_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::read_link", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::try_exists", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::exists", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::env::set_current_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::env::temp_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::canonicalize", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::copy", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::create_dir_all", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::create_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::hard_link", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::read_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::read_link", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::read_to_string", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::read", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::remove_dir_all", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::remove_dir", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::remove_file", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::rename", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::set_permissions", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::symlink_metadata", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::fs::write", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoLibSys" }, + { path = "std::path::Path::exists", reason = "File system operations should be done using DenoLibSys" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead" }, +] diff --git a/cli/lib/env.rs b/cli/lib/env.rs deleted file mode 100644 index 9c6001478b..0000000000 --- a/cli/lib/env.rs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -pub fn has_trace_permissions_enabled() -> bool { - has_flag_env_var("DENO_TRACE_PERMISSIONS") -} - -pub fn has_flag_env_var(name: &str) -> bool { - let value = std::env::var(name); - matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) -} diff --git a/cli/lib/lib.rs b/cli/lib/lib.rs index 5453bddaee..6b9267805d 100644 --- a/cli/lib/lib.rs +++ b/cli/lib/lib.rs @@ -1,9 +1,11 @@ // Copyright 2018-2025 the Deno authors. MIT license. -pub mod cache; -pub mod env; +pub mod args; +pub mod loader; pub mod npm; +pub mod shared; pub mod standalone; pub mod sys; pub mod util; +pub mod version; pub mod worker; diff --git a/cli/lib/loader.rs b/cli/lib/loader.rs new file mode 100644 index 0000000000..32f97dc718 --- /dev/null +++ b/cli/lib/loader.rs @@ -0,0 +1,213 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_media_type::MediaType; +use deno_resolver::cjs::CjsTracker; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_runtime::deno_core::ModuleSourceCode; +use node_resolver::analyze::CjsCodeAnalyzer; +use node_resolver::analyze::NodeCodeTranslator; +use node_resolver::InNpmPackageChecker; +use node_resolver::IsBuiltInNodeModuleChecker; +use node_resolver::NpmPackageFolderResolver; +use thiserror::Error; +use url::Url; + +use crate::sys::DenoLibSys; +use crate::util::text_encoding::from_utf8_lossy_cow; + +pub struct ModuleCodeStringSource { + pub code: ModuleSourceCode, + pub found_url: Url, + pub media_type: MediaType, +} + +#[derive(Debug, Error, deno_error::JsError)] +#[class(type)] +#[error("{media_type} files are not supported in npm packages: {specifier}")] +pub struct NotSupportedKindInNpmError { + pub media_type: MediaType, + pub specifier: Url, +} + +#[derive(Debug, Error, deno_error::JsError)] +pub enum NpmModuleLoadError { + #[class(inherit)] + #[error(transparent)] + UrlToFilePath(#[from] deno_path_util::UrlToFilePathError), + #[class(inherit)] + #[error(transparent)] + NotSupportedKindInNpm(#[from] NotSupportedKindInNpmError), + #[class(inherit)] + #[error(transparent)] + ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), + #[class(inherit)] + #[error(transparent)] + TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError), + #[class(inherit)] + #[error("Unable to load {}{}", file_path.display(), maybe_referrer.as_ref().map(|r| format!(" imported from {}", r)).unwrap_or_default())] + UnableToLoad { + file_path: PathBuf, + maybe_referrer: Option, + #[source] + #[inherit] + source: std::io::Error, + }, + #[class(inherit)] + #[error( + "{}", + format_dir_import_message(file_path, maybe_referrer, suggestion) + )] + DirImport { + file_path: PathBuf, + maybe_referrer: Option, + suggestion: Option<&'static str>, + #[source] + #[inherit] + source: std::io::Error, + }, +} + +fn format_dir_import_message( + file_path: &std::path::Path, + maybe_referrer: &Option, + suggestion: &Option<&'static str>, +) -> String { + // directory imports are not allowed when importing from an + // ES module, so provide the user with a helpful error message + let dir_path = file_path; + let mut msg = "Directory import ".to_string(); + msg.push_str(&dir_path.to_string_lossy()); + if let Some(referrer) = maybe_referrer { + msg.push_str(" is not supported resolving import from "); + msg.push_str(referrer.as_str()); + if let Some(entrypoint_name) = suggestion { + msg.push_str("\nDid you mean to import "); + msg.push_str(entrypoint_name); + msg.push_str(" within the directory?"); + } + } + msg +} + +#[derive(Clone)] +pub struct NpmModuleLoader< + TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TSys: DenoLibSys, +> { + cjs_tracker: Arc>, + sys: TSys, + node_code_translator: Arc< + NodeCodeTranslator< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + >, +} + +impl< + TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TSys: DenoLibSys, + > + NpmModuleLoader< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + > +{ + pub fn new( + cjs_tracker: Arc>, + node_code_translator: Arc< + NodeCodeTranslator< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + >, + sys: TSys, + ) -> Self { + Self { + cjs_tracker, + node_code_translator, + sys, + } + } + + pub async fn load( + &self, + specifier: &Url, + maybe_referrer: Option<&Url>, + ) -> Result { + let file_path = deno_path_util::url_to_file_path(specifier)?; + let code = self.sys.fs_read(&file_path).map_err(|source| { + if self.sys.fs_is_dir_no_err(&file_path) { + let suggestion = ["index.mjs", "index.js", "index.cjs"] + .into_iter() + .find(|e| self.sys.fs_is_file_no_err(file_path.join(e))); + NpmModuleLoadError::DirImport { + file_path, + maybe_referrer: maybe_referrer.cloned(), + suggestion, + source, + } + } else { + NpmModuleLoadError::UnableToLoad { + file_path, + maybe_referrer: maybe_referrer.cloned(), + source, + } + } + })?; + + let media_type = MediaType::from_specifier(specifier); + if media_type.is_emittable() { + return Err(NpmModuleLoadError::NotSupportedKindInNpm( + NotSupportedKindInNpmError { + media_type, + specifier: specifier.clone(), + }, + )); + } + + let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { + // translate cjs to esm if it's cjs and inject node globals + let code = from_utf8_lossy_cow(code); + ModuleSourceCode::String( + self + .node_code_translator + .translate_cjs_to_esm(specifier, Some(code)) + .await? + .into_owned() + .into(), + ) + } else { + // esm and json code is untouched + ModuleSourceCode::Bytes(match code { + Cow::Owned(bytes) => bytes.into_boxed_slice().into(), + Cow::Borrowed(bytes) => bytes.into(), + }) + }; + + Ok(ModuleCodeStringSource { + code, + found_url: specifier.clone(), + media_type: MediaType::from_specifier(specifier), + }) + } +} diff --git a/cli/lib/npm/mod.rs b/cli/lib/npm/mod.rs index e7d4d8d9d1..b6ad5d1be5 100644 --- a/cli/lib/npm/mod.rs +++ b/cli/lib/npm/mod.rs @@ -2,5 +2,79 @@ mod permission_checker; +use std::path::Path; +use std::sync::Arc; + +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_resolver::npm::ByonmNpmResolver; +use deno_resolver::npm::ManagedNpmResolverRc; +use deno_resolver::npm::NpmResolver; +use deno_runtime::deno_process::NpmProcessStateProvider; +use deno_runtime::deno_process::NpmProcessStateProviderRc; pub use permission_checker::NpmRegistryReadPermissionChecker; pub use permission_checker::NpmRegistryReadPermissionCheckerMode; + +use crate::args::NpmProcessState; +use crate::args::NpmProcessStateKind; +use crate::sys::DenoLibSys; + +pub fn create_npm_process_state_provider( + npm_resolver: &NpmResolver, +) -> NpmProcessStateProviderRc { + match npm_resolver { + NpmResolver::Byonm(byonm_npm_resolver) => { + Arc::new(ByonmNpmProcessStateProvider(byonm_npm_resolver.clone())) + } + NpmResolver::Managed(managed_npm_resolver) => { + Arc::new(ManagedNpmProcessStateProvider(managed_npm_resolver.clone())) + } + } +} + +pub fn npm_process_state( + snapshot: ValidSerializedNpmResolutionSnapshot, + node_modules_path: Option<&Path>, +) -> String { + serde_json::to_string(&NpmProcessState { + kind: NpmProcessStateKind::Snapshot(snapshot.into_serialized()), + local_node_modules_path: node_modules_path + .map(|p| p.to_string_lossy().to_string()), + }) + .unwrap() +} + +#[derive(Debug)] +pub struct ManagedNpmProcessStateProvider( + pub ManagedNpmResolverRc, +); + +impl NpmProcessStateProvider + for ManagedNpmProcessStateProvider +{ + fn get_npm_process_state(&self) -> String { + npm_process_state( + self.0.resolution().serialized_valid_snapshot(), + self.0.root_node_modules_path(), + ) + } +} + +#[derive(Debug)] +pub struct ByonmNpmProcessStateProvider( + pub Arc>, +); + +impl NpmProcessStateProvider + for ByonmNpmProcessStateProvider +{ + fn get_npm_process_state(&self) -> String { + serde_json::to_string(&NpmProcessState { + kind: NpmProcessStateKind::Byonm, + local_node_modules_path: self + .0 + .root_node_modules_path() + .map(|p| p.to_string_lossy().to_string()), + }) + .unwrap() + } +} diff --git a/cli/shared.rs b/cli/lib/shared.rs similarity index 79% rename from cli/shared.rs rename to cli/lib/shared.rs index 6a28473edd..15ec3c2440 100644 --- a/cli/shared.rs +++ b/cli/lib/shared.rs @@ -1,8 +1,11 @@ // Copyright 2018-2025 the Deno authors. MIT license. /// This module is shared between build script and the binaries. Use it sparsely. -use deno_core::anyhow::bail; -use deno_core::error::AnyError; +use thiserror::Error; + +#[derive(Debug, Error)] +#[error("Unrecognized release channel: {0}")] +pub struct UnrecognizedReleaseChannelError(pub String); #[derive(Debug, Clone, Copy, PartialEq)] pub enum ReleaseChannel { @@ -50,13 +53,17 @@ impl ReleaseChannel { // NOTE(bartlomieju): do not ever change these values, tools like `patchver` // rely on them. #[allow(unused)] - pub fn deserialize(str_: &str) -> Result { + pub fn deserialize( + str_: &str, + ) -> Result { Ok(match str_ { "stable" => Self::Stable, "canary" => Self::Canary, "rc" => Self::Rc, "lts" => Self::Lts, - unknown => bail!("Unrecognized release channel: {}", unknown), + unknown => { + return Err(UnrecognizedReleaseChannelError(unknown.to_string())) + } }) } } diff --git a/cli/lib/standalone/binary.rs b/cli/lib/standalone/binary.rs new file mode 100644 index 0000000000..ae02197bf4 --- /dev/null +++ b/cli/lib/standalone/binary.rs @@ -0,0 +1,389 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::collections::BTreeMap; + +use deno_config::workspace::PackageJsonDepResolution; +use deno_media_type::MediaType; +use deno_runtime::deno_permissions::PermissionsOptions; +use deno_runtime::deno_telemetry::OtelConfig; +use deno_semver::Version; +use indexmap::IndexMap; +use node_resolver::analyze::CjsAnalysisExports; +use serde::Deserialize; +use serde::Serialize; +use url::Url; + +use super::virtual_fs::FileSystemCaseSensitivity; +use crate::args::UnstableConfig; + +pub const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; + +pub trait DenoRtDeserializable<'a>: Sized { + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)>; +} + +impl<'a> DenoRtDeserializable<'a> for Cow<'a, [u8]> { + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)> { + let (input, data) = read_bytes_with_u32_len(input)?; + Ok((input, Cow::Borrowed(data))) + } +} + +pub trait DenoRtSerializable<'a> { + fn serialize( + &'a self, + builder: &mut capacity_builder::BytesBuilder<'a, Vec>, + ); +} + +#[derive(Deserialize, Serialize)] +pub enum NodeModules { + Managed { + /// Relative path for the node_modules directory in the vfs. + node_modules_dir: Option, + }, + Byonm { + root_node_modules_dir: Option, + }, +} + +#[derive(Deserialize, Serialize)] +pub struct SerializedWorkspaceResolverImportMap { + pub specifier: String, + pub json: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SerializedResolverWorkspaceJsrPackage { + pub relative_base: String, + pub name: String, + pub version: Option, + pub exports: IndexMap, +} + +#[derive(Deserialize, Serialize)] +pub struct SerializedWorkspaceResolver { + pub import_map: Option, + pub jsr_pkgs: Vec, + pub package_jsons: BTreeMap, + pub pkg_json_resolution: PackageJsonDepResolution, +} + +// Note: Don't use hashmaps/hashsets. Ensure the serialization +// is deterministic. +#[derive(Deserialize, Serialize)] +pub struct Metadata { + pub argv: Vec, + pub seed: Option, + pub code_cache_key: Option, + pub permissions: PermissionsOptions, + pub location: Option, + pub v8_flags: Vec, + pub log_level: Option, + pub ca_stores: Option>, + pub ca_data: Option>, + pub unsafely_ignore_certificate_errors: Option>, + pub env_vars_from_env_file: IndexMap, + pub workspace_resolver: SerializedWorkspaceResolver, + pub entrypoint_key: String, + pub node_modules: Option, + pub unstable_config: UnstableConfig, + pub otel_config: OtelConfig, + pub vfs_case_sensitivity: FileSystemCaseSensitivity, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct SpecifierId(u32); + +impl SpecifierId { + pub fn new(id: u32) -> Self { + Self(id) + } +} + +impl<'a> capacity_builder::BytesAppendable<'a> for SpecifierId { + fn append_to_builder( + self, + builder: &mut capacity_builder::BytesBuilder<'a, TBytes>, + ) { + builder.append_le(self.0); + } +} + +impl<'a> DenoRtSerializable<'a> for SpecifierId { + fn serialize( + &'a self, + builder: &mut capacity_builder::BytesBuilder<'a, Vec>, + ) { + builder.append_le(self.0); + } +} + +impl<'a> DenoRtDeserializable<'a> for SpecifierId { + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)> { + let (input, id) = read_u32(input)?; + Ok((input, Self(id))) + } +} + +#[derive(Deserialize, Serialize)] +pub enum CjsExportAnalysisEntry { + Esm, + Cjs(CjsAnalysisExports), +} + +const HAS_TRANSPILED_FLAG: u8 = 1 << 0; +const HAS_SOURCE_MAP_FLAG: u8 = 1 << 1; +const HAS_CJS_EXPORT_ANALYSIS_FLAG: u8 = 1 << 2; + +pub struct RemoteModuleEntry<'a> { + pub media_type: MediaType, + pub data: Cow<'a, [u8]>, + pub maybe_transpiled: Option>, + pub maybe_source_map: Option>, + pub maybe_cjs_export_analysis: Option>, +} + +impl<'a> DenoRtSerializable<'a> for RemoteModuleEntry<'a> { + fn serialize( + &'a self, + builder: &mut capacity_builder::BytesBuilder<'a, Vec>, + ) { + fn append_maybe_data<'a>( + builder: &mut capacity_builder::BytesBuilder<'a, Vec>, + maybe_data: Option<&'a [u8]>, + ) { + if let Some(data) = maybe_data { + builder.append_le(data.len() as u32); + builder.append(data); + } + } + + let mut has_data_flags = 0; + if self.maybe_transpiled.is_some() { + has_data_flags |= HAS_TRANSPILED_FLAG; + } + if self.maybe_source_map.is_some() { + has_data_flags |= HAS_SOURCE_MAP_FLAG; + } + if self.maybe_cjs_export_analysis.is_some() { + has_data_flags |= HAS_CJS_EXPORT_ANALYSIS_FLAG; + } + builder.append(serialize_media_type(self.media_type)); + builder.append_le(self.data.len() as u32); + builder.append(self.data.as_ref()); + builder.append(has_data_flags); + append_maybe_data(builder, self.maybe_transpiled.as_deref()); + append_maybe_data(builder, self.maybe_source_map.as_deref()); + append_maybe_data(builder, self.maybe_cjs_export_analysis.as_deref()); + } +} + +impl<'a> DenoRtDeserializable<'a> for RemoteModuleEntry<'a> { + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)> { + #[allow(clippy::type_complexity)] + fn deserialize_data_if_has_flag( + input: &[u8], + has_data_flags: u8, + flag: u8, + ) -> std::io::Result<(&[u8], Option>)> { + if has_data_flags & flag != 0 { + let (input, bytes) = read_bytes_with_u32_len(input)?; + Ok((input, Some(Cow::Borrowed(bytes)))) + } else { + Ok((input, None)) + } + } + + let (input, media_type) = MediaType::deserialize(input)?; + let (input, data) = read_bytes_with_u32_len(input)?; + let (input, has_data_flags) = read_u8(input)?; + let (input, maybe_transpiled) = + deserialize_data_if_has_flag(input, has_data_flags, HAS_TRANSPILED_FLAG)?; + let (input, maybe_source_map) = + deserialize_data_if_has_flag(input, has_data_flags, HAS_SOURCE_MAP_FLAG)?; + let (input, maybe_cjs_export_analysis) = deserialize_data_if_has_flag( + input, + has_data_flags, + HAS_CJS_EXPORT_ANALYSIS_FLAG, + )?; + Ok(( + input, + Self { + media_type, + data: Cow::Borrowed(data), + maybe_transpiled, + maybe_source_map, + maybe_cjs_export_analysis, + }, + )) + } +} + +fn serialize_media_type(media_type: MediaType) -> u8 { + match media_type { + MediaType::JavaScript => 0, + MediaType::Jsx => 1, + MediaType::Mjs => 2, + MediaType::Cjs => 3, + MediaType::TypeScript => 4, + MediaType::Mts => 5, + MediaType::Cts => 6, + MediaType::Dts => 7, + MediaType::Dmts => 8, + MediaType::Dcts => 9, + MediaType::Tsx => 10, + MediaType::Json => 11, + MediaType::Wasm => 12, + MediaType::Css => 13, + MediaType::SourceMap => 14, + MediaType::Unknown => 15, + } +} + +impl<'a> DenoRtDeserializable<'a> for MediaType { + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)> { + let (input, value) = read_u8(input)?; + let value = match value { + 0 => MediaType::JavaScript, + 1 => MediaType::Jsx, + 2 => MediaType::Mjs, + 3 => MediaType::Cjs, + 4 => MediaType::TypeScript, + 5 => MediaType::Mts, + 6 => MediaType::Cts, + 7 => MediaType::Dts, + 8 => MediaType::Dmts, + 9 => MediaType::Dcts, + 10 => MediaType::Tsx, + 11 => MediaType::Json, + 12 => MediaType::Wasm, + 13 => MediaType::Css, + 14 => MediaType::SourceMap, + 15 => MediaType::Unknown, + value => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Unknown media type value: {value}"), + )) + } + }; + Ok((input, value)) + } +} + +/// Data stored keyed by specifier. +pub struct SpecifierDataStore { + data: IndexMap, +} + +impl Default for SpecifierDataStore { + fn default() -> Self { + Self { + data: IndexMap::new(), + } + } +} + +impl SpecifierDataStore { + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: IndexMap::with_capacity(capacity), + } + } + + pub fn iter(&self) -> impl Iterator { + self.data.iter().map(|(k, v)| (*k, v)) + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn contains(&self, specifier: SpecifierId) -> bool { + self.data.contains_key(&specifier) + } + + pub fn add(&mut self, specifier: SpecifierId, value: TData) { + self.data.insert(specifier, value); + } + + pub fn get(&self, specifier: SpecifierId) -> Option<&TData> { + self.data.get(&specifier) + } +} + +impl<'a, TData> SpecifierDataStore +where + TData: DenoRtSerializable<'a> + 'a, +{ + pub fn serialize( + &'a self, + builder: &mut capacity_builder::BytesBuilder<'a, Vec>, + ) { + builder.append_le(self.len() as u32); + for (specifier, value) in self.iter() { + builder.append(specifier); + value.serialize(builder); + } + } +} + +impl<'a, TData> DenoRtDeserializable<'a> for SpecifierDataStore +where + TData: DenoRtDeserializable<'a>, +{ + fn deserialize(input: &'a [u8]) -> std::io::Result<(&'a [u8], Self)> { + let (input, len) = read_u32_as_usize(input)?; + let mut data = IndexMap::with_capacity(len); + let mut input = input; + for _ in 0..len { + let (new_input, specifier) = SpecifierId::deserialize(input)?; + let (new_input, value) = TData::deserialize(new_input)?; + data.insert(specifier, value); + input = new_input; + } + Ok((input, Self { data })) + } +} + +fn read_bytes_with_u32_len(input: &[u8]) -> std::io::Result<(&[u8], &[u8])> { + let (input, len) = read_u32_as_usize(input)?; + let (input, data) = read_bytes(input, len)?; + Ok((input, data)) +} + +fn read_u32_as_usize(input: &[u8]) -> std::io::Result<(&[u8], usize)> { + read_u32(input).map(|(input, len)| (input, len as usize)) +} + +fn read_u32(input: &[u8]) -> std::io::Result<(&[u8], u32)> { + let (input, len_bytes) = read_bytes(input, 4)?; + let len = u32::from_le_bytes(len_bytes.try_into().unwrap()); + Ok((input, len)) +} + +fn read_u8(input: &[u8]) -> std::io::Result<(&[u8], u8)> { + check_has_len(input, 1)?; + Ok((&input[1..], input[0])) +} + +fn read_bytes(input: &[u8], len: usize) -> std::io::Result<(&[u8], &[u8])> { + check_has_len(input, len)?; + let (len_bytes, input) = input.split_at(len); + Ok((input, len_bytes)) +} + +#[inline(always)] +fn check_has_len(input: &[u8], len: usize) -> std::io::Result<()> { + if input.len() < len { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Unexpected end of data", + )) + } else { + Ok(()) + } +} diff --git a/cli/lib/standalone/mod.rs b/cli/lib/standalone/mod.rs index 6e173a457a..42a5f20122 100644 --- a/cli/lib/standalone/mod.rs +++ b/cli/lib/standalone/mod.rs @@ -1,3 +1,4 @@ // Copyright 2018-2025 the Deno authors. MIT license. +pub mod binary; pub mod virtual_fs; diff --git a/cli/lib/standalone/virtual_fs.rs b/cli/lib/standalone/virtual_fs.rs index 5fc17f27b7..124c2a0002 100644 --- a/cli/lib/standalone/virtual_fs.rs +++ b/cli/lib/standalone/virtual_fs.rs @@ -1,20 +1,27 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::cmp::Ordering; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::fmt; use std::path::Path; use std::path::PathBuf; +use deno_path_util::normalize_path; +use deno_path_util::strip_unc_prefix; +use deno_runtime::colors; +use deno_runtime::deno_core::anyhow::bail; +use deno_runtime::deno_core::anyhow::Context; +use deno_runtime::deno_core::error::AnyError; +use indexmap::IndexSet; +use serde::de; +use serde::de::SeqAccess; +use serde::de::Visitor; use serde::Deserialize; +use serde::Deserializer; use serde::Serialize; - -#[derive(Debug, Copy, Clone)] -pub enum VfsFileSubDataKind { - /// Raw bytes of the file. - Raw, - /// Bytes to use for module loading. For example, for TypeScript - /// files this will be the transpiled JavaScript source. - ModuleGraph, -} +use serde::Serializer; #[derive(Debug, PartialEq, Eq)] pub enum WindowsSystemRootablePath { @@ -24,6 +31,14 @@ pub enum WindowsSystemRootablePath { } impl WindowsSystemRootablePath { + pub fn root_for_current_os() -> Self { + if cfg!(windows) { + WindowsSystemRootablePath::WindowSystemRoot + } else { + WindowsSystemRootablePath::Path(PathBuf::from("/")) + } + } + pub fn join(&self, name_component: &str) -> PathBuf { // this method doesn't handle multiple components debug_assert!( @@ -110,6 +125,10 @@ impl VirtualDirectoryEntries { self.0.get_mut(index) } + pub fn get_by_index(&self, index: usize) -> Option<&VfsEntry> { + self.0.get(index) + } + pub fn binary_search( &self, name: &str, @@ -180,27 +199,67 @@ pub struct VirtualDirectory { pub entries: VirtualDirectoryEntries, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy)] pub struct OffsetWithLength { - #[serde(rename = "o")] pub offset: u64, - #[serde(rename = "l")] pub len: u64, } +// serialize as an array in order to save space +impl Serialize for OffsetWithLength { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let array = [self.offset, self.len]; + array.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for OffsetWithLength { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct OffsetWithLengthVisitor; + + impl<'de> Visitor<'de> for OffsetWithLengthVisitor { + type Value = OffsetWithLength; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an array with two elements: [offset, len]") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let offset = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let len = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + Ok(OffsetWithLength { offset, len }) + } + } + + deserializer.deserialize_seq(OffsetWithLengthVisitor) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VirtualFile { #[serde(rename = "n")] pub name: String, #[serde(rename = "o")] pub offset: OffsetWithLength, - /// Offset file to use for module loading when it differs from the - /// raw file. Often this will be the same offset as above for data - /// such as JavaScript files, but for TypeScript files the `offset` - /// will be the original raw bytes when included as an asset and this - /// offset will be to the transpiled JavaScript source. - #[serde(rename = "m")] - pub module_graph_offset: OffsetWithLength, + #[serde(rename = "m", skip_serializing_if = "Option::is_none")] + pub transpiled_offset: Option, + #[serde(rename = "c", skip_serializing_if = "Option::is_none")] + pub cjs_export_analysis_offset: Option, + #[serde(rename = "s", skip_serializing_if = "Option::is_none")] + pub source_map_offset: Option, } #[derive(Debug, Serialize, Deserialize)] @@ -294,3 +353,647 @@ impl VfsEntry { } } } + +pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = + ".deno_compile_node_modules"; + +#[derive(Debug)] +pub struct BuiltVfs { + pub root_path: WindowsSystemRootablePath, + pub case_sensitivity: FileSystemCaseSensitivity, + pub entries: VirtualDirectoryEntries, + pub files: Vec>, +} + +#[derive(Debug, Default)] +struct FilesData { + files: Vec>, + current_offset: u64, + file_offsets: HashMap<(String, usize), OffsetWithLength>, +} + +impl FilesData { + pub fn file_bytes(&self, offset: OffsetWithLength) -> Option<&[u8]> { + if offset.len == 0 { + return Some(&[]); + } + + // the debug assertions in this method should never happen + // because it would indicate providing an offset not in the vfs + let mut count: u64 = 0; + for file in &self.files { + // clippy wanted a match + match count.cmp(&offset.offset) { + Ordering::Equal => { + debug_assert_eq!(offset.len, file.len() as u64); + if offset.len == file.len() as u64 { + return Some(file); + } else { + return None; + } + } + Ordering::Less => { + count += file.len() as u64; + } + Ordering::Greater => { + debug_assert!(false); + return None; + } + } + } + debug_assert!(false); + None + } + + pub fn add_data(&mut self, data: Vec) -> OffsetWithLength { + if data.is_empty() { + return OffsetWithLength { offset: 0, len: 0 }; + } + let checksum = crate::util::checksum::gen(&[&data]); + match self.file_offsets.entry((checksum, data.len())) { + Entry::Occupied(occupied_entry) => { + let offset_and_len = *occupied_entry.get(); + debug_assert_eq!(data.len() as u64, offset_and_len.len); + offset_and_len + } + Entry::Vacant(vacant_entry) => { + let offset_and_len = OffsetWithLength { + offset: self.current_offset, + len: data.len() as u64, + }; + vacant_entry.insert(offset_and_len); + self.current_offset += offset_and_len.len; + self.files.push(data); + offset_and_len + } + } + } +} + +pub struct AddFileDataOptions { + pub data: Vec, + pub maybe_transpiled: Option>, + pub maybe_source_map: Option>, + pub maybe_cjs_export_analysis: Option>, +} + +#[derive(Debug)] +pub struct VfsBuilder { + executable_root: VirtualDirectory, + files: FilesData, + /// The minimum root directory that should be included in the VFS. + min_root_dir: Option, + case_sensitivity: FileSystemCaseSensitivity, +} + +impl Default for VfsBuilder { + fn default() -> Self { + Self::new() + } +} + +impl VfsBuilder { + pub fn new() -> Self { + Self { + executable_root: VirtualDirectory { + name: "/".to_string(), + entries: Default::default(), + }, + files: Default::default(), + min_root_dir: Default::default(), + // This is not exactly correct because file systems on these OSes + // may be case-sensitive or not based on the directory, but this + // is a good enough approximation and limitation. In the future, + // we may want to store this information per directory instead + // depending on the feedback we get. + case_sensitivity: if cfg!(windows) || cfg!(target_os = "macos") { + FileSystemCaseSensitivity::Insensitive + } else { + FileSystemCaseSensitivity::Sensitive + }, + } + } + + pub fn case_sensitivity(&self) -> FileSystemCaseSensitivity { + self.case_sensitivity + } + + pub fn files_len(&self) -> usize { + self.files.files.len() + } + + pub fn file_bytes(&self, offset: OffsetWithLength) -> Option<&[u8]> { + self.files.file_bytes(offset) + } + + /// Add a directory that might be the minimum root directory + /// of the VFS. + /// + /// For example, say the user has a deno.json and specifies an + /// import map in a parent directory. The import map won't be + /// included in the VFS, but its base will meaning we need to + /// tell the VFS builder to include the base of the import map + /// by calling this method. + pub fn add_possible_min_root_dir(&mut self, path: &Path) { + self.add_dir_raw(path); + + match &self.min_root_dir { + Some(WindowsSystemRootablePath::WindowSystemRoot) => { + // already the root dir + } + Some(WindowsSystemRootablePath::Path(current_path)) => { + let mut common_components = Vec::new(); + for (a, b) in current_path.components().zip(path.components()) { + if a != b { + break; + } + common_components.push(a); + } + if common_components.is_empty() { + self.min_root_dir = + Some(WindowsSystemRootablePath::root_for_current_os()); + } else { + self.min_root_dir = Some(WindowsSystemRootablePath::Path( + common_components.iter().collect(), + )); + } + } + None => { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(path.to_path_buf())); + } + } + } + + pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { + let target_path = self.resolve_target_path(path)?; + self.add_dir_recursive_not_symlink(&target_path) + } + + fn add_dir_recursive_not_symlink( + &mut self, + path: &Path, + ) -> Result<(), AnyError> { + self.add_dir_raw(path); + // ok, building fs implementation + #[allow(clippy::disallowed_methods)] + let read_dir = std::fs::read_dir(path) + .with_context(|| format!("Reading {}", path.display()))?; + + let mut dir_entries = + read_dir.into_iter().collect::, _>>()?; + dir_entries.sort_by_cached_key(|entry| entry.file_name()); // determinism + + for entry in dir_entries { + let file_type = entry.file_type()?; + let path = entry.path(); + + if file_type.is_dir() { + self.add_dir_recursive_not_symlink(&path)?; + } else if file_type.is_file() { + self.add_file_at_path_not_symlink(&path)?; + } else if file_type.is_symlink() { + match self.add_symlink(&path) { + Ok(target) => match target { + SymlinkTarget::File(target) => { + self.add_file_at_path_not_symlink(&target)? + } + SymlinkTarget::Dir(target) => { + self.add_dir_recursive_not_symlink(&target)?; + } + }, + Err(err) => { + log::warn!( + "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", + colors::yellow("Warning"), + path.display(), + err + ); + } + } + } + } + + Ok(()) + } + + fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { + log::debug!("Ensuring directory '{}'", path.display()); + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let index = current_dir.entries.insert_or_modify( + &name, + self.case_sensitivity, + || { + VfsEntry::Dir(VirtualDirectory { + name: name.to_string(), + entries: Default::default(), + }) + }, + |_| { + // ignore + }, + ); + match current_dir.entries.get_mut_by_index(index) { + Some(VfsEntry::Dir(dir)) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + current_dir + } + + pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { + &mut self.executable_root + } + + pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let entry = current_dir + .entries + .get_mut_by_name(&name, self.case_sensitivity)?; + match entry { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!("{}", path.display()), + }; + } + + Some(current_dir) + } + + pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { + // ok, building fs implementation + #[allow(clippy::disallowed_methods)] + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data( + path, + AddFileDataOptions { + data: file_bytes, + maybe_cjs_export_analysis: None, + maybe_transpiled: None, + maybe_source_map: None, + }, + ) + } + + fn add_file_at_path_not_symlink( + &mut self, + path: &Path, + ) -> Result<(), AnyError> { + // ok, building fs implementation + #[allow(clippy::disallowed_methods)] + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data_raw(path, file_bytes) + } + + pub fn add_file_with_data( + &mut self, + path: &Path, + options: AddFileDataOptions, + ) -> Result<(), AnyError> { + // ok, fs implementation + #[allow(clippy::disallowed_methods)] + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + let target = self.add_symlink(path)?.into_path_buf(); + self.add_file_with_data_raw_options(&target, options) + } else { + self.add_file_with_data_raw_options(path, options) + } + } + + pub fn add_file_with_data_raw( + &mut self, + path: &Path, + data: Vec, + ) -> Result<(), AnyError> { + self.add_file_with_data_raw_options( + path, + AddFileDataOptions { + data, + maybe_transpiled: None, + maybe_cjs_export_analysis: None, + maybe_source_map: None, + }, + ) + } + + fn add_file_with_data_raw_options( + &mut self, + path: &Path, + options: AddFileDataOptions, + ) -> Result<(), AnyError> { + log::debug!("Adding file '{}'", path.display()); + let case_sensitivity = self.case_sensitivity; + + let offset_and_len = self.files.add_data(options.data); + let transpiled_offset = options + .maybe_transpiled + .map(|data| self.files.add_data(data)); + let source_map_offset = options + .maybe_source_map + .map(|data| self.files.add_data(data)); + let cjs_export_analysis_offset = options + .maybe_cjs_export_analysis + .map(|data| self.files.add_data(data)); + let dir = self.add_dir_raw(path.parent().unwrap()); + let name = path.file_name().unwrap().to_string_lossy(); + + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::File(VirtualFile { + name: name.to_string(), + offset: offset_and_len, + transpiled_offset, + cjs_export_analysis_offset, + source_map_offset, + }) + }, + |entry| match entry { + VfsEntry::File(virtual_file) => { + virtual_file.offset = offset_and_len; + // doesn't overwrite to None + if transpiled_offset.is_some() { + virtual_file.transpiled_offset = transpiled_offset; + } + if source_map_offset.is_some() { + virtual_file.source_map_offset = source_map_offset; + } + if cjs_export_analysis_offset.is_some() { + virtual_file.cjs_export_analysis_offset = + cjs_export_analysis_offset; + } + } + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), + }, + ); + + Ok(()) + } + + fn resolve_target_path(&mut self, path: &Path) -> Result { + // ok, fs implementation + #[allow(clippy::disallowed_methods)] + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + Ok(self.add_symlink(path)?.into_path_buf()) + } else { + Ok(path.to_path_buf()) + } + } + + pub fn add_symlink( + &mut self, + path: &Path, + ) -> Result { + self.add_symlink_inner(path, &mut IndexSet::new()) + } + + fn add_symlink_inner( + &mut self, + path: &Path, + visited: &mut IndexSet, + ) -> Result { + log::debug!("Adding symlink '{}'", path.display()); + let target = strip_unc_prefix( + // ok, fs implementation + #[allow(clippy::disallowed_methods)] + std::fs::read_link(path) + .with_context(|| format!("Reading symlink '{}'", path.display()))?, + ); + let case_sensitivity = self.case_sensitivity; + let target = normalize_path(path.parent().unwrap().join(&target)); + let dir = self.add_dir_raw(path.parent().unwrap()); + let name = path.file_name().unwrap().to_string_lossy(); + dir.entries.insert_or_modify( + &name, + case_sensitivity, + || { + VfsEntry::Symlink(VirtualSymlink { + name: name.to_string(), + dest_parts: VirtualSymlinkParts::from_path(&target), + }) + }, + |_| { + // ignore previously inserted + }, + ); + // ok, fs implementation + #[allow(clippy::disallowed_methods)] + let target_metadata = + std::fs::symlink_metadata(&target).with_context(|| { + format!("Reading symlink target '{}'", target.display()) + })?; + if target_metadata.is_symlink() { + if !visited.insert(target.clone()) { + // todo: probably don't error in this scenario + bail!( + "Circular symlink detected: {} -> {}", + visited + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> "), + target.display() + ); + } + self.add_symlink_inner(&target, visited) + } else if target_metadata.is_dir() { + Ok(SymlinkTarget::Dir(target)) + } else { + Ok(SymlinkTarget::File(target)) + } + } + + /// Adds the CJS export analysis to the provided file. + /// + /// Warning: This will panic if the file wasn't properly + /// setup before calling this. + pub fn add_cjs_export_analysis(&mut self, path: &Path, data: Vec) { + self.add_data_for_file_or_panic(path, data, |file, offset_with_length| { + file.cjs_export_analysis_offset = Some(offset_with_length); + }) + } + + fn add_data_for_file_or_panic( + &mut self, + path: &Path, + data: Vec, + update_file: impl FnOnce(&mut VirtualFile, OffsetWithLength), + ) { + let offset_with_length = self.files.add_data(data); + let case_sensitivity = self.case_sensitivity; + let dir = self.get_dir_mut(path.parent().unwrap()).unwrap(); + let name = path.file_name().unwrap().to_string_lossy(); + let file = dir + .entries + .get_mut_by_name(&name, case_sensitivity) + .unwrap(); + match file { + VfsEntry::File(virtual_file) => { + update_file(virtual_file, offset_with_length); + } + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => { + unreachable!() + } + } + } + + /// Iterates through all the files in the virtual file system. + pub fn iter_files( + &self, + ) -> impl Iterator + '_ { + FileIterator { + pending_dirs: VecDeque::from([( + WindowsSystemRootablePath::root_for_current_os(), + &self.executable_root, + )]), + current_dir_index: 0, + } + } + + pub fn build(self) -> BuiltVfs { + fn strip_prefix_from_symlinks( + dir: &mut VirtualDirectory, + parts: &[String], + ) { + for entry in dir.entries.iter_mut() { + match entry { + VfsEntry::Dir(dir) => { + strip_prefix_from_symlinks(dir, parts); + } + VfsEntry::File(_) => {} + VfsEntry::Symlink(symlink) => { + let parts = symlink + .dest_parts + .take_parts() + .into_iter() + .skip(parts.len()) + .collect(); + symlink.dest_parts.set_parts(parts); + } + } + } + } + + let mut current_dir = self.executable_root; + let mut current_path = WindowsSystemRootablePath::root_for_current_os(); + loop { + if current_dir.entries.len() != 1 { + break; + } + if self.min_root_dir.as_ref() == Some(¤t_path) { + break; + } + match current_dir.entries.iter().next().unwrap() { + VfsEntry::Dir(dir) => { + if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + // special directory we want to maintain + break; + } + match current_dir.entries.remove(0) { + VfsEntry::Dir(dir) => { + current_path = + WindowsSystemRootablePath::Path(current_path.join(&dir.name)); + current_dir = dir; + } + _ => unreachable!(), + }; + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => break, + } + } + if let WindowsSystemRootablePath::Path(path) = ¤t_path { + strip_prefix_from_symlinks( + &mut current_dir, + VirtualSymlinkParts::from_path(path).parts(), + ); + } + BuiltVfs { + root_path: current_path, + case_sensitivity: self.case_sensitivity, + entries: current_dir.entries, + files: self.files.files, + } + } +} + +struct FileIterator<'a> { + pending_dirs: VecDeque<(WindowsSystemRootablePath, &'a VirtualDirectory)>, + current_dir_index: usize, +} + +impl<'a> Iterator for FileIterator<'a> { + type Item = (PathBuf, &'a VirtualFile); + + fn next(&mut self) -> Option { + while !self.pending_dirs.is_empty() { + let (dir_path, current_dir) = self.pending_dirs.front()?; + if let Some(entry) = + current_dir.entries.get_by_index(self.current_dir_index) + { + self.current_dir_index += 1; + match entry { + VfsEntry::Dir(virtual_directory) => { + self.pending_dirs.push_back(( + WindowsSystemRootablePath::Path( + dir_path.join(&virtual_directory.name), + ), + virtual_directory, + )); + } + VfsEntry::File(virtual_file) => { + return Some((dir_path.join(&virtual_file.name), virtual_file)); + } + VfsEntry::Symlink(_) => { + // ignore + } + } + } else { + self.pending_dirs.pop_front(); + self.current_dir_index = 0; + } + } + None + } +} + +#[derive(Debug)] +pub enum SymlinkTarget { + File(PathBuf), + Dir(PathBuf), +} + +impl SymlinkTarget { + pub fn into_path_buf(self) -> PathBuf { + match self { + Self::File(path) => path, + Self::Dir(path) => path, + } + } +} diff --git a/cli/cache/common.rs b/cli/lib/util/hash.rs similarity index 100% rename from cli/cache/common.rs rename to cli/lib/util/hash.rs diff --git a/cli/util/logger.rs b/cli/lib/util/logger.rs similarity index 74% rename from cli/util/logger.rs rename to cli/lib/util/logger.rs index 2bd4760ebd..b280dc22ed 100644 --- a/cli/util/logger.rs +++ b/cli/lib/util/logger.rs @@ -2,44 +2,33 @@ use std::io::Write; -use deno_telemetry::OtelConfig; -use deno_telemetry::OtelConsoleConfig; +use deno_runtime::deno_telemetry; +use deno_runtime::deno_telemetry::OtelConfig; +use deno_runtime::deno_telemetry::OtelConsoleConfig; -use super::draw_thread::DrawThread; - -struct CliLogger { +struct CliLogger { otel_console_config: OtelConsoleConfig, logger: env_logger::Logger, + on_log_start: FnOnLogStart, + on_log_end: FnOnLogEnd, } -impl CliLogger { - pub fn new( - logger: env_logger::Logger, - otel_console_config: OtelConsoleConfig, - ) -> Self { - Self { - logger, - otel_console_config, - } - } - +impl CliLogger { pub fn filter(&self) -> log::LevelFilter { self.logger.filter() } } -impl log::Log for CliLogger { +impl log::Log + for CliLogger +{ fn enabled(&self, metadata: &log::Metadata) -> bool { self.logger.enabled(metadata) } fn log(&self, record: &log::Record) { if self.enabled(record.metadata()) { - // it was considered to hold the draw thread's internal lock - // across logging, but if outputting to stderr blocks then that - // could potentially block other threads that access the draw - // thread's state - DrawThread::hide(); + (self.on_log_start)(); match self.otel_console_config { OtelConsoleConfig::Ignore => { @@ -54,7 +43,7 @@ impl log::Log for CliLogger { } } - DrawThread::show(); + (self.on_log_end)(); } } @@ -63,8 +52,20 @@ impl log::Log for CliLogger { } } -pub fn init(maybe_level: Option, otel_config: Option) { - let log_level = maybe_level.unwrap_or(log::Level::Info); +pub struct InitLoggingOptions { + pub on_log_start: FnOnLogStart, + pub on_log_end: FnOnLogEnd, + pub maybe_level: Option, + pub otel_config: Option, +} + +pub fn init< + FOnLogStart: Fn() + Send + Sync + 'static, + FnOnLogEnd: Fn() + Send + Sync + 'static, +>( + options: InitLoggingOptions, +) { + let log_level = options.maybe_level.unwrap_or(log::Level::Info); let logger = env_logger::Builder::from_env( env_logger::Env::new() // Use `DENO_LOG` and `DENO_LOG_STYLE` instead of `RUST_` prefix @@ -117,12 +118,15 @@ pub fn init(maybe_level: Option, otel_config: Option) { }) .build(); - let cli_logger = CliLogger::new( + let cli_logger = CliLogger { + on_log_start: options.on_log_start, + on_log_end: options.on_log_end, logger, - otel_config + otel_console_config: options + .otel_config .map(|c| c.console) .unwrap_or(OtelConsoleConfig::Ignore), - ); + }; let max_level = cli_logger.filter(); let r = log::set_boxed_logger(Box::new(cli_logger)); if r.is_ok() { diff --git a/cli/lib/util/mod.rs b/cli/lib/util/mod.rs index 8371440750..27643a2009 100644 --- a/cli/lib/util/mod.rs +++ b/cli/lib/util/mod.rs @@ -1,3 +1,8 @@ // Copyright 2018-2025 the Deno authors. MIT license. pub mod checksum; +pub mod hash; +pub mod logger; +pub mod result; +pub mod text_encoding; +pub mod v8; diff --git a/cli/util/result.rs b/cli/lib/util/result.rs similarity index 90% rename from cli/util/result.rs rename to cli/lib/util/result.rs index e6d45be470..3e302e5dfc 100644 --- a/cli/util/result.rs +++ b/cli/lib/util/result.rs @@ -4,10 +4,10 @@ use std::convert::Infallible; use std::fmt::Debug; use std::fmt::Display; -use deno_core::error::AnyError; -use deno_core::error::CoreError; use deno_error::JsErrorBox; use deno_error::JsErrorClass; +use deno_runtime::deno_core::error::AnyError; +use deno_runtime::deno_core::error::CoreError; pub trait InfallibleResultExt { fn unwrap_infallible(self) -> T; diff --git a/cli/lib/util/text_encoding.rs b/cli/lib/util/text_encoding.rs new file mode 100644 index 0000000000..5b6e5f43b6 --- /dev/null +++ b/cli/lib/util/text_encoding.rs @@ -0,0 +1,45 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::sync::Arc; + +#[inline(always)] +pub fn from_utf8_lossy_owned(bytes: Vec) -> String { + match String::from_utf8_lossy(&bytes) { + Cow::Owned(code) => code, + // SAFETY: `String::from_utf8_lossy` guarantees that the result is valid + // UTF-8 if `Cow::Borrowed` is returned. + Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(bytes) }, + } +} + +#[inline(always)] +pub fn from_utf8_lossy_cow(bytes: Cow<[u8]>) -> Cow { + match bytes { + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + Cow::Owned(bytes) => Cow::Owned(from_utf8_lossy_owned(bytes)), + } +} + +/// Converts an `Arc` to an `Arc<[u8]>`. +#[allow(dead_code)] +pub fn arc_str_to_bytes(arc_str: Arc) -> Arc<[u8]> { + let raw = Arc::into_raw(arc_str); + // SAFETY: This is safe because they have the same memory layout. + unsafe { Arc::from_raw(raw as *const [u8]) } +} + +/// Converts an `Arc` to an `Arc` if able. +#[allow(dead_code)] +pub fn arc_u8_to_arc_str( + arc_u8: Arc<[u8]>, +) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { + std::mem::transmute::, std::sync::Arc>(arc_u8) + }) +} diff --git a/cli/lib/util/v8.rs b/cli/lib/util/v8.rs new file mode 100644 index 0000000000..976fbf531b --- /dev/null +++ b/cli/lib/util/v8.rs @@ -0,0 +1,14 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +#[inline(always)] +pub fn construct_v8_flags( + default_v8_flags: &[String], + v8_flags: &[String], + env_v8_flags: Vec, +) -> Vec { + std::iter::once("UNUSED_BUT_NECESSARY_ARG0".to_owned()) + .chain(default_v8_flags.iter().cloned()) + .chain(env_v8_flags) + .chain(v8_flags.iter().cloned()) + .collect::>() +} diff --git a/cli/lib/version.rs b/cli/lib/version.rs new file mode 100644 index 0000000000..88a25dffeb --- /dev/null +++ b/cli/lib/version.rs @@ -0,0 +1,94 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; + +use deno_runtime::deno_telemetry::OtelRuntimeConfig; + +use crate::shared::ReleaseChannel; + +pub fn otel_runtime_config() -> OtelRuntimeConfig { + OtelRuntimeConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + } +} + +const GIT_COMMIT_HASH: &str = env!("GIT_COMMIT_HASH"); +const TYPESCRIPT: &str = "5.6.2"; +const DENO_VERSION: &str = env!("DENO_VERSION"); +// TODO(bartlomieju): ideally we could remove this const. +const IS_CANARY: bool = option_env!("DENO_CANARY").is_some(); +// TODO(bartlomieju): this is temporary, to allow Homebrew to cut RC releases as well +const IS_RC: bool = option_env!("DENO_RC").is_some(); + +pub static DENO_VERSION_INFO: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + let release_channel = libsui::find_section("denover") + .and_then(|buf| std::str::from_utf8(buf).ok()) + .and_then(|str_| ReleaseChannel::deserialize(str_).ok()) + .unwrap_or({ + if IS_CANARY { + ReleaseChannel::Canary + } else if IS_RC { + ReleaseChannel::Rc + } else { + ReleaseChannel::Stable + } + }); + + DenoVersionInfo { + deno: if release_channel == ReleaseChannel::Canary { + concat!(env!("DENO_VERSION"), "+", env!("GIT_COMMIT_HASH_SHORT")) + } else { + env!("DENO_VERSION") + }, + + release_channel, + + git_hash: GIT_COMMIT_HASH, + + // Keep in sync with `deno` field. + user_agent: if release_channel == ReleaseChannel::Canary { + concat!( + "Deno/", + env!("DENO_VERSION"), + "+", + env!("GIT_COMMIT_HASH_SHORT") + ) + } else { + concat!("Deno/", env!("DENO_VERSION")) + }, + + typescript: TYPESCRIPT, + } + }); + +pub struct DenoVersionInfo { + /// Human-readable version of the current Deno binary. + /// + /// For stable release, a semver, eg. `v1.46.2`. + /// For canary release, a semver + 7-char git hash, eg. `v1.46.3+asdfqwq`. + pub deno: &'static str, + + pub release_channel: ReleaseChannel, + + /// A full git hash. + pub git_hash: &'static str, + + /// A user-agent header that will be used in HTTP client. + pub user_agent: &'static str, + + pub typescript: &'static str, +} + +impl DenoVersionInfo { + /// For stable release, a semver like, eg. `v1.46.2`. + /// For canary release a full git hash, eg. `9bdab6fb6b93eb43b1930f40987fa4997287f9c8`. + pub fn version_or_git_hash(&self) -> &'static str { + if self.release_channel == ReleaseChannel::Canary { + self.git_hash + } else { + DENO_VERSION + } + } +} diff --git a/cli/lib/version.txt b/cli/lib/version.txt new file mode 100644 index 0000000000..b6da51221f --- /dev/null +++ b/cli/lib/version.txt @@ -0,0 +1 @@ +2.1.6 \ No newline at end of file diff --git a/cli/lib/worker.rs b/cli/lib/worker.rs index 180d3eef8c..01862bfd82 100644 --- a/cli/lib/worker.rs +++ b/cli/lib/worker.rs @@ -1,5 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. +use std::path::Path; use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; @@ -42,9 +43,10 @@ use deno_runtime::BootstrapOptions; use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; use deno_runtime::UNSTABLE_GRANULAR_FLAGS; +use node_resolver::errors::ResolvePkgJsonBinExportError; use url::Url; -use crate::env::has_trace_permissions_enabled; +use crate::args::has_trace_permissions_enabled; use crate::sys::DenoLibSys; use crate::util::checksum; @@ -113,9 +115,9 @@ impl StorageKeyResolver { } } -// TODO(bartlomieju): this should be moved to some other place, added to avoid string -// duplication between worker setups and `deno info` output. pub fn get_cache_storage_dir() -> PathBuf { + // ok because this won't ever be used by the js runtime + #[allow(clippy::disallowed_methods)] // Note: we currently use temp_dir() to avoid managing storage size. std::env::temp_dir().join("deno_cache") } @@ -131,10 +133,31 @@ pub fn create_isolate_create_params() -> Option { }) } +#[derive(Debug, thiserror::Error, deno_error::JsError)] +pub enum ResolveNpmBinaryEntrypointError { + #[class(inherit)] + #[error(transparent)] + ResolvePkgJsonBinExport(ResolvePkgJsonBinExportError), + #[class(generic)] + #[error("{original:#}\n\nFallback failed: {fallback:#}")] + Fallback { + fallback: ResolveNpmBinaryEntrypointFallbackError, + original: ResolvePkgJsonBinExportError, + }, +} + +#[derive(Debug, thiserror::Error, deno_error::JsError)] +pub enum ResolveNpmBinaryEntrypointFallbackError { + #[class(inherit)] + #[error(transparent)] + PackageSubpathResolve(node_resolver::errors::PackageSubpathResolveError), + #[class(generic)] + #[error("Cannot find module '{0}'")] + ModuleNotFound(Url), +} + pub struct LibMainWorkerOptions { pub argv: Vec, - pub deno_version: &'static str, - pub deno_user_agent: &'static str, pub log_level: WorkerLogLevel, pub enable_op_summary_metrics: bool, pub enable_testing_features: bool, @@ -263,7 +286,7 @@ impl LibWorkerFactorySharedState { main_module: args.main_module.clone(), worker_id: args.worker_id, bootstrap: BootstrapOptions { - deno_version: shared.options.deno_version.to_string(), + deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), args: shared.options.argv.clone(), cpu_count: std::thread::available_parallelism() .map(|p| p.get()) @@ -278,7 +301,7 @@ impl LibWorkerFactorySharedState { is_stdout_tty: deno_terminal::is_stdout_tty(), is_stderr_tty: deno_terminal::is_stderr_tty(), unstable_features, - user_agent: shared.options.deno_user_agent.to_string(), + user_agent: crate::version::DENO_VERSION_INFO.user_agent.to_string(), inspect: shared.options.is_inspecting, has_node_modules_dir: shared.options.has_node_modules_dir, argv0: shared.options.argv0.clone(), @@ -359,6 +382,21 @@ impl LibMainWorkerFactory { } } + pub fn create_main_worker( + &self, + mode: WorkerExecutionMode, + permissions: PermissionsContainer, + main_module: Url, + ) -> Result { + self.create_custom_worker( + mode, + main_module, + permissions, + vec![], + Default::default(), + ) + } + pub fn create_custom_worker( &self, mode: WorkerExecutionMode, @@ -420,7 +458,7 @@ impl LibMainWorkerFactory { let options = WorkerOptions { bootstrap: BootstrapOptions { - deno_version: shared.options.deno_version.to_string(), + deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), args: shared.options.argv.clone(), cpu_count: std::thread::available_parallelism() .map(|p| p.get()) @@ -435,7 +473,7 @@ impl LibMainWorkerFactory { is_stderr_tty: deno_terminal::is_stderr_tty(), color_level: colors::get_color_level(), unstable_features, - user_agent: shared.options.deno_user_agent.to_string(), + user_agent: crate::version::DENO_VERSION_INFO.user_agent.to_string(), inspect: shared.options.is_inspecting, has_node_modules_dir: shared.options.has_node_modules_dir, argv0: shared.options.argv0.clone(), @@ -476,6 +514,76 @@ impl LibMainWorkerFactory { worker, }) } + + pub fn resolve_npm_binary_entrypoint( + &self, + package_folder: &Path, + sub_path: Option<&str>, + ) -> Result { + match self + .shared + .node_resolver + .resolve_binary_export(package_folder, sub_path) + { + Ok(specifier) => Ok(specifier), + Err(original_err) => { + // if the binary entrypoint was not found, fallback to regular node resolution + let result = + self.resolve_binary_entrypoint_fallback(package_folder, sub_path); + match result { + Ok(Some(specifier)) => Ok(specifier), + Ok(None) => { + Err(ResolveNpmBinaryEntrypointError::ResolvePkgJsonBinExport( + original_err, + )) + } + Err(fallback_err) => Err(ResolveNpmBinaryEntrypointError::Fallback { + original: original_err, + fallback: fallback_err, + }), + } + } + } + } + + /// resolve the binary entrypoint using regular node resolution + fn resolve_binary_entrypoint_fallback( + &self, + package_folder: &Path, + sub_path: Option<&str>, + ) -> Result, ResolveNpmBinaryEntrypointFallbackError> { + // only fallback if the user specified a sub path + if sub_path.is_none() { + // it's confusing to users if the package doesn't have any binary + // entrypoint and we just execute the main script which will likely + // have blank output, so do not resolve the entrypoint in this case + return Ok(None); + } + + let specifier = self + .shared + .node_resolver + .resolve_package_subpath_from_deno_module( + package_folder, + sub_path, + /* referrer */ None, + node_resolver::ResolutionMode::Import, + node_resolver::NodeResolutionKind::Execution, + ) + .map_err( + ResolveNpmBinaryEntrypointFallbackError::PackageSubpathResolve, + )?; + if deno_path_util::url_to_file_path(&specifier) + .map(|p| self.shared.sys.fs_exists_no_err(p)) + .unwrap_or(false) + { + Ok(Some(specifier)) + } else { + Err(ResolveNpmBinaryEntrypointFallbackError::ModuleNotFound( + specifier, + )) + } + } } pub struct LibMainWorker { @@ -536,6 +644,33 @@ impl LibMainWorker { self.worker.evaluate_module(id).await } + pub async fn run(&mut self) -> Result { + log::debug!("main_module {}", self.main_module); + + self.execute_main_module().await?; + self.worker.dispatch_load_event()?; + + loop { + self + .worker + .run_event_loop(/* wait for inspector */ false) + .await?; + + let web_continue = self.worker.dispatch_beforeunload_event()?; + if !web_continue { + let node_continue = self.worker.dispatch_process_beforeexit_event()?; + if !node_continue { + break; + } + } + } + + self.worker.dispatch_unload_event()?; + self.worker.dispatch_process_exit_event()?; + + Ok(self.worker.exit_code()) + } + #[inline] pub async fn run_event_loop( &mut self, diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index a65bbd5efe..97fbbaff14 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -8,9 +8,9 @@ use std::time::SystemTime; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_lib::cache::DenoDir; use deno_path_util::url_to_file_path; +use crate::cache::DenoDir; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; use crate::cache::LocalLspHttpCache; @@ -70,7 +70,7 @@ fn calculate_fs_version_in_cache( #[derive(Debug, Clone)] pub struct LspCache { - deno_dir: DenoDir, + deno_dir: DenoDir, global: Arc, vendors_by_scope: BTreeMap>>, } @@ -121,7 +121,7 @@ impl LspCache { .collect(); } - pub fn deno_dir(&self) -> &DenoDir { + pub fn deno_dir(&self) -> &DenoDir { &self.deno_dir } diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 98c4498a1a..a0456c14ed 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -41,7 +41,8 @@ use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_lib::env::has_flag_env_var; +use deno_lib::args::has_flag_env_var; +use deno_lib::util::hash::FastInsecureHasher; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; @@ -61,7 +62,6 @@ use crate::args::CliLockfileReadFromPathOptions; use crate::args::ConfigFile; use crate::args::LintFlags; use crate::args::LintOptions; -use crate::cache::FastInsecureHasher; use crate::file_fetcher::CliFileFetcher; use crate::lsp::logging::lsp_warn; use crate::resolver::CliSloppyImportsResolver; diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index 3e3e31de28..126e8ef01d 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -265,7 +265,7 @@ impl TsDiagnosticsStore { } pub fn should_send_diagnostic_batch_index_notifications() -> bool { - deno_lib::env::has_flag_env_var( + deno_lib::args::has_flag_env_var( "DENO_DONT_USE_INTERNAL_LSP_DIAGNOSTIC_SYNC_FLAG", ) } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index c2fddc08bd..d14f0e9246 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -27,7 +27,10 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; -use deno_lib::env::has_flag_env_var; +use deno_lib::args::get_root_cert_store; +use deno_lib::args::has_flag_env_var; +use deno_lib::args::CaData; +use deno_lib::version::DENO_VERSION_INFO; use deno_path_util::url_to_file_path; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; @@ -95,8 +98,6 @@ use super::urls; use super::urls::uri_to_url; use super::urls::url_to_uri; use crate::args::create_default_npmrc; -use crate::args::get_root_cert_store; -use crate::args::CaData; use crate::args::CliOptions; use crate::args::Flags; use crate::args::InternalFlags; @@ -703,7 +704,7 @@ impl Inner { let version = format!( "{} ({}, {})", - crate::version::DENO_VERSION_INFO.deno, + DENO_VERSION_INFO.deno, env!("PROFILE"), env!("TARGET") ); diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 32352d9f26..e1b3691c0a 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -39,6 +39,7 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; use deno_core::RuntimeOptions; +use deno_lib::util::result::InfallibleResultExt; use deno_lib::worker::create_isolate_create_params; use deno_path_util::url_to_file_path; use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES; @@ -96,7 +97,6 @@ use crate::tsc::ResolveArgs; use crate::tsc::MISSING_DEPENDENCY_SPECIFIER; use crate::util::path::relative_specifier; use crate::util::path::to_percent_decoded_str; -use crate::util::result::InfallibleResultExt; use crate::util::v8::convert; static BRACKET_ACCESSOR_RE: Lazy = diff --git a/cli/main.rs b/cli/main.rs index a6b552c636..8501a3487f 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -17,16 +17,18 @@ mod node; mod npm; mod ops; mod resolver; -mod shared; mod standalone; -mod sys; mod task_runner; mod tools; mod tsc; mod util; -mod version; mod worker; +pub mod sys { + #[allow(clippy::disallowed_types)] // ok, definition + pub type CliSys = sys_traits::impls::RealSys; +} + use std::env; use std::future::Future; use std::io::IsTerminal; @@ -40,18 +42,22 @@ use deno_core::error::AnyError; use deno_core::error::CoreError; use deno_core::futures::FutureExt; use deno_core::unsync::JoinHandle; +use deno_lib::util::result::any_and_jserrorbox_downcast_ref; use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics; use deno_runtime::WorkerExecutionMode; pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS; +use deno_telemetry::OtelConfig; use deno_terminal::colors; use factory::CliFactory; -use standalone::MODULE_NOT_FOUND; -use standalone::UNSUPPORTED_SCHEME; + +const MODULE_NOT_FOUND: &str = "Module not found"; +const UNSUPPORTED_SCHEME: &str = "Unsupported scheme"; use self::npm::ResolveSnapshotError; +use self::util::draw_thread::DrawThread; use crate::args::flags_from_vec; use crate::args::DenoSubcommand; use crate::args::Flags; @@ -201,7 +207,7 @@ async fn run_subcommand(flags: Arc) -> Result { match result { Ok(v) => Ok(v), Err(script_err) => { - if let Some(worker::CreateCustomWorkerError::ResolvePkgFolderFromDenoReq(ResolvePkgFolderFromDenoReqError::Byonm(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq(_)))) = util::result::any_and_jserrorbox_downcast_ref::(&script_err) { + if let Some(worker::CreateCustomWorkerError::ResolvePkgFolderFromDenoReq(ResolvePkgFolderFromDenoReqError::Byonm(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq(_)))) = any_and_jserrorbox_downcast_ref::(&script_err) { if flags.node_modules_dir.is_none() { let mut flags = flags.deref().clone(); let watch = match &flags.subcommand { @@ -351,7 +357,7 @@ fn setup_panic_hook() { eprintln!("var set and include the backtrace in your report."); eprintln!(); eprintln!("Platform: {} {}", env::consts::OS, env::consts::ARCH); - eprintln!("Version: {}", version::DENO_VERSION_INFO.deno); + eprintln!("Version: {}", deno_lib::version::DENO_VERSION_INFO.deno); eprintln!("Args: {:?}", env::args().collect::>()); eprintln!(); orig_hook(panic_info); @@ -373,13 +379,11 @@ fn exit_for_error(error: AnyError) -> ! { let mut error_code = 1; if let Some(CoreError::Js(e)) = - util::result::any_and_jserrorbox_downcast_ref::(&error) + any_and_jserrorbox_downcast_ref::(&error) { error_string = format_js_error(e); } else if let Some(e @ ResolveSnapshotError { .. }) = - util::result::any_and_jserrorbox_downcast_ref::( - &error, - ) + any_and_jserrorbox_downcast_ref::(&error) { if let Some(e) = e.maybe_integrity_check_error() { error_string = e.to_string(); @@ -442,19 +446,19 @@ fn resolve_flags_and_init( if err.kind() == clap::error::ErrorKind::DisplayVersion => { // Ignore results to avoid BrokenPipe errors. - util::logger::init(None, None); + init_logging(None, None); let _ = err.print(); deno_runtime::exit(0); } Err(err) => { - util::logger::init(None, None); + init_logging(None, None); exit_for_error(AnyError::from(err)) } }; let otel_config = flags.otel_config(); - deno_telemetry::init(crate::args::otel_runtime_config(), &otel_config)?; - util::logger::init(flags.log_level, Some(otel_config)); + deno_telemetry::init(deno_lib::version::otel_runtime_config(), &otel_config)?; + init_logging(flags.log_level, Some(otel_config)); // TODO(bartlomieju): remove in Deno v2.5 and hard error then. if flags.unstable_config.legacy_flag_enabled { @@ -487,3 +491,19 @@ fn resolve_flags_and_init( Ok(flags) } + +fn init_logging( + maybe_level: Option, + otel_config: Option, +) { + deno_lib::util::logger::init(deno_lib::util::logger::InitLoggingOptions { + maybe_level, + otel_config, + // it was considered to hold the draw thread's internal lock + // across logging, but if outputting to stderr blocks then that + // could potentially block other threads that access the draw + // thread's state + on_log_start: DrawThread::hide, + on_log_end: DrawThread::show, + }) +} diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 7acb947491..279fc6422e 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -37,7 +37,11 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::Resolution; use deno_graph::WasmModule; +use deno_lib::loader::ModuleCodeStringSource; +use deno_lib::loader::NotSupportedKindInNpmError; +use deno_lib::loader::NpmModuleLoadError; use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::util::hash::FastInsecureHasher; use deno_lib::worker::CreateModuleLoaderResult; use deno_lib::worker::ModuleLoaderFactory; use deno_resolver::npm::DenoInNpmPackageChecker; @@ -45,6 +49,7 @@ use deno_runtime::code_cache; use deno_runtime::deno_node::create_host_defined_options; use deno_runtime::deno_node::ops::require::UnableToGetCwdError; use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::ClosestPkgJsonError; @@ -59,7 +64,6 @@ use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::TsTypeLib; use crate::cache::CodeCache; -use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; use crate::graph_container::MainModuleGraphContainer; @@ -69,15 +73,13 @@ use crate::graph_util::enhance_graph_error; use crate::graph_util::CreateGraphOptions; use crate::graph_util::EnhanceGraphErrorMode; use crate::graph_util::ModuleGraphBuilder; +use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; -use crate::resolver::ModuleCodeStringSource; -use crate::resolver::NotSupportedKindInNpmError; -use crate::resolver::NpmModuleLoader; use crate::sys::CliSys; use crate::tools::check; use crate::tools::check::CheckError; @@ -86,6 +88,14 @@ use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; +pub type CliNpmModuleLoader = deno_lib::loader::NpmModuleLoader< + CliCjsCodeAnalyzer, + DenoInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + CliNpmResolver, + CliSys, +>; + #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum PrepareModuleLoadError { #[class(inherit)] @@ -246,7 +256,7 @@ struct SharedCliModuleLoaderState { module_load_preparer: Arc, node_code_translator: Arc, node_resolver: Arc, - npm_module_loader: NpmModuleLoader, + npm_module_loader: CliNpmModuleLoader, npm_registry_permission_checker: Arc>, npm_req_resolver: Arc, @@ -308,7 +318,7 @@ impl CliModuleLoaderFactory { module_load_preparer: Arc, node_code_translator: Arc, node_resolver: Arc, - npm_module_loader: NpmModuleLoader, + npm_module_loader: CliNpmModuleLoader, npm_registry_permission_checker: Arc< NpmRegistryReadPermissionChecker, >, @@ -427,7 +437,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { pub enum LoadCodeSourceError { #[class(inherit)] #[error(transparent)] - NpmModuleLoad(crate::resolver::NpmModuleLoadError), + NpmModuleLoad(NpmModuleLoadError), #[class(inherit)] #[error(transparent)] LoadPreparedModule(#[from] LoadPreparedModuleError), diff --git a/cli/node.rs b/cli/node.rs index f6411f5e53..90fd5645c7 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -12,7 +12,6 @@ use deno_runtime::deno_fs; use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; use node_resolver::analyze::CjsAnalysisExports; -use node_resolver::analyze::CjsCodeAnalysisError; use node_resolver::analyze::CjsCodeAnalyzer; use node_resolver::analyze::NodeCodeTranslator; use serde::Deserialize; @@ -76,7 +75,7 @@ impl CliCjsCodeAnalyzer { &self, specifier: &ModuleSpecifier, source: &str, - ) -> Result { + ) -> Result { let source_hash = CacheDBHash::from_hashable(source); if let Some(analysis) = self.cache.get_cjs_analysis(specifier.as_str(), source_hash) @@ -93,7 +92,9 @@ impl CliCjsCodeAnalyzer { } let cjs_tracker = self.cjs_tracker.clone(); - let is_maybe_cjs = cjs_tracker.is_maybe_cjs(specifier, media_type)?; + let is_maybe_cjs = cjs_tracker + .is_maybe_cjs(specifier, media_type) + .map_err(JsErrorBox::from_err)?; let analysis = if is_maybe_cjs { let maybe_parsed_source = self .parsed_source_cache @@ -103,7 +104,7 @@ impl CliCjsCodeAnalyzer { deno_core::unsync::spawn_blocking({ let specifier = specifier.clone(); let source: Arc = source.into(); - move || -> Result<_, CjsCodeAnalysisError> { + move || -> Result<_, JsErrorBox> { let parsed_source = maybe_parsed_source .map(Ok) .unwrap_or_else(|| { @@ -118,11 +119,13 @@ impl CliCjsCodeAnalyzer { }) .map_err(JsErrorBox::from_err)?; let is_script = parsed_source.compute_is_script(); - let is_cjs = cjs_tracker.is_cjs_with_known_is_script( - parsed_source.specifier(), - media_type, - is_script, - )?; + let is_cjs = cjs_tracker + .is_cjs_with_known_is_script( + parsed_source.specifier(), + media_type, + is_script, + ) + .map_err(JsErrorBox::from_err)?; if is_cjs { let analysis = parsed_source.analyze_cjs(); Ok(CliCjsAnalysis::Cjs { @@ -154,7 +157,7 @@ impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { &self, specifier: &ModuleSpecifier, source: Option>, - ) -> Result, CjsCodeAnalysisError> { + ) -> Result, JsErrorBox> { let source = match source { Some(source) => source, None => { diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs deleted file mode 100644 index d52b222074..0000000000 --- a/cli/npm/byonm.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -use std::sync::Arc; - -use deno_core::serde_json; -use deno_resolver::npm::ByonmNpmResolver; -use deno_resolver::npm::ByonmNpmResolverCreateOptions; -use deno_runtime::deno_process::NpmProcessStateProvider; - -use crate::args::NpmProcessState; -use crate::args::NpmProcessStateKind; -use crate::sys::CliSys; - -pub type CliByonmNpmResolverCreateOptions = - ByonmNpmResolverCreateOptions; -pub type CliByonmNpmResolver = ByonmNpmResolver; - -#[derive(Debug)] -pub struct CliByonmNpmProcessStateProvider(pub Arc); - -impl NpmProcessStateProvider for CliByonmNpmProcessStateProvider { - fn get_npm_process_state(&self) -> String { - serde_json::to_string(&NpmProcessState { - kind: NpmProcessStateKind::Byonm, - local_node_modules_path: self - .0 - .root_node_modules_path() - .map(|p| p.to_string_lossy().to_string()), - }) - .unwrap() - } -} diff --git a/cli/npm/installer/common/lifecycle_scripts.rs b/cli/npm/installer/common/lifecycle_scripts.rs index 3238b8d023..64b06aecbf 100644 --- a/cli/npm/installer/common/lifecycle_scripts.rs +++ b/cli/npm/installer/common/lifecycle_scripts.rs @@ -220,7 +220,7 @@ impl<'a> LifecycleScripts<'a> { get_package_path, ); let init_cwd = &self.config.initial_cwd; - let process_state = crate::npm::managed::npm_process_state( + let process_state = deno_lib::npm::npm_process_state( snapshot.as_valid_serialized(), Some(root_node_modules_dir_path), ); diff --git a/cli/npm/installer/global.rs b/cli/npm/installer/global.rs index a6b296c6d8..f074c62174 100644 --- a/cli/npm/installer/global.rs +++ b/cli/npm/installer/global.rs @@ -9,6 +9,7 @@ use async_trait::async_trait; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::StreamExt; use deno_error::JsErrorBox; +use deno_lib::util::hash::FastInsecureHasher; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; use deno_resolver::npm::managed::NpmResolutionCell; @@ -17,7 +18,6 @@ use super::common::lifecycle_scripts::LifecycleScriptsStrategy; use super::common::NpmPackageFsInstaller; use super::PackageCaching; use crate::args::LifecycleScriptsConfig; -use crate::cache::FastInsecureHasher; use crate::colors; use crate::npm::CliNpmCache; use crate::npm::CliNpmTarballCache; diff --git a/cli/npm/managed.rs b/cli/npm/managed.rs index 049e3541db..14ba088d89 100644 --- a/cli/npm/managed.rs +++ b/cli/npm/managed.rs @@ -1,11 +1,9 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use deno_core::parking_lot::Mutex; -use deno_core::serde_json; use deno_error::JsError; use deno_error::JsErrorBox; use deno_npm::registry::NpmRegistryApi; @@ -13,14 +11,10 @@ use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions; use deno_resolver::npm::managed::NpmResolutionCell; -use deno_resolver::npm::ManagedNpmResolverRc; -use deno_runtime::deno_process::NpmProcessStateProvider; use thiserror::Error; use super::CliNpmRegistryInfoProvider; use crate::args::CliLockfile; -use crate::args::NpmProcessState; -use crate::args::NpmProcessStateKind; use crate::sys::CliSys; pub type CliManagedNpmResolverCreateOptions = @@ -207,27 +201,3 @@ async fn snapshot_from_lockfile( .await?; Ok(snapshot) } - -pub fn npm_process_state( - snapshot: ValidSerializedNpmResolutionSnapshot, - node_modules_path: Option<&Path>, -) -> String { - serde_json::to_string(&NpmProcessState { - kind: NpmProcessStateKind::Snapshot(snapshot.into_serialized()), - local_node_modules_path: node_modules_path - .map(|p| p.to_string_lossy().to_string()), - }) - .unwrap() -} - -#[derive(Debug)] -pub struct CliManagedNpmProcessStateProvider(pub ManagedNpmResolverRc); - -impl NpmProcessStateProvider for CliManagedNpmProcessStateProvider { - fn get_npm_process_state(&self) -> String { - npm_process_state( - self.0.resolution().serialized_valid_snapshot(), - self.0.root_node_modules_path(), - ) - } -} diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index a2cbd81d5b..388b007f81 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -1,6 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -mod byonm; pub mod installer; mod managed; @@ -10,15 +9,15 @@ use dashmap::DashMap; use deno_core::serde_json; use deno_core::url::Url; use deno_error::JsErrorBox; +use deno_lib::version::DENO_VERSION_INFO; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; -use deno_runtime::deno_process::NpmProcessStateProviderRc; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use http::HeaderName; use http::HeaderValue; -pub use self::byonm::CliByonmNpmResolverCreateOptions; pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::NpmResolutionInitializer; @@ -37,6 +36,8 @@ pub type CliNpmResolver = deno_resolver::npm::NpmResolver; pub type CliManagedNpmResolver = deno_resolver::npm::ManagedNpmResolver; pub type CliNpmResolverCreateOptions = deno_resolver::npm::NpmResolverCreateOptions; +pub type CliByonmNpmResolverCreateOptions = + ByonmNpmResolverCreateOptions; #[derive(Debug)] pub struct CliNpmCacheHttpClient { @@ -56,19 +57,6 @@ impl CliNpmCacheHttpClient { } } -pub fn create_npm_process_state_provider( - npm_resolver: &CliNpmResolver, -) -> NpmProcessStateProviderRc { - match npm_resolver { - CliNpmResolver::Byonm(byonm_npm_resolver) => Arc::new( - byonm::CliByonmNpmProcessStateProvider(byonm_npm_resolver.clone()), - ), - CliNpmResolver::Managed(managed_npm_resolver) => Arc::new( - managed::CliManagedNpmProcessStateProvider(managed_npm_resolver.clone()), - ), - } -} - #[async_trait::async_trait(?Send)] impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient { async fn download_with_retries_on_any_tokio_runtime( @@ -195,8 +183,8 @@ pub const NPM_CONFIG_USER_AGENT_ENV_VAR: &str = "npm_config_user_agent"; pub fn get_npm_config_user_agent() -> String { format!( "deno/{} npm/? deno/{} {} {}", - env!("CARGO_PKG_VERSION"), - env!("CARGO_PKG_VERSION"), + DENO_VERSION_INFO.deno, + DENO_VERSION_INFO.deno, std::env::consts::OS, std::env::consts::ARCH ) diff --git a/cli/resolver.rs b/cli/resolver.rs index b837ba51b1..35ae25da9e 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -1,16 +1,11 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::borrow::Cow; -use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; use dashmap::DashSet; -use deno_ast::MediaType; use deno_config::workspace::MappedResolutionDiagnostic; use deno_config::workspace::MappedResolutionError; -use deno_core::url::Url; -use deno_core::ModuleSourceCode; use deno_core::ModuleSpecifier; use deno_error::JsErrorBox; use deno_graph::source::ResolveError; @@ -22,23 +17,19 @@ use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::sloppy_imports::SloppyImportsCachedFs; use deno_resolver::sloppy_imports::SloppyImportsResolver; use deno_runtime::colors; -use deno_runtime::deno_fs; use deno_runtime::deno_node::is_builtin_node_module; use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_semver::package::PackageReq; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; -use thiserror::Error; use crate::args::NpmCachingStrategy; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; -use crate::node::CliNodeCodeTranslator; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; use crate::npm::CliNpmResolver; use crate::sys::CliSys; use crate::util::sync::AtomicFlag; -use crate::util::text_encoding::from_utf8_lossy_cow; pub type CliCjsTracker = deno_resolver::cjs::CjsTracker; @@ -61,150 +52,6 @@ pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver< CliSys, >; -pub struct ModuleCodeStringSource { - pub code: ModuleSourceCode, - pub found_url: ModuleSpecifier, - pub media_type: MediaType, -} - -#[derive(Debug, Error, deno_error::JsError)] -#[class(type)] -#[error("{media_type} files are not supported in npm packages: {specifier}")] -pub struct NotSupportedKindInNpmError { - pub media_type: MediaType, - pub specifier: Url, -} - -// todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone) -#[derive(Clone)] -pub struct NpmModuleLoader { - cjs_tracker: Arc, - fs: Arc, - node_code_translator: Arc, -} - -#[derive(Debug, Error, deno_error::JsError)] -pub enum NpmModuleLoadError { - #[class(inherit)] - #[error(transparent)] - NotSupportedKindInNpm(#[from] NotSupportedKindInNpmError), - #[class(inherit)] - #[error(transparent)] - ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), - #[class(inherit)] - #[error(transparent)] - TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError), - #[class(inherit)] - #[error("{}", format_message(file_path, maybe_referrer))] - Fs { - file_path: PathBuf, - maybe_referrer: Option, - #[source] - #[inherit] - source: deno_runtime::deno_io::fs::FsError, - }, -} - -fn format_message( - file_path: &std::path::Path, - maybe_referrer: &Option, -) -> String { - if file_path.is_dir() { - // directory imports are not allowed when importing from an - // ES module, so provide the user with a helpful error message - let dir_path = file_path; - let mut msg = "Directory import ".to_string(); - msg.push_str(&dir_path.to_string_lossy()); - if let Some(referrer) = maybe_referrer { - msg.push_str(" is not supported resolving import from "); - msg.push_str(referrer.as_str()); - let entrypoint_name = ["index.mjs", "index.js", "index.cjs"] - .iter() - .find(|e| dir_path.join(e).is_file()); - if let Some(entrypoint_name) = entrypoint_name { - msg.push_str("\nDid you mean to import "); - msg.push_str(entrypoint_name); - msg.push_str(" within the directory?"); - } - } - msg - } else { - let mut msg = "Unable to load ".to_string(); - msg.push_str(&file_path.to_string_lossy()); - if let Some(referrer) = maybe_referrer { - msg.push_str(" imported from "); - msg.push_str(referrer.as_str()); - } - msg - } -} - -impl NpmModuleLoader { - pub fn new( - cjs_tracker: Arc, - fs: Arc, - node_code_translator: Arc, - ) -> Self { - Self { - cjs_tracker, - node_code_translator, - fs, - } - } - - pub async fn load( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result { - let file_path = specifier.to_file_path().unwrap(); - let code = self - .fs - .read_file_async(file_path.clone(), None) - .await - .map_err(|source| NpmModuleLoadError::Fs { - file_path, - maybe_referrer: maybe_referrer.cloned(), - source, - })?; - - let media_type = MediaType::from_specifier(specifier); - if media_type.is_emittable() { - return Err(NpmModuleLoadError::NotSupportedKindInNpm( - NotSupportedKindInNpmError { - media_type, - specifier: specifier.clone(), - }, - )); - } - - let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { - // translate cjs to esm if it's cjs and inject node globals - let code = from_utf8_lossy_cow(code); - ModuleSourceCode::String( - self - .node_code_translator - .translate_cjs_to_esm(specifier, Some(code)) - .await? - .into_owned() - .into(), - ) - } else { - // esm and json code is untouched - ModuleSourceCode::Bytes(match code { - Cow::Owned(bytes) => bytes.into_boxed_slice().into(), - Cow::Borrowed(bytes) => bytes.into(), - }) - }; - - Ok(ModuleCodeStringSource { - code, - found_url: specifier.clone(), - media_type: MediaType::from_specifier(specifier), - }) - } -} - #[derive(Debug, Default)] pub struct FoundPackageJsonDepFlag(AtomicFlag); diff --git a/cli/rt/Cargo.toml b/cli/rt/Cargo.toml new file mode 100644 index 0000000000..c10a1640c4 --- /dev/null +++ b/cli/rt/Cargo.toml @@ -0,0 +1,63 @@ +# Copyright 2018-2025 the Deno authors. MIT license. + +[package] +name = "denort" +version = "2.1.5" +authors.workspace = true +default-run = "denort" +edition.workspace = true +license.workspace = true +publish = false +repository.workspace = true +description = "Provides the denort executable" + +[[bin]] +name = "denort" +path = "main.rs" +doc = false + +[[test]] +name = "integration" +path = "integration_tests_runner.rs" +harness = false + +[build-dependencies] +deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting", "only_snapshotted_js_sources"] } +deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } + +[dependencies] +deno_cache_dir.workspace = true +deno_config.workspace = true +deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } +deno_error.workspace = true +deno_lib.workspace = true +deno_media_type = { workspace = true, features = ["data_url", "decoding"] } +deno_npm.workspace = true +deno_package_json.workspace = true +deno_path_util.workspace = true +deno_resolver = { workspace = true, features = ["sync"] } +deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } +deno_semver.workspace = true +deno_snapshots.workspace = true +deno_terminal.workspace = true +libsui = "0.5.0" +node_resolver.workspace = true + +async-trait.workspace = true +bincode = "=1.3.3" +import_map = { version = "=0.21.0", features = ["ext"] } +indexmap.workspace = true +log = { workspace = true, features = ["serde"] } +serde.workspace = true +serde_json.workspace = true +sys_traits = { workspace = true, features = ["getrandom", "filetime", "libc", "real", "strip_unc", "winapi"] } +thiserror.workspace = true +tokio.workspace = true +tokio-util.workspace = true +twox-hash.workspace = true +url.workspace = true + +[dev-dependencies] +pretty_assertions.workspace = true +sys_traits = { workspace = true, features = ["memory"] } +test_util.workspace = true diff --git a/cli/rt/binary.rs b/cli/rt/binary.rs new file mode 100644 index 0000000000..19aad257ca --- /dev/null +++ b/cli/rt/binary.rs @@ -0,0 +1,682 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::ffi::OsString; +use std::io::ErrorKind; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_core::url::Url; +use deno_core::FastString; +use deno_core::ModuleSourceCode; +use deno_core::ModuleType; +use deno_error::JsError; +use deno_error::JsErrorBox; +use deno_lib::standalone::binary::DenoRtDeserializable; +use deno_lib::standalone::binary::Metadata; +use deno_lib::standalone::binary::RemoteModuleEntry; +use deno_lib::standalone::binary::SpecifierDataStore; +use deno_lib::standalone::binary::SpecifierId; +use deno_lib::standalone::binary::MAGIC_BYTES; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; +use deno_media_type::MediaType; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::RealFs; +use deno_runtime::deno_io::fs::FsError; +use deno_semver::package::PackageReq; +use deno_semver::StackString; +use indexmap::IndexMap; +use thiserror::Error; + +use crate::file_system::FileBackedVfs; +use crate::file_system::VfsRoot; + +pub struct StandaloneData { + pub metadata: Metadata, + pub modules: Arc, + pub npm_snapshot: Option, + pub root_path: PathBuf, + pub vfs: Arc, +} + +/// This function will try to run this binary as a standalone binary +/// produced by `deno compile`. It determines if this is a standalone +/// binary by skipping over the trailer width at the end of the file, +/// then checking for the magic trailer string `d3n0l4nd`. If found, +/// the bundle is executed. If not, this function exits with `Ok(None)`. +pub fn extract_standalone( + cli_args: Cow>, +) -> Result, AnyError> { + let Some(data) = libsui::find_section("d3n0l4nd") else { + return Ok(None); + }; + + let root_path = { + let maybe_current_exe = std::env::current_exe().ok(); + let current_exe_name = maybe_current_exe + .as_ref() + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy()) + // should never happen + .unwrap_or_else(|| Cow::Borrowed("binary")); + std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)) + }; + let root_url = deno_path_util::url_from_directory_path(&root_path)?; + + let DeserializedDataSection { + mut metadata, + npm_snapshot, + modules_store: remote_modules, + vfs_root_entries, + vfs_files_data, + } = match deserialize_binary_data_section(&root_url, data)? { + Some(data_section) => data_section, + None => return Ok(None), + }; + + let cli_args = cli_args.into_owned(); + metadata.argv.reserve(cli_args.len() - 1); + for arg in cli_args.into_iter().skip(1) { + metadata.argv.push(arg.into_string().unwrap()); + } + let vfs = { + let fs_root = VfsRoot { + dir: VirtualDirectory { + // align the name of the directory with the root dir + name: root_path.file_name().unwrap().to_string_lossy().to_string(), + entries: vfs_root_entries, + }, + root_path: root_path.clone(), + start_file_offset: 0, + }; + Arc::new(FileBackedVfs::new( + Cow::Borrowed(vfs_files_data), + fs_root, + metadata.vfs_case_sensitivity, + )) + }; + Ok(Some(StandaloneData { + metadata, + modules: Arc::new(StandaloneModules { + modules: remote_modules, + vfs: vfs.clone(), + }), + npm_snapshot, + root_path, + vfs, + })) +} + +pub struct DeserializedDataSection { + pub metadata: Metadata, + pub npm_snapshot: Option, + pub modules_store: RemoteModulesStore, + pub vfs_root_entries: VirtualDirectoryEntries, + pub vfs_files_data: &'static [u8], +} + +pub fn deserialize_binary_data_section( + root_dir_url: &Url, + data: &'static [u8], +) -> Result, AnyError> { + fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> { + if input.len() < MAGIC_BYTES.len() { + bail!("Unexpected end of data. Could not find magic bytes."); + } + let (magic_bytes, input) = input.split_at(MAGIC_BYTES.len()); + if magic_bytes != MAGIC_BYTES { + return Ok((input, false)); + } + Ok((input, true)) + } + + let (input, found) = read_magic_bytes(data)?; + if !found { + return Ok(None); + } + + // 1. Metadata + let (input, data) = + read_bytes_with_u64_len(input).context("reading metadata")?; + let metadata: Metadata = + serde_json::from_slice(data).context("deserializing metadata")?; + // 2. Npm snapshot + let (input, data) = + read_bytes_with_u64_len(input).context("reading npm snapshot")?; + let npm_snapshot = if data.is_empty() { + None + } else { + Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?) + }; + // 3. Specifiers + let (input, specifiers_store) = + SpecifierStore::deserialize(root_dir_url, input) + .context("deserializing specifiers")?; + // 4. Redirects + let (input, redirects_store) = + SpecifierDataStore::::deserialize(input) + .context("deserializing redirects")?; + // 5. Remote modules + let (input, remote_modules_store) = + SpecifierDataStore::>::deserialize(input) + .context("deserializing remote modules")?; + // 6. VFS + let (input, data) = read_bytes_with_u64_len(input).context("vfs")?; + let vfs_root_entries: VirtualDirectoryEntries = + serde_json::from_slice(data).context("deserializing vfs data")?; + let (input, vfs_files_data) = + read_bytes_with_u64_len(input).context("reading vfs files data")?; + + // finally ensure we read the magic bytes at the end + let (_input, found) = read_magic_bytes(input)?; + if !found { + bail!("Could not find magic bytes at the end of the data."); + } + + let modules_store = RemoteModulesStore::new( + specifiers_store, + redirects_store, + remote_modules_store, + ); + + Ok(Some(DeserializedDataSection { + metadata, + npm_snapshot, + modules_store, + vfs_root_entries, + vfs_files_data, + })) +} + +struct SpecifierStore { + data: IndexMap, SpecifierId>, + reverse: IndexMap>, +} + +impl SpecifierStore { + pub fn deserialize<'a>( + root_dir_url: &Url, + input: &'a [u8], + ) -> std::io::Result<(&'a [u8], Self)> { + let (input, len) = read_u32_as_usize(input)?; + let mut data = IndexMap::with_capacity(len); + let mut reverse = IndexMap::with_capacity(len); + let mut input = input; + for _ in 0..len { + let (new_input, specifier_str) = read_string_lossy(input)?; + let specifier = match Url::parse(&specifier_str) { + Ok(url) => url, + Err(err) => match root_dir_url.join(&specifier_str) { + Ok(url) => url, + Err(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + err, + )); + } + }, + }; + let (new_input, id) = SpecifierId::deserialize(new_input)?; + let specifier = Arc::new(specifier); + data.insert(specifier.clone(), id); + reverse.insert(id, specifier); + input = new_input; + } + Ok((input, Self { data, reverse })) + } + + pub fn get_id(&self, specifier: &Url) -> Option { + self.data.get(specifier).cloned() + } + + pub fn get_specifier(&self, specifier_id: SpecifierId) -> Option<&Url> { + self.reverse.get(&specifier_id).map(|url| url.as_ref()) + } +} + +pub struct StandaloneModules { + modules: RemoteModulesStore, + vfs: Arc, +} + +impl StandaloneModules { + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a Url, + ) -> Result, TooManyRedirectsError> { + if specifier.scheme() == "file" { + Ok(Some(specifier)) + } else { + self.modules.resolve_specifier(specifier) + } + } + + pub fn has_file(&self, path: &Path) -> bool { + self.vfs.file_entry(path).is_ok() + } + + pub fn read<'a>( + &'a self, + specifier: &'a Url, + ) -> Result>, JsErrorBox> { + if specifier.scheme() == "file" { + let path = deno_path_util::url_to_file_path(specifier) + .map_err(JsErrorBox::from_err)?; + let mut transpiled = None; + let mut source_map = None; + let mut cjs_export_analysis = None; + let bytes = match self.vfs.file_entry(&path) { + Ok(entry) => { + let bytes = self + .vfs + .read_file_all(entry) + .map_err(JsErrorBox::from_err)?; + transpiled = entry + .transpiled_offset + .and_then(|t| self.vfs.read_file_offset_with_len(t).ok()); + source_map = entry + .source_map_offset + .and_then(|t| self.vfs.read_file_offset_with_len(t).ok()); + cjs_export_analysis = entry + .cjs_export_analysis_offset + .and_then(|t| self.vfs.read_file_offset_with_len(t).ok()); + bytes + } + Err(err) if err.kind() == ErrorKind::NotFound => { + match RealFs.read_file_sync(&path, None) { + Ok(bytes) => bytes, + Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => { + return Ok(None) + } + Err(err) => return Err(JsErrorBox::from_err(err)), + } + } + Err(err) => return Err(JsErrorBox::from_err(err)), + }; + Ok(Some(DenoCompileModuleData { + media_type: MediaType::from_specifier(specifier), + specifier, + data: bytes, + transpiled, + source_map, + cjs_export_analysis, + })) + } else { + self.modules.read(specifier).map_err(JsErrorBox::from_err) + } + } +} + +pub struct DenoCompileModuleData<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub data: Cow<'static, [u8]>, + pub transpiled: Option>, + pub source_map: Option>, + pub cjs_export_analysis: Option>, +} + +impl<'a> DenoCompileModuleData<'a> { + pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) { + fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource { + match data { + Cow::Borrowed(d) => DenoCompileModuleSource::String( + // SAFETY: we know this is a valid utf8 string + unsafe { std::str::from_utf8_unchecked(d) }, + ), + Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)), + } + } + + let data = self.transpiled.unwrap_or(self.data); + let (media_type, source) = match self.media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx => (ModuleType::JavaScript, into_string_unsafe(data)), + MediaType::Json => (ModuleType::Json, into_string_unsafe(data)), + MediaType::Wasm => { + (ModuleType::Wasm, DenoCompileModuleSource::Bytes(data)) + } + // just assume javascript if we made it here + MediaType::Css | MediaType::SourceMap | MediaType::Unknown => { + (ModuleType::JavaScript, DenoCompileModuleSource::Bytes(data)) + } + }; + (self.specifier, media_type, source) + } +} + +pub enum DenoCompileModuleSource { + String(&'static str), + Bytes(Cow<'static, [u8]>), +} + +impl DenoCompileModuleSource { + pub fn into_for_v8(self) -> ModuleSourceCode { + fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode { + ModuleSourceCode::Bytes(match data { + Cow::Borrowed(d) => d.into(), + Cow::Owned(d) => d.into_boxed_slice().into(), + }) + } + + match self { + // todo(https://github.com/denoland/deno_core/pull/943): store whether + // the string is ascii or not ahead of time so we can avoid the is_ascii() + // check in FastString::from_static + Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)), + Self::Bytes(b) => into_bytes(b), + } + } +} + +#[derive(Debug, Error, JsError)] +#[class(generic)] +#[error("Too many redirects resolving: {0}")] +pub struct TooManyRedirectsError(Url); + +pub struct RemoteModulesStore { + specifiers: SpecifierStore, + redirects: SpecifierDataStore, + remote_modules: SpecifierDataStore>, +} + +impl RemoteModulesStore { + fn new( + specifiers: SpecifierStore, + redirects: SpecifierDataStore, + remote_modules: SpecifierDataStore>, + ) -> Self { + Self { + specifiers, + redirects, + remote_modules, + } + } + + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a Url, + ) -> Result, TooManyRedirectsError> { + let Some(mut current) = self.specifiers.get_id(specifier) else { + return Ok(None); + }; + let mut count = 0; + loop { + if count > 10 { + return Err(TooManyRedirectsError(specifier.clone())); + } + match self.redirects.get(current) { + Some(to) => { + current = *to; + count += 1; + } + None => { + if count == 0 { + return Ok(Some(specifier)); + } else { + return Ok(self.specifiers.get_specifier(current)); + } + } + } + } + } + + pub fn read<'a>( + &'a self, + original_specifier: &'a Url, + ) -> Result>, TooManyRedirectsError> { + #[allow(clippy::ptr_arg)] + fn handle_cow_ref(data: &Cow<'static, [u8]>) -> Cow<'static, [u8]> { + match data { + Cow::Borrowed(data) => Cow::Borrowed(data), + Cow::Owned(data) => { + // this variant should never happen because the data + // should always be borrowed static in denort + debug_assert!(false); + Cow::Owned(data.clone()) + } + } + } + + let mut count = 0; + let Some(mut specifier) = self.specifiers.get_id(original_specifier) else { + return Ok(None); + }; + loop { + if count > 10 { + return Err(TooManyRedirectsError(original_specifier.clone())); + } + match self.redirects.get(specifier) { + Some(to) => { + specifier = *to; + count += 1; + } + None => { + let Some(entry) = self.remote_modules.get(specifier) else { + return Ok(None); + }; + return Ok(Some(DenoCompileModuleData { + specifier: if count == 0 { + original_specifier + } else { + self.specifiers.get_specifier(specifier).unwrap() + }, + media_type: entry.media_type, + data: handle_cow_ref(&entry.data), + transpiled: entry.maybe_transpiled.as_ref().map(handle_cow_ref), + source_map: entry.maybe_source_map.as_ref().map(handle_cow_ref), + cjs_export_analysis: entry + .maybe_cjs_export_analysis + .as_ref() + .map(handle_cow_ref), + })); + } + } + } + } +} + +fn deserialize_npm_snapshot( + input: &[u8], +) -> Result { + fn parse_id(input: &[u8]) -> Result<(&[u8], NpmPackageId), AnyError> { + let (input, id) = read_string_lossy(input)?; + let id = NpmPackageId::from_serialized(&id)?; + Ok((input, id)) + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_root_package<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let req = PackageReq::from_str(&req)?; + let (input, id) = read_u32_as_usize(input)?; + Ok((input, (req, id_to_npm_id(id)?))) + } + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_package_dep<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (StackString, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let (input, id) = read_u32_as_usize(input)?; + let req = StackString::from_cow(req); + Ok((input, (req, id_to_npm_id(id)?))) + } + } + + fn parse_package<'a>( + input: &'a [u8], + id: NpmPackageId, + id_to_npm_id: &impl Fn(usize) -> Result, + ) -> Result<(&'a [u8], SerializedNpmResolutionSnapshotPackage), AnyError> { + let (input, deps_len) = read_u32_as_usize(input)?; + let (input, dependencies) = + parse_hashmap_n_times(input, deps_len, parse_package_dep(id_to_npm_id))?; + Ok(( + input, + SerializedNpmResolutionSnapshotPackage { + id, + system: Default::default(), + dist: Default::default(), + dependencies, + optional_dependencies: Default::default(), + bin: None, + scripts: Default::default(), + deprecated: Default::default(), + }, + )) + } + + let (input, packages_len) = read_u32_as_usize(input)?; + + // get a hashmap of all the npm package ids to their serialized ids + let (input, data_ids_to_npm_ids) = + parse_vec_n_times(input, packages_len, parse_id) + .context("deserializing id")?; + let data_id_to_npm_id = |id: usize| { + data_ids_to_npm_ids + .get(id) + .cloned() + .ok_or_else(|| deno_core::anyhow::anyhow!("Invalid npm package id")) + }; + + let (input, root_packages_len) = read_u32_as_usize(input)?; + let (input, root_packages) = parse_hashmap_n_times( + input, + root_packages_len, + parse_root_package(&data_id_to_npm_id), + ) + .context("deserializing root package")?; + let (input, packages) = + parse_vec_n_times_with_index(input, packages_len, |input, index| { + parse_package(input, data_id_to_npm_id(index)?, &data_id_to_npm_id) + }) + .context("deserializing package")?; + + if !input.is_empty() { + bail!("Unexpected data left over"); + } + + Ok( + SerializedNpmResolutionSnapshot { + packages, + root_packages, + } + // this is ok because we have already verified that all the + // identifiers found in the snapshot are valid via the + // npm package id -> npm package id mapping + .into_valid_unsafe(), + ) +} + +fn parse_hashmap_n_times( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], (TKey, TValue)), AnyError>, +) -> Result<(&[u8], HashMap), AnyError> { + let mut results = HashMap::with_capacity(times); + for _ in 0..times { + let result = parse(input); + let (new_input, (key, value)) = result?; + results.insert(key, value); + input = new_input; + } + Ok((input, results)) +} + +fn parse_vec_n_times( + input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + parse_vec_n_times_with_index(input, times, |input, _index| parse(input)) +} + +fn parse_vec_n_times_with_index( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8], usize) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + let mut results = Vec::with_capacity(times); + for i in 0..times { + let result = parse(input, i); + let (new_input, result) = result?; + results.push(result); + input = new_input; + } + Ok((input, results)) +} + +fn read_bytes_with_u64_len(input: &[u8]) -> std::io::Result<(&[u8], &[u8])> { + let (input, len) = read_u64(input)?; + let (input, data) = read_bytes(input, len as usize)?; + Ok((input, data)) +} + +fn read_bytes_with_u32_len(input: &[u8]) -> std::io::Result<(&[u8], &[u8])> { + let (input, len) = read_u32_as_usize(input)?; + let (input, data) = read_bytes(input, len)?; + Ok((input, data)) +} + +fn read_bytes(input: &[u8], len: usize) -> std::io::Result<(&[u8], &[u8])> { + check_has_len(input, len)?; + let (len_bytes, input) = input.split_at(len); + Ok((input, len_bytes)) +} + +#[inline(always)] +fn check_has_len(input: &[u8], len: usize) -> std::io::Result<()> { + if input.len() < len { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Unexpected end of data", + )) + } else { + Ok(()) + } +} + +fn read_string_lossy(input: &[u8]) -> std::io::Result<(&[u8], Cow)> { + let (input, data_bytes) = read_bytes_with_u32_len(input)?; + Ok((input, String::from_utf8_lossy(data_bytes))) +} + +fn read_u32_as_usize(input: &[u8]) -> std::io::Result<(&[u8], usize)> { + let (input, len_bytes) = read_bytes(input, 4)?; + let len = u32::from_le_bytes(len_bytes.try_into().unwrap()); + Ok((input, len as usize)) +} + +fn read_u64(input: &[u8]) -> std::io::Result<(&[u8], u64)> { + let (input, len_bytes) = read_bytes(input, 8)?; + let len = u64::from_le_bytes(len_bytes.try_into().unwrap()); + Ok((input, len)) +} diff --git a/cli/rt/build.rs b/cli/rt/build.rs new file mode 100644 index 0000000000..486e203dd6 --- /dev/null +++ b/cli/rt/build.rs @@ -0,0 +1,11 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +fn main() { + // Skip building from docs.rs. + if std::env::var_os("DOCS_RS").is_some() { + return; + } + + deno_runtime::deno_napi::print_linker_flags("denort"); + deno_runtime::deno_webgpu::print_linker_flags("denort"); +} diff --git a/cli/standalone/code_cache.rs b/cli/rt/code_cache.rs similarity index 96% rename from cli/standalone/code_cache.rs rename to cli/rt/code_cache.rs index de9ff2a141..c97638abd1 100644 --- a/cli/standalone/code_cache.rs +++ b/cli/rt/code_cache.rs @@ -1,6 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::collections::BTreeMap; use std::collections::HashMap; use std::io::BufReader; use std::io::BufWriter; @@ -10,17 +9,15 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::unsync::sync::AtomicFlag; +use deno_lib::util::hash::FastInsecureHasher; use deno_path_util::get_atomic_path; use deno_runtime::code_cache::CodeCache; use deno_runtime::code_cache::CodeCacheType; - -use crate::cache::FastInsecureHasher; -use crate::worker::CliCodeCache; +use url::Url; enum CodeCacheStrategy { FirstRun(FirstRunCodeCacheStrategy), @@ -76,12 +73,27 @@ impl DenoCompileCodeCache { } } } + + pub fn for_deno_core(self: Arc) -> Arc { + self.clone() + } + + pub fn enabled(&self) -> bool { + match &self.strategy { + CodeCacheStrategy::FirstRun(strategy) => { + !strategy.is_finished.is_raised() + } + CodeCacheStrategy::SubsequentRun(strategy) => { + !strategy.is_finished.is_raised() + } + } + } } impl CodeCache for DenoCompileCodeCache { fn get_sync( &self, - specifier: &ModuleSpecifier, + specifier: &Url, code_cache_type: CodeCacheType, source_hash: u64, ) -> Option> { @@ -106,7 +118,7 @@ impl CodeCache for DenoCompileCodeCache { fn set_sync( &self, - specifier: ModuleSpecifier, + specifier: Url, code_cache_type: CodeCacheType, source_hash: u64, bytes: &[u8], @@ -152,23 +164,6 @@ impl CodeCache for DenoCompileCodeCache { } } -impl CliCodeCache for DenoCompileCodeCache { - fn enabled(&self) -> bool { - match &self.strategy { - CodeCacheStrategy::FirstRun(strategy) => { - !strategy.is_finished.is_raised() - } - CodeCacheStrategy::SubsequentRun(strategy) => { - !strategy.is_finished.is_raised() - } - } - } - - fn as_code_cache(self: Arc) -> Arc { - self - } -} - type CodeCacheKey = (String, CodeCacheType); struct FirstRunCodeCacheData { @@ -216,7 +211,7 @@ struct SubsequentRunCodeCacheStrategy { impl SubsequentRunCodeCacheStrategy { fn take_from_cache( &self, - specifier: &ModuleSpecifier, + specifier: &Url, code_cache_type: CodeCacheType, source_hash: u64, ) -> Option> { @@ -395,8 +390,6 @@ fn deserialize_with_reader( #[cfg(test)] mod test { - use std::fs::File; - use test_util::TempDir; use super::*; @@ -463,8 +456,8 @@ mod test { fn code_cache() { let temp_dir = TempDir::new(); let file_path = temp_dir.path().join("cache.bin").to_path_buf(); - let url1 = ModuleSpecifier::parse("https://deno.land/example1.js").unwrap(); - let url2 = ModuleSpecifier::parse("https://deno.land/example2.js").unwrap(); + let url1 = Url::parse("https://deno.land/example1.js").unwrap(); + let url2 = Url::parse("https://deno.land/example2.js").unwrap(); // first run { let code_cache = DenoCompileCodeCache::new(file_path.clone(), 1234); diff --git a/cli/rt/file_system.rs b/cli/rt/file_system.rs new file mode 100644 index 0000000000..097674f015 --- /dev/null +++ b/cli/rt/file_system.rs @@ -0,0 +1,1713 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::cell::RefCell; +use std::collections::HashSet; +use std::io::ErrorKind; +use std::io::SeekFrom; +use std::ops::Range; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; +use std::time::SystemTime; + +use deno_core::BufMutView; +use deno_core::BufView; +use deno_core::ResourceHandleFd; +use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::OffsetWithLength; +use deno_lib::standalone::virtual_fs::VfsEntry; +use deno_lib::standalone::virtual_fs::VfsEntryRef; +use deno_lib::standalone::virtual_fs::VirtualDirectory; +use deno_lib::standalone::virtual_fs::VirtualFile; +use deno_lib::sys::DenoLibSys; +use deno_runtime::deno_fs::AccessCheckCb; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsDirEntry; +use deno_runtime::deno_fs::FsFileType; +use deno_runtime::deno_fs::OpenOptions; +use deno_runtime::deno_fs::RealFs; +use deno_runtime::deno_io; +use deno_runtime::deno_io::fs::File as DenoFile; +use deno_runtime::deno_io::fs::FsError; +use deno_runtime::deno_io::fs::FsResult; +use deno_runtime::deno_io::fs::FsStat; +use deno_runtime::deno_node::ExtNodeSys; +use sys_traits::boxed::BoxedFsDirEntry; +use sys_traits::boxed::BoxedFsMetadataValue; +use sys_traits::boxed::FsMetadataBoxed; +use sys_traits::boxed::FsReadDirBoxed; +use sys_traits::FsCopy; +use url::Url; + +#[derive(Debug, Clone)] +pub struct DenoRtSys(Arc); + +impl DenoRtSys { + pub fn new(vfs: Arc) -> Self { + Self(vfs) + } + + pub fn is_specifier_in_vfs(&self, specifier: &Url) -> bool { + deno_path_util::url_to_file_path(specifier) + .map(|p| self.is_in_vfs(&p)) + .unwrap_or(false) + } + + pub fn is_in_vfs(&self, path: &Path) -> bool { + self.0.is_path_within(path) + } + + fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> { + if self.0.is_path_within(path) { + Err(FsError::NotSupported) + } else { + Ok(()) + } + } + + fn copy_to_real_path( + &self, + oldpath: &Path, + newpath: &Path, + ) -> std::io::Result { + let old_file = self.0.file_entry(oldpath)?; + let old_file_bytes = self.0.read_file_all(old_file)?; + let len = old_file_bytes.len() as u64; + RealFs + .write_file_sync( + newpath, + OpenOptions { + read: false, + write: true, + create: true, + truncate: true, + append: false, + create_new: false, + mode: None, + }, + None, + &old_file_bytes, + ) + .map_err(|err| err.into_io_error())?; + Ok(len) + } +} + +#[async_trait::async_trait(?Send)] +impl FileSystem for DenoRtSys { + fn cwd(&self) -> FsResult { + RealFs.cwd() + } + + fn tmp_dir(&self) -> FsResult { + RealFs.tmp_dir() + } + + fn chdir(&self, path: &Path) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chdir(path) + } + + fn umask(&self, mask: Option) -> FsResult { + RealFs.umask(mask) + } + + fn open_sync( + &self, + path: &Path, + options: OpenOptions, + access_check: Option, + ) -> FsResult> { + if self.0.is_path_within(path) { + Ok(Rc::new(self.0.open_file(path)?)) + } else { + RealFs.open_sync(path, options, access_check) + } + } + async fn open_async<'a>( + &'a self, + path: PathBuf, + options: OpenOptions, + access_check: Option>, + ) -> FsResult> { + if self.0.is_path_within(&path) { + Ok(Rc::new(self.0.open_file(&path)?)) + } else { + RealFs.open_async(path, options, access_check).await + } + } + + fn mkdir_sync( + &self, + path: &Path, + recursive: bool, + mode: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.mkdir_sync(path, recursive, mode) + } + async fn mkdir_async( + &self, + path: PathBuf, + recursive: bool, + mode: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.mkdir_async(path, recursive, mode).await + } + + fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chmod_sync(path, mode) + } + async fn chmod_async(&self, path: PathBuf, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.chmod_async(path, mode).await + } + + fn chown_sync( + &self, + path: &Path, + uid: Option, + gid: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chown_sync(path, uid, gid) + } + async fn chown_async( + &self, + path: PathBuf, + uid: Option, + gid: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.chown_async(path, uid, gid).await + } + + fn lchown_sync( + &self, + path: &Path, + uid: Option, + gid: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.lchown_sync(path, uid, gid) + } + + async fn lchown_async( + &self, + path: PathBuf, + uid: Option, + gid: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.lchown_async(path, uid, gid).await + } + + fn remove_sync(&self, path: &Path, recursive: bool) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.remove_sync(path, recursive) + } + async fn remove_async(&self, path: PathBuf, recursive: bool) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.remove_async(path, recursive).await + } + + fn copy_file_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(newpath)?; + if self.0.is_path_within(oldpath) { + self + .copy_to_real_path(oldpath, newpath) + .map(|_| ()) + .map_err(FsError::Io) + } else { + RealFs.copy_file_sync(oldpath, newpath) + } + } + async fn copy_file_async( + &self, + oldpath: PathBuf, + newpath: PathBuf, + ) -> FsResult<()> { + self.error_if_in_vfs(&newpath)?; + if self.0.is_path_within(&oldpath) { + let fs = self.clone(); + tokio::task::spawn_blocking(move || { + fs.copy_to_real_path(&oldpath, &newpath) + .map(|_| ()) + .map_err(FsError::Io) + }) + .await? + } else { + RealFs.copy_file_async(oldpath, newpath).await + } + } + + fn cp_sync(&self, from: &Path, to: &Path) -> FsResult<()> { + self.error_if_in_vfs(to)?; + + RealFs.cp_sync(from, to) + } + async fn cp_async(&self, from: PathBuf, to: PathBuf) -> FsResult<()> { + self.error_if_in_vfs(&to)?; + + RealFs.cp_async(from, to).await + } + + fn stat_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.stat(path)?.as_fs_stat()) + } else { + RealFs.stat_sync(path) + } + } + async fn stat_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.stat(&path)?.as_fs_stat()) + } else { + RealFs.stat_async(path).await + } + } + + fn lstat_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.lstat(path)?.as_fs_stat()) + } else { + RealFs.lstat_sync(path) + } + } + async fn lstat_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.lstat(&path)?.as_fs_stat()) + } else { + RealFs.lstat_async(path).await + } + } + + fn realpath_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.canonicalize(path)?) + } else { + RealFs.realpath_sync(path) + } + } + async fn realpath_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.canonicalize(&path)?) + } else { + RealFs.realpath_async(path).await + } + } + + fn read_dir_sync(&self, path: &Path) -> FsResult> { + if self.0.is_path_within(path) { + Ok(self.0.read_dir(path)?) + } else { + RealFs.read_dir_sync(path) + } + } + async fn read_dir_async(&self, path: PathBuf) -> FsResult> { + if self.0.is_path_within(&path) { + Ok(self.0.read_dir(&path)?) + } else { + RealFs.read_dir_async(path).await + } + } + + fn rename_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.rename_sync(oldpath, newpath) + } + async fn rename_async( + &self, + oldpath: PathBuf, + newpath: PathBuf, + ) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.rename_async(oldpath, newpath).await + } + + fn link_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.link_sync(oldpath, newpath) + } + async fn link_async( + &self, + oldpath: PathBuf, + newpath: PathBuf, + ) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.link_async(oldpath, newpath).await + } + + fn symlink_sync( + &self, + oldpath: &Path, + newpath: &Path, + file_type: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.symlink_sync(oldpath, newpath, file_type) + } + async fn symlink_async( + &self, + oldpath: PathBuf, + newpath: PathBuf, + file_type: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.symlink_async(oldpath, newpath, file_type).await + } + + fn read_link_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.read_link(path)?) + } else { + RealFs.read_link_sync(path) + } + } + async fn read_link_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.read_link(&path)?) + } else { + RealFs.read_link_async(path).await + } + } + + fn truncate_sync(&self, path: &Path, len: u64) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.truncate_sync(path, len) + } + async fn truncate_async(&self, path: PathBuf, len: u64) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.truncate_async(path, len).await + } + + fn utime_sync( + &self, + path: &Path, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.utime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + } + async fn utime_async( + &self, + path: PathBuf, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs + .utime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + .await + } + + fn lutime_sync( + &self, + path: &Path, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.lutime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + } + async fn lutime_async( + &self, + path: PathBuf, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs + .lutime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + .await + } +} + +impl ExtNodeSys for DenoRtSys {} +impl DenoLibSys for DenoRtSys {} + +impl sys_traits::BaseFsHardLink for DenoRtSys { + #[inline] + fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> { + self.link_sync(src, dst).map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRead for DenoRtSys { + #[inline] + fn base_fs_read(&self, path: &Path) -> std::io::Result> { + self + .read_file_sync(path, None) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::FsMetadataValue for FileBackedVfsMetadata { + fn file_type(&self) -> sys_traits::FileType { + self.file_type + } + + fn len(&self) -> u64 { + self.len + } + + fn accessed(&self) -> std::io::Result { + Err(not_supported("accessed time")) + } + + fn created(&self) -> std::io::Result { + Err(not_supported("created time")) + } + + fn changed(&self) -> std::io::Result { + Err(not_supported("changed time")) + } + + fn modified(&self) -> std::io::Result { + Err(not_supported("modified time")) + } + + fn dev(&self) -> std::io::Result { + Ok(0) + } + + fn ino(&self) -> std::io::Result { + Ok(0) + } + + fn mode(&self) -> std::io::Result { + Ok(0) + } + + fn nlink(&self) -> std::io::Result { + Ok(0) + } + + fn uid(&self) -> std::io::Result { + Ok(0) + } + + fn gid(&self) -> std::io::Result { + Ok(0) + } + + fn rdev(&self) -> std::io::Result { + Ok(0) + } + + fn blksize(&self) -> std::io::Result { + Ok(0) + } + + fn blocks(&self) -> std::io::Result { + Ok(0) + } + + fn is_block_device(&self) -> std::io::Result { + Ok(false) + } + + fn is_char_device(&self) -> std::io::Result { + Ok(false) + } + + fn is_fifo(&self) -> std::io::Result { + Ok(false) + } + + fn is_socket(&self) -> std::io::Result { + Ok(false) + } + + fn file_attributes(&self) -> std::io::Result { + Ok(0) + } +} + +fn not_supported(name: &str) -> std::io::Error { + std::io::Error::new( + ErrorKind::Unsupported, + format!( + "{} is not supported for an embedded deno compile file", + name + ), + ) +} + +impl sys_traits::FsDirEntry for FileBackedVfsDirEntry { + type Metadata = BoxedFsMetadataValue; + + fn file_name(&self) -> Cow { + Cow::Borrowed(self.metadata.name.as_ref()) + } + + fn file_type(&self) -> std::io::Result { + Ok(self.metadata.file_type) + } + + fn metadata(&self) -> std::io::Result { + Ok(BoxedFsMetadataValue(Box::new(self.metadata.clone()))) + } + + fn path(&self) -> Cow { + Cow::Owned(self.parent_path.join(&self.metadata.name)) + } +} + +impl sys_traits::BaseFsReadDir for DenoRtSys { + type ReadDirEntry = BoxedFsDirEntry; + + fn base_fs_read_dir( + &self, + path: &Path, + ) -> std::io::Result< + Box> + '_>, + > { + if self.0.is_path_within(path) { + let entries = self.0.read_dir_with_metadata(path)?; + Ok(Box::new( + entries.map(|entry| Ok(BoxedFsDirEntry::new(entry))), + )) + } else { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.fs_read_dir_boxed(path) + } + } +} + +impl sys_traits::BaseFsCanonicalize for DenoRtSys { + #[inline] + fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result { + self.realpath_sync(path).map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsMetadata for DenoRtSys { + type Metadata = BoxedFsMetadataValue; + + #[inline] + fn base_fs_metadata(&self, path: &Path) -> std::io::Result { + if self.0.is_path_within(path) { + Ok(BoxedFsMetadataValue::new(self.0.stat(path)?)) + } else { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.fs_metadata_boxed(path) + } + } + + #[inline] + fn base_fs_symlink_metadata( + &self, + path: &Path, + ) -> std::io::Result { + if self.0.is_path_within(path) { + Ok(BoxedFsMetadataValue::new(self.0.lstat(path)?)) + } else { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.fs_symlink_metadata_boxed(path) + } + } +} + +impl sys_traits::BaseFsCopy for DenoRtSys { + #[inline] + fn base_fs_copy(&self, from: &Path, to: &Path) -> std::io::Result { + self + .error_if_in_vfs(to) + .map_err(|err| err.into_io_error())?; + if self.0.is_path_within(from) { + self.copy_to_real_path(from, to) + } else { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.fs_copy(from, to) + } + } +} + +impl sys_traits::BaseFsCloneFile for DenoRtSys { + fn base_fs_clone_file( + &self, + _from: &Path, + _to: &Path, + ) -> std::io::Result<()> { + // will cause a fallback in the code that uses this + Err(not_supported("cloning files")) + } +} + +impl sys_traits::BaseFsCreateDir for DenoRtSys { + #[inline] + fn base_fs_create_dir( + &self, + path: &Path, + options: &sys_traits::CreateDirOptions, + ) -> std::io::Result<()> { + self + .mkdir_sync(path, options.recursive, options.mode) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRemoveFile for DenoRtSys { + #[inline] + fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> { + self + .remove_sync(path, false) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRename for DenoRtSys { + #[inline] + fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self + .rename_sync(from, to) + .map_err(|err| err.into_io_error()) + } +} + +pub enum FsFileAdapter { + Real(sys_traits::impls::RealFsFile), + Vfs(FileBackedVfsFile), +} + +impl sys_traits::FsFile for FsFileAdapter {} + +impl sys_traits::FsFileAsRaw for FsFileAdapter { + #[cfg(windows)] + fn fs_file_as_raw_handle(&self) -> Option { + match self { + Self::Real(file) => file.fs_file_as_raw_handle(), + Self::Vfs(_) => None, + } + } + + #[cfg(unix)] + fn fs_file_as_raw_fd(&self) -> Option { + match self { + Self::Real(file) => file.fs_file_as_raw_fd(), + Self::Vfs(_) => None, + } + } +} + +impl sys_traits::FsFileSyncData for FsFileAdapter { + fn fs_file_sync_data(&mut self) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_sync_data(), + Self::Vfs(_) => Ok(()), + } + } +} + +impl sys_traits::FsFileSyncAll for FsFileAdapter { + fn fs_file_sync_all(&mut self) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_sync_all(), + Self::Vfs(_) => Ok(()), + } + } +} + +impl sys_traits::FsFileSetPermissions for FsFileAdapter { + #[inline] + fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_set_permissions(mode), + Self::Vfs(_) => Ok(()), + } + } +} + +impl std::io::Read for FsFileAdapter { + #[inline] + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Self::Real(file) => file.read(buf), + Self::Vfs(file) => file.read_to_buf(buf), + } + } +} + +impl std::io::Seek for FsFileAdapter { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + match self { + Self::Real(file) => file.seek(pos), + Self::Vfs(file) => file.seek(pos), + } + } +} + +impl std::io::Write for FsFileAdapter { + #[inline] + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Self::Real(file) => file.write(buf), + Self::Vfs(_) => Err(not_supported("writing files")), + } + } + + #[inline] + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Real(file) => file.flush(), + Self::Vfs(_) => Err(not_supported("writing files")), + } + } +} + +impl sys_traits::FsFileSetLen for FsFileAdapter { + #[inline] + fn fs_file_set_len(&mut self, len: u64) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_set_len(len), + Self::Vfs(_) => Err(not_supported("setting file length")), + } + } +} + +impl sys_traits::FsFileSetTimes for FsFileAdapter { + fn fs_file_set_times( + &mut self, + times: sys_traits::FsFileTimes, + ) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_set_times(times), + Self::Vfs(_) => Err(not_supported("setting file times")), + } + } +} + +impl sys_traits::FsFileLock for FsFileAdapter { + fn fs_file_lock( + &mut self, + mode: sys_traits::FsFileLockMode, + ) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_lock(mode), + Self::Vfs(_) => Err(not_supported("locking files")), + } + } + + fn fs_file_try_lock( + &mut self, + mode: sys_traits::FsFileLockMode, + ) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_try_lock(mode), + Self::Vfs(_) => Err(not_supported("locking files")), + } + } + + fn fs_file_unlock(&mut self) -> std::io::Result<()> { + match self { + Self::Real(file) => file.fs_file_unlock(), + Self::Vfs(_) => Err(not_supported("unlocking files")), + } + } +} + +impl sys_traits::FsFileIsTerminal for FsFileAdapter { + #[inline] + fn fs_file_is_terminal(&self) -> bool { + match self { + Self::Real(file) => file.fs_file_is_terminal(), + Self::Vfs(_) => false, + } + } +} + +impl sys_traits::BaseFsOpen for DenoRtSys { + type File = FsFileAdapter; + + fn base_fs_open( + &self, + path: &Path, + options: &sys_traits::OpenOptions, + ) -> std::io::Result { + if self.0.is_path_within(path) { + Ok(FsFileAdapter::Vfs(self.0.open_file(path)?)) + } else { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + Ok(FsFileAdapter::Real( + sys_traits::impls::RealSys.base_fs_open(path, options)?, + )) + } + } +} + +impl sys_traits::BaseFsSymlinkDir for DenoRtSys { + fn base_fs_symlink_dir(&self, src: &Path, dst: &Path) -> std::io::Result<()> { + self + .symlink_sync(src, dst, Some(FsFileType::Directory)) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::SystemRandom for DenoRtSys { + #[inline] + fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.sys_random(buf) + } +} + +impl sys_traits::SystemTimeNow for DenoRtSys { + #[inline] + fn sys_time_now(&self) -> SystemTime { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.sys_time_now() + } +} + +impl sys_traits::ThreadSleep for DenoRtSys { + #[inline] + fn thread_sleep(&self, dur: Duration) { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.thread_sleep(dur) + } +} + +impl sys_traits::EnvCurrentDir for DenoRtSys { + fn env_current_dir(&self) -> std::io::Result { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.env_current_dir() + } +} + +impl sys_traits::BaseEnvVar for DenoRtSys { + fn base_env_var_os( + &self, + key: &std::ffi::OsStr, + ) -> Option { + #[allow(clippy::disallowed_types)] // ok because we're implementing the fs + sys_traits::impls::RealSys.base_env_var_os(key) + } +} + +#[derive(Debug)] +pub struct VfsRoot { + pub dir: VirtualDirectory, + pub root_path: PathBuf, + pub start_file_offset: u64, +} + +impl VfsRoot { + fn find_entry<'a>( + &'a self, + path: &Path, + case_sensitivity: FileSystemCaseSensitivity, + ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + self.find_entry_inner(path, &mut HashSet::new(), case_sensitivity) + } + + fn find_entry_inner<'a>( + &'a self, + path: &Path, + seen: &mut HashSet, + case_sensitivity: FileSystemCaseSensitivity, + ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + let mut path = Cow::Borrowed(path); + loop { + let (resolved_path, entry) = + self.find_entry_no_follow_inner(&path, seen, case_sensitivity)?; + match entry { + VfsEntryRef::Symlink(symlink) => { + if !seen.insert(path.to_path_buf()) { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "circular symlinks", + )); + } + path = Cow::Owned(symlink.resolve_dest_from_root(&self.root_path)); + } + _ => { + return Ok((resolved_path, entry)); + } + } + } + } + + fn find_entry_no_follow( + &self, + path: &Path, + case_sensitivity: FileSystemCaseSensitivity, + ) -> std::io::Result<(PathBuf, VfsEntryRef)> { + self.find_entry_no_follow_inner(path, &mut HashSet::new(), case_sensitivity) + } + + fn find_entry_no_follow_inner<'a>( + &'a self, + path: &Path, + seen: &mut HashSet, + case_sensitivity: FileSystemCaseSensitivity, + ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + let relative_path = match path.strip_prefix(&self.root_path) { + Ok(p) => p, + Err(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + }; + let mut final_path = self.root_path.clone(); + let mut current_entry = VfsEntryRef::Dir(&self.dir); + for component in relative_path.components() { + let component = component.as_os_str(); + let current_dir = match current_entry { + VfsEntryRef::Dir(dir) => { + final_path.push(component); + dir + } + VfsEntryRef::Symlink(symlink) => { + let dest = symlink.resolve_dest_from_root(&self.root_path); + let (resolved_path, entry) = + self.find_entry_inner(&dest, seen, case_sensitivity)?; + final_path = resolved_path; // overwrite with the new resolved path + match entry { + VfsEntryRef::Dir(dir) => { + final_path.push(component); + dir + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + } + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + }; + let component = component.to_string_lossy(); + current_entry = current_dir + .entries + .get_by_name(&component, case_sensitivity) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "path not found") + })? + .as_ref(); + } + + Ok((final_path, current_entry)) + } +} + +pub struct FileBackedVfsFile { + file: VirtualFile, + pos: RefCell, + vfs: Arc, +} + +impl FileBackedVfsFile { + pub fn seek(&self, pos: SeekFrom) -> std::io::Result { + match pos { + SeekFrom::Start(pos) => { + *self.pos.borrow_mut() = pos; + Ok(pos) + } + SeekFrom::End(offset) => { + if offset < 0 && -offset as u64 > self.file.offset.len { + let msg = "An attempt was made to move the file pointer before the beginning of the file."; + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + msg, + )) + } else { + let mut current_pos = self.pos.borrow_mut(); + *current_pos = if offset >= 0 { + self.file.offset.len - (offset as u64) + } else { + self.file.offset.len + (-offset as u64) + }; + Ok(*current_pos) + } + } + SeekFrom::Current(offset) => { + let mut current_pos = self.pos.borrow_mut(); + if offset >= 0 { + *current_pos += offset as u64; + } else if -offset as u64 > *current_pos { + return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.")); + } else { + *current_pos -= -offset as u64; + } + Ok(*current_pos) + } + } + } + + pub fn read_to_buf(&self, buf: &mut [u8]) -> std::io::Result { + let read_pos = { + let mut pos = self.pos.borrow_mut(); + let read_pos = *pos; + // advance the position due to the read + *pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64); + read_pos + }; + self.vfs.read_file(&self.file, read_pos, buf) + } + + fn read_to_end(&self) -> FsResult> { + let read_pos = { + let mut pos = self.pos.borrow_mut(); + let read_pos = *pos; + // todo(dsherret): should this always set it to the end of the file? + if *pos < self.file.offset.len { + // advance the position due to the read + *pos = self.file.offset.len; + } + read_pos + }; + if read_pos > self.file.offset.len { + return Ok(Cow::Borrowed(&[])); + } + if read_pos == 0 { + Ok(self.vfs.read_file_all(&self.file)?) + } else { + let size = (self.file.offset.len - read_pos) as usize; + let mut buf = vec![0; size]; + self.vfs.read_file(&self.file, read_pos, &mut buf)?; + Ok(Cow::Owned(buf)) + } + } +} + +#[async_trait::async_trait(?Send)] +impl deno_io::fs::File for FileBackedVfsFile { + fn read_sync(self: Rc, buf: &mut [u8]) -> FsResult { + self.read_to_buf(buf).map_err(Into::into) + } + async fn read_byob( + self: Rc, + mut buf: BufMutView, + ) -> FsResult<(usize, BufMutView)> { + // this is fast, no need to spawn a task + let nread = self.read_to_buf(&mut buf)?; + Ok((nread, buf)) + } + + fn write_sync(self: Rc, _buf: &[u8]) -> FsResult { + Err(FsError::NotSupported) + } + async fn write( + self: Rc, + _buf: BufView, + ) -> FsResult { + Err(FsError::NotSupported) + } + + fn write_all_sync(self: Rc, _buf: &[u8]) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn write_all(self: Rc, _buf: BufView) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn read_all_sync(self: Rc) -> FsResult> { + self.read_to_end() + } + async fn read_all_async(self: Rc) -> FsResult> { + // this is fast, no need to spawn a task + self.read_to_end() + } + + fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn chmod_async(self: Rc, _mode: u32) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn seek_sync(self: Rc, pos: SeekFrom) -> FsResult { + self.seek(pos).map_err(|err| err.into()) + } + async fn seek_async(self: Rc, pos: SeekFrom) -> FsResult { + self.seek(pos).map_err(|err| err.into()) + } + + fn datasync_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn datasync_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn sync_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn sync_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn stat_sync(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + async fn stat_async(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + + fn lock_sync(self: Rc, _exclusive: bool) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn lock_async(self: Rc, _exclusive: bool) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn unlock_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn unlock_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn truncate_sync(self: Rc, _len: u64) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn truncate_async(self: Rc, _len: u64) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn utime_sync( + self: Rc, + _atime_secs: i64, + _atime_nanos: u32, + _mtime_secs: i64, + _mtime_nanos: u32, + ) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn utime_async( + self: Rc, + _atime_secs: i64, + _atime_nanos: u32, + _mtime_secs: i64, + _mtime_nanos: u32, + ) -> FsResult<()> { + Err(FsError::NotSupported) + } + + // lower level functionality + fn as_stdio(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + fn backing_fd(self: Rc) -> Option { + None + } + fn try_clone_inner(self: Rc) -> FsResult> { + Ok(self) + } +} + +#[derive(Debug, Clone)] +pub struct FileBackedVfsDirEntry { + pub parent_path: PathBuf, + pub metadata: FileBackedVfsMetadata, +} + +#[derive(Debug, Clone)] +pub struct FileBackedVfsMetadata { + pub name: String, + pub file_type: sys_traits::FileType, + pub len: u64, +} + +impl FileBackedVfsMetadata { + pub fn from_vfs_entry_ref(vfs_entry: VfsEntryRef) -> Self { + FileBackedVfsMetadata { + file_type: match vfs_entry { + VfsEntryRef::Dir(_) => sys_traits::FileType::Dir, + VfsEntryRef::File(_) => sys_traits::FileType::File, + VfsEntryRef::Symlink(_) => sys_traits::FileType::Symlink, + }, + name: vfs_entry.name().to_string(), + len: match vfs_entry { + VfsEntryRef::Dir(_) => 0, + VfsEntryRef::File(file) => file.offset.len, + VfsEntryRef::Symlink(_) => 0, + }, + } + } + pub fn as_fs_stat(&self) -> FsStat { + FsStat { + is_directory: self.file_type == sys_traits::FileType::Dir, + is_file: self.file_type == sys_traits::FileType::File, + is_symlink: self.file_type == sys_traits::FileType::Symlink, + atime: None, + birthtime: None, + mtime: None, + ctime: None, + blksize: 0, + size: self.len, + dev: 0, + ino: 0, + mode: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blocks: 0, + is_block_device: false, + is_char_device: false, + is_fifo: false, + is_socket: false, + } + } +} + +#[derive(Debug)] +pub struct FileBackedVfs { + vfs_data: Cow<'static, [u8]>, + fs_root: VfsRoot, + case_sensitivity: FileSystemCaseSensitivity, +} + +impl FileBackedVfs { + pub fn new( + data: Cow<'static, [u8]>, + fs_root: VfsRoot, + case_sensitivity: FileSystemCaseSensitivity, + ) -> Self { + Self { + vfs_data: data, + fs_root, + case_sensitivity, + } + } + + pub fn root(&self) -> &Path { + &self.fs_root.root_path + } + + pub fn is_path_within(&self, path: &Path) -> bool { + path.starts_with(&self.fs_root.root_path) + } + + pub fn open_file( + self: &Arc, + path: &Path, + ) -> std::io::Result { + let file = self.file_entry(path)?; + Ok(FileBackedVfsFile { + file: file.clone(), + vfs: self.clone(), + pos: Default::default(), + }) + } + + pub fn read_dir(&self, path: &Path) -> std::io::Result> { + let dir = self.dir_entry(path)?; + Ok( + dir + .entries + .iter() + .map(|entry| FsDirEntry { + name: entry.name().to_string(), + is_file: matches!(entry, VfsEntry::File(_)), + is_directory: matches!(entry, VfsEntry::Dir(_)), + is_symlink: matches!(entry, VfsEntry::Symlink(_)), + }) + .collect(), + ) + } + + pub fn read_dir_with_metadata<'a>( + &'a self, + path: &Path, + ) -> std::io::Result + 'a> { + let dir = self.dir_entry(path)?; + let path = path.to_path_buf(); + Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry { + parent_path: path.to_path_buf(), + metadata: FileBackedVfsMetadata::from_vfs_entry_ref(entry.as_ref()), + })) + } + + pub fn read_link(&self, path: &Path) -> std::io::Result { + let (_, entry) = self + .fs_root + .find_entry_no_follow(path, self.case_sensitivity)?; + match entry { + VfsEntryRef::Symlink(symlink) => { + Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path)) + } + VfsEntryRef::Dir(_) | VfsEntryRef::File(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "not a symlink", + )), + } + } + + pub fn lstat(&self, path: &Path) -> std::io::Result { + let (_, entry) = self + .fs_root + .find_entry_no_follow(path, self.case_sensitivity)?; + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) + } + + pub fn stat(&self, path: &Path) -> std::io::Result { + let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; + Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) + } + + pub fn canonicalize(&self, path: &Path) -> std::io::Result { + let (path, _) = self.fs_root.find_entry(path, self.case_sensitivity)?; + Ok(path) + } + + pub fn read_file_all( + &self, + file: &VirtualFile, + ) -> std::io::Result> { + self.read_file_offset_with_len(file.offset) + } + + pub fn read_file_offset_with_len( + &self, + offset_with_len: OffsetWithLength, + ) -> std::io::Result> { + let read_range = + self.get_read_range(offset_with_len, 0, offset_with_len.len)?; + match &self.vfs_data { + Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), + Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), + } + } + + pub fn read_file( + &self, + file: &VirtualFile, + pos: u64, + buf: &mut [u8], + ) -> std::io::Result { + let read_range = self.get_read_range(file.offset, pos, buf.len() as u64)?; + let read_len = read_range.len(); + buf[..read_len].copy_from_slice(&self.vfs_data[read_range]); + Ok(read_len) + } + + fn get_read_range( + &self, + file_offset_and_len: OffsetWithLength, + pos: u64, + len: u64, + ) -> std::io::Result> { + if pos > file_offset_and_len.len { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected EOF", + )); + } + let file_offset = + self.fs_root.start_file_offset + file_offset_and_len.offset; + let start = file_offset + pos; + let end = file_offset + std::cmp::min(pos + len, file_offset_and_len.len); + Ok(start as usize..end as usize) + } + + pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> { + let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; + match entry { + VfsEntryRef::Dir(dir) => Ok(dir), + VfsEntryRef::Symlink(_) => unreachable!(), + VfsEntryRef::File(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "path is a file", + )), + } + } + + pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> { + let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; + match entry { + VfsEntryRef::Dir(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "path is a directory", + )), + VfsEntryRef::Symlink(_) => unreachable!(), + VfsEntryRef::File(file) => Ok(file), + } + } +} + +#[cfg(test)] +mod test { + use std::io::Write; + + use deno_lib::standalone::virtual_fs::VfsBuilder; + use test_util::assert_contains; + use test_util::TempDir; + + use super::*; + + #[track_caller] + fn read_file(vfs: &FileBackedVfs, path: &Path) -> String { + let file = vfs.file_entry(path).unwrap(); + String::from_utf8(vfs.read_file_all(file).unwrap().into_owned()).unwrap() + } + + #[test] + fn builds_and_uses_virtual_fs() { + let temp_dir = TempDir::new(); + // we canonicalize the temp directory because the vfs builder + // will canonicalize the root path + let src_path = temp_dir.path().canonicalize().join("src"); + src_path.create_dir_all(); + src_path.join("sub_dir").create_dir_all(); + src_path.join("e.txt").write("e"); + src_path.symlink_file("e.txt", "sub_dir/e.txt"); + let src_path = src_path.to_path_buf(); + let mut builder = VfsBuilder::new(); + builder + .add_file_with_data_raw(&src_path.join("a.txt"), "data".into()) + .unwrap(); + builder + .add_file_with_data_raw(&src_path.join("b.txt"), "data".into()) + .unwrap(); + assert_eq!(builder.files_len(), 1); // because duplicate data + builder + .add_file_with_data_raw(&src_path.join("c.txt"), "c".into()) + .unwrap(); + builder + .add_file_with_data_raw( + &src_path.join("sub_dir").join("d.txt"), + "d".into(), + ) + .unwrap(); + builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); + builder + .add_symlink(&src_path.join("sub_dir").join("e.txt")) + .unwrap(); + + // get the virtual fs + let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); + + assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data"); + assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data"); + + // attempt reading a symlink + assert_eq!( + read_file(&virtual_fs, &dest_path.join("sub_dir").join("e.txt")), + "e", + ); + + // canonicalize symlink + assert_eq!( + virtual_fs + .canonicalize(&dest_path.join("sub_dir").join("e.txt")) + .unwrap(), + dest_path.join("e.txt"), + ); + + // metadata + assert_eq!( + virtual_fs + .lstat(&dest_path.join("sub_dir").join("e.txt")) + .unwrap() + .file_type, + sys_traits::FileType::Symlink, + ); + assert_eq!( + virtual_fs + .stat(&dest_path.join("sub_dir").join("e.txt")) + .unwrap() + .file_type, + sys_traits::FileType::File, + ); + assert_eq!( + virtual_fs + .stat(&dest_path.join("sub_dir")) + .unwrap() + .file_type, + sys_traits::FileType::Dir, + ); + assert_eq!( + virtual_fs.stat(&dest_path.join("e.txt")).unwrap().file_type, + sys_traits::FileType::File + ); + } + + #[test] + fn test_include_dir_recursive() { + let temp_dir = TempDir::new(); + let temp_dir_path = temp_dir.path().canonicalize(); + temp_dir.create_dir_all("src/nested/sub_dir"); + temp_dir.write("src/a.txt", "data"); + temp_dir.write("src/b.txt", "data"); + temp_dir.path().symlink_dir( + temp_dir_path.join("src/nested/sub_dir"), + temp_dir_path.join("src/sub_dir_link"), + ); + temp_dir.write("src/nested/sub_dir/c.txt", "c"); + + // build and create the virtual fs + let src_path = temp_dir_path.join("src").to_path_buf(); + let mut builder = VfsBuilder::new(); + builder.add_dir_recursive(&src_path).unwrap(); + let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); + + assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data",); + assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data",); + + assert_eq!( + read_file( + &virtual_fs, + &dest_path.join("nested").join("sub_dir").join("c.txt") + ), + "c", + ); + assert_eq!( + read_file(&virtual_fs, &dest_path.join("sub_dir_link").join("c.txt")), + "c", + ); + assert_eq!( + virtual_fs + .lstat(&dest_path.join("sub_dir_link")) + .unwrap() + .file_type, + sys_traits::FileType::Symlink, + ); + + assert_eq!( + virtual_fs + .canonicalize(&dest_path.join("sub_dir_link").join("c.txt")) + .unwrap(), + dest_path.join("nested").join("sub_dir").join("c.txt"), + ); + } + + fn into_virtual_fs( + builder: VfsBuilder, + temp_dir: &TempDir, + ) -> (PathBuf, FileBackedVfs) { + let virtual_fs_file = temp_dir.path().join("virtual_fs"); + let vfs = builder.build(); + { + let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); + for file_data in &vfs.files { + file.write_all(file_data).unwrap(); + } + } + let dest_path = temp_dir.path().join("dest"); + let data = std::fs::read(&virtual_fs_file).unwrap(); + ( + dest_path.to_path_buf(), + FileBackedVfs::new( + Cow::Owned(data), + VfsRoot { + dir: VirtualDirectory { + name: "".to_string(), + entries: vfs.entries, + }, + root_path: dest_path.to_path_buf(), + start_file_offset: 0, + }, + FileSystemCaseSensitivity::Sensitive, + ), + ) + } + + #[test] + fn circular_symlink() { + let temp_dir = TempDir::new(); + let src_path = temp_dir.path().canonicalize().join("src"); + src_path.create_dir_all(); + src_path.symlink_file("a.txt", "b.txt"); + src_path.symlink_file("b.txt", "c.txt"); + src_path.symlink_file("c.txt", "a.txt"); + let src_path = src_path.to_path_buf(); + let mut builder = VfsBuilder::new(); + let err = builder + .add_symlink(src_path.join("a.txt").as_path()) + .unwrap_err(); + assert_contains!(err.to_string(), "Circular symlink detected",); + } + + #[tokio::test] + async fn test_open_file() { + let temp_dir = TempDir::new(); + let temp_path = temp_dir.path().canonicalize(); + let mut builder = VfsBuilder::new(); + builder + .add_file_with_data_raw( + temp_path.join("a.txt").as_path(), + "0123456789".to_string().into_bytes(), + ) + .unwrap(); + let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); + let virtual_fs = Arc::new(virtual_fs); + let file = virtual_fs.open_file(&dest_path.join("a.txt")).unwrap(); + file.seek(SeekFrom::Current(2)).unwrap(); + let mut buf = vec![0; 2]; + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"23"); + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"45"); + file.seek(SeekFrom::Current(-4)).unwrap(); + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"23"); + file.seek(SeekFrom::Start(2)).unwrap(); + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"23"); + file.seek(SeekFrom::End(2)).unwrap(); + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"89"); + file.seek(SeekFrom::Current(-8)).unwrap(); + file.read_to_buf(&mut buf).unwrap(); + assert_eq!(buf, b"23"); + assert_eq!( + file + .seek(SeekFrom::Current(-5)) + .unwrap_err() + .to_string(), + "An attempt was made to move the file pointer before the beginning of the file." + ); + // go beyond the file length, then back + file.seek(SeekFrom::Current(40)).unwrap(); + file.seek(SeekFrom::Current(-38)).unwrap(); + let file = Rc::new(file); + let read_buf = file.clone().read(2).await.unwrap(); + assert_eq!(read_buf.to_vec(), b"67"); + file.clone().seek_sync(SeekFrom::Current(-2)).unwrap(); + + // read to the end of the file + let all_buf = file.clone().read_all_sync().unwrap(); + assert_eq!(all_buf.to_vec(), b"6789"); + file.clone().seek_sync(SeekFrom::Current(-9)).unwrap(); + + // try try_clone_inner and read_all_async + let all_buf = file + .try_clone_inner() + .unwrap() + .read_all_async() + .await + .unwrap(); + assert_eq!(all_buf.to_vec(), b"123456789"); + } +} diff --git a/cli/rt/integration_tests_runner.rs b/cli/rt/integration_tests_runner.rs new file mode 100644 index 0000000000..63f2abe460 --- /dev/null +++ b/cli/rt/integration_tests_runner.rs @@ -0,0 +1,5 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub fn main() { + // this file exists to cause the executable to be built when running cargo test +} diff --git a/cli/mainrt.rs b/cli/rt/main.rs similarity index 66% rename from cli/mainrt.rs rename to cli/rt/main.rs index 8eea3f85ed..60b5a2fb96 100644 --- a/cli/mainrt.rs +++ b/cli/rt/main.rs @@ -1,46 +1,27 @@ // Copyright 2018-2025 the Deno authors. MIT license. -// Allow unused code warnings because we share -// code between the two bin targets. -#![allow(dead_code)] -#![allow(unused_imports)] - -mod standalone; - -mod args; -mod cache; -mod emit; -mod file_fetcher; -mod http_util; -mod js; -mod node; -mod npm; -mod resolver; -mod shared; -mod sys; -mod task_runner; -mod util; -mod version; -mod worker; - use std::borrow::Cow; -use std::collections::HashMap; use std::env; -use std::env::current_exe; use std::sync::Arc; use deno_core::error::AnyError; use deno_core::error::CoreError; -use deno_core::error::JsError; +use deno_lib::util::result::any_and_jserrorbox_downcast_ref; +use deno_lib::version::otel_runtime_config; +use deno_runtime::deno_telemetry::OtelConfig; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics; -pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS; use deno_terminal::colors; use indexmap::IndexMap; -use standalone::DenoCompileFileSystem; -use crate::args::Flags; -use crate::util::result::any_and_jserrorbox_downcast_ref; +use self::binary::extract_standalone; +use self::file_system::DenoRtSys; + +mod binary; +mod code_cache; +mod file_system; +mod node; +mod run; pub(crate) fn unstable_exit_cb(feature: &str, api_name: &str) { log::error!( @@ -87,27 +68,26 @@ fn load_env_vars(env_vars: &IndexMap) { fn main() { deno_runtime::deno_permissions::mark_standalone(); let args: Vec<_> = env::args_os().collect(); - let standalone = standalone::extract_standalone(Cow::Owned(args)); + let standalone = extract_standalone(Cow::Owned(args)); let future = async move { match standalone { Ok(Some(data)) => { - deno_telemetry::init( - crate::args::otel_runtime_config(), + deno_runtime::deno_telemetry::init( + otel_runtime_config(), &data.metadata.otel_config, )?; - util::logger::init( + init_logging( data.metadata.log_level, Some(data.metadata.otel_config.clone()), ); load_env_vars(&data.metadata.env_vars_from_env_file); - let fs = DenoCompileFileSystem::new(data.vfs.clone()); - let sys = crate::sys::CliSys::DenoCompile(fs.clone()); - let exit_code = standalone::run(Arc::new(fs), sys, data).await?; + let sys = DenoRtSys::new(data.vfs.clone()); + let exit_code = run::run(Arc::new(sys.clone()), sys, data).await?; deno_runtime::exit(exit_code); } Ok(None) => Ok(()), Err(err) => { - util::logger::init(None, None); + init_logging(None, None); Err(err) } } @@ -115,3 +95,15 @@ fn main() { unwrap_or_exit(create_and_run_current_thread_with_maybe_metrics(future)); } + +fn init_logging( + maybe_level: Option, + otel_config: Option, +) { + deno_lib::util::logger::init(deno_lib::util::logger::InitLoggingOptions { + maybe_level, + otel_config, + on_log_start: || {}, + on_log_end: || {}, + }) +} diff --git a/cli/rt/node.rs b/cli/rt/node.rs new file mode 100644 index 0000000000..ef4f99cc8a --- /dev/null +++ b/cli/rt/node.rs @@ -0,0 +1,165 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::sync::Arc; + +use deno_core::url::Url; +use deno_error::JsErrorBox; +use deno_lib::loader::NpmModuleLoader; +use deno_lib::standalone::binary::CjsExportAnalysisEntry; +use deno_media_type::MediaType; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmReqResolver; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; +use node_resolver::analyze::CjsAnalysis; +use node_resolver::analyze::CjsAnalysisExports; +use node_resolver::analyze::NodeCodeTranslator; + +use crate::binary::StandaloneModules; +use crate::file_system::DenoRtSys; + +pub type DenoRtCjsTracker = + deno_resolver::cjs::CjsTracker; +pub type DenoRtNpmResolver = deno_resolver::npm::NpmResolver; +pub type DenoRtNpmModuleLoader = NpmModuleLoader< + CjsCodeAnalyzer, + DenoInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + DenoRtNpmResolver, + DenoRtSys, +>; +pub type DenoRtNodeCodeTranslator = NodeCodeTranslator< + CjsCodeAnalyzer, + DenoInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + DenoRtNpmResolver, + DenoRtSys, +>; +pub type DenoRtNodeResolver = deno_runtime::deno_node::NodeResolver< + DenoInNpmPackageChecker, + DenoRtNpmResolver, + DenoRtSys, +>; +pub type DenoRtNpmReqResolver = NpmReqResolver< + DenoInNpmPackageChecker, + RealIsBuiltInNodeModuleChecker, + DenoRtNpmResolver, + DenoRtSys, +>; + +pub struct CjsCodeAnalyzer { + cjs_tracker: Arc, + modules: Arc, + sys: DenoRtSys, +} + +impl CjsCodeAnalyzer { + pub fn new( + cjs_tracker: Arc, + modules: Arc, + sys: DenoRtSys, + ) -> Self { + Self { + cjs_tracker, + modules, + sys, + } + } + + fn inner_cjs_analysis<'a>( + &self, + specifier: &Url, + source: Cow<'a, str>, + ) -> Result, JsErrorBox> { + let media_type = MediaType::from_specifier(specifier); + if media_type == MediaType::Json { + return Ok(CjsAnalysis::Cjs(CjsAnalysisExports { + exports: vec![], + reexports: vec![], + })); + } + + let cjs_tracker = self.cjs_tracker.clone(); + let is_maybe_cjs = cjs_tracker + .is_maybe_cjs(specifier, media_type) + .map_err(JsErrorBox::from_err)?; + let analysis = if is_maybe_cjs { + let data = self + .modules + .read(specifier)? + .and_then(|d| d.cjs_export_analysis); + match data { + Some(data) => { + let data: CjsExportAnalysisEntry = bincode::deserialize(&data) + .map_err(|err| JsErrorBox::generic(err.to_string()))?; + match data { + CjsExportAnalysisEntry::Esm => { + cjs_tracker.set_is_known_script(specifier, false); + CjsAnalysis::Esm(source) + } + CjsExportAnalysisEntry::Cjs(analysis) => { + cjs_tracker.set_is_known_script(specifier, true); + CjsAnalysis::Cjs(analysis) + } + } + } + None => { + if log::log_enabled!(log::Level::Debug) { + if self.sys.is_specifier_in_vfs(specifier) { + log::debug!( + "No CJS export analysis was stored for '{}'. Assuming ESM. This might indicate a bug in Deno.", + specifier + ); + } else { + log::debug!( + "Analyzing potentially CommonJS files is not supported at runtime in a compiled executable ({}). Assuming ESM.", + specifier + ); + } + } + // assume ESM as we don't have access to swc here + CjsAnalysis::Esm(source) + } + } + } else { + CjsAnalysis::Esm(source) + }; + + Ok(analysis) + } +} + +#[async_trait::async_trait(?Send)] +impl node_resolver::analyze::CjsCodeAnalyzer for CjsCodeAnalyzer { + async fn analyze_cjs<'a>( + &self, + specifier: &Url, + source: Option>, + ) -> Result, JsErrorBox> { + let source = match source { + Some(source) => source, + None => { + if let Ok(path) = deno_path_util::url_to_file_path(specifier) { + // todo(dsherret): should this use the sync method instead? + if let Ok(source_from_file) = + self.sys.read_text_file_lossy_async(path, None).await + { + source_from_file + } else { + return Ok(CjsAnalysis::Cjs(CjsAnalysisExports { + exports: vec![], + reexports: vec![], + })); + } + } else { + return Ok(CjsAnalysis::Cjs(CjsAnalysisExports { + exports: vec![], + reexports: vec![], + })); + } + } + }; + self.inner_cjs_analysis(specifier, source) + } +} diff --git a/cli/rt/run.rs b/cli/rt/run.rs new file mode 100644 index 0000000000..6f5c05b467 --- /dev/null +++ b/cli/rt/run.rs @@ -0,0 +1,990 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::OnceLock; + +use deno_cache_dir::npm::NpmCacheDir; +use deno_config::workspace::MappedResolution; +use deno_config::workspace::ResolverWorkspaceJsrPackage; +use deno_config::workspace::WorkspaceResolver; +use deno_core::error::AnyError; +use deno_core::error::ModuleLoaderError; +use deno_core::futures::future::LocalBoxFuture; +use deno_core::futures::FutureExt; +use deno_core::url::Url; +use deno_core::v8_set_flags; +use deno_core::FastString; +use deno_core::FeatureChecker; +use deno_core::ModuleLoader; +use deno_core::ModuleSourceCode; +use deno_core::ModuleType; +use deno_core::RequestedModuleType; +use deno_core::ResolutionKind; +use deno_core::SourceCodeCacheInfo; +use deno_error::JsErrorBox; +use deno_lib::args::get_root_cert_store; +use deno_lib::args::npm_pkg_req_ref_to_binary_command; +use deno_lib::args::CaData; +use deno_lib::args::RootCertStoreLoadError; +use deno_lib::loader::NpmModuleLoader; +use deno_lib::npm::create_npm_process_state_provider; +use deno_lib::npm::NpmRegistryReadPermissionChecker; +use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; +use deno_lib::standalone::binary::NodeModules; +use deno_lib::util::hash::FastInsecureHasher; +use deno_lib::util::text_encoding::from_utf8_lossy_cow; +use deno_lib::util::text_encoding::from_utf8_lossy_owned; +use deno_lib::util::v8::construct_v8_flags; +use deno_lib::worker::CreateModuleLoaderResult; +use deno_lib::worker::LibMainWorkerFactory; +use deno_lib::worker::LibMainWorkerOptions; +use deno_lib::worker::ModuleLoaderFactory; +use deno_lib::worker::StorageKeyResolver; +use deno_media_type::MediaType; +use deno_npm::npm_rc::ResolvedNpmRc; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_package_json::PackageJsonDepValue; +use deno_resolver::cjs::CjsTracker; +use deno_resolver::cjs::IsCjsResolutionMode; +use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; +use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions; +use deno_resolver::npm::managed::NpmResolutionCell; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; +use deno_resolver::npm::CreateInNpmPkgCheckerOptions; +use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::npm::NpmReqResolver; +use deno_resolver::npm::NpmReqResolverOptions; +use deno_resolver::npm::NpmResolver; +use deno_resolver::npm::NpmResolverCreateOptions; +use deno_runtime::code_cache::CodeCache; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_node::create_host_defined_options; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; +use deno_runtime::deno_permissions::Permissions; +use deno_runtime::deno_permissions::PermissionsContainer; +use deno_runtime::deno_tls::rustls::RootCertStore; +use deno_runtime::deno_tls::RootCertStoreProvider; +use deno_runtime::deno_web::BlobStore; +use deno_runtime::permissions::RuntimePermissionDescriptorParser; +use deno_runtime::WorkerExecutionMode; +use deno_runtime::WorkerLogLevel; +use deno_semver::npm::NpmPackageReqReference; +use node_resolver::analyze::NodeCodeTranslator; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::NodeResolutionKind; +use node_resolver::NodeResolver; +use node_resolver::PackageJsonResolver; +use node_resolver::ResolutionMode; + +use crate::binary::DenoCompileModuleSource; +use crate::binary::StandaloneData; +use crate::binary::StandaloneModules; +use crate::code_cache::DenoCompileCodeCache; +use crate::file_system::DenoRtSys; +use crate::file_system::FileBackedVfs; +use crate::node::CjsCodeAnalyzer; +use crate::node::DenoRtCjsTracker; +use crate::node::DenoRtNodeCodeTranslator; +use crate::node::DenoRtNodeResolver; +use crate::node::DenoRtNpmModuleLoader; +use crate::node::DenoRtNpmReqResolver; + +struct SharedModuleLoaderState { + cjs_tracker: Arc, + code_cache: Option>, + modules: Arc, + node_code_translator: Arc, + node_resolver: Arc, + npm_module_loader: Arc, + npm_registry_permission_checker: NpmRegistryReadPermissionChecker, + npm_req_resolver: Arc, + vfs: Arc, + workspace_resolver: WorkspaceResolver, +} + +impl SharedModuleLoaderState { + fn get_code_cache( + &self, + specifier: &Url, + source: &[u8], + ) -> Option { + let Some(code_cache) = &self.code_cache else { + return None; + }; + if !code_cache.enabled() { + return None; + } + // deno version is already included in the root cache key + let hash = FastInsecureHasher::new_without_deno_version() + .write_hashable(source) + .finish(); + let data = code_cache.get_sync( + specifier, + deno_runtime::code_cache::CodeCacheType::EsModule, + hash, + ); + Some(SourceCodeCacheInfo { + hash, + data: data.map(Cow::Owned), + }) + } +} + +#[derive(Clone)] +struct EmbeddedModuleLoader { + shared: Arc, +} + +impl std::fmt::Debug for EmbeddedModuleLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbeddedModuleLoader").finish() + } +} + +impl ModuleLoader for EmbeddedModuleLoader { + fn resolve( + &self, + raw_specifier: &str, + referrer: &str, + kind: ResolutionKind, + ) -> Result { + let referrer = if referrer == "." { + if kind != ResolutionKind::MainModule { + return Err( + JsErrorBox::generic(format!( + "Expected to resolve main module, got {:?} instead.", + kind + )) + .into(), + ); + } + let current_dir = std::env::current_dir().unwrap(); + deno_core::resolve_path(".", ¤t_dir)? + } else { + Url::parse(referrer).map_err(|err| { + JsErrorBox::type_error(format!( + "Referrer uses invalid specifier: {}", + err + )) + })? + }; + let referrer_kind = if self + .shared + .cjs_tracker + .is_maybe_cjs(&referrer, MediaType::from_specifier(&referrer)) + .map_err(JsErrorBox::from_err)? + { + ResolutionMode::Require + } else { + ResolutionMode::Import + }; + + if self.shared.node_resolver.in_npm_package(&referrer) { + return Ok( + self + .shared + .node_resolver + .resolve( + raw_specifier, + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(JsErrorBox::from_err)? + .into_url(), + ); + } + + let mapped_resolution = self + .shared + .workspace_resolver + .resolve(raw_specifier, &referrer); + + match mapped_resolution { + Ok(MappedResolution::WorkspaceJsrPackage { specifier, .. }) => { + Ok(specifier) + } + Ok(MappedResolution::WorkspaceNpmPackage { + target_pkg_json: pkg_json, + sub_path, + .. + }) => Ok( + self + .shared + .node_resolver + .resolve_package_subpath_from_deno_module( + pkg_json.dir_path(), + sub_path.as_deref(), + Some(&referrer), + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(JsErrorBox::from_err)?, + ), + Ok(MappedResolution::PackageJson { + dep_result, + sub_path, + alias, + .. + }) => match dep_result + .as_ref() + .map_err(|e| JsErrorBox::from_err(e.clone()))? + { + PackageJsonDepValue::Req(req) => self + .shared + .npm_req_resolver + .resolve_req_with_sub_path( + req, + sub_path.as_deref(), + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(|e| JsErrorBox::from_err(e).into()), + PackageJsonDepValue::Workspace(version_req) => { + let pkg_folder = self + .shared + .workspace_resolver + .resolve_workspace_pkg_json_folder_for_pkg_json_dep( + alias, + version_req, + ) + .map_err(JsErrorBox::from_err)?; + Ok( + self + .shared + .node_resolver + .resolve_package_subpath_from_deno_module( + pkg_folder, + sub_path.as_deref(), + Some(&referrer), + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(JsErrorBox::from_err)?, + ) + } + }, + Ok(MappedResolution::Normal { specifier, .. }) + | Ok(MappedResolution::ImportMap { specifier, .. }) => { + if let Ok(reference) = + NpmPackageReqReference::from_specifier(&specifier) + { + return Ok( + self + .shared + .npm_req_resolver + .resolve_req_reference( + &reference, + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(JsErrorBox::from_err)?, + ); + } + + if specifier.scheme() == "jsr" { + if let Some(specifier) = self + .shared + .modules + .resolve_specifier(&specifier) + .map_err(JsErrorBox::from_err)? + { + return Ok(specifier.clone()); + } + } + + Ok( + self + .shared + .node_resolver + .handle_if_in_node_modules(&specifier) + .unwrap_or(specifier), + ) + } + Err(err) + if err.is_unmapped_bare_specifier() && referrer.scheme() == "file" => + { + let maybe_res = self + .shared + .npm_req_resolver + .resolve_if_for_npm_pkg( + raw_specifier, + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + ) + .map_err(JsErrorBox::from_err)?; + if let Some(res) = maybe_res { + return Ok(res.into_url()); + } + Err(JsErrorBox::from_err(err).into()) + } + Err(err) => Err(JsErrorBox::from_err(err).into()), + } + } + + fn get_host_defined_options<'s>( + &self, + scope: &mut deno_core::v8::HandleScope<'s>, + name: &str, + ) -> Option> { + let name = Url::parse(name).ok()?; + if self.shared.node_resolver.in_npm_package(&name) { + Some(create_host_defined_options(scope)) + } else { + None + } + } + + fn load( + &self, + original_specifier: &Url, + maybe_referrer: Option<&Url>, + _is_dynamic: bool, + _requested_module_type: RequestedModuleType, + ) -> deno_core::ModuleLoadResponse { + if original_specifier.scheme() == "data" { + let data_url_text = + match deno_media_type::data_url::RawDataUrl::parse(original_specifier) + .and_then(|url| url.decode()) + { + Ok(response) => response, + Err(err) => { + return deno_core::ModuleLoadResponse::Sync(Err( + JsErrorBox::type_error(format!("{:#}", err)).into(), + )); + } + }; + return deno_core::ModuleLoadResponse::Sync(Ok( + deno_core::ModuleSource::new( + deno_core::ModuleType::JavaScript, + ModuleSourceCode::String(data_url_text.into()), + original_specifier, + None, + ), + )); + } + + if self.shared.node_resolver.in_npm_package(original_specifier) { + let shared = self.shared.clone(); + let original_specifier = original_specifier.clone(); + let maybe_referrer = maybe_referrer.cloned(); + return deno_core::ModuleLoadResponse::Async( + async move { + let code_source = shared + .npm_module_loader + .load(&original_specifier, maybe_referrer.as_ref()) + .await + .map_err(JsErrorBox::from_err)?; + let code_cache_entry = shared.get_code_cache( + &code_source.found_url, + code_source.code.as_bytes(), + ); + Ok(deno_core::ModuleSource::new_with_redirect( + match code_source.media_type { + MediaType::Json => ModuleType::Json, + _ => ModuleType::JavaScript, + }, + code_source.code, + &original_specifier, + &code_source.found_url, + code_cache_entry, + )) + } + .boxed_local(), + ); + } + + match self.shared.modules.read(original_specifier) { + Ok(Some(module)) => { + let media_type = module.media_type; + let (module_specifier, module_type, module_source) = + module.into_parts(); + let is_maybe_cjs = match self + .shared + .cjs_tracker + .is_maybe_cjs(original_specifier, media_type) + { + Ok(is_maybe_cjs) => is_maybe_cjs, + Err(err) => { + return deno_core::ModuleLoadResponse::Sync(Err( + JsErrorBox::type_error(format!("{:?}", err)).into(), + )); + } + }; + if is_maybe_cjs { + let original_specifier = original_specifier.clone(); + let module_specifier = module_specifier.clone(); + let shared = self.shared.clone(); + deno_core::ModuleLoadResponse::Async( + async move { + let source = match module_source { + DenoCompileModuleSource::String(string) => { + Cow::Borrowed(string) + } + DenoCompileModuleSource::Bytes(module_code_bytes) => { + match module_code_bytes { + Cow::Owned(bytes) => { + Cow::Owned(from_utf8_lossy_owned(bytes)) + } + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + } + } + }; + let source = shared + .node_code_translator + .translate_cjs_to_esm(&module_specifier, Some(source)) + .await + .map_err(JsErrorBox::from_err)?; + let module_source = match source { + Cow::Owned(source) => ModuleSourceCode::String(source.into()), + Cow::Borrowed(source) => { + ModuleSourceCode::String(FastString::from_static(source)) + } + }; + let code_cache_entry = shared + .get_code_cache(&module_specifier, module_source.as_bytes()); + Ok(deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + &original_specifier, + &module_specifier, + code_cache_entry, + )) + } + .boxed_local(), + ) + } else { + let module_source = module_source.into_for_v8(); + let code_cache_entry = self + .shared + .get_code_cache(module_specifier, module_source.as_bytes()); + deno_core::ModuleLoadResponse::Sync(Ok( + deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + original_specifier, + module_specifier, + code_cache_entry, + ), + )) + } + } + Ok(None) => deno_core::ModuleLoadResponse::Sync(Err( + JsErrorBox::type_error(format!( + "Module not found: {}", + original_specifier + )) + .into(), + )), + Err(err) => deno_core::ModuleLoadResponse::Sync(Err( + JsErrorBox::type_error(format!("{:?}", err)).into(), + )), + } + } + + fn code_cache_ready( + &self, + specifier: Url, + source_hash: u64, + code_cache_data: &[u8], + ) -> LocalBoxFuture<'static, ()> { + if let Some(code_cache) = &self.shared.code_cache { + code_cache.set_sync( + specifier, + deno_runtime::code_cache::CodeCacheType::EsModule, + source_hash, + code_cache_data, + ); + } + std::future::ready(()).boxed_local() + } + + fn get_source_map(&self, file_name: &str) -> Option> { + let url = Url::parse(file_name).ok()?; + let data = self.shared.modules.read(&url).ok()??; + data.source_map + } + + fn get_source_mapped_source_line( + &self, + file_name: &str, + line_number: usize, + ) -> Option { + let specifier = Url::parse(file_name).ok()?; + let data = self.shared.modules.read(&specifier).ok()??; + + let source = String::from_utf8_lossy(&data.data); + // Do NOT use .lines(): it skips the terminating empty line. + // (due to internally using_terminator() instead of .split()) + let lines: Vec<&str> = source.split('\n').collect(); + if line_number >= lines.len() { + Some(format!( + "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)", + crate::colors::yellow("Warning"), line_number + 1, + )) + } else { + Some(lines[line_number].to_string()) + } + } +} + +impl NodeRequireLoader for EmbeddedModuleLoader { + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn deno_runtime::deno_node::NodePermissions, + path: &'a std::path::Path, + ) -> Result, JsErrorBox> { + if self.shared.modules.has_file(path) { + // allow reading if the file is in the snapshot + return Ok(Cow::Borrowed(path)); + } + + self + .shared + .npm_registry_permission_checker + .ensure_read_permission(permissions, path) + .map_err(JsErrorBox::from_err) + } + + fn load_text_file_lossy( + &self, + path: &std::path::Path, + ) -> Result, JsErrorBox> { + let file_entry = self + .shared + .vfs + .file_entry(path) + .map_err(JsErrorBox::from_err)?; + let file_bytes = self + .shared + .vfs + .read_file_offset_with_len( + file_entry.transpiled_offset.unwrap_or(file_entry.offset), + ) + .map_err(JsErrorBox::from_err)?; + Ok(from_utf8_lossy_cow(file_bytes)) + } + + fn is_maybe_cjs(&self, specifier: &Url) -> Result { + let media_type = MediaType::from_specifier(specifier); + self.shared.cjs_tracker.is_maybe_cjs(specifier, media_type) + } +} + +struct StandaloneModuleLoaderFactory { + shared: Arc, +} + +impl StandaloneModuleLoaderFactory { + pub fn create_result(&self) -> CreateModuleLoaderResult { + let loader = Rc::new(EmbeddedModuleLoader { + shared: self.shared.clone(), + }); + CreateModuleLoaderResult { + module_loader: loader.clone(), + node_require_loader: loader, + } + } +} + +impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { + fn create_for_main( + &self, + _root_permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { + self.create_result() + } + + fn create_for_worker( + &self, + _parent_permissions: PermissionsContainer, + _permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { + self.create_result() + } +} + +struct StandaloneRootCertStoreProvider { + ca_stores: Option>, + ca_data: Option, + cell: OnceLock>, +} + +impl RootCertStoreProvider for StandaloneRootCertStoreProvider { + fn get_or_try_init(&self) -> Result<&RootCertStore, JsErrorBox> { + self + .cell + // get_or_try_init was not stable yet when this was written + .get_or_init(|| { + get_root_cert_store(None, self.ca_stores.clone(), self.ca_data.clone()) + }) + .as_ref() + .map_err(|err| JsErrorBox::from_err(err.clone())) + } +} + +pub async fn run( + fs: Arc, + sys: DenoRtSys, + data: StandaloneData, +) -> Result { + let StandaloneData { + metadata, + modules, + npm_snapshot, + root_path, + vfs, + } = data; + let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { + ca_stores: metadata.ca_stores, + ca_data: metadata.ca_data.map(CaData::Bytes), + cell: Default::default(), + }); + // use a dummy npm registry url + let npm_registry_url = Url::parse("https://localhost/").unwrap(); + let root_dir_url = Arc::new(Url::from_directory_path(&root_path).unwrap()); + let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); + let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone())); + let npm_registry_permission_checker = { + let mode = match &metadata.node_modules { + Some(NodeModules::Managed { + node_modules_dir: Some(path), + }) => NpmRegistryReadPermissionCheckerMode::Local(PathBuf::from(path)), + Some(NodeModules::Byonm { .. }) => { + NpmRegistryReadPermissionCheckerMode::Byonm + } + Some(NodeModules::Managed { + node_modules_dir: None, + }) + | None => NpmRegistryReadPermissionCheckerMode::Global( + npm_global_cache_dir.clone(), + ), + }; + NpmRegistryReadPermissionChecker::new(sys.clone(), mode) + }; + let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules { + Some(NodeModules::Managed { node_modules_dir }) => { + // create an npmrc that uses the fake npm_registry_url to resolve packages + let npmrc = Arc::new(ResolvedNpmRc { + default_config: deno_npm::npm_rc::RegistryConfigWithUrl { + registry_url: npm_registry_url.clone(), + config: Default::default(), + }, + scopes: Default::default(), + registry_configs: Default::default(), + }); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &sys, + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); + let snapshot = npm_snapshot.unwrap(); + let maybe_node_modules_path = node_modules_dir + .map(|node_modules_dir| root_path.join(node_modules_dir)); + let in_npm_pkg_checker = + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( + ManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: maybe_node_modules_path.as_deref(), + }, + )); + let npm_resolution = + Arc::new(NpmResolutionCell::new(NpmResolutionSnapshot::new(snapshot))); + let npm_resolver = NpmResolver::::new::( + NpmResolverCreateOptions::Managed(ManagedNpmResolverCreateOptions { + npm_resolution, + npm_cache_dir, + sys: sys.clone(), + maybe_node_modules_path, + npm_system_info: Default::default(), + npmrc, + }), + ); + (in_npm_pkg_checker, npm_resolver) + } + Some(NodeModules::Byonm { + root_node_modules_dir, + }) => { + let root_node_modules_dir = + root_node_modules_dir.map(|p| vfs.root().join(p)); + let in_npm_pkg_checker = + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Byonm); + let npm_resolver = NpmResolver::::new::( + NpmResolverCreateOptions::Byonm(ByonmNpmResolverCreateOptions { + sys: sys.clone(), + pkg_json_resolver: pkg_json_resolver.clone(), + root_node_modules_dir, + }), + ); + (in_npm_pkg_checker, npm_resolver) + } + None => { + // Packages from different registries are already inlined in the binary, + // so no need to create actual `.npmrc` configuration. + let npmrc = create_default_npmrc(); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &sys, + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); + let in_npm_pkg_checker = + DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( + ManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: None, + }, + )); + let npm_resolution = Arc::new(NpmResolutionCell::default()); + let npm_resolver = NpmResolver::::new::( + NpmResolverCreateOptions::Managed(ManagedNpmResolverCreateOptions { + npm_resolution, + sys: sys.clone(), + npm_cache_dir, + maybe_node_modules_path: None, + npm_system_info: Default::default(), + npmrc: create_default_npmrc(), + }), + ); + (in_npm_pkg_checker, npm_resolver) + } + }; + + let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some(); + let node_resolver = Arc::new(NodeResolver::new( + in_npm_pkg_checker.clone(), + RealIsBuiltInNodeModuleChecker, + npm_resolver.clone(), + pkg_json_resolver.clone(), + sys.clone(), + node_resolver::ConditionsFromResolutionMode::default(), + )); + let cjs_tracker = Arc::new(CjsTracker::new( + in_npm_pkg_checker.clone(), + pkg_json_resolver.clone(), + if metadata.unstable_config.detect_cjs { + IsCjsResolutionMode::ImplicitTypeCommonJs + } else if metadata.workspace_resolver.package_jsons.is_empty() { + IsCjsResolutionMode::Disabled + } else { + IsCjsResolutionMode::ExplicitTypeCommonJs + }, + )); + let npm_req_resolver = Arc::new(NpmReqResolver::new(NpmReqResolverOptions { + sys: sys.clone(), + in_npm_pkg_checker: in_npm_pkg_checker.clone(), + node_resolver: node_resolver.clone(), + npm_resolver: npm_resolver.clone(), + })); + let cjs_esm_code_analyzer = + CjsCodeAnalyzer::new(cjs_tracker.clone(), modules.clone(), sys.clone()); + let node_code_translator = Arc::new(NodeCodeTranslator::new( + cjs_esm_code_analyzer, + in_npm_pkg_checker, + node_resolver.clone(), + npm_resolver.clone(), + pkg_json_resolver.clone(), + sys.clone(), + )); + let workspace_resolver = { + let import_map = match metadata.workspace_resolver.import_map { + Some(import_map) => Some( + import_map::parse_from_json_with_options( + root_dir_url.join(&import_map.specifier).unwrap(), + &import_map.json, + import_map::ImportMapOptions { + address_hook: None, + expand_imports: true, + }, + )? + .import_map, + ), + None => None, + }; + let pkg_jsons = metadata + .workspace_resolver + .package_jsons + .into_iter() + .map(|(relative_path, json)| { + let path = root_dir_url + .join(&relative_path) + .unwrap() + .to_file_path() + .unwrap(); + let pkg_json = + deno_package_json::PackageJson::load_from_value(path, json); + Arc::new(pkg_json) + }) + .collect(); + WorkspaceResolver::new_raw( + root_dir_url.clone(), + import_map, + metadata + .workspace_resolver + .jsr_pkgs + .iter() + .map(|pkg| ResolverWorkspaceJsrPackage { + is_patch: false, // only used for enhancing the diagnostic, which isn't shown in deno compile + base: root_dir_url.join(&pkg.relative_base).unwrap(), + name: pkg.name.clone(), + version: pkg.version.clone(), + exports: pkg.exports.clone(), + }) + .collect(), + pkg_jsons, + metadata.workspace_resolver.pkg_json_resolution, + ) + }; + let code_cache = match metadata.code_cache_key { + Some(code_cache_key) => Some(Arc::new(DenoCompileCodeCache::new( + root_path.with_file_name(format!( + "{}.cache", + root_path.file_name().unwrap().to_string_lossy() + )), + code_cache_key, + ))), + None => { + log::debug!("Code cache disabled."); + None + } + }; + let module_loader_factory = StandaloneModuleLoaderFactory { + shared: Arc::new(SharedModuleLoaderState { + cjs_tracker: cjs_tracker.clone(), + code_cache: code_cache.clone(), + modules, + node_code_translator: node_code_translator.clone(), + node_resolver: node_resolver.clone(), + npm_module_loader: Arc::new(NpmModuleLoader::new( + cjs_tracker.clone(), + node_code_translator, + sys.clone(), + )), + npm_registry_permission_checker, + npm_req_resolver, + vfs: vfs.clone(), + workspace_resolver, + }), + }; + + let permissions = { + let mut permissions = metadata.permissions; + // grant read access to the vfs + match &mut permissions.allow_read { + Some(vec) if vec.is_empty() => { + // do nothing, already granted + } + Some(vec) => { + vec.push(root_path.to_string_lossy().to_string()); + } + None => { + permissions.allow_read = + Some(vec![root_path.to_string_lossy().to_string()]); + } + } + + let desc_parser = + Arc::new(RuntimePermissionDescriptorParser::new(sys.clone())); + let permissions = + Permissions::from_options(desc_parser.as_ref(), &permissions)?; + PermissionsContainer::new(desc_parser, permissions) + }; + let feature_checker = Arc::new({ + let mut checker = FeatureChecker::default(); + checker.set_exit_cb(Box::new(crate::unstable_exit_cb)); + for feature in metadata.unstable_config.features { + // `metadata` is valid for the whole lifetime of the program, so we + // can leak the string here. + checker.enable_feature(feature.leak()); + } + checker + }); + let lib_main_worker_options = LibMainWorkerOptions { + argv: metadata.argv, + log_level: WorkerLogLevel::Info, + enable_op_summary_metrics: false, + enable_testing_features: false, + has_node_modules_dir, + inspect_brk: false, + inspect_wait: false, + strace_ops: None, + is_inspecting: false, + skip_op_registration: true, + location: metadata.location, + argv0: NpmPackageReqReference::from_specifier(&main_module) + .ok() + .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) + .or(std::env::args().next()), + node_debug: std::env::var("NODE_DEBUG").ok(), + origin_data_folder_path: None, + seed: metadata.seed, + unsafely_ignore_certificate_errors: metadata + .unsafely_ignore_certificate_errors, + node_ipc: None, + serve_port: None, + serve_host: None, + otel_config: metadata.otel_config, + startup_snapshot: deno_snapshots::CLI_SNAPSHOT, + }; + let worker_factory = LibMainWorkerFactory::new( + Arc::new(BlobStore::default()), + code_cache.map(|c| c.for_deno_core()), + feature_checker, + fs, + None, + Box::new(module_loader_factory), + node_resolver.clone(), + create_npm_process_state_provider(&npm_resolver), + pkg_json_resolver, + root_cert_store_provider, + StorageKeyResolver::empty(), + sys.clone(), + lib_main_worker_options, + ); + + // Initialize v8 once from the main thread. + v8_set_flags(construct_v8_flags(&[], &metadata.v8_flags, vec![])); + // TODO(bartlomieju): remove last argument once Deploy no longer needs it + deno_core::JsRuntime::init_platform(None, true); + + let main_module = match NpmPackageReqReference::from_specifier(&main_module) { + Ok(package_ref) => { + let pkg_folder = npm_resolver.resolve_pkg_folder_from_deno_module_req( + package_ref.req(), + &deno_path_util::url_from_file_path(&vfs.root().join("package.json"))?, + )?; + worker_factory + .resolve_npm_binary_entrypoint(&pkg_folder, package_ref.sub_path())? + } + Err(_) => main_module, + }; + + let mut worker = worker_factory.create_main_worker( + WorkerExecutionMode::Run, + permissions, + main_module, + )?; + + let exit_code = worker.run().await?; + Ok(exit_code) +} + +fn create_default_npmrc() -> Arc { + // this is fine because multiple registries are combined into + // one when compiling the binary + Arc::new(ResolvedNpmRc { + default_config: deno_npm::npm_rc::RegistryConfigWithUrl { + registry_url: Url::parse("https://registry.npmjs.org").unwrap(), + config: Default::default(), + }, + scopes: Default::default(), + registry_configs: Default::default(), + }) +} diff --git a/cli/snapshot/Cargo.toml b/cli/snapshot/Cargo.toml new file mode 100644 index 0000000000..06c968ef63 --- /dev/null +++ b/cli/snapshot/Cargo.toml @@ -0,0 +1,20 @@ +# Copyright 2018-2025 the Deno authors. MIT license. + +[package] +name = "deno_snapshots" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "v8 snapshot used by the Deno CLI" + +[lib] +path = "lib.rs" + +[features] +disable = [] + +[build-dependencies] +deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting", "only_snapshotted_js_sources", "snapshotting"] } diff --git a/cli/snapshot/README.md b/cli/snapshot/README.md new file mode 100644 index 0000000000..d52dead6b2 --- /dev/null +++ b/cli/snapshot/README.md @@ -0,0 +1,3 @@ +# deno_snapshots + +v8 snapshot used in the Deno CLI. diff --git a/cli/snapshot/build.rs b/cli/snapshot/build.rs new file mode 100644 index 0000000000..9f08ac0e9e --- /dev/null +++ b/cli/snapshot/build.rs @@ -0,0 +1,30 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +#[cfg(not(feature = "disable"))] +mod shared; + +fn main() { + #[cfg(not(feature = "disable"))] + { + let o = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin"); + create_cli_snapshot(cli_snapshot_path); + } +} + +#[cfg(not(feature = "disable"))] +fn create_cli_snapshot(snapshot_path: std::path::PathBuf) { + use deno_runtime::ops::bootstrap::SnapshotOptions; + + let snapshot_options = SnapshotOptions { + ts_version: shared::TS_VERSION.to_string(), + v8_version: deno_runtime::deno_core::v8::VERSION_STRING, + target: std::env::var("TARGET").unwrap(), + }; + + deno_runtime::snapshot::create_runtime_snapshot( + snapshot_path, + snapshot_options, + vec![], + ); +} diff --git a/cli/snapshot/lib.rs b/cli/snapshot/lib.rs new file mode 100644 index 0000000000..e5af4bcf6b --- /dev/null +++ b/cli/snapshot/lib.rs @@ -0,0 +1,13 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +#[cfg(not(feature = "disable"))] +pub static CLI_SNAPSHOT: Option<&[u8]> = Some(include_bytes!(concat!( + env!("OUT_DIR"), + "/CLI_SNAPSHOT.bin" +))); +#[cfg(feature = "disable")] +pub static CLI_SNAPSHOT: Option<&[u8]> = None; + +mod shared; + +pub use shared::TS_VERSION; diff --git a/cli/snapshot/shared.rs b/cli/snapshot/shared.rs new file mode 100644 index 0000000000..eec982776a --- /dev/null +++ b/cli/snapshot/shared.rs @@ -0,0 +1,3 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub static TS_VERSION: &str = "5.6.2"; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 54c82d17a3..4cb30316c1 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -1,111 +1,70 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::borrow::Cow; -use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::VecDeque; use std::env; -use std::env::current_exe; use std::ffi::OsString; use std::fs; use std::fs::File; -use std::future::Future; -use std::io::ErrorKind; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::io::Write; -use std::ops::Range; use std::path::Component; use std::path::Path; use std::path::PathBuf; -use std::process::Command; -use std::sync::Arc; +use capacity_builder::BytesAppendable; use deno_ast::MediaType; use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; -use deno_config::workspace::PackageJsonDepResolution; -use deno_config::workspace::ResolverWorkspaceJsrPackage; -use deno_config::workspace::Workspace; use deno_config::workspace::WorkspaceResolver; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::futures::io::AllowStdIo; -use deno_core::futures::AsyncReadExt; -use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; -use deno_error::JsErrorBox; use deno_graph::ModuleGraph; -use deno_lib::cache::DenoDir; -use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::args::CaData; +use deno_lib::args::UnstableConfig; +use deno_lib::shared::ReleaseChannel; +use deno_lib::standalone::binary::CjsExportAnalysisEntry; +use deno_lib::standalone::binary::Metadata; +use deno_lib::standalone::binary::NodeModules; +use deno_lib::standalone::binary::RemoteModuleEntry; +use deno_lib::standalone::binary::SerializedResolverWorkspaceJsrPackage; +use deno_lib::standalone::binary::SerializedWorkspaceResolver; +use deno_lib::standalone::binary::SerializedWorkspaceResolverImportMap; +use deno_lib::standalone::binary::SpecifierDataStore; +use deno_lib::standalone::binary::SpecifierId; +use deno_lib::standalone::binary::MAGIC_BYTES; +use deno_lib::standalone::virtual_fs::BuiltVfs; +use deno_lib::standalone::virtual_fs::VfsBuilder; use deno_lib::standalone::virtual_fs::VfsEntry; -use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; use deno_lib::standalone::virtual_fs::VirtualDirectory; use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; +use deno_lib::standalone::virtual_fs::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; +use deno_lib::util::hash::FastInsecureHasher; +use deno_lib::version::DENO_VERSION_INFO; use deno_npm::resolution::SerializedNpmResolutionSnapshot; -use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; -use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; -use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; use deno_path_util::url_from_directory_path; -use deno_path_util::url_from_file_path; use deno_path_util::url_to_file_path; -use deno_runtime::deno_fs; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_fs::RealFs; -use deno_runtime::deno_io::fs::FsError; -use deno_runtime::deno_node::PackageJson; -use deno_runtime::deno_permissions::PermissionsOptions; -use deno_semver::npm::NpmVersionReqParseError; -use deno_semver::package::PackageReq; -use deno_semver::Version; -use deno_semver::VersionReqSpecifierParseError; -use deno_telemetry::OtelConfig; use indexmap::IndexMap; -use log::Level; -use serde::Deserialize; -use serde::Serialize; +use node_resolver::analyze::CjsAnalysis; +use node_resolver::analyze::CjsCodeAnalyzer; -use super::file_system::DenoCompileFileSystem; -use super::serialization::deserialize_binary_data_section; -use super::serialization::serialize_binary_data_section; -use super::serialization::DenoCompileModuleData; -use super::serialization::DeserializedDataSection; -use super::serialization::RemoteModulesStore; -use super::serialization::RemoteModulesStoreBuilder; -use super::serialization::SourceMapStore; use super::virtual_fs::output_vfs; -use super::virtual_fs::BuiltVfs; -use super::virtual_fs::FileBackedVfs; -use super::virtual_fs::VfsBuilder; -use super::virtual_fs::VfsRoot; -use crate::args::CaData; use crate::args::CliOptions; use crate::args::CompileFlags; -use crate::args::NpmInstallDepsProvider; -use crate::args::PermissionFlags; -use crate::args::UnstableConfig; -use crate::cache::FastInsecureHasher; +use crate::cache::DenoDir; use crate::emit::Emitter; -use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; +use crate::node::CliCjsCodeAnalyzer; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; -use crate::shared::ReleaseChannel; -use crate::sys::CliSys; use crate::util::archive; -use crate::util::fs::canonicalize_path; -use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; -pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = - ".deno_compile_node_modules"; - /// A URL that can be designated as the base for relative URLs. /// /// After creation, this URL may be used to get the key for a @@ -151,104 +110,60 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> { } } -#[derive(Deserialize, Serialize)] -pub enum NodeModules { - Managed { - /// Relative path for the node_modules directory in the vfs. - node_modules_dir: Option, - }, - Byonm { - root_node_modules_dir: Option, - }, +struct SpecifierStore<'a> { + data: IndexMap<&'a Url, SpecifierId>, } -#[derive(Deserialize, Serialize)] -pub struct SerializedWorkspaceResolverImportMap { - pub specifier: String, - pub json: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SerializedResolverWorkspaceJsrPackage { - pub relative_base: String, - pub name: String, - pub version: Option, - pub exports: IndexMap, -} - -#[derive(Deserialize, Serialize)] -pub struct SerializedWorkspaceResolver { - pub import_map: Option, - pub jsr_pkgs: Vec, - pub package_jsons: BTreeMap, - pub pkg_json_resolution: PackageJsonDepResolution, -} - -// Note: Don't use hashmaps/hashsets. Ensure the serialization -// is deterministic. -#[derive(Deserialize, Serialize)] -pub struct Metadata { - pub argv: Vec, - pub seed: Option, - pub code_cache_key: Option, - pub permissions: PermissionsOptions, - pub location: Option, - pub v8_flags: Vec, - pub log_level: Option, - pub ca_stores: Option>, - pub ca_data: Option>, - pub unsafely_ignore_certificate_errors: Option>, - pub env_vars_from_env_file: IndexMap, - pub workspace_resolver: SerializedWorkspaceResolver, - pub entrypoint_key: String, - pub node_modules: Option, - pub unstable_config: UnstableConfig, - pub otel_config: OtelConfig, - pub vfs_case_sensitivity: FileSystemCaseSensitivity, -} - -#[allow(clippy::too_many_arguments)] -fn write_binary_bytes( - mut file_writer: File, - original_bin: Vec, - metadata: &Metadata, - npm_snapshot: Option, - remote_modules: &RemoteModulesStoreBuilder, - source_map_store: &SourceMapStore, - vfs: &BuiltVfs, - compile_flags: &CompileFlags, -) -> Result<(), AnyError> { - let data_section_bytes = serialize_binary_data_section( - metadata, - npm_snapshot, - remote_modules, - source_map_store, - vfs, - ) - .context("Serializing binary data section.")?; - - let target = compile_flags.resolve_target(); - if target.contains("linux") { - libsui::Elf::new(&original_bin).append( - "d3n0l4nd", - &data_section_bytes, - &mut file_writer, - )?; - } else if target.contains("windows") { - let mut pe = libsui::PortableExecutable::from(&original_bin)?; - if let Some(icon) = compile_flags.icon.as_ref() { - let icon = std::fs::read(icon)?; - pe = pe.set_icon(&icon)?; +impl<'a> SpecifierStore<'a> { + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: IndexMap::with_capacity(capacity), + } + } + + pub fn get_or_add(&mut self, specifier: &'a Url) -> SpecifierId { + let len = self.data.len(); + let entry = self.data.entry(specifier); + match entry { + indexmap::map::Entry::Occupied(occupied_entry) => *occupied_entry.get(), + indexmap::map::Entry::Vacant(vacant_entry) => { + let new_id = SpecifierId::new(len as u32); + vacant_entry.insert(new_id); + new_id + } + } + } + + pub fn for_serialization( + self, + base_url: &StandaloneRelativeFileBaseUrl<'a>, + ) -> SpecifierStoreForSerialization<'a> { + SpecifierStoreForSerialization { + data: self + .data + .into_iter() + .map(|(specifier, id)| (base_url.specifier_key(specifier), id)) + .collect(), + } + } +} + +struct SpecifierStoreForSerialization<'a> { + data: Vec<(Cow<'a, str>, SpecifierId)>, +} + +impl<'a> BytesAppendable<'a> for &'a SpecifierStoreForSerialization<'a> { + fn append_to_builder( + self, + builder: &mut capacity_builder::BytesBuilder<'a, TBytes>, + ) { + builder.append_le(self.data.len() as u32); + for (specifier_str, id) in &self.data { + builder.append_le(specifier_str.len() as u32); + builder.append(specifier_str.as_ref()); + builder.append(*id); } - - pe.write_resource("d3n0l4nd", data_section_bytes)? - .build(&mut file_writer)?; - } else if target.contains("darwin") { - libsui::Macho::from(original_bin)? - .write_section("d3n0l4nd", data_section_bytes)? - .build_and_sign(&mut file_writer)?; } - Ok(()) } pub fn is_standalone_binary(exe_path: &Path) -> bool { @@ -261,146 +176,6 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { || libsui::utils::is_macho(&data) } -pub struct StandaloneData { - pub metadata: Metadata, - pub modules: StandaloneModules, - pub npm_snapshot: Option, - pub root_path: PathBuf, - pub source_maps: SourceMapStore, - pub vfs: Arc, -} - -pub struct StandaloneModules { - remote_modules: RemoteModulesStore, - vfs: Arc, -} - -impl StandaloneModules { - pub fn resolve_specifier<'a>( - &'a self, - specifier: &'a ModuleSpecifier, - ) -> Result, JsErrorBox> { - if specifier.scheme() == "file" { - Ok(Some(specifier)) - } else { - self.remote_modules.resolve_specifier(specifier) - } - } - - pub fn has_file(&self, path: &Path) -> bool { - self.vfs.file_entry(path).is_ok() - } - - pub fn read<'a>( - &'a self, - specifier: &'a ModuleSpecifier, - kind: VfsFileSubDataKind, - ) -> Result>, AnyError> { - if specifier.scheme() == "file" { - let path = deno_path_util::url_to_file_path(specifier)?; - let bytes = match self.vfs.file_entry(&path) { - Ok(entry) => self.vfs.read_file_all(entry, kind)?, - Err(err) if err.kind() == ErrorKind::NotFound => { - match RealFs.read_file_sync(&path, None) { - Ok(bytes) => bytes, - Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => { - return Ok(None) - } - Err(err) => return Err(err.into()), - } - } - Err(err) => return Err(err.into()), - }; - Ok(Some(DenoCompileModuleData { - media_type: MediaType::from_specifier(specifier), - specifier, - data: bytes, - })) - } else { - self.remote_modules.read(specifier).map(|maybe_entry| { - maybe_entry.map(|entry| DenoCompileModuleData { - media_type: entry.media_type, - specifier: entry.specifier, - data: match kind { - VfsFileSubDataKind::Raw => entry.data, - VfsFileSubDataKind::ModuleGraph => { - entry.transpiled_data.unwrap_or(entry.data) - } - }, - }) - }) - } - } -} - -/// This function will try to run this binary as a standalone binary -/// produced by `deno compile`. It determines if this is a standalone -/// binary by skipping over the trailer width at the end of the file, -/// then checking for the magic trailer string `d3n0l4nd`. If found, -/// the bundle is executed. If not, this function exits with `Ok(None)`. -pub fn extract_standalone( - cli_args: Cow>, -) -> Result, AnyError> { - let Some(data) = libsui::find_section("d3n0l4nd") else { - return Ok(None); - }; - - let DeserializedDataSection { - mut metadata, - npm_snapshot, - remote_modules, - source_maps, - vfs_root_entries, - vfs_files_data, - } = match deserialize_binary_data_section(data)? { - Some(data_section) => data_section, - None => return Ok(None), - }; - - let root_path = { - let maybe_current_exe = std::env::current_exe().ok(); - let current_exe_name = maybe_current_exe - .as_ref() - .and_then(|p| p.file_name()) - .map(|p| p.to_string_lossy()) - // should never happen - .unwrap_or_else(|| Cow::Borrowed("binary")); - std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)) - }; - let cli_args = cli_args.into_owned(); - metadata.argv.reserve(cli_args.len() - 1); - for arg in cli_args.into_iter().skip(1) { - metadata.argv.push(arg.into_string().unwrap()); - } - let vfs = { - let fs_root = VfsRoot { - dir: VirtualDirectory { - // align the name of the directory with the root dir - name: root_path.file_name().unwrap().to_string_lossy().to_string(), - entries: vfs_root_entries, - }, - root_path: root_path.clone(), - start_file_offset: 0, - }; - Arc::new(FileBackedVfs::new( - Cow::Borrowed(vfs_files_data), - fs_root, - metadata.vfs_case_sensitivity, - )) - }; - Ok(Some(StandaloneData { - metadata, - modules: StandaloneModules { - remote_modules, - vfs: vfs.clone(), - }, - npm_snapshot, - root_path, - source_maps, - vfs, - })) -} - pub struct WriteBinOptions<'a> { pub writer: File, pub display_output_filename: &'a str, @@ -411,11 +186,11 @@ pub struct WriteBinOptions<'a> { } pub struct DenoCompileBinaryWriter<'a> { + cjs_code_analyzer: CliCjsCodeAnalyzer, cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, - file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, @@ -425,22 +200,22 @@ pub struct DenoCompileBinaryWriter<'a> { impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( + cjs_code_analyzer: CliCjsCodeAnalyzer, cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, - deno_dir: &'a DenoDir, + deno_dir: &'a DenoDir, emitter: &'a Emitter, - file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a CliNpmResolver, workspace_resolver: &'a WorkspaceResolver, npm_system_info: NpmSystemInfo, ) -> Self { Self { + cjs_code_analyzer, cjs_tracker, cli_options, deno_dir, emitter, - file_fetcher, http_client_provider, npm_resolver, workspace_resolver, @@ -476,7 +251,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self.write_standalone_binary(options, original_binary) + self.write_standalone_binary(options, original_binary).await } async fn get_base_binary( @@ -496,19 +271,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { let target = compile_flags.resolve_target(); let binary_name = format!("denort-{target}.zip"); - let binary_path_suffix = - match crate::version::DENO_VERSION_INFO.release_channel { - ReleaseChannel::Canary => { - format!( - "canary/{}/{}", - crate::version::DENO_VERSION_INFO.git_hash, - binary_name - ) - } - _ => { - format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) - } - }; + let binary_path_suffix = match DENO_VERSION_INFO.release_channel { + ReleaseChannel::Canary => { + format!("canary/{}/{}", DENO_VERSION_INFO.git_hash, binary_name) + } + _ => { + format!("release/v{}/{}", DENO_VERSION_INFO.deno, binary_name) + } + }; let download_directory = self.deno_dir.dl_folder_path(); let binary_path = download_directory.join(&binary_path_suffix); @@ -579,7 +349,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] - fn write_standalone_binary( + async fn write_standalone_binary( &self, options: WriteBinOptions<'_>, original_bin: Vec, @@ -623,23 +393,50 @@ impl<'a> DenoCompileBinaryWriter<'a> { .add_file_at_path(&path) .with_context(|| format!("Including {}", path.display()))?; } - let mut remote_modules_store = RemoteModulesStoreBuilder::default(); - let mut source_maps = Vec::with_capacity(graph.specifiers_count()); - // todo(dsherret): transpile in parallel + let specifiers_count = graph.specifiers_count(); + let mut specifier_store = SpecifierStore::with_capacity(specifiers_count); + let mut remote_modules_store = + SpecifierDataStore::with_capacity(specifiers_count); + // todo(dsherret): transpile and analyze CJS in parallel for module in graph.modules() { if module.specifier().scheme() == "data" { continue; // don't store data urls as an entry as they're in the code } - let (maybe_original_source, maybe_transpiled, media_type) = match module { + let mut maybe_source_map = None; + let mut maybe_transpiled = None; + let mut maybe_cjs_analysis = None; + let (maybe_original_source, media_type) = match module { deno_graph::Module::Js(m) => { - let original_bytes = m.source.as_bytes().to_vec(); - let maybe_transpiled = if m.media_type.is_emittable() { - let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( - &m.specifier, + let specifier = &m.specifier; + let original_bytes = m.source.as_bytes(); + if self.cjs_tracker.is_maybe_cjs(specifier, m.media_type)? { + if self.cjs_tracker.is_cjs_with_known_is_script( + specifier, m.media_type, m.is_script, - )?; - let module_kind = ModuleKind::from_is_cjs(is_cjs); + )? { + let cjs_analysis = self + .cjs_code_analyzer + .analyze_cjs( + module.specifier(), + Some(Cow::Borrowed(m.source.as_ref())), + ) + .await?; + maybe_cjs_analysis = Some(match cjs_analysis { + CjsAnalysis::Esm(_) => CjsExportAnalysisEntry::Esm, + CjsAnalysis::Cjs(exports) => { + CjsExportAnalysisEntry::Cjs(exports) + } + }); + } else { + maybe_cjs_analysis = Some(CjsExportAnalysisEntry::Esm); + } + } + if m.media_type.is_emittable() { + let module_kind = match maybe_cjs_analysis.as_ref() { + Some(CjsExportAnalysisEntry::Cjs(_)) => ModuleKind::Cjs, + _ => ModuleKind::Esm, + }; let (source, source_map) = self.emitter.emit_parsed_source_for_deno_compile( &m.specifier, @@ -648,60 +445,67 @@ impl<'a> DenoCompileBinaryWriter<'a> { &m.source, )?; if source != m.source.as_ref() { - source_maps.push((&m.specifier, source_map)); - Some(source.into_bytes()) - } else { - None + maybe_source_map = Some(source_map.into_bytes()); + maybe_transpiled = Some(source.into_bytes()); } - } else { - None - }; - (Some(original_bytes), maybe_transpiled, m.media_type) + } + (Some(original_bytes), m.media_type) } deno_graph::Module::Json(m) => { - (Some(m.source.as_bytes().to_vec()), None, m.media_type) + (Some(m.source.as_bytes()), m.media_type) } deno_graph::Module::Wasm(m) => { - (Some(m.source.to_vec()), None, MediaType::Wasm) + (Some(m.source.as_ref()), MediaType::Wasm) } deno_graph::Module::Npm(_) | deno_graph::Module::Node(_) - | deno_graph::Module::External(_) => (None, None, MediaType::Unknown), + | deno_graph::Module::External(_) => (None, MediaType::Unknown), }; if let Some(original_source) = maybe_original_source { + let maybe_cjs_export_analysis = maybe_cjs_analysis + .as_ref() + .map(bincode::serialize) + .transpose()?; if module.specifier().scheme() == "file" { let file_path = deno_path_util::url_to_file_path(module.specifier())?; vfs .add_file_with_data( &file_path, - original_source, - VfsFileSubDataKind::Raw, + deno_lib::standalone::virtual_fs::AddFileDataOptions { + data: original_source.to_vec(), + maybe_transpiled, + maybe_source_map, + maybe_cjs_export_analysis, + }, ) .with_context(|| { format!("Failed adding '{}'", file_path.display()) })?; - if let Some(transpiled_source) = maybe_transpiled { - vfs - .add_file_with_data( - &file_path, - transpiled_source, - VfsFileSubDataKind::ModuleGraph, - ) - .with_context(|| { - format!("Failed adding '{}'", file_path.display()) - })?; - } } else { + let specifier_id = specifier_store.get_or_add(module.specifier()); remote_modules_store.add( - module.specifier(), - media_type, - original_source, - maybe_transpiled, + specifier_id, + RemoteModuleEntry { + media_type, + data: Cow::Borrowed(original_source), + maybe_transpiled: maybe_transpiled.map(Cow::Owned), + maybe_source_map: maybe_source_map.map(Cow::Owned), + maybe_cjs_export_analysis: maybe_cjs_export_analysis + .map(Cow::Owned), + }, ); } } } - remote_modules_store.add_redirects(&graph.redirects); + + let mut redirects_store = + SpecifierDataStore::with_capacity(graph.redirects.len()); + for (from, to) in &graph.redirects { + redirects_store.add( + specifier_store.get_or_add(from), + specifier_store.get_or_add(to), + ); + } if let Some(import_map) = self.workspace_resolver.maybe_import_map() { if let Ok(file_path) = url_to_file_path(import_map.base_url()) { @@ -719,7 +523,48 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } + // do CJS export analysis on all the files in the VFS + // todo(dsherret): analyze cjs in parallel + let mut to_add = Vec::new(); + for (file_path, file) in vfs.iter_files() { + if file.cjs_export_analysis_offset.is_some() { + continue; // already analyzed + } + let specifier = deno_path_util::url_from_file_path(&file_path)?; + let media_type = MediaType::from_specifier(&specifier); + if self.cjs_tracker.is_maybe_cjs(&specifier, media_type)? { + let maybe_source = vfs + .file_bytes(file.offset) + .map(|text| String::from_utf8_lossy(text)); + let cjs_analysis_result = self + .cjs_code_analyzer + .analyze_cjs(&specifier, maybe_source) + .await; + let maybe_analysis = match cjs_analysis_result { + Ok(CjsAnalysis::Esm(_)) => Some(CjsExportAnalysisEntry::Esm), + Ok(CjsAnalysis::Cjs(exports)) => { + Some(CjsExportAnalysisEntry::Cjs(exports)) + } + Err(err) => { + log::debug!( + "Ignoring cjs export analysis for '{}': {}", + specifier, + err + ); + None + } + }; + if let Some(analysis) = &maybe_analysis { + to_add.push((file_path, bincode::serialize(analysis)?)); + } + } + } + for (file_path, analysis) in to_add { + vfs.add_cjs_export_analysis(&file_path, analysis); + } + let vfs = self.build_vfs_consolidating_global_npm_cache(vfs); + let root_dir_url = match &vfs.root_path { WindowsSystemRootablePath::Path(dir) => { Some(url_from_directory_path(dir)?) @@ -745,14 +590,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { None }; - let mut source_map_store = SourceMapStore::with_capacity(source_maps.len()); - for (specifier, source_map) in source_maps { - source_map_store.add( - Cow::Owned(root_dir_url.specifier_key(specifier).into_owned()), - Cow::Owned(source_map.into_bytes()), - ); - } - let node_modules = match &self.npm_resolver { CliNpmResolver::Managed(_) => { npm_snapshot.as_ref().map(|_| NodeModules::Managed { @@ -862,17 +699,18 @@ impl<'a> DenoCompileBinaryWriter<'a> { vfs_case_sensitivity: vfs.case_sensitivity, }; - write_binary_bytes( - writer, - original_bin, + let data_section_bytes = serialize_binary_data_section( &metadata, npm_snapshot.map(|s| s.into_serialized()), + &specifier_store.for_serialization(&root_dir_url), + &redirects_store, &remote_modules_store, - &source_map_store, &vfs, - compile_flags, ) - .context("Writing binary bytes") + .context("Serializing binary data section.")?; + + write_binary_bytes(writer, original_bin, data_section_bytes, compile_flags) + .context("Writing binary bytes") } fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> { @@ -1043,6 +881,146 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } +#[allow(clippy::too_many_arguments)] +fn write_binary_bytes( + mut file_writer: File, + original_bin: Vec, + data_section_bytes: Vec, + compile_flags: &CompileFlags, +) -> Result<(), AnyError> { + let target = compile_flags.resolve_target(); + if target.contains("linux") { + libsui::Elf::new(&original_bin).append( + "d3n0l4nd", + &data_section_bytes, + &mut file_writer, + )?; + } else if target.contains("windows") { + let mut pe = libsui::PortableExecutable::from(&original_bin)?; + if let Some(icon) = compile_flags.icon.as_ref() { + let icon = std::fs::read(icon)?; + pe = pe.set_icon(&icon)?; + } + + pe.write_resource("d3n0l4nd", data_section_bytes)? + .build(&mut file_writer)?; + } else if target.contains("darwin") { + libsui::Macho::from(original_bin)? + .write_section("d3n0l4nd", data_section_bytes)? + .build_and_sign(&mut file_writer)?; + } + Ok(()) +} + +/// Binary format: +/// * d3n0l4nd +/// * +/// * +/// * +/// * +/// * +/// * +/// * +/// * d3n0l4nd +#[allow(clippy::too_many_arguments)] +fn serialize_binary_data_section( + metadata: &Metadata, + npm_snapshot: Option, + specifiers: &SpecifierStoreForSerialization, + redirects: &SpecifierDataStore, + remote_modules: &SpecifierDataStore>, + vfs: &BuiltVfs, +) -> Result, AnyError> { + let metadata = serde_json::to_string(metadata)?; + let npm_snapshot = + npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); + let serialized_vfs = serde_json::to_string(&vfs.entries)?; + + let bytes = capacity_builder::BytesBuilder::build(|builder| { + builder.append(MAGIC_BYTES); + // 1. Metadata + { + builder.append_le(metadata.len() as u64); + builder.append(&metadata); + } + // 2. Npm snapshot + { + builder.append_le(npm_snapshot.len() as u64); + builder.append(&npm_snapshot); + } + // 3. Specifiers + builder.append(specifiers); + // 4. Redirects + redirects.serialize(builder); + // 5. Remote modules + remote_modules.serialize(builder); + // 6. VFS + { + builder.append_le(serialized_vfs.len() as u64); + builder.append(&serialized_vfs); + let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); + builder.append_le(vfs_bytes_len); + for file in &vfs.files { + builder.append(file); + } + } + + // write the magic bytes at the end so we can use it + // to make sure we've deserialized correctly + builder.append(MAGIC_BYTES); + })?; + + Ok(bytes) +} + +fn serialize_npm_snapshot( + mut snapshot: SerializedNpmResolutionSnapshot, +) -> Vec { + fn append_string(bytes: &mut Vec, string: &str) { + let len = string.len() as u32; + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(string.as_bytes()); + } + + snapshot.packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism + let ids_to_stored_ids = snapshot + .packages + .iter() + .enumerate() + .map(|(i, pkg)| (&pkg.id, i as u32)) + .collect::>(); + + let mut root_packages: Vec<_> = snapshot.root_packages.iter().collect(); + root_packages.sort(); + let mut bytes = Vec::new(); + + bytes.extend_from_slice(&(snapshot.packages.len() as u32).to_le_bytes()); + for pkg in &snapshot.packages { + append_string(&mut bytes, &pkg.id.as_serialized()); + } + + bytes.extend_from_slice(&(root_packages.len() as u32).to_le_bytes()); + for (req, id) in root_packages { + append_string(&mut bytes, &req.to_string()); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + + for pkg in &snapshot.packages { + let deps_len = pkg.dependencies.len() as u32; + bytes.extend_from_slice(&deps_len.to_le_bytes()); + let mut deps: Vec<_> = pkg.dependencies.iter().collect(); + deps.sort(); + for (req, id) in deps { + append_string(&mut bytes, req); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + } + + bytes +} + fn get_denort_path(deno_exe: PathBuf) -> Option { let mut denort = deno_exe; denort.set_file_name(if cfg!(windows) { diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs deleted file mode 100644 index c4b3ebe728..0000000000 --- a/cli/standalone/file_system.rs +++ /dev/null @@ -1,884 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -use std::borrow::Cow; -use std::io::ErrorKind; -use std::path::Path; -use std::path::PathBuf; -use std::rc::Rc; -use std::sync::Arc; -use std::time::Duration; -use std::time::SystemTime; - -use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; -use deno_runtime::deno_fs::AccessCheckCb; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_fs::FsDirEntry; -use deno_runtime::deno_fs::FsFileType; -use deno_runtime::deno_fs::OpenOptions; -use deno_runtime::deno_fs::RealFs; -use deno_runtime::deno_io::fs::File; -use deno_runtime::deno_io::fs::FsError; -use deno_runtime::deno_io::fs::FsResult; -use deno_runtime::deno_io::fs::FsStat; -use sys_traits::boxed::BoxedFsDirEntry; -use sys_traits::boxed::BoxedFsMetadataValue; -use sys_traits::boxed::FsMetadataBoxed; -use sys_traits::boxed::FsReadDirBoxed; -use sys_traits::FsCopy; -use sys_traits::FsMetadata; - -use super::virtual_fs::FileBackedVfs; -use super::virtual_fs::FileBackedVfsDirEntry; -use super::virtual_fs::FileBackedVfsFile; -use super::virtual_fs::FileBackedVfsMetadata; - -#[derive(Debug, Clone)] -pub struct DenoCompileFileSystem(Arc); - -impl DenoCompileFileSystem { - pub fn new(vfs: Arc) -> Self { - Self(vfs) - } - - fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> { - if self.0.is_path_within(path) { - Err(FsError::NotSupported) - } else { - Ok(()) - } - } - - fn copy_to_real_path( - &self, - oldpath: &Path, - newpath: &Path, - ) -> std::io::Result { - let old_file = self.0.file_entry(oldpath)?; - let old_file_bytes = - self.0.read_file_all(old_file, VfsFileSubDataKind::Raw)?; - let len = old_file_bytes.len() as u64; - RealFs - .write_file_sync( - newpath, - OpenOptions { - read: false, - write: true, - create: true, - truncate: true, - append: false, - create_new: false, - mode: None, - }, - None, - &old_file_bytes, - ) - .map_err(|err| err.into_io_error())?; - Ok(len) - } -} - -#[async_trait::async_trait(?Send)] -impl FileSystem for DenoCompileFileSystem { - fn cwd(&self) -> FsResult { - RealFs.cwd() - } - - fn tmp_dir(&self) -> FsResult { - RealFs.tmp_dir() - } - - fn chdir(&self, path: &Path) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.chdir(path) - } - - fn umask(&self, mask: Option) -> FsResult { - RealFs.umask(mask) - } - - fn open_sync( - &self, - path: &Path, - options: OpenOptions, - access_check: Option, - ) -> FsResult> { - if self.0.is_path_within(path) { - Ok(Rc::new(self.0.open_file(path)?)) - } else { - RealFs.open_sync(path, options, access_check) - } - } - async fn open_async<'a>( - &'a self, - path: PathBuf, - options: OpenOptions, - access_check: Option>, - ) -> FsResult> { - if self.0.is_path_within(&path) { - Ok(Rc::new(self.0.open_file(&path)?)) - } else { - RealFs.open_async(path, options, access_check).await - } - } - - fn mkdir_sync( - &self, - path: &Path, - recursive: bool, - mode: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.mkdir_sync(path, recursive, mode) - } - async fn mkdir_async( - &self, - path: PathBuf, - recursive: bool, - mode: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.mkdir_async(path, recursive, mode).await - } - - fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.chmod_sync(path, mode) - } - async fn chmod_async(&self, path: PathBuf, mode: u32) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.chmod_async(path, mode).await - } - - fn chown_sync( - &self, - path: &Path, - uid: Option, - gid: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.chown_sync(path, uid, gid) - } - async fn chown_async( - &self, - path: PathBuf, - uid: Option, - gid: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.chown_async(path, uid, gid).await - } - - fn lchown_sync( - &self, - path: &Path, - uid: Option, - gid: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.lchown_sync(path, uid, gid) - } - - async fn lchown_async( - &self, - path: PathBuf, - uid: Option, - gid: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.lchown_async(path, uid, gid).await - } - - fn remove_sync(&self, path: &Path, recursive: bool) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.remove_sync(path, recursive) - } - async fn remove_async(&self, path: PathBuf, recursive: bool) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.remove_async(path, recursive).await - } - - fn copy_file_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { - self.error_if_in_vfs(newpath)?; - if self.0.is_path_within(oldpath) { - self - .copy_to_real_path(oldpath, newpath) - .map(|_| ()) - .map_err(FsError::Io) - } else { - RealFs.copy_file_sync(oldpath, newpath) - } - } - async fn copy_file_async( - &self, - oldpath: PathBuf, - newpath: PathBuf, - ) -> FsResult<()> { - self.error_if_in_vfs(&newpath)?; - if self.0.is_path_within(&oldpath) { - let fs = self.clone(); - tokio::task::spawn_blocking(move || { - fs.copy_to_real_path(&oldpath, &newpath) - .map(|_| ()) - .map_err(FsError::Io) - }) - .await? - } else { - RealFs.copy_file_async(oldpath, newpath).await - } - } - - fn cp_sync(&self, from: &Path, to: &Path) -> FsResult<()> { - self.error_if_in_vfs(to)?; - - RealFs.cp_sync(from, to) - } - async fn cp_async(&self, from: PathBuf, to: PathBuf) -> FsResult<()> { - self.error_if_in_vfs(&to)?; - - RealFs.cp_async(from, to).await - } - - fn stat_sync(&self, path: &Path) -> FsResult { - if self.0.is_path_within(path) { - Ok(self.0.stat(path)?.as_fs_stat()) - } else { - RealFs.stat_sync(path) - } - } - async fn stat_async(&self, path: PathBuf) -> FsResult { - if self.0.is_path_within(&path) { - Ok(self.0.stat(&path)?.as_fs_stat()) - } else { - RealFs.stat_async(path).await - } - } - - fn lstat_sync(&self, path: &Path) -> FsResult { - if self.0.is_path_within(path) { - Ok(self.0.lstat(path)?.as_fs_stat()) - } else { - RealFs.lstat_sync(path) - } - } - async fn lstat_async(&self, path: PathBuf) -> FsResult { - if self.0.is_path_within(&path) { - Ok(self.0.lstat(&path)?.as_fs_stat()) - } else { - RealFs.lstat_async(path).await - } - } - - fn realpath_sync(&self, path: &Path) -> FsResult { - if self.0.is_path_within(path) { - Ok(self.0.canonicalize(path)?) - } else { - RealFs.realpath_sync(path) - } - } - async fn realpath_async(&self, path: PathBuf) -> FsResult { - if self.0.is_path_within(&path) { - Ok(self.0.canonicalize(&path)?) - } else { - RealFs.realpath_async(path).await - } - } - - fn read_dir_sync(&self, path: &Path) -> FsResult> { - if self.0.is_path_within(path) { - Ok(self.0.read_dir(path)?) - } else { - RealFs.read_dir_sync(path) - } - } - async fn read_dir_async(&self, path: PathBuf) -> FsResult> { - if self.0.is_path_within(&path) { - Ok(self.0.read_dir(&path)?) - } else { - RealFs.read_dir_async(path).await - } - } - - fn rename_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { - self.error_if_in_vfs(oldpath)?; - self.error_if_in_vfs(newpath)?; - RealFs.rename_sync(oldpath, newpath) - } - async fn rename_async( - &self, - oldpath: PathBuf, - newpath: PathBuf, - ) -> FsResult<()> { - self.error_if_in_vfs(&oldpath)?; - self.error_if_in_vfs(&newpath)?; - RealFs.rename_async(oldpath, newpath).await - } - - fn link_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { - self.error_if_in_vfs(oldpath)?; - self.error_if_in_vfs(newpath)?; - RealFs.link_sync(oldpath, newpath) - } - async fn link_async( - &self, - oldpath: PathBuf, - newpath: PathBuf, - ) -> FsResult<()> { - self.error_if_in_vfs(&oldpath)?; - self.error_if_in_vfs(&newpath)?; - RealFs.link_async(oldpath, newpath).await - } - - fn symlink_sync( - &self, - oldpath: &Path, - newpath: &Path, - file_type: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(oldpath)?; - self.error_if_in_vfs(newpath)?; - RealFs.symlink_sync(oldpath, newpath, file_type) - } - async fn symlink_async( - &self, - oldpath: PathBuf, - newpath: PathBuf, - file_type: Option, - ) -> FsResult<()> { - self.error_if_in_vfs(&oldpath)?; - self.error_if_in_vfs(&newpath)?; - RealFs.symlink_async(oldpath, newpath, file_type).await - } - - fn read_link_sync(&self, path: &Path) -> FsResult { - if self.0.is_path_within(path) { - Ok(self.0.read_link(path)?) - } else { - RealFs.read_link_sync(path) - } - } - async fn read_link_async(&self, path: PathBuf) -> FsResult { - if self.0.is_path_within(&path) { - Ok(self.0.read_link(&path)?) - } else { - RealFs.read_link_async(path).await - } - } - - fn truncate_sync(&self, path: &Path, len: u64) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.truncate_sync(path, len) - } - async fn truncate_async(&self, path: PathBuf, len: u64) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs.truncate_async(path, len).await - } - - fn utime_sync( - &self, - path: &Path, - atime_secs: i64, - atime_nanos: u32, - mtime_secs: i64, - mtime_nanos: u32, - ) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.utime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) - } - async fn utime_async( - &self, - path: PathBuf, - atime_secs: i64, - atime_nanos: u32, - mtime_secs: i64, - mtime_nanos: u32, - ) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs - .utime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) - .await - } - - fn lutime_sync( - &self, - path: &Path, - atime_secs: i64, - atime_nanos: u32, - mtime_secs: i64, - mtime_nanos: u32, - ) -> FsResult<()> { - self.error_if_in_vfs(path)?; - RealFs.lutime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) - } - async fn lutime_async( - &self, - path: PathBuf, - atime_secs: i64, - atime_nanos: u32, - mtime_secs: i64, - mtime_nanos: u32, - ) -> FsResult<()> { - self.error_if_in_vfs(&path)?; - RealFs - .lutime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) - .await - } -} - -impl sys_traits::BaseFsHardLink for DenoCompileFileSystem { - #[inline] - fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> { - self.link_sync(src, dst).map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::BaseFsRead for DenoCompileFileSystem { - #[inline] - fn base_fs_read(&self, path: &Path) -> std::io::Result> { - self - .read_file_sync(path, None) - .map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::FsMetadataValue for FileBackedVfsMetadata { - fn file_type(&self) -> sys_traits::FileType { - self.file_type - } - - fn len(&self) -> u64 { - self.len - } - - fn accessed(&self) -> std::io::Result { - Err(not_supported("accessed time")) - } - - fn created(&self) -> std::io::Result { - Err(not_supported("created time")) - } - - fn changed(&self) -> std::io::Result { - Err(not_supported("changed time")) - } - - fn modified(&self) -> std::io::Result { - Err(not_supported("modified time")) - } - - fn dev(&self) -> std::io::Result { - Ok(0) - } - - fn ino(&self) -> std::io::Result { - Ok(0) - } - - fn mode(&self) -> std::io::Result { - Ok(0) - } - - fn nlink(&self) -> std::io::Result { - Ok(0) - } - - fn uid(&self) -> std::io::Result { - Ok(0) - } - - fn gid(&self) -> std::io::Result { - Ok(0) - } - - fn rdev(&self) -> std::io::Result { - Ok(0) - } - - fn blksize(&self) -> std::io::Result { - Ok(0) - } - - fn blocks(&self) -> std::io::Result { - Ok(0) - } - - fn is_block_device(&self) -> std::io::Result { - Ok(false) - } - - fn is_char_device(&self) -> std::io::Result { - Ok(false) - } - - fn is_fifo(&self) -> std::io::Result { - Ok(false) - } - - fn is_socket(&self) -> std::io::Result { - Ok(false) - } - - fn file_attributes(&self) -> std::io::Result { - Ok(0) - } -} - -fn not_supported(name: &str) -> std::io::Error { - std::io::Error::new( - ErrorKind::Unsupported, - format!( - "{} is not supported for an embedded deno compile file", - name - ), - ) -} - -impl sys_traits::FsDirEntry for FileBackedVfsDirEntry { - type Metadata = BoxedFsMetadataValue; - - fn file_name(&self) -> Cow { - Cow::Borrowed(self.metadata.name.as_ref()) - } - - fn file_type(&self) -> std::io::Result { - Ok(self.metadata.file_type) - } - - fn metadata(&self) -> std::io::Result { - Ok(BoxedFsMetadataValue(Box::new(self.metadata.clone()))) - } - - fn path(&self) -> Cow { - Cow::Owned(self.parent_path.join(&self.metadata.name)) - } -} - -impl sys_traits::BaseFsReadDir for DenoCompileFileSystem { - type ReadDirEntry = BoxedFsDirEntry; - - fn base_fs_read_dir( - &self, - path: &Path, - ) -> std::io::Result< - Box> + '_>, - > { - if self.0.is_path_within(path) { - let entries = self.0.read_dir_with_metadata(path)?; - Ok(Box::new( - entries.map(|entry| Ok(BoxedFsDirEntry::new(entry))), - )) - } else { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.fs_read_dir_boxed(path) - } - } -} - -impl sys_traits::BaseFsCanonicalize for DenoCompileFileSystem { - #[inline] - fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result { - self.realpath_sync(path).map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::BaseFsMetadata for DenoCompileFileSystem { - type Metadata = BoxedFsMetadataValue; - - #[inline] - fn base_fs_metadata(&self, path: &Path) -> std::io::Result { - if self.0.is_path_within(path) { - Ok(BoxedFsMetadataValue::new(self.0.stat(path)?)) - } else { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.fs_metadata_boxed(path) - } - } - - #[inline] - fn base_fs_symlink_metadata( - &self, - path: &Path, - ) -> std::io::Result { - if self.0.is_path_within(path) { - Ok(BoxedFsMetadataValue::new(self.0.lstat(path)?)) - } else { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.fs_symlink_metadata_boxed(path) - } - } -} - -impl sys_traits::BaseFsCopy for DenoCompileFileSystem { - #[inline] - fn base_fs_copy(&self, from: &Path, to: &Path) -> std::io::Result { - self - .error_if_in_vfs(to) - .map_err(|err| err.into_io_error())?; - if self.0.is_path_within(from) { - self.copy_to_real_path(from, to) - } else { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.fs_copy(from, to) - } - } -} - -impl sys_traits::BaseFsCloneFile for DenoCompileFileSystem { - fn base_fs_clone_file( - &self, - _from: &Path, - _to: &Path, - ) -> std::io::Result<()> { - // will cause a fallback in the code that uses this - Err(not_supported("cloning files")) - } -} - -impl sys_traits::BaseFsCreateDir for DenoCompileFileSystem { - #[inline] - fn base_fs_create_dir( - &self, - path: &Path, - options: &sys_traits::CreateDirOptions, - ) -> std::io::Result<()> { - self - .mkdir_sync(path, options.recursive, options.mode) - .map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::BaseFsRemoveFile for DenoCompileFileSystem { - #[inline] - fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> { - self - .remove_sync(path, false) - .map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::BaseFsRename for DenoCompileFileSystem { - #[inline] - fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { - self - .rename_sync(from, to) - .map_err(|err| err.into_io_error()) - } -} - -pub enum FsFileAdapter { - Real(sys_traits::impls::RealFsFile), - Vfs(FileBackedVfsFile), -} - -impl sys_traits::FsFile for FsFileAdapter {} - -impl sys_traits::FsFileAsRaw for FsFileAdapter { - #[cfg(windows)] - fn fs_file_as_raw_handle(&self) -> Option { - match self { - Self::Real(file) => file.fs_file_as_raw_handle(), - Self::Vfs(_) => None, - } - } - - #[cfg(unix)] - fn fs_file_as_raw_fd(&self) -> Option { - match self { - Self::Real(file) => file.fs_file_as_raw_fd(), - Self::Vfs(_) => None, - } - } -} - -impl sys_traits::FsFileSyncData for FsFileAdapter { - fn fs_file_sync_data(&mut self) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_sync_data(), - Self::Vfs(_) => Ok(()), - } - } -} - -impl sys_traits::FsFileSyncAll for FsFileAdapter { - fn fs_file_sync_all(&mut self) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_sync_all(), - Self::Vfs(_) => Ok(()), - } - } -} - -impl sys_traits::FsFileSetPermissions for FsFileAdapter { - #[inline] - fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_set_permissions(mode), - Self::Vfs(_) => Ok(()), - } - } -} - -impl std::io::Read for FsFileAdapter { - #[inline] - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - Self::Real(file) => file.read(buf), - Self::Vfs(file) => file.read_to_buf(buf), - } - } -} - -impl std::io::Seek for FsFileAdapter { - fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { - match self { - Self::Real(file) => file.seek(pos), - Self::Vfs(file) => file.seek(pos), - } - } -} - -impl std::io::Write for FsFileAdapter { - #[inline] - fn write(&mut self, buf: &[u8]) -> std::io::Result { - match self { - Self::Real(file) => file.write(buf), - Self::Vfs(_) => Err(not_supported("writing files")), - } - } - - #[inline] - fn flush(&mut self) -> std::io::Result<()> { - match self { - Self::Real(file) => file.flush(), - Self::Vfs(_) => Err(not_supported("writing files")), - } - } -} - -impl sys_traits::FsFileSetLen for FsFileAdapter { - #[inline] - fn fs_file_set_len(&mut self, len: u64) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_set_len(len), - Self::Vfs(_) => Err(not_supported("setting file length")), - } - } -} - -impl sys_traits::FsFileSetTimes for FsFileAdapter { - fn fs_file_set_times( - &mut self, - times: sys_traits::FsFileTimes, - ) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_set_times(times), - Self::Vfs(_) => Err(not_supported("setting file times")), - } - } -} - -impl sys_traits::FsFileLock for FsFileAdapter { - fn fs_file_lock( - &mut self, - mode: sys_traits::FsFileLockMode, - ) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_lock(mode), - Self::Vfs(_) => Err(not_supported("locking files")), - } - } - - fn fs_file_try_lock( - &mut self, - mode: sys_traits::FsFileLockMode, - ) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_try_lock(mode), - Self::Vfs(_) => Err(not_supported("locking files")), - } - } - - fn fs_file_unlock(&mut self) -> std::io::Result<()> { - match self { - Self::Real(file) => file.fs_file_unlock(), - Self::Vfs(_) => Err(not_supported("unlocking files")), - } - } -} - -impl sys_traits::FsFileIsTerminal for FsFileAdapter { - #[inline] - fn fs_file_is_terminal(&self) -> bool { - match self { - Self::Real(file) => file.fs_file_is_terminal(), - Self::Vfs(_) => false, - } - } -} - -impl sys_traits::BaseFsOpen for DenoCompileFileSystem { - type File = FsFileAdapter; - - fn base_fs_open( - &self, - path: &Path, - options: &sys_traits::OpenOptions, - ) -> std::io::Result { - if self.0.is_path_within(path) { - Ok(FsFileAdapter::Vfs(self.0.open_file(path)?)) - } else { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - Ok(FsFileAdapter::Real( - sys_traits::impls::RealSys.base_fs_open(path, options)?, - )) - } - } -} - -impl sys_traits::BaseFsSymlinkDir for DenoCompileFileSystem { - fn base_fs_symlink_dir(&self, src: &Path, dst: &Path) -> std::io::Result<()> { - self - .symlink_sync(src, dst, Some(FsFileType::Directory)) - .map_err(|err| err.into_io_error()) - } -} - -impl sys_traits::SystemRandom for DenoCompileFileSystem { - #[inline] - fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.sys_random(buf) - } -} - -impl sys_traits::SystemTimeNow for DenoCompileFileSystem { - #[inline] - fn sys_time_now(&self) -> SystemTime { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.sys_time_now() - } -} - -impl sys_traits::ThreadSleep for DenoCompileFileSystem { - #[inline] - fn thread_sleep(&self, dur: Duration) { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.thread_sleep(dur) - } -} - -impl sys_traits::EnvCurrentDir for DenoCompileFileSystem { - fn env_current_dir(&self) -> std::io::Result { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.env_current_dir() - } -} - -impl sys_traits::BaseEnvVar for DenoCompileFileSystem { - fn base_env_var_os( - &self, - key: &std::ffi::OsStr, - ) -> Option { - #[allow(clippy::disallowed_types)] // ok because we're implementing the fs - sys_traits::impls::RealSys.base_env_var_os(key) - } -} diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 1ab77a5a4c..81ca2b4ff1 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -1,1044 +1,4 @@ // Copyright 2018-2025 the Deno authors. MIT license. -// Allow unused code warnings because we share -// code between the two bin targets. -#![allow(dead_code)] -#![allow(unused_imports)] - -use std::borrow::Cow; -use std::path::PathBuf; -use std::rc::Rc; -use std::sync::Arc; - -use binary::StandaloneData; -use binary::StandaloneModules; -use code_cache::DenoCompileCodeCache; -use deno_ast::MediaType; -use deno_cache_dir::file_fetcher::CacheSetting; -use deno_cache_dir::npm::NpmCacheDir; -use deno_config::workspace::MappedResolution; -use deno_config::workspace::MappedResolutionError; -use deno_config::workspace::ResolverWorkspaceJsrPackage; -use deno_config::workspace::WorkspaceResolver; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::error::ModuleLoaderError; -use deno_core::futures::future::LocalBoxFuture; -use deno_core::futures::FutureExt; -use deno_core::v8_set_flags; -use deno_core::FastString; -use deno_core::FeatureChecker; -use deno_core::ModuleLoader; -use deno_core::ModuleSourceCode; -use deno_core::ModuleSpecifier; -use deno_core::ModuleType; -use deno_core::RequestedModuleType; -use deno_core::ResolutionKind; -use deno_core::SourceCodeCacheInfo; -use deno_error::JsErrorBox; -use deno_lib::cache::DenoDirProvider; -use deno_lib::npm::NpmRegistryReadPermissionChecker; -use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; -use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; -use deno_lib::worker::CreateModuleLoaderResult; -use deno_lib::worker::LibMainWorkerFactory; -use deno_lib::worker::LibMainWorkerOptions; -use deno_lib::worker::ModuleLoaderFactory; -use deno_lib::worker::StorageKeyResolver; -use deno_npm::npm_rc::ResolvedNpmRc; -use deno_npm::resolution::NpmResolutionSnapshot; -use deno_package_json::PackageJsonDepValue; -use deno_resolver::cjs::IsCjsResolutionMode; -use deno_resolver::npm::managed::ManagedInNpmPkgCheckerCreateOptions; -use deno_resolver::npm::managed::NpmResolutionCell; -use deno_resolver::npm::ByonmNpmResolverCreateOptions; -use deno_resolver::npm::CreateInNpmPkgCheckerOptions; -use deno_resolver::npm::DenoInNpmPackageChecker; -use deno_resolver::npm::NpmReqResolverOptions; -use deno_runtime::deno_fs; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::create_host_defined_options; -use deno_runtime::deno_node::NodeRequireLoader; -use deno_runtime::deno_node::NodeResolver; -use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; -use deno_runtime::deno_permissions::Permissions; -use deno_runtime::deno_permissions::PermissionsContainer; -use deno_runtime::deno_tls::rustls::RootCertStore; -use deno_runtime::deno_tls::RootCertStoreProvider; -use deno_runtime::deno_web::BlobStore; -use deno_runtime::permissions::RuntimePermissionDescriptorParser; -use deno_runtime::WorkerExecutionMode; -use deno_runtime::WorkerLogLevel; -use deno_semver::npm::NpmPackageReqReference; -use import_map::parse_from_json; -use node_resolver::analyze::NodeCodeTranslator; -use node_resolver::errors::ClosestPkgJsonError; -use node_resolver::NodeResolutionKind; -use node_resolver::ResolutionMode; -use serialization::DenoCompileModuleSource; -use serialization::SourceMapStore; -use virtual_fs::FileBackedVfs; - -use crate::args::create_default_npmrc; -use crate::args::get_root_cert_store; -use crate::args::npm_pkg_req_ref_to_binary_command; -use crate::args::CaData; -use crate::args::NpmInstallDepsProvider; -use crate::cache::Caches; -use crate::cache::FastInsecureHasher; -use crate::cache::NodeAnalysisCache; -use crate::http_util::HttpClientProvider; -use crate::node::CliCjsCodeAnalyzer; -use crate::node::CliNodeCodeTranslator; -use crate::node::CliNodeResolver; -use crate::node::CliPackageJsonResolver; -use crate::npm::create_npm_process_state_provider; -use crate::npm::CliByonmNpmResolverCreateOptions; -use crate::npm::CliManagedNpmResolverCreateOptions; -use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::NpmResolutionInitializer; -use crate::resolver::CliCjsTracker; -use crate::resolver::CliNpmReqResolver; -use crate::resolver::NpmModuleLoader; -use crate::sys::CliSys; -use crate::util::progress_bar::ProgressBar; -use crate::util::progress_bar::ProgressBarStyle; -use crate::util::text_encoding::from_utf8_lossy_cow; -use crate::util::v8::construct_v8_flags; -use crate::worker::CliCodeCache; -use crate::worker::CliMainWorkerFactory; -use crate::worker::CliMainWorkerOptions; - pub mod binary; -mod code_cache; -mod file_system; -mod serialization; mod virtual_fs; - -pub use binary::extract_standalone; -pub use binary::is_standalone_binary; -pub use binary::DenoCompileBinaryWriter; - -use self::binary::Metadata; -pub use self::file_system::DenoCompileFileSystem; - -struct SharedModuleLoaderState { - cjs_tracker: Arc, - code_cache: Option>, - fs: Arc, - modules: StandaloneModules, - node_code_translator: Arc, - node_resolver: Arc, - npm_module_loader: Arc, - npm_registry_permission_checker: NpmRegistryReadPermissionChecker, - npm_req_resolver: Arc, - npm_resolver: CliNpmResolver, - source_maps: SourceMapStore, - vfs: Arc, - workspace_resolver: WorkspaceResolver, -} - -impl SharedModuleLoaderState { - fn get_code_cache( - &self, - specifier: &ModuleSpecifier, - source: &[u8], - ) -> Option { - let Some(code_cache) = &self.code_cache else { - return None; - }; - if !code_cache.enabled() { - return None; - } - // deno version is already included in the root cache key - let hash = FastInsecureHasher::new_without_deno_version() - .write_hashable(source) - .finish(); - let data = code_cache.get_sync( - specifier, - deno_runtime::code_cache::CodeCacheType::EsModule, - hash, - ); - Some(SourceCodeCacheInfo { - hash, - data: data.map(Cow::Owned), - }) - } -} - -#[derive(Clone)] -struct EmbeddedModuleLoader { - shared: Arc, -} - -impl std::fmt::Debug for EmbeddedModuleLoader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("EmbeddedModuleLoader").finish() - } -} - -pub const MODULE_NOT_FOUND: &str = "Module not found"; -pub const UNSUPPORTED_SCHEME: &str = "Unsupported scheme"; - -impl ModuleLoader for EmbeddedModuleLoader { - fn resolve( - &self, - raw_specifier: &str, - referrer: &str, - kind: ResolutionKind, - ) -> Result { - let referrer = if referrer == "." { - if kind != ResolutionKind::MainModule { - return Err( - JsErrorBox::generic(format!( - "Expected to resolve main module, got {:?} instead.", - kind - )) - .into(), - ); - } - let current_dir = std::env::current_dir().unwrap(); - deno_core::resolve_path(".", ¤t_dir)? - } else { - ModuleSpecifier::parse(referrer).map_err(|err| { - JsErrorBox::type_error(format!( - "Referrer uses invalid specifier: {}", - err - )) - })? - }; - let referrer_kind = if self - .shared - .cjs_tracker - .is_maybe_cjs(&referrer, MediaType::from_specifier(&referrer)) - .map_err(JsErrorBox::from_err)? - { - ResolutionMode::Require - } else { - ResolutionMode::Import - }; - - if self.shared.node_resolver.in_npm_package(&referrer) { - return Ok( - self - .shared - .node_resolver - .resolve( - raw_specifier, - &referrer, - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(JsErrorBox::from_err)? - .into_url(), - ); - } - - let mapped_resolution = self - .shared - .workspace_resolver - .resolve(raw_specifier, &referrer); - - match mapped_resolution { - Ok(MappedResolution::WorkspaceJsrPackage { specifier, .. }) => { - Ok(specifier) - } - Ok(MappedResolution::WorkspaceNpmPackage { - target_pkg_json: pkg_json, - sub_path, - .. - }) => Ok( - self - .shared - .node_resolver - .resolve_package_subpath_from_deno_module( - pkg_json.dir_path(), - sub_path.as_deref(), - Some(&referrer), - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(JsErrorBox::from_err)?, - ), - Ok(MappedResolution::PackageJson { - dep_result, - sub_path, - alias, - .. - }) => match dep_result - .as_ref() - .map_err(|e| JsErrorBox::from_err(e.clone()))? - { - PackageJsonDepValue::Req(req) => self - .shared - .npm_req_resolver - .resolve_req_with_sub_path( - req, - sub_path.as_deref(), - &referrer, - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(|e| JsErrorBox::from_err(e).into()), - PackageJsonDepValue::Workspace(version_req) => { - let pkg_folder = self - .shared - .workspace_resolver - .resolve_workspace_pkg_json_folder_for_pkg_json_dep( - alias, - version_req, - ) - .map_err(JsErrorBox::from_err)?; - Ok( - self - .shared - .node_resolver - .resolve_package_subpath_from_deno_module( - pkg_folder, - sub_path.as_deref(), - Some(&referrer), - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(JsErrorBox::from_err)?, - ) - } - }, - Ok(MappedResolution::Normal { specifier, .. }) - | Ok(MappedResolution::ImportMap { specifier, .. }) => { - if let Ok(reference) = - NpmPackageReqReference::from_specifier(&specifier) - { - return Ok( - self - .shared - .npm_req_resolver - .resolve_req_reference( - &reference, - &referrer, - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(JsErrorBox::from_err)?, - ); - } - - if specifier.scheme() == "jsr" { - if let Some(specifier) = - self.shared.modules.resolve_specifier(&specifier)? - { - return Ok(specifier.clone()); - } - } - - Ok( - self - .shared - .node_resolver - .handle_if_in_node_modules(&specifier) - .unwrap_or(specifier), - ) - } - Err(err) - if err.is_unmapped_bare_specifier() && referrer.scheme() == "file" => - { - let maybe_res = self - .shared - .npm_req_resolver - .resolve_if_for_npm_pkg( - raw_specifier, - &referrer, - referrer_kind, - NodeResolutionKind::Execution, - ) - .map_err(JsErrorBox::from_err)?; - if let Some(res) = maybe_res { - return Ok(res.into_url()); - } - Err(JsErrorBox::from_err(err).into()) - } - Err(err) => Err(JsErrorBox::from_err(err).into()), - } - } - - fn get_host_defined_options<'s>( - &self, - scope: &mut deno_core::v8::HandleScope<'s>, - name: &str, - ) -> Option> { - let name = deno_core::ModuleSpecifier::parse(name).ok()?; - if self.shared.node_resolver.in_npm_package(&name) { - Some(create_host_defined_options(scope)) - } else { - None - } - } - - fn load( - &self, - original_specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - _is_dynamic: bool, - _requested_module_type: RequestedModuleType, - ) -> deno_core::ModuleLoadResponse { - if original_specifier.scheme() == "data" { - let data_url_text = - match deno_graph::source::RawDataUrl::parse(original_specifier) - .and_then(|url| url.decode()) - { - Ok(response) => response, - Err(err) => { - return deno_core::ModuleLoadResponse::Sync(Err( - JsErrorBox::type_error(format!("{:#}", err)).into(), - )); - } - }; - return deno_core::ModuleLoadResponse::Sync(Ok( - deno_core::ModuleSource::new( - deno_core::ModuleType::JavaScript, - ModuleSourceCode::String(data_url_text.into()), - original_specifier, - None, - ), - )); - } - - if self.shared.node_resolver.in_npm_package(original_specifier) { - let shared = self.shared.clone(); - let original_specifier = original_specifier.clone(); - let maybe_referrer = maybe_referrer.cloned(); - return deno_core::ModuleLoadResponse::Async( - async move { - let code_source = shared - .npm_module_loader - .load(&original_specifier, maybe_referrer.as_ref()) - .await - .map_err(JsErrorBox::from_err)?; - let code_cache_entry = shared.get_code_cache( - &code_source.found_url, - code_source.code.as_bytes(), - ); - Ok(deno_core::ModuleSource::new_with_redirect( - match code_source.media_type { - MediaType::Json => ModuleType::Json, - _ => ModuleType::JavaScript, - }, - code_source.code, - &original_specifier, - &code_source.found_url, - code_cache_entry, - )) - } - .boxed_local(), - ); - } - - match self - .shared - .modules - .read(original_specifier, VfsFileSubDataKind::ModuleGraph) - { - Ok(Some(module)) => { - let media_type = module.media_type; - let (module_specifier, module_type, module_source) = - module.into_parts(); - let is_maybe_cjs = match self - .shared - .cjs_tracker - .is_maybe_cjs(original_specifier, media_type) - { - Ok(is_maybe_cjs) => is_maybe_cjs, - Err(err) => { - return deno_core::ModuleLoadResponse::Sync(Err( - JsErrorBox::type_error(format!("{:?}", err)).into(), - )); - } - }; - if is_maybe_cjs { - let original_specifier = original_specifier.clone(); - let module_specifier = module_specifier.clone(); - let shared = self.shared.clone(); - deno_core::ModuleLoadResponse::Async( - async move { - let source = match module_source { - DenoCompileModuleSource::String(string) => { - Cow::Borrowed(string) - } - DenoCompileModuleSource::Bytes(module_code_bytes) => { - match module_code_bytes { - Cow::Owned(bytes) => Cow::Owned( - crate::util::text_encoding::from_utf8_lossy_owned(bytes), - ), - Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), - } - } - }; - let source = shared - .node_code_translator - .translate_cjs_to_esm(&module_specifier, Some(source)) - .await - .map_err(JsErrorBox::from_err)?; - let module_source = match source { - Cow::Owned(source) => ModuleSourceCode::String(source.into()), - Cow::Borrowed(source) => { - ModuleSourceCode::String(FastString::from_static(source)) - } - }; - let code_cache_entry = shared - .get_code_cache(&module_specifier, module_source.as_bytes()); - Ok(deno_core::ModuleSource::new_with_redirect( - module_type, - module_source, - &original_specifier, - &module_specifier, - code_cache_entry, - )) - } - .boxed_local(), - ) - } else { - let module_source = module_source.into_for_v8(); - let code_cache_entry = self - .shared - .get_code_cache(module_specifier, module_source.as_bytes()); - deno_core::ModuleLoadResponse::Sync(Ok( - deno_core::ModuleSource::new_with_redirect( - module_type, - module_source, - original_specifier, - module_specifier, - code_cache_entry, - ), - )) - } - } - Ok(None) => deno_core::ModuleLoadResponse::Sync(Err( - JsErrorBox::type_error(format!( - "{MODULE_NOT_FOUND}: {}", - original_specifier - )) - .into(), - )), - Err(err) => deno_core::ModuleLoadResponse::Sync(Err( - JsErrorBox::type_error(format!("{:?}", err)).into(), - )), - } - } - - fn code_cache_ready( - &self, - specifier: ModuleSpecifier, - source_hash: u64, - code_cache_data: &[u8], - ) -> LocalBoxFuture<'static, ()> { - if let Some(code_cache) = &self.shared.code_cache { - code_cache.set_sync( - specifier, - deno_runtime::code_cache::CodeCacheType::EsModule, - source_hash, - code_cache_data, - ); - } - std::future::ready(()).boxed_local() - } - - fn get_source_map(&self, file_name: &str) -> Option> { - if file_name.starts_with("file:///") { - let url = - deno_path_util::url_from_directory_path(self.shared.vfs.root()).ok()?; - let file_url = ModuleSpecifier::parse(file_name).ok()?; - let relative_path = url.make_relative(&file_url)?; - self.shared.source_maps.get(&relative_path) - } else { - self.shared.source_maps.get(file_name) - } - .map(Cow::Borrowed) - } - - fn get_source_mapped_source_line( - &self, - file_name: &str, - line_number: usize, - ) -> Option { - let specifier = ModuleSpecifier::parse(file_name).ok()?; - let data = self - .shared - .modules - .read(&specifier, VfsFileSubDataKind::Raw) - .ok()??; - - let source = String::from_utf8_lossy(&data.data); - // Do NOT use .lines(): it skips the terminating empty line. - // (due to internally using_terminator() instead of .split()) - let lines: Vec<&str> = source.split('\n').collect(); - if line_number >= lines.len() { - Some(format!( - "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)", - crate::colors::yellow("Warning"), line_number + 1, - )) - } else { - Some(lines[line_number].to_string()) - } - } -} - -impl NodeRequireLoader for EmbeddedModuleLoader { - fn ensure_read_permission<'a>( - &self, - permissions: &mut dyn deno_runtime::deno_node::NodePermissions, - path: &'a std::path::Path, - ) -> Result, JsErrorBox> { - if self.shared.modules.has_file(path) { - // allow reading if the file is in the snapshot - return Ok(Cow::Borrowed(path)); - } - - self - .shared - .npm_registry_permission_checker - .ensure_read_permission(permissions, path) - .map_err(JsErrorBox::from_err) - } - - fn load_text_file_lossy( - &self, - path: &std::path::Path, - ) -> Result, JsErrorBox> { - let file_entry = self - .shared - .vfs - .file_entry(path) - .map_err(JsErrorBox::from_err)?; - let file_bytes = self - .shared - .vfs - .read_file_all(file_entry, VfsFileSubDataKind::ModuleGraph) - .map_err(JsErrorBox::from_err)?; - Ok(from_utf8_lossy_cow(file_bytes)) - } - - fn is_maybe_cjs( - &self, - specifier: &ModuleSpecifier, - ) -> Result { - let media_type = MediaType::from_specifier(specifier); - self.shared.cjs_tracker.is_maybe_cjs(specifier, media_type) - } -} - -struct StandaloneModuleLoaderFactory { - shared: Arc, -} - -impl StandaloneModuleLoaderFactory { - pub fn create_result(&self) -> CreateModuleLoaderResult { - let loader = Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - }); - CreateModuleLoaderResult { - module_loader: loader.clone(), - node_require_loader: loader, - } - } -} - -impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { - fn create_for_main( - &self, - _root_permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult { - self.create_result() - } - - fn create_for_worker( - &self, - _parent_permissions: PermissionsContainer, - _permissions: PermissionsContainer, - ) -> CreateModuleLoaderResult { - self.create_result() - } -} - -struct StandaloneRootCertStoreProvider { - ca_stores: Option>, - ca_data: Option, - cell: once_cell::sync::OnceCell, -} - -impl RootCertStoreProvider for StandaloneRootCertStoreProvider { - fn get_or_try_init(&self) -> Result<&RootCertStore, JsErrorBox> { - self.cell.get_or_try_init(|| { - get_root_cert_store(None, self.ca_stores.clone(), self.ca_data.clone()) - .map_err(JsErrorBox::from_err) - }) - } -} - -pub async fn run( - fs: Arc, - sys: CliSys, - data: StandaloneData, -) -> Result { - let StandaloneData { - metadata, - modules, - npm_snapshot, - root_path, - source_maps, - vfs, - } = data; - let deno_dir_provider = Arc::new(DenoDirProvider::new(sys.clone(), None)); - let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { - ca_stores: metadata.ca_stores, - ca_data: metadata.ca_data.map(CaData::Bytes), - cell: Default::default(), - }); - // use a dummy npm registry url - let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); - let root_dir_url = - Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap()); - let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); - let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); - let pkg_json_resolver = Arc::new(CliPackageJsonResolver::new(sys.clone())); - let npm_registry_permission_checker = { - let mode = match &metadata.node_modules { - Some(binary::NodeModules::Managed { - node_modules_dir: Some(path), - }) => NpmRegistryReadPermissionCheckerMode::Local(PathBuf::from(path)), - Some(binary::NodeModules::Byonm { .. }) => { - NpmRegistryReadPermissionCheckerMode::Byonm - } - Some(binary::NodeModules::Managed { - node_modules_dir: None, - }) - | None => NpmRegistryReadPermissionCheckerMode::Global( - npm_global_cache_dir.clone(), - ), - }; - NpmRegistryReadPermissionChecker::new(sys.clone(), mode) - }; - let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules { - Some(binary::NodeModules::Managed { node_modules_dir }) => { - // create an npmrc that uses the fake npm_registry_url to resolve packages - let npmrc = Arc::new(ResolvedNpmRc { - default_config: deno_npm::npm_rc::RegistryConfigWithUrl { - registry_url: npm_registry_url.clone(), - config: Default::default(), - }, - scopes: Default::default(), - registry_configs: Default::default(), - }); - let npm_cache_dir = Arc::new(NpmCacheDir::new( - &sys, - npm_global_cache_dir, - npmrc.get_all_known_registries_urls(), - )); - let snapshot = npm_snapshot.unwrap(); - let maybe_node_modules_path = node_modules_dir - .map(|node_modules_dir| root_path.join(node_modules_dir)); - let in_npm_pkg_checker = - DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( - ManagedInNpmPkgCheckerCreateOptions { - root_cache_dir_url: npm_cache_dir.root_dir_url(), - maybe_node_modules_path: maybe_node_modules_path.as_deref(), - }, - )); - let npm_resolution = - Arc::new(NpmResolutionCell::new(NpmResolutionSnapshot::new(snapshot))); - let npm_resolver = - CliNpmResolver::new(CliNpmResolverCreateOptions::Managed( - CliManagedNpmResolverCreateOptions { - npm_resolution, - npm_cache_dir, - sys: sys.clone(), - maybe_node_modules_path, - npm_system_info: Default::default(), - npmrc, - }, - )); - (in_npm_pkg_checker, npm_resolver) - } - Some(binary::NodeModules::Byonm { - root_node_modules_dir, - }) => { - let root_node_modules_dir = - root_node_modules_dir.map(|p| vfs.root().join(p)); - let in_npm_pkg_checker = - DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Byonm); - let npm_resolver = CliNpmResolver::new( - CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { - sys: sys.clone(), - pkg_json_resolver: pkg_json_resolver.clone(), - root_node_modules_dir, - }), - ); - (in_npm_pkg_checker, npm_resolver) - } - None => { - // Packages from different registries are already inlined in the binary, - // so no need to create actual `.npmrc` configuration. - let npmrc = create_default_npmrc(); - let npm_cache_dir = Arc::new(NpmCacheDir::new( - &sys, - npm_global_cache_dir, - npmrc.get_all_known_registries_urls(), - )); - let in_npm_pkg_checker = - DenoInNpmPackageChecker::new(CreateInNpmPkgCheckerOptions::Managed( - ManagedInNpmPkgCheckerCreateOptions { - root_cache_dir_url: npm_cache_dir.root_dir_url(), - maybe_node_modules_path: None, - }, - )); - let npm_resolution = Arc::new(NpmResolutionCell::default()); - let npm_resolver = - CliNpmResolver::new(CliNpmResolverCreateOptions::Managed( - CliManagedNpmResolverCreateOptions { - npm_resolution, - sys: sys.clone(), - npm_cache_dir, - maybe_node_modules_path: None, - npm_system_info: Default::default(), - npmrc: create_default_npmrc(), - }, - )); - (in_npm_pkg_checker, npm_resolver) - } - }; - - let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some(); - let node_resolver = Arc::new(NodeResolver::new( - in_npm_pkg_checker.clone(), - RealIsBuiltInNodeModuleChecker, - npm_resolver.clone(), - pkg_json_resolver.clone(), - sys.clone(), - node_resolver::ConditionsFromResolutionMode::default(), - )); - let cjs_tracker = Arc::new(CliCjsTracker::new( - in_npm_pkg_checker.clone(), - pkg_json_resolver.clone(), - if metadata.unstable_config.detect_cjs { - IsCjsResolutionMode::ImplicitTypeCommonJs - } else if metadata.workspace_resolver.package_jsons.is_empty() { - IsCjsResolutionMode::Disabled - } else { - IsCjsResolutionMode::ExplicitTypeCommonJs - }, - )); - let cache_db = Caches::new(deno_dir_provider.clone()); - let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); - let npm_req_resolver = - Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { - sys: sys.clone(), - in_npm_pkg_checker: in_npm_pkg_checker.clone(), - node_resolver: node_resolver.clone(), - npm_resolver: npm_resolver.clone(), - })); - let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new( - node_analysis_cache, - cjs_tracker.clone(), - fs.clone(), - None, - ); - let node_code_translator = Arc::new(NodeCodeTranslator::new( - cjs_esm_code_analyzer, - in_npm_pkg_checker, - node_resolver.clone(), - npm_resolver.clone(), - pkg_json_resolver.clone(), - sys.clone(), - )); - let workspace_resolver = { - let import_map = match metadata.workspace_resolver.import_map { - Some(import_map) => Some( - import_map::parse_from_json_with_options( - root_dir_url.join(&import_map.specifier).unwrap(), - &import_map.json, - import_map::ImportMapOptions { - address_hook: None, - expand_imports: true, - }, - )? - .import_map, - ), - None => None, - }; - let pkg_jsons = metadata - .workspace_resolver - .package_jsons - .into_iter() - .map(|(relative_path, json)| { - let path = root_dir_url - .join(&relative_path) - .unwrap() - .to_file_path() - .unwrap(); - let pkg_json = - deno_package_json::PackageJson::load_from_value(path, json); - Arc::new(pkg_json) - }) - .collect(); - WorkspaceResolver::new_raw( - root_dir_url.clone(), - import_map, - metadata - .workspace_resolver - .jsr_pkgs - .iter() - .map(|pkg| ResolverWorkspaceJsrPackage { - is_patch: false, // only used for enhancing the diagnostic, which isn't shown in deno compile - base: root_dir_url.join(&pkg.relative_base).unwrap(), - name: pkg.name.clone(), - version: pkg.version.clone(), - exports: pkg.exports.clone(), - }) - .collect(), - pkg_jsons, - metadata.workspace_resolver.pkg_json_resolution, - ) - }; - let code_cache = match metadata.code_cache_key { - Some(code_cache_key) => Some(Arc::new(DenoCompileCodeCache::new( - root_path.with_file_name(format!( - "{}.cache", - root_path.file_name().unwrap().to_string_lossy() - )), - code_cache_key, - )) as Arc), - None => { - log::debug!("Code cache disabled."); - None - } - }; - let module_loader_factory = StandaloneModuleLoaderFactory { - shared: Arc::new(SharedModuleLoaderState { - cjs_tracker: cjs_tracker.clone(), - code_cache: code_cache.clone(), - fs: fs.clone(), - modules, - node_code_translator: node_code_translator.clone(), - node_resolver: node_resolver.clone(), - npm_module_loader: Arc::new(NpmModuleLoader::new( - cjs_tracker.clone(), - fs.clone(), - node_code_translator, - )), - npm_registry_permission_checker, - npm_resolver: npm_resolver.clone(), - npm_req_resolver, - source_maps, - vfs, - workspace_resolver, - }), - }; - - let permissions = { - let mut permissions = metadata.permissions; - // grant read access to the vfs - match &mut permissions.allow_read { - Some(vec) if vec.is_empty() => { - // do nothing, already granted - } - Some(vec) => { - vec.push(root_path.to_string_lossy().to_string()); - } - None => { - permissions.allow_read = - Some(vec![root_path.to_string_lossy().to_string()]); - } - } - - let desc_parser = - Arc::new(RuntimePermissionDescriptorParser::new(sys.clone())); - let permissions = - Permissions::from_options(desc_parser.as_ref(), &permissions)?; - PermissionsContainer::new(desc_parser, permissions) - }; - let feature_checker = Arc::new({ - let mut checker = FeatureChecker::default(); - checker.set_exit_cb(Box::new(crate::unstable_exit_cb)); - for feature in metadata.unstable_config.features { - // `metadata` is valid for the whole lifetime of the program, so we - // can leak the string here. - checker.enable_feature(feature.leak()); - } - checker - }); - let lib_main_worker_options = LibMainWorkerOptions { - argv: metadata.argv, - log_level: WorkerLogLevel::Info, - enable_op_summary_metrics: false, - enable_testing_features: false, - has_node_modules_dir, - inspect_brk: false, - inspect_wait: false, - strace_ops: None, - is_inspecting: false, - skip_op_registration: true, - location: metadata.location, - argv0: NpmPackageReqReference::from_specifier(&main_module) - .ok() - .map(|req_ref| npm_pkg_req_ref_to_binary_command(&req_ref)) - .or(std::env::args().next()), - node_debug: std::env::var("NODE_DEBUG").ok(), - origin_data_folder_path: None, - seed: metadata.seed, - unsafely_ignore_certificate_errors: metadata - .unsafely_ignore_certificate_errors, - node_ipc: None, - serve_port: None, - serve_host: None, - deno_version: crate::version::DENO_VERSION_INFO.deno, - deno_user_agent: crate::version::DENO_VERSION_INFO.user_agent, - otel_config: metadata.otel_config, - startup_snapshot: crate::js::deno_isolate_init(), - }; - let lib_main_worker_factory = LibMainWorkerFactory::new( - Arc::new(BlobStore::default()), - code_cache.map(|c| c.as_code_cache()), - feature_checker, - fs, - None, - Box::new(module_loader_factory), - node_resolver.clone(), - create_npm_process_state_provider(&npm_resolver), - pkg_json_resolver, - root_cert_store_provider, - StorageKeyResolver::empty(), - sys.clone(), - lib_main_worker_options, - ); - // todo(dsherret): use LibMainWorker directly here and don't use CliMainWorkerFactory - let cli_main_worker_options = CliMainWorkerOptions { - create_hmr_runner: None, - create_coverage_collector: None, - needs_test_modules: false, - default_npm_caching_strategy: crate::args::NpmCachingStrategy::Lazy, - }; - let worker_factory = CliMainWorkerFactory::new( - lib_main_worker_factory, - None, - None, - node_resolver, - None, - npm_resolver, - sys, - cli_main_worker_options, - permissions, - ); - - // Initialize v8 once from the main thread. - v8_set_flags(construct_v8_flags(&[], &metadata.v8_flags, vec![])); - // TODO(bartlomieju): remove last argument once Deploy no longer needs it - deno_core::JsRuntime::init_platform(None, true); - - let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Run, main_module) - .await?; - - let exit_code = worker.run().await?; - Ok(exit_code) -} diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs deleted file mode 100644 index 00a0a04997..0000000000 --- a/cli/standalone/serialization.rs +++ /dev/null @@ -1,788 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -use std::borrow::Cow; -use std::cell::Cell; -use std::collections::BTreeMap; -use std::collections::HashMap; -use std::io::Write; - -use capacity_builder::BytesAppendable; -use deno_ast::swc::common::source_map; -use deno_ast::MediaType; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::serde_json; -use deno_core::url::Url; -use deno_core::FastString; -use deno_core::ModuleSourceCode; -use deno_core::ModuleType; -use deno_error::JsErrorBox; -use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; -use deno_npm::resolution::SerializedNpmResolutionSnapshot; -use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; -use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; -use deno_npm::NpmPackageId; -use deno_semver::package::PackageReq; -use deno_semver::StackString; -use indexmap::IndexMap; - -use super::binary::Metadata; -use super::virtual_fs::BuiltVfs; -use super::virtual_fs::VfsBuilder; - -const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; - -/// Binary format: -/// * d3n0l4nd -/// * -/// * -/// * -/// * -/// * -/// * -/// * d3n0l4nd -pub fn serialize_binary_data_section( - metadata: &Metadata, - npm_snapshot: Option, - remote_modules: &RemoteModulesStoreBuilder, - source_map_store: &SourceMapStore, - vfs: &BuiltVfs, -) -> Result, AnyError> { - let metadata = serde_json::to_string(metadata)?; - let npm_snapshot = - npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); - let serialized_vfs = serde_json::to_string(&vfs.entries)?; - - let bytes = capacity_builder::BytesBuilder::build(|builder| { - builder.append(MAGIC_BYTES); - // 1. Metadata - { - builder.append_le(metadata.len() as u64); - builder.append(&metadata); - } - // 2. Npm snapshot - { - builder.append_le(npm_snapshot.len() as u64); - builder.append(&npm_snapshot); - } - // 3. Remote modules - { - remote_modules.write(builder); - } - // 4. VFS - { - builder.append_le(serialized_vfs.len() as u64); - builder.append(&serialized_vfs); - let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); - builder.append_le(vfs_bytes_len); - for file in &vfs.files { - builder.append(file); - } - } - // 5. Source maps - { - builder.append_le(source_map_store.data.len() as u32); - for (specifier, source_map) in &source_map_store.data { - builder.append_le(specifier.len() as u32); - builder.append(specifier); - builder.append_le(source_map.len() as u32); - builder.append(source_map.as_ref()); - } - } - - // write the magic bytes at the end so we can use it - // to make sure we've deserialized correctly - builder.append(MAGIC_BYTES); - })?; - - Ok(bytes) -} - -pub struct DeserializedDataSection { - pub metadata: Metadata, - pub npm_snapshot: Option, - pub remote_modules: RemoteModulesStore, - pub source_maps: SourceMapStore, - pub vfs_root_entries: VirtualDirectoryEntries, - pub vfs_files_data: &'static [u8], -} - -pub fn deserialize_binary_data_section( - data: &'static [u8], -) -> Result, AnyError> { - fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> { - if input.len() < MAGIC_BYTES.len() { - bail!("Unexpected end of data. Could not find magic bytes."); - } - let (magic_bytes, input) = input.split_at(MAGIC_BYTES.len()); - if magic_bytes != MAGIC_BYTES { - return Ok((input, false)); - } - Ok((input, true)) - } - - #[allow(clippy::type_complexity)] - fn read_source_map_entry( - input: &[u8], - ) -> Result<(&[u8], (Cow, &[u8])), AnyError> { - let (input, specifier) = read_string_lossy(input)?; - let (input, source_map) = read_bytes_with_u32_len(input)?; - Ok((input, (specifier, source_map))) - } - - let (input, found) = read_magic_bytes(data)?; - if !found { - return Ok(None); - } - - // 1. Metadata - let (input, data) = - read_bytes_with_u64_len(input).context("reading metadata")?; - let metadata: Metadata = - serde_json::from_slice(data).context("deserializing metadata")?; - // 2. Npm snapshot - let (input, data) = - read_bytes_with_u64_len(input).context("reading npm snapshot")?; - let npm_snapshot = if data.is_empty() { - None - } else { - Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?) - }; - // 3. Remote modules - let (input, remote_modules) = - RemoteModulesStore::build(input).context("deserializing remote modules")?; - // 4. VFS - let (input, data) = read_bytes_with_u64_len(input).context("vfs")?; - let vfs_root_entries: VirtualDirectoryEntries = - serde_json::from_slice(data).context("deserializing vfs data")?; - let (input, vfs_files_data) = - read_bytes_with_u64_len(input).context("reading vfs files data")?; - // 5. Source maps - let (mut input, source_map_data_len) = read_u32_as_usize(input)?; - let mut source_maps = SourceMapStore::with_capacity(source_map_data_len); - for _ in 0..source_map_data_len { - let (current_input, (specifier, source_map)) = - read_source_map_entry(input)?; - input = current_input; - source_maps.add(specifier, Cow::Borrowed(source_map)); - } - - // finally ensure we read the magic bytes at the end - let (_input, found) = read_magic_bytes(input)?; - if !found { - bail!("Could not find magic bytes at the end of the data."); - } - - Ok(Some(DeserializedDataSection { - metadata, - npm_snapshot, - remote_modules, - source_maps, - vfs_root_entries, - vfs_files_data, - })) -} - -#[derive(Default)] -pub struct RemoteModulesStoreBuilder { - specifiers: Vec<(String, u64)>, - data: Vec<(MediaType, Vec, Option>)>, - data_byte_len: u64, - redirects: Vec<(String, String)>, - redirects_len: u64, -} - -impl RemoteModulesStoreBuilder { - pub fn add( - &mut self, - specifier: &Url, - media_type: MediaType, - data: Vec, - maybe_transpiled: Option>, - ) { - log::debug!("Adding '{}' ({})", specifier, media_type); - let specifier = specifier.to_string(); - self.specifiers.push((specifier, self.data_byte_len)); - let maybe_transpiled_len = match &maybe_transpiled { - // data length (4 bytes), data - Some(data) => 4 + data.len() as u64, - None => 0, - }; - // media type (1 byte), data length (4 bytes), data, has transpiled (1 byte), transpiled length - self.data_byte_len += 1 + 4 + data.len() as u64 + 1 + maybe_transpiled_len; - self.data.push((media_type, data, maybe_transpiled)); - } - - pub fn add_redirects(&mut self, redirects: &BTreeMap) { - self.redirects.reserve(redirects.len()); - for (from, to) in redirects { - log::debug!("Adding redirect '{}' -> '{}'", from, to); - let from = from.to_string(); - let to = to.to_string(); - self.redirects_len += (4 + from.len() + 4 + to.len()) as u64; - self.redirects.push((from, to)); - } - } - - fn write<'a, TBytes: capacity_builder::BytesType>( - &'a self, - builder: &mut capacity_builder::BytesBuilder<'a, TBytes>, - ) { - builder.append_le(self.specifiers.len() as u32); - builder.append_le(self.redirects.len() as u32); - for (specifier, offset) in &self.specifiers { - builder.append_le(specifier.len() as u32); - builder.append(specifier); - builder.append_le(*offset); - } - for (from, to) in &self.redirects { - builder.append_le(from.len() as u32); - builder.append(from); - builder.append_le(to.len() as u32); - builder.append(to); - } - builder.append_le( - self - .data - .iter() - .map(|(_, data, maybe_transpiled)| { - 1 + 4 - + (data.len() as u64) - + 1 - + match maybe_transpiled { - Some(transpiled) => 4 + (transpiled.len() as u64), - None => 0, - } - }) - .sum::(), - ); - for (media_type, data, maybe_transpiled) in &self.data { - builder.append(serialize_media_type(*media_type)); - builder.append_le(data.len() as u32); - builder.append(data); - if let Some(transpiled) = maybe_transpiled { - builder.append(1); - builder.append_le(transpiled.len() as u32); - builder.append(transpiled); - } else { - builder.append(0); - } - } - } -} - -pub enum DenoCompileModuleSource { - String(&'static str), - Bytes(Cow<'static, [u8]>), -} - -impl DenoCompileModuleSource { - pub fn into_for_v8(self) -> ModuleSourceCode { - fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode { - ModuleSourceCode::Bytes(match data { - Cow::Borrowed(d) => d.into(), - Cow::Owned(d) => d.into_boxed_slice().into(), - }) - } - - match self { - // todo(https://github.com/denoland/deno_core/pull/943): store whether - // the string is ascii or not ahead of time so we can avoid the is_ascii() - // check in FastString::from_static - Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)), - Self::Bytes(b) => into_bytes(b), - } - } -} - -pub struct SourceMapStore { - data: IndexMap, Cow<'static, [u8]>>, -} - -impl SourceMapStore { - pub fn with_capacity(capacity: usize) -> Self { - Self { - data: IndexMap::with_capacity(capacity), - } - } - - pub fn add( - &mut self, - specifier: Cow<'static, str>, - source_map: Cow<'static, [u8]>, - ) { - self.data.insert(specifier, source_map); - } - - pub fn get(&self, specifier: &str) -> Option<&[u8]> { - self.data.get(specifier).map(|v| v.as_ref()) - } -} - -pub struct DenoCompileModuleData<'a> { - pub specifier: &'a Url, - pub media_type: MediaType, - pub data: Cow<'static, [u8]>, -} - -impl<'a> DenoCompileModuleData<'a> { - pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) { - fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource { - match data { - Cow::Borrowed(d) => DenoCompileModuleSource::String( - // SAFETY: we know this is a valid utf8 string - unsafe { std::str::from_utf8_unchecked(d) }, - ), - Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)), - } - } - - let (media_type, source) = match self.media_type { - MediaType::JavaScript - | MediaType::Jsx - | MediaType::Mjs - | MediaType::Cjs - | MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Dts - | MediaType::Dmts - | MediaType::Dcts - | MediaType::Tsx => { - (ModuleType::JavaScript, into_string_unsafe(self.data)) - } - MediaType::Json => (ModuleType::Json, into_string_unsafe(self.data)), - MediaType::Wasm => { - (ModuleType::Wasm, DenoCompileModuleSource::Bytes(self.data)) - } - // just assume javascript if we made it here - MediaType::Css | MediaType::SourceMap | MediaType::Unknown => ( - ModuleType::JavaScript, - DenoCompileModuleSource::Bytes(self.data), - ), - }; - (self.specifier, media_type, source) - } -} - -pub struct RemoteModuleEntry<'a> { - pub specifier: &'a Url, - pub media_type: MediaType, - pub data: Cow<'static, [u8]>, - pub transpiled_data: Option>, -} - -enum RemoteModulesStoreSpecifierValue { - Data(usize), - Redirect(Url), -} - -pub struct RemoteModulesStore { - specifiers: HashMap, - files_data: &'static [u8], -} - -impl RemoteModulesStore { - fn build(input: &'static [u8]) -> Result<(&'static [u8], Self), AnyError> { - fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> { - let (input, specifier) = read_string_lossy(input)?; - let specifier = Url::parse(&specifier)?; - let (input, offset) = read_u64(input)?; - Ok((input, (specifier, offset))) - } - - fn read_redirect(input: &[u8]) -> Result<(&[u8], (Url, Url)), AnyError> { - let (input, from) = read_string_lossy(input)?; - let from = Url::parse(&from)?; - let (input, to) = read_string_lossy(input)?; - let to = Url::parse(&to)?; - Ok((input, (from, to))) - } - - fn read_headers( - input: &[u8], - ) -> Result<(&[u8], HashMap), AnyError> - { - let (input, specifiers_len) = read_u32_as_usize(input)?; - let (mut input, redirects_len) = read_u32_as_usize(input)?; - let mut specifiers = - HashMap::with_capacity(specifiers_len + redirects_len); - for _ in 0..specifiers_len { - let (current_input, (specifier, offset)) = - read_specifier(input).context("reading specifier")?; - input = current_input; - specifiers.insert( - specifier, - RemoteModulesStoreSpecifierValue::Data(offset as usize), - ); - } - - for _ in 0..redirects_len { - let (current_input, (from, to)) = read_redirect(input)?; - input = current_input; - specifiers.insert(from, RemoteModulesStoreSpecifierValue::Redirect(to)); - } - - Ok((input, specifiers)) - } - - let (input, specifiers) = read_headers(input)?; - let (input, files_data) = read_bytes_with_u64_len(input)?; - - Ok(( - input, - Self { - specifiers, - files_data, - }, - )) - } - - pub fn resolve_specifier<'a>( - &'a self, - specifier: &'a Url, - ) -> Result, JsErrorBox> { - let mut count = 0; - let mut current = specifier; - loop { - if count > 10 { - return Err(JsErrorBox::generic(format!( - "Too many redirects resolving '{}'", - specifier - ))); - } - match self.specifiers.get(current) { - Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { - current = to; - count += 1; - } - Some(RemoteModulesStoreSpecifierValue::Data(_)) => { - return Ok(Some(current)); - } - None => { - return Ok(None); - } - } - } - } - - pub fn read<'a>( - &'a self, - original_specifier: &'a Url, - ) -> Result>, AnyError> { - let mut count = 0; - let mut specifier = original_specifier; - loop { - if count > 10 { - bail!("Too many redirects resolving '{}'", original_specifier); - } - match self.specifiers.get(specifier) { - Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { - specifier = to; - count += 1; - } - Some(RemoteModulesStoreSpecifierValue::Data(offset)) => { - let input = &self.files_data[*offset..]; - let (input, media_type_byte) = read_bytes(input, 1)?; - let media_type = deserialize_media_type(media_type_byte[0])?; - let (input, data) = read_bytes_with_u32_len(input)?; - check_has_len(input, 1)?; - let (input, has_transpiled) = (&input[1..], input[0]); - let (_, transpiled_data) = match has_transpiled { - 0 => (input, None), - 1 => { - let (input, data) = read_bytes_with_u32_len(input)?; - (input, Some(data)) - } - value => bail!( - "Invalid transpiled data flag: {}. Compiled data is corrupt.", - value - ), - }; - return Ok(Some(RemoteModuleEntry { - specifier, - media_type, - data: Cow::Borrowed(data), - transpiled_data: transpiled_data.map(Cow::Borrowed), - })); - } - None => { - return Ok(None); - } - } - } - } -} - -fn serialize_npm_snapshot( - mut snapshot: SerializedNpmResolutionSnapshot, -) -> Vec { - fn append_string(bytes: &mut Vec, string: &str) { - let len = string.len() as u32; - bytes.extend_from_slice(&len.to_le_bytes()); - bytes.extend_from_slice(string.as_bytes()); - } - - snapshot.packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism - let ids_to_stored_ids = snapshot - .packages - .iter() - .enumerate() - .map(|(i, pkg)| (&pkg.id, i as u32)) - .collect::>(); - - let mut root_packages: Vec<_> = snapshot.root_packages.iter().collect(); - root_packages.sort(); - let mut bytes = Vec::new(); - - bytes.extend_from_slice(&(snapshot.packages.len() as u32).to_le_bytes()); - for pkg in &snapshot.packages { - append_string(&mut bytes, &pkg.id.as_serialized()); - } - - bytes.extend_from_slice(&(root_packages.len() as u32).to_le_bytes()); - for (req, id) in root_packages { - append_string(&mut bytes, &req.to_string()); - let id = ids_to_stored_ids.get(&id).unwrap(); - bytes.extend_from_slice(&id.to_le_bytes()); - } - - for pkg in &snapshot.packages { - let deps_len = pkg.dependencies.len() as u32; - bytes.extend_from_slice(&deps_len.to_le_bytes()); - let mut deps: Vec<_> = pkg.dependencies.iter().collect(); - deps.sort(); - for (req, id) in deps { - append_string(&mut bytes, req); - let id = ids_to_stored_ids.get(&id).unwrap(); - bytes.extend_from_slice(&id.to_le_bytes()); - } - } - - bytes -} - -fn deserialize_npm_snapshot( - input: &[u8], -) -> Result { - fn parse_id(input: &[u8]) -> Result<(&[u8], NpmPackageId), AnyError> { - let (input, id) = read_string_lossy(input)?; - let id = NpmPackageId::from_serialized(&id)?; - Ok((input, id)) - } - - #[allow(clippy::needless_lifetimes)] // clippy bug - fn parse_root_package<'a>( - id_to_npm_id: &'a impl Fn(usize) -> Result, - ) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a - { - |input| { - let (input, req) = read_string_lossy(input)?; - let req = PackageReq::from_str(&req)?; - let (input, id) = read_u32_as_usize(input)?; - Ok((input, (req, id_to_npm_id(id)?))) - } - } - - #[allow(clippy::needless_lifetimes)] // clippy bug - fn parse_package_dep<'a>( - id_to_npm_id: &'a impl Fn(usize) -> Result, - ) -> impl Fn(&[u8]) -> Result<(&[u8], (StackString, NpmPackageId)), AnyError> + 'a - { - |input| { - let (input, req) = read_string_lossy(input)?; - let (input, id) = read_u32_as_usize(input)?; - let req = StackString::from_cow(req); - Ok((input, (req, id_to_npm_id(id)?))) - } - } - - fn parse_package<'a>( - input: &'a [u8], - id: NpmPackageId, - id_to_npm_id: &impl Fn(usize) -> Result, - ) -> Result<(&'a [u8], SerializedNpmResolutionSnapshotPackage), AnyError> { - let (input, deps_len) = read_u32_as_usize(input)?; - let (input, dependencies) = - parse_hashmap_n_times(input, deps_len, parse_package_dep(id_to_npm_id))?; - Ok(( - input, - SerializedNpmResolutionSnapshotPackage { - id, - system: Default::default(), - dist: Default::default(), - dependencies, - optional_dependencies: Default::default(), - bin: None, - scripts: Default::default(), - deprecated: Default::default(), - }, - )) - } - - let (input, packages_len) = read_u32_as_usize(input)?; - - // get a hashmap of all the npm package ids to their serialized ids - let (input, data_ids_to_npm_ids) = - parse_vec_n_times(input, packages_len, parse_id) - .context("deserializing id")?; - let data_id_to_npm_id = |id: usize| { - data_ids_to_npm_ids - .get(id) - .cloned() - .ok_or_else(|| deno_core::anyhow::anyhow!("Invalid npm package id")) - }; - - let (input, root_packages_len) = read_u32_as_usize(input)?; - let (input, root_packages) = parse_hashmap_n_times( - input, - root_packages_len, - parse_root_package(&data_id_to_npm_id), - ) - .context("deserializing root package")?; - let (input, packages) = - parse_vec_n_times_with_index(input, packages_len, |input, index| { - parse_package(input, data_id_to_npm_id(index)?, &data_id_to_npm_id) - }) - .context("deserializing package")?; - - if !input.is_empty() { - bail!("Unexpected data left over"); - } - - Ok( - SerializedNpmResolutionSnapshot { - packages, - root_packages, - } - // this is ok because we have already verified that all the - // identifiers found in the snapshot are valid via the - // npm package id -> npm package id mapping - .into_valid_unsafe(), - ) -} - -fn serialize_media_type(media_type: MediaType) -> u8 { - match media_type { - MediaType::JavaScript => 0, - MediaType::Jsx => 1, - MediaType::Mjs => 2, - MediaType::Cjs => 3, - MediaType::TypeScript => 4, - MediaType::Mts => 5, - MediaType::Cts => 6, - MediaType::Dts => 7, - MediaType::Dmts => 8, - MediaType::Dcts => 9, - MediaType::Tsx => 10, - MediaType::Json => 11, - MediaType::Wasm => 12, - MediaType::Css => 13, - MediaType::SourceMap => 14, - MediaType::Unknown => 15, - } -} - -fn deserialize_media_type(value: u8) -> Result { - match value { - 0 => Ok(MediaType::JavaScript), - 1 => Ok(MediaType::Jsx), - 2 => Ok(MediaType::Mjs), - 3 => Ok(MediaType::Cjs), - 4 => Ok(MediaType::TypeScript), - 5 => Ok(MediaType::Mts), - 6 => Ok(MediaType::Cts), - 7 => Ok(MediaType::Dts), - 8 => Ok(MediaType::Dmts), - 9 => Ok(MediaType::Dcts), - 10 => Ok(MediaType::Tsx), - 11 => Ok(MediaType::Json), - 12 => Ok(MediaType::Wasm), - 13 => Ok(MediaType::Css), - 14 => Ok(MediaType::SourceMap), - 15 => Ok(MediaType::Unknown), - _ => bail!("Unknown media type value: {}", value), - } -} - -fn parse_hashmap_n_times( - mut input: &[u8], - times: usize, - parse: impl Fn(&[u8]) -> Result<(&[u8], (TKey, TValue)), AnyError>, -) -> Result<(&[u8], HashMap), AnyError> { - let mut results = HashMap::with_capacity(times); - for _ in 0..times { - let result = parse(input); - let (new_input, (key, value)) = result?; - results.insert(key, value); - input = new_input; - } - Ok((input, results)) -} - -fn parse_vec_n_times( - input: &[u8], - times: usize, - parse: impl Fn(&[u8]) -> Result<(&[u8], TResult), AnyError>, -) -> Result<(&[u8], Vec), AnyError> { - parse_vec_n_times_with_index(input, times, |input, _index| parse(input)) -} - -fn parse_vec_n_times_with_index( - mut input: &[u8], - times: usize, - parse: impl Fn(&[u8], usize) -> Result<(&[u8], TResult), AnyError>, -) -> Result<(&[u8], Vec), AnyError> { - let mut results = Vec::with_capacity(times); - for i in 0..times { - let result = parse(input, i); - let (new_input, result) = result?; - results.push(result); - input = new_input; - } - Ok((input, results)) -} - -fn read_bytes_with_u64_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { - let (input, len) = read_u64(input)?; - let (input, data) = read_bytes(input, len as usize)?; - Ok((input, data)) -} - -fn read_bytes_with_u32_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { - let (input, len) = read_u32_as_usize(input)?; - let (input, data) = read_bytes(input, len)?; - Ok((input, data)) -} - -fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> { - check_has_len(input, len)?; - let (len_bytes, input) = input.split_at(len); - Ok((input, len_bytes)) -} - -#[inline(always)] -fn check_has_len(input: &[u8], len: usize) -> Result<(), AnyError> { - if input.len() < len { - bail!("Unexpected end of data."); - } - Ok(()) -} - -fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow), AnyError> { - let (input, data_bytes) = read_bytes_with_u32_len(input)?; - Ok((input, String::from_utf8_lossy(data_bytes))) -} - -fn read_u32_as_usize(input: &[u8]) -> Result<(&[u8], usize), AnyError> { - let (input, len_bytes) = read_bytes(input, 4)?; - let len = u32::from_le_bytes(len_bytes.try_into()?); - Ok((input, len as usize)) -} - -fn read_u64(input: &[u8]) -> Result<(&[u8], u64), AnyError> { - let (input, len_bytes) = read_bytes(input, 8)?; - let len = u64::from_le_bytes(len_bytes.try_into()?); - Ok((input, len)) -} diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 4f761d0d15..fa79b784dc 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -1,497 +1,21 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::borrow::Cow; -use std::cell::RefCell; -use std::cmp::Ordering; -use std::collections::HashMap; use std::collections::HashSet; -use std::fs::File; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::ops::Range; -use std::path::Path; use std::path::PathBuf; -use std::rc::Rc; -use std::sync::Arc; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::parking_lot::Mutex; -use deno_core::BufMutView; -use deno_core::BufView; -use deno_core::ResourceHandleFd; -use deno_lib::standalone::virtual_fs::FileSystemCaseSensitivity; +use deno_lib::standalone::virtual_fs::BuiltVfs; use deno_lib::standalone::virtual_fs::OffsetWithLength; use deno_lib::standalone::virtual_fs::VfsEntry; -use deno_lib::standalone::virtual_fs::VfsEntryRef; -use deno_lib::standalone::virtual_fs::VfsFileSubDataKind; use deno_lib::standalone::virtual_fs::VirtualDirectory; use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries; use deno_lib::standalone::virtual_fs::VirtualFile; -use deno_lib::standalone::virtual_fs::VirtualSymlink; use deno_lib::standalone::virtual_fs::VirtualSymlinkParts; use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath; -use deno_path_util::normalize_path; -use deno_path_util::strip_unc_prefix; -use deno_runtime::deno_fs::FsDirEntry; -use deno_runtime::deno_io; -use deno_runtime::deno_io::fs::FsError; -use deno_runtime::deno_io::fs::FsResult; -use deno_runtime::deno_io::fs::FsStat; -use indexmap::IndexSet; -use serde::Deserialize; -use serde::Serialize; -use thiserror::Error; +use deno_lib::standalone::virtual_fs::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; -use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; -use crate::util; use crate::util::display::human_size; use crate::util::display::DisplayTreeNode; -use crate::util::fs::canonicalize_path; - -#[derive(Debug)] -pub struct BuiltVfs { - pub root_path: WindowsSystemRootablePath, - pub case_sensitivity: FileSystemCaseSensitivity, - pub entries: VirtualDirectoryEntries, - pub files: Vec>, -} - -#[derive(Debug)] -pub struct VfsBuilder { - executable_root: VirtualDirectory, - files: Vec>, - current_offset: u64, - file_offsets: HashMap, - /// The minimum root directory that should be included in the VFS. - min_root_dir: Option, - case_sensitivity: FileSystemCaseSensitivity, -} - -impl VfsBuilder { - pub fn new() -> Self { - Self { - executable_root: VirtualDirectory { - name: "/".to_string(), - entries: Default::default(), - }, - files: Vec::new(), - current_offset: 0, - file_offsets: Default::default(), - min_root_dir: Default::default(), - // This is not exactly correct because file systems on these OSes - // may be case-sensitive or not based on the directory, but this - // is a good enough approximation and limitation. In the future, - // we may want to store this information per directory instead - // depending on the feedback we get. - case_sensitivity: if cfg!(windows) || cfg!(target_os = "macos") { - FileSystemCaseSensitivity::Insensitive - } else { - FileSystemCaseSensitivity::Sensitive - }, - } - } - - pub fn case_sensitivity(&self) -> FileSystemCaseSensitivity { - self.case_sensitivity - } - - /// Add a directory that might be the minimum root directory - /// of the VFS. - /// - /// For example, say the user has a deno.json and specifies an - /// import map in a parent directory. The import map won't be - /// included in the VFS, but its base will meaning we need to - /// tell the VFS builder to include the base of the import map - /// by calling this method. - pub fn add_possible_min_root_dir(&mut self, path: &Path) { - self.add_dir_raw(path); - - match &self.min_root_dir { - Some(WindowsSystemRootablePath::WindowSystemRoot) => { - // already the root dir - } - Some(WindowsSystemRootablePath::Path(current_path)) => { - let mut common_components = Vec::new(); - for (a, b) in current_path.components().zip(path.components()) { - if a != b { - break; - } - common_components.push(a); - } - if common_components.is_empty() { - if cfg!(windows) { - self.min_root_dir = - Some(WindowsSystemRootablePath::WindowSystemRoot); - } else { - self.min_root_dir = - Some(WindowsSystemRootablePath::Path(PathBuf::from("/"))); - } - } else { - self.min_root_dir = Some(WindowsSystemRootablePath::Path( - common_components.iter().collect(), - )); - } - } - None => { - self.min_root_dir = - Some(WindowsSystemRootablePath::Path(path.to_path_buf())); - } - } - } - - pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = self.resolve_target_path(path)?; - self.add_dir_recursive_not_symlink(&target_path) - } - - fn add_dir_recursive_not_symlink( - &mut self, - path: &Path, - ) -> Result<(), AnyError> { - self.add_dir_raw(path); - let read_dir = std::fs::read_dir(path) - .with_context(|| format!("Reading {}", path.display()))?; - - let mut dir_entries = - read_dir.into_iter().collect::, _>>()?; - dir_entries.sort_by_cached_key(|entry| entry.file_name()); // determinism - - for entry in dir_entries { - let file_type = entry.file_type()?; - let path = entry.path(); - - if file_type.is_dir() { - self.add_dir_recursive_not_symlink(&path)?; - } else if file_type.is_file() { - self.add_file_at_path_not_symlink(&path)?; - } else if file_type.is_symlink() { - match self.add_symlink(&path) { - Ok(target) => match target { - SymlinkTarget::File(target) => { - self.add_file_at_path_not_symlink(&target)? - } - SymlinkTarget::Dir(target) => { - self.add_dir_recursive_not_symlink(&target)?; - } - }, - Err(err) => { - log::warn!( - "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", - crate::colors::yellow("Warning"), - path.display(), - err - ); - } - } - } - } - - Ok(()) - } - - fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { - log::debug!("Ensuring directory '{}'", path.display()); - debug_assert!(path.is_absolute()); - let mut current_dir = &mut self.executable_root; - - for component in path.components() { - if matches!(component, std::path::Component::RootDir) { - continue; - } - let name = component.as_os_str().to_string_lossy(); - let index = current_dir.entries.insert_or_modify( - &name, - self.case_sensitivity, - || { - VfsEntry::Dir(VirtualDirectory { - name: name.to_string(), - entries: Default::default(), - }) - }, - |_| { - // ignore - }, - ); - match current_dir.entries.get_mut_by_index(index) { - Some(VfsEntry::Dir(dir)) => { - current_dir = dir; - } - _ => unreachable!(), - }; - } - - current_dir - } - - pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { - &mut self.executable_root - } - - pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { - debug_assert!(path.is_absolute()); - let mut current_dir = &mut self.executable_root; - - for component in path.components() { - if matches!(component, std::path::Component::RootDir) { - continue; - } - let name = component.as_os_str().to_string_lossy(); - let entry = current_dir - .entries - .get_mut_by_name(&name, self.case_sensitivity)?; - match entry { - VfsEntry::Dir(dir) => { - current_dir = dir; - } - _ => unreachable!(), - }; - } - - Some(current_dir) - } - - pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { - let file_bytes = std::fs::read(path) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw) - } - - fn add_file_at_path_not_symlink( - &mut self, - path: &Path, - ) -> Result<(), AnyError> { - let file_bytes = std::fs::read(path) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner(path, file_bytes, VfsFileSubDataKind::Raw) - } - - pub fn add_file_with_data( - &mut self, - path: &Path, - data: Vec, - sub_data_kind: VfsFileSubDataKind, - ) -> Result<(), AnyError> { - let metadata = std::fs::symlink_metadata(path).with_context(|| { - format!("Resolving target path for '{}'", path.display()) - })?; - if metadata.is_symlink() { - let target = self.add_symlink(path)?.into_path_buf(); - self.add_file_with_data_inner(&target, data, sub_data_kind) - } else { - self.add_file_with_data_inner(path, data, sub_data_kind) - } - } - - fn add_file_with_data_inner( - &mut self, - path: &Path, - data: Vec, - sub_data_kind: VfsFileSubDataKind, - ) -> Result<(), AnyError> { - log::debug!("Adding file '{}'", path.display()); - let checksum = deno_lib::util::checksum::gen(&[&data]); - let case_sensitivity = self.case_sensitivity; - let offset = if let Some(offset) = self.file_offsets.get(&checksum) { - // duplicate file, reuse an old offset - *offset - } else { - self.file_offsets.insert(checksum, self.current_offset); - self.current_offset - }; - - let dir = self.add_dir_raw(path.parent().unwrap()); - let name = path.file_name().unwrap().to_string_lossy(); - let offset_and_len = OffsetWithLength { - offset, - len: data.len() as u64, - }; - dir.entries.insert_or_modify( - &name, - case_sensitivity, - || { - VfsEntry::File(VirtualFile { - name: name.to_string(), - offset: offset_and_len, - module_graph_offset: offset_and_len, - }) - }, - |entry| match entry { - VfsEntry::File(virtual_file) => match sub_data_kind { - VfsFileSubDataKind::Raw => { - virtual_file.offset = offset_and_len; - } - VfsFileSubDataKind::ModuleGraph => { - virtual_file.module_graph_offset = offset_and_len; - } - }, - VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), - }, - ); - - // new file, update the list of files - if self.current_offset == offset { - self.files.push(data); - self.current_offset += offset_and_len.len; - } - - Ok(()) - } - - fn resolve_target_path(&mut self, path: &Path) -> Result { - let metadata = std::fs::symlink_metadata(path).with_context(|| { - format!("Resolving target path for '{}'", path.display()) - })?; - if metadata.is_symlink() { - Ok(self.add_symlink(path)?.into_path_buf()) - } else { - Ok(path.to_path_buf()) - } - } - - fn add_symlink(&mut self, path: &Path) -> Result { - self.add_symlink_inner(path, &mut IndexSet::new()) - } - - fn add_symlink_inner( - &mut self, - path: &Path, - visited: &mut IndexSet, - ) -> Result { - log::debug!("Adding symlink '{}'", path.display()); - let target = strip_unc_prefix( - std::fs::read_link(path) - .with_context(|| format!("Reading symlink '{}'", path.display()))?, - ); - let case_sensitivity = self.case_sensitivity; - let target = normalize_path(path.parent().unwrap().join(&target)); - let dir = self.add_dir_raw(path.parent().unwrap()); - let name = path.file_name().unwrap().to_string_lossy(); - dir.entries.insert_or_modify( - &name, - case_sensitivity, - || { - VfsEntry::Symlink(VirtualSymlink { - name: name.to_string(), - dest_parts: VirtualSymlinkParts::from_path(&target), - }) - }, - |_| { - // ignore previously inserted - }, - ); - let target_metadata = - std::fs::symlink_metadata(&target).with_context(|| { - format!("Reading symlink target '{}'", target.display()) - })?; - if target_metadata.is_symlink() { - if !visited.insert(target.clone()) { - // todo: probably don't error in this scenario - bail!( - "Circular symlink detected: {} -> {}", - visited - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(" -> "), - target.display() - ); - } - self.add_symlink_inner(&target, visited) - } else if target_metadata.is_dir() { - Ok(SymlinkTarget::Dir(target)) - } else { - Ok(SymlinkTarget::File(target)) - } - } - - pub fn build(self) -> BuiltVfs { - fn strip_prefix_from_symlinks( - dir: &mut VirtualDirectory, - parts: &[String], - ) { - for entry in dir.entries.iter_mut() { - match entry { - VfsEntry::Dir(dir) => { - strip_prefix_from_symlinks(dir, parts); - } - VfsEntry::File(_) => {} - VfsEntry::Symlink(symlink) => { - let parts = symlink - .dest_parts - .take_parts() - .into_iter() - .skip(parts.len()) - .collect(); - symlink.dest_parts.set_parts(parts); - } - } - } - } - - let mut current_dir = self.executable_root; - let mut current_path = if cfg!(windows) { - WindowsSystemRootablePath::WindowSystemRoot - } else { - WindowsSystemRootablePath::Path(PathBuf::from("/")) - }; - loop { - if current_dir.entries.len() != 1 { - break; - } - if self.min_root_dir.as_ref() == Some(¤t_path) { - break; - } - match current_dir.entries.iter().next().unwrap() { - VfsEntry::Dir(dir) => { - if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { - // special directory we want to maintain - break; - } - match current_dir.entries.remove(0) { - VfsEntry::Dir(dir) => { - current_path = - WindowsSystemRootablePath::Path(current_path.join(&dir.name)); - current_dir = dir; - } - _ => unreachable!(), - }; - } - VfsEntry::File(_) | VfsEntry::Symlink(_) => break, - } - } - if let WindowsSystemRootablePath::Path(path) = ¤t_path { - strip_prefix_from_symlinks( - &mut current_dir, - VirtualSymlinkParts::from_path(path).parts(), - ); - } - BuiltVfs { - root_path: current_path, - case_sensitivity: self.case_sensitivity, - entries: current_dir.entries, - files: self.files, - } - } -} - -#[derive(Debug)] -enum SymlinkTarget { - File(PathBuf), - Dir(PathBuf), -} - -impl SymlinkTarget { - pub fn into_path_buf(self) -> PathBuf { - match self { - Self::File(path) => path, - Self::Dir(path) => path, - } - } -} pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { if !log::log_enabled!(log::Level::Info) { @@ -680,8 +204,13 @@ fn vfs_as_display_tree( let mut size = Size::default(); add_offset_to_size(file.offset, &mut size, seen_offsets); - if file.module_graph_offset.offset != file.offset.offset { - add_offset_to_size(file.module_graph_offset, &mut size, seen_offsets); + let maybe_offsets = [ + file.transpiled_offset, + file.source_map_offset, + file.cjs_export_analysis_offset, + ]; + for offset in maybe_offsets.into_iter().flatten() { + add_offset_to_size(offset, &mut size, seen_offsets); } size } @@ -841,846 +370,14 @@ fn vfs_as_display_tree( } } -#[derive(Debug)] -pub struct VfsRoot { - pub dir: VirtualDirectory, - pub root_path: PathBuf, - pub start_file_offset: u64, -} - -impl VfsRoot { - fn find_entry<'a>( - &'a self, - path: &Path, - case_sensitivity: FileSystemCaseSensitivity, - ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { - self.find_entry_inner(path, &mut HashSet::new(), case_sensitivity) - } - - fn find_entry_inner<'a>( - &'a self, - path: &Path, - seen: &mut HashSet, - case_sensitivity: FileSystemCaseSensitivity, - ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { - let mut path = Cow::Borrowed(path); - loop { - let (resolved_path, entry) = - self.find_entry_no_follow_inner(&path, seen, case_sensitivity)?; - match entry { - VfsEntryRef::Symlink(symlink) => { - if !seen.insert(path.to_path_buf()) { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "circular symlinks", - )); - } - path = Cow::Owned(symlink.resolve_dest_from_root(&self.root_path)); - } - _ => { - return Ok((resolved_path, entry)); - } - } - } - } - - fn find_entry_no_follow( - &self, - path: &Path, - case_sensitivity: FileSystemCaseSensitivity, - ) -> std::io::Result<(PathBuf, VfsEntryRef)> { - self.find_entry_no_follow_inner(path, &mut HashSet::new(), case_sensitivity) - } - - fn find_entry_no_follow_inner<'a>( - &'a self, - path: &Path, - seen: &mut HashSet, - case_sensitivity: FileSystemCaseSensitivity, - ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { - let relative_path = match path.strip_prefix(&self.root_path) { - Ok(p) => p, - Err(_) => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "path not found", - )); - } - }; - let mut final_path = self.root_path.clone(); - let mut current_entry = VfsEntryRef::Dir(&self.dir); - for component in relative_path.components() { - let component = component.as_os_str(); - let current_dir = match current_entry { - VfsEntryRef::Dir(dir) => { - final_path.push(component); - dir - } - VfsEntryRef::Symlink(symlink) => { - let dest = symlink.resolve_dest_from_root(&self.root_path); - let (resolved_path, entry) = - self.find_entry_inner(&dest, seen, case_sensitivity)?; - final_path = resolved_path; // overwrite with the new resolved path - match entry { - VfsEntryRef::Dir(dir) => { - final_path.push(component); - dir - } - _ => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "path not found", - )); - } - } - } - _ => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "path not found", - )); - } - }; - let component = component.to_string_lossy(); - current_entry = current_dir - .entries - .get_by_name(&component, case_sensitivity) - .ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::NotFound, "path not found") - })? - .as_ref(); - } - - Ok((final_path, current_entry)) - } -} - -pub struct FileBackedVfsFile { - file: VirtualFile, - pos: RefCell, - vfs: Arc, -} - -impl FileBackedVfsFile { - pub fn seek(&self, pos: SeekFrom) -> std::io::Result { - match pos { - SeekFrom::Start(pos) => { - *self.pos.borrow_mut() = pos; - Ok(pos) - } - SeekFrom::End(offset) => { - if offset < 0 && -offset as u64 > self.file.offset.len { - let msg = "An attempt was made to move the file pointer before the beginning of the file."; - Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - msg, - )) - } else { - let mut current_pos = self.pos.borrow_mut(); - *current_pos = if offset >= 0 { - self.file.offset.len - (offset as u64) - } else { - self.file.offset.len + (-offset as u64) - }; - Ok(*current_pos) - } - } - SeekFrom::Current(offset) => { - let mut current_pos = self.pos.borrow_mut(); - if offset >= 0 { - *current_pos += offset as u64; - } else if -offset as u64 > *current_pos { - return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.")); - } else { - *current_pos -= -offset as u64; - } - Ok(*current_pos) - } - } - } - - pub fn read_to_buf(&self, buf: &mut [u8]) -> std::io::Result { - let read_pos = { - let mut pos = self.pos.borrow_mut(); - let read_pos = *pos; - // advance the position due to the read - *pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64); - read_pos - }; - self.vfs.read_file(&self.file, read_pos, buf) - } - - fn read_to_end(&self) -> FsResult> { - let read_pos = { - let mut pos = self.pos.borrow_mut(); - let read_pos = *pos; - // todo(dsherret): should this always set it to the end of the file? - if *pos < self.file.offset.len { - // advance the position due to the read - *pos = self.file.offset.len; - } - read_pos - }; - if read_pos > self.file.offset.len { - return Ok(Cow::Borrowed(&[])); - } - if read_pos == 0 { - Ok( - self - .vfs - .read_file_all(&self.file, VfsFileSubDataKind::Raw)?, - ) - } else { - let size = (self.file.offset.len - read_pos) as usize; - let mut buf = vec![0; size]; - self.vfs.read_file(&self.file, read_pos, &mut buf)?; - Ok(Cow::Owned(buf)) - } - } -} - -#[async_trait::async_trait(?Send)] -impl deno_io::fs::File for FileBackedVfsFile { - fn read_sync(self: Rc, buf: &mut [u8]) -> FsResult { - self.read_to_buf(buf).map_err(Into::into) - } - async fn read_byob( - self: Rc, - mut buf: BufMutView, - ) -> FsResult<(usize, BufMutView)> { - // this is fast, no need to spawn a task - let nread = self.read_to_buf(&mut buf)?; - Ok((nread, buf)) - } - - fn write_sync(self: Rc, _buf: &[u8]) -> FsResult { - Err(FsError::NotSupported) - } - async fn write( - self: Rc, - _buf: BufView, - ) -> FsResult { - Err(FsError::NotSupported) - } - - fn write_all_sync(self: Rc, _buf: &[u8]) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn write_all(self: Rc, _buf: BufView) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn read_all_sync(self: Rc) -> FsResult> { - self.read_to_end() - } - async fn read_all_async(self: Rc) -> FsResult> { - // this is fast, no need to spawn a task - self.read_to_end() - } - - fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn chmod_async(self: Rc, _mode: u32) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn seek_sync(self: Rc, pos: SeekFrom) -> FsResult { - self.seek(pos).map_err(|err| err.into()) - } - async fn seek_async(self: Rc, pos: SeekFrom) -> FsResult { - self.seek(pos).map_err(|err| err.into()) - } - - fn datasync_sync(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn datasync_async(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn sync_sync(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn sync_async(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn stat_sync(self: Rc) -> FsResult { - Err(FsError::NotSupported) - } - async fn stat_async(self: Rc) -> FsResult { - Err(FsError::NotSupported) - } - - fn lock_sync(self: Rc, _exclusive: bool) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn lock_async(self: Rc, _exclusive: bool) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn unlock_sync(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn unlock_async(self: Rc) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn truncate_sync(self: Rc, _len: u64) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn truncate_async(self: Rc, _len: u64) -> FsResult<()> { - Err(FsError::NotSupported) - } - - fn utime_sync( - self: Rc, - _atime_secs: i64, - _atime_nanos: u32, - _mtime_secs: i64, - _mtime_nanos: u32, - ) -> FsResult<()> { - Err(FsError::NotSupported) - } - async fn utime_async( - self: Rc, - _atime_secs: i64, - _atime_nanos: u32, - _mtime_secs: i64, - _mtime_nanos: u32, - ) -> FsResult<()> { - Err(FsError::NotSupported) - } - - // lower level functionality - fn as_stdio(self: Rc) -> FsResult { - Err(FsError::NotSupported) - } - fn backing_fd(self: Rc) -> Option { - None - } - fn try_clone_inner(self: Rc) -> FsResult> { - Ok(self) - } -} - -#[derive(Debug, Clone)] -pub struct FileBackedVfsDirEntry { - pub parent_path: PathBuf, - pub metadata: FileBackedVfsMetadata, -} - -#[derive(Debug, Clone)] -pub struct FileBackedVfsMetadata { - pub name: String, - pub file_type: sys_traits::FileType, - pub len: u64, -} - -impl FileBackedVfsMetadata { - pub fn from_vfs_entry_ref(vfs_entry: VfsEntryRef) -> Self { - FileBackedVfsMetadata { - file_type: match vfs_entry { - VfsEntryRef::Dir(_) => sys_traits::FileType::Dir, - VfsEntryRef::File(_) => sys_traits::FileType::File, - VfsEntryRef::Symlink(_) => sys_traits::FileType::Symlink, - }, - name: vfs_entry.name().to_string(), - len: match vfs_entry { - VfsEntryRef::Dir(_) => 0, - VfsEntryRef::File(file) => file.offset.len, - VfsEntryRef::Symlink(_) => 0, - }, - } - } - pub fn as_fs_stat(&self) -> FsStat { - FsStat { - is_directory: self.file_type == sys_traits::FileType::Dir, - is_file: self.file_type == sys_traits::FileType::File, - is_symlink: self.file_type == sys_traits::FileType::Symlink, - atime: None, - birthtime: None, - mtime: None, - ctime: None, - blksize: 0, - size: self.len, - dev: 0, - ino: 0, - mode: 0, - nlink: 0, - uid: 0, - gid: 0, - rdev: 0, - blocks: 0, - is_block_device: false, - is_char_device: false, - is_fifo: false, - is_socket: false, - } - } -} - -#[derive(Debug)] -pub struct FileBackedVfs { - vfs_data: Cow<'static, [u8]>, - fs_root: VfsRoot, - case_sensitivity: FileSystemCaseSensitivity, -} - -impl FileBackedVfs { - pub fn new( - data: Cow<'static, [u8]>, - fs_root: VfsRoot, - case_sensitivity: FileSystemCaseSensitivity, - ) -> Self { - Self { - vfs_data: data, - fs_root, - case_sensitivity, - } - } - - pub fn root(&self) -> &Path { - &self.fs_root.root_path - } - - pub fn is_path_within(&self, path: &Path) -> bool { - path.starts_with(&self.fs_root.root_path) - } - - pub fn open_file( - self: &Arc, - path: &Path, - ) -> std::io::Result { - let file = self.file_entry(path)?; - Ok(FileBackedVfsFile { - file: file.clone(), - vfs: self.clone(), - pos: Default::default(), - }) - } - - pub fn read_dir(&self, path: &Path) -> std::io::Result> { - let dir = self.dir_entry(path)?; - Ok( - dir - .entries - .iter() - .map(|entry| FsDirEntry { - name: entry.name().to_string(), - is_file: matches!(entry, VfsEntry::File(_)), - is_directory: matches!(entry, VfsEntry::Dir(_)), - is_symlink: matches!(entry, VfsEntry::Symlink(_)), - }) - .collect(), - ) - } - - pub fn read_dir_with_metadata<'a>( - &'a self, - path: &Path, - ) -> std::io::Result + 'a> { - let dir = self.dir_entry(path)?; - let path = path.to_path_buf(); - Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry { - parent_path: path.to_path_buf(), - metadata: FileBackedVfsMetadata::from_vfs_entry_ref(entry.as_ref()), - })) - } - - pub fn read_link(&self, path: &Path) -> std::io::Result { - let (_, entry) = self - .fs_root - .find_entry_no_follow(path, self.case_sensitivity)?; - match entry { - VfsEntryRef::Symlink(symlink) => { - Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path)) - } - VfsEntryRef::Dir(_) | VfsEntryRef::File(_) => Err(std::io::Error::new( - std::io::ErrorKind::Other, - "not a symlink", - )), - } - } - - pub fn lstat(&self, path: &Path) -> std::io::Result { - let (_, entry) = self - .fs_root - .find_entry_no_follow(path, self.case_sensitivity)?; - Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) - } - - pub fn stat(&self, path: &Path) -> std::io::Result { - let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; - Ok(FileBackedVfsMetadata::from_vfs_entry_ref(entry)) - } - - pub fn canonicalize(&self, path: &Path) -> std::io::Result { - let (path, _) = self.fs_root.find_entry(path, self.case_sensitivity)?; - Ok(path) - } - - pub fn read_file_all( - &self, - file: &VirtualFile, - sub_data_kind: VfsFileSubDataKind, - ) -> std::io::Result> { - let read_len = match sub_data_kind { - VfsFileSubDataKind::Raw => file.offset.len, - VfsFileSubDataKind::ModuleGraph => file.module_graph_offset.len, - }; - let read_range = self.get_read_range(file, sub_data_kind, 0, read_len)?; - match &self.vfs_data { - Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), - Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), - } - } - - pub fn read_file( - &self, - file: &VirtualFile, - pos: u64, - buf: &mut [u8], - ) -> std::io::Result { - let read_range = self.get_read_range( - file, - VfsFileSubDataKind::Raw, - pos, - buf.len() as u64, - )?; - let read_len = read_range.len(); - buf[..read_len].copy_from_slice(&self.vfs_data[read_range]); - Ok(read_len) - } - - fn get_read_range( - &self, - file: &VirtualFile, - sub_data_kind: VfsFileSubDataKind, - pos: u64, - len: u64, - ) -> std::io::Result> { - let file_offset_and_len = match sub_data_kind { - VfsFileSubDataKind::Raw => file.offset, - VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, - }; - if pos > file_offset_and_len.len { - return Err(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "unexpected EOF", - )); - } - let file_offset = - self.fs_root.start_file_offset + file_offset_and_len.offset; - let start = file_offset + pos; - let end = file_offset + std::cmp::min(pos + len, file_offset_and_len.len); - Ok(start as usize..end as usize) - } - - pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> { - let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; - match entry { - VfsEntryRef::Dir(dir) => Ok(dir), - VfsEntryRef::Symlink(_) => unreachable!(), - VfsEntryRef::File(_) => Err(std::io::Error::new( - std::io::ErrorKind::Other, - "path is a file", - )), - } - } - - pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> { - let (_, entry) = self.fs_root.find_entry(path, self.case_sensitivity)?; - match entry { - VfsEntryRef::Dir(_) => Err(std::io::Error::new( - std::io::ErrorKind::Other, - "path is a directory", - )), - VfsEntryRef::Symlink(_) => unreachable!(), - VfsEntryRef::File(file) => Ok(file), - } - } -} - #[cfg(test)] mod test { - use std::io::Write; - use console_static_text::ansi::strip_ansi_codes; - use deno_io::fs::File; - use test_util::assert_contains; + use deno_lib::standalone::virtual_fs::VfsBuilder; use test_util::TempDir; use super::*; - #[track_caller] - fn read_file(vfs: &FileBackedVfs, path: &Path) -> String { - let file = vfs.file_entry(path).unwrap(); - String::from_utf8( - vfs - .read_file_all(file, VfsFileSubDataKind::Raw) - .unwrap() - .into_owned(), - ) - .unwrap() - } - - #[test] - fn builds_and_uses_virtual_fs() { - let temp_dir = TempDir::new(); - // we canonicalize the temp directory because the vfs builder - // will canonicalize the root path - let src_path = temp_dir.path().canonicalize().join("src"); - src_path.create_dir_all(); - src_path.join("sub_dir").create_dir_all(); - src_path.join("e.txt").write("e"); - src_path.symlink_file("e.txt", "sub_dir/e.txt"); - let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(); - builder - .add_file_with_data_inner( - &src_path.join("a.txt"), - "data".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_file_with_data_inner( - &src_path.join("b.txt"), - "data".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - assert_eq!(builder.files.len(), 1); // because duplicate data - builder - .add_file_with_data_inner( - &src_path.join("c.txt"), - "c".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_file_with_data_inner( - &src_path.join("sub_dir").join("d.txt"), - "d".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); - builder - .add_symlink(&src_path.join("sub_dir").join("e.txt")) - .unwrap(); - - // get the virtual fs - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - - assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data"); - assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data"); - - // attempt reading a symlink - assert_eq!( - read_file(&virtual_fs, &dest_path.join("sub_dir").join("e.txt")), - "e", - ); - - // canonicalize symlink - assert_eq!( - virtual_fs - .canonicalize(&dest_path.join("sub_dir").join("e.txt")) - .unwrap(), - dest_path.join("e.txt"), - ); - - // metadata - assert_eq!( - virtual_fs - .lstat(&dest_path.join("sub_dir").join("e.txt")) - .unwrap() - .file_type, - sys_traits::FileType::Symlink, - ); - assert_eq!( - virtual_fs - .stat(&dest_path.join("sub_dir").join("e.txt")) - .unwrap() - .file_type, - sys_traits::FileType::File, - ); - assert_eq!( - virtual_fs - .stat(&dest_path.join("sub_dir")) - .unwrap() - .file_type, - sys_traits::FileType::Dir, - ); - assert_eq!( - virtual_fs.stat(&dest_path.join("e.txt")).unwrap().file_type, - sys_traits::FileType::File - ); - } - - #[test] - fn test_include_dir_recursive() { - let temp_dir = TempDir::new(); - let temp_dir_path = temp_dir.path().canonicalize(); - temp_dir.create_dir_all("src/nested/sub_dir"); - temp_dir.write("src/a.txt", "data"); - temp_dir.write("src/b.txt", "data"); - util::fs::symlink_dir( - &crate::sys::CliSys::default(), - temp_dir_path.join("src/nested/sub_dir").as_path(), - temp_dir_path.join("src/sub_dir_link").as_path(), - ) - .unwrap(); - temp_dir.write("src/nested/sub_dir/c.txt", "c"); - - // build and create the virtual fs - let src_path = temp_dir_path.join("src").to_path_buf(); - let mut builder = VfsBuilder::new(); - builder.add_dir_recursive(&src_path).unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - - assert_eq!(read_file(&virtual_fs, &dest_path.join("a.txt")), "data",); - assert_eq!(read_file(&virtual_fs, &dest_path.join("b.txt")), "data",); - - assert_eq!( - read_file( - &virtual_fs, - &dest_path.join("nested").join("sub_dir").join("c.txt") - ), - "c", - ); - assert_eq!( - read_file(&virtual_fs, &dest_path.join("sub_dir_link").join("c.txt")), - "c", - ); - assert_eq!( - virtual_fs - .lstat(&dest_path.join("sub_dir_link")) - .unwrap() - .file_type, - sys_traits::FileType::Symlink, - ); - - assert_eq!( - virtual_fs - .canonicalize(&dest_path.join("sub_dir_link").join("c.txt")) - .unwrap(), - dest_path.join("nested").join("sub_dir").join("c.txt"), - ); - } - - fn into_virtual_fs( - builder: VfsBuilder, - temp_dir: &TempDir, - ) -> (PathBuf, FileBackedVfs) { - let virtual_fs_file = temp_dir.path().join("virtual_fs"); - let vfs = builder.build(); - { - let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); - for file_data in &vfs.files { - file.write_all(file_data).unwrap(); - } - } - let dest_path = temp_dir.path().join("dest"); - let data = std::fs::read(&virtual_fs_file).unwrap(); - ( - dest_path.to_path_buf(), - FileBackedVfs::new( - Cow::Owned(data), - VfsRoot { - dir: VirtualDirectory { - name: "".to_string(), - entries: vfs.entries, - }, - root_path: dest_path.to_path_buf(), - start_file_offset: 0, - }, - FileSystemCaseSensitivity::Sensitive, - ), - ) - } - - #[test] - fn circular_symlink() { - let temp_dir = TempDir::new(); - let src_path = temp_dir.path().canonicalize().join("src"); - src_path.create_dir_all(); - src_path.symlink_file("a.txt", "b.txt"); - src_path.symlink_file("b.txt", "c.txt"); - src_path.symlink_file("c.txt", "a.txt"); - let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(); - let err = builder - .add_symlink(src_path.join("a.txt").as_path()) - .unwrap_err(); - assert_contains!(err.to_string(), "Circular symlink detected",); - } - - #[tokio::test] - async fn test_open_file() { - let temp_dir = TempDir::new(); - let temp_path = temp_dir.path().canonicalize(); - let mut builder = VfsBuilder::new(); - builder - .add_file_with_data_inner( - temp_path.join("a.txt").as_path(), - "0123456789".to_string().into_bytes(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - let virtual_fs = Arc::new(virtual_fs); - let file = virtual_fs.open_file(&dest_path.join("a.txt")).unwrap(); - file.seek(SeekFrom::Current(2)).unwrap(); - let mut buf = vec![0; 2]; - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"23"); - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"45"); - file.seek(SeekFrom::Current(-4)).unwrap(); - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"23"); - file.seek(SeekFrom::Start(2)).unwrap(); - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"23"); - file.seek(SeekFrom::End(2)).unwrap(); - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"89"); - file.seek(SeekFrom::Current(-8)).unwrap(); - file.read_to_buf(&mut buf).unwrap(); - assert_eq!(buf, b"23"); - assert_eq!( - file - .seek(SeekFrom::Current(-5)) - .unwrap_err() - .to_string(), - "An attempt was made to move the file pointer before the beginning of the file." - ); - // go beyond the file length, then back - file.seek(SeekFrom::Current(40)).unwrap(); - file.seek(SeekFrom::Current(-38)).unwrap(); - let file = Rc::new(file); - let read_buf = file.clone().read(2).await.unwrap(); - assert_eq!(read_buf.to_vec(), b"67"); - file.clone().seek_sync(SeekFrom::Current(-2)).unwrap(); - - // read to the end of the file - let all_buf = file.clone().read_all_sync().unwrap(); - assert_eq!(all_buf.to_vec(), b"6789"); - file.clone().seek_sync(SeekFrom::Current(-9)).unwrap(); - - // try try_clone_inner and read_all_async - let all_buf = file - .try_clone_inner() - .unwrap() - .read_all_async() - .await - .unwrap(); - assert_eq!(all_buf.to_vec(), b"123456789"); - } - #[test] fn test_vfs_as_display_tree() { let temp_dir = TempDir::new(); diff --git a/cli/sys.rs b/cli/sys.rs deleted file mode 100644 index e551eab2e8..0000000000 --- a/cli/sys.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -// todo(dsherret): this should instead use conditional compilation and directly -// surface the underlying implementation. -// -// The problem atm is that there's no way to have conditional compilation for -// denort or the deno binary. We should extract out denort to a separate binary. - -use std::borrow::Cow; -use std::path::Path; -use std::path::PathBuf; - -use sys_traits::boxed::BoxedFsDirEntry; -use sys_traits::boxed::BoxedFsFile; -use sys_traits::boxed::BoxedFsMetadataValue; -use sys_traits::boxed::FsMetadataBoxed; -use sys_traits::boxed::FsOpenBoxed; -use sys_traits::boxed::FsReadDirBoxed; -use sys_traits::CreateDirOptions; - -use crate::standalone::DenoCompileFileSystem; - -#[derive(Debug, Clone)] -pub enum CliSys { - #[allow(dead_code)] // will be dead code for denort - #[allow(clippy::disallowed_types)] // ok because sys impl - Real(sys_traits::impls::RealSys), - #[allow(dead_code)] // will be dead code for deno - DenoCompile(DenoCompileFileSystem), -} - -impl deno_lib::sys::DenoLibSys for CliSys {} - -impl Default for CliSys { - fn default() -> Self { - Self::Real(sys_traits::impls::RealSys) - } -} - -impl deno_runtime::deno_node::ExtNodeSys for CliSys {} - -impl sys_traits::BaseFsCloneFile for CliSys { - fn base_fs_clone_file(&self, src: &Path, dst: &Path) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_clone_file(src, dst), - Self::DenoCompile(sys) => sys.base_fs_clone_file(src, dst), - } - } -} - -impl sys_traits::BaseFsSymlinkDir for CliSys { - fn base_fs_symlink_dir(&self, src: &Path, dst: &Path) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_symlink_dir(src, dst), - Self::DenoCompile(sys) => sys.base_fs_symlink_dir(src, dst), - } - } -} - -impl sys_traits::BaseFsCopy for CliSys { - fn base_fs_copy(&self, src: &Path, dst: &Path) -> std::io::Result { - match self { - Self::Real(sys) => sys.base_fs_copy(src, dst), - Self::DenoCompile(sys) => sys.base_fs_copy(src, dst), - } - } -} - -impl sys_traits::BaseFsHardLink for CliSys { - fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_hard_link(src, dst), - Self::DenoCompile(sys) => sys.base_fs_hard_link(src, dst), - } - } -} - -impl sys_traits::BaseFsRead for CliSys { - fn base_fs_read(&self, p: &Path) -> std::io::Result> { - match self { - Self::Real(sys) => sys.base_fs_read(p), - Self::DenoCompile(sys) => sys.base_fs_read(p), - } - } -} - -impl sys_traits::BaseFsReadDir for CliSys { - type ReadDirEntry = BoxedFsDirEntry; - - fn base_fs_read_dir( - &self, - p: &Path, - ) -> std::io::Result< - Box> + '_>, - > { - match self { - Self::Real(sys) => sys.fs_read_dir_boxed(p), - Self::DenoCompile(sys) => sys.fs_read_dir_boxed(p), - } - } -} - -impl sys_traits::BaseFsCanonicalize for CliSys { - fn base_fs_canonicalize(&self, p: &Path) -> std::io::Result { - match self { - Self::Real(sys) => sys.base_fs_canonicalize(p), - Self::DenoCompile(sys) => sys.base_fs_canonicalize(p), - } - } -} - -impl sys_traits::BaseFsMetadata for CliSys { - type Metadata = BoxedFsMetadataValue; - - fn base_fs_metadata(&self, path: &Path) -> std::io::Result { - match self { - Self::Real(sys) => sys.fs_metadata_boxed(path), - Self::DenoCompile(sys) => sys.fs_metadata_boxed(path), - } - } - - fn base_fs_symlink_metadata( - &self, - path: &Path, - ) -> std::io::Result { - match self { - Self::Real(sys) => sys.fs_symlink_metadata_boxed(path), - Self::DenoCompile(sys) => sys.fs_symlink_metadata_boxed(path), - } - } -} - -impl sys_traits::BaseFsCreateDir for CliSys { - fn base_fs_create_dir( - &self, - p: &Path, - options: &CreateDirOptions, - ) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_create_dir(p, options), - Self::DenoCompile(sys) => sys.base_fs_create_dir(p, options), - } - } -} - -impl sys_traits::BaseFsOpen for CliSys { - type File = BoxedFsFile; - - fn base_fs_open( - &self, - path: &Path, - options: &sys_traits::OpenOptions, - ) -> std::io::Result { - match self { - Self::Real(sys) => sys.fs_open_boxed(path, options), - Self::DenoCompile(sys) => sys.fs_open_boxed(path, options), - } - } -} - -impl sys_traits::BaseFsRemoveFile for CliSys { - fn base_fs_remove_file(&self, p: &Path) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_remove_file(p), - Self::DenoCompile(sys) => sys.base_fs_remove_file(p), - } - } -} - -impl sys_traits::BaseFsRename for CliSys { - fn base_fs_rename(&self, old: &Path, new: &Path) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.base_fs_rename(old, new), - Self::DenoCompile(sys) => sys.base_fs_rename(old, new), - } - } -} - -impl sys_traits::SystemRandom for CliSys { - fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> { - match self { - Self::Real(sys) => sys.sys_random(buf), - Self::DenoCompile(sys) => sys.sys_random(buf), - } - } -} - -impl sys_traits::SystemTimeNow for CliSys { - fn sys_time_now(&self) -> std::time::SystemTime { - match self { - Self::Real(sys) => sys.sys_time_now(), - Self::DenoCompile(sys) => sys.sys_time_now(), - } - } -} - -impl sys_traits::ThreadSleep for CliSys { - fn thread_sleep(&self, dur: std::time::Duration) { - match self { - Self::Real(sys) => sys.thread_sleep(dur), - Self::DenoCompile(sys) => sys.thread_sleep(dur), - } - } -} - -impl sys_traits::EnvCurrentDir for CliSys { - fn env_current_dir(&self) -> std::io::Result { - match self { - Self::Real(sys) => sys.env_current_dir(), - Self::DenoCompile(sys) => sys.env_current_dir(), - } - } -} - -impl sys_traits::BaseEnvVar for CliSys { - fn base_env_var_os( - &self, - key: &std::ffi::OsStr, - ) -> Option { - match self { - Self::Real(sys) => sys.base_env_var_os(key), - Self::DenoCompile(sys) => sys.base_env_var_os(key), - } - } -} - -impl sys_traits::EnvHomeDir for CliSys { - fn env_home_dir(&self) -> Option { - #[allow(clippy::disallowed_types)] // ok because sys impl - sys_traits::impls::RealSys.env_home_dir() - } -} diff --git a/cli/tools/bench/reporters.rs b/cli/tools/bench/reporters.rs index 68a0c56bce..c3df53b76a 100644 --- a/cli/tools/bench/reporters.rs +++ b/cli/tools/bench/reporters.rs @@ -1,10 +1,10 @@ // Copyright 2018-2025 the Deno authors. MIT license. +use deno_lib::version::DENO_VERSION_INFO; use serde::Serialize; use super::*; use crate::tools::test::TestFailureFormatOptions; -use crate::version; pub trait BenchReporter { fn report_group_summary(&mut self); @@ -31,11 +31,7 @@ impl Default for JsonReporterOutput { fn default() -> Self { Self { version: JSON_SCHEMA_VERSION, - runtime: format!( - "{} {}", - version::DENO_VERSION_INFO.user_agent, - env!("TARGET") - ), + runtime: format!("{} {}", DENO_VERSION_INFO.user_agent, env!("TARGET")), cpu: mitata::cpu::name(), benches: vec![], } @@ -163,7 +159,7 @@ impl BenchReporter for ConsoleReporter { "{}\n", colors::gray(format!( "Runtime | Deno {} ({})", - crate::version::DENO_VERSION_INFO.deno, + DENO_VERSION_INFO.deno, env!("TARGET") )) ); diff --git a/cli/tools/check.rs b/cli/tools/check.rs index e850b1900f..d8128432b3 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -13,6 +13,7 @@ use deno_graph::Module; use deno_graph::ModuleError; use deno_graph::ModuleGraph; use deno_graph::ModuleLoadError; +use deno_lib::util::hash::FastInsecureHasher; use deno_semver::npm::NpmPackageNvReference; use deno_terminal::colors; use once_cell::sync::Lazy; @@ -28,7 +29,6 @@ use crate::args::TsTypeLib; use crate::args::TypeCheckMode; use crate::cache::CacheDBHash; use crate::cache::Caches; -use crate::cache::FastInsecureHasher; use crate::cache::TypeCheckCache; use crate::factory::CliFactory; use crate::graph_util::maybe_additional_sloppy_imports_message; diff --git a/cli/tools/clean.rs b/cli/tools/clean.rs index a550d2826a..e6f8c1e52b 100644 --- a/cli/tools/clean.rs +++ b/cli/tools/clean.rs @@ -4,8 +4,8 @@ use std::path::Path; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_lib::cache::DenoDir; +use crate::cache::DenoDir; use crate::colors; use crate::display; use crate::sys::CliSys; diff --git a/cli/tools/coverage/reporter.rs b/cli/tools/coverage/reporter.rs index bc6e85b47e..4f0325117c 100644 --- a/cli/tools/coverage/reporter.rs +++ b/cli/tools/coverage/reporter.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use deno_core::error::AnyError; use deno_core::url::Url; +use deno_lib::version::DENO_VERSION_INFO; use super::util; use super::CoverageReport; @@ -559,7 +560,7 @@ impl HtmlCoverageReporter { /// Creates footer part of the contents for html report. pub fn create_html_footer(&self, now: &str) -> String { - let version = env!("CARGO_PKG_VERSION"); + let version = DENO_VERSION_INFO.deno; format!( "
diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 114c8f958a..ebb90c08e9 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -20,6 +20,7 @@ use deno_graph::EsParser; use deno_graph::GraphKind; use deno_graph::ModuleAnalyzer; use deno_graph::ModuleSpecifier; +use deno_lib::version::DENO_VERSION_INFO; use doc::html::ShortPath; use doc::DocDiagnostic; use indexmap::IndexMap; @@ -280,7 +281,7 @@ impl deno_doc::html::HrefResolver for DocResolver { if self.deno_ns.contains_key(symbol) { Some(format!( "https://deno.land/api@v{}?s={}", - env!("CARGO_PKG_VERSION"), + DENO_VERSION_INFO.deno, symbol.join(".") )) } else { diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index 596d087589..509158779a 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -18,6 +18,7 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_core::url::Url; +use deno_lib::args::CaData; use deno_semver::npm::NpmPackageReqReference; use log::Level; use once_cell::sync::Lazy; @@ -26,7 +27,6 @@ use regex::RegexBuilder; use crate::args::resolve_no_prompt; use crate::args::AddFlags; -use crate::args::CaData; use crate::args::ConfigFlag; use crate::args::Flags; use crate::args::InstallFlags; @@ -657,6 +657,7 @@ fn is_in_path(dir: &Path) -> bool { mod tests { use std::process::Command; + use deno_lib::args::UnstableConfig; use test_util::testdata_path; use test_util::TempDir; @@ -664,7 +665,6 @@ mod tests { use crate::args::ConfigFlag; use crate::args::PermissionFlags; use crate::args::UninstallFlagsGlobal; - use crate::args::UnstableConfig; use crate::util::fs::canonicalize_path; #[tokio::test] diff --git a/cli/tools/jupyter/server.rs b/cli/tools/jupyter/server.rs index bc045d9d9b..d7d55c255a 100644 --- a/cli/tools/jupyter/server.rs +++ b/cli/tools/jupyter/server.rs @@ -18,6 +18,7 @@ use deno_core::parking_lot::Mutex; use deno_core::serde_json; use deno_core::CancelFuture; use deno_core::CancelHandle; +use deno_lib::version::DENO_VERSION_INFO; use jupyter_runtime::messaging; use jupyter_runtime::ConnectionInfo; use jupyter_runtime::ExecutionCount; @@ -679,10 +680,10 @@ fn kernel_info() -> messaging::KernelInfoReply { status: ReplyStatus::Ok, protocol_version: "5.3".to_string(), implementation: "Deno kernel".to_string(), - implementation_version: crate::version::DENO_VERSION_INFO.deno.to_string(), + implementation_version: DENO_VERSION_INFO.deno.to_string(), language_info: messaging::LanguageInfo { name: "typescript".to_string(), - version: crate::version::DENO_VERSION_INFO.typescript.to_string(), + version: DENO_VERSION_INFO.typescript.to_string(), mimetype: "text/x.typescript".to_string(), file_extension: ".ts".to_string(), pygments_lexer: "typescript".to_string(), diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index 05e2f320eb..6a42688aca 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -8,6 +8,7 @@ use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::serde_json; use deno_core::unsync::spawn_blocking; +use deno_lib::version::DENO_VERSION_INFO; use deno_runtime::WorkerExecutionMode; use rustyline::error::ReadlineError; @@ -244,7 +245,7 @@ pub async fn run( if !cli_options.is_quiet() { let mut handle = io::stdout().lock(); - writeln!(handle, "Deno {}", crate::version::DENO_VERSION_INFO.deno)?; + writeln!(handle, "Deno {}", DENO_VERSION_INFO.deno)?; writeln!(handle, "exit using ctrl+d, ctrl+c, or close()")?; if repl_flags.is_default_command { diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index 5ea4523c48..2d969123ab 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -32,6 +32,7 @@ use deno_error::JsErrorBox; use deno_graph::Position; use deno_graph::PositionRange; use deno_graph::SpecifierWithRange; +use deno_lib::util::result::any_and_jserrorbox_downcast_ref; use deno_runtime::worker::MainWorker; use deno_semver::npm::NpmPackageReqReference; use node_resolver::NodeResolutionKind; @@ -402,18 +403,16 @@ impl ReplSession { } Err(err) => { // handle a parsing diagnostic - match crate::util::result::any_and_jserrorbox_downcast_ref::< - deno_ast::ParseDiagnostic, - >(&err) - { + match any_and_jserrorbox_downcast_ref::( + &err, + ) { Some(diagnostic) => { Ok(EvaluationOutput::Error(format_diagnostic(diagnostic))) } None => { - match crate::util::result::any_and_jserrorbox_downcast_ref::< - ParseDiagnosticsError, - >(&err) - { + match any_and_jserrorbox_downcast_ref::( + &err, + ) { Some(diagnostics) => Ok(EvaluationOutput::Error( diagnostics .0 diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index 521c3217fe..7149c152d9 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -19,6 +19,8 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::unsync::spawn; use deno_core::url::Url; +use deno_lib::shared::ReleaseChannel; +use deno_lib::version; use deno_semver::SmallStackString; use deno_semver::Version; use once_cell::sync::Lazy; @@ -30,11 +32,9 @@ use crate::colors; use crate::factory::CliFactory; use crate::http_util::HttpClient; use crate::http_util::HttpClientProvider; -use crate::shared::ReleaseChannel; use crate::util::archive; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; -use crate::version; const RELEASE_URL: &str = "https://github.com/denoland/deno/releases"; const CANARY_URL: &str = "https://dl.deno.land/canary"; diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 1b76b640d3..868610cb40 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -29,6 +29,7 @@ use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; use deno_lib::util::checksum; +use deno_lib::util::hash::FastInsecureHasher; use deno_lib::worker::create_isolate_create_params; use deno_resolver::npm::managed::ResolvePkgFolderFromDenoModuleError; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; @@ -44,7 +45,6 @@ use thiserror::Error; use crate::args::TsConfig; use crate::args::TypeCheckMode; -use crate::cache::FastInsecureHasher; use crate::cache::ModuleInfoCache; use crate::node::CliNodeResolver; use crate::npm::CliNpmResolver; diff --git a/cli/util/file_watcher.rs b/cli/util/file_watcher.rs index d3ff1bae77..20de748b4a 100644 --- a/cli/util/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -14,6 +14,7 @@ use deno_core::error::CoreError; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::parking_lot::Mutex; +use deno_lib::util::result::any_and_jserrorbox_downcast_ref; use deno_runtime::fmt_errors::format_js_error; use log::info; use notify::event::Event as NotifyEvent; @@ -82,13 +83,11 @@ where { let result = watch_future.await; if let Err(err) = result { - let error_string = - match crate::util::result::any_and_jserrorbox_downcast_ref::( - &err, - ) { - Some(CoreError::Js(e)) => format_js_error(e), - _ => format!("{err:?}"), - }; + let error_string = match any_and_jserrorbox_downcast_ref::(&err) + { + Some(CoreError::Js(e)) => format_js_error(e), + _ => format!("{err:?}"), + }; log::error!( "{}: {}", colors::red_bold("error"), diff --git a/cli/util/mod.rs b/cli/util/mod.rs index 702e5673c9..13c2eee5cb 100644 --- a/cli/util/mod.rs +++ b/cli/util/mod.rs @@ -9,10 +9,8 @@ pub mod draw_thread; pub mod extract; pub mod file_watcher; pub mod fs; -pub mod logger; pub mod path; pub mod progress_bar; -pub mod result; pub mod retry; pub mod sync; pub mod text_encoding; diff --git a/cli/util/text_encoding.rs b/cli/util/text_encoding.rs index b5a288be7b..a984e5a9f5 100644 --- a/cli/util/text_encoding.rs +++ b/cli/util/text_encoding.rs @@ -1,8 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::borrow::Cow; use std::ops::Range; -use std::sync::Arc; use base64::prelude::BASE64_STANDARD; use base64::Engine; @@ -11,24 +9,6 @@ use deno_core::ModuleSourceCode; static SOURCE_MAP_PREFIX: &[u8] = b"//# sourceMappingURL=data:application/json;base64,"; -#[inline(always)] -pub fn from_utf8_lossy_cow(bytes: Cow<[u8]>) -> Cow { - match bytes { - Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), - Cow::Owned(bytes) => Cow::Owned(from_utf8_lossy_owned(bytes)), - } -} - -#[inline(always)] -pub fn from_utf8_lossy_owned(bytes: Vec) -> String { - match String::from_utf8_lossy(&bytes) { - Cow::Owned(code) => code, - // SAFETY: `String::from_utf8_lossy` guarantees that the result is valid - // UTF-8 if `Cow::Borrowed` is returned. - Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(bytes) }, - } -} - pub fn source_map_from_code(code: &[u8]) -> Option> { let range = find_source_map_range(code)?; let source_map_range = &code[range]; @@ -105,29 +85,6 @@ fn find_source_map_range(code: &[u8]) -> Option> { } } -/// Converts an `Arc` to an `Arc<[u8]>`. -#[allow(dead_code)] -pub fn arc_str_to_bytes(arc_str: Arc) -> Arc<[u8]> { - let raw = Arc::into_raw(arc_str); - // SAFETY: This is safe because they have the same memory layout. - unsafe { Arc::from_raw(raw as *const [u8]) } -} - -/// Converts an `Arc` to an `Arc` if able. -#[allow(dead_code)] -pub fn arc_u8_to_arc_str( - arc_u8: Arc<[u8]>, -) -> Result, std::str::Utf8Error> { - // Check that the string is valid UTF-8. - std::str::from_utf8(&arc_u8)?; - // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as - // Arc. This is proven by the From> impl for Arc<[u8]> from the - // standard library. - Ok(unsafe { - std::mem::transmute::, std::sync::Arc>(arc_u8) - }) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/cli/util/v8.rs b/cli/util/v8.rs index 403d1955a5..33dd2a206a 100644 --- a/cli/util/v8.rs +++ b/cli/util/v8.rs @@ -1,5 +1,7 @@ // Copyright 2018-2025 the Deno authors. MIT license. +use deno_lib::util::v8::construct_v8_flags; + pub mod convert; #[inline(always)] @@ -10,19 +12,6 @@ pub fn get_v8_flags_from_env() -> Vec { .unwrap_or_default() } -#[inline(always)] -pub fn construct_v8_flags( - default_v8_flags: &[String], - v8_flags: &[String], - env_v8_flags: Vec, -) -> Vec { - std::iter::once("UNUSED_BUT_NECESSARY_ARG0".to_owned()) - .chain(default_v8_flags.iter().cloned()) - .chain(env_v8_flags) - .chain(v8_flags.iter().cloned()) - .collect::>() -} - pub fn init_v8_flags( default_v8_flags: &[String], v8_flags: &[String], diff --git a/cli/version.rs b/cli/version.rs deleted file mode 100644 index fb0fe98b29..0000000000 --- a/cli/version.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2018-2025 the Deno authors. MIT license. - -use once_cell::sync::Lazy; - -use crate::shared::ReleaseChannel; - -const GIT_COMMIT_HASH: &str = env!("GIT_COMMIT_HASH"); -const TYPESCRIPT: &str = env!("TS_VERSION"); -const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); -// TODO(bartlomieju): ideally we could remove this const. -const IS_CANARY: bool = option_env!("DENO_CANARY").is_some(); -// TODO(bartlomieju): this is temporary, to allow Homebrew to cut RC releases as well -const IS_RC: bool = option_env!("DENO_RC").is_some(); - -pub static DENO_VERSION_INFO: Lazy = Lazy::new(|| { - let release_channel = libsui::find_section("denover") - .and_then(|buf| std::str::from_utf8(buf).ok()) - .and_then(|str_| ReleaseChannel::deserialize(str_).ok()) - .unwrap_or({ - if IS_CANARY { - ReleaseChannel::Canary - } else if IS_RC { - ReleaseChannel::Rc - } else { - ReleaseChannel::Stable - } - }); - - DenoVersionInfo { - deno: if release_channel == ReleaseChannel::Canary { - concat!( - env!("CARGO_PKG_VERSION"), - "+", - env!("GIT_COMMIT_HASH_SHORT") - ) - } else { - env!("CARGO_PKG_VERSION") - }, - - release_channel, - - git_hash: GIT_COMMIT_HASH, - - // Keep in sync with `deno` field. - user_agent: if release_channel == ReleaseChannel::Canary { - concat!( - "Deno/", - env!("CARGO_PKG_VERSION"), - "+", - env!("GIT_COMMIT_HASH_SHORT") - ) - } else { - concat!("Deno/", env!("CARGO_PKG_VERSION")) - }, - - typescript: TYPESCRIPT, - } -}); - -pub struct DenoVersionInfo { - /// Human-readable version of the current Deno binary. - /// - /// For stable release, a semver, eg. `v1.46.2`. - /// For canary release, a semver + 7-char git hash, eg. `v1.46.3+asdfqwq`. - pub deno: &'static str, - - pub release_channel: ReleaseChannel, - - /// A full git hash. - pub git_hash: &'static str, - - /// A user-agent header that will be used in HTTP client. - pub user_agent: &'static str, - - pub typescript: &'static str, -} - -impl DenoVersionInfo { - /// For stable release, a semver like, eg. `v1.46.2`. - /// For canary release a full git hash, eg. `9bdab6fb6b93eb43b1930f40987fa4997287f9c8`. - pub fn version_or_git_hash(&self) -> &'static str { - if self.release_channel == ReleaseChannel::Canary { - self.git_hash - } else { - CARGO_PKG_VERSION - } - } -} diff --git a/cli/worker.rs b/cli/worker.rs index 3a11f56c71..b83bf9620d 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -1,6 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::path::Path; use std::sync::Arc; use deno_ast::ModuleSpecifier; @@ -12,20 +11,16 @@ use deno_core::PollEventLoopOptions; use deno_error::JsErrorBox; use deno_lib::worker::LibMainWorker; use deno_lib::worker::LibMainWorkerFactory; -use deno_runtime::code_cache; +use deno_lib::worker::ResolveNpmBinaryEntrypointError; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::worker::MainWorker; use deno_runtime::WorkerExecutionMode; use deno_semver::npm::NpmPackageReqReference; -use node_resolver::errors::ResolvePkgJsonBinExportError; -use node_resolver::NodeResolutionKind; -use node_resolver::ResolutionMode; use sys_traits::EnvCurrentDir; use tokio::select; use crate::args::CliLockfile; use crate::args::NpmCachingStrategy; -use crate::node::CliNodeResolver; use crate::npm::installer::NpmInstaller; use crate::npm::installer::PackageCaching; use crate::npm::CliNpmResolver; @@ -40,15 +35,6 @@ pub trait HmrRunner: Send + Sync { async fn run(&mut self) -> Result<(), CoreError>; } -pub trait CliCodeCache: code_cache::CodeCache { - /// Gets if the code cache is still enabled. - fn enabled(&self) -> bool { - true - } - - fn as_code_cache(self: Arc) -> Arc; -} - #[async_trait::async_trait(?Send)] pub trait CoverageCollector: Send + Sync { async fn start_collecting(&mut self) -> Result<(), CoreError>; @@ -102,6 +88,8 @@ impl CliMainWorker { log::debug!("main_module {}", self.worker.main_module()); + // WARNING: Remember to update cli/lib/worker.rs to align with + // changes made here so that they affect deno_compile as well. self.execute_main_module().await?; self.worker.dispatch_load_event()?; @@ -295,29 +283,6 @@ impl CliMainWorker { } } -#[derive(Debug, thiserror::Error, deno_error::JsError)] -pub enum ResolveBinaryEntrypointError { - #[class(inherit)] - #[error(transparent)] - ResolvePkgJsonBinExport(ResolvePkgJsonBinExportError), - #[class(generic)] - #[error("{original:#}\n\nFallback failed: {fallback:#}")] - Fallback { - fallback: ResolveBinaryEntrypointFallbackError, - original: ResolvePkgJsonBinExportError, - }, -} - -#[derive(Debug, thiserror::Error, deno_error::JsError)] -pub enum ResolveBinaryEntrypointFallbackError { - #[class(inherit)] - #[error(transparent)] - PackageSubpathResolve(node_resolver::errors::PackageSubpathResolveError), - #[class(generic)] - #[error("Cannot find module '{0}'")] - ModuleNotFound(ModuleSpecifier), -} - #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum CreateCustomWorkerError { #[class(inherit)] @@ -336,7 +301,7 @@ pub enum CreateCustomWorkerError { UrlParse(#[from] deno_core::url::ParseError), #[class(inherit)] #[error(transparent)] - ResolveBinaryEntrypoint(#[from] ResolveBinaryEntrypointError), + ResolveNpmBinaryEntrypoint(#[from] ResolveNpmBinaryEntrypointError), #[class(inherit)] #[error(transparent)] NpmPackageReq(JsErrorBox), @@ -350,7 +315,6 @@ pub enum CreateCustomWorkerError { pub struct CliMainWorkerFactory { lib_main_worker_factory: LibMainWorkerFactory, maybe_lockfile: Option>, - node_resolver: Arc, npm_installer: Option>, npm_resolver: CliNpmResolver, root_permissions: PermissionsContainer, @@ -366,7 +330,6 @@ impl CliMainWorkerFactory { lib_main_worker_factory: LibMainWorkerFactory, maybe_file_watcher_communicator: Option>, maybe_lockfile: Option>, - node_resolver: Arc, npm_installer: Option>, npm_resolver: CliNpmResolver, sys: CliSys, @@ -376,7 +339,6 @@ impl CliMainWorkerFactory { Self { lib_main_worker_factory, maybe_lockfile, - node_resolver, npm_installer, npm_resolver, root_permissions, @@ -446,8 +408,11 @@ impl CliMainWorkerFactory { package_ref.req(), &referrer, )?; - let main_module = self - .resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?; + let main_module = + self.lib_main_worker_factory.resolve_npm_binary_entrypoint( + &package_folder, + package_ref.sub_path(), + )?; if let Some(lockfile) = &self.maybe_lockfile { // For npm binary commands, ensure that the lockfile gets updated @@ -494,71 +459,6 @@ impl CliMainWorkerFactory { shared: self.shared.clone(), }) } - - fn resolve_binary_entrypoint( - &self, - package_folder: &Path, - sub_path: Option<&str>, - ) -> Result { - match self - .node_resolver - .resolve_binary_export(package_folder, sub_path) - { - Ok(specifier) => Ok(specifier), - Err(original_err) => { - // if the binary entrypoint was not found, fallback to regular node resolution - let result = - self.resolve_binary_entrypoint_fallback(package_folder, sub_path); - match result { - Ok(Some(specifier)) => Ok(specifier), - Ok(None) => Err( - ResolveBinaryEntrypointError::ResolvePkgJsonBinExport(original_err), - ), - Err(fallback_err) => Err(ResolveBinaryEntrypointError::Fallback { - original: original_err, - fallback: fallback_err, - }), - } - } - } - } - - /// resolve the binary entrypoint using regular node resolution - fn resolve_binary_entrypoint_fallback( - &self, - package_folder: &Path, - sub_path: Option<&str>, - ) -> Result, ResolveBinaryEntrypointFallbackError> { - // only fallback if the user specified a sub path - if sub_path.is_none() { - // it's confusing to users if the package doesn't have any binary - // entrypoint and we just execute the main script which will likely - // have blank output, so do not resolve the entrypoint in this case - return Ok(None); - } - - let specifier = self - .node_resolver - .resolve_package_subpath_from_deno_module( - package_folder, - sub_path, - /* referrer */ None, - ResolutionMode::Import, - NodeResolutionKind::Execution, - ) - .map_err(ResolveBinaryEntrypointFallbackError::PackageSubpathResolve)?; - if specifier - .to_file_path() - .map(|p| p.exists()) - .unwrap_or(false) - { - Ok(Some(specifier)) - } else { - Err(ResolveBinaryEntrypointFallbackError::ModuleNotFound( - specifier, - )) - } - } } #[allow(clippy::print_stdout)] diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index d4152ce58c..d37bbb620e 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -33,7 +33,6 @@ deno_error.workspace = true deno_fetch.workspace = true deno_fs.workspace = true deno_io.workspace = true -deno_media_type.workspace = true deno_net.workspace = true deno_package_json.workspace = true deno_path_util.workspace = true @@ -47,7 +46,6 @@ ecb.workspace = true ecdsa = "0.16.9" ed25519-dalek = { version = "2.1.1", features = ["digest", "pkcs8", "rand_core", "signature"] } elliptic-curve.workspace = true -errno = "0.2.8" faster-hex.workspace = true h2.workspace = true hkdf.workspace = true @@ -104,6 +102,9 @@ x25519-dalek = { version = "2.0.0", features = ["static_secrets"] } x509-parser = "0.15.0" yoke.workspace = true +[target.'cfg(unix)'.dependencies] +errno = "0.3.8" + [target.'cfg(windows)'.dependencies] windows-sys.workspace = true winapi = { workspace = true, features = ["consoleapi"] } diff --git a/ext/os/Cargo.toml b/ext/os/Cargo.toml index bc809f3485..fd065ee6cc 100644 --- a/ext/os/Cargo.toml +++ b/ext/os/Cargo.toml @@ -23,11 +23,13 @@ libc.workspace = true netif = "0.1.6" once_cell.workspace = true serde.workspace = true -signal-hook = "0.3.17" -signal-hook-registry = "1.4.0" thiserror.workspace = true tokio.workspace = true [target.'cfg(windows)'.dependencies] winapi = { workspace = true, features = ["commapi", "knownfolders", "mswsock", "objbase", "psapi", "shlobj", "tlhelp32", "winbase", "winerror", "winuser", "winsock2"] } ntapi = "0.4.0" + +[target.'cfg(unix)'.dependencies] +signal-hook = "0.3.17" +signal-hook-registry = "1.4.0" diff --git a/ext/webgpu/lib.rs b/ext/webgpu/lib.rs index 4535c0296b..796a6282e8 100644 --- a/ext/webgpu/lib.rs +++ b/ext/webgpu/lib.rs @@ -39,6 +39,25 @@ mod webidl; pub const UNSTABLE_FEATURE_NAME: &str = "webgpu"; +#[allow(clippy::print_stdout)] +pub fn print_linker_flags(name: &str) { + if cfg!(windows) { + // these dls load slowly, so delay loading them + let dlls = [ + // webgpu + "d3dcompiler_47", + "OPENGL32", + // network related functions + "iphlpapi", + ]; + for dll in dlls { + println!("cargo:rustc-link-arg-bin={name}=/delayload:{dll}.dll"); + } + // enable delay loading + println!("cargo:rustc-link-arg-bin={name}=delayimp.lib"); + } +} + pub type Instance = Arc; deno_core::extension!( diff --git a/resolvers/deno/cjs.rs b/resolvers/deno/cjs.rs index 4358b0ced2..f3347fc9a0 100644 --- a/resolvers/deno/cjs.rs +++ b/resolvers/deno/cjs.rs @@ -50,6 +50,26 @@ impl self.treat_as_cjs_with_is_script(specifier, media_type, None) } + /// Mark a file as being known CJS or ESM. + pub fn set_is_known_script(&self, specifier: &Url, is_script: bool) { + let new_value = if is_script { + ResolutionMode::Require + } else { + ResolutionMode::Import + }; + // block to really ensure dashmap is not borrowed while trying to insert + { + if let Some(value) = self.known.get(specifier) { + // you shouldn't be insert a value in here that's + // already known and is a different value than what + // was previously determined + debug_assert_eq!(*value, new_value); + return; + } + } + self.known.insert(specifier.clone(), new_value); + } + /// Gets whether the file is CJS. If true, this is for sure /// cjs because `is_script` is provided. /// @@ -233,7 +253,7 @@ impl } } else if is_script == Some(false) { // we know this is esm - known_cache.insert(specifier.clone(), ResolutionMode::Import); + known_cache.insert(specifier.clone(), ResolutionMode::Import); Some(ResolutionMode::Import) } else { None diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index ee3a783860..e44303a8eb 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -21,7 +21,6 @@ anyhow.workspace = true async-trait.workspace = true boxed_error.workspace = true deno_error.workspace = true -deno_media_type.workspace = true deno_package_json.workspace = true deno_path_util.workspace = true futures.workspace = true @@ -29,6 +28,7 @@ lazy-regex.workspace = true once_cell.workspace = true path-clean = "=0.1.0" regex.workspace = true +serde.workspace = true serde_json.workspace = true sys_traits.workspace = true thiserror.workspace = true diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index a67023d1e1..b5fc224037 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -14,6 +14,8 @@ use futures::stream::FuturesUnordered; use futures::FutureExt; use futures::StreamExt; use once_cell::sync::Lazy; +use serde::Deserialize; +use serde::Serialize; use sys_traits::FsCanonicalize; use sys_traits::FsMetadata; use sys_traits::FsRead; @@ -36,22 +38,12 @@ pub enum CjsAnalysis<'a> { Cjs(CjsAnalysisExports), } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CjsAnalysisExports { pub exports: Vec, pub reexports: Vec, } -#[derive(Debug, thiserror::Error, deno_error::JsError)] -pub enum CjsCodeAnalysisError { - #[class(inherit)] - #[error(transparent)] - ClosestPkgJson(#[from] crate::errors::ClosestPkgJsonError), - #[class(inherit)] - #[error(transparent)] - Other(#[from] JsErrorBox), -} - /// Code analyzer for CJS and ESM files. #[async_trait::async_trait(?Send)] pub trait CjsCodeAnalyzer { @@ -66,17 +58,17 @@ pub trait CjsCodeAnalyzer { &self, specifier: &Url, maybe_source: Option>, - ) -> Result, CjsCodeAnalysisError>; + ) -> Result, JsErrorBox>; } #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum TranslateCjsToEsmError { #[class(inherit)] #[error(transparent)] - CjsCodeAnalysis(#[from] CjsCodeAnalysisError), + CjsCodeAnalysis(JsErrorBox), #[class(inherit)] #[error(transparent)] - ExportAnalysis(#[from] JsErrorBox), + ExportAnalysis(JsErrorBox), } #[derive(Debug, thiserror::Error, deno_error::JsError)] @@ -87,7 +79,7 @@ pub struct CjsAnalysisCouldNotLoadError { reexport_specifier: Url, referrer: Url, #[source] - source: CjsCodeAnalysisError, + source: JsErrorBox, } pub struct NodeCodeTranslator< @@ -164,7 +156,8 @@ impl< let analysis = self .cjs_code_analyzer .analyze_cjs(entry_specifier, source) - .await?; + .await + .map_err(TranslateCjsToEsmError::CjsCodeAnalysis)?; let analysis = match analysis { CjsAnalysis::Esm(source) => return Ok(source), diff --git a/resolvers/npm_cache/Cargo.toml b/resolvers/npm_cache/Cargo.toml index 4c6cca2416..69679689eb 100644 --- a/resolvers/npm_cache/Cargo.toml +++ b/resolvers/npm_cache/Cargo.toml @@ -35,6 +35,8 @@ ring.workspace = true serde_json.workspace = true sys_traits.workspace = true tar.workspace = true -tempfile = "3.4.0" thiserror.workspace = true url.workspace = true + +[dev-dependencies] +tempfile = "3.4.0" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index b87d4cfbdf..83ed3da495 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -27,6 +27,7 @@ hmr = ["include_js_files_for_snapshotting"] # conditionally exclude the runtime source transpilation logic, and add an # assertion that a snapshot is provided. only_snapshotted_js_sources = ["include_js_files_for_snapshotting"] +snapshotting = ["deno_ast"] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } @@ -39,41 +40,8 @@ path = "lib.rs" name = "extension" path = "examples/extension/main.rs" -[build-dependencies] -deno_ast.workspace = true -deno_broadcast_channel.workspace = true -deno_cache.workspace = true -deno_canvas.workspace = true -deno_console.workspace = true -deno_core.workspace = true -deno_cron.workspace = true -deno_crypto.workspace = true -deno_fetch.workspace = true -deno_os.workspace = true -deno_ffi.workspace = true -deno_fs = { workspace = true, features = ["sync_fs"] } -deno_http.workspace = true -deno_io.workspace = true -deno_net.workspace = true -deno_node.workspace = true -deno_kv.workspace = true -deno_tls.workspace = true -deno_url.workspace = true -deno_web.workspace = true -deno_process.workspace = true -deno_webgpu.workspace = true -deno_webidl.workspace = true -deno_websocket.workspace = true -deno_webstorage.workspace = true -deno_napi.workspace = true -flate2 = { workspace = true, features = ["default"] } -serde.workspace = true - -[target.'cfg(windows)'.build-dependencies] -winapi.workspace = true - [dependencies] -deno_ast.workspace = true +deno_ast = { workspace = true, optional = true } deno_broadcast_channel.workspace = true deno_cache.workspace = true deno_canvas.workspace = true diff --git a/runtime/lib.rs b/runtime/lib.rs index c83fe5d60b..a2f3e54353 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -36,6 +36,7 @@ pub mod inspector_server; pub mod js; pub mod ops; pub mod permissions; +#[cfg(feature = "snapshotting")] pub mod snapshot; pub mod tokio_util; pub mod web_worker; diff --git a/runtime/shared.rs b/runtime/shared.rs index ecf2088fe1..f712cef65b 100644 --- a/runtime/shared.rs +++ b/runtime/shared.rs @@ -1,17 +1,8 @@ // Copyright 2018-2025 the Deno authors. MIT license. // Utilities shared between `build.rs` and the rest of the crate. -use std::path::Path; - -use deno_ast::MediaType; -use deno_ast::ParseParams; -use deno_ast::SourceMapOption; use deno_core::extension; use deno_core::Extension; -use deno_core::ModuleCodeString; -use deno_core::ModuleName; -use deno_core::SourceMapData; -use deno_error::JsErrorBox; extension!(runtime, deps = [ @@ -60,70 +51,3 @@ extension!(runtime, } } ); - -deno_error::js_error_wrapper!( - deno_ast::ParseDiagnostic, - JsParseDiagnostic, - "Error" -); -deno_error::js_error_wrapper!( - deno_ast::TranspileError, - JsTranspileError, - "Error" -); - -pub fn maybe_transpile_source( - name: ModuleName, - source: ModuleCodeString, -) -> Result<(ModuleCodeString, Option), JsErrorBox> { - // Always transpile `node:` built-in modules, since they might be TypeScript. - let media_type = if name.starts_with("node:") { - MediaType::TypeScript - } else { - MediaType::from_path(Path::new(&name)) - }; - - match media_type { - MediaType::TypeScript => {} - MediaType::JavaScript => return Ok((source, None)), - MediaType::Mjs => return Ok((source, None)), - _ => panic!( - "Unsupported media type for snapshotting {media_type:?} for file {}", - name - ), - } - - let parsed = deno_ast::parse_module(ParseParams { - specifier: deno_core::url::Url::parse(&name).unwrap(), - text: source.into(), - media_type, - capture_tokens: false, - scope_analysis: false, - maybe_syntax: None, - }) - .map_err(|e| JsErrorBox::from_err(JsParseDiagnostic(e)))?; - let transpiled_source = parsed - .transpile( - &deno_ast::TranspileOptions { - imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, - ..Default::default() - }, - &deno_ast::TranspileModuleOptions::default(), - &deno_ast::EmitOptions { - source_map: if cfg!(debug_assertions) { - SourceMapOption::Separate - } else { - SourceMapOption::None - }, - ..Default::default() - }, - ) - .map_err(|e| JsErrorBox::from_err(JsTranspileError(e)))? - .into_source(); - - let maybe_source_map: Option = transpiled_source - .source_map - .map(|sm| sm.into_bytes().into()); - let source_text = transpiled_source.text; - Ok((source_text.into(), maybe_source_map)) -} diff --git a/runtime/snapshot.rs b/runtime/snapshot.rs index eec8579e59..a2f0322763 100644 --- a/runtime/snapshot.rs +++ b/runtime/snapshot.rs @@ -7,10 +7,17 @@ use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; +use deno_ast::MediaType; +use deno_ast::ParseParams; +use deno_ast::SourceMapOption; use deno_cache::SqliteBackedCache; use deno_core::snapshot::*; use deno_core::v8; use deno_core::Extension; +use deno_core::ModuleCodeString; +use deno_core::ModuleName; +use deno_core::SourceMapData; +use deno_error::JsErrorBox; use deno_http::DefaultHttpPropertyExtractor; use deno_io::fs::FsError; use deno_permissions::PermissionCheckError; @@ -19,7 +26,6 @@ use deno_resolver::npm::NpmResolver; use crate::ops; use crate::ops::bootstrap::SnapshotOptions; -use crate::shared::maybe_transpile_source; use crate::shared::runtime; #[derive(Clone)] @@ -370,3 +376,70 @@ pub fn create_runtime_snapshot( println!("cargo:rerun-if-changed={}", path.display()); } } + +deno_error::js_error_wrapper!( + deno_ast::ParseDiagnostic, + JsParseDiagnostic, + "Error" +); +deno_error::js_error_wrapper!( + deno_ast::TranspileError, + JsTranspileError, + "Error" +); + +pub fn maybe_transpile_source( + name: ModuleName, + source: ModuleCodeString, +) -> Result<(ModuleCodeString, Option), JsErrorBox> { + // Always transpile `node:` built-in modules, since they might be TypeScript. + let media_type = if name.starts_with("node:") { + MediaType::TypeScript + } else { + MediaType::from_path(Path::new(&name)) + }; + + match media_type { + MediaType::TypeScript => {} + MediaType::JavaScript => return Ok((source, None)), + MediaType::Mjs => return Ok((source, None)), + _ => panic!( + "Unsupported media type for snapshotting {media_type:?} for file {}", + name + ), + } + + let parsed = deno_ast::parse_module(ParseParams { + specifier: deno_core::url::Url::parse(&name).unwrap(), + text: source.into(), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + }) + .map_err(|e| JsErrorBox::from_err(JsParseDiagnostic(e)))?; + let transpiled_source = parsed + .transpile( + &deno_ast::TranspileOptions { + imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, + ..Default::default() + }, + &deno_ast::TranspileModuleOptions::default(), + &deno_ast::EmitOptions { + source_map: if cfg!(debug_assertions) { + SourceMapOption::Separate + } else { + SourceMapOption::None + }, + ..Default::default() + }, + ) + .map_err(|e| JsErrorBox::from_err(JsTranspileError(e)))? + .into_source(); + + let maybe_source_map: Option = transpiled_source + .source_map + .map(|sm| sm.into_bytes().into()); + let source_text = transpiled_source.text; + Ok((source_text.into(), maybe_source_map)) +} diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index bb769c46a9..d7cf85bab9 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -60,7 +60,6 @@ use node_resolver::NpmPackageFolderResolver; use crate::inspector_server::InspectorServer; use crate::ops; -use crate::shared::maybe_transpile_source; use crate::shared::runtime; use crate::tokio_util::create_and_run_current_thread; use crate::worker::create_op_metrics; @@ -595,9 +594,7 @@ impl WebWorker { shared_array_buffer_store: services.shared_array_buffer_store, compiled_wasm_module_store: services.compiled_wasm_module_store, extensions, - extension_transpiler: Some(Rc::new(|specifier, source| { - maybe_transpile_source(specifier, source) - })), + extension_transpiler: None, inspector: true, feature_checker: Some(services.feature_checker), op_metrics_factory_fn, diff --git a/runtime/worker.rs b/runtime/worker.rs index 72eb54ec47..270c8b5392 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -52,7 +52,6 @@ use crate::code_cache::CodeCache; use crate::code_cache::CodeCacheType; use crate::inspector_server::InspectorServer; use crate::ops; -use crate::shared::maybe_transpile_source; use crate::shared::runtime; use crate::BootstrapOptions; @@ -502,9 +501,7 @@ impl MainWorker { shared_array_buffer_store: services.shared_array_buffer_store.clone(), compiled_wasm_module_store: services.compiled_wasm_module_store.clone(), extensions, - extension_transpiler: Some(Rc::new(|specifier, source| { - maybe_transpile_source(specifier, source) - })), + extension_transpiler: None, inspector: true, is_main: true, feature_checker: Some(services.feature_checker.clone()), diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index ff90804f3b..f263f6a0db 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -129,6 +129,8 @@ impl Default for BootstrapOptions { .map(|p| p.get()) .unwrap_or(1); + // this version is not correct as its the version of deno_runtime + // and the implementor should supply a user agent that makes sense let runtime_version = env!("CARGO_PKG_VERSION"); let user_agent = format!("Deno/{runtime_version}"); diff --git a/tests/Cargo.toml b/tests/Cargo.toml index cff778b2de..41bed83834 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -13,37 +13,31 @@ repository.workspace = true path = "lib.rs" [features] -run = [] upgrade = [] [[test]] name = "integration_tests" path = "integration/mod.rs" -required-features = ["run"] [[test]] name = "specs" path = "specs/mod.rs" -required-features = ["run"] harness = false [[test]] name = "node_compat_tests" path = "node_compat/test_runner.rs" -required-features = ["run"] [dev-dependencies] +anyhow.workspace = true bytes.workspace = true chrono = { workspace = true, features = ["now"] } -deno_ast.workspace = true deno_bench_util.workspace = true deno_cache_dir = { workspace = true } -deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } -deno_fetch.workspace = true deno_lockfile.workspace = true deno_semver.workspace = true deno_terminal.workspace = true -deno_tls.workspace = true +deno_unsync.workspace = true fastwebsockets = { workspace = true, features = ["upgrade", "unstable-split"] } file_test_runner = "0.7.3" flaky_test = "=0.2.2" @@ -59,7 +53,11 @@ os_pipe.workspace = true pretty_assertions.workspace = true regex.workspace = true reqwest.workspace = true +rustls.workspace = true +rustls-pemfile.workspace = true +rustls-tokio-stream.workspace = true serde.workspace = true +serde_json.workspace = true sys_traits = { workspace = true, features = ["real", "getrandom", "libc", "winapi"] } test_util.workspace = true tokio.workspace = true diff --git a/tests/integration/bench_tests.rs b/tests/integration/bench_tests.rs index b8d38a7f91..a87fbd28c9 100644 --- a/tests/integration/bench_tests.rs +++ b/tests/integration/bench_tests.rs @@ -1,8 +1,8 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json::json; -use deno_core::url::Url; +use serde_json::json; use test_util as util; +use url::Url; use util::assert_contains; use util::assert_not_contains; use util::TestContext; diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index e61a1ed9eb..7519ca4e86 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -1,6 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json; use test_util as util; use util::assert_not_contains; use util::testdata_path; diff --git a/tests/integration/coverage_tests.rs b/tests/integration/coverage_tests.rs index ab09d54267..67b63ffab6 100644 --- a/tests/integration/coverage_tests.rs +++ b/tests/integration/coverage_tests.rs @@ -1,6 +1,5 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json; use test_util as util; use test_util::TempDir; use util::assert_contains; diff --git a/tests/integration/fmt_tests.rs b/tests/integration/fmt_tests.rs index 468c9cfbc8..333776e0bf 100644 --- a/tests/integration/fmt_tests.rs +++ b/tests/integration/fmt_tests.rs @@ -1,6 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json::json; +use serde_json::json; use test_util as util; use test_util::itest; use util::assert_contains; diff --git a/tests/integration/inspector_tests.rs b/tests/integration/inspector_tests.rs index 7c1be193a1..4cbd3e3342 100644 --- a/tests/integration/inspector_tests.rs +++ b/tests/integration/inspector_tests.rs @@ -4,12 +4,9 @@ use std::io::BufRead; use std::process::ChildStderr; use std::time::Duration; +use anyhow::anyhow; +use anyhow::Error as AnyError; use bytes::Bytes; -use deno_core::anyhow::anyhow; -use deno_core::error::AnyError; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::url; use fastwebsockets::FragmentCollector; use fastwebsockets::Frame; use fastwebsockets::WebSocket; @@ -18,6 +15,7 @@ use hyper::upgrade::Upgraded; use hyper::Request; use hyper::Response; use hyper_util::rt::TokioIo; +use serde_json::json; use test_util as util; use tokio::net::TcpStream; use tokio::time::timeout; @@ -35,7 +33,7 @@ where Fut::Output: Send + 'static, { fn execute(&self, fut: Fut) { - deno_core::unsync::spawn(fut); + deno_unsync::spawn(fut); } } @@ -742,7 +740,7 @@ async fn inspector_json() { } let resp = client.execute(req).await.unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::OK); - let endpoint_list: Vec = + let endpoint_list: Vec = serde_json::from_str(&resp.text().await.unwrap()).unwrap(); let matching_endpoint = endpoint_list.iter().find(|e| { e["webSocketDebuggerUrl"] @@ -775,7 +773,7 @@ async fn inspector_json_list() { url.set_path("/json/list"); let resp = reqwest::get(url).await.unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::OK); - let endpoint_list: Vec = + let endpoint_list: Vec = serde_json::from_str(&resp.text().await.unwrap()).unwrap(); let matching_endpoint = endpoint_list .iter() diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index 0902b7d0a2..683c8ef85d 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -1,13 +1,12 @@ // Copyright 2018-2025 the Deno authors. MIT license. use deno_cache_dir::HttpCache; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; use deno_lockfile::Lockfile; use deno_lockfile::NewLockfileOptions; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; +use serde_json::json; +use serde_json::Value; use test_util as util; use url::Url; use util::assert_contains; diff --git a/tests/integration/jupyter_tests.rs b/tests/integration/jupyter_tests.rs index 4ef993c157..db5574f50f 100644 --- a/tests/integration/jupyter_tests.rs +++ b/tests/integration/jupyter_tests.rs @@ -4,15 +4,14 @@ use std::process::Output; use std::sync::Arc; use std::time::Duration; +use anyhow::Result; use bytes::Bytes; use chrono::DateTime; use chrono::Utc; -use deno_core::anyhow::Result; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; use serde::Deserialize; use serde::Serialize; +use serde_json::json; +use serde_json::Value; use test_util::assertions::assert_json_subset; use test_util::DenoChild; use test_util::TestContext; diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 9607207163..196184f3e0 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -3,13 +3,10 @@ use std::fs; use std::str::FromStr; -use deno_ast::ModuleSpecifier; -use deno_core::serde::Deserialize; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; -use deno_core::url::Url; use pretty_assertions::assert_eq; +use serde::Deserialize; +use serde_json::json; +use serde_json::Value; use test_util::assert_starts_with; use test_util::assertions::assert_json_subset; use test_util::deno_cmd_with_deno_dir; @@ -20,6 +17,7 @@ use test_util::lsp::LspClient; use test_util::testdata_path; use test_util::TestContextBuilder; use tower_lsp::lsp_types as lsp; +use url::Url; #[test] fn lsp_startup_shutdown() { @@ -13149,7 +13147,7 @@ fn lsp_lint_exclude_with_config() { let diagnostics = client.did_open( json!({ "textDocument": { - "uri": ModuleSpecifier::from_file_path(temp_dir.path().join("ignored.ts")).unwrap().to_string(), + "uri": Url::from_file_path(temp_dir.path().join("ignored.ts")).unwrap().to_string(), "languageId": "typescript", "version": 1, "text": "// TODO: fixme\nexport async function non_camel_case() {\nconsole.log(\"finished!\")\n}" @@ -14133,10 +14131,7 @@ fn lsp_node_modules_dir() { .unwrap(); // canonicalize for mac let path = temp_dir.path().join("node_modules").canonicalize(); - assert_starts_with!( - uri, - ModuleSpecifier::from_file_path(&path).unwrap().as_str() - ); + assert_starts_with!(uri, Url::from_file_path(&path).unwrap().as_str()); client.shutdown(); } diff --git a/tests/integration/npm_tests.rs b/tests/integration/npm_tests.rs index 033d948b2e..4406982cc8 100644 --- a/tests/integration/npm_tests.rs +++ b/tests/integration/npm_tests.rs @@ -1,9 +1,8 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; use pretty_assertions::assert_eq; +use serde_json::json; +use serde_json::Value; use test_util as util; use test_util::itest; use url::Url; diff --git a/tests/integration/pm_tests.rs b/tests/integration/pm_tests.rs index e8985ebfb1..c84c023f32 100644 --- a/tests/integration/pm_tests.rs +++ b/tests/integration/pm_tests.rs @@ -1,6 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use deno_core::serde_json::json; +use serde_json::json; use test_util::assert_contains; use test_util::env_vars_for_jsr_npm_tests; use test_util::TestContextBuilder; diff --git a/tests/integration/publish_tests.rs b/tests/integration/publish_tests.rs index 332b7c6fa6..0cfb5db5ba 100644 --- a/tests/integration/publish_tests.rs +++ b/tests/integration/publish_tests.rs @@ -2,7 +2,7 @@ use std::process::Command; -use deno_core::serde_json::json; +use serde_json::json; use test_util::assert_contains; use test_util::assert_not_contains; use test_util::env_vars_for_jsr_provenance_tests; diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index abf1c200d9..8d4bfa20cb 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -9,15 +9,12 @@ use std::process::Stdio; use std::sync::Arc; use bytes::Bytes; -use deno_core::serde_json::json; -use deno_core::url; -use deno_tls::rustls; -use deno_tls::rustls::ClientConnection; -use deno_tls::rustls_pemfile; -use deno_tls::TlsStream; use hickory_proto::serialize::txt::Parser; use hickory_server::authority::AuthorityObject; use pretty_assertions::assert_eq; +use rustls::ClientConnection; +use rustls_tokio_stream::TlsStream; +use serde_json::json; use test_util as util; use test_util::itest; use test_util::TempDir; @@ -2615,7 +2612,7 @@ where Fut::Output: Send + 'static, { fn execute(&self, fut: Fut) { - deno_core::unsync::spawn(fut); + deno_unsync::spawn(fut); } } diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs index 4da5a87d14..fac26ff40e 100644 --- a/tests/specs/mod.rs +++ b/tests/specs/mod.rs @@ -7,8 +7,7 @@ use std::collections::HashSet; use std::panic::AssertUnwindSafe; use std::rc::Rc; -use deno_core::anyhow::Context; -use deno_core::serde_json; +use anyhow::Context; use file_test_runner::collection::collect_tests_or_exit; use file_test_runner::collection::strategies::FileTestMapperStrategy; use file_test_runner::collection::strategies::TestPerDirectoryCollectionStrategy; diff --git a/tests/util/server/Cargo.toml b/tests/util/server/Cargo.toml index aec9f6e91c..6e31dd4587 100644 --- a/tests/util/server/Cargo.toml +++ b/tests/util/server/Cargo.toml @@ -13,6 +13,11 @@ repository.workspace = true name = "test_server" path = "src/test_server.rs" +[[test]] +name = "integration" +path = "integration_tests_runner.rs" +harness = false + [dependencies] anyhow.workspace = true async-stream = "0.3.3" @@ -36,7 +41,6 @@ lazy-regex.workspace = true libc.workspace = true lsp-types.workspace = true monch.workspace = true -nix = { workspace = true, features = ["fs", "term", "signal"] } once_cell.workspace = true os_pipe.workspace = true parking_lot.workspace = true @@ -60,5 +64,8 @@ win32job = "2" [target.'cfg(windows)'.dependencies] winapi = { workspace = true, features = ["consoleapi", "synchapi", "handleapi", "namedpipeapi", "winbase", "winerror"] } +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, features = ["fs", "term", "signal"] } + [build-dependencies] prost-build.workspace = true diff --git a/tests/util/server/integration_tests_runner.rs b/tests/util/server/integration_tests_runner.rs new file mode 100644 index 0000000000..63f2abe460 --- /dev/null +++ b/tests/util/server/integration_tests_runner.rs @@ -0,0 +1,5 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +pub fn main() { + // this file exists to cause the executable to be built when running cargo test +} diff --git a/tools/release/01_bump_crate_versions.ts b/tools/release/01_bump_crate_versions.ts index ddbc785c75..3d1bf0cf9a 100755 --- a/tools/release/01_bump_crate_versions.ts +++ b/tools/release/01_bump_crate_versions.ts @@ -6,6 +6,8 @@ import { $, GitLogOutput, semver } from "./deps.ts"; const workspace = await DenoWorkspace.load(); const repo = workspace.repo; const cliCrate = workspace.getCliCrate(); +const denoRtCrate = workspace.getDenoRtCrate(); +const denoLibCrate = workspace.getDenoLibCrate(); const originalCliVersion = cliCrate.version; await bumpCiCacheVersion(); @@ -21,6 +23,9 @@ if (Deno.args.some((a) => a === "--patch")) { await cliCrate.promptAndIncrement(); } +denoRtCrate.setVersion(cliCrate.version); +denoLibCrate.folderPath.join("version.txt").writeTextSync(cliCrate.version); + // increment the dependency crate versions for (const crate of workspace.getCliDependencyCrates()) { await crate.increment("minor"); diff --git a/tools/release/deno_workspace.ts b/tools/release/deno_workspace.ts index e5d9f1b928..32a21a87de 100644 --- a/tools/release/deno_workspace.ts +++ b/tools/release/deno_workspace.ts @@ -42,6 +42,14 @@ export class DenoWorkspace { return this.getCrate("deno"); } + getDenoRtCrate() { + return this.getCrate("denort"); + } + + getDenoLibCrate() { + return this.getCrate("deno_lib"); + } + getCrate(name: string) { return this.#repo.getCrate(name); } diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts index 606ff6dbe8..84b2f52bda 100755 --- a/tools/release/npm/build.ts +++ b/tools/release/npm/build.ts @@ -48,8 +48,8 @@ const packages: Package[] = [{ const markdownText = `# Deno -[Deno](https://www.deno.com) -([/ˈdiːnoʊ/](http://ipa-reader.xyz/?text=%CB%88di%CB%90no%CA%8A), pronounced +[Deno](https://deno.com) +([/ˈdiːnoʊ/](https://ipa-reader.com/?text=%CB%88di%CB%90no%CA%8A), pronounced \`dee-no\`) is a JavaScript, TypeScript, and WebAssembly runtime with secure defaults and a great developer experience. It's built on [V8](https://v8.dev/), [Rust](https://www.rust-lang.org/), and [Tokio](https://tokio.rs/).