diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 1e7b0d3f70..85a22cf837 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; @@ -76,6 +79,7 @@ use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; +use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -88,31 +92,28 @@ use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::output_vfs; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; +use super::virtual_fs::WindowsSystemRootablePath; + +pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = + ".deno_compile_node_modules"; /// A URL that can be designated as the base for relative URLs. /// /// After creation, this URL may be used to get the key for a /// module in the binary. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); - -impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { - fn from(url: &'a Url) -> Self { - Self(url) - } +pub enum StandaloneRelativeFileBaseUrl<'a> { + WindowsSystemRoot, + Path(&'a Url), } impl<'a> StandaloneRelativeFileBaseUrl<'a> { - pub fn new(url: &'a Url) -> Self { - debug_assert_eq!(url.scheme(), "file"); - Self(url) - } - /// Gets the module map key of the provided specifier. /// /// * Descendant file specifiers will be made relative to the base. @@ -122,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> { if target.scheme() != "file" { return Cow::Borrowed(target.as_str()); } + let base = match self { + Self::Path(base) => base, + Self::WindowsSystemRoot => return Cow::Borrowed(target.path()), + }; - match self.0.make_relative(target) { + match base.make_relative(target) { Some(relative) => { - if relative.starts_with("../") { - Cow::Borrowed(target.as_str()) - } else { - Cow::Owned(relative) - } + // This is not a great scenario to have because it means that the + // specifier is outside the vfs and could cause the binary to act + // strangely. If you encounter this, the fix is to add more paths + // to the vfs builder by calling `add_possible_min_root_dir`. + debug_assert!( + !relative.starts_with("../"), + "{} -> {} ({})", + base.as_str(), + target.as_str(), + relative, + ); + Cow::Owned(relative) } None => Cow::Borrowed(target.as_str()), } } - - pub fn inner(&self) -> &Url { - self.0 - } } #[derive(Deserialize, Serialize)] @@ -201,7 +209,7 @@ fn write_binary_bytes( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = @@ -372,7 +380,6 @@ pub struct WriteBinOptions<'a> { pub writer: File, pub display_output_filename: &'a str, pub graph: &'a ModuleGraph, - pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>, pub entrypoint: &'a ModuleSpecifier, pub include_files: &'a [ModuleSpecifier], pub compile_flags: &'a CompileFlags, @@ -556,7 +563,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { writer, display_output_filename, graph, - root_dir_url, entrypoint, include_files, compile_flags, @@ -568,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; - let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self - .build_npm_vfs(&root_path) - .context("Building npm vfs.")?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self - .npm_resolver - .root_node_modules_path() - .map(|path| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), - ) - .into_owned() - }), - }), - Some(snapshot), - ) - } else { - (None, None, None) - } + let mut vfs = VfsBuilder::new(); + let npm_snapshot = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; + Some(snapshot) + } else { + None } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; - let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { - npm_vfs - } else { - VfsBuilder::new(root_path.clone())? + } + InnerCliNpmResolverRef::Byonm(_) => { + self.fill_npm_vfs(&mut vfs)?; + None + } }; for include_file in include_files { let path = deno_path_util::url_to_file_path(include_file)?; - if path.is_dir() { - // TODO(#26941): we should analyze if any of these are - // modules in order to include their dependencies - vfs - .add_dir_recursive(&path) - .with_context(|| format!("Including {}", path.display()))?; - } else { - vfs - .add_file_at_path(&path) - .with_context(|| format!("Including {}", path.display()))?; - } + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { @@ -707,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); + if let Some(import_map) = self.workspace_resolver.maybe_import_map() { + if let Ok(file_path) = url_to_file_path(import_map.base_url()) { + if let Some(import_map_parent_dir) = file_path.parent() { + // tell the vfs about the import map's parent directory in case it + // falls outside what the root of where the VFS will be based + vfs.add_possible_min_root_dir(import_map_parent_dir); + } + } + } + if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() { + // ensure the vfs doesn't go below the node_modules directory's parent + if let Some(parent) = node_modules_dir.parent() { + vfs.add_possible_min_root_dir(parent); + } + } + + let vfs = self.build_vfs_consolidating_global_npm_cache(vfs); + let root_dir_url = match &vfs.root_path { + WindowsSystemRootablePath::Path(dir) => { + Some(url_from_directory_path(dir)?) + } + WindowsSystemRootablePath::WindowSystemRoot => None, + }; + let root_dir_url = match &root_dir_url { + Some(url) => StandaloneRelativeFileBaseUrl::Path(url), + None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot, + }; + + let node_modules = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(_) => { + npm_snapshot.as_ref().map(|_| NodeModules::Managed { + node_modules_dir: self.npm_resolver.root_node_modules_path().map( + |path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }, + ), + }) + } + InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), + ) + .into_owned() + }, + ), + }), + }; + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); @@ -721,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => Default::default(), }; + output_vfs(&vfs, display_output_filename); + let metadata = Metadata { argv: compile_flags.args.clone(), seed: self.cli_options.seed(), @@ -785,21 +803,19 @@ impl<'a> DenoCompileBinaryWriter<'a> { otel_config: self.cli_options.otel_config(), }; - output_vfs(&vfs, display_output_filename); - write_binary_bytes( writer, original_bin, &metadata, npm_snapshot.map(|s| s.into_serialized()), &remote_modules_store, - vfs, + &vfs, compile_flags, ) .context("Writing binary bytes") } - fn build_npm_vfs(&self, root_path: &Path) -> Result { + fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -810,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Managed(npm_resolver) => { if let Some(node_modules_path) = npm_resolver.root_node_modules_path() { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; builder.add_dir_recursive(node_modules_path)?; - Ok(builder) + Ok(()) } else { - // DO NOT include the user's registry url as it may contain credentials, - // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_path(); - let mut builder = - VfsBuilder::new(global_cache_root_path.to_path_buf())?; + // we'll flatten to remove any custom registries later let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -827,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?; builder.add_dir_recursive(&folder)?; } - - // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder - // that will be used by denort when loading the npm cache. This avoids us exposing - // the user's private registry information and means we don't have to bother - // serializing all the different registry config into the binary. - builder.with_root_dir(|root_dir| { - root_dir.name = ".deno_compile_node_modules".to_string(); - let mut new_entries = Vec::with_capacity(root_dir.entries.len()); - let mut localhost_entries = IndexMap::new(); - for entry in std::mem::take(&mut root_dir.entries) { - match entry { - VfsEntry::Dir(dir) => { - for entry in dir.entries { - log::debug!( - "Flattening {} into node_modules", - entry.name() - ); - if let Some(existing) = - localhost_entries.insert(entry.name().to_string(), entry) - { - panic!( - "Unhandled scenario where a duplicate entry was found: {:?}", - existing - ); - } - } - } - VfsEntry::File(_) | VfsEntry::Symlink(_) => { - new_entries.push(entry); - } - } - } - new_entries.push(VfsEntry::Dir(VirtualDirectory { - name: "localhost".to_string(), - entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), - })); - // needs to be sorted by name - new_entries.sort_by(|a, b| a.name().cmp(b.name())); - root_dir.entries = new_entries; - }); - - builder.set_new_root_path(root_path.to_path_buf())?; - - Ok(builder) + Ok(()) } } InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } @@ -908,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } } - Ok(builder) + Ok(()) } } } + + fn build_vfs_consolidating_global_npm_cache( + &self, + mut vfs: VfsBuilder, + ) -> BuiltVfs { + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(npm_resolver) => { + if npm_resolver.root_node_modules_path().is_some() { + return vfs.build(); + } + + let global_cache_root_path = npm_resolver.global_cache_root_path(); + + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder + // that will be used by denort when loading the npm cache. This avoids us exposing + // the user's private registry information and means we don't have to bother + // serializing all the different registry config into the binary. + let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else { + return vfs.build(); + }; + + root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string(); + let mut new_entries = Vec::with_capacity(root_dir.entries.len()); + let mut localhost_entries = IndexMap::new(); + for entry in std::mem::take(&mut root_dir.entries) { + match entry { + VfsEntry::Dir(dir) => { + for entry in dir.entries { + log::debug!("Flattening {} into node_modules", entry.name()); + if let Some(existing) = + localhost_entries.insert(entry.name().to_string(), entry) + { + panic!( + "Unhandled scenario where a duplicate entry was found: {:?}", + existing + ); + } + } + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => { + new_entries.push(entry); + } + } + } + new_entries.push(VfsEntry::Dir(VirtualDirectory { + name: "localhost".to_string(), + entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), + })); + // needs to be sorted by name + new_entries.sort_by(|a, b| a.name().cmp(b.name())); + root_dir.entries = new_entries; + + // it's better to not expose the user's cache directory, so take it out + // of there + let parent = global_cache_root_path.parent().unwrap(); + let parent_dir = vfs.get_dir_mut(parent).unwrap(); + let index = parent_dir + .entries + .iter() + .position(|entry| { + entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME + }) + .unwrap(); + let npm_global_cache_dir_entry = parent_dir.entries.remove(index); + + // go up from the ancestors removing empty directories... + // this is not as optimized as it could be + let mut last_name = + Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME); + for ancestor in parent.ancestors() { + let dir = vfs.get_dir_mut(ancestor).unwrap(); + if let Some(index) = dir + .entries + .iter() + .position(|entry| entry.name() == last_name) + { + dir.entries.remove(index); + } + last_name = Cow::Owned(dir.name.clone()); + if !dir.entries.is_empty() { + break; + } + } + + // now build the vfs and add the global cache dir entry there + let mut built_vfs = vfs.build(); + built_vfs.root.insert_entry(npm_global_cache_dir_entry); + built_vfs + } + InnerCliNpmResolverRef::Byonm(_) => vfs.build(), + } + } } fn get_denort_path(deno_exe: PathBuf) -> Option { diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index a5eb649bfd..6062e21019 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -23,6 +23,7 @@ use deno_semver::package::PackageReq; use crate::standalone::virtual_fs::VirtualDirectory; use super::binary::Metadata; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::VfsBuilder; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; @@ -39,7 +40,7 @@ pub fn serialize_binary_data_section( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, ) -> Result, AnyError> { fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); @@ -73,12 +74,11 @@ pub fn serialize_binary_data_section( } // 4. VFS { - let (vfs, vfs_files) = vfs.into_dir_and_files(); - let vfs = serde_json::to_string(&vfs)?; - write_bytes_with_len(&mut bytes, vfs.as_bytes()); - let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + let serialized_vfs = serde_json::to_string(&vfs.root)?; + write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes()); + let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); - for file in &vfs_files { + for file in &vfs.files { bytes.extend_from_slice(file); } } diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index ce7c0bb625..8ddd179c7a 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -15,17 +15,21 @@ use std::rc::Rc; use std::sync::Arc; use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_path_util::normalize_path; +use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; use deno_runtime::deno_io; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; +use indexmap::IndexSet; use serde::Deserialize; use serde::Serialize; use thiserror::Error; @@ -34,6 +38,38 @@ use crate::util; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; +use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!(!name_component.contains('\\')); + debug_assert!(!name_component.contains('/')); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug)] +pub struct BuiltVfs { + pub root_path: WindowsSystemRootablePath, + pub root: VirtualDirectory, + pub files: Vec>, +} + #[derive(Debug, Copy, Clone)] pub enum VfsFileSubDataKind { /// Raw bytes of the file. @@ -43,84 +79,84 @@ pub enum VfsFileSubDataKind { ModuleGraph, } -#[derive(Error, Debug)] -#[error( - "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() -)] -pub struct StripRootError { - root_path: PathBuf, - target: PathBuf, -} - #[derive(Debug)] pub struct VfsBuilder { - root_path: PathBuf, - root_dir: VirtualDirectory, + executable_root: VirtualDirectory, files: Vec>, current_offset: u64, file_offsets: HashMap, + /// The minimum root directory that should be included in the VFS. + min_root_dir: Option, } impl VfsBuilder { - pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path) - .with_context(|| format!("Canonicalizing {}", root_path.display()))?; - log::debug!("Building vfs with root '{}'", root_path.display()); - Ok(Self { - root_dir: VirtualDirectory { - name: root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), + pub fn new() -> Self { + Self { + executable_root: VirtualDirectory { + name: "/".to_string(), entries: Vec::new(), }, - root_path, files: Vec::new(), current_offset: 0, file_offsets: Default::default(), - }) + min_root_dir: Default::default(), + } } - pub fn set_new_root_path( - &mut self, - root_path: PathBuf, - ) -> Result<(), AnyError> { - let root_path = canonicalize_path(&root_path)?; - self.root_path = root_path; - self.root_dir = VirtualDirectory { - name: self - .root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), - entries: vec![VfsEntry::Dir(VirtualDirectory { - name: std::mem::take(&mut self.root_dir.name), - entries: std::mem::take(&mut self.root_dir.entries), - })], - }; - Ok(()) - } + /// Add a directory that might be the minimum root directory + /// of the VFS. + /// + /// For example, say the user has a deno.json and specifies an + /// import map in a parent directory. The import map won't be + /// included in the VFS, but its base will meaning we need to + /// tell the VFS builder to include the base of the import map + /// by calling this method. + pub fn add_possible_min_root_dir(&mut self, path: &Path) { + self.add_dir_raw(path); - pub fn with_root_dir( - &mut self, - with_root: impl FnOnce(&mut VirtualDirectory) -> R, - ) -> R { - with_root(&mut self.root_dir) + match &self.min_root_dir { + Some(WindowsSystemRootablePath::WindowSystemRoot) => { + // already the root dir + } + Some(WindowsSystemRootablePath::Path(current_path)) => { + let mut common_components = Vec::new(); + for (a, b) in current_path.components().zip(path.components()) { + if a != b { + break; + } + common_components.push(a); + } + if common_components.is_empty() { + if cfg!(windows) { + self.min_root_dir = + Some(WindowsSystemRootablePath::WindowSystemRoot); + } else { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(PathBuf::from("/"))); + } + } else { + self.min_root_dir = Some(WindowsSystemRootablePath::Path( + common_components.iter().collect(), + )); + } + } + None => { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(path.to_path_buf())); + } + } } pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if path != target_path { - self.add_symlink(path, &target_path)?; - } - self.add_dir_recursive_internal(&target_path) + let target_path = self.resolve_target_path(path)?; + self.add_dir_recursive_not_symlink(&target_path) } - fn add_dir_recursive_internal( + fn add_dir_recursive_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { - self.add_dir(path)?; + self.add_dir_raw(path); let read_dir = std::fs::read_dir(path) .with_context(|| format!("Reading {}", path.display()))?; @@ -133,49 +169,26 @@ impl VfsBuilder { let path = entry.path(); if file_type.is_dir() { - self.add_dir_recursive_internal(&path)?; + self.add_dir_recursive_not_symlink(&path)?; } else if file_type.is_file() { self.add_file_at_path_not_symlink(&path)?; } else if file_type.is_symlink() { - match util::fs::canonicalize_path(&path) { - Ok(target) => { - if let Err(StripRootError { .. }) = self.add_symlink(&path, &target) - { - if target.is_file() { - // this may change behavior, so warn the user about it - log::warn!( - "{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - // inline the symlink and make the target file - let file_bytes = std::fs::read(&target) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner( - &path, - file_bytes, - VfsFileSubDataKind::Raw, - )?; - } else { - log::warn!( - "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - } + match self.add_symlink(&path) { + Ok(target) => match target { + SymlinkTarget::File(target) => { + self.add_file_at_path_not_symlink(&target)? } - } + SymlinkTarget::Dir(target) => { + self.add_dir_recursive_not_symlink(&target)?; + } + }, Err(err) => { log::warn!( - "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", - crate::colors::yellow("Warning"), - path.display(), - err - ); + "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", + crate::colors::yellow("Warning"), + path.display(), + err + ); } } } @@ -184,15 +197,15 @@ impl VfsBuilder { Ok(()) } - fn add_dir( - &mut self, - path: &Path, - ) -> Result<&mut VirtualDirectory, StripRootError> { + fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { log::debug!("Ensuring directory '{}'", path.display()); - let path = self.path_relative_root(path)?; - let mut current_dir = &mut self.root_dir; + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } let name = component.as_os_str().to_string_lossy(); let index = match current_dir .entries @@ -218,15 +231,44 @@ impl VfsBuilder { }; } - Ok(current_dir) + current_dir + } + + pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { + &mut self.executable_root + } + + pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let index = match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&name)) + { + Ok(index) => index, + Err(_) => return None, + }; + match &mut current_dir.entries[index] { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + Some(current_dir) } pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; - } - self.add_file_at_path_not_symlink(&target_path) + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw) } fn add_file_at_path_not_symlink( @@ -244,11 +286,15 @@ impl VfsBuilder { data: Vec, sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + let target = self.add_symlink(path)?.into_path_buf(); + self.add_file_with_data_inner(&target, data, sub_data_kind) + } else { + self.add_file_with_data_inner(path, data, sub_data_kind) } - self.add_file_with_data_inner(&target_path, data, sub_data_kind) } fn add_file_with_data_inner( @@ -267,7 +313,7 @@ impl VfsBuilder { self.current_offset }; - let dir = self.add_dir(path.parent().unwrap())?; + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); let offset_and_len = OffsetWithLength { offset, @@ -309,74 +355,162 @@ impl VfsBuilder { Ok(()) } - fn add_symlink( + fn resolve_target_path(&mut self, path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + Ok(self.add_symlink(path)?.into_path_buf()) + } else { + Ok(path.to_path_buf()) + } + } + + fn add_symlink(&mut self, path: &Path) -> Result { + self.add_symlink_inner(path, &mut IndexSet::new()) + } + + fn add_symlink_inner( &mut self, path: &Path, - target: &Path, - ) -> Result<(), StripRootError> { - log::debug!( - "Adding symlink '{}' to '{}'", - path.display(), - target.display() + visited: &mut IndexSet, + ) -> Result { + log::debug!("Adding symlink '{}'", path.display()); + let target = strip_unc_prefix( + std::fs::read_link(path) + .with_context(|| format!("Reading symlink '{}'", path.display()))?, ); - let relative_target = self.path_relative_root(target)?; - let relative_path = match self.path_relative_root(path) { - Ok(path) => path, - Err(StripRootError { .. }) => { - // ignore if the original path is outside the root directory - return Ok(()); - } - }; - if relative_target == relative_path { - // it's the same, ignore - return Ok(()); - } - let dir = self.add_dir(path.parent().unwrap())?; + let target = normalize_path(path.parent().unwrap().join(&target)); + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => Ok(()), // previously inserted + Ok(_) => {} // previously inserted Err(insert_index) => { dir.entries.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: relative_target - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(), + dest_parts: VirtualSymlinkParts::from_path(&target), }), ); - Ok(()) } } + let target_metadata = + std::fs::symlink_metadata(&target).with_context(|| { + format!("Reading symlink target '{}'", target.display()) + })?; + if target_metadata.is_symlink() { + if !visited.insert(target.clone()) { + // todo: probably don't error in this scenario + bail!( + "Circular symlink detected: {} -> {}", + visited + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> "), + target.display() + ); + } + self.add_symlink_inner(&target, visited) + } else if target_metadata.is_dir() { + Ok(SymlinkTarget::Dir(target)) + } else { + Ok(SymlinkTarget::File(target)) + } } - pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { - (self.root_dir, self.files) - } + pub fn build(self) -> BuiltVfs { + fn strip_prefix_from_symlinks( + dir: &mut VirtualDirectory, + parts: &[String], + ) { + for entry in &mut dir.entries { + match entry { + VfsEntry::Dir(dir) => { + strip_prefix_from_symlinks(dir, parts); + } + VfsEntry::File(_) => {} + VfsEntry::Symlink(symlink) => { + let old_parts = std::mem::take(&mut symlink.dest_parts.0); + symlink.dest_parts.0 = + old_parts.into_iter().skip(parts.len()).collect(); + } + } + } + } - fn path_relative_root(&self, path: &Path) -> Result { - match path.strip_prefix(&self.root_path) { - Ok(p) => Ok(p.to_path_buf()), - Err(_) => Err(StripRootError { - root_path: self.root_path.clone(), - target: path.to_path_buf(), - }), + let mut current_dir = self.executable_root; + let mut current_path = if cfg!(windows) { + WindowsSystemRootablePath::WindowSystemRoot + } else { + WindowsSystemRootablePath::Path(PathBuf::from("/")) + }; + loop { + if current_dir.entries.len() != 1 { + break; + } + if self.min_root_dir.as_ref() == Some(¤t_path) { + break; + } + match ¤t_dir.entries[0] { + VfsEntry::Dir(dir) => { + if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + // special directory we want to maintain + break; + } + match current_dir.entries.remove(0) { + VfsEntry::Dir(dir) => { + current_path = + WindowsSystemRootablePath::Path(current_path.join(&dir.name)); + current_dir = dir; + } + _ => unreachable!(), + }; + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => break, + } + } + if let WindowsSystemRootablePath::Path(path) = ¤t_path { + strip_prefix_from_symlinks( + &mut current_dir, + &VirtualSymlinkParts::from_path(path).0, + ); + } + BuiltVfs { + root_path: current_path, + root: current_dir, + files: self.files, } } } -pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { +#[derive(Debug)] +enum SymlinkTarget { + File(PathBuf), + Dir(PathBuf), +} + +impl SymlinkTarget { + pub fn into_path_buf(self) -> PathBuf { + match self { + Self::File(path) => path, + Self::Dir(path) => path, + } + } +} + +pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { if !log::log_enabled!(log::Level::Info) { return; // no need to compute if won't output } - if builder.root_dir.entries.is_empty() { + if vfs.root.entries.is_empty() { return; // nothing to output } let mut text = String::new(); - let display_tree = vfs_as_display_tree(builder, executable_name); + let display_tree = vfs_as_display_tree(vfs, executable_name); display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string log::info!( "\n{}\n", @@ -386,7 +520,7 @@ pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) { } fn vfs_as_display_tree( - builder: &VfsBuilder, + vfs: &BuiltVfs, executable_name: &str, ) -> DisplayTreeNode { enum EntryOutput<'a> { @@ -398,20 +532,38 @@ fn vfs_as_display_tree( impl<'a> EntryOutput<'a> { pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + let mut children = match self { + EntryOutput::Subset(vec) => vec + .iter() + .map(|e| e.output.as_display_tree(e.name.to_string())) + .collect(), + EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => { + vec![] + } + }; + // we only want to collapse leafs so that nodes of the + // same depth have the same indentation + let collapse_single_child = + children.len() == 1 && children[0].children.is_empty(); DisplayTreeNode { text: match self { - EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name, + EntryOutput::All => format!("{}/*", name), + EntryOutput::Subset(_) => { + if collapse_single_child { + format!("{}/{}", name, children[0].text) + } else { + name + } + } + EntryOutput::File => name, EntryOutput::Symlink(parts) => { format!("{} --> {}", name, parts.join("/")) } }, - children: match self { - EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())], - EntryOutput::Subset(vec) => vec - .iter() - .map(|e| e.output.as_display_tree(e.name.to_string())) - .collect(), - EntryOutput::File | EntryOutput::Symlink(_) => vec![], + children: if collapse_single_child { + children.remove(0).children + } else { + children }, } } @@ -422,37 +574,81 @@ fn vfs_as_display_tree( output: EntryOutput<'a>, } - fn include_all_entries<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - EntryOutput::Subset( + fn show_global_node_modules_dir( + vfs_dir: &VirtualDirectory, + ) -> Vec { + fn show_subset_deep( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> EntryOutput { + if depth == 0 { + EntryOutput::All + } else { + EntryOutput::Subset(show_subset(vfs_dir, depth)) + } + } + + fn show_subset( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> Vec { vfs_dir .entries .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: match entry { + VfsEntry::Dir(virtual_directory) => { + show_subset_deep(virtual_directory, depth - 1) + } + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + } + }, }) - .collect(), - ) + .collect() + } + + // in this scenario, we want to show + // .deno_compile_node_modules/localhost///* + show_subset(vfs_dir, 3) } - fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> { + fn include_all_entries<'a>( + dir_path: &WindowsSystemRootablePath, + vfs_dir: &'a VirtualDirectory, + ) -> Vec> { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return show_global_node_modules_dir(vfs_dir); + } + + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(dir_path.join(entry.name()), entry), + }) + .collect() + } + + fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput { match entry { VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), VfsEntry::File(_) => EntryOutput::File, VfsEntry::Symlink(virtual_symlink) => { - EntryOutput::Symlink(&virtual_symlink.dest_parts) + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) } } } - fn analyze_dir<'a>( - dir: &Path, - vfs_dir: &'a VirtualDirectory, - ) -> EntryOutput<'a> { - let real_entry_count = std::fs::read_dir(dir) + fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir)); + } + + let real_entry_count = std::fs::read_dir(&dir) .ok() .map(|entries| entries.flat_map(|e| e.ok()).count()) .unwrap_or(0); @@ -462,7 +658,7 @@ fn vfs_as_display_tree( .iter() .map(|entry| DirEntryOutput { name: entry.name(), - output: analyze_entry(&dir.join(entry.name()), entry), + output: analyze_entry(dir.join(entry.name()), entry), }) .collect::>(); if children @@ -474,15 +670,23 @@ fn vfs_as_display_tree( EntryOutput::Subset(children) } } else { - include_all_entries(dir, vfs_dir) + EntryOutput::Subset(include_all_entries( + &WindowsSystemRootablePath::Path(dir), + vfs_dir, + )) } } // always include all the entries for the root directory, otherwise the // user might not have context about what's being shown - let output = include_all_entries(&builder.root_path, &builder.root_dir); - output - .as_display_tree(deno_terminal::colors::italic(executable_name).to_string()) + let child_entries = include_all_entries(&vfs.root_path, &vfs.root); + DisplayTreeNode { + text: deno_terminal::colors::italic(executable_name).to_string(), + children: child_entries + .iter() + .map(|entry| entry.output.as_display_tree(entry.name.to_string())) + .collect(), + } } #[derive(Debug)] @@ -603,6 +807,20 @@ pub struct VirtualDirectory { pub entries: Vec, } +impl VirtualDirectory { + pub fn insert_entry(&mut self, entry: VfsEntry) { + let name = entry.name(); + match self.entries.binary_search_by(|e| e.name().cmp(name)) { + Ok(index) => { + self.entries[index] = entry; + } + Err(insert_index) => { + self.entries.insert(insert_index, entry); + } + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct OffsetWithLength { #[serde(rename = "o")] @@ -626,18 +844,33 @@ pub struct VirtualFile { pub module_graph_offset: OffsetWithLength, } +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct VirtualSymlink { #[serde(rename = "n")] pub name: String, #[serde(rename = "p")] - pub dest_parts: Vec, + pub dest_parts: VirtualSymlinkParts, } impl VirtualSymlink { pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { let mut dest = root.to_path_buf(); - for part in &self.dest_parts { + for part in &self.dest_parts.0 { dest.push(part); } dest @@ -709,10 +942,10 @@ impl VfsRoot { let mut final_path = self.root_path.clone(); let mut current_entry = VfsEntryRef::Dir(&self.dir); for component in relative_path.components() { - let component = component.as_os_str().to_string_lossy(); + let component = component.as_os_str(); let current_dir = match current_entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } VfsEntryRef::Symlink(symlink) => { @@ -721,7 +954,7 @@ impl VfsRoot { final_path = resolved_path; // overwrite with the new resolved path match entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } _ => { @@ -739,6 +972,7 @@ impl VfsRoot { )); } }; + let component = component.to_string_lossy(); match current_dir .entries .binary_search_by(|e| e.name().cmp(&component)) @@ -1136,6 +1370,7 @@ impl FileBackedVfs { mod test { use console_static_text::ansi::strip_ansi_codes; use std::io::Write; + use test_util::assert_contains; use test_util::TempDir; use super::*; @@ -1159,8 +1394,11 @@ mod test { // will canonicalize the root path let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.join("sub_dir").create_dir_all(); + src_path.join("e.txt").write("e"); + src_path.symlink_file("e.txt", "sub_dir/e.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( &src_path.join("a.txt"), @@ -1190,18 +1428,9 @@ mod test { VfsFileSubDataKind::Raw, ) .unwrap(); + builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); builder - .add_file_with_data_inner( - &src_path.join("e.txt"), - "e".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_symlink( - &src_path.join("sub_dir").join("e.txt"), - &src_path.join("e.txt"), - ) + .add_symlink(&src_path.join("sub_dir").join("e.txt")) .unwrap(); // get the virtual fs @@ -1262,7 +1491,7 @@ mod test { // build and create the virtual fs let src_path = temp_dir_path.join("src").to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder.add_dir_recursive(&src_path).unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); @@ -1300,10 +1529,10 @@ mod test { temp_dir: &TempDir, ) -> (PathBuf, FileBackedVfs) { let virtual_fs_file = temp_dir.path().join("virtual_fs"); - let (root_dir, files) = builder.into_dir_and_files(); + let vfs = builder.build(); { let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); - for file_data in &files { + for file_data in &vfs.files { file.write_all(file_data).unwrap(); } } @@ -1314,7 +1543,7 @@ mod test { FileBackedVfs::new( Cow::Owned(data), VfsRoot { - dir: root_dir, + dir: vfs.root, root_path: dest_path.to_path_buf(), start_file_offset: 0, }, @@ -1327,41 +1556,22 @@ mod test { let temp_dir = TempDir::new(); let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.symlink_file("a.txt", "b.txt"); + src_path.symlink_file("b.txt", "c.txt"); + src_path.symlink_file("c.txt", "a.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); - builder - .add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt")) - .unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - assert_eq!( - virtual_fs - .file_entry(&dest_path.join("a.txt")) - .err() - .unwrap() - .to_string(), - "circular symlinks", - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(), - dest_path.join("b.txt") - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(), - dest_path.join("c.txt") - ); + let mut builder = VfsBuilder::new(); + let err = builder + .add_symlink(src_path.join("a.txt").as_path()) + .unwrap_err(); + assert_contains!(err.to_string(), "Circular symlink detected",); } #[tokio::test] async fn test_open_file() { let temp_dir = TempDir::new(); let temp_path = temp_dir.path().canonicalize(); - let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( temp_path.join("a.txt").as_path(), @@ -1436,8 +1646,7 @@ mod test { temp_dir.write("c/a.txt", "contents"); temp_dir.symlink_file("c/a.txt", "c/b.txt"); assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works - let mut vfs_builder = - VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap(); + let mut vfs_builder = VfsBuilder::new(); // full dir vfs_builder .add_dir_recursive(temp_dir.path().join("a").as_path()) @@ -1451,16 +1660,14 @@ mod test { .add_dir_recursive(temp_dir.path().join("c").as_path()) .unwrap(); temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory - let node = vfs_as_display_tree(&vfs_builder, "executable"); + let node = vfs_as_display_tree(&vfs_builder.build(), "executable"); let mut text = String::new(); node.print(&mut text).unwrap(); assert_eq!( strip_ansi_codes(&text), r#"executable -├─┬ a -│ └── * -├─┬ b -│ └── a.txt +├── a/* +├── b/a.txt └─┬ c ├── a.txt └── b.txt --> c/a.txt diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 4d0607ba71..7a463a7b09 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,7 +5,6 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; -use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::binary::WriteBinOptions; use crate::standalone::is_standalone_binary; use deno_ast::MediaType; @@ -17,8 +16,11 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_terminal::colors; use rand::Rng; +use std::collections::HashSet; +use std::collections::VecDeque; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -84,29 +86,6 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let root_dir_url = resolve_root_dir_from_specifiers( - cli_options.workspace().root_dir(), - graph - .specifiers() - .map(|(s, _)| s) - .chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ) - .chain(include_files.iter()) - .chain( - // sometimes the import map path is outside the root dir - cli_options - .workspace() - .to_import_map_path() - .ok() - .and_then(|p| p.and_then(|p| url_from_file_path(&p).ok())) - .iter(), - ), - ); - log::debug!("Binary root dir: {}", root_dir_url); log::info!( "{} {} to {}", colors::green("Compile"), @@ -138,7 +117,6 @@ pub async fn compile( .unwrap() .to_string_lossy(), graph: &graph, - root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url), entrypoint, include_files: &include_files, compile_flags: &compile_flags, @@ -261,15 +239,58 @@ fn get_module_roots_and_include_files( } } - let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); - let mut include_files = Vec::with_capacity(compile_flags.include.len()); + fn analyze_path( + url: &ModuleSpecifier, + module_roots: &mut Vec, + include_files: &mut Vec, + searched_paths: &mut HashSet, + ) -> Result<(), AnyError> { + let Ok(path) = url_to_file_path(url) else { + return Ok(()); + }; + let mut pending = VecDeque::from([path]); + while let Some(path) = pending.pop_front() { + if !searched_paths.insert(path.clone()) { + continue; + } + if !path.is_dir() { + let url = url_from_file_path(&path)?; + include_files.push(url.clone()); + if is_module_graph_module(&url) { + module_roots.push(url); + } + continue; + } + for entry in std::fs::read_dir(&path).with_context(|| { + format!("Failed reading directory '{}'", path.display()) + })? { + let entry = entry.with_context(|| { + format!("Failed reading entry in directory '{}'", path.display()) + })?; + pending.push_back(entry.path()); + } + } + Ok(()) + } + + let mut searched_paths = HashSet::new(); + let mut module_roots = Vec::new(); + let mut include_files = Vec::new(); module_roots.push(entrypoint.clone()); for side_module in &compile_flags.include { let url = resolve_url_or_path(side_module, initial_cwd)?; if is_module_graph_module(&url) { - module_roots.push(url); + module_roots.push(url.clone()); + if url.scheme() == "file" { + include_files.push(url); + } } else { - include_files.push(url); + analyze_path( + &url, + &mut module_roots, + &mut include_files, + &mut searched_paths, + )?; } } Ok((module_roots, include_files)) @@ -335,57 +356,6 @@ fn get_os_specific_filepath( } } -fn resolve_root_dir_from_specifiers<'a>( - starting_dir: &ModuleSpecifier, - specifiers: impl Iterator, -) -> ModuleSpecifier { - fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str { - let min_length = a.len().min(b.len()); - - let mut last_slash = 0; - for i in 0..min_length { - if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' { - last_slash = i; - } else if a.as_bytes()[i] != b.as_bytes()[i] { - break; - } - } - - // Return the common root path up to the last common slash. - // This returns a slice of the original string 'a', up to and including the last matching '/'. - let common = &a[..=last_slash]; - if cfg!(windows) && common == "file:///" { - a - } else { - common - } - } - - fn is_file_system_root(url: &str) -> bool { - let Some(path) = url.strip_prefix("file:///") else { - return false; - }; - if cfg!(windows) { - let Some((_drive, path)) = path.split_once('/') else { - return true; - }; - path.is_empty() - } else { - path.is_empty() - } - } - - let mut found_dir = starting_dir.as_str(); - if !is_file_system_root(found_dir) { - for specifier in specifiers { - if specifier.scheme() == "file" { - found_dir = select_common_root(found_dir, specifier.as_str()); - } - } - } - ModuleSpecifier::parse(found_dir).unwrap() -} - #[cfg(test)] mod test { pub use super::*; @@ -462,41 +432,4 @@ mod test { run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); } - - #[test] - fn test_resolve_root_dir_from_specifiers() { - fn resolve(start: &str, specifiers: &[&str]) -> String { - let specifiers = specifiers - .iter() - .map(|s| ModuleSpecifier::parse(s).unwrap()) - .collect::>(); - resolve_root_dir_from_specifiers( - &ModuleSpecifier::parse(start).unwrap(), - specifiers.iter(), - ) - .to_string() - } - - assert_eq!( - resolve("file:///a/b/e", &["file:///a/b/c/d"]), - "file:///a/b/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d"]), - "file:///a/b/c/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]), - "file:///a/b/c/" - ); - assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///"); - if cfg!(windows) { - assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/"); - // this will ignore the other one because it's on a separate drive - assert_eq!( - resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]), - "file:///c:/a/b/c/" - ); - } - } } diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index 62c5cf8fab..a34d2cdd1d 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -846,21 +846,6 @@ testing[WILDCARD]this .assert_matches_text("2\n"); } -#[test] -fn compile_npm_file_system() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/npm_fs/main.ts", - copy_temp_dir: Some("compile/npm_fs"), - compile_args: vec!["-A"], - run_args: vec![], - output_file: "compile/npm_fs/main.out", - node_modules_local: true, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_bin_esm() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() { }); } -#[test] -fn compile_npm_vfs_implicit_read_permissions() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/vfs_implicit_read_permission/main.ts", - copy_temp_dir: Some("compile/vfs_implicit_read_permission"), - compile_args: vec![], - run_args: vec![], - output_file: "compile/vfs_implicit_read_permission/main.out", - node_modules_local: false, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_no_permissions() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() { let symlink_target_dir = temp_dir.path().join("some_folder"); project_dir.join("node_modules").create_dir_all(); symlink_target_dir.create_dir_all(); + symlink_target_dir.join("file.txt").write("5"); let symlink_target_file = temp_dir.path().join("target.txt"); symlink_target_file.write("5"); let symlink_dir = project_dir.join("node_modules").join("symlink_dir"); diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc new file mode 100644 index 0000000000..d346c3ad20 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc @@ -0,0 +1,22 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out new file mode 100644 index 0000000000..c29c878593 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -0,0 +1,47 @@ +[WILDCARD] +Compile file:///[WILDLINE]/main.ts to [WILDLINE] + +Embedded File System + +main[WILDLINE] +├─┬ .deno_compile_node_modules +│ └─┬ localhost +│ ├─┬ ansi-regex +│ │ ├── 3.0.1/* +│ │ └── 5.0.1/* +│ ├── ansi-styles/4.3.0/* +│ ├── camelcase/5.3.1/* +│ ├── cliui/6.0.0/* +│ ├── color-convert/2.0.1/* +│ ├── color-name/1.1.4/* +│ ├── cowsay/1.5.0/* +│ ├── decamelize/1.2.0/* +│ ├── emoji-regex/8.0.0/* +│ ├── find-up/4.1.0/* +│ ├── get-caller-file/2.0.5/* +│ ├── get-stdin/8.0.0/* +│ ├─┬ is-fullwidth-code-point +│ │ ├── 2.0.0/* +│ │ └── 3.0.0/* +│ ├── locate-path/5.0.0/* +│ ├── p-limit/2.3.0/* +│ ├── p-locate/4.1.0/* +│ ├── p-try/2.2.0/* +│ ├── path-exists/4.0.0/* +│ ├── require-directory/2.1.1/* +│ ├── require-main-filename/2.0.0/* +│ ├── set-blocking/2.0.0/* +│ ├─┬ string-width +│ │ ├── 2.1.1/* +│ │ └── 4.2.3/* +│ ├─┬ strip-ansi +│ │ ├── 4.0.0/* +│ │ └── 6.0.1/* +│ ├── strip-final-newline/2.0.0/* +│ ├── which-module/2.0.0/* +│ ├── wrap-ansi/6.2.0/* +│ ├── y18n/4.0.3/* +│ ├── yargs/15.4.1/* +│ └── yargs-parser/18.1.3/* +└── main.ts + diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.out similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.out rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.out diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.ts b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.ts rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts diff --git a/tests/specs/compile/include/data_files/non_existent.out b/tests/specs/compile/include/data_files/non_existent.out index a88b441ba8..54bc69ef09 100644 --- a/tests/specs/compile/include/data_files/non_existent.out +++ b/tests/specs/compile/include/data_files/non_existent.out @@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]' Caused by: 0: Including [WILDLINE]does_not_exist.txt - 1: [WILDLINE] + 1: Reading [WILDLINE]does_not_exist.txt + 2: [WILDLINE] diff --git a/tests/specs/compile/include/folder_ts_file/__test__.jsonc b/tests/specs/compile/include/folder_ts_file/__test__.jsonc new file mode 100644 index 0000000000..f02ed1efc3 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/__test__.jsonc @@ -0,0 +1,25 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + // notice how the math folder is not included + "args": "compile --allow-read=data --include src --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=data --include src --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/include/folder_ts_file/main.js b/tests/specs/compile/include/folder_ts_file/main.js new file mode 100644 index 0000000000..23b490e390 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/main.js @@ -0,0 +1,14 @@ +const mathDir = import.meta.dirname + "/math"; +const files = Array.from( + Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name), +); +files.sort(); +for (const file of files) { + console.log(file); +} + +function nonAnalyzable() { + return "./src/main.ts"; +} + +await import(nonAnalyzable()); diff --git a/tests/specs/compile/include/folder_ts_file/math/add.ts b/tests/specs/compile/include/folder_ts_file/math/add.ts new file mode 100644 index 0000000000..3b399665dc --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/math/add.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a + b; +} diff --git a/tests/specs/compile/include/folder_ts_file/output.out b/tests/specs/compile/include/folder_ts_file/output.out new file mode 100644 index 0000000000..959e3d5c76 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/output.out @@ -0,0 +1,2 @@ +[WILDLINE]add.ts +3 diff --git a/tests/specs/compile/include/folder_ts_file/src/main.ts b/tests/specs/compile/include/folder_ts_file/src/main.ts new file mode 100644 index 0000000000..38868c3d82 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/src/main.ts @@ -0,0 +1,2 @@ +import { add } from "../math/add.ts"; +console.log(add(1, 2)); diff --git a/tests/specs/compile/include/symlink_twice/__test__.jsonc b/tests/specs/compile/include/symlink_twice/__test__.jsonc index ebdf824f43..f0f57292a6 100644 --- a/tests/specs/compile/include/symlink_twice/__test__.jsonc +++ b/tests/specs/compile/include/symlink_twice/__test__.jsonc @@ -6,7 +6,7 @@ }, { "if": "unix", "args": "compile --allow-read=data --include . --output main link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "unix", "commandName": "./main", @@ -16,7 +16,7 @@ }, { "if": "windows", "args": "compile --allow-read=data --include . --output main.exe link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "windows", "commandName": "./main.exe", diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out new file mode 100644 index 0000000000..c57eb9b2f1 --- /dev/null +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -0,0 +1,9 @@ +Compile [WILDLINE] + +Embedded File System + +main[WILDLINE] +├── index.js +├── link.js --> index.js +└── setup.js + diff --git a/tests/specs/compile/include/symlink_twice/setup.js b/tests/specs/compile/include/symlink_twice/setup.js index 3e713dd63e..4c7cebfaf5 100644 --- a/tests/specs/compile/include/symlink_twice/setup.js +++ b/tests/specs/compile/include/symlink_twice/setup.js @@ -1,3 +1,2 @@ -Deno.mkdirSync("data"); Deno.writeTextFileSync("index.js", "console.log(1);"); Deno.symlinkSync("index.js", "link.js"); diff --git a/tests/specs/compile/npm_fs/__test__.jsonc b/tests/specs/compile/npm_fs/__test__.jsonc new file mode 100644 index 0000000000..a8198bfb5d --- /dev/null +++ b/tests/specs/compile/npm_fs/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + // use this so the vfs output is all in the same folder + "canonicalizedTempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out new file mode 100644 index 0000000000..4944146788 --- /dev/null +++ b/tests/specs/compile/npm_fs/compile.out @@ -0,0 +1,8 @@ +[WILDCARD] + +Embedded File System + +main[WILDLINE] +├── main.ts +└── node_modules/* + diff --git a/tests/specs/compile/npm_fs/deno.json b/tests/specs/compile/npm_fs/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/compile/npm_fs/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/testdata/compile/npm_fs/main.out b/tests/specs/compile/npm_fs/main.out similarity index 100% rename from tests/testdata/compile/npm_fs/main.out rename to tests/specs/compile/npm_fs/main.out diff --git a/tests/testdata/compile/npm_fs/main.ts b/tests/specs/compile/npm_fs/main.ts similarity index 100% rename from tests/testdata/compile/npm_fs/main.ts rename to tests/specs/compile/npm_fs/main.ts diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs index f5820e4d88..985a6c7c40 100644 --- a/tests/specs/mod.rs +++ b/tests/specs/mod.rs @@ -118,6 +118,12 @@ struct MultiStepMetaData { /// steps. #[serde(default)] pub temp_dir: bool, + /// Whether the temporary directory should be canonicalized. + /// + /// This should be used sparingly, but is sometimes necessary + /// on the CI. + #[serde(default)] + pub canonicalized_temp_dir: bool, /// Whether the temporary directory should be symlinked to another path. #[serde(default)] pub symlinked_temp_dir: bool, @@ -144,6 +150,8 @@ struct SingleTestMetaData { #[serde(default)] pub temp_dir: bool, #[serde(default)] + pub canonicalized_temp_dir: bool, + #[serde(default)] pub symlinked_temp_dir: bool, #[serde(default)] pub repeat: Option, @@ -159,6 +167,7 @@ impl SingleTestMetaData { base: self.base, cwd: None, temp_dir: self.temp_dir, + canonicalized_temp_dir: self.canonicalized_temp_dir, symlinked_temp_dir: self.symlinked_temp_dir, repeat: self.repeat, envs: Default::default(), @@ -326,6 +335,13 @@ fn test_context_from_metadata( builder = builder.cwd(cwd.to_string_lossy()); } + if metadata.canonicalized_temp_dir { + // not actually deprecated, we just want to discourage its use + #[allow(deprecated)] + { + builder = builder.use_canonicalized_temp_dir(); + } + } if metadata.symlinked_temp_dir { // not actually deprecated, we just want to discourage its use // because it's mostly used for testing purposes locally diff --git a/tests/specs/schema.json b/tests/specs/schema.json index 2b35d9bd7d..77ffc59530 100644 --- a/tests/specs/schema.json +++ b/tests/specs/schema.json @@ -36,6 +36,9 @@ "flaky": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, "symlinkedTempDir": { "type": "boolean" }, @@ -66,6 +69,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, @@ -94,6 +103,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index 70de321361..633c2cca62 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,5 +1,4 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Warning Symlink target is outside '[WILDCARD]node_modules_symlink_outside'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. Embedded File System diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 205c6a9281..61f0a2456a 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -3,8 +3,13 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Warning Symlink target is outside '[WILDLINE]node_modules_symlink_outside'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. Embedded File System -[WILDCARD] +bin[WILDLINE] +├─┬ compile +│ └─┬ node_modules_symlink_outside +│ ├── main.ts +│ └── node_modules/* +└── some_folder/* + diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 953896cffd..531944bf6a 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -816,15 +816,17 @@ pub fn wildcard_match_detailed( } let actual_next_text = ¤t_text[max_current_text_found_index..]; - let max_next_text_len = 40; - let next_text_len = - std::cmp::min(max_next_text_len, actual_next_text.len()); + let next_text_len = actual_next_text + .chars() + .take(40) + .map(|c| c.len_utf8()) + .sum::(); output_lines.push(format!( "==== NEXT ACTUAL TEXT ====\n{}{}", colors::red(annotate_whitespace( &actual_next_text[..next_text_len] )), - if actual_next_text.len() > max_next_text_len { + if actual_next_text.len() > next_text_len { "[TRUNCATED]" } else { ""