foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-21 04:52:26 -05:00
Code Issues Packages 1 Wiki Activity

fix(compile): analyze modules in directory specified in --include (#27296)

Browse source 
I ended up changing the file system implementation to determine
its root directory as the last step of building it instead of being the
first step which makes it much more reliable.
This commit is contained in:
David Sherret 2024-12-12 13:07:35 -05:00 • committed by GitHub
parent 5f8be055db
commit 4cfa34052d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 907 additions and 541 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

349
cli/standalone/binary.rs
View file

@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmSystemInfo;
use deno_path_util::url_from_directory_path;
use deno_path_util::url_from_file_path;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::RealFs;
@ -76,6 +79,7 @@ use crate::resolver::CjsTracker;
use crate::shared::ReleaseChannel;
use crate::standalone::virtual_fs::VfsEntry;
use crate::util::archive;
use crate::util::fs::canonicalize_path;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -88,31 +92,28 @@ use super::serialization::DeserializedDataSection;
use super::serialization::RemoteModulesStore;
use super::serialization::RemoteModulesStoreBuilder;
use super::virtual_fs::output_vfs;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsFileSubDataKind;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
use super::virtual_fs::WindowsSystemRootablePath;
pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str =
".deno_compile_node_modules";
/// A URL that can be designated as the base for relative URLs.
///
/// After creation, this URL may be used to get the key for a
/// module in the binary.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url);
impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> {
fn from(url: &'a Url) -> Self {
Self(url)
}
pub enum StandaloneRelativeFileBaseUrl<'a> {
WindowsSystemRoot,
Path(&'a Url),
}
impl<'a> StandaloneRelativeFileBaseUrl<'a> {
pub fn new(url: &'a Url) -> Self {
debug_assert_eq!(url.scheme(), "file");
Self(url)
}
/// Gets the module map key of the provided specifier.
///
/// * Descendant file specifiers will be made relative to the base.
@ -122,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> {
if target.scheme() != "file" {
return Cow::Borrowed(target.as_str());
}
let base = match self {
Self::Path(base) => base,
Self::WindowsSystemRoot => return Cow::Borrowed(target.path()),
};
match self.0.make_relative(target) {
match base.make_relative(target) {
Some(relative) => {
if relative.starts_with("../") {
Cow::Borrowed(target.as_str())
} else {
Cow::Owned(relative)
}
// This is not a great scenario to have because it means that the
// specifier is outside the vfs and could cause the binary to act
// strangely. If you encounter this, the fix is to add more paths
// to the vfs builder by calling `add_possible_min_root_dir`.
debug_assert!(
!relative.starts_with("../"),
"{} -> {} ({})",
base.as_str(),
target.as_str(),
relative,
);
Cow::Owned(relative)
}
None => Cow::Borrowed(target.as_str()),
}
}
pub fn inner(&self) -> &Url {
self.0
}
}
#[derive(Deserialize, Serialize)]
@ -201,7 +209,7 @@ fn write_binary_bytes(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
vfs: &BuiltVfs,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let data_section_bytes =
@ -372,7 +380,6 @@ pub struct WriteBinOptions<'a> {
pub writer: File,
pub display_output_filename: &'a str,
pub graph: &'a ModuleGraph,
pub root_dir_url: StandaloneRelativeFileBaseUrl<'a>,
pub entrypoint: &'a ModuleSpecifier,
pub include_files: &'a [ModuleSpecifier],
pub compile_flags: &'a CompileFlags,
@ -556,7 +563,6 @@ impl<'a> DenoCompileBinaryWriter<'a> {
writer,
display_output_filename,
graph,
root_dir_url,
entrypoint,
include_files,
compile_flags,
@ -568,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> {
Some(CaData::Bytes(bytes)) => Some(bytes.clone()),
None => None,
};
let root_path = root_dir_url.inner().to_file_path().unwrap();
let (maybe_npm_vfs, node_modules, npm_snapshot) =
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
let npm_vfs_builder = self
.build_npm_vfs(&root_path)
.context("Building npm vfs.")?;
(
Some(npm_vfs_builder),
Some(NodeModules::Managed {
node_modules_dir: self
.npm_resolver
.root_node_modules_path()
.map(|path| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(path).unwrap(),
)
.into_owned()
}),
}),
Some(snapshot),
)
} else {
(None, None, None)
}
let mut vfs = VfsBuilder::new();
let npm_snapshot = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(managed) => {
let snapshot =
managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
if !snapshot.as_serialized().packages.is_empty() {
self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?;
Some(snapshot)
} else {
None
}
InnerCliNpmResolverRef::Byonm(resolver) => {
let npm_vfs_builder = self.build_npm_vfs(&root_path)?;
(
Some(npm_vfs_builder),
Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(node_modules_dir)
.unwrap(),
)
.into_owned()
},
),
}),
None,
)
}
};
let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs {
npm_vfs
} else {
VfsBuilder::new(root_path.clone())?
}
InnerCliNpmResolverRef::Byonm(_) => {
self.fill_npm_vfs(&mut vfs)?;
None
}
};
for include_file in include_files {
let path = deno_path_util::url_to_file_path(include_file)?;
if path.is_dir() {
// TODO(#26941): we should analyze if any of these are
// modules in order to include their dependencies
vfs
.add_dir_recursive(&path)
.with_context(|| format!("Including {}", path.display()))?;
} else {
vfs
.add_file_at_path(&path)
.with_context(|| format!("Including {}", path.display()))?;
}
vfs
.add_file_at_path(&path)
.with_context(|| format!("Including {}", path.display()))?;
}
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() {
@ -707,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
remote_modules_store.add_redirects(&graph.redirects);
if let Some(import_map) = self.workspace_resolver.maybe_import_map() {
if let Ok(file_path) = url_to_file_path(import_map.base_url()) {
if let Some(import_map_parent_dir) = file_path.parent() {
// tell the vfs about the import map's parent directory in case it
// falls outside what the root of where the VFS will be based
vfs.add_possible_min_root_dir(import_map_parent_dir);
}
}
}
if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() {
// ensure the vfs doesn't go below the node_modules directory's parent
if let Some(parent) = node_modules_dir.parent() {
vfs.add_possible_min_root_dir(parent);
}
}
let vfs = self.build_vfs_consolidating_global_npm_cache(vfs);
let root_dir_url = match &vfs.root_path {
WindowsSystemRootablePath::Path(dir) => {
Some(url_from_directory_path(dir)?)
}
WindowsSystemRootablePath::WindowSystemRoot => None,
};
let root_dir_url = match &root_dir_url {
Some(url) => StandaloneRelativeFileBaseUrl::Path(url),
None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot,
};
let node_modules = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(_) => {
npm_snapshot.as_ref().map(|_| NodeModules::Managed {
node_modules_dir: self.npm_resolver.root_node_modules_path().map(
|path| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(path).unwrap(),
)
.into_owned()
},
),
})
}
InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm {
root_node_modules_dir: resolver.root_node_modules_path().map(
|node_modules_dir| {
root_dir_url
.specifier_key(
&ModuleSpecifier::from_directory_path(node_modules_dir)
.unwrap(),
)
.into_owned()
},
),
}),
};
let env_vars_from_env_file = match self.cli_options.env_file_name() {
Some(env_filenames) => {
let mut aggregated_env_vars = IndexMap::new();
@ -721,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => Default::default(),
};
output_vfs(&vfs, display_output_filename);
let metadata = Metadata {
argv: compile_flags.args.clone(),
seed: self.cli_options.seed(),
@ -785,21 +803,19 @@ impl<'a> DenoCompileBinaryWriter<'a> {
otel_config: self.cli_options.otel_config(),
};
output_vfs(&vfs, display_output_filename);
write_binary_bytes(
writer,
original_bin,
&metadata,
npm_snapshot.map(|s| s.into_serialized()),
&remote_modules_store,
vfs,
&vfs,
compile_flags,
)
.context("Writing binary bytes")
}
fn build_npm_vfs(&self, root_path: &Path) -> Result<VfsBuilder, AnyError> {
fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> {
fn maybe_warn_different_system(system_info: &NpmSystemInfo) {
if system_info != &NpmSystemInfo::default() {
log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning"));
@ -810,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
if let Some(node_modules_path) = npm_resolver.root_node_modules_path() {
maybe_warn_different_system(&self.npm_system_info);
let mut builder = VfsBuilder::new(root_path.to_path_buf())?;
builder.add_dir_recursive(node_modules_path)?;
Ok(builder)
Ok(())
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let global_cache_root_path = npm_resolver.global_cache_root_path();
let mut builder =
VfsBuilder::new(global_cache_root_path.to_path_buf())?;
// we'll flatten to remove any custom registries later
let mut packages =
npm_resolver.all_system_packages(&self.npm_system_info);
packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
@ -827,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?;
builder.add_dir_recursive(&folder)?;
}
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
builder.with_root_dir(|root_dir| {
root_dir.name = ".deno_compile_node_modules".to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
log::debug!(
"Flattening {} into node_modules",
entry.name()
);
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
{
panic!(
"Unhandled scenario where a duplicate entry was found: {:?}",
existing
);
}
}
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => {
new_entries.push(entry);
}
}
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
});
builder.set_new_root_path(root_path.to_path_buf())?;
Ok(builder)
Ok(())
}
}
InnerCliNpmResolverRef::Byonm(_) => {
maybe_warn_different_system(&self.npm_system_info);
let mut builder = VfsBuilder::new(root_path.to_path_buf())?;
for pkg_json in self.cli_options.workspace().package_jsons() {
builder.add_file_at_path(&pkg_json.path)?;
}
@ -908,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
}
}
Ok(builder)
Ok(())
}
}
}
fn build_vfs_consolidating_global_npm_cache(
&self,
mut vfs: VfsBuilder,
) -> BuiltVfs {
match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(npm_resolver) => {
if npm_resolver.root_node_modules_path().is_some() {
return vfs.build();
}
let global_cache_root_path = npm_resolver.global_cache_root_path();
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else {
return vfs.build();
};
root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
log::debug!("Flattening {} into node_modules", entry.name());
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
{
panic!(
"Unhandled scenario where a duplicate entry was found: {:?}",
existing
);
}
}
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => {
new_entries.push(entry);
}
}
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
// it's better to not expose the user's cache directory, so take it out
// of there
let parent = global_cache_root_path.parent().unwrap();
let parent_dir = vfs.get_dir_mut(parent).unwrap();
let index = parent_dir
.entries
.iter()
.position(|entry| {
entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME
})
.unwrap();
let npm_global_cache_dir_entry = parent_dir.entries.remove(index);
// go up from the ancestors removing empty directories...
// this is not as optimized as it could be
let mut last_name =
Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME);
for ancestor in parent.ancestors() {
let dir = vfs.get_dir_mut(ancestor).unwrap();
if let Some(index) = dir
.entries
.iter()
.position(|entry| entry.name() == last_name)
{
dir.entries.remove(index);
}
last_name = Cow::Owned(dir.name.clone());
if !dir.entries.is_empty() {
break;
}
}
// now build the vfs and add the global cache dir entry there
let mut built_vfs = vfs.build();
built_vfs.root.insert_entry(npm_global_cache_dir_entry);
built_vfs
}
InnerCliNpmResolverRef::Byonm(_) => vfs.build(),
}
}
}
fn get_denort_path(deno_exe: PathBuf) -> Option<OsString> {

12
cli/standalone/serialization.rs
View file

@ -23,6 +23,7 @@ use deno_semver::package::PackageReq;
use crate::standalone::virtual_fs::VirtualDirectory;
use super::binary::Metadata;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::VfsBuilder;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
@ -39,7 +40,7 @@ pub fn serialize_binary_data_section(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
vfs: VfsBuilder,
vfs: &BuiltVfs,
) -> Result<Vec<u8>, AnyError> {
fn write_bytes_with_len(bytes: &mut Vec<u8>, data: &[u8]) {
bytes.extend_from_slice(&(data.len() as u64).to_le_bytes());
@ -73,12 +74,11 @@ pub fn serialize_binary_data_section(
}
// 4. VFS
{
let (vfs, vfs_files) = vfs.into_dir_and_files();
let vfs = serde_json::to_string(&vfs)?;
write_bytes_with_len(&mut bytes, vfs.as_bytes());
let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::<u64>();
let serialized_vfs = serde_json::to_string(&vfs.root)?;
write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes());
let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::<u64>();
bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes());
for file in &vfs_files {
for file in &vfs.files {
bytes.extend_from_slice(file);
}
}

671
cli/standalone/virtual_fs.rs
View file

@ -15,17 +15,21 @@ use std::rc::Rc;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::BufMutView;
use deno_core::BufView;
use deno_core::ResourceHandleFd;
use deno_path_util::normalize_path;
use deno_path_util::strip_unc_prefix;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_io;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use indexmap::IndexSet;
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
@ -34,6 +38,38 @@ use crate::util;
use crate::util::display::DisplayTreeNode;
use crate::util::fs::canonicalize_path;
use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME;
#[derive(Debug, PartialEq, Eq)]
pub enum WindowsSystemRootablePath {
/// The root of the system above any drive letters.
WindowSystemRoot,
Path(PathBuf),
}
impl WindowsSystemRootablePath {
pub fn join(&self, name_component: &str) -> PathBuf {
// this method doesn't handle multiple components
debug_assert!(!name_component.contains('\\'));
debug_assert!(!name_component.contains('/'));
match self {
WindowsSystemRootablePath::WindowSystemRoot => {
// windows drive letter
PathBuf::from(&format!("{}\\", name_component))
}
WindowsSystemRootablePath::Path(path) => path.join(name_component),
}
}
}
#[derive(Debug)]
pub struct BuiltVfs {
pub root_path: WindowsSystemRootablePath,
pub root: VirtualDirectory,
pub files: Vec<Vec<u8>>,
}
#[derive(Debug, Copy, Clone)]
pub enum VfsFileSubDataKind {
/// Raw bytes of the file.
@ -43,84 +79,84 @@ pub enum VfsFileSubDataKind {
ModuleGraph,
}
#[derive(Error, Debug)]
#[error(
"Failed to strip prefix '{}' from '{}'", root_path.display(), target.display()
)]
pub struct StripRootError {
root_path: PathBuf,
target: PathBuf,
}
#[derive(Debug)]
pub struct VfsBuilder {
root_path: PathBuf,
root_dir: VirtualDirectory,
executable_root: VirtualDirectory,
files: Vec<Vec<u8>>,
current_offset: u64,
file_offsets: HashMap<String, u64>,
/// The minimum root directory that should be included in the VFS.
min_root_dir: Option<WindowsSystemRootablePath>,
}
impl VfsBuilder {
pub fn new(root_path: PathBuf) -> Result<Self, AnyError> {
let root_path = canonicalize_path(&root_path)
.with_context(|| format!("Canonicalizing {}", root_path.display()))?;
log::debug!("Building vfs with root '{}'", root_path.display());
Ok(Self {
root_dir: VirtualDirectory {
name: root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
pub fn new() -> Self {
Self {
executable_root: VirtualDirectory {
name: "/".to_string(),
entries: Vec::new(),
},
root_path,
files: Vec::new(),
current_offset: 0,
file_offsets: Default::default(),
})
min_root_dir: Default::default(),
}
}
pub fn set_new_root_path(
&mut self,
root_path: PathBuf,
) -> Result<(), AnyError> {
let root_path = canonicalize_path(&root_path)?;
self.root_path = root_path;
self.root_dir = VirtualDirectory {
name: self
.root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
entries: vec![VfsEntry::Dir(VirtualDirectory {
name: std::mem::take(&mut self.root_dir.name),
entries: std::mem::take(&mut self.root_dir.entries),
})],
};
Ok(())
}
/// Add a directory that might be the minimum root directory
/// of the VFS.
///
/// For example, say the user has a deno.json and specifies an
/// import map in a parent directory. The import map won't be
/// included in the VFS, but its base will meaning we need to
/// tell the VFS builder to include the base of the import map
/// by calling this method.
pub fn add_possible_min_root_dir(&mut self, path: &Path) {
self.add_dir_raw(path);
pub fn with_root_dir<R>(
&mut self,
with_root: impl FnOnce(&mut VirtualDirectory) -> R,
) -> R {
with_root(&mut self.root_dir)
match &self.min_root_dir {
Some(WindowsSystemRootablePath::WindowSystemRoot) => {
// already the root dir
}
Some(WindowsSystemRootablePath::Path(current_path)) => {
let mut common_components = Vec::new();
for (a, b) in current_path.components().zip(path.components()) {
if a != b {
break;
}
common_components.push(a);
}
if common_components.is_empty() {
if cfg!(windows) {
self.min_root_dir =
Some(WindowsSystemRootablePath::WindowSystemRoot);
} else {
self.min_root_dir =
Some(WindowsSystemRootablePath::Path(PathBuf::from("/")));
}
} else {
self.min_root_dir = Some(WindowsSystemRootablePath::Path(
common_components.iter().collect(),
));
}
}
None => {
self.min_root_dir =
Some(WindowsSystemRootablePath::Path(path.to_path_buf()));
}
}
}
pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if path != target_path {
self.add_symlink(path, &target_path)?;
}
self.add_dir_recursive_internal(&target_path)
let target_path = self.resolve_target_path(path)?;
self.add_dir_recursive_not_symlink(&target_path)
}
fn add_dir_recursive_internal(
fn add_dir_recursive_not_symlink(
&mut self,
path: &Path,
) -> Result<(), AnyError> {
self.add_dir(path)?;
self.add_dir_raw(path);
let read_dir = std::fs::read_dir(path)
.with_context(|| format!("Reading {}", path.display()))?;
@ -133,49 +169,26 @@ impl VfsBuilder {
let path = entry.path();
if file_type.is_dir() {
self.add_dir_recursive_internal(&path)?;
self.add_dir_recursive_not_symlink(&path)?;
} else if file_type.is_file() {
self.add_file_at_path_not_symlink(&path)?;
} else if file_type.is_symlink() {
match util::fs::canonicalize_path(&path) {
Ok(target) => {
if let Err(StripRootError { .. }) = self.add_symlink(&path, &target)
{
if target.is_file() {
// this may change behavior, so warn the user about it
log::warn!(
"{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.",
crate::colors::yellow("Warning"),
self.root_path.display(),
path.display(),
target.display(),
);
// inline the symlink and make the target file
let file_bytes = std::fs::read(&target)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file_with_data_inner(
&path,
file_bytes,
VfsFileSubDataKind::Raw,
)?;
} else {
log::warn!(
"{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.",
crate::colors::yellow("Warning"),
self.root_path.display(),
path.display(),
target.display(),
);
}
match self.add_symlink(&path) {
Ok(target) => match target {
SymlinkTarget::File(target) => {
self.add_file_at_path_not_symlink(&target)?
}
}
SymlinkTarget::Dir(target) => {
self.add_dir_recursive_not_symlink(&target)?;
}
},
Err(err) => {
log::warn!(
"{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}",
crate::colors::yellow("Warning"),
path.display(),
err
);
"{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}",
crate::colors::yellow("Warning"),
path.display(),
err
);
}
}
}
@ -184,15 +197,15 @@ impl VfsBuilder {
Ok(())
}
fn add_dir(
&mut self,
path: &Path,
) -> Result<&mut VirtualDirectory, StripRootError> {
fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory {
log::debug!("Ensuring directory '{}'", path.display());
let path = self.path_relative_root(path)?;
let mut current_dir = &mut self.root_dir;
debug_assert!(path.is_absolute());
let mut current_dir = &mut self.executable_root;
for component in path.components() {
if matches!(component, std::path::Component::RootDir) {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
@ -218,15 +231,44 @@ impl VfsBuilder {
};
}
Ok(current_dir)
current_dir
}
pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory {
&mut self.executable_root
}
pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> {
debug_assert!(path.is_absolute());
let mut current_dir = &mut self.executable_root;
for component in path.components() {
if matches!(component, std::path::Component::RootDir) {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(_) => return None,
};
match &mut current_dir.entries[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
_ => unreachable!(),
};
}
Some(current_dir)
}
pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if target_path != path {
self.add_symlink(path, &target_path)?;
}
self.add_file_at_path_not_symlink(&target_path)
let file_bytes = std::fs::read(path)
.with_context(|| format!("Reading {}", path.display()))?;
self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw)
}
fn add_file_at_path_not_symlink(
@ -244,11 +286,15 @@ impl VfsBuilder {
data: Vec<u8>,
sub_data_kind: VfsFileSubDataKind,
) -> Result<(), AnyError> {
let target_path = canonicalize_path(path)?;
if target_path != path {
self.add_symlink(path, &target_path)?;
let metadata = std::fs::symlink_metadata(path).with_context(|| {
format!("Resolving target path for '{}'", path.display())
})?;
if metadata.is_symlink() {
let target = self.add_symlink(path)?.into_path_buf();
self.add_file_with_data_inner(&target, data, sub_data_kind)
} else {
self.add_file_with_data_inner(path, data, sub_data_kind)
}
self.add_file_with_data_inner(&target_path, data, sub_data_kind)
}
fn add_file_with_data_inner(
@ -267,7 +313,7 @@ impl VfsBuilder {
self.current_offset
};
let dir = self.add_dir(path.parent().unwrap())?;
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
let offset_and_len = OffsetWithLength {
offset,
@ -309,74 +355,162 @@ impl VfsBuilder {
Ok(())
}
fn add_symlink(
fn resolve_target_path(&mut self, path: &Path) -> Result<PathBuf, AnyError> {
let metadata = std::fs::symlink_metadata(path).with_context(|| {
format!("Resolving target path for '{}'", path.display())
})?;
if metadata.is_symlink() {
Ok(self.add_symlink(path)?.into_path_buf())
} else {
Ok(path.to_path_buf())
}
}
fn add_symlink(&mut self, path: &Path) -> Result<SymlinkTarget, AnyError> {
self.add_symlink_inner(path, &mut IndexSet::new())
}
fn add_symlink_inner(
&mut self,
path: &Path,
target: &Path,
) -> Result<(), StripRootError> {
log::debug!(
"Adding symlink '{}' to '{}'",
path.display(),
target.display()
visited: &mut IndexSet<PathBuf>,
) -> Result<SymlinkTarget, AnyError> {
log::debug!("Adding symlink '{}'", path.display());
let target = strip_unc_prefix(
std::fs::read_link(path)
.with_context(|| format!("Reading symlink '{}'", path.display()))?,
);
let relative_target = self.path_relative_root(target)?;
let relative_path = match self.path_relative_root(path) {
Ok(path) => path,
Err(StripRootError { .. }) => {
// ignore if the original path is outside the root directory
return Ok(());
}
};
if relative_target == relative_path {
// it's the same, ignore
return Ok(());
}
let dir = self.add_dir(path.parent().unwrap())?;
let target = normalize_path(path.parent().unwrap().join(&target));
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
Ok(_) => Ok(()), // previously inserted
Ok(_) => {} // previously inserted
Err(insert_index) => {
dir.entries.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
dest_parts: relative_target
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
dest_parts: VirtualSymlinkParts::from_path(&target),
}),
);
Ok(())
}
}
let target_metadata =
std::fs::symlink_metadata(&target).with_context(|| {
format!("Reading symlink target '{}'", target.display())
})?;
if target_metadata.is_symlink() {
if !visited.insert(target.clone()) {
// todo: probably don't error in this scenario
bail!(
"Circular symlink detected: {} -> {}",
visited
.iter()
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(" -> "),
target.display()
);
}
self.add_symlink_inner(&target, visited)
} else if target_metadata.is_dir() {
Ok(SymlinkTarget::Dir(target))
} else {
Ok(SymlinkTarget::File(target))
}
}
pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec<Vec<u8>>) {
(self.root_dir, self.files)
}
pub fn build(self) -> BuiltVfs {
fn strip_prefix_from_symlinks(
dir: &mut VirtualDirectory,
parts: &[String],
) {
for entry in &mut dir.entries {
match entry {
VfsEntry::Dir(dir) => {
strip_prefix_from_symlinks(dir, parts);
}
VfsEntry::File(_) => {}
VfsEntry::Symlink(symlink) => {
let old_parts = std::mem::take(&mut symlink.dest_parts.0);
symlink.dest_parts.0 =
old_parts.into_iter().skip(parts.len()).collect();
}
}
}
}
fn path_relative_root(&self, path: &Path) -> Result<PathBuf, StripRootError> {
match path.strip_prefix(&self.root_path) {
Ok(p) => Ok(p.to_path_buf()),
Err(_) => Err(StripRootError {
root_path: self.root_path.clone(),
target: path.to_path_buf(),
}),
let mut current_dir = self.executable_root;
let mut current_path = if cfg!(windows) {
WindowsSystemRootablePath::WindowSystemRoot
} else {
WindowsSystemRootablePath::Path(PathBuf::from("/"))
};
loop {
if current_dir.entries.len() != 1 {
break;
}
if self.min_root_dir.as_ref() == Some(&current_path) {
break;
}
match &current_dir.entries[0] {
VfsEntry::Dir(dir) => {
if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
// special directory we want to maintain
break;
}
match current_dir.entries.remove(0) {
VfsEntry::Dir(dir) => {
current_path =
WindowsSystemRootablePath::Path(current_path.join(&dir.name));
current_dir = dir;
}
_ => unreachable!(),
};
}
VfsEntry::File(_) | VfsEntry::Symlink(_) => break,
}
}
if let WindowsSystemRootablePath::Path(path) = &current_path {
strip_prefix_from_symlinks(
&mut current_dir,
&VirtualSymlinkParts::from_path(path).0,
);
}
BuiltVfs {
root_path: current_path,
root: current_dir,
files: self.files,
}
}
}
pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) {
#[derive(Debug)]
enum SymlinkTarget {
File(PathBuf),
Dir(PathBuf),
}
impl SymlinkTarget {
pub fn into_path_buf(self) -> PathBuf {
match self {
Self::File(path) => path,
Self::Dir(path) => path,
}
}
}
pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) {
if !log::log_enabled!(log::Level::Info) {
return; // no need to compute if won't output
}
if builder.root_dir.entries.is_empty() {
if vfs.root.entries.is_empty() {
return; // nothing to output
}
let mut text = String::new();
let display_tree = vfs_as_display_tree(builder, executable_name);
let display_tree = vfs_as_display_tree(vfs, executable_name);
display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string
log::info!(
"\n{}\n",
@ -386,7 +520,7 @@ pub fn output_vfs(builder: &VfsBuilder, executable_name: &str) {
}
fn vfs_as_display_tree(
builder: &VfsBuilder,
vfs: &BuiltVfs,
executable_name: &str,
) -> DisplayTreeNode {
enum EntryOutput<'a> {
@ -398,20 +532,38 @@ fn vfs_as_display_tree(
impl<'a> EntryOutput<'a> {
pub fn as_display_tree(&self, name: String) -> DisplayTreeNode {
let mut children = match self {
EntryOutput::Subset(vec) => vec
.iter()
.map(|e| e.output.as_display_tree(e.name.to_string()))
.collect(),
EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => {
vec![]
}
};
// we only want to collapse leafs so that nodes of the
// same depth have the same indentation
let collapse_single_child =
children.len() == 1 && children[0].children.is_empty();
DisplayTreeNode {
text: match self {
EntryOutput::All | EntryOutput::Subset(_) | EntryOutput::File => name,
EntryOutput::All => format!("{}/*", name),
EntryOutput::Subset(_) => {
if collapse_single_child {
format!("{}/{}", name, children[0].text)
} else {
name
}
}
EntryOutput::File => name,
EntryOutput::Symlink(parts) => {
format!("{} --> {}", name, parts.join("/"))
}
},
children: match self {
EntryOutput::All => vec![DisplayTreeNode::from_text("*".to_string())],
EntryOutput::Subset(vec) => vec
.iter()
.map(|e| e.output.as_display_tree(e.name.to_string()))
.collect(),
EntryOutput::File | EntryOutput::Symlink(_) => vec![],
children: if collapse_single_child {
children.remove(0).children
} else {
children
},
}
}
@ -422,37 +574,81 @@ fn vfs_as_display_tree(
output: EntryOutput<'a>,
}
fn include_all_entries<'a>(
dir: &Path,
vfs_dir: &'a VirtualDirectory,
) -> EntryOutput<'a> {
EntryOutput::Subset(
fn show_global_node_modules_dir(
vfs_dir: &VirtualDirectory,
) -> Vec<DirEntryOutput> {
fn show_subset_deep(
vfs_dir: &VirtualDirectory,
depth: usize,
) -> EntryOutput {
if depth == 0 {
EntryOutput::All
} else {
EntryOutput::Subset(show_subset(vfs_dir, depth))
}
}
fn show_subset(
vfs_dir: &VirtualDirectory,
depth: usize,
) -> Vec<DirEntryOutput> {
vfs_dir
.entries
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(&dir.join(entry.name()), entry),
output: match entry {
VfsEntry::Dir(virtual_directory) => {
show_subset_deep(virtual_directory, depth - 1)
}
VfsEntry::File(_) => EntryOutput::File,
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
}
},
})
.collect(),
)
.collect()
}
// in this scenario, we want to show
// .deno_compile_node_modules/localhost/<package_name>/<version>/*
show_subset(vfs_dir, 3)
}
fn analyze_entry<'a>(path: &Path, entry: &'a VfsEntry) -> EntryOutput<'a> {
fn include_all_entries<'a>(
dir_path: &WindowsSystemRootablePath,
vfs_dir: &'a VirtualDirectory,
) -> Vec<DirEntryOutput<'a>> {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return show_global_node_modules_dir(vfs_dir);
}
vfs_dir
.entries
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(dir_path.join(entry.name()), entry),
})
.collect()
}
fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput {
match entry {
VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory),
VfsEntry::File(_) => EntryOutput::File,
VfsEntry::Symlink(virtual_symlink) => {
EntryOutput::Symlink(&virtual_symlink.dest_parts)
EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
}
}
}
fn analyze_dir<'a>(
dir: &Path,
vfs_dir: &'a VirtualDirectory,
) -> EntryOutput<'a> {
let real_entry_count = std::fs::read_dir(dir)
fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir));
}
let real_entry_count = std::fs::read_dir(&dir)
.ok()
.map(|entries| entries.flat_map(|e| e.ok()).count())
.unwrap_or(0);
@ -462,7 +658,7 @@ fn vfs_as_display_tree(
.iter()
.map(|entry| DirEntryOutput {
name: entry.name(),
output: analyze_entry(&dir.join(entry.name()), entry),
output: analyze_entry(dir.join(entry.name()), entry),
})
.collect::<Vec<_>>();
if children
@ -474,15 +670,23 @@ fn vfs_as_display_tree(
EntryOutput::Subset(children)
}
} else {
include_all_entries(dir, vfs_dir)
EntryOutput::Subset(include_all_entries(
&WindowsSystemRootablePath::Path(dir),
vfs_dir,
))
}
}
// always include all the entries for the root directory, otherwise the
// user might not have context about what's being shown
let output = include_all_entries(&builder.root_path, &builder.root_dir);
output
.as_display_tree(deno_terminal::colors::italic(executable_name).to_string())
let child_entries = include_all_entries(&vfs.root_path, &vfs.root);
DisplayTreeNode {
text: deno_terminal::colors::italic(executable_name).to_string(),
children: child_entries
.iter()
.map(|entry| entry.output.as_display_tree(entry.name.to_string()))
.collect(),
}
}
#[derive(Debug)]
@ -603,6 +807,20 @@ pub struct VirtualDirectory {
pub entries: Vec<VfsEntry>,
}
impl VirtualDirectory {
pub fn insert_entry(&mut self, entry: VfsEntry) {
let name = entry.name();
match self.entries.binary_search_by(|e| e.name().cmp(name)) {
Ok(index) => {
self.entries[index] = entry;
}
Err(insert_index) => {
self.entries.insert(insert_index, entry);
}
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct OffsetWithLength {
#[serde(rename = "o")]
@ -626,18 +844,33 @@ pub struct VirtualFile {
pub module_graph_offset: OffsetWithLength,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlinkParts(Vec<String>);
impl VirtualSymlinkParts {
pub fn from_path(path: &Path) -> Self {
Self(
path
.components()
.filter(|c| !matches!(c, std::path::Component::RootDir))
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect(),
)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualSymlink {
#[serde(rename = "n")]
pub name: String,
#[serde(rename = "p")]
pub dest_parts: Vec<String>,
pub dest_parts: VirtualSymlinkParts,
}
impl VirtualSymlink {
pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf {
let mut dest = root.to_path_buf();
for part in &self.dest_parts {
for part in &self.dest_parts.0 {
dest.push(part);
}
dest
@ -709,10 +942,10 @@ impl VfsRoot {
let mut final_path = self.root_path.clone();
let mut current_entry = VfsEntryRef::Dir(&self.dir);
for component in relative_path.components() {
let component = component.as_os_str().to_string_lossy();
let component = component.as_os_str();
let current_dir = match current_entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
final_path.push(component);
dir
}
VfsEntryRef::Symlink(symlink) => {
@ -721,7 +954,7 @@ impl VfsRoot {
final_path = resolved_path; // overwrite with the new resolved path
match entry {
VfsEntryRef::Dir(dir) => {
final_path.push(component.as_ref());
final_path.push(component);
dir
}
_ => {
@ -739,6 +972,7 @@ impl VfsRoot {
));
}
};
let component = component.to_string_lossy();
match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
@ -1136,6 +1370,7 @@ impl FileBackedVfs {
mod test {
use console_static_text::ansi::strip_ansi_codes;
use std::io::Write;
use test_util::assert_contains;
use test_util::TempDir;
use super::*;
@ -1159,8 +1394,11 @@ mod test {
// will canonicalize the root path
let src_path = temp_dir.path().canonicalize().join("src");
src_path.create_dir_all();
src_path.join("sub_dir").create_dir_all();
src_path.join("e.txt").write("e");
src_path.symlink_file("e.txt", "sub_dir/e.txt");
let src_path = src_path.to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
let mut builder = VfsBuilder::new();
builder
.add_file_with_data_inner(
&src_path.join("a.txt"),
@ -1190,18 +1428,9 @@ mod test {
VfsFileSubDataKind::Raw,
)
.unwrap();
builder.add_file_at_path(&src_path.join("e.txt")).unwrap();
builder
.add_file_with_data_inner(
&src_path.join("e.txt"),
"e".into(),
VfsFileSubDataKind::Raw,
)
.unwrap();
builder
.add_symlink(
&src_path.join("sub_dir").join("e.txt"),
&src_path.join("e.txt"),
)
.add_symlink(&src_path.join("sub_dir").join("e.txt"))
.unwrap();
// get the virtual fs
@ -1262,7 +1491,7 @@ mod test {
// build and create the virtual fs
let src_path = temp_dir_path.join("src").to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
let mut builder = VfsBuilder::new();
builder.add_dir_recursive(&src_path).unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
@ -1300,10 +1529,10 @@ mod test {
temp_dir: &TempDir,
) -> (PathBuf, FileBackedVfs) {
let virtual_fs_file = temp_dir.path().join("virtual_fs");
let (root_dir, files) = builder.into_dir_and_files();
let vfs = builder.build();
{
let mut file = std::fs::File::create(&virtual_fs_file).unwrap();
for file_data in &files {
for file_data in &vfs.files {
file.write_all(file_data).unwrap();
}
}
@ -1314,7 +1543,7 @@ mod test {
FileBackedVfs::new(
Cow::Owned(data),
VfsRoot {
dir: root_dir,
dir: vfs.root,
root_path: dest_path.to_path_buf(),
start_file_offset: 0,
},
@ -1327,41 +1556,22 @@ mod test {
let temp_dir = TempDir::new();
let src_path = temp_dir.path().canonicalize().join("src");
src_path.create_dir_all();
src_path.symlink_file("a.txt", "b.txt");
src_path.symlink_file("b.txt", "c.txt");
src_path.symlink_file("c.txt", "a.txt");
let src_path = src_path.to_path_buf();
let mut builder = VfsBuilder::new(src_path.clone()).unwrap();
builder
.add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt"))
.unwrap();
builder
.add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt"))
.unwrap();
builder
.add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt"))
.unwrap();
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
assert_eq!(
virtual_fs
.file_entry(&dest_path.join("a.txt"))
.err()
.unwrap()
.to_string(),
"circular symlinks",
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(),
dest_path.join("b.txt")
);
assert_eq!(
virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(),
dest_path.join("c.txt")
);
let mut builder = VfsBuilder::new();
let err = builder
.add_symlink(src_path.join("a.txt").as_path())
.unwrap_err();
assert_contains!(err.to_string(), "Circular symlink detected",);
}
#[tokio::test]
async fn test_open_file() {
let temp_dir = TempDir::new();
let temp_path = temp_dir.path().canonicalize();
let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap();
let mut builder = VfsBuilder::new();
builder
.add_file_with_data_inner(
temp_path.join("a.txt").as_path(),
@ -1436,8 +1646,7 @@ mod test {
temp_dir.write("c/a.txt", "contents");
temp_dir.symlink_file("c/a.txt", "c/b.txt");
assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works
let mut vfs_builder =
VfsBuilder::new(temp_dir.path().to_path_buf()).unwrap();
let mut vfs_builder = VfsBuilder::new();
// full dir
vfs_builder
.add_dir_recursive(temp_dir.path().join("a").as_path())
@ -1451,16 +1660,14 @@ mod test {
.add_dir_recursive(temp_dir.path().join("c").as_path())
.unwrap();
temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory
let node = vfs_as_display_tree(&vfs_builder, "executable");
let node = vfs_as_display_tree(&vfs_builder.build(), "executable");
let mut text = String::new();
node.print(&mut text).unwrap();
assert_eq!(
strip_ansi_codes(&text),
r#"executable
├─┬ a
│ └── *
├─┬ b
│ └── a.txt
├── a/*
├── b/a.txt
└─┬ c
├── a.txt
└── b.txt --> c/a.txt

167
cli/tools/compile.rs
View file

@ -5,7 +5,6 @@ use crate::args::CompileFlags;
use crate::args::Flags;
use crate::factory::CliFactory;
use crate::http_util::HttpClientProvider;
use crate::standalone::binary::StandaloneRelativeFileBaseUrl;
use crate::standalone::binary::WriteBinOptions;
use crate::standalone::is_standalone_binary;
use deno_ast::MediaType;
@ -17,8 +16,11 @@ use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_graph::GraphKind;
use deno_path_util::url_from_file_path;
use deno_path_util::url_to_file_path;
use deno_terminal::colors;
use rand::Rng;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@ -84,29 +86,6 @@ pub async fn compile(
let ts_config_for_emit = cli_options
.resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?;
check_warn_tsconfig(&ts_config_for_emit);
let root_dir_url = resolve_root_dir_from_specifiers(
cli_options.workspace().root_dir(),
graph
.specifiers()
.map(|(s, _)| s)
.chain(
cli_options
.node_modules_dir_path()
.and_then(|p| ModuleSpecifier::from_directory_path(p).ok())
.iter(),
)
.chain(include_files.iter())
.chain(
// sometimes the import map path is outside the root dir
cli_options
.workspace()
.to_import_map_path()
.ok()
.and_then(|p| p.and_then(|p| url_from_file_path(&p).ok()))
.iter(),
),
);
log::debug!("Binary root dir: {}", root_dir_url);
log::info!(
"{} {} to {}",
colors::green("Compile"),
@ -138,7 +117,6 @@ pub async fn compile(
.unwrap()
.to_string_lossy(),
graph: &graph,
root_dir_url: StandaloneRelativeFileBaseUrl::from(&root_dir_url),
entrypoint,
include_files: &include_files,
compile_flags: &compile_flags,
@ -261,15 +239,58 @@ fn get_module_roots_and_include_files(
}
}
let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1);
let mut include_files = Vec::with_capacity(compile_flags.include.len());
fn analyze_path(
url: &ModuleSpecifier,
module_roots: &mut Vec<ModuleSpecifier>,
include_files: &mut Vec<ModuleSpecifier>,
searched_paths: &mut HashSet<PathBuf>,
) -> Result<(), AnyError> {
let Ok(path) = url_to_file_path(url) else {
return Ok(());
};
let mut pending = VecDeque::from([path]);
while let Some(path) = pending.pop_front() {
if !searched_paths.insert(path.clone()) {
continue;
}
if !path.is_dir() {
let url = url_from_file_path(&path)?;
include_files.push(url.clone());
if is_module_graph_module(&url) {
module_roots.push(url);
}
continue;
}
for entry in std::fs::read_dir(&path).with_context(|| {
format!("Failed reading directory '{}'", path.display())
})? {
let entry = entry.with_context(|| {
format!("Failed reading entry in directory '{}'", path.display())
})?;
pending.push_back(entry.path());
}
}
Ok(())
}
let mut searched_paths = HashSet::new();
let mut module_roots = Vec::new();
let mut include_files = Vec::new();
module_roots.push(entrypoint.clone());
for side_module in &compile_flags.include {
let url = resolve_url_or_path(side_module, initial_cwd)?;
if is_module_graph_module(&url) {
module_roots.push(url);
module_roots.push(url.clone());
if url.scheme() == "file" {
include_files.push(url);
}
} else {
include_files.push(url);
analyze_path(
&url,
&mut module_roots,
&mut include_files,
&mut searched_paths,
)?;
}
}
Ok((module_roots, include_files))
@ -335,57 +356,6 @@ fn get_os_specific_filepath(
}
}
fn resolve_root_dir_from_specifiers<'a>(
starting_dir: &ModuleSpecifier,
specifiers: impl Iterator<Item = &'a ModuleSpecifier>,
) -> ModuleSpecifier {
fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str {
let min_length = a.len().min(b.len());
let mut last_slash = 0;
for i in 0..min_length {
if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' {
last_slash = i;
} else if a.as_bytes()[i] != b.as_bytes()[i] {
break;
}
}
// Return the common root path up to the last common slash.
// This returns a slice of the original string 'a', up to and including the last matching '/'.
let common = &a[..=last_slash];
if cfg!(windows) && common == "file:///" {
a
} else {
common
}
}
fn is_file_system_root(url: &str) -> bool {
let Some(path) = url.strip_prefix("file:///") else {
return false;
};
if cfg!(windows) {
let Some((_drive, path)) = path.split_once('/') else {
return true;
};
path.is_empty()
} else {
path.is_empty()
}
}
let mut found_dir = starting_dir.as_str();
if !is_file_system_root(found_dir) {
for specifier in specifiers {
if specifier.scheme() == "file" {
found_dir = select_common_root(found_dir, specifier.as_str());
}
}
}
ModuleSpecifier::parse(found_dir).unwrap()
}
#[cfg(test)]
mod test {
pub use super::*;
@ -462,41 +432,4 @@ mod test {
run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe");
run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2");
}
#[test]
fn test_resolve_root_dir_from_specifiers() {
fn resolve(start: &str, specifiers: &[&str]) -> String {
let specifiers = specifiers
.iter()
.map(|s| ModuleSpecifier::parse(s).unwrap())
.collect::<Vec<_>>();
resolve_root_dir_from_specifiers(
&ModuleSpecifier::parse(start).unwrap(),
specifiers.iter(),
)
.to_string()
}
assert_eq!(
resolve("file:///a/b/e", &["file:///a/b/c/d"]),
"file:///a/b/"
);
assert_eq!(
resolve("file:///a/b/c/", &["file:///a/b/c/d"]),
"file:///a/b/c/"
);
assert_eq!(
resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]),
"file:///a/b/c/"
);
assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///");
if cfg!(windows) {
assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/");
// this will ignore the other one because it's on a separate drive
assert_eq!(
resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]),
"file:///c:/a/b/c/"
);
}
}
}

31
tests/integration/compile_tests.rs
View file

@ -846,21 +846,6 @@ testing[WILDCARD]this
.assert_matches_text("2\n");
}
#[test]
fn compile_npm_file_system() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/npm_fs/main.ts",
copy_temp_dir: Some("compile/npm_fs"),
compile_args: vec!["-A"],
run_args: vec![],
output_file: "compile/npm_fs/main.out",
node_modules_local: true,
input_name: Some("binary"),
expected_name: "binary",
exit_code: 0,
});
}
#[test]
fn compile_npm_bin_esm() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() {
});
}
#[test]
fn compile_npm_vfs_implicit_read_permissions() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
input_specifier: "compile/vfs_implicit_read_permission/main.ts",
copy_temp_dir: Some("compile/vfs_implicit_read_permission"),
compile_args: vec![],
run_args: vec![],
output_file: "compile/vfs_implicit_read_permission/main.out",
node_modules_local: false,
input_name: Some("binary"),
expected_name: "binary",
exit_code: 0,
});
}
#[test]
fn compile_npm_no_permissions() {
run_npm_bin_compile_test(RunNpmBinCompileOptions {
@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() {
let symlink_target_dir = temp_dir.path().join("some_folder");
project_dir.join("node_modules").create_dir_all();
symlink_target_dir.create_dir_all();
symlink_target_dir.join("file.txt").write("5");
let symlink_target_file = temp_dir.path().join("target.txt");
symlink_target_file.write("5");
let symlink_dir = project_dir.join("node_modules").join("symlink_dir");

22
tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc Normal file
View file

@ -0,0 +1,22 @@
{
"tempDir": true,
"steps": [{
"if": "unix",
"args": "compile --output main main.ts",
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "main.out"
}, {
"if": "windows",
"args": "compile --output main.exe main.ts",
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "main.out"
}]
}

47
tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out Normal file
View file

@ -0,0 +1,47 @@
[WILDCARD]
Compile file:///[WILDLINE]/main.ts to [WILDLINE]
Embedded File System
main[WILDLINE]
├─┬ .deno_compile_node_modules
│ └─┬ localhost
│ ├─┬ ansi-regex
│ │ ├── 3.0.1/*
│ │ └── 5.0.1/*
│ ├── ansi-styles/4.3.0/*
│ ├── camelcase/5.3.1/*
│ ├── cliui/6.0.0/*
│ ├── color-convert/2.0.1/*
│ ├── color-name/1.1.4/*
│ ├── cowsay/1.5.0/*
│ ├── decamelize/1.2.0/*
│ ├── emoji-regex/8.0.0/*
│ ├── find-up/4.1.0/*
│ ├── get-caller-file/2.0.5/*
│ ├── get-stdin/8.0.0/*
│ ├─┬ is-fullwidth-code-point
│ │ ├── 2.0.0/*
│ │ └── 3.0.0/*
│ ├── locate-path/5.0.0/*
│ ├── p-limit/2.3.0/*
│ ├── p-locate/4.1.0/*
│ ├── p-try/2.2.0/*
│ ├── path-exists/4.0.0/*
│ ├── require-directory/2.1.1/*
│ ├── require-main-filename/2.0.0/*
│ ├── set-blocking/2.0.0/*
│ ├─┬ string-width
│ │ ├── 2.1.1/*
│ │ └── 4.2.3/*
│ ├─┬ strip-ansi
│ │ ├── 4.0.0/*
│ │ └── 6.0.1/*
│ ├── strip-final-newline/2.0.0/*
│ ├── which-module/2.0.0/*
│ ├── wrap-ansi/6.2.0/*
│ ├── y18n/4.0.3/*
│ ├── yargs/15.4.1/*
│ └── yargs-parser/18.1.3/*
└── main.ts

0
tests/testdata/compile/vfs_implicit_read_permission/main.out → tests/specs/compile/global_npm_cache_implicit_read_permission/main.out
View file

0
tests/testdata/compile/vfs_implicit_read_permission/main.ts → tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts
View file

3
tests/specs/compile/include/data_files/non_existent.out
View file

@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]'
Caused by:
0: Including [WILDLINE]does_not_exist.txt
1: [WILDLINE]
1: Reading [WILDLINE]does_not_exist.txt
2: [WILDLINE]

25
tests/specs/compile/include/folder_ts_file/__test__.jsonc Normal file
View file

@ -0,0 +1,25 @@
{
"tempDir": true,
"steps": [{
"if": "unix",
// notice how the math folder is not included
"args": "compile --allow-read=data --include src --output main main.js",
"output": "[WILDCARD]"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "output.out",
"exitCode": 0
}, {
"if": "windows",
"args": "compile --allow-read=data --include src --output main.exe main.js",
"output": "[WILDCARD]"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "output.out",
"exitCode": 0
}]
}

14
tests/specs/compile/include/folder_ts_file/main.js Normal file
View file

@ -0,0 +1,14 @@
const mathDir = import.meta.dirname + "/math";
const files = Array.from(
Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name),
);
files.sort();
for (const file of files) {
console.log(file);
}
function nonAnalyzable() {
return "./src/main.ts";
}
await import(nonAnalyzable());

3
tests/specs/compile/include/folder_ts_file/math/add.ts Normal file
View file

@ -0,0 +1,3 @@
export function add(a: number, b: number) {
return a + b;
}

2
tests/specs/compile/include/folder_ts_file/output.out Normal file
View file

@ -0,0 +1,2 @@
[WILDLINE]add.ts
3

2
tests/specs/compile/include/folder_ts_file/src/main.ts Normal file
View file

@ -0,0 +1,2 @@
import { add } from "../math/add.ts";
console.log(add(1, 2));

4
tests/specs/compile/include/symlink_twice/__test__.jsonc
View file

@ -6,7 +6,7 @@
}, {
"if": "unix",
"args": "compile --allow-read=data --include . --output main link.js",
"output": "[WILDCARD]"
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
@ -16,7 +16,7 @@
}, {
"if": "windows",
"args": "compile --allow-read=data --include . --output main.exe link.js",
"output": "[WILDCARD]"
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",

9
tests/specs/compile/include/symlink_twice/compile.out Normal file
View file

@ -0,0 +1,9 @@
Compile [WILDLINE]
Embedded File System
main[WILDLINE]
├── index.js
├── link.js --> index.js
└── setup.js

1
tests/specs/compile/include/symlink_twice/setup.js
View file

@ -1,3 +1,2 @@
Deno.mkdirSync("data");
Deno.writeTextFileSync("index.js", "console.log(1);");
Deno.symlinkSync("index.js", "link.js");

24
tests/specs/compile/npm_fs/__test__.jsonc Normal file
View file

@ -0,0 +1,24 @@
{
"tempDir": true,
// use this so the vfs output is all in the same folder
"canonicalizedTempDir": true,
"steps": [{
"if": "unix",
"args": "compile -A --output main main.ts",
"output": "compile.out"
}, {
"if": "unix",
"commandName": "./main",
"args": [],
"output": "main.out"
}, {
"if": "windows",
"args": "compile -A --output main.exe main.ts",
"output": "compile.out"
}, {
"if": "windows",
"commandName": "./main.exe",
"args": [],
"output": "main.out"
}]
}

8
tests/specs/compile/npm_fs/compile.out Normal file
View file

@ -0,0 +1,8 @@
[WILDCARD]
Embedded File System
main[WILDLINE]
├── main.ts
└── node_modules/*

3
tests/specs/compile/npm_fs/deno.json Normal file
View file

@ -0,0 +1,3 @@
{
"nodeModulesDir": "auto"
}

0
tests/testdata/compile/npm_fs/main.out → tests/specs/compile/npm_fs/main.out
View file

0
tests/testdata/compile/npm_fs/main.ts → tests/specs/compile/npm_fs/main.ts
View file

16
tests/specs/mod.rs
View file

@ -118,6 +118,12 @@ struct MultiStepMetaData {
/// steps.
#[serde(default)]
pub temp_dir: bool,
/// Whether the temporary directory should be canonicalized.
///
/// This should be used sparingly, but is sometimes necessary
/// on the CI.
#[serde(default)]
pub canonicalized_temp_dir: bool,
/// Whether the temporary directory should be symlinked to another path.
#[serde(default)]
pub symlinked_temp_dir: bool,
@ -144,6 +150,8 @@ struct SingleTestMetaData {
#[serde(default)]
pub temp_dir: bool,
#[serde(default)]
pub canonicalized_temp_dir: bool,
#[serde(default)]
pub symlinked_temp_dir: bool,
#[serde(default)]
pub repeat: Option<usize>,
@ -159,6 +167,7 @@ impl SingleTestMetaData {
base: self.base,
cwd: None,
temp_dir: self.temp_dir,
canonicalized_temp_dir: self.canonicalized_temp_dir,
symlinked_temp_dir: self.symlinked_temp_dir,
repeat: self.repeat,
envs: Default::default(),
@ -326,6 +335,13 @@ fn test_context_from_metadata(
builder = builder.cwd(cwd.to_string_lossy());
}
if metadata.canonicalized_temp_dir {
// not actually deprecated, we just want to discourage its use
#[allow(deprecated)]
{
builder = builder.use_canonicalized_temp_dir();
}
}
if metadata.symlinked_temp_dir {
// not actually deprecated, we just want to discourage its use
// because it's mostly used for testing purposes locally

15
tests/specs/schema.json
View file

@ -36,6 +36,9 @@
"flaky": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
@ -66,6 +69,12 @@
"tempDir": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
"base": {
"type": "string"
},
@ -94,6 +103,12 @@
"tempDir": {
"type": "boolean"
},
"canonicalizedTempDir": {
"type": "boolean"
},
"symlinkedTempDir": {
"type": "boolean"
},
"base": {
"type": "string"
},

1
tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out vendored
View file

@ -1,5 +1,4 @@
Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD]
Warning Symlink target is outside '[WILDCARD]node_modules_symlink_outside'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file.
Embedded File System

9
tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out vendored
View file

@ -3,8 +3,13 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz
Initialize @denotest/esm-basic@1.0.0
Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts
Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE]
Warning Symlink target is outside '[WILDLINE]node_modules_symlink_outside'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'.
Embedded File System
[WILDCARD]
bin[WILDLINE]
├─┬ compile
│ └─┬ node_modules_symlink_outside
│ ├── main.ts
│ └── node_modules/*
└── some_folder/*

10
tests/util/server/src/lib.rs
View file

@ -816,15 +816,17 @@ pub fn wildcard_match_detailed(
}
let actual_next_text =
&current_text[max_current_text_found_index..];
let max_next_text_len = 40;
let next_text_len =
std::cmp::min(max_next_text_len, actual_next_text.len());
let next_text_len = actual_next_text
.chars()
.take(40)
.map(|c| c.len_utf8())
.sum::<usize>();
output_lines.push(format!(
"==== NEXT ACTUAL TEXT ====\n{}{}",
colors::red(annotate_whitespace(
&actual_next_text[..next_text_len]
)),
if actual_next_text.len() > max_next_text_len {
if actual_next_text.len() > next_text_len {
"[TRUNCATED]"
} else {
""