From 32d7856189ea47fff41f79632ed90e3837ef7287 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Mon, 25 Nov 2024 20:42:03 +0100 Subject: [PATCH] WIP --- cli/args/flags.rs | 38 ++++ cli/tools/bundle/mod.rs | 359 ++++++++++++++++++++++++++++++----- resolvers/node/resolution.rs | 5 +- 3 files changed, 350 insertions(+), 52 deletions(-) diff --git a/cli/args/flags.rs b/cli/args/flags.rs index daac1a44a8..0ac746cdf7 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -368,9 +368,17 @@ pub enum BundlePlatform { Browser, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum BundleSourceMap { + Inline, + External, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct BundleFlags { pub files: FileFlags, + pub minify: bool, + pub source_map: Option, pub platform: BundlePlatform, pub out_dir: String, pub watch: Option, @@ -387,6 +395,8 @@ impl BundleFlags { files, platform, out_dir, + minify: false, + source_map: None, watch: None, } } @@ -1860,6 +1870,18 @@ fn bundle_subcommand() -> Command { .help("Target platform. Must be one of: 'deno' or 'browser' (Default: 'deno')") .action(ArgAction::Set) ) + .arg( + Arg::new("minify") + .long("minify") + .help("Minify bundled code") + .action(ArgAction::SetTrue) + ) + .arg( + Arg::new("source-map") + .long("source-map") + .help("Minify bundled code") + .action(ArgAction::Set) + ) .arg( Arg::new("files") .num_args(1..) @@ -4631,6 +4653,20 @@ fn bundle_parse( None => Ok(BundlePlatform::Deno), }?; + let minify = matches.get_flag("minify"); + let source_map = + if let Some(value) = matches.remove_one::("source-map") { + match value.as_str() { + "inline" => Ok(Some(BundleSourceMap::Inline)), + "external" => Ok(Some(BundleSourceMap::External)), + _ => Err(clap::error::Error::new( + clap::error::ErrorKind::InvalidValue, + )), + } + } else { + Ok(None) + }?; + flags.subcommand = DenoSubcommand::Bundle(BundleFlags { files: FileFlags { include: files, @@ -4638,6 +4674,8 @@ fn bundle_parse( }, platform, out_dir, + minify, + source_map, watch: watch_arg_parse(matches)?, }); diff --git a/cli/tools/bundle/mod.rs b/cli/tools/bundle/mod.rs index eeb6fe5652..12c0978bed 100644 --- a/cli/tools/bundle/mod.rs +++ b/cli/tools/bundle/mod.rs @@ -1,21 +1,54 @@ use std::{ + cmp::Ordering, collections::{HashMap, HashSet}, fs, - path::Path, + io::Write, + path::{Path, PathBuf}, sync::Arc, }; -use deno_core::{error::AnyError, url::Url}; -use deno_graph::{GraphKind, Module, ModuleGraph}; -use deno_runtime::colors; +use deno_core::{anyhow::Context, error::AnyError, url::Url}; +use deno_graph::{GraphKind, Module, ModuleGraph, NpmModule, Resolution}; +use deno_runtime::{colors, deno_node::NodeResolver}; +use deno_semver::package; +use flate2::{write::ZlibEncoder, Compression}; +use indexmap::IndexSet; +use node_resolver::{NodeModuleKind, NodeResolutionMode}; use crate::{ args::{BundleFlags, BundlePlatform, Flags}, factory::CliFactory, graph_util::CreateGraphOptions, + npm::CliNpmResolver, + resolver::CjsTracker, util::{fs::collect_specifiers, path::matches_pattern_or_exact_path}, }; +#[derive(Debug)] +struct BundleChunkStat { + name: PathBuf, + size: usize, + gzip: usize, + brotli: usize, +} + +#[derive(Debug)] +enum BundleGraphModKind { + Asset(String), + Js, + Json, + Wasm, + Node, +} + +#[derive(Debug)] +struct BundleGraphMod { + id: usize, + kind: BundleGraphModKind, + used_count: usize, + has_side_effects: bool, +} + pub async fn bundle( flags: Arc, bundle_flags: BundleFlags, @@ -23,8 +56,9 @@ pub async fn bundle( // FIXME: Permissions let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; - let deno_dir = factory.deno_dir()?; - let http_client = factory.http_client_provider(); + let npm_resolver = factory.npm_resolver().await?; + let node_resolver = factory.node_resolver().await?; + let cjs_tracker = factory.cjs_tracker()?; // TODO: Ensure that dependencies are installed @@ -53,12 +87,110 @@ pub async fn bundle( graph.valid()?; + let bundle_graph: HashMap = HashMap::new(); + + let mut id = 0; + let mut all_modules: HashMap = HashMap::new(); + let mut module_to_id: HashMap = HashMap::new(); + let mut id_to_module: HashMap = HashMap::new(); + + let mut npm_modules: IndexSet = IndexSet::new(); + let mut seen: IndexSet = IndexSet::new(); + + fn resolve_npm_module( + module: &NpmModule, + npm_resolver: &Arc, + node_resolver: &Arc, + ) -> Url { + let nv = module.nv_reference.nv(); + let managed = npm_resolver.as_managed().unwrap(); + let package_folder = + managed.resolve_pkg_folder_from_deno_module(nv).unwrap(); + + let resolved = node_resolver + .resolve_package_subpath_from_deno_module( + &package_folder, + module.nv_reference.sub_path(), + None, // FIXME + NodeModuleKind::Esm, // FIXME + NodeResolutionMode::Execution, + ) + .with_context(|| format!("Could not resolve '{}'.", module.nv_reference)) + .unwrap(); + + resolved + } + + // Hack: Create sub graphs for every npm module we encounter and + // expand npm specifiers to the actual file + for module in graph.modules() { + let url = module.specifier(); + seen.insert(url.clone()); + + let current_id = id; + module_to_id.insert(url.clone(), current_id); + id_to_module.insert(current_id, url.clone()); + + id += 1; + + match module { + Module::Npm(module) => { + let resolved = resolve_npm_module(&module, npm_resolver, node_resolver); + npm_modules.insert(resolved.clone()); + } + _ => { + all_modules.insert(url.clone(), module.clone()); + } + } + } + + let npm_modules_vec = + npm_modules.iter().map(|u| u.clone()).collect::>(); + + eprintln!("npm vec: {:#?}", npm_modules_vec); + while let Some(url) = npm_modules.pop() { + let npm_graph = module_graph_creator + .create_graph_with_options(CreateGraphOptions { + graph_kind: GraphKind::CodeOnly, + roots: vec![url.clone()], + is_dynamic: false, + loader: None, + }) + .await?; + + for module in npm_graph.modules() { + if seen.contains(module.specifier()) { + continue; + } + + match module { + Module::Npm(module) => { + let resolved = + resolve_npm_module(&module, npm_resolver, node_resolver); + npm_modules.insert(resolved.clone()); + } + _ => { + all_modules.insert(url.clone(), module.clone()); + } + } + } + + eprintln!("RES {:#?}", all_modules); + } + let mut chunk_graph = ChunkGraph::new(); for file in files { - let chunk_id = chunk_graph.new_chunk(None); - if let Some(module) = graph.get(&file) { - assign_chunks(&bundle_flags, &mut chunk_graph, &graph, module, chunk_id); - } + assign_chunks( + &bundle_flags, + &mut chunk_graph, + &all_modules, + npm_resolver, + node_resolver, + cjs_tracker, + &file, + None, + true, + ); } // Hoist shared modules into common parent chunk that is not a root chunk @@ -68,15 +200,18 @@ pub async fn bundle( let out_dir = Path::new(&bundle_flags.out_dir); fs::create_dir_all(out_dir)?; + let mut stats: Vec = vec![]; + let mut cols = (8, 4, 4, 6); + // Write out chunks // TODO: Walk topo for chunk hashes for (_id, chunk) in &chunk_graph.chunks { //chunk let mut source = String::new(); - for spec in &chunk.specifiers { + for spec in chunk.specifiers.iter().rev() { if let Some(module) = graph.get(&spec) { - // + // FIXME: don't print module urls by default source.push_str(&format!("// {}\n", spec.to_string())); if let Some(contents) = &module.source() { source.push_str(contents); @@ -84,58 +219,123 @@ pub async fn bundle( } } - let out_path = out_dir.join("out.js"); - fs::write(&out_path, source).unwrap(); - log::log!( - log::Level::Info, - "{} {}", - colors::green("Filename"), - colors::green("Size") - ); - log::log!( - log::Level::Info, - " {} {}", - out_path.to_string_lossy(), - colors::cyan(0) - ); - log::log!(log::Level::Info, ""); + let out_path = out_dir.join(chunk.name.to_string()); + fs::write(&out_path, &source).unwrap(); + + let out_len = out_path.to_string_lossy().len(); + if out_len > cols.0 { + cols.0 = out_len; + } + + let mut gzip_writer = ZlibEncoder::new(vec![], Compression::default()); + gzip_writer.write_all(source.as_bytes())?; + let gzip_compressed = gzip_writer.finish()?; + + stats.push(BundleChunkStat { + name: out_path.clone(), + size: source.len(), + gzip: gzip_compressed.len(), + brotli: 0, + }); } + // Sort to show biggest files first + stats.sort_by(|a, b| { + if a.gzip > b.gzip { + Ordering::Greater + } else if a.gzip < b.gzip { + Ordering::Less + } else { + Ordering::Equal + } + }); + + log::log!( + log::Level::Info, + "{} {} {} {}", + colors::green(&format!("{:width$}", stat.size, width = cols.1)), + colors::cyan(&format!("{:>width$}", stat.gzip, width = cols.2)), + colors::cyan(&format!("{:>width$}", stat.brotli, width = cols.3)) + ); + } + log::log!(log::Level::Info, ""); + + // eprintln!("chunk {:#?}", chunk_graph); + Ok(()) } fn assign_chunks( bundle_flags: &BundleFlags, chunk_graph: &mut ChunkGraph, - graph: &ModuleGraph, - module: &Module, - chunk_id: usize, + graph: &HashMap, + npm_resolver: &Arc, + node_resolver: &Arc, + cjs_tracker: &Arc, + url: &Url, + parent_chunk_id: Option, + is_dynamic: bool, ) { + let module = graph.get(url).unwrap(); + match module { Module::Js(js_module) => { - chunk_graph.assign_specifier(js_module.specifier.clone(), chunk_id); + let chunk_id = chunk_graph.assign_specifier_to_chunk( + url, + parent_chunk_id, + ChunkKind::Js, + is_dynamic, + ); - for (value, dep) in &js_module.dependencies { - let url = Url::parse(value).unwrap(); - let chunk_id = if dep.is_dynamic { - chunk_graph.new_chunk(Some(chunk_id)) - } else { - chunk_id - }; - - if let Some(module) = graph.get(&url) { - assign_chunks(bundle_flags, chunk_graph, graph, module, chunk_id); + for (_, dep) in &js_module.dependencies { + match &dep.maybe_code { + Resolution::None => todo!(), + Resolution::Ok(resolution_resolved) => { + assign_chunks( + bundle_flags, + chunk_graph, + graph, + npm_resolver, + node_resolver, + cjs_tracker, + &resolution_resolved.specifier, + Some(chunk_id), + dep.is_dynamic, + ); + } + Resolution::Err(resolution_error) => todo!(), } } } Module::Json(json_module) => { - chunk_graph.assign_specifier(json_module.specifier.clone(), chunk_id); + chunk_graph.assign_specifier_to_chunk( + url, + parent_chunk_id, + ChunkKind::Js, + is_dynamic, + ); } Module::Wasm(wasm_module) => { - let chunk_id = chunk_graph.new_chunk(Some(chunk_id)); - chunk_graph.assign_specifier(wasm_module.specifier.clone(), chunk_id); + chunk_graph.assign_specifier_to_chunk( + url, + parent_chunk_id, + ChunkKind::Asset("wasm".to_string()), + true, + ); + } + Module::Npm(_) => { + unreachable!() } - Module::Npm(npm_module) => todo!(), Module::Node(built_in_node_module) => { if let BundlePlatform::Browser = bundle_flags.platform { // TODO: Show where it was imported from @@ -150,12 +350,20 @@ fn assign_chunks( } } +#[derive(Debug, Eq, PartialEq)] +enum ChunkKind { + Asset(String), + Js, +} + #[derive(Debug)] struct Chunk { id: usize, + name: String, + pub kind: ChunkKind, parent_ids: HashSet, children: Vec, // TODO: IndexSet? - specifiers: HashSet, + specifiers: IndexSet, } #[derive(Debug)] @@ -176,7 +384,49 @@ impl ChunkGraph { } } - fn assign_specifier(&mut self, url: Url, chunk_id: usize) { + fn get_or_create_chunk( + &mut self, + url: &Url, + parent_chunk_id: Option, + kind: ChunkKind, + is_dynamic: bool, + ) -> usize { + if let Some(parent_chunk_id) = parent_chunk_id { + if !is_dynamic { + return parent_chunk_id; + } + } + + let name = if let Ok(f) = url.to_file_path() { + if let Some(name) = f.file_stem() { + name.to_string_lossy().to_string() + } else { + format!("chunk_{}", self.id) + } + } else { + format!("chunk_{}", self.id) + }; + + let ext = match &kind { + ChunkKind::Asset(ext) => ext, + ChunkKind::Js => "js", + }; + + let full_name = format!("{}.{}", name, ext); + + self.new_chunk(full_name, parent_chunk_id, kind) + } + + fn assign_specifier_to_chunk( + &mut self, + url: &Url, + parent_chunk_id: Option, + kind: ChunkKind, + is_dynamic: bool, + ) -> usize { + let chunk_id = + self.get_or_create_chunk(&url, parent_chunk_id, kind, is_dynamic); + if let Some(value) = self.specifier_to_chunks.get_mut(&url) { value.push(chunk_id) } else { @@ -185,11 +435,18 @@ impl ChunkGraph { } if let Some(chunk) = self.chunks.get_mut(&chunk_id) { - chunk.specifiers.insert(url); + chunk.specifiers.insert(url.clone()); } + + chunk_id } - fn new_chunk(&mut self, parent_id: Option) -> usize { + fn new_chunk( + &mut self, + name: String, + parent_id: Option, + kind: ChunkKind, + ) -> usize { let id = self.id; self.id += 1; @@ -202,9 +459,11 @@ impl ChunkGraph { let chunk = Chunk { id, + name, + kind, parent_ids, children: vec![], - specifiers: HashSet::new(), + specifiers: IndexSet::new(), }; self.chunks.insert(id, chunk); diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index c2ec25aca4..61ed1d0aa9 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -46,8 +46,9 @@ use crate::PackageJsonResolverRc; use crate::PathClean; use deno_package_json::PackageJson; -pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"]; -pub static REQUIRE_CONDITIONS: &[&str] = &["require", "node"]; +// FIXME: Wire this through +pub static DEFAULT_CONDITIONS: &[&str] = &["browser", "deno", "node", "import"]; +pub static REQUIRE_CONDITIONS: &[&str] = &["browser", "require", "node"]; static TYPES_ONLY_CONDITIONS: &[&str] = &["types"]; fn conditions_from_module_kind(