diff --git a/Cargo.lock b/Cargo.lock index cff3891e9c..910c1df06c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1156,6 +1156,7 @@ name = "deno_runtime" version = "0.97.0" dependencies = [ "atty", + "deno_ast", "deno_broadcast_channel", "deno_cache", "deno_console", diff --git a/bench_util/js_runtime.rs b/bench_util/js_runtime.rs index 103b974304..376d30593d 100644 --- a/bench_util/js_runtime.rs +++ b/bench_util/js_runtime.rs @@ -11,7 +11,7 @@ pub fn create_js_runtime(setup: impl FnOnce() -> Vec) -> JsRuntime { JsRuntime::new(RuntimeOptions { extensions_with_js: setup(), module_loader: Some(std::rc::Rc::new( - deno_core::InternalModuleLoader::new(None), + deno_core::InternalModuleLoader::default(), )), ..Default::default() }) diff --git a/cli/build.rs b/cli/build.rs index d4cfc47e95..d54e22be89 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -4,8 +4,10 @@ use std::env; use std::path::Path; use std::path::PathBuf; +use deno_core::include_js_files_dir; use deno_core::snapshot_util::*; use deno_core::Extension; +use deno_core::ExtensionFileSource; use deno_runtime::deno_cache::SqliteBackedCache; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::*; @@ -15,6 +17,7 @@ mod ts { use crate::deno_webgpu_get_declaration; use deno_core::error::custom_error; use deno_core::error::AnyError; + use deno_core::include_js_files_dir; use deno_core::op; use deno_core::OpState; use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES; @@ -32,11 +35,7 @@ mod ts { specifier: String, } - pub fn create_compiler_snapshot( - snapshot_path: PathBuf, - files: Vec, - cwd: &Path, - ) { + pub fn create_compiler_snapshot(snapshot_path: PathBuf, cwd: &Path) { // libs that are being provided by op crates. let mut op_crate_libs = HashMap::new(); op_crate_libs.insert("deno.cache", deno_cache::get_declaration()); @@ -252,36 +251,42 @@ mod ts { } } + let tsc_extension = Extension::builder("deno_tsc") + .ops(vec![ + op_build_info::decl(), + op_cwd::decl(), + op_exists::decl(), + op_is_node_file::decl(), + op_load::decl(), + op_script_version::decl(), + ]) + .js(include_js_files_dir! { + dir "tsc", + "00_typescript.js", + "99_main_compiler.js", + }) + .state(move |state| { + state.put(op_crate_libs.clone()); + state.put(build_libs.clone()); + state.put(path_dts.clone()); + + Ok(()) + }) + .build(); + create_snapshot(CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), snapshot_path, startup_snapshot: None, - extensions: vec![Extension::builder("deno_tsc") - .ops(vec![ - op_build_info::decl(), - op_cwd::decl(), - op_exists::decl(), - op_is_node_file::decl(), - op_load::decl(), - op_script_version::decl(), - ]) - .state(move |state| { - state.put(op_crate_libs.clone()); - state.put(build_libs.clone()); - state.put(path_dts.clone()); - - Ok(()) - }) - .build()], - extensions_with_js: vec![], - additional_files: files, - additional_esm_files: vec![], + extensions: vec![], + extensions_with_js: vec![tsc_extension], compression_cb: Some(Box::new(|vec, snapshot_slice| { vec.extend_from_slice( &zstd::bulk::compress(snapshot_slice, 22) .expect("snapshot compression failed"), ); })), + snapshot_module_load_cb: None, }); } @@ -307,7 +312,7 @@ mod ts { } } -fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec) { +fn create_cli_snapshot(snapshot_path: PathBuf) { let extensions: Vec = vec![ deno_webidl::init(), deno_console::init(), @@ -338,14 +343,23 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec) { deno_flash::init::(false), // No --unstable ]; + let mut esm_files = include_js_files_dir!( + dir "js", + "40_testing.js", + ); + esm_files.push(ExtensionFileSource { + specifier: "runtime/js/99_main.js".to_string(), + code: deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS, + }); + let extensions_with_js = + vec![Extension::builder("cli").esm(esm_files).build()]; + create_snapshot(CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), snapshot_path, startup_snapshot: Some(deno_runtime::js::deno_isolate_init()), extensions, - extensions_with_js: vec![], - additional_files: vec![], - additional_esm_files: esm_files, + extensions_with_js, compression_cb: Some(Box::new(|vec, snapshot_slice| { lzzzz::lz4_hc::compress_to_vec( snapshot_slice, @@ -354,6 +368,7 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec) { ) .expect("snapshot compression failed"); })), + snapshot_module_load_cb: None, }) } @@ -450,13 +465,10 @@ fn main() { let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin"); - let js_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "tsc", None); - ts::create_compiler_snapshot(compiler_snapshot_path, js_files, &c); + ts::create_compiler_snapshot(compiler_snapshot_path, &c); let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin"); - let mut esm_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "js", None); - esm_files.push(deno_runtime::js::get_99_main()); - create_cli_snapshot(cli_snapshot_path, esm_files); + create_cli_snapshot(cli_snapshot_path); #[cfg(target_os = "windows")] { diff --git a/core/extensions.rs b/core/extensions.rs index e08e8c5667..16cca924d0 100644 --- a/core/extensions.rs +++ b/core/extensions.rs @@ -6,6 +6,7 @@ use std::rc::Rc; use std::task::Context; use v8::fast_api::FastFunction; +#[derive(Clone, Debug)] pub struct ExtensionFileSource { pub specifier: String, pub code: &'static str, @@ -244,8 +245,9 @@ impl ExtensionBuilder { } } } -/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)> -/// representing the filename and source code. + +/// Helps embed JS files in an extension. Returns a vector of +/// `ExtensionFileSource`, that represent the filename and source code. /// /// Example: /// ```ignore @@ -265,3 +267,29 @@ macro_rules! include_js_files { ] }; } + +/// Helps embed JS files in an extension. Returns a vector of +/// `ExtensionFileSource`, that represent the filename and source code. +/// Additional "dir" option is required, that specifies which directory in the +/// crate root contains the listed files. "dir" option will be prepended to +/// each file name. +/// +/// Example: +/// ```ignore +/// include_js_files_dir!( +/// dir "example", +/// "01_hello.js", +/// "02_goodbye.js", +/// ) +/// ``` +#[macro_export] +macro_rules! include_js_files_dir { + (dir $dir:literal, $($file:literal,)+) => { + vec![ + $($crate::ExtensionFileSource { + specifier: concat!($dir, "/", $file).to_string(), + code: include_str!(concat!($dir, "/", $file)), + },)+ + ] + }; +} diff --git a/core/lib.rs b/core/lib.rs index 86c432d43d..a777013399 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -75,6 +75,7 @@ pub use crate::module_specifier::ModuleSpecifier; pub use crate::module_specifier::DUMMY_SPECIFIER; pub use crate::modules::FsModuleLoader; pub use crate::modules::InternalModuleLoader; +pub use crate::modules::InternalModuleLoaderCb; pub use crate::modules::ModuleId; pub use crate::modules::ModuleLoader; pub use crate::modules::ModuleSource; diff --git a/core/modules.rs b/core/modules.rs index da0d8cba39..b604e8c13c 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -2,6 +2,7 @@ use crate::bindings; use crate::error::generic_error; +use crate::extensions::ExtensionFileSource; use crate::module_specifier::ModuleSpecifier; use crate::resolve_import; use crate::resolve_url; @@ -312,13 +313,38 @@ pub(crate) fn resolve_helper( loader.resolve(specifier, referrer, kind) } -pub struct InternalModuleLoader(Rc); +/// Function that can be passed to the `InternalModuleLoader` that allows to +/// transpile sources before passing to V8. +pub type InternalModuleLoaderCb = + Box Result>; + +pub struct InternalModuleLoader { + module_loader: Rc, + esm_sources: Vec, + maybe_load_callback: Option, +} + +impl Default for InternalModuleLoader { + fn default() -> Self { + Self { + module_loader: Rc::new(NoopModuleLoader), + esm_sources: vec![], + maybe_load_callback: None, + } + } +} impl InternalModuleLoader { - pub fn new(module_loader: Option>) -> Self { - InternalModuleLoader( - module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)), - ) + pub fn new( + module_loader: Option>, + esm_sources: Vec, + maybe_load_callback: Option, + ) -> Self { + InternalModuleLoader { + module_loader: module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)), + esm_sources, + maybe_load_callback, + } } } @@ -343,7 +369,7 @@ impl ModuleLoader for InternalModuleLoader { } } - self.0.resolve(specifier, referrer, kind) + self.module_loader.resolve(specifier, referrer, kind) } fn load( @@ -352,7 +378,46 @@ impl ModuleLoader for InternalModuleLoader { maybe_referrer: Option, is_dyn_import: bool, ) -> Pin> { - self.0.load(module_specifier, maybe_referrer, is_dyn_import) + if module_specifier.scheme() != "internal" { + return self.module_loader.load( + module_specifier, + maybe_referrer, + is_dyn_import, + ); + } + + let specifier = module_specifier.to_string(); + let maybe_file_source = self + .esm_sources + .iter() + .find(|file_source| file_source.specifier == module_specifier.as_str()); + + if let Some(file_source) = maybe_file_source { + let result = if let Some(load_callback) = &self.maybe_load_callback { + load_callback(file_source) + } else { + Ok(file_source.code.to_string()) + }; + + return async move { + let code = result?; + let source = ModuleSource { + code: code.into_bytes().into_boxed_slice(), + module_type: ModuleType::JavaScript, + module_url_specified: specifier.clone(), + module_url_found: specifier.clone(), + }; + Ok(source) + } + .boxed_local(); + } + + async move { + Err(generic_error(format!( + "Cannot find internal module source for specifier {specifier}" + ))) + } + .boxed_local() } fn prepare_load( @@ -366,7 +431,7 @@ impl ModuleLoader for InternalModuleLoader { return async { Ok(()) }.boxed_local(); } - self.0.prepare_load( + self.module_loader.prepare_load( op_state, module_specifier, maybe_referrer, @@ -2639,7 +2704,7 @@ if (import.meta.url != 'file:///main_with_code.js') throw Error(); #[test] fn internal_module_loader() { - let loader = InternalModuleLoader::new(None); + let loader = InternalModuleLoader::default(); assert!(loader .resolve("internal:foo", "internal:bar", ResolutionKind::Import) .is_ok()); diff --git a/core/runtime.rs b/core/runtime.rs index 377137a012..d922d0ca5a 100644 --- a/core/runtime.rs +++ b/core/runtime.rs @@ -8,6 +8,7 @@ use crate::extensions::OpDecl; use crate::extensions::OpEventLoopFn; use crate::inspector::JsRuntimeInspector; use crate::module_specifier::ModuleSpecifier; +use crate::modules::InternalModuleLoaderCb; use crate::modules::ModuleError; use crate::modules::ModuleId; use crate::modules::ModuleLoadId; @@ -19,6 +20,7 @@ use crate::ops::*; use crate::source_map::SourceMapCache; use crate::source_map::SourceMapGetter; use crate::Extension; +use crate::ExtensionFileSource; use crate::NoopModuleLoader; use crate::OpMiddlewareFn; use crate::OpResult; @@ -270,6 +272,11 @@ pub struct RuntimeOptions { /// The snapshot is deterministic and uses predictable random numbers. pub will_snapshot: bool, + /// An optional callback that will be called for each module that is loaded + /// during snapshotting. This callback can be used to transpile source on the + /// fly, during snapshotting, eg. to transpile TypeScript to JavaScript. + pub snapshot_module_load_cb: Option, + /// Isolate creation parameters. pub create_params: Option, @@ -607,8 +614,16 @@ impl JsRuntime { }; let loader = if snapshot_options != SnapshotOptions::Load { + let esm_sources = options + .extensions_with_js + .iter() + .flat_map(|ext| ext.get_esm_sources().to_owned()) + .collect::>(); + Rc::new(crate::modules::InternalModuleLoader::new( options.module_loader, + esm_sources, + options.snapshot_module_load_cb, )) } else { options @@ -818,13 +833,13 @@ impl JsRuntime { let extensions = std::mem::take(&mut self.extensions_with_js); for ext in &extensions { { - let js_files = ext.get_esm_sources(); - for file_source in js_files { + let esm_files = ext.get_esm_sources(); + for file_source in esm_files { futures::executor::block_on(async { let id = self .load_side_module( &ModuleSpecifier::parse(&file_source.specifier)?, - Some(file_source.code.to_string()), + None, ) .await?; let receiver = self.mod_evaluate(id); diff --git a/core/snapshot_util.rs b/core/snapshot_util.rs index 8daaa98660..0a1793b513 100644 --- a/core/snapshot_util.rs +++ b/core/snapshot_util.rs @@ -1,12 +1,11 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use anyhow::Context; use std::path::Path; use std::path::PathBuf; use crate::Extension; +use crate::InternalModuleLoaderCb; use crate::JsRuntime; -use crate::ModuleSpecifier; use crate::RuntimeOptions; use crate::Snapshot; @@ -18,57 +17,20 @@ pub struct CreateSnapshotOptions { pub startup_snapshot: Option, pub extensions: Vec, pub extensions_with_js: Vec, - pub additional_files: Vec, - pub additional_esm_files: Vec, pub compression_cb: Option>, + pub snapshot_module_load_cb: Option, } pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) { - let mut js_runtime = JsRuntime::new(RuntimeOptions { + let js_runtime = JsRuntime::new(RuntimeOptions { will_snapshot: true, startup_snapshot: create_snapshot_options.startup_snapshot, extensions: create_snapshot_options.extensions, extensions_with_js: create_snapshot_options.extensions_with_js, + snapshot_module_load_cb: create_snapshot_options.snapshot_module_load_cb, ..Default::default() }); - // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the - // workspace root. - let display_root = Path::new(create_snapshot_options.cargo_manifest_dir) - .parent() - .unwrap(); - for file in create_snapshot_options.additional_files { - let display_path = file.strip_prefix(display_root).unwrap_or(&file); - let display_path_str = display_path.display().to_string(); - js_runtime - .execute_script( - &("internal:".to_string() + &display_path_str.replace('\\', "/")), - &std::fs::read_to_string(&file).unwrap(), - ) - .unwrap(); - } - for file in create_snapshot_options.additional_esm_files { - let display_path = file.strip_prefix(display_root).unwrap_or(&file); - let display_path_str = display_path.display().to_string(); - - let filename = - &("internal:".to_string() + &display_path_str.replace('\\', "/")); - - futures::executor::block_on(async { - let id = js_runtime - .load_side_module( - &ModuleSpecifier::parse(filename)?, - Some(std::fs::read_to_string(&file)?), - ) - .await?; - let receiver = js_runtime.mod_evaluate(id); - js_runtime.run_event_loop(false).await?; - receiver.await? - }) - .with_context(|| format!("Couldn't execute '{}'", file.display())) - .unwrap(); - } - let snapshot = js_runtime.snapshot(); let snapshot_slice: &[u8] = &snapshot; println!("Snapshot size: {}", snapshot_slice.len()); diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 039bb52931..2d93f9dff2 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -24,6 +24,7 @@ name = "hello_runtime" path = "examples/hello_runtime.rs" [build-dependencies] +deno_ast.workspace = true deno_broadcast_channel.workspace = true deno_cache.workspace = true deno_console.workspace = true diff --git a/runtime/build.rs b/runtime/build.rs index bba4394f82..e892e74854 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,7 +1,7 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +use deno_core::include_js_files_dir; use std::env; - use std::path::PathBuf; // This is a shim that allows to generate documentation on docs.rs @@ -13,6 +13,39 @@ mod not_docs { use deno_core::snapshot_util::*; use deno_core::Extension; + use deno_ast::MediaType; + use deno_ast::ParseParams; + use deno_ast::SourceTextInfo; + use deno_core::error::AnyError; + use deno_core::ExtensionFileSource; + + fn transpile_ts_for_snapshotting( + file_source: &ExtensionFileSource, + ) -> Result { + let media_type = MediaType::from(Path::new(&file_source.specifier)); + + let should_transpile = match media_type { + MediaType::JavaScript => false, + MediaType::TypeScript => true, + _ => panic!("Unsupported media type for snapshotting {media_type:?}"), + }; + + if !should_transpile { + return Ok(file_source.code.to_string()); + } + + let parsed = deno_ast::parse_module(ParseParams { + specifier: file_source.specifier.to_string(), + text_info: SourceTextInfo::from_string(file_source.code.to_string()), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + })?; + let transpiled_source = parsed.transpile(&Default::default())?; + Ok(transpiled_source.text) + } + struct Permissions; impl deno_fetch::FetchPermissions for Permissions { @@ -120,7 +153,10 @@ mod not_docs { } } - fn create_runtime_snapshot(snapshot_path: PathBuf, esm_files: Vec) { + fn create_runtime_snapshot( + snapshot_path: PathBuf, + additional_extension: Extension, + ) { let extensions_with_js: Vec = vec![ deno_webidl::init(), deno_console::init(), @@ -149,6 +185,7 @@ mod not_docs { deno_napi::init::(false), deno_http::init(), deno_flash::init::(false), // No --unstable + additional_extension, ]; create_snapshot(CreateSnapshotOptions { @@ -157,8 +194,6 @@ mod not_docs { startup_snapshot: None, extensions: vec![], extensions_with_js, - additional_files: vec![], - additional_esm_files: esm_files, compression_cb: Some(Box::new(|vec, snapshot_slice| { lzzzz::lz4_hc::compress_to_vec( snapshot_slice, @@ -167,24 +202,50 @@ mod not_docs { ) .expect("snapshot compression failed"); })), + snapshot_module_load_cb: Some(Box::new(transpile_ts_for_snapshotting)), }); } pub fn build_snapshot(runtime_snapshot_path: PathBuf) { - #[allow(unused_mut)] - let mut esm_files = get_js_files( - env!("CARGO_MANIFEST_DIR"), - "js", - Some(Box::new(|path| !path.ends_with("99_main.js"))), + #[allow(unused_mut, unused_assignments)] + let mut esm_files = include_js_files_dir!( + dir "js", + "01_build.js", + "01_errors.js", + "01_version.ts", + "06_util.js", + "10_permissions.js", + "11_workers.js", + "12_io.js", + "13_buffer.js", + "30_fs.js", + "30_os.js", + "40_diagnostics.js", + "40_files.js", + "40_fs_events.js", + "40_http.js", + "40_process.js", + "40_read_file.js", + "40_signals.js", + "40_spawn.js", + "40_tty.js", + "40_write_file.js", + "41_prompt.js", + "90_deno_ns.js", + "98_global_scope.js", ); #[cfg(not(feature = "snapshot_from_snapshot"))] { - let manifest = env!("CARGO_MANIFEST_DIR"); - let path = PathBuf::from(manifest); - esm_files.push(path.join("js").join("99_main.js")); + esm_files.push(ExtensionFileSource { + specifier: "js/99_main.js".to_string(), + code: include_str!("js/99_main.js"), + }); } - create_runtime_snapshot(runtime_snapshot_path, esm_files); + + let additional_extension = + Extension::builder("runtime").esm(esm_files).build(); + create_runtime_snapshot(runtime_snapshot_path, additional_extension); } } diff --git a/runtime/js.rs b/runtime/js.rs index 3d14c744cb..82e58a09cc 100644 --- a/runtime/js.rs +++ b/runtime/js.rs @@ -2,7 +2,6 @@ use deno_core::Snapshot; use log::debug; use once_cell::sync::Lazy; -use std::path::PathBuf; pub static RUNTIME_SNAPSHOT: Lazy> = Lazy::new( #[allow(clippy::uninit_vec)] @@ -35,8 +34,5 @@ pub fn deno_isolate_init() -> Snapshot { Snapshot::Static(&RUNTIME_SNAPSHOT) } -pub fn get_99_main() -> PathBuf { - let manifest = env!("CARGO_MANIFEST_DIR"); - let path = PathBuf::from(manifest); - path.join("js").join("99_main.js") -} +#[cfg(feature = "snapshot_from_snapshot")] +pub static SOURCE_CODE_FOR_99_MAIN_JS: &str = include_str!("js/99_main.js"); diff --git a/runtime/js/01_version.js b/runtime/js/01_version.ts similarity index 80% rename from runtime/js/01_version.js rename to runtime/js/01_version.ts index 62f3df17c8..cbbbd8d03e 100644 --- a/runtime/js/01_version.js +++ b/runtime/js/01_version.ts @@ -3,7 +3,13 @@ const primordials = globalThis.__bootstrap.primordials; const { ObjectFreeze } = primordials; -const version = { +interface Version { + deno: string; + v8: string; + typescript: string; +} + +const version: Version = { deno: "", v8: "", typescript: "", diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index 5321bf1d3f..fe59ea1a79 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -12,7 +12,7 @@ import * as http from "internal:deno_http/01_http.js"; import * as flash from "internal:deno_flash/01_http.js"; import * as build from "internal:runtime/js/01_build.js"; import * as errors from "internal:runtime/js/01_errors.js"; -import * as version from "internal:runtime/js/01_version.js"; +import * as version from "internal:runtime/js/01_version.ts"; import * as permissions from "internal:runtime/js/10_permissions.js"; import * as io from "internal:runtime/js/12_io.js"; import * as buffer from "internal:runtime/js/13_buffer.js"; diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 5d9012bc4c..fd7b93e24e 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -43,7 +43,7 @@ import * as util from "internal:runtime/js/06_util.js"; import * as event from "internal:deno_web/02_event.js"; import * as location from "internal:deno_web/12_location.js"; import * as build from "internal:runtime/js/01_build.js"; -import * as version from "internal:runtime/js/01_version.js"; +import * as version from "internal:runtime/js/01_version.ts"; import * as os from "internal:runtime/js/30_os.js"; import * as timers from "internal:deno_web/02_timers.js"; import * as colors from "internal:deno_console/01_colors.js";