fix(publish): lazily parse sources (#22301)

Closes #22290

cli/cache/mod.rs

@@ -46,6 +46,7 @@ pub use emit::EmitCache;
 pub use incremental::IncrementalCache;
 pub use module_info::ModuleInfoCache;
 pub use node::NodeAnalysisCache;
+pub use parsed_source::LazyGraphSourceParser;
 pub use parsed_source::ParsedSourceCache;
 
 /// Permissions used to save a file in the disk caches.

cli/cache/parsed_source.rs

@@ -11,6 +11,39 @@ use deno_graph::CapturingModuleParser;
 use deno_graph::ModuleParser;
 use deno_graph::ParseOptions;
 
+/// Lazily parses JS/TS sources from a `deno_graph::ModuleGraph` given
+/// a `ParsedSourceCache`. Note that deno_graph doesn't necessarily cause
+/// files to end up in the `ParsedSourceCache` because it might have all
+/// the information it needs via caching in order to skip parsing.
+#[derive(Clone, Copy)]
+pub struct LazyGraphSourceParser<'a> {
+  cache: &'a ParsedSourceCache,
+  graph: &'a deno_graph::ModuleGraph,
+}
+
+impl<'a> LazyGraphSourceParser<'a> {
+  pub fn new(
+    cache: &'a ParsedSourceCache,
+    graph: &'a deno_graph::ModuleGraph,
+  ) -> Self {
+    Self { cache, graph }
+  }
+
+  pub fn get_or_parse_source(
+    &self,
+    module_specifier: &ModuleSpecifier,
+  ) -> Result<Option<deno_ast::ParsedSource>, deno_ast::Diagnostic> {
+    let Some(deno_graph::Module::Js(module)) = self.graph.get(module_specifier)
+    else {
+      return Ok(None);
+    };
+    self
+      .cache
+      .get_parsed_source_from_js_module(module)
+      .map(Some)
+  }
+}
+
 #[derive(Default)]
 pub struct ParsedSourceCache {
   sources: Mutex<HashMap<ModuleSpecifier, ParsedSource>>,

cli/diagnostics.rs

@@ -11,10 +11,11 @@ use deno_ast::SourcePos;
 use deno_ast::SourceRange;
 use deno_ast::SourceRanged;
 use deno_ast::SourceTextInfo;
-use deno_graph::ParsedSourceStore;
 use deno_runtime::colors;
 use unicode_width::UnicodeWidthStr;
 
+use crate::cache::LazyGraphSourceParser;
+
 pub trait SourceTextStore {
   fn get_source_text<'a>(
     &'a self,
@@ -22,14 +23,14 @@ pub trait SourceTextStore {
   ) -> Option<Cow<'a, SourceTextInfo>>;
 }
 
-pub struct SourceTextParsedSourceStore<'a>(pub &'a dyn ParsedSourceStore);
+pub struct SourceTextParsedSourceStore<'a>(pub LazyGraphSourceParser<'a>);
 
 impl SourceTextStore for SourceTextParsedSourceStore<'_> {
   fn get_source_text<'a>(
     &'a self,
     specifier: &ModuleSpecifier,
   ) -> Option<Cow<'a, SourceTextInfo>> {
-    let parsed_source = self.0.get_parsed_source(specifier)?;
+    let parsed_source = self.0.get_or_parse_source(specifier).ok()??;
     Some(Cow::Owned(parsed_source.text_info().clone()))
   }
 }

cli/tests/integration/publish_tests.rs

@@ -66,6 +66,37 @@ itest!(invalid_import {
   http_server: true,
 });
 
+#[test]
+fn publish_non_exported_files_using_import_map() {
+  let context = publish_context_builder().build();
+  let temp_dir = context.temp_dir().path();
+  temp_dir.join("deno.json").write_json(&json!({
+    "name": "@foo/bar",
+    "version": "1.0.0",
+    "exports": "./mod.ts",
+    "imports": {
+      "@denotest/add": "jsr:@denotest/add@1"
+    }
+  }));
+  // file not in the graph
+  let other_ts = temp_dir.join("_other.ts");
+  other_ts
+    .write("import { add } from '@denotest/add'; console.log(add(1, 3));");
+  let mod_ts = temp_dir.join("mod.ts");
+  mod_ts.write("import { add } from '@denotest/add'; console.log(add(1, 2));");
+  let output = context
+    .new_command()
+    .args("publish --log-level=debug --token 'sadfasdf'")
+    .run();
+  let lines = output.combined_output().split('\n').collect::<Vec<_>>();
+  assert!(lines
+    .iter()
+    .any(|l| l.contains("Unfurling") && l.ends_with("mod.ts")));
+  assert!(lines
+    .iter()
+    .any(|l| l.contains("Unfurling") && l.ends_with("other.ts")));
+}
+
 itest!(javascript_missing_decl_file {
   args: "publish --token 'sadfasdf'",
   output: "publish/javascript_missing_decl_file.out",

cli/tests/testdata/jsr/registry/@denotest/add/1.0.0/mod.ts

@@ -0,0 +1,3 @@
+export function add(a: number, b: number): number {
+  return a + b;
+}

cli/tests/testdata/jsr/registry/@denotest/add/1.0.0_meta.json

@@ -0,0 +1,8 @@
+{
+  "exports": {
+    ".": "./mod.ts"
+  },
+  "moduleGraph1": {
+    "/mod.ts": {}
+  }
+}

cli/tests/testdata/jsr/registry/@denotest/add/meta.json

@@ -0,0 +1,5 @@
+{
+  "versions": {
+    "1.0.0": {}
+  }
+}

cli/tools/doc.rs

@@ -4,6 +4,7 @@ use crate::args::DocFlags;
 use crate::args::DocHtmlFlag;
 use crate::args::DocSourceFileFlag;
 use crate::args::Flags;
+use crate::cache::LazyGraphSourceParser;
 use crate::colors;
 use crate::diagnostics::Diagnostic;
 use crate::diagnostics::DiagnosticLevel;
@@ -142,7 +143,10 @@ pub async fn doc(flags: Flags, doc_flags: DocFlags) -> Result<(), AnyError> {
 
       if doc_flags.lint {
         let diagnostics = doc_parser.take_diagnostics();
-        check_diagnostics(&**parsed_source_cache, &diagnostics)?;
+        check_diagnostics(
+          LazyGraphSourceParser::new(parsed_source_cache, &graph),
+          &diagnostics,
+        )?;
       }
 
       doc_nodes_by_url
@@ -413,7 +417,7 @@ impl Diagnostic for DocDiagnostic {
 }
 
 fn check_diagnostics(
-  parsed_source_cache: &dyn deno_graph::ParsedSourceStore,
+  source_parser: LazyGraphSourceParser,
   diagnostics: &[DocDiagnostic],
 ) -> Result<(), AnyError> {
   if diagnostics.is_empty() {
@@ -437,8 +441,8 @@ fn check_diagnostics(
     for (_, diagnostics_by_col) in diagnostics_by_lc {
       for (_, diagnostics) in diagnostics_by_col {
         for diagnostic in diagnostics {
-          let sources = SourceTextParsedSourceStore(parsed_source_cache);
+          let sources = SourceTextParsedSourceStore(source_parser);
-          eprintln!("{}", diagnostic.display(&sources));
+          log::error!("{}", diagnostic.display(&sources));
         }
       }
     }

cli/tools/registry/diagnostics.rs

@@ -10,9 +10,9 @@ use deno_ast::swc::common::util::take::Take;
 use deno_core::anyhow::anyhow;
 use deno_core::error::AnyError;
 use deno_graph::FastCheckDiagnostic;
-use deno_graph::ParsedSourceStore;
 use lsp_types::Url;
 
+use crate::cache::LazyGraphSourceParser;
 use crate::diagnostics::Diagnostic;
 use crate::diagnostics::DiagnosticLevel;
 use crate::diagnostics::DiagnosticLocation;
@@ -33,7 +33,7 @@ pub struct PublishDiagnosticsCollector {
 impl PublishDiagnosticsCollector {
   pub fn print_and_error(
     &self,
-    sources: &dyn ParsedSourceStore,
+    sources: LazyGraphSourceParser,
   ) -> Result<(), AnyError> {
     let mut errors = 0;
     let mut has_zap_errors = false;

cli/tools/registry/mod.rs

@@ -27,6 +27,7 @@ use crate::args::deno_registry_url;
 use crate::args::CliOptions;
 use crate::args::Flags;
 use crate::args::PublishFlags;
+use crate::cache::LazyGraphSourceParser;
 use crate::cache::ParsedSourceCache;
 use crate::factory::CliFactory;
 use crate::graph_util::ModuleGraphBuilder;
@@ -90,6 +91,7 @@ fn get_deno_json_package_name(
 async fn prepare_publish(
   deno_json: &ConfigFile,
   source_cache: Arc<ParsedSourceCache>,
+  graph: Arc<deno_graph::ModuleGraph>,
   import_map: Arc<ImportMap>,
   diagnostics_collector: &PublishDiagnosticsCollector,
 ) -> Result<Rc<PreparedPublishPackage>, AnyError> {
@@ -140,7 +142,7 @@ async fn prepare_publish(
     let unfurler = ImportMapUnfurler::new(&import_map);
     tar::create_gzipped_tarball(
       &dir_path,
-      &*source_cache,
+      LazyGraphSourceParser::new(&source_cache, &graph),
       &diagnostics_collector,
       &unfurler,
       file_patterns,
@@ -639,19 +641,19 @@ async fn publish_package(
   Ok(())
 }
 
+struct PreparePackagesData {
+  publish_order_graph: PublishOrderGraph,
+  graph: Arc<deno_graph::ModuleGraph>,
+  package_by_name: HashMap<String, Rc<PreparedPublishPackage>>,
+}
+
 async fn prepare_packages_for_publishing(
   cli_factory: &CliFactory,
   no_zap: bool,
   diagnostics_collector: &PublishDiagnosticsCollector,
   deno_json: ConfigFile,
   import_map: Arc<ImportMap>,
-) -> Result<
+) -> Result<PreparePackagesData, AnyError> {
-  (
-    PublishOrderGraph,
-    HashMap<String, Rc<PreparedPublishPackage>>,
-  ),
-  AnyError,
-> {
   let maybe_workspace_config = deno_json.to_workspace_config()?;
   let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref();
   let source_cache = cli_factory.parsed_source_cache();
@@ -660,7 +662,7 @@ async fn prepare_packages_for_publishing(
 
   let Some(workspace_config) = maybe_workspace_config else {
     let roots = resolve_config_file_roots_from_exports(&deno_json)?;
-    build_and_check_graph_for_publish(
+    let graph = build_and_check_graph_for_publish(
       module_graph_builder,
       type_checker,
       cli_options,
@@ -673,10 +675,10 @@ async fn prepare_packages_for_publishing(
       }],
     )
     .await?;
-    let mut prepared_package_by_name = HashMap::with_capacity(1);
     let package = prepare_publish(
       &deno_json,
       source_cache.clone(),
+      graph.clone(),
       import_map,
       diagnostics_collector,
     )
@@ -684,8 +686,12 @@ async fn prepare_packages_for_publishing(
     let package_name = format!("@{}/{}", package.scope, package.package);
     let publish_order_graph =
       PublishOrderGraph::new_single(package_name.clone());
-    prepared_package_by_name.insert(package_name, package);
+    let package_by_name = HashMap::from([(package_name, package)]);
-    return Ok((publish_order_graph, prepared_package_by_name));
+    return Ok(PreparePackagesData {
+      publish_order_graph,
+      graph,
+      package_by_name,
+    });
   };
 
   println!("Publishing a workspace...");
@@ -701,7 +707,7 @@ async fn prepare_packages_for_publishing(
   )
   .await?;
 
-  let mut prepared_package_by_name =
+  let mut package_by_name =
     HashMap::with_capacity(workspace_config.members.len());
   let publish_order_graph =
     publish_order::build_publish_order_graph(&graph, &roots)?;
@@ -712,11 +718,13 @@ async fn prepare_packages_for_publishing(
     .cloned()
     .map(|member| {
       let import_map = import_map.clone();
+      let graph = graph.clone();
       async move {
         let package = prepare_publish(
           &member.config_file,
           source_cache.clone(),
-          import_map.clone(),
+          graph,
+          import_map,
           diagnostics_collector,
         )
         .await
@@ -731,9 +739,13 @@ async fn prepare_packages_for_publishing(
   let results = deno_core::futures::future::join_all(results).await;
   for result in results {
     let (package_name, package) = result?;
-    prepared_package_by_name.insert(package_name, package);
+    package_by_name.insert(package_name, package);
   }
-  Ok((publish_order_graph, prepared_package_by_name))
+  Ok(PreparePackagesData {
+    publish_order_graph,
+    graph,
+    package_by_name,
+  })
 }
 
 async fn build_and_check_graph_for_publish(
@@ -828,20 +840,22 @@ pub async fn publish(
 
   let diagnostics_collector = PublishDiagnosticsCollector::default();
 
-  let (publish_order_graph, prepared_package_by_name) =
+  let prepared_data = prepare_packages_for_publishing(
-    prepare_packages_for_publishing(
+    &cli_factory,
-      &cli_factory,
+    publish_flags.no_zap,
-      publish_flags.no_zap,
+    &diagnostics_collector,
-      &diagnostics_collector,
+    config_file.clone(),
-      config_file.clone(),
+    import_map,
-      import_map,
+  )
-    )
+  .await?;
-    .await?;
 
-  diagnostics_collector
+  let source_parser = LazyGraphSourceParser::new(
-    .print_and_error(&**cli_factory.parsed_source_cache())?;
+    cli_factory.parsed_source_cache(),
+    &prepared_data.graph,
+  );
+  diagnostics_collector.print_and_error(source_parser)?;
 
-  if prepared_package_by_name.is_empty() {
+  if prepared_data.package_by_name.is_empty() {
     bail!("No packages to publish");
   }
 
@@ -855,8 +869,8 @@ pub async fn publish(
 
   perform_publish(
     cli_factory.http_client(),
-    publish_order_graph,
+    prepared_data.publish_order_graph,
-    prepared_package_by_name,
+    prepared_data.package_by_name,
     auth_method,
   )
   .await

cli/tools/registry/tar.rs

@@ -1,6 +1,7 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
 use bytes::Bytes;
+use deno_ast::MediaType;
 use deno_config::glob::FilePatterns;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
@@ -13,6 +14,7 @@ use std::io::Write;
 use std::path::Path;
 use tar::Header;
 
+use crate::cache::LazyGraphSourceParser;
 use crate::tools::registry::paths::PackagePath;
 use crate::util::import_map::ImportMapUnfurler;
 
@@ -34,7 +36,7 @@ pub struct PublishableTarball {
 
 pub fn create_gzipped_tarball(
   dir: &Path,
-  source_cache: &dyn deno_graph::ParsedSourceStore,
+  source_parser: LazyGraphSourceParser,
   diagnostics_collector: &PublishDiagnosticsCollector,
   unfurler: &ImportMapUnfurler,
   file_patterns: Option<FilePatterns>,
@@ -122,25 +124,17 @@ pub fn create_gzipped_tarball(
         }
       }
 
-      let data = std::fs::read(path).with_context(|| {
+      let content = resolve_content_maybe_unfurling(
-        format!("Unable to read file '{}'", entry.path().display())
+        path,
-      })?;
+        &specifier,
+        unfurler,
+        source_parser,
+        diagnostics_collector,
+      )?;
       files.push(PublishableTarballFile {
         specifier: specifier.clone(),
-        size: data.len(),
+        size: content.len(),
       });
-      let content = match source_cache.get_parsed_source(&specifier) {
-        Some(parsed_source) => {
-          let mut reporter = |diagnostic| {
-            diagnostics_collector
-              .push(PublishDiagnostic::ImportMapUnfurl(diagnostic));
-          };
-          let content =
-            unfurler.unfurl(&specifier, &parsed_source, &mut reporter);
-          content.into_bytes()
-        }
-        None => data,
-      };
       tar
         .add_file(format!(".{}", path_str), &content)
         .with_context(|| {
@@ -172,6 +166,64 @@ pub fn create_gzipped_tarball(
   })
 }
 
+fn resolve_content_maybe_unfurling(
+  path: &Path,
+  specifier: &Url,
+  unfurler: &ImportMapUnfurler,
+  source_parser: LazyGraphSourceParser,
+  diagnostics_collector: &PublishDiagnosticsCollector,
+) -> Result<Vec<u8>, AnyError> {
+  let parsed_source = match source_parser.get_or_parse_source(specifier)? {
+    Some(parsed_source) => parsed_source,
+    None => {
+      let data = std::fs::read(path)
+        .with_context(|| format!("Unable to read file '{}'", path.display()))?;
+      let media_type = MediaType::from_specifier(specifier);
+
+      match media_type {
+        MediaType::JavaScript
+        | MediaType::Jsx
+        | MediaType::Mjs
+        | MediaType::Cjs
+        | MediaType::TypeScript
+        | MediaType::Mts
+        | MediaType::Cts
+        | MediaType::Dts
+        | MediaType::Dmts
+        | MediaType::Dcts
+        | MediaType::Tsx => {
+          // continue
+        }
+        MediaType::SourceMap
+        | MediaType::Unknown
+        | MediaType::Json
+        | MediaType::Wasm
+        | MediaType::TsBuildInfo => {
+          // not unfurlable data
+          return Ok(data);
+        }
+      }
+
+      let text = String::from_utf8(data)?;
+      deno_ast::parse_module(deno_ast::ParseParams {
+        specifier: specifier.to_string(),
+        text_info: deno_ast::SourceTextInfo::from_string(text),
+        media_type,
+        capture_tokens: false,
+        maybe_syntax: None,
+        scope_analysis: false,
+      })?
+    }
+  };
+
+  log::debug!("Unfurling {}", specifier);
+  let mut reporter = |diagnostic| {
+    diagnostics_collector.push(PublishDiagnostic::ImportMapUnfurl(diagnostic));
+  };
+  let content = unfurler.unfurl(specifier, &parsed_source, &mut reporter);
+  Ok(content.into_bytes())
+}
+
 struct TarGzArchive {
   builder: tar::Builder<Vec<u8>>,
 }

test_util/src/servers/registry.rs

@@ -1,5 +1,7 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
+use crate::testdata_path;
+
 use super::run_server;
 use super::ServerKind;
 use super::ServerOptions;
@@ -59,6 +61,16 @@ async fn registry_server_handler(
     return Ok(res);
   }
 
+  // serve the registry package files
+  let mut file_path =
+    testdata_path().to_path_buf().join("jsr").join("registry");
+  file_path.push(&req.uri().path()[1..].replace("%2f", "/"));
+  if let Ok(body) = tokio::fs::read(&file_path).await {
+    return Ok(Response::new(UnsyncBoxBody::new(
+      http_body_util::Full::new(Bytes::from(body)),
+    )));
+  }
+
   let empty_body = UnsyncBoxBody::new(Empty::new());
   let res = Response::builder()
     .status(StatusCode::NOT_FOUND)