From 19d52b9a55a6d5a67f27cbcc6cbe9c6c15865d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 14 Dec 2023 12:05:59 +0100 Subject: [PATCH] refactor: split registry into multiple modules (#21572) Co-authored-by: David Sherret Co-authored-by: Luca Casonato --- cli/tools/registry/api.rs | 110 +++++++++++++ cli/tools/registry/auth.rs | 51 ++++++ cli/tools/registry/mod.rs | 326 +++++++++++++------------------------ 3 files changed, 271 insertions(+), 216 deletions(-) create mode 100644 cli/tools/registry/api.rs create mode 100644 cli/tools/registry/auth.rs diff --git a/cli/tools/registry/api.rs b/cli/tools/registry/api.rs new file mode 100644 index 0000000000..b174ea367a --- /dev/null +++ b/cli/tools/registry/api.rs @@ -0,0 +1,110 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use deno_core::serde_json; +use deno_runtime::deno_fetch::reqwest; +use serde::de::DeserializeOwned; + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateAuthorizationResponse { + pub verification_url: String, + pub code: String, + pub exchange_token: String, + pub poll_interval: u64, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExchangeAuthorizationResponse { + pub token: String, + pub user: User, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct User { + pub name: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OidcTokenResponse { + pub value: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PublishingTaskError { + pub code: String, + pub message: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PublishingTask { + pub id: String, + pub status: String, + pub error: Option, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApiError { + pub code: String, + pub message: String, + #[serde(skip)] + pub x_deno_ray: Option, +} + +impl std::fmt::Display for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.message, self.code)?; + if let Some(x_deno_ray) = &self.x_deno_ray { + write!(f, "[x-deno-ray: {}]", x_deno_ray)?; + } + Ok(()) + } +} + +impl std::fmt::Debug for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +impl std::error::Error for ApiError {} + +pub async fn parse_response( + response: reqwest::Response, +) -> Result { + let status = response.status(); + let x_deno_ray = response + .headers() + .get("x-deno-ray") + .and_then(|value| value.to_str().ok()) + .map(|s| s.to_string()); + let text = response.text().await.unwrap(); + + if !status.is_success() { + match serde_json::from_str::(&text) { + Ok(mut err) => { + err.x_deno_ray = x_deno_ray; + return Err(err); + } + Err(_) => { + let err = ApiError { + code: "unknown".to_string(), + message: format!("{}: {}", status, text), + x_deno_ray, + }; + return Err(err); + } + } + } + + serde_json::from_str(&text).map_err(|err| ApiError { + code: "unknown".to_string(), + message: format!("Failed to parse response: {}, response: '{}'", err, text), + x_deno_ray, + }) +} diff --git a/cli/tools/registry/auth.rs b/cli/tools/registry/auth.rs new file mode 100644 index 0000000000..df0f849dbe --- /dev/null +++ b/cli/tools/registry/auth.rs @@ -0,0 +1,51 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::io::IsTerminal; + +use deno_core::anyhow; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; + +pub enum AuthMethod { + Interactive, + Token(String), + Oidc(OidcConfig), +} + +pub struct OidcConfig { + pub url: String, + pub token: String, +} + +fn get_gh_oidc_env_vars() -> Option> { + if std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true" { + let url = std::env::var("ACTIONS_ID_TOKEN_REQUEST_URL"); + let token = std::env::var("ACTIONS_ID_TOKEN_REQUEST_TOKEN"); + match (url, token) { + (Ok(url), Ok(token)) => Some(Ok((url, token))), + (Err(_), Err(_)) => Some(Err(anyhow::anyhow!( + "No means to authenticate. Pass a token to `--token`, or enable tokenless publishing from GitHub Actions using OIDC. Learn more at https://deno.co/ghoidc" + ))), + _ => None, + } + } else { + None + } +} + +pub fn get_auth_method( + maybe_token: Option, +) -> Result { + if let Some(token) = maybe_token { + return Ok(AuthMethod::Token(token)); + } + + match get_gh_oidc_env_vars() { + Some(Ok((url, token))) => Ok(AuthMethod::Oidc(OidcConfig { url, token })), + Some(Err(err)) => Err(err), + None if std::io::stdin().is_terminal() => Ok(AuthMethod::Interactive), + None => { + bail!("No means to authenticate. Pass a token to `--token`.") + } + } +} diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index 0e58e601e6..012294a3e8 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -2,14 +2,12 @@ use std::collections::HashMap; use std::fmt::Write; -use std::io::IsTerminal; use std::rc::Rc; use std::sync::Arc; use base64::prelude::BASE64_STANDARD; use base64::Engine; use deno_config::ConfigFile; -use deno_core::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -24,7 +22,6 @@ use http::header::CONTENT_ENCODING; use hyper::body::Bytes; use import_map::ImportMap; use lsp_types::Url; -use serde::de::DeserializeOwned; use serde::Serialize; use sha2::Digest; @@ -35,21 +32,14 @@ use crate::factory::CliFactory; use crate::http_util::HttpClient; use crate::util::import_map::ImportMapUnfurler; -use self::publish_order::PublishOrderGraph; - +mod api; +mod auth; mod publish_order; mod tar; -enum AuthMethod { - Interactive, - Token(String), - Oidc(OidcConfig), -} - -struct OidcConfig { - url: String, - token: String, -} +use auth::get_auth_method; +use auth::AuthMethod; +use publish_order::PublishOrderGraph; struct PreparedPublishPackage { scope: String, @@ -60,28 +50,13 @@ struct PreparedPublishPackage { diagnostics: Vec, } -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PublishingTaskError { - pub code: String, - pub message: String, -} - -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PublishingTask { - pub id: String, - pub status: String, - pub error: Option, -} - static SUGGESTED_ENTRYPOINTS: [&str; 4] = ["mod.ts", "mod.js", "index.ts", "index.js"]; async fn prepare_publish( deno_json: &ConfigFile, import_map: Arc, -) -> Result { +) -> Result, AnyError> { let config_path = deno_json.specifier.to_file_path().unwrap(); let dir_path = config_path.parent().unwrap().to_path_buf(); let Some(version) = deno_json.json.version.clone() else { @@ -139,14 +114,14 @@ async fn prepare_publish( write!(&mut tarball_hash, "{:02x}", byte).unwrap(); } - Ok(PreparedPublishPackage { + Ok(Rc::new(PreparedPublishPackage { scope: scope.to_string(), package: package_name.to_string(), version: version.to_string(), tarball_hash, tarball, diagnostics, - }) + })) } #[derive(Serialize)] @@ -161,96 +136,6 @@ pub enum Permission<'s> { }, } -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct CreateAuthorizationResponse { - verification_url: String, - code: String, - exchange_token: String, - poll_interval: u64, -} - -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct ExchangeAuthorizationResponse { - token: String, - user: User, -} - -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct User { - name: String, -} - -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct ApiError { - pub code: String, - pub message: String, - #[serde(skip)] - pub x_deno_ray: Option, -} - -impl std::fmt::Display for ApiError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({})", self.message, self.code)?; - if let Some(x_deno_ray) = &self.x_deno_ray { - write!(f, "[x-deno-ray: {}]", x_deno_ray)?; - } - Ok(()) - } -} - -impl std::fmt::Debug for ApiError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self, f) - } -} - -impl std::error::Error for ApiError {} - -async fn parse_response( - response: reqwest::Response, -) -> Result { - let status = response.status(); - let x_deno_ray = response - .headers() - .get("x-deno-ray") - .and_then(|value| value.to_str().ok()) - .map(|s| s.to_string()); - let text = response.text().await.unwrap(); - - if !status.is_success() { - match serde_json::from_str::(&text) { - Ok(mut err) => { - err.x_deno_ray = x_deno_ray; - return Err(err); - } - Err(_) => { - let err = ApiError { - code: "unknown".to_string(), - message: format!("{}: {}", status, text), - x_deno_ray, - }; - return Err(err); - } - } - } - - serde_json::from_str(&text).map_err(|err| ApiError { - code: "unknown".to_string(), - message: format!("Failed to parse response: {}, response: '{}'", err, text), - x_deno_ray, - }) -} - -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct OidcTokenResponse { - value: String, -} - /// Prints diagnostics like so: /// ``` /// @@ -292,22 +177,12 @@ fn print_diagnostics(diagnostics: Vec) { } } -async fn perform_publish( - http_client: &Arc, - mut publish_order_graph: PublishOrderGraph, - mut prepared_package_by_name: HashMap, +async fn get_auth_headers( + client: &reqwest::Client, + registry_url: String, + packages: Vec>, auth_method: AuthMethod, -) -> Result<(), AnyError> { - let client = http_client.client()?; - let registry_url = deno_registry_api_url().to_string(); - - let packages = prepared_package_by_name.values().collect::>(); - let diagnostics = packages - .iter() - .flat_map(|p| p.diagnostics.clone()) - .collect::>(); - print_diagnostics(diagnostics); - +) -> Result>, AnyError> { let permissions = packages .iter() .map(|package| Permission::VersionPublish { @@ -334,9 +209,10 @@ async fn perform_publish( .send() .await .context("Failed to create interactive authorization")?; - let auth = parse_response::(response) - .await - .context("Failed to create interactive authorization")?; + let auth = + api::parse_response::(response) + .await + .context("Failed to create interactive authorization")?; print!( "Visit {} to authorize publishing of", @@ -366,7 +242,8 @@ async fn perform_publish( .await .context("Failed to exchange authorization")?; let res = - parse_response::(response).await; + api::parse_response::(response) + .await; match res { Ok(res) => { println!( @@ -433,7 +310,7 @@ async fn perform_publish( text ); } - let OidcTokenResponse { value } = serde_json::from_str(&text) + let api::OidcTokenResponse { value } = serde_json::from_str(&text) .with_context(|| { format!( "Failed to parse OIDC token: '{}' (status {})", @@ -452,6 +329,32 @@ async fn perform_publish( } }; + Ok(authorizations) +} + +async fn perform_publish( + http_client: &Arc, + mut publish_order_graph: PublishOrderGraph, + mut prepared_package_by_name: HashMap>, + auth_method: AuthMethod, +) -> Result<(), AnyError> { + let client = http_client.client()?; + let registry_url = deno_registry_api_url().to_string(); + + let packages = prepared_package_by_name + .values() + .cloned() + .collect::>(); + let diagnostics = packages + .iter() + .flat_map(|p| p.diagnostics.clone()) + .collect::>(); + print_diagnostics(diagnostics); + + let mut authorizations = + get_auth_headers(client, registry_url.clone(), packages, auth_method) + .await?; + assert_eq!(prepared_package_by_name.len(), authorizations.len()); let mut futures: JoinSet> = JoinSet::default(); loop { @@ -493,7 +396,7 @@ async fn perform_publish( async fn publish_package( http_client: &HttpClient, - package: PreparedPublishPackage, + package: Rc, registry_url: &str, authorization: &str, ) -> Result<(), AnyError> { @@ -515,11 +418,11 @@ async fn publish_package( .post(url) .header(AUTHORIZATION, authorization) .header(CONTENT_ENCODING, "gzip") - .body(package.tarball) + .body(package.tarball.clone()) .send() .await?; - let res = parse_response::(response).await; + let res = api::parse_response::(response).await; let mut task = match res { Ok(task) => task, Err(err) if err.code == "duplicateVersionPublish" => { @@ -555,7 +458,7 @@ async fn publish_package( package.scope, package.package, package.version ) })?; - task = parse_response::(resp) + task = api::parse_response::(resp) .await .with_context(|| { format!( @@ -583,6 +486,7 @@ async fn publish_package( package.package, package.version ); + // TODO(bartlomieju): return something more useful here println!( "{}@{}/{}/{}_meta.json", registry_url, package.scope, package.package, package.version @@ -590,20 +494,63 @@ async fn publish_package( Ok(()) } -fn get_gh_oidc_env_vars() -> Option> { - if std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true" { - let url = std::env::var("ACTIONS_ID_TOKEN_REQUEST_URL"); - let token = std::env::var("ACTIONS_ID_TOKEN_REQUEST_TOKEN"); - match (url, token) { - (Ok(url), Ok(token)) => Some(Ok((url, token))), - (Err(_), Err(_)) => Some(Err(anyhow::anyhow!( - "No means to authenticate. Pass a token to `--token`, or enable tokenless publishing from GitHub Actions using OIDC. Learn more at https://deno.co/ghoidc" - ))), - _ => None, - } - } else { - None +async fn prepare_packages_for_publishing( + cli_factory: &CliFactory, + deno_json: ConfigFile, + import_map: Arc, +) -> Result< + ( + PublishOrderGraph, + HashMap>, + ), + AnyError, +> { + let maybe_workspace_config = deno_json.to_workspace_config()?; + + let Some(workspace_config) = maybe_workspace_config else { + let mut prepared_package_by_name = HashMap::with_capacity(1); + let package = prepare_publish(&deno_json, import_map).await?; + let package_name = package.package.clone(); + let publish_order_graph = + PublishOrderGraph::new_single(package_name.clone()); + prepared_package_by_name.insert(package_name, package); + return Ok((publish_order_graph, prepared_package_by_name)); + }; + + println!("Publishing a workspace..."); + let mut prepared_package_by_name = + HashMap::with_capacity(workspace_config.members.len()); + let publish_order_graph = publish_order::build_publish_graph( + &workspace_config, + cli_factory.module_graph_builder().await?.as_ref(), + ) + .await?; + + let results = + workspace_config + .members + .iter() + .cloned() + .map(|member| { + let import_map = import_map.clone(); + deno_core::unsync::spawn(async move { + let package = prepare_publish(&member.config_file, import_map) + .await + .with_context(|| { + format!("Failed preparing '{}'.", member.package_name) + })?; + Ok((member.package_name, package)) + }) + }) + .collect::), AnyError>>, + >>(); + let results = deno_core::futures::future::join_all(results).await; + for result in results { + let (package_name, package) = result??; + prepared_package_by_name.insert(package_name, package); } + Ok((publish_order_graph, prepared_package_by_name)) } pub async fn publish( @@ -612,17 +559,7 @@ pub async fn publish( ) -> Result<(), AnyError> { let cli_factory = CliFactory::from_flags(flags).await?; - let auth_method = match publish_flags.token { - Some(token) => AuthMethod::Token(token), - None => match get_gh_oidc_env_vars() { - Some(Ok((url, token))) => AuthMethod::Oidc(OidcConfig { url, token }), - Some(Err(err)) => return Err(err), - None if std::io::stdin().is_terminal() => AuthMethod::Interactive, - None => { - bail!("No means to authenticate. Pass a token to `--token`.") - } - }, - }; + let auth_method = get_auth_method(publish_flags.token)?; let import_map = cli_factory .maybe_import_map() @@ -645,53 +582,10 @@ pub async fn publish( ) })?; - let workspace_config = deno_json.to_workspace_config()?; - - let (publish_order_graph, prepared_package_by_name) = match workspace_config { - Some(workspace_config) => { - println!("Publishing a workspace..."); - let mut prepared_package_by_name = - HashMap::with_capacity(workspace_config.members.len()); - let publish_order_graph = publish_order::build_publish_graph( - &workspace_config, - cli_factory.module_graph_builder().await?.as_ref(), - ) + let (publish_order_graph, prepared_package_by_name) = + prepare_packages_for_publishing(&cli_factory, deno_json, import_map) .await?; - let results = workspace_config - .members - .iter() - .cloned() - .map(|member| { - let import_map = import_map.clone(); - deno_core::unsync::spawn(async move { - let package = prepare_publish(&member.config_file, import_map) - .await - .with_context(|| { - format!("Failed preparing '{}'.", member.package_name) - })?; - Ok((member.package_name, package)) - }) - }) - .collect::>>>(); - let results = deno_core::futures::future::join_all(results).await; - for result in results { - let (package_name, package) = result??; - prepared_package_by_name.insert(package_name, package); - } - (publish_order_graph, prepared_package_by_name) - } - None => { - let mut prepared_package_by_name = HashMap::with_capacity(1); - let package = prepare_publish(&deno_json, import_map).await?; - let package_name = package.package.clone(); - let publish_order_graph = - PublishOrderGraph::new_single(package_name.clone()); - prepared_package_by_name.insert(package_name, package); - (publish_order_graph, prepared_package_by_name) - } - }; - if prepared_package_by_name.is_empty() { bail!("No packages to publish"); }