foster/deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-03-03 09:31:22 -05:00
Code Issues Packages 1 Wiki Activity

refactor: split registry into multiple modules (#21572)

Browse source 
Co-authored-by: David Sherret <dsherret@gmail.com>
Co-authored-by: Luca Casonato <hello@lcas.dev>
This commit is contained in:
Bartek Iwańczuk 2023-12-14 12:05:59 +01:00 • committed by GitHub
parent f2c56dc3d8
commit 19d52b9a55
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 271 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
cli/tools/registry/api.rs Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use deno_core::serde_json;
use deno_runtime::deno_fetch::reqwest;
use serde::de::DeserializeOwned;
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CreateAuthorizationResponse {
pub verification_url: String,
pub code: String,
pub exchange_token: String,
pub poll_interval: u64,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ExchangeAuthorizationResponse {
pub token: String,
pub user: User,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct User {
pub name: String,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct OidcTokenResponse {
pub value: String,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PublishingTaskError {
pub code: String,
pub message: String,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PublishingTask {
pub id: String,
pub status: String,
pub error: Option<PublishingTaskError>,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ApiError {
pub code: String,
pub message: String,
#[serde(skip)]
pub x_deno_ray: Option<String>,
}
impl std::fmt::Display for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} ({})", self.message, self.code)?;
if let Some(x_deno_ray) = &self.x_deno_ray {
write!(f, "[x-deno-ray: {}]", x_deno_ray)?;
}
Ok(())
}
}
impl std::fmt::Debug for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self, f)
}
}
impl std::error::Error for ApiError {}
pub async fn parse_response<T: DeserializeOwned>(
response: reqwest::Response,
) -> Result<T, ApiError> {
let status = response.status();
let x_deno_ray = response
.headers()
.get("x-deno-ray")
.and_then(|value| value.to_str().ok())
.map(|s| s.to_string());
let text = response.text().await.unwrap();
if !status.is_success() {
match serde_json::from_str::<ApiError>(&text) {
Ok(mut err) => {
err.x_deno_ray = x_deno_ray;
return Err(err);
}
Err(_) => {
let err = ApiError {
code: "unknown".to_string(),
message: format!("{}: {}", status, text),
x_deno_ray,
};
return Err(err);
}
}
}
serde_json::from_str(&text).map_err(|err| ApiError {
code: "unknown".to_string(),
message: format!("Failed to parse response: {}, response: '{}'", err, text),
x_deno_ray,
})
}

51
cli/tools/registry/auth.rs Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
use std::io::IsTerminal;
use deno_core::anyhow;
use deno_core::anyhow::bail;
use deno_core::error::AnyError;
pub enum AuthMethod {
Interactive,
Token(String),
Oidc(OidcConfig),
}
pub struct OidcConfig {
pub url: String,
pub token: String,
}
fn get_gh_oidc_env_vars() -> Option<Result<(String, String), AnyError>> {
if std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true" {
let url = std::env::var("ACTIONS_ID_TOKEN_REQUEST_URL");
let token = std::env::var("ACTIONS_ID_TOKEN_REQUEST_TOKEN");
match (url, token) {
(Ok(url), Ok(token)) => Some(Ok((url, token))),
(Err(_), Err(_)) => Some(Err(anyhow::anyhow!(
"No means to authenticate. Pass a token to `--token`, or enable tokenless publishing from GitHub Actions using OIDC. Learn more at https://deno.co/ghoidc"
))),
_ => None,
}
} else {
None
}
}
pub fn get_auth_method(
maybe_token: Option<String>,
) -> Result<AuthMethod, AnyError> {
if let Some(token) = maybe_token {
return Ok(AuthMethod::Token(token));
}
match get_gh_oidc_env_vars() {
Some(Ok((url, token))) => Ok(AuthMethod::Oidc(OidcConfig { url, token })),
Some(Err(err)) => Err(err),
None if std::io::stdin().is_terminal() => Ok(AuthMethod::Interactive),
None => {
bail!("No means to authenticate. Pass a token to `--token`.")
}
}
}

326
cli/tools/registry/mod.rs
View file

@ -2,14 +2,12 @@
use std::collections::HashMap;
use std::fmt::Write;
use std::io::IsTerminal;
use std::rc::Rc;
use std::sync::Arc;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_config::ConfigFile;
use deno_core::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
@ -24,7 +22,6 @@ use http::header::CONTENT_ENCODING;
use hyper::body::Bytes;
use import_map::ImportMap;
use lsp_types::Url;
use serde::de::DeserializeOwned;
use serde::Serialize;
use sha2::Digest;
@ -35,21 +32,14 @@ use crate::factory::CliFactory;
use crate::http_util::HttpClient;
use crate::util::import_map::ImportMapUnfurler;
use self::publish_order::PublishOrderGraph;
mod api;
mod auth;
mod publish_order;
mod tar;
enum AuthMethod {
Interactive,
Token(String),
Oidc(OidcConfig),
}
struct OidcConfig {
url: String,
token: String,
}
use auth::get_auth_method;
use auth::AuthMethod;
use publish_order::PublishOrderGraph;
struct PreparedPublishPackage {
scope: String,
@ -60,28 +50,13 @@ struct PreparedPublishPackage {
diagnostics: Vec<String>,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PublishingTaskError {
pub code: String,
pub message: String,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PublishingTask {
pub id: String,
pub status: String,
pub error: Option<PublishingTaskError>,
}
static SUGGESTED_ENTRYPOINTS: [&str; 4] =
["mod.ts", "mod.js", "index.ts", "index.js"];
async fn prepare_publish(
deno_json: &ConfigFile,
import_map: Arc<ImportMap>,
) -> Result<PreparedPublishPackage, AnyError> {
) -> Result<Rc<PreparedPublishPackage>, AnyError> {
let config_path = deno_json.specifier.to_file_path().unwrap();
let dir_path = config_path.parent().unwrap().to_path_buf();
let Some(version) = deno_json.json.version.clone() else {
@ -139,14 +114,14 @@ async fn prepare_publish(
write!(&mut tarball_hash, "{:02x}", byte).unwrap();
}
Ok(PreparedPublishPackage {
Ok(Rc::new(PreparedPublishPackage {
scope: scope.to_string(),
package: package_name.to_string(),
version: version.to_string(),
tarball_hash,
tarball,
diagnostics,
})
}))
}
#[derive(Serialize)]
@ -161,96 +136,6 @@ pub enum Permission<'s> {
},
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct CreateAuthorizationResponse {
verification_url: String,
code: String,
exchange_token: String,
poll_interval: u64,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct ExchangeAuthorizationResponse {
token: String,
user: User,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct User {
name: String,
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct ApiError {
pub code: String,
pub message: String,
#[serde(skip)]
pub x_deno_ray: Option<String>,
}
impl std::fmt::Display for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} ({})", self.message, self.code)?;
if let Some(x_deno_ray) = &self.x_deno_ray {
write!(f, "[x-deno-ray: {}]", x_deno_ray)?;
}
Ok(())
}
}
impl std::fmt::Debug for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self, f)
}
}
impl std::error::Error for ApiError {}
async fn parse_response<T: DeserializeOwned>(
response: reqwest::Response,
) -> Result<T, ApiError> {
let status = response.status();
let x_deno_ray = response
.headers()
.get("x-deno-ray")
.and_then(|value| value.to_str().ok())
.map(|s| s.to_string());
let text = response.text().await.unwrap();
if !status.is_success() {
match serde_json::from_str::<ApiError>(&text) {
Ok(mut err) => {
err.x_deno_ray = x_deno_ray;
return Err(err);
}
Err(_) => {
let err = ApiError {
code: "unknown".to_string(),
message: format!("{}: {}", status, text),
x_deno_ray,
};
return Err(err);
}
}
}
serde_json::from_str(&text).map_err(|err| ApiError {
code: "unknown".to_string(),
message: format!("Failed to parse response: {}, response: '{}'", err, text),
x_deno_ray,
})
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct OidcTokenResponse {
value: String,
}
/// Prints diagnostics like so:
/// ```
///
@ -292,22 +177,12 @@ fn print_diagnostics(diagnostics: Vec<String>) {
}
}
async fn perform_publish(
http_client: &Arc<HttpClient>,
mut publish_order_graph: PublishOrderGraph,
mut prepared_package_by_name: HashMap<String, PreparedPublishPackage>,
async fn get_auth_headers(
client: &reqwest::Client,
registry_url: String,
packages: Vec<Rc<PreparedPublishPackage>>,
auth_method: AuthMethod,
) -> Result<(), AnyError> {
let client = http_client.client()?;
let registry_url = deno_registry_api_url().to_string();
let packages = prepared_package_by_name.values().collect::<Vec<_>>();
let diagnostics = packages
.iter()
.flat_map(|p| p.diagnostics.clone())
.collect::<Vec<_>>();
print_diagnostics(diagnostics);
) -> Result<HashMap<(String, String, String), Rc<str>>, AnyError> {
let permissions = packages
.iter()
.map(|package| Permission::VersionPublish {
@ -334,9 +209,10 @@ async fn perform_publish(
.send()
.await
.context("Failed to create interactive authorization")?;
let auth = parse_response::<CreateAuthorizationResponse>(response)
.await
.context("Failed to create interactive authorization")?;
let auth =
api::parse_response::<api::CreateAuthorizationResponse>(response)
.await
.context("Failed to create interactive authorization")?;
print!(
"Visit {} to authorize publishing of",
@ -366,7 +242,8 @@ async fn perform_publish(
.await
.context("Failed to exchange authorization")?;
let res =
parse_response::<ExchangeAuthorizationResponse>(response).await;
api::parse_response::<api::ExchangeAuthorizationResponse>(response)
.await;
match res {
Ok(res) => {
println!(
@ -433,7 +310,7 @@ async fn perform_publish(
text
);
}
let OidcTokenResponse { value } = serde_json::from_str(&text)
let api::OidcTokenResponse { value } = serde_json::from_str(&text)
.with_context(|| {
format!(
"Failed to parse OIDC token: '{}' (status {})",
@ -452,6 +329,32 @@ async fn perform_publish(
}
};
Ok(authorizations)
}
async fn perform_publish(
http_client: &Arc<HttpClient>,
mut publish_order_graph: PublishOrderGraph,
mut prepared_package_by_name: HashMap<String, Rc<PreparedPublishPackage>>,
auth_method: AuthMethod,
) -> Result<(), AnyError> {
let client = http_client.client()?;
let registry_url = deno_registry_api_url().to_string();
let packages = prepared_package_by_name
.values()
.cloned()
.collect::<Vec<_>>();
let diagnostics = packages
.iter()
.flat_map(|p| p.diagnostics.clone())
.collect::<Vec<_>>();
print_diagnostics(diagnostics);
let mut authorizations =
get_auth_headers(client, registry_url.clone(), packages, auth_method)
.await?;
assert_eq!(prepared_package_by_name.len(), authorizations.len());
let mut futures: JoinSet<Result<String, AnyError>> = JoinSet::default();
loop {
@ -493,7 +396,7 @@ async fn perform_publish(
async fn publish_package(
http_client: &HttpClient,
package: PreparedPublishPackage,
package: Rc<PreparedPublishPackage>,
registry_url: &str,
authorization: &str,
) -> Result<(), AnyError> {
@ -515,11 +418,11 @@ async fn publish_package(
.post(url)
.header(AUTHORIZATION, authorization)
.header(CONTENT_ENCODING, "gzip")
.body(package.tarball)
.body(package.tarball.clone())
.send()
.await?;
let res = parse_response::<PublishingTask>(response).await;
let res = api::parse_response::<api::PublishingTask>(response).await;
let mut task = match res {
Ok(task) => task,
Err(err) if err.code == "duplicateVersionPublish" => {
@ -555,7 +458,7 @@ async fn publish_package(
package.scope, package.package, package.version
)
})?;
task = parse_response::<PublishingTask>(resp)
task = api::parse_response::<api::PublishingTask>(resp)
.await
.with_context(|| {
format!(
@ -583,6 +486,7 @@ async fn publish_package(
package.package,
package.version
);
// TODO(bartlomieju): return something more useful here
println!(
"{}@{}/{}/{}_meta.json",
registry_url, package.scope, package.package, package.version
@ -590,20 +494,63 @@ async fn publish_package(
Ok(())
}
fn get_gh_oidc_env_vars() -> Option<Result<(String, String), AnyError>> {
if std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true" {
let url = std::env::var("ACTIONS_ID_TOKEN_REQUEST_URL");
let token = std::env::var("ACTIONS_ID_TOKEN_REQUEST_TOKEN");
match (url, token) {
(Ok(url), Ok(token)) => Some(Ok((url, token))),
(Err(_), Err(_)) => Some(Err(anyhow::anyhow!(
"No means to authenticate. Pass a token to `--token`, or enable tokenless publishing from GitHub Actions using OIDC. Learn more at https://deno.co/ghoidc"
))),
_ => None,
}
} else {
None
async fn prepare_packages_for_publishing(
cli_factory: &CliFactory,
deno_json: ConfigFile,
import_map: Arc<ImportMap>,
) -> Result<
(
PublishOrderGraph,
HashMap<String, Rc<PreparedPublishPackage>>,
),
AnyError,
> {
let maybe_workspace_config = deno_json.to_workspace_config()?;
let Some(workspace_config) = maybe_workspace_config else {
let mut prepared_package_by_name = HashMap::with_capacity(1);
let package = prepare_publish(&deno_json, import_map).await?;
let package_name = package.package.clone();
let publish_order_graph =
PublishOrderGraph::new_single(package_name.clone());
prepared_package_by_name.insert(package_name, package);
return Ok((publish_order_graph, prepared_package_by_name));
};
println!("Publishing a workspace...");
let mut prepared_package_by_name =
HashMap::with_capacity(workspace_config.members.len());
let publish_order_graph = publish_order::build_publish_graph(
&workspace_config,
cli_factory.module_graph_builder().await?.as_ref(),
)
.await?;
let results =
workspace_config
.members
.iter()
.cloned()
.map(|member| {
let import_map = import_map.clone();
deno_core::unsync::spawn(async move {
let package = prepare_publish(&member.config_file, import_map)
.await
.with_context(|| {
format!("Failed preparing '{}'.", member.package_name)
})?;
Ok((member.package_name, package))
})
})
.collect::<Vec<
JoinHandle<Result<(String, Rc<PreparedPublishPackage>), AnyError>>,
>>();
let results = deno_core::futures::future::join_all(results).await;
for result in results {
let (package_name, package) = result??;
prepared_package_by_name.insert(package_name, package);
}
Ok((publish_order_graph, prepared_package_by_name))
}
pub async fn publish(
@ -612,17 +559,7 @@ pub async fn publish(
) -> Result<(), AnyError> {
let cli_factory = CliFactory::from_flags(flags).await?;
let auth_method = match publish_flags.token {
Some(token) => AuthMethod::Token(token),
None => match get_gh_oidc_env_vars() {
Some(Ok((url, token))) => AuthMethod::Oidc(OidcConfig { url, token }),
Some(Err(err)) => return Err(err),
None if std::io::stdin().is_terminal() => AuthMethod::Interactive,
None => {
bail!("No means to authenticate. Pass a token to `--token`.")
}
},
};
let auth_method = get_auth_method(publish_flags.token)?;
let import_map = cli_factory
.maybe_import_map()
@ -645,53 +582,10 @@ pub async fn publish(
)
})?;
let workspace_config = deno_json.to_workspace_config()?;
let (publish_order_graph, prepared_package_by_name) = match workspace_config {
Some(workspace_config) => {
println!("Publishing a workspace...");
let mut prepared_package_by_name =
HashMap::with_capacity(workspace_config.members.len());
let publish_order_graph = publish_order::build_publish_graph(
&workspace_config,
cli_factory.module_graph_builder().await?.as_ref(),
)
let (publish_order_graph, prepared_package_by_name) =
prepare_packages_for_publishing(&cli_factory, deno_json, import_map)
.await?;
let results = workspace_config
.members
.iter()
.cloned()
.map(|member| {
let import_map = import_map.clone();
deno_core::unsync::spawn(async move {
let package = prepare_publish(&member.config_file, import_map)
.await
.with_context(|| {
format!("Failed preparing '{}'.", member.package_name)
})?;
Ok((member.package_name, package))
})
})
.collect::<Vec<JoinHandle<Result<(String, PreparedPublishPackage), AnyError>>>>();
let results = deno_core::futures::future::join_all(results).await;
for result in results {
let (package_name, package) = result??;
prepared_package_by_name.insert(package_name, package);
}
(publish_order_graph, prepared_package_by_name)
}
None => {
let mut prepared_package_by_name = HashMap::with_capacity(1);
let package = prepare_publish(&deno_json, import_map).await?;
let package_name = package.package.clone();
let publish_order_graph =
PublishOrderGraph::new_single(package_name.clone());
prepared_package_by_name.insert(package_name, package);
(publish_order_graph, prepared_package_by_name)
}
};
if prepared_package_by_name.is_empty() {
bail!("No packages to publish");
}