From 8bc11a77d795d3d2d680faf62dc0c487c400f0bd Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Thu, 15 Feb 2024 12:06:22 -0600 Subject: [PATCH 1/9] client namespace support --- crates/client/src/api.rs | 171 ++++++++++++++++++++++---- crates/client/src/config.rs | 32 +++++ crates/client/src/lib.rs | 144 ++++++++++++++++++---- crates/client/src/storage.rs | 48 ++++++-- crates/client/src/storage/fs.rs | 133 ++++++++++++++++++-- crates/protocol/src/operator/state.rs | 17 ++- src/commands/config.rs | 36 +++++- src/commands/download.rs | 12 +- src/commands/info.rs | 56 ++++++--- src/commands/publish.rs | 57 ++++++--- src/commands/reset.rs | 5 +- src/commands/update.rs | 15 ++- tests/client.rs | 2 +- tests/server.rs | 2 +- tests/support/mod.rs | 1 + 15 files changed, 624 insertions(+), 107 deletions(-) diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 03b33e39..758d7d74 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -4,8 +4,8 @@ use anyhow::{anyhow, Result}; use bytes::Bytes; use futures_util::{Stream, TryStreamExt}; use reqwest::{ - header::{HeaderMap, HeaderValue}, - Body, IntoUrl, Method, Response, StatusCode, + header::{HeaderMap, HeaderName, HeaderValue}, + Body, IntoUrl, Method, RequestBuilder, Response, StatusCode, }; use serde::de::DeserializeOwned; use std::{borrow::Cow, collections::HashMap}; @@ -156,8 +156,9 @@ async fn into_result, } impl Client { @@ -165,23 +166,60 @@ impl Client { pub fn new(url: impl IntoUrl) -> Result { let url = RegistryUrl::new(url)?; Ok(Self { - url, + home_url: url, client: reqwest::Client::new(), + namespace_registry: None, }) } /// Gets the URL of the API client. - pub fn url(&self) -> &RegistryUrl { - &self.url + pub fn home_url(&self) -> &RegistryUrl { + &self.home_url } /// Gets the latest checkpoint from the registry. pub async fn latest_checkpoint( &self, ) -> Result, ClientError> { - let url = self.url.join(paths::fetch_checkpoint()); + let url = self.home_url.join(paths::fetch_checkpoint()); tracing::debug!("getting latest checkpoint at `{url}`"); - into_result::<_, FetchError>(reqwest::get(url).await?).await + if let Some(nm) = self.namespace_registry() { + let registry_header = HeaderName::try_from("warg-registry").unwrap(); + let header_val = HeaderValue::try_from(nm).unwrap(); + into_result::<_, FetchError>( + self.client + .get(url) + .header(registry_header, header_val) + .send() + .await?, + ) + .await + } else { + into_result::<_, FetchError>(reqwest::get(url).await?).await + } + } + + /// Gets the latest checkpoints from registries. + pub async fn latest_checkpoints( + &mut self, + registries: impl Iterator, + ) -> Result>> { + let mut timestamps = HashMap::new(); + for reg in registries.into_iter() { + let url = self.home_url.join(paths::fetch_checkpoint()); + let registry_header = HeaderName::try_from("warg-registry").unwrap(); + let header_val = HeaderValue::try_from(reg).unwrap(); + let res: SerdeEnvelope = into_result::<_, FetchError>( + self.client + .get(url) + .header(registry_header, header_val) + .send() + .await?, + ) + .await?; + timestamps.insert(reg.clone(), res); + } + Ok(timestamps) } /// Verify checkpoint of the registry. @@ -189,22 +227,66 @@ impl Client { &self, request: SerdeEnvelope, ) -> Result { - let url = self.url.join(paths::verify_checkpoint()); + let url = self.home_url.join(paths::verify_checkpoint()); tracing::debug!("verifying checkpoint at `{url}`"); let response = self.client.post(url).json(&request).send().await?; into_result::<_, MonitorError>(response).await } + /// Add warg header to request + pub fn warg_header( + &self, + namespace_registry: &Option, + req: RequestBuilder, + ) -> RequestBuilder { + if let Some(nm) = namespace_registry { + let registry_header = HeaderName::try_from("warg-registry").unwrap(); + let header_val = HeaderValue::try_from(nm).unwrap(); + req.header(registry_header, header_val) + } else { + req + } + } + /// Fetches package log entries from the registry. pub async fn fetch_logs( &self, request: FetchLogsRequest<'_>, ) -> Result { - let url = self.url.join(paths::fetch_logs()); + let url = self.home_url.join(paths::fetch_logs()); tracing::debug!("fetching logs at `{url}`"); + let response = self + .warg_header(self.namespace_registry(), self.client.post(&url)) + .json(&request) + .send() + .await?; - let response = self.client.post(url).json(&request).send().await?; + if let Some(hint) = response.headers().get("Warg-Registry-Hint") { + let hint_reg = hint.to_str().unwrap().to_owned(); + let mut terms = hint_reg.split('='); + let namespace = terms.next(); + let registry = terms.next(); + if let (Some(namespace), Some(registry)) = (namespace, registry) { + print!( + "One of the packages you're requesting does not exist in the registry you're using. + However, the package namespace `{namespace}` does exist in the registry at {registry}.\nWould you like to configure your warg cli to use this registry for packages with this namespace in the future? y/N\n", + ); + std::io::Write::flush(&mut std::io::stdout()).expect("flush failed!"); + let mut buf = String::new(); + std::io::stdin().read_line(&mut buf).unwrap(); + let lowered = buf.to_lowercase(); + if lowered == "y" || lowered == "yes" { + return into_result::<_, FetchError>( + self.warg_header(&Some(hint_reg), self.client.post(url)) + .json(&request) + .send() + .await?, + ) + .await; + } + } + } into_result::<_, FetchError>(response).await } @@ -213,7 +295,7 @@ impl Client { &self, request: FetchPackageNamesRequest<'_>, ) -> Result { - let url = self.url.join(paths::fetch_package_names()); + let url = self.home_url.join(paths::fetch_package_names()); tracing::debug!("fetching package names at `{url}`"); let response = self.client.post(url).json(&request).send().await?; @@ -222,7 +304,7 @@ impl Client { /// Gets ledger sources from the registry. pub async fn ledger_sources(&self) -> Result { - let url = self.url.join(paths::ledger_sources()); + let url = self.home_url.join(paths::ledger_sources()); tracing::debug!("getting ledger sources at `{url}`"); let response = reqwest::get(url).await?; @@ -235,13 +317,17 @@ impl Client { log_id: &LogId, request: PublishRecordRequest<'_>, ) -> Result { - let url = self.url.join(&paths::publish_package_record(log_id)); + let url = self.home_url.join(&paths::publish_package_record(log_id)); tracing::debug!( "appending record to package `{name}` at `{url}`", name = request.package_name ); - let response = self.client.post(url).json(&request).send().await?; + let response = self + .warg_header(&self.namespace_registry, self.client.post(url)) + .json(&request) + .send() + .await?; into_result::<_, PackageError>(response).await } @@ -251,10 +337,22 @@ impl Client { log_id: &LogId, record_id: &RecordId, ) -> Result { - let url = self.url.join(&paths::package_record(log_id, record_id)); + let url = self + .home_url + .join(&paths::package_record(log_id, record_id)); tracing::debug!("getting record `{record_id}` for package `{log_id}` at `{url}`"); - let response = reqwest::get(url).await?; + let response = if let Some(nm) = self.namespace_registry() { + let registry_header = HeaderName::try_from("warg-registry").unwrap(); + let header_val = HeaderValue::try_from(nm).unwrap(); + self.client + .get(url) + .header(registry_header, header_val) + .send() + .await? + } else { + reqwest::get(url).await? + }; into_result::<_, PackageError>(response).await } @@ -263,7 +361,7 @@ impl Client { &self, digest: &AnyHash, ) -> Result { - let url = self.url.join(&paths::content_sources(digest)); + let url = self.home_url.join(&paths::content_sources(digest)); tracing::debug!("getting content sources for digest `{digest}` at `{url}`"); let response = reqwest::get(url).await?; @@ -303,6 +401,16 @@ impl Client { Err(ClientError::AllSourcesFailed(digest.clone())) } + /// Map namespace + pub fn map_namespace(&mut self, registry: Option) { + self.namespace_registry = registry; + } + + /// Get namespace registry + pub fn namespace_registry(&self) -> &Option { + &self.namespace_registry + } + /// Proves the inclusion of the given package log heads in the registry. pub async fn prove_inclusion( &self, @@ -310,11 +418,14 @@ impl Client { checkpoint: &Checkpoint, leafs: &[LogLeaf], ) -> Result<(), ClientError> { - let url = self.url.join(paths::prove_inclusion()); + let url = self.home_url.join(paths::prove_inclusion()); tracing::debug!("proving checkpoint inclusion at `{url}`"); let response = into_result::( - self.client.post(url).json(&request).send().await?, + self.warg_header(self.namespace_registry(), self.client.post(url)) + .json(&request) + .send() + .await?, ) .await?; @@ -328,9 +439,12 @@ impl Client { from_log_root: Cow<'_, AnyHash>, to_log_root: Cow<'_, AnyHash>, ) -> Result<(), ClientError> { - let url = self.url.join(paths::prove_consistency()); + let url = self.home_url.join(paths::prove_consistency()); let response = into_result::( - self.client.post(url).json(&request).send().await?, + self.warg_header(self.namespace_registry(), self.client.post(url)) + .json(&request) + .send() + .await?, ) .await?; @@ -380,7 +494,7 @@ impl Client { content: impl Into, ) -> Result<(), ClientError> { // Upload URLs may be relative to the registry URL. - let url = self.url.join(url); + let url = self.home_url.join(url); let method = match method { "POST" => Method::POST, @@ -405,10 +519,13 @@ impl Client { tracing::debug!("uploading content to `{url}`"); let response = self - .client - .request(method, url) - .headers(headers) - .body(content) + .warg_header( + self.namespace_registry(), + self.client + .request(method, url) + .headers(headers) + .body(content), + ) .send() .await?; if !response.status().is_success() { diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 10ae0b49..c0438efc 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -67,6 +67,8 @@ pub struct StoragePaths { pub registries_dir: PathBuf, /// The path to the content storage directory. pub content_dir: PathBuf, + /// The path to the namespace map storage directory. + pub namespace_map_path: PathBuf, } /// Represents the Warg client configuration. @@ -94,6 +96,15 @@ pub struct Config { /// `$CACHE_DIR` is the platform-specific cache directory. #[serde(default, skip_serializing_if = "Option::is_none")] pub content_dir: Option, + + /// The path to the directory where namespace map is stored. + /// + /// This path is expected to be relative to the configuration file. + /// + /// If `None`, the default of `$CACHE_DIR/warg/namespace` is used, where + /// `$CACHE_DIR` is the platform-specific cache directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace_map_path: Option, } impl Config { @@ -165,6 +176,11 @@ impl Config { assert!(p.is_absolute()); pathdiff::diff_paths(&p, &parent).unwrap() }), + namespace_map_path: self.namespace_map_path.as_ref().map(|p| { + let p = normalize_path(parent.join(p).as_path()); + assert!(p.is_absolute()); + pathdiff::diff_paths(&p, &parent).unwrap() + }), }; serde_json::to_writer_pretty( @@ -237,6 +253,20 @@ impl Config { }) } + /// Gets the path to the directory where namespace mapping is stored. + pub fn namespace_map_path(&self) -> Result { + self.namespace_map_path + .as_ref() + .cloned() + .map(Ok) + .unwrap_or_else(|| { + CACHE_DIR + .as_ref() + .map(|p| p.join("warg/namespaces")) + .ok_or_else(|| anyhow!("failed to determine operating system cache directory")) + }) + } + pub(crate) fn storage_paths_for_url( &self, url: Option<&str>, @@ -249,10 +279,12 @@ impl Config { let label = registry_url.safe_label(); let registries_dir = self.registries_dir()?.join(label); let content_dir = self.content_dir()?; + let namespace_map_path = self.namespace_map_path()?; Ok(StoragePaths { registry_url, registries_dir, content_dir, + namespace_map_path, }) } } diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index e3af19ec..38d81f05 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -8,8 +8,8 @@ use reqwest::{Body, IntoUrl}; use std::cmp::Ordering; use std::{borrow::Cow, collections::HashMap, path::PathBuf, time::Duration}; use storage::{ - ContentStorage, FileSystemContentStorage, FileSystemRegistryStorage, PublishInfo, - RegistryStorage, + ContentStorage, FileSystemContentStorage, FileSystemNamespaceMapStorage, + FileSystemRegistryStorage, NamespaceMapStorage, PublishInfo, RegistryStorage, }; use thiserror::Error; use warg_api::v1::{ @@ -39,26 +39,33 @@ pub use self::config::*; pub use self::registry_url::RegistryUrl; /// A client for a Warg registry. -pub struct Client { +pub struct Client +where + R: RegistryStorage, + C: ContentStorage, + N: NamespaceMapStorage, +{ registry: R, content: C, + namespace_map: N, api: api::Client, } -impl Client { +impl Client { /// Creates a new client for the given URL, registry storage, and /// content storage. - pub fn new(url: impl IntoUrl, registry: R, content: C) -> ClientResult { + pub fn new(url: impl IntoUrl, registry: R, content: C, namespace_map: N) -> ClientResult { Ok(Self { registry, content, + namespace_map, api: api::Client::new(url)?, }) } /// Gets the URL of the client. - pub fn url(&self) -> &RegistryUrl { - self.api.url() + pub fn home_url(&self) -> &RegistryUrl { + self.api.home_url() } /// Gets the registry storage used by the client. @@ -71,6 +78,11 @@ impl Client { &self.content } + /// Gets the namespace map + pub fn namespace_map(&self) -> &N { + &self.namespace_map + } + /// Reset client storage for the registry. pub async fn reset_registry(&self, all_registries: bool) -> ClientResult<()> { tracing::info!("resetting registry local state"); @@ -89,6 +101,39 @@ impl Client { .or(Err(ClientError::ClearContentCacheFailed)) } + /// Check operator log for namespace mapping + pub async fn fetch_namespace(&mut self, namespace: &str) -> ClientResult<()> { + self.update_checkpoint(&self.api.latest_checkpoint().await?, vec![]) + .await?; + let operator = self.registry().load_operator(&None).await.unwrap(); + if let Some(op) = operator { + for (name, namespace_def) in op.state.namespaces().clone() { + if name == namespace { + match namespace_def.state().clone() { + warg_protocol::operator::NamespaceState::Defined => {} + warg_protocol::operator::NamespaceState::Imported { registry } => { + self.api.map_namespace(Some(registry)); + break; + } + } + } + } + } + if self.api.namespace_registry().is_none() { + let map = self.namespace_map().load_namespace_map().await?; + if let Some(map) = map { + let namespace = map.get(namespace); + self.api.map_namespace(namespace.cloned()); + } + } + Ok(()) + } + + /// Get namespace registry + pub fn namespace_registry(&self) -> &Option { + self.api.namespace_registry() + } + /// Submits the publish information in client storage. /// /// If there's no publishing information in client storage, an error is returned. @@ -137,7 +182,7 @@ impl Client { let mut package = self .registry - .load_package(&info.name) + .load_package(self.namespace_registry(), &info.name) .await? .unwrap_or_else(|| PackageInfo::new(info.name.clone())); @@ -259,11 +304,19 @@ impl Client { } } + /// Updates every package log in every client registry storage to the latest registry checkpoint. + pub async fn update_all(&mut self) -> ClientResult<()> { + let packages = self.registry.load_all_packages().await?; + let checkpoints = self.api.latest_checkpoints(packages.keys()).await?; + self.update_checkpoints(checkpoints, packages).await?; + Ok(()) + } /// Updates every package log in client storage to the latest registry checkpoint. pub async fn update(&self) -> ClientResult<()> { tracing::info!("updating all packages to latest checkpoint"); let mut updating = self.registry.load_packages().await?; + self.update_checkpoint(&self.api.latest_checkpoint().await?, &mut updating) .await?; @@ -284,7 +337,7 @@ impl Client { for package in packages { updating.push( self.registry - .load_package(package) + .load_package(self.namespace_registry(), package) .await? .unwrap_or_else(|| PackageInfo::new(package.clone())), ); @@ -384,7 +437,11 @@ impl Client { checkpoint.log_length ); - let mut operator = self.registry.load_operator().await?.unwrap_or_default(); + let mut operator = self + .registry + .load_operator(self.namespace_registry()) + .await? + .unwrap_or_default(); // Map package names to package logs that need to be updated let mut packages = packages @@ -540,7 +597,11 @@ impl Client { .await?; } - if let Some(from) = self.registry.load_checkpoint().await? { + if let Some(from) = self + .registry + .load_checkpoint(self.namespace_registry()) + .await? + { let from_log_length = from.as_ref().checkpoint.log_length; let to_log_length = ts_checkpoint.as_ref().checkpoint.log_length; @@ -577,20 +638,56 @@ impl Client { } } - self.registry.store_operator(operator).await?; + self.registry + .store_operator(self.namespace_registry(), operator) + .await?; for package in packages.values_mut() { package.checkpoint = Some(checkpoint.clone()); - self.registry.store_package(package).await?; + self.registry + .store_package(self.namespace_registry(), package) + .await?; } - self.registry.store_checkpoint(ts_checkpoint).await?; + self.registry + .store_checkpoint(self.namespace_registry(), ts_checkpoint) + .await?; + + Ok(()) + } + + /// Update client namespace + pub fn map_namespace(&mut self, namespace: &Option) { + self.api.map_namespace(namespace.clone()); + } + + async fn update_checkpoints<'a>( + &mut self, + ts_checkpoints: HashMap>, + mut packages: HashMap>, + ) -> Result<(), ClientError> { + for (name, ts_checkpoint) in ts_checkpoints { + if self.home_url().safe_label() != name { + self.map_namespace(&Some(name.clone())); + } else { + self.map_namespace(&None) + } + let mut packages = packages.get_mut(&name.clone()); + if let Some(pkgs) = &mut packages { + self.update_checkpoint(&ts_checkpoint, pkgs.as_mut_slice()) + .await?; + } + } Ok(()) } async fn fetch_package(&self, name: &PackageName) -> Result { - match self.registry.load_package(name).await? { + match self + .registry + .load_package(self.namespace_registry(), name) + .await? + { Some(info) => { tracing::info!("log for package `{name}` already exists in storage"); Ok(info) @@ -654,10 +751,10 @@ impl Client { } } } - /// A Warg registry client that uses the local file system to store /// package logs and content. -pub type FileSystemClient = Client; +pub type FileSystemClient = + Client; /// A result of an attempt to lock client storage. pub enum StorageLockResult { @@ -684,21 +781,24 @@ impl FileSystemClient { registry_url: url, registries_dir, content_dir, + namespace_map_path, } = config.storage_paths_for_url(url)?; - let (packages, content) = match ( + let (packages, content, namespace_map) = match ( FileSystemRegistryStorage::try_lock(registries_dir.clone())?, FileSystemContentStorage::try_lock(content_dir.clone())?, + FileSystemNamespaceMapStorage::new(namespace_map_path.clone()), ) { - (Some(packages), Some(content)) => (packages, content), - (None, _) => return Ok(StorageLockResult::NotAcquired(registries_dir)), - (_, None) => return Ok(StorageLockResult::NotAcquired(content_dir)), + (Some(packages), Some(content), namespace_map) => (packages, content, namespace_map), + (None, _, _) => return Ok(StorageLockResult::NotAcquired(registries_dir)), + (_, None, _) => return Ok(StorageLockResult::NotAcquired(content_dir)), }; Ok(StorageLockResult::Acquired(Self::new( url.into_url(), packages, content, + namespace_map, )?)) } @@ -713,11 +813,13 @@ impl FileSystemClient { registry_url, registries_dir, content_dir, + namespace_map_path, } = config.storage_paths_for_url(url)?; Self::new( registry_url.into_url(), FileSystemRegistryStorage::lock(registries_dir)?, FileSystemContentStorage::lock(content_dir)?, + FileSystemNamespaceMapStorage::new(namespace_map_path), ) } } diff --git a/crates/client/src/storage.rs b/crates/client/src/storage.rs index 3adc954d..0c4ebf6c 100644 --- a/crates/client/src/storage.rs +++ b/crates/client/src/storage.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures_util::Stream; use serde::{Deserialize, Serialize}; -use std::{path::PathBuf, pin::Pin, time::SystemTime}; +use std::{collections::HashMap, path::PathBuf, pin::Pin, time::SystemTime}; use warg_crypto::{ hash::{AnyHash, HashAlgorithm}, signing::{self, KeyID, PublicKey}, @@ -32,33 +32,57 @@ pub trait RegistryStorage: Send + Sync { /// Reset registry local data async fn reset(&self, all_registries: bool) -> Result<()>; + // /// Directory where all registries are stored + // fn registries_dir(&self) -> PathBuf; /// Loads most recent checkpoint - async fn load_checkpoint(&self) -> Result>>; + async fn load_checkpoint( + &self, + namespace_registry: &Option, + ) -> Result>>; /// Stores most recent checkpoint async fn store_checkpoint( &self, + namespace_registry: &Option, ts_checkpoint: &SerdeEnvelope, ) -> Result<()>; /// Loads the operator information from the storage. /// /// Returns `Ok(None)` if the information is not present. - async fn load_operator(&self) -> Result>; + async fn load_operator( + &self, + namespace_registry: &Option, + ) -> Result>; /// Stores the operator information in the storage. - async fn store_operator(&self, operator: OperatorInfo) -> Result<()>; + async fn store_operator( + &self, + namespace_registry: &Option, + operator: OperatorInfo, + ) -> Result<()>; - /// Loads the package information for all packages in the storage. + /// Loads the package information for all packages in the home registry storage . async fn load_packages(&self) -> Result>; + /// Loads the package information for all packages in all registry storages. + async fn load_all_packages(&self) -> Result>>; + /// Loads the package information from the storage. /// /// Returns `Ok(None)` if the information is not present. - async fn load_package(&self, package: &PackageName) -> Result>; + async fn load_package( + &self, + namespace_registry: &Option, + package: &PackageName, + ) -> Result>; /// Stores the package information in the storage. - async fn store_package(&self, info: &PackageInfo) -> Result<()>; + async fn store_package( + &self, + namespace_registry: &Option, + info: &PackageInfo, + ) -> Result<()>; /// Loads information about a pending publish operation. /// @@ -108,6 +132,16 @@ pub trait ContentStorage: Send + Sync { ) -> Result; } +/// Trait for namespace map storage implementations. +/// +/// Namespace Map storage data must be synchronized if shared between +/// multiple threads an +#[async_trait] +pub trait NamespaceMapStorage: Send + Sync { + /// Loads namespace map + async fn load_namespace_map(&self) -> Result>>; +} + /// Represents information about a registry operator. #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] diff --git a/crates/client/src/storage/fs.rs b/crates/client/src/storage/fs.rs index a6ab4bb5..fadf7cf9 100644 --- a/crates/client/src/storage/fs.rs +++ b/crates/client/src/storage/fs.rs @@ -1,6 +1,8 @@ //! A module for file system client storage. -use super::{ContentStorage, OperatorInfo, PackageInfo, PublishInfo, RegistryStorage}; +use super::{ + ContentStorage, NamespaceMapStorage, OperatorInfo, PackageInfo, PublishInfo, RegistryStorage, +}; use crate::lock::FileLock; use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; @@ -8,6 +10,7 @@ use bytes::Bytes; use futures_util::{Stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use std::{ + collections::HashMap, ffi::OsStr, fs, path::{Path, PathBuf}, @@ -32,6 +35,7 @@ const PACKAGE_LOGS_DIR: &str = "package-logs"; pub struct FileSystemRegistryStorage { _lock: FileLock, base_dir: PathBuf, + registries_dir: PathBuf, } impl FileSystemRegistryStorage { @@ -42,10 +46,13 @@ impl FileSystemRegistryStorage { /// If the lock cannot be acquired, `Ok(None)` is returned. pub fn try_lock(base_dir: impl Into) -> Result> { let base_dir = base_dir.into(); + let mut registries_dir = base_dir.clone(); + registries_dir.pop(); match FileLock::try_open_rw(base_dir.join(LOCK_FILE_NAME))? { Some(lock) => Ok(Some(Self { _lock: lock, base_dir, + registries_dir, })), None => Ok(None), } @@ -60,17 +67,30 @@ impl FileSystemRegistryStorage { pub fn lock(base_dir: impl Into) -> Result { let base_dir = base_dir.into(); let lock = FileLock::open_rw(base_dir.join(LOCK_FILE_NAME))?; + let mut registries_dir = base_dir.clone(); + registries_dir.pop(); Ok(Self { _lock: lock, base_dir, + registries_dir, }) } - fn operator_path(&self) -> PathBuf { + fn operator_path(&self, namespace_registry: &Option) -> PathBuf { + if let Some(nm) = namespace_registry { + return self.registries_dir.join(nm).join("operator.log"); + } self.base_dir.join("operator.log") } - fn package_path(&self, name: &PackageName) -> PathBuf { + fn package_path(&self, namespace_registry: &Option, name: &PackageName) -> PathBuf { + if let Some(nm) = namespace_registry { + return self.registries_dir.join(nm).join(PACKAGE_LOGS_DIR).join( + LogId::package_log::(name) + .to_string() + .replace(':', "/"), + ); + } self.base_dir.join(PACKAGE_LOGS_DIR).join( LogId::package_log::(name) .to_string() @@ -93,14 +113,28 @@ impl RegistryStorage for FileSystemRegistryStorage { } } - async fn load_checkpoint(&self) -> Result>> { + async fn load_checkpoint( + &self, + namespace_registry: &Option, + ) -> Result>> { + if let Some(nm) = namespace_registry { + return load(&self.registries_dir.join(nm).join("checkpoint")).await; + } load(&self.base_dir.join("checkpoint")).await } async fn store_checkpoint( &self, + namespace_registry: &Option, ts_checkpoint: &SerdeEnvelope, ) -> Result<()> { + if let Some(nm) = namespace_registry { + return store( + &self.registries_dir.join(nm).join("checkpoint"), + ts_checkpoint, + ) + .await; + } store(&self.base_dir.join("checkpoint"), ts_checkpoint).await } @@ -142,20 +176,72 @@ impl RegistryStorage for FileSystemRegistryStorage { Ok(packages) } - async fn load_operator(&self) -> Result> { - Ok(load(&self.operator_path()).await?) + async fn load_all_packages(&self) -> Result>> { + let mut all_packages = HashMap::new(); + let regs = fs::read_dir(self.registries_dir.clone())?; + for reg in regs { + let folder = reg?; + if let Some(name) = folder.file_name().to_str() { + let packages_dir = self + .registries_dir + .join(folder.file_name()) + .join(PACKAGE_LOGS_DIR); + let mut packages = Vec::new(); + for entry in WalkDir::new(&packages_dir).into_iter().flatten() { + let path = entry.path(); + if !path.is_file() { + continue; + } + + if let Some(name) = path.file_name().and_then(OsStr::to_str) { + if name.starts_with('.') { + continue; + } + } + + let info: PackageInfo = load(path).await?.ok_or_else(|| { + anyhow!( + "failed to load package state from `{path}`", + path = path.display() + ) + })?; + packages.push(info); + } + all_packages.insert(name.to_string(), packages); + }; + } + Ok(all_packages) } - async fn store_operator(&self, info: OperatorInfo) -> Result<()> { - store(&self.operator_path(), info).await + async fn load_operator( + &self, + namespace_registry: &Option, + ) -> Result> { + Ok(load(&self.operator_path(namespace_registry)).await?) + } + + async fn store_operator( + &self, + namespace_registry: &Option, + info: OperatorInfo, + ) -> Result<()> { + store(&self.operator_path(namespace_registry), info).await } - async fn load_package(&self, package: &PackageName) -> Result> { - Ok(load(&self.package_path(package)).await?) + async fn load_package( + &self, + namespace_registry: &Option, + package: &PackageName, + ) -> Result> { + Ok(load(&self.package_path(namespace_registry, package)).await?) } - async fn store_package(&self, info: &PackageInfo) -> Result<()> { - store(&self.package_path(&info.name), info).await + async fn store_package( + &self, + namespace_registry: &Option, + info: &PackageInfo, + ) -> Result<()> { + store(&self.package_path(namespace_registry, &info.name), info).await } async fn load_publish(&self) -> Result> { @@ -328,6 +414,29 @@ impl ContentStorage for FileSystemContentStorage { } } +/// Represents a namespace_domain map storage using the local file system. +pub struct FileSystemNamespaceMapStorage { + base_dir: PathBuf, +} + +impl FileSystemNamespaceMapStorage { + /// Creates new namespace_domain mapping config + pub fn new(base_dir: impl Into) -> Self { + Self { + base_dir: base_dir.into(), + } + } +} + +#[async_trait] +impl NamespaceMapStorage for FileSystemNamespaceMapStorage { + async fn load_namespace_map(&self) -> Result>> { + let namespace_path = &self.base_dir; + let namespace_map = load(namespace_path).await?; + Ok(namespace_map) + } +} + async fn remove(path: &Path) -> Result<()> { if path.is_file() { return tokio::fs::remove_file(path) diff --git a/crates/protocol/src/operator/state.rs b/crates/protocol/src/operator/state.rs index 11545d26..88debc8e 100644 --- a/crates/protocol/src/operator/state.rs +++ b/crates/protocol/src/operator/state.rs @@ -4,6 +4,7 @@ use crate::registry::RecordId; use crate::ProtoEnvelope; use indexmap::{IndexMap, IndexSet}; use serde::{Deserialize, Serialize}; +use std::hash::RandomState; use std::time::SystemTime; use thiserror::Error; use warg_crypto::hash::{HashAlgorithm, Sha256}; @@ -72,13 +73,23 @@ pub enum ValidationError { /// The namespace definition. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -struct NamespaceDefinition { +pub struct NamespaceDefinition { /// Case sensitive namespace name. namespace: String, /// Namespace state. state: NamespaceState, } +impl NamespaceDefinition { + pub fn namespace(&self) -> &String { + &self.namespace + } + + pub fn state(&self) -> &NamespaceState { + &self.state + } +} + /// The namespace state for defining or importing from other registries. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -163,6 +174,10 @@ impl LogState { self.keys.get(key_id) } + pub fn namespaces(&self) -> &IndexMap { + &self.namespaces + } + /// Gets the namespace state. pub fn namespace_state(&self, namespace: &str) -> Result, &str> { if let Some(def) = self.namespaces.get(&namespace.to_ascii_lowercase()) { diff --git a/src/commands/config.rs b/src/commands/config.rs index eb926c0b..f7eaa6cf 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,8 +1,31 @@ use anyhow::{bail, Context, Result}; use clap::Args; -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use warg_client::{Config, RegistryUrl}; +#[derive(Clone)] +struct Namespace { + namespace: String, + domain: String, +} + +impl FromStr for Namespace { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::prelude::v1::Result { + let mut split = s.split('='); + let namespace = split.next(); + let domain = split.next(); + if let (Some(namespace), Some(domain)) = (namespace, domain) { + Ok(Namespace { + namespace: namespace.to_owned(), + domain: domain.to_owned(), + }) + } else { + bail!("expected namesape argument to be of form ="); + } + } +} /// Creates a new warg configuration file. #[derive(Args)] pub struct ConfigCommand { @@ -27,6 +50,14 @@ pub struct ConfigCommand { /// If not specified, the default of `$CONFIG_DIR/warg/config.json` is used. #[clap(value_name = "PATH")] pub path: Option, + + /// The namespace and domain to map + #[clap(long, long, value_name = "NAMESPACE")] + namespace: Option, + + /// The path to the namespace map + #[clap(long, value_name = "NAMESPACE_PATH")] + pub namespace_path: Option, } impl ConfigCommand { @@ -37,7 +68,7 @@ impl ConfigCommand { .map(Ok) .unwrap_or_else(Config::default_config_path)?; - if !self.overwrite && path.is_file() { + if !self.overwrite && path.is_file() && self.namespace.is_none() { bail!( "configuration file `{path}` already exists; use `--overwrite` to overwrite it", path = path.display() @@ -60,6 +91,7 @@ impl ConfigCommand { default_url, registries_dir: self.registries_dir.map(|p| cwd.join(p)), content_dir: self.content_dir.map(|p| cwd.join(p)), + namespace_map_path: self.namespace_path.map(|p| cwd.join(p)), }; config.write_to_file(&path)?; diff --git a/src/commands/download.rs b/src/commands/download.rs index 67c615d0..005b840b 100644 --- a/src/commands/download.rs +++ b/src/commands/download.rs @@ -1,6 +1,7 @@ use super::CommonOptions; use anyhow::{anyhow, Result}; use clap::Args; +use warg_client::storage::NamespaceMapStorage; use warg_protocol::{registry::PackageName, VersionReq}; /// Download a warg registry package. @@ -22,7 +23,16 @@ impl DownloadCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + if client.namespace_registry().is_none() { + let map = client.namespace_map().load_namespace_map().await?; + if let Some(map) = map { + let namespace = map.get(self.name.namespace()); + client.map_namespace(&namespace.cloned()); + dbg!(&client.namespace_registry()); + } + } println!("downloading package `{name}`...", name = self.name); diff --git a/src/commands/info.rs b/src/commands/info.rs index 0d328129..41539ef9 100644 --- a/src/commands/info.rs +++ b/src/commands/info.rs @@ -1,7 +1,10 @@ use super::CommonOptions; use anyhow::Result; -use clap::Args; -use warg_client::storage::{PackageInfo, RegistryStorage}; +use clap::{ArgAction, Args}; +use warg_client::{ + storage::{ContentStorage, NamespaceMapStorage, PackageInfo, RegistryStorage}, + Client, +}; use warg_crypto::hash::AnyHash; use warg_protocol::{registry::PackageName, Version}; @@ -15,6 +18,10 @@ pub struct InfoCommand { /// Only show information for the specified package. #[clap(value_name = "PACKAGE")] pub package: Option, + + /// Only show the namespace map + #[clap(short, long, value_name = "NAMESPACES", action = ArgAction::SetTrue)] + pub namespaces: bool, } impl InfoCommand { @@ -23,23 +30,30 @@ impl InfoCommand { let config = self.common.read_config()?; let client = self.common.create_client(&config)?; - println!("registry: {url}", url = client.url()); + println!("registry: {url}", url = client.home_url()); println!("\npackages in client storage:"); - match self.package { - Some(package) => { - if let Some(info) = client.registry().load_package(&package).await? { - Self::print_package_info(&info); + if !self.namespaces { + match self.package { + Some(package) => { + if let Some(info) = client + .registry() + .load_package(client.namespace_registry(), &package) + .await? + { + Self::print_package_info(&info); + } + } + None => { + client + .registry() + .load_packages() + .await? + .iter() + .for_each(Self::print_package_info); } - } - None => { - client - .registry() - .load_packages() - .await? - .iter() - .for_each(Self::print_package_info); } } + Self::print_namespace_map(&client).await?; Ok(()) } @@ -57,4 +71,16 @@ impl InfoCommand { fn print_release(version: &Version, content: &AnyHash) { println!(" {version} ({content})"); } + + async fn print_namespace_map( + client: &Client, + ) -> Result<()> { + if let Some(map) = client.namespace_map().load_namespace_map().await? { + for (namespace, registry) in map { + println!(" {namespace}={registry}"); + } + }; + + Ok(()) + } } diff --git a/src/commands/publish.rs b/src/commands/publish.rs index dab66c9d..ac09af97 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -8,7 +8,7 @@ use tokio::io::BufReader; use tokio_util::io::ReaderStream; use warg_client::{ storage::{ContentStorage as _, PublishEntry, PublishInfo, RegistryStorage as _}, - FileSystemClient, + FileSystemClient, RegistryUrl, }; use warg_crypto::{ hash::AnyHash, @@ -118,15 +118,20 @@ impl PublishInitCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; - + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + + let signing_key = if let Some(nm) = client.namespace_registry() { + self.common.signing_key(&RegistryUrl::new(nm)?)? + } else { + self.common.signing_key(client.home_url())? + }; match enqueue(&client, &self.name, |_| { std::future::ready(Ok(PublishEntry::Init)) }) .await? { Some(entry) => { - let signing_key = self.common.signing_key(client.url())?; let record_id = client .publish_with_info( &signing_key, @@ -188,7 +193,13 @@ impl PublishReleaseCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + let signing_key = if let Some(nm) = client.namespace_registry() { + self.common.signing_key(&RegistryUrl::new(nm)?)? + } else { + self.common.signing_key(client.home_url())? + }; let path = self.path.clone(); let version = self.version.clone(); @@ -213,7 +224,6 @@ impl PublishReleaseCommand { .await? { Some(entry) => { - let signing_key = self.common.signing_key(client.url())?; let record_id = client .publish_with_info( &signing_key, @@ -274,7 +284,13 @@ impl PublishYankCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + let signing_key = if let Some(nm) = client.namespace_registry() { + self.common.signing_key(&RegistryUrl::new(nm)?)? + } else { + self.common.signing_key(client.home_url())? + }; let version = self.version.clone(); match enqueue(&client, &self.name, move |_| async move { @@ -283,7 +299,6 @@ impl PublishYankCommand { .await? { Some(entry) => { - let signing_key = self.common.signing_key(client.url())?; let record_id = client .publish_with_info( &signing_key, @@ -351,7 +366,13 @@ impl PublishGrantCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + let signing_key = if let Some(nm) = client.namespace_registry() { + self.common.signing_key(&RegistryUrl::new(nm)?)? + } else { + self.common.signing_key(client.home_url())? + }; match enqueue(&client, &self.name, |_| async { Ok(PublishEntry::Grant { @@ -362,7 +383,6 @@ impl PublishGrantCommand { .await? { Some(entry) => { - let signing_key = self.common.signing_key(client.url())?; let record_id = client .publish_with_info( &signing_key, @@ -432,7 +452,13 @@ impl PublishRevokeCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; + let signing_key = if let Some(nm) = client.namespace_registry() { + self.common.signing_key(&RegistryUrl::new(nm)?)? + } else { + self.common.signing_key(client.home_url())? + }; match enqueue(&client, &self.name, |_| async { Ok(PublishEntry::Revoke { @@ -443,7 +469,6 @@ impl PublishRevokeCommand { .await? { Some(entry) => { - let signing_key = self.common.signing_key(client.url())?; let record_id = client .publish_with_info( &signing_key, @@ -500,7 +525,8 @@ impl PublishStartCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; match client.registry().load_publish().await? { Some(info) => bail!("a publish is already in progress for package `{name}`; use `publish abort` to abort the current publish", name = info.name), @@ -631,7 +657,7 @@ impl PublishSubmitCommand { name = info.name ); - let signing_key = self.common.signing_key(client.url())?; + let signing_key = self.common.signing_key(client.home_url())?; let record_id = client.publish_with_info(&signing_key, info.clone()).await?; client.registry().store_publish(None).await?; @@ -700,7 +726,8 @@ impl PublishWaitCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.name.namespace()).await?; let record_id = RecordId::from(self.record_id); println!( diff --git a/src/commands/reset.rs b/src/commands/reset.rs index 4a7bff4c..e36ffd92 100644 --- a/src/commands/reset.rs +++ b/src/commands/reset.rs @@ -23,7 +23,10 @@ impl ResetCommand { println!("resetting local data for all registries..."); client.reset_registry(true).await?; } else { - println!("resetting local data for registry `{}`...", client.url()); + println!( + "resetting local data for registry `{}`...", + client.home_url() + ); client.reset_registry(false).await?; } diff --git a/src/commands/update.rs b/src/commands/update.rs index 3eba8e17..bb918eac 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -1,6 +1,6 @@ use super::CommonOptions; use anyhow::Result; -use clap::Args; +use clap::{ArgAction, Args}; /// Update all local package logs for a registry. #[derive(Args)] @@ -8,16 +8,25 @@ pub struct UpdateCommand { /// The common command options. #[clap(flatten)] pub common: CommonOptions, + + /// The common command options. + #[clap(short, long, value_name = "ALL", action = ArgAction::SetTrue)] + pub all: bool, } impl UpdateCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; println!("updating package logs to the latest available versions..."); - client.update().await?; + if self.all { + client.update_all().await?; + } else { + client.update().await?; + } + Ok(()) } } diff --git a/tests/client.rs b/tests/client.rs index 66ad3c0f..fe7f9011 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -93,7 +93,7 @@ async fn client_incrementally_fetches() -> Result<()> { // Ensure the package log exists and has releases with all with the same digest let package = client .registry() - .load_package(&name) + .load_package(client.namespace_registry(), &name) .await? .context("package does not exist in client storage")?; diff --git a/tests/server.rs b/tests/server.rs index 2b04bda1..3c468883 100644 --- a/tests/server.rs +++ b/tests/server.rs @@ -441,7 +441,7 @@ async fn test_custom_content_url(config: &Config) -> Result<()> { client.upsert([&name]).await?; let package = client .registry() - .load_package(&name) + .load_package(client.namespace_registry(), &name) .await? .expect("expected the package to exist"); package diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 59258ca5..fe5a952d 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -163,6 +163,7 @@ pub async fn spawn_server( default_url: Some(format!("http://{addr}")), registries_dir: Some(root.join("registries")), content_dir: Some(root.join("content")), + namespace_map_path: Some(root.join("namespaces")), }; Ok((instance, config)) From e8f08855222dd5b7d1745d5841b182bf348b612b Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Fri, 16 Feb 2024 10:52:46 -0600 Subject: [PATCH 2/9] handle error properly --- .gitignore | 3 + crates/api/src/v1/mod.rs | 3 + crates/client/src/api.rs | 224 +++++++++--------- crates/client/src/config.rs | 8 +- crates/client/src/depsolve.rs | 45 ++-- crates/client/src/lib.rs | 325 ++++++++++++++++---------- crates/client/src/storage.rs | 17 +- crates/client/src/storage/fs.rs | 73 ++++-- crates/protocol/src/operator/state.rs | 17 +- src/commands/bundle.rs | 15 +- src/commands/config.rs | 35 +-- src/commands/dependencies.rs | 19 +- src/commands/download.rs | 9 - src/commands/info.rs | 4 +- src/commands/lock.rs | 15 +- src/commands/publish.rs | 32 +-- src/commands/reset.rs | 12 +- tests/client.rs | 2 +- tests/depsolve.rs | 12 +- tests/memory/mod.rs | 6 +- tests/postgres/mod.rs | 4 +- tests/server.rs | 16 +- tests/support/mod.rs | 2 +- 23 files changed, 518 insertions(+), 380 deletions(-) diff --git a/.gitignore b/.gitignore index 0168ef04..6f848019 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ .cargo vendor/ publish +# Test generated files +bundled.wasm +locked.wasm diff --git a/crates/api/src/v1/mod.rs b/crates/api/src/v1/mod.rs index 71af9127..9288b10d 100644 --- a/crates/api/src/v1/mod.rs +++ b/crates/api/src/v1/mod.rs @@ -14,6 +14,9 @@ use serde::{Deserialize, Serialize}; /// subject of the request. This header is only expected to be used if referring to a different /// registry than the host registry. pub const REGISTRY_HEADER_NAME: &str = "warg-registry"; +/// The HTTP response header name that specifies that the client should +/// try another registry +pub const REGISTRY_HINT_HEADER_NAME: &str = "warg-registry-hint"; /// Represents the supported kinds of content sources. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 758d7d74..0be98ca3 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -23,6 +23,7 @@ use warg_api::v1::{ proof::{ ConsistencyRequest, ConsistencyResponse, InclusionRequest, InclusionResponse, ProofError, }, + REGISTRY_HEADER_NAME, REGISTRY_HINT_HEADER_NAME, }; use warg_crypto::hash::{AnyHash, HashError, Sha256}; use warg_protocol::{ @@ -35,7 +36,6 @@ use warg_transparency::{ }; use crate::registry_url::RegistryUrl; - /// Represents an error that occurred while communicating with the registry. #[derive(Debug, Error)] pub enum ClientError { @@ -102,6 +102,9 @@ pub enum ClientError { /// Invalid upload HTTP method. #[error("server returned an invalid HTTP header `{0}: {1}`")] InvalidHttpHeader(String, String), + /// The provided log was not found with hint header. + #[error("log `{0}` was not found in this registry, but the registry provided the hint header: `{1:?}`")] + LogNotFoundWithHint(LogId, HeaderValue), /// An other error occurred during the requested operation. #[error(transparent)] Other(#[from] anyhow::Error), @@ -153,12 +156,29 @@ async fn into_result) -> RequestBuilder; +} + +impl WithWargHeader for RequestBuilder { + type Client = Client; + fn warg_header(self, registry_header: &Option) -> reqwest::RequestBuilder { + if let Some(reg) = registry_header { + let registry_header = HeaderName::try_from(REGISTRY_HEADER_NAME).unwrap(); + self.header(registry_header, reg) + } else { + self + } + } +} + /// Represents a Warg API client for communicating with /// a Warg registry server. pub struct Client { - home_url: RegistryUrl, + url: RegistryUrl, client: reqwest::Client, - namespace_registry: Option, + warg_header: Option, } impl Client { @@ -166,37 +186,31 @@ impl Client { pub fn new(url: impl IntoUrl) -> Result { let url = RegistryUrl::new(url)?; Ok(Self { - home_url: url, + url, client: reqwest::Client::new(), - namespace_registry: None, + warg_header: None, }) } /// Gets the URL of the API client. - pub fn home_url(&self) -> &RegistryUrl { - &self.home_url + pub fn url(&self) -> &RegistryUrl { + &self.url } /// Gets the latest checkpoint from the registry. pub async fn latest_checkpoint( &self, ) -> Result, ClientError> { - let url = self.home_url.join(paths::fetch_checkpoint()); + let url = self.url.join(paths::fetch_checkpoint()); tracing::debug!("getting latest checkpoint at `{url}`"); - if let Some(nm) = self.namespace_registry() { - let registry_header = HeaderName::try_from("warg-registry").unwrap(); - let header_val = HeaderValue::try_from(nm).unwrap(); - into_result::<_, FetchError>( - self.client - .get(url) - .header(registry_header, header_val) - .send() - .await?, - ) - .await - } else { - into_result::<_, FetchError>(reqwest::get(url).await?).await - } + into_result::<_, FetchError>( + self.client + .get(url) + .warg_header(self.get_warg_header()) + .send() + .await?, + ) + .await } /// Gets the latest checkpoints from registries. @@ -206,8 +220,8 @@ impl Client { ) -> Result>> { let mut timestamps = HashMap::new(); for reg in registries.into_iter() { - let url = self.home_url.join(paths::fetch_checkpoint()); - let registry_header = HeaderName::try_from("warg-registry").unwrap(); + let url = self.url.join(paths::fetch_checkpoint()); + let registry_header = HeaderName::try_from(REGISTRY_HEADER_NAME).unwrap(); let header_val = HeaderValue::try_from(reg).unwrap(); let res: SerdeEnvelope = into_result::<_, FetchError>( self.client @@ -227,67 +241,43 @@ impl Client { &self, request: SerdeEnvelope, ) -> Result { - let url = self.home_url.join(paths::verify_checkpoint()); + let url = self.url.join(paths::verify_checkpoint()); tracing::debug!("verifying checkpoint at `{url}`"); - let response = self.client.post(url).json(&request).send().await?; + let response = self + .client + .post(url) + .json(&request) + .warg_header(self.get_warg_header()) + .send() + .await?; into_result::<_, MonitorError>(response).await } - /// Add warg header to request - pub fn warg_header( - &self, - namespace_registry: &Option, - req: RequestBuilder, - ) -> RequestBuilder { - if let Some(nm) = namespace_registry { - let registry_header = HeaderName::try_from("warg-registry").unwrap(); - let header_val = HeaderValue::try_from(nm).unwrap(); - req.header(registry_header, header_val) - } else { - req - } - } - /// Fetches package log entries from the registry. pub async fn fetch_logs( &self, request: FetchLogsRequest<'_>, ) -> Result { - let url = self.home_url.join(paths::fetch_logs()); + let url = self.url.join(paths::fetch_logs()); tracing::debug!("fetching logs at `{url}`"); let response = self - .warg_header(self.namespace_registry(), self.client.post(&url)) + .client + .post(&url) .json(&request) + .warg_header(self.get_warg_header()) .send() .await?; - if let Some(hint) = response.headers().get("Warg-Registry-Hint") { - let hint_reg = hint.to_str().unwrap().to_owned(); - let mut terms = hint_reg.split('='); - let namespace = terms.next(); - let registry = terms.next(); - if let (Some(namespace), Some(registry)) = (namespace, registry) { - print!( - "One of the packages you're requesting does not exist in the registry you're using. - However, the package namespace `{namespace}` does exist in the registry at {registry}.\nWould you like to configure your warg cli to use this registry for packages with this namespace in the future? y/N\n", - ); - std::io::Write::flush(&mut std::io::stdout()).expect("flush failed!"); - let mut buf = String::new(); - std::io::stdin().read_line(&mut buf).unwrap(); - let lowered = buf.to_lowercase(); - if lowered == "y" || lowered == "yes" { - return into_result::<_, FetchError>( - self.warg_header(&Some(hint_reg), self.client.post(url)) - .json(&request) - .send() - .await?, - ) - .await; + let header = response.headers().get(REGISTRY_HINT_HEADER_NAME).cloned(); + into_result::<_, FetchError>(response) + .await + .map_err(|err| match err { + ClientError::Fetch(FetchError::LogNotFound(log_id)) if header.is_some() => { + ClientError::LogNotFoundWithHint(log_id, header.unwrap()) } - } - } - into_result::<_, FetchError>(response).await + _ => err, + }) } /// Fetches package names from the registry. @@ -295,20 +285,32 @@ impl Client { &self, request: FetchPackageNamesRequest<'_>, ) -> Result { - let url = self.home_url.join(paths::fetch_package_names()); + let url = self.url.join(paths::fetch_package_names()); tracing::debug!("fetching package names at `{url}`"); - let response = self.client.post(url).json(&request).send().await?; + let response = self + .client + .post(url) + .warg_header(self.get_warg_header()) + .json(&request) + .send() + .await?; into_result::<_, FetchError>(response).await } /// Gets ledger sources from the registry. pub async fn ledger_sources(&self) -> Result { - let url = self.home_url.join(paths::ledger_sources()); + let url = self.url.join(paths::ledger_sources()); tracing::debug!("getting ledger sources at `{url}`"); - let response = reqwest::get(url).await?; - into_result::<_, LedgerError>(response).await + into_result::<_, LedgerError>( + self.client + .get(url) + .warg_header(self.get_warg_header()) + .send() + .await?, + ) + .await } /// Publish a new record to a package log. @@ -317,15 +319,17 @@ impl Client { log_id: &LogId, request: PublishRecordRequest<'_>, ) -> Result { - let url = self.home_url.join(&paths::publish_package_record(log_id)); + let url = self.url.join(&paths::publish_package_record(log_id)); tracing::debug!( "appending record to package `{name}` at `{url}`", name = request.package_name ); let response = self - .warg_header(&self.namespace_registry, self.client.post(url)) + .client + .post(url) .json(&request) + .warg_header(self.get_warg_header()) .send() .await?; into_result::<_, PackageError>(response).await @@ -337,23 +341,17 @@ impl Client { log_id: &LogId, record_id: &RecordId, ) -> Result { - let url = self - .home_url - .join(&paths::package_record(log_id, record_id)); + let url = self.url.join(&paths::package_record(log_id, record_id)); tracing::debug!("getting record `{record_id}` for package `{log_id}` at `{url}`"); - let response = if let Some(nm) = self.namespace_registry() { - let registry_header = HeaderName::try_from("warg-registry").unwrap(); - let header_val = HeaderValue::try_from(nm).unwrap(); + into_result::<_, PackageError>( self.client .get(url) - .header(registry_header, header_val) + .warg_header(self.get_warg_header()) .send() - .await? - } else { - reqwest::get(url).await? - }; - into_result::<_, PackageError>(response).await + .await?, + ) + .await } /// Gets a content sources from the registry. @@ -361,11 +359,17 @@ impl Client { &self, digest: &AnyHash, ) -> Result { - let url = self.home_url.join(&paths::content_sources(digest)); + let url = self.url.join(&paths::content_sources(digest)); tracing::debug!("getting content sources for digest `{digest}` at `{url}`"); - let response = reqwest::get(url).await?; - into_result::<_, ContentError>(response).await + into_result::<_, ContentError>( + self.client + .get(url) + .warg_header(self.get_warg_header()) + .send() + .await?, + ) + .await } /// Downloads the content associated with a given record. @@ -386,7 +390,12 @@ impl Client { tracing::debug!("downloading content `{digest}` from `{url}`"); - let response = self.client.get(url).send().await?; + let response = self + .client + .get(url) + .warg_header(self.get_warg_header()) + .send() + .await?; if !response.status().is_success() { tracing::debug!( "failed to download content `{digest}` from `{url}`: {status}", @@ -402,13 +411,13 @@ impl Client { } /// Map namespace - pub fn map_namespace(&mut self, registry: Option) { - self.namespace_registry = registry; + pub fn map_warg_header(&mut self, registry: Option) { + self.warg_header = registry; } /// Get namespace registry - pub fn namespace_registry(&self) -> &Option { - &self.namespace_registry + pub fn get_warg_header(&self) -> &Option { + &self.warg_header } /// Proves the inclusion of the given package log heads in the registry. @@ -418,12 +427,14 @@ impl Client { checkpoint: &Checkpoint, leafs: &[LogLeaf], ) -> Result<(), ClientError> { - let url = self.home_url.join(paths::prove_inclusion()); + let url = self.url.join(paths::prove_inclusion()); tracing::debug!("proving checkpoint inclusion at `{url}`"); let response = into_result::( - self.warg_header(self.namespace_registry(), self.client.post(url)) + self.client + .post(url) .json(&request) + .warg_header(&self.get_warg_header()) .send() .await?, ) @@ -439,10 +450,12 @@ impl Client { from_log_root: Cow<'_, AnyHash>, to_log_root: Cow<'_, AnyHash>, ) -> Result<(), ClientError> { - let url = self.home_url.join(paths::prove_consistency()); + let url = self.url.join(paths::prove_consistency()); let response = into_result::( - self.warg_header(self.namespace_registry(), self.client.post(url)) + self.client + .post(url) .json(&request) + .warg_header(self.get_warg_header()) .send() .await?, ) @@ -494,7 +507,7 @@ impl Client { content: impl Into, ) -> Result<(), ClientError> { // Upload URLs may be relative to the registry URL. - let url = self.home_url.join(url); + let url = self.url.join(url); let method = match method { "POST" => Method::POST, @@ -519,13 +532,10 @@ impl Client { tracing::debug!("uploading content to `{url}`"); let response = self - .warg_header( - self.namespace_registry(), - self.client - .request(method, url) - .headers(headers) - .body(content), - ) + .client + .request(method, url) + .headers(headers) + .body(content) .send() .await?; if !response.status().is_success() { diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index c0438efc..3a44dca6 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -77,7 +77,7 @@ pub struct StoragePaths { pub struct Config { /// The default Warg registry server URL. #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_url: Option, + pub home_url: Option, /// The path to the top-level directory where per-registry information is stored. /// @@ -101,7 +101,7 @@ pub struct Config { /// /// This path is expected to be relative to the configuration file. /// - /// If `None`, the default of `$CACHE_DIR/warg/namespace` is used, where + /// If `None`, the default of `$CACHE_DIR/warg/namespaces` is used, where /// `$CACHE_DIR` is the platform-specific cache directory. #[serde(default, skip_serializing_if = "Option::is_none")] pub namespace_map_path: Option, @@ -165,7 +165,7 @@ impl Config { assert!(parent.is_absolute()); let config = Config { - default_url: self.default_url.clone(), + home_url: self.home_url.clone(), registries_dir: self.registries_dir.as_ref().map(|p| { let p = normalize_path(parent.join(p).as_path()); assert!(p.is_absolute()); @@ -272,7 +272,7 @@ impl Config { url: Option<&str>, ) -> Result { let registry_url = RegistryUrl::new( - url.or(self.default_url.as_deref()) + url.or(self.home_url.as_deref()) .ok_or(ClientError::NoDefaultUrl)?, )?; diff --git a/crates/client/src/depsolve.rs b/crates/client/src/depsolve.rs index cb72c6e5..c1d98e67 100644 --- a/crates/client/src/depsolve.rs +++ b/crates/client/src/depsolve.rs @@ -11,7 +11,7 @@ use wasm_encoder::{ use wasmparser::{Chunk, ComponentImportSectionReader, Parser, Payload}; use super::Client; -use crate::storage::{ContentStorage, PackageInfo, RegistryStorage}; +use crate::storage::{ContentStorage, NamespaceMapStorage, PackageInfo, RegistryStorage}; use crate::version_util::{DependencyImportParser, Import, ImportKind}; /// Import Kinds found in components @@ -32,7 +32,7 @@ impl Default for LockListBuilder { impl LockListBuilder { fn parse_import( - &mut self, + &self, parser: &ComponentImportSectionReader, imports: &mut Vec, ) -> Result<()> { @@ -45,9 +45,9 @@ impl LockListBuilder { } #[async_recursion] - async fn parse_package( + async fn parse_package( &mut self, - client: &Client, + client: &Client, mut bytes: &[u8], ) -> Result<()> { let mut parser = Parser::new(0); @@ -101,7 +101,11 @@ impl LockListBuilder { match import.kind { ImportKind::Locked(_) | ImportKind::Unlocked => { let id = PackageName::new(import.name.clone())?; - if let Some(info) = client.registry().load_package(&id).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &id) + .await? + { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { @@ -111,7 +115,11 @@ impl LockListBuilder { self.lock_list.insert(import); } else { client.download(&id, &VersionReq::STAR).await?; - if let Some(info) = client.registry().load_package(&id).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &id) + .await? + { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { @@ -128,10 +136,10 @@ impl LockListBuilder { Ok(()) } - fn release_bytes( + fn release_bytes( &self, release: &Release, - client: &Client, + client: &Client, ) -> Result>> { let state = &release.state; if let ReleaseState::Released { content } = state { @@ -145,9 +153,9 @@ impl LockListBuilder { /// List of deps for building #[async_recursion] - pub async fn build_list( + pub async fn build_list( &mut self, - client: &Client, + client: &Client, info: &PackageInfo, ) -> Result<()> { let release = info.state.releases().last(); @@ -166,22 +174,24 @@ impl LockListBuilder { } /// Bundles Dependencies -pub struct Bundler<'a, R, C> +pub struct Bundler<'a, R, C, N> where R: RegistryStorage, C: ContentStorage, + N: NamespaceMapStorage, { /// Warg client used for bundling - client: &'a Client, + client: &'a Client, } -impl<'a, R, C> Bundler<'a, R, C> +impl<'a, R, C, N> Bundler<'a, R, C, N> where R: RegistryStorage, C: ContentStorage, + N: NamespaceMapStorage, { /// New Bundler - pub fn new(client: &'a Client) -> Self { + pub fn new(client: &'a Client) -> Self { Self { client } } @@ -200,7 +210,12 @@ where let parsed_imp = dep_parser.parse()?; if !parsed_imp.name.contains('/') { let pkg_id = PackageName::new(parsed_imp.name)?; - if let Some(info) = self.client.registry().load_package(&pkg_id).await? { + if let Some(info) = self + .client + .registry() + .load_package(&self.client.get_warg_header(), &pkg_id) + .await? + { let release = if parsed_imp.req != VersionReq::STAR { info.state .releases() diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 8158c822..495e5da4 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -3,6 +3,7 @@ #![deny(missing_docs)] use crate::storage::PackageInfo; use anyhow::{anyhow, Context, Result}; +use reqwest::header::HeaderValue; use reqwest::{Body, IntoUrl}; use semver::{Version, VersionReq}; use std::cmp::Ordering; @@ -72,8 +73,8 @@ impl Client &RegistryUrl { - self.api.home_url() + pub fn url(&self) -> &RegistryUrl { + self.api.url() } /// Gets the registry storage used by the client. @@ -91,6 +92,25 @@ impl Client Result<()> { + self.namespace_map + .store_namespace(namespace, registry_domain) + .await?; + Ok(()) + } + + // /// Gets the namespace map mutably + // pub fn namespace_map_mut(&mut self) -> &mut N { + // &mut self.namespace_map + // } + + /// Gets the namespace map + pub async fn reset_namespaces(&self) -> Result<()> { + self.namespace_map.reset_namespaces().await?; + Ok(()) + } + /// Reset client storage for the registry. pub async fn reset_registry(&self, all_registries: bool) -> ClientResult<()> { tracing::info!("resetting registry local state"); @@ -109,129 +129,136 @@ impl Client ClientResult<()> { + /// Check operator log for namespace mapping + pub async fn fetch_namespace(&mut self, namespace: &str) -> ClientResult<()> { self.update_checkpoint(&self.api.latest_checkpoint().await?, vec![]) .await?; let operator = self.registry().load_operator(&None).await.unwrap(); if let Some(op) = operator { - for (name, namespace_def) in op.state.namespaces().clone() { - if name == namespace { - match namespace_def.state().clone() { - warg_protocol::operator::NamespaceState::Defined => {} - warg_protocol::operator::NamespaceState::Imported { registry } => { - self.api.map_namespace(Some(registry)); - break; - } + let namespace_state = op.state.namespace_state(namespace); + if let Ok(Some(nm)) = namespace_state { + match nm { + warg_protocol::operator::NamespaceState::Defined => {} + warg_protocol::operator::NamespaceState::Imported { registry } => { + self.api + .map_warg_header(Some(HeaderValue::from_str(registry).unwrap())); } } } } - if self.api.namespace_registry().is_none() { + if self.api.get_warg_header().is_none() { let map = self.namespace_map().load_namespace_map().await?; if let Some(map) = map { let namespace = map.get(namespace); - self.api.map_namespace(namespace.cloned()); + if let Some(nm) = namespace { + self.api + .map_warg_header(Some(HeaderValue::from_str(nm).unwrap())); + } else { + self.api.map_warg_header(None); + } } } Ok(()) } /// Get namespace registry - pub fn namespace_registry(&self) -> &Option { - self.api.namespace_registry() + pub fn get_warg_header(&self) -> &Option { + self.api.get_warg_header() } - + /// Locks component pub async fn lock_component(&self, info: &PackageInfo) -> ClientResult> { - let mut builder = LockListBuilder::default(); - builder.build_list(self, info).await?; - let top = Import { - name: format!("{}:{}", info.name.namespace(), info.name.name()), - req: VersionReq::STAR, - kind: ImportKind::Unlocked, - }; - builder.lock_list.insert(top); - let mut composer = CompositionGraph::new(); - let mut handled = HashMap::::new(); - for package in builder.lock_list { - let name = package.name.clone(); - let version = package.req; - let id = PackageName::new(name)?; - let info = self.registry().load_package(self.namespace_registry(), &id).await?; - if let Some(inf) = info { - let release = if version != VersionReq::STAR { - inf.state - .releases() - .filter(|r| version.matches(&r.version)) - .last() - } else { - inf.state.releases().last() - }; - - if let Some(r) = release { - let state = &r.state; - if let ReleaseState::Released { content } = state { - let locked_package = locked_package(&package.name, r, content); - let path = self.content().content_location(content); - if let Some(p) = path { - let bytes = fs::read(&p).map_err(|_| ClientError::ContentNotFound { - digest: content.clone(), - })?; - - let read_digest = - AnyHash::from_str(&format!("sha256:{}", sha256::digest(bytes))) - .unwrap(); - if content != &read_digest { - return Err(ClientError::IncorrectContent { - digest: read_digest, - expected: content.clone(), - }); - } - let component = - wasm_compose::graph::Component::from_file(&locked_package, p)?; - let component_id = if let Some((id, _)) = - composer.get_component_by_name(&locked_package) - { - id - } else { - composer.add_component(component)? - }; - let instance_id = composer.instantiate(component_id)?; - let added = composer.get_component(component_id); - handled.insert(versioned_package(&package.name, version), instance_id); - let mut args = Vec::new(); - if let Some(added) = added { - for (index, name, _) in added.imports() { - let iid = handled.get(kindless_name(name)); - if let Some(arg) = iid { - args.push((arg, index)); - } - } - } - for arg in args { - composer.connect( - *arg.0, - None::, - instance_id, - arg.1, - )?; - } - } - } - } - } - } - let final_name = &format!("{}:{}", info.name.namespace(), &info.name.name()); - let id = handled.get(final_name); - let options = EncodeOptions { - export: id.copied(), - ..Default::default() - }; - let locked = composer.encode(options)?; - fs::write("./locked.wasm", locked.as_slice()).map_err(|e| ClientError::Other(e.into()))?; - Ok(locked) - } + let mut builder = LockListBuilder::default(); + builder.build_list(self, info).await?; + let top = Import { + name: format!("{}:{}", info.name.namespace(), info.name.name()), + req: VersionReq::STAR, + kind: ImportKind::Unlocked, + }; + builder.lock_list.insert(top); + let mut composer = CompositionGraph::new(); + let mut handled = HashMap::::new(); + for package in builder.lock_list { + let name = package.name.clone(); + let version = package.req; + let id = PackageName::new(name)?; + let info = self + .registry() + .load_package(self.get_warg_header(), &id) + .await?; + if let Some(inf) = info { + let release = if version != VersionReq::STAR { + inf.state + .releases() + .filter(|r| version.matches(&r.version)) + .last() + } else { + inf.state.releases().last() + }; + + if let Some(r) = release { + let state = &r.state; + if let ReleaseState::Released { content } = state { + let locked_package = locked_package(&package.name, r, content); + let path = self.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(&p).map_err(|_| ClientError::ContentNotFound { + digest: content.clone(), + })?; + + let read_digest = + AnyHash::from_str(&format!("sha256:{}", sha256::digest(bytes))) + .unwrap(); + if content != &read_digest { + return Err(ClientError::IncorrectContent { + digest: read_digest, + expected: content.clone(), + }); + } + let component = + wasm_compose::graph::Component::from_file(&locked_package, p)?; + let component_id = if let Some((id, _)) = + composer.get_component_by_name(&locked_package) + { + id + } else { + composer.add_component(component)? + }; + let instance_id = composer.instantiate(component_id)?; + let added = composer.get_component(component_id); + handled.insert(versioned_package(&package.name, version), instance_id); + let mut args = Vec::new(); + if let Some(added) = added { + for (index, name, _) in added.imports() { + let iid = handled.get(kindless_name(name)); + if let Some(arg) = iid { + args.push((arg, index)); + } + } + } + for arg in args { + composer.connect( + *arg.0, + None::, + instance_id, + arg.1, + )?; + } + } + } + } + } + } + let final_name = &format!("{}:{}", info.name.namespace(), &info.name.name()); + let id = handled.get(final_name); + let options = EncodeOptions { + export: id.copied(), + ..Default::default() + }; + let locked = composer.encode(options)?; + fs::write("./locked.wasm", locked.as_slice()).map_err(|e| ClientError::Other(e.into()))?; + Ok(locked) + } /// Bundles component pub async fn bundle_component(&self, info: &PackageInfo) -> ClientResult> { @@ -255,7 +282,7 @@ impl Client ClientResult { + pub async fn publish(&mut self, signing_key: &signing::PrivateKey) -> ClientResult { let info = self .registry .load_publish() @@ -296,7 +323,7 @@ impl Client Client Client Result { @@ -554,7 +581,7 @@ impl Client Client>(); loop { - let response: FetchLogsResponse = self + let response: FetchLogsResponse = match self .api .fetch_logs(FetchLogsRequest { log_length: checkpoint.log_length, @@ -590,11 +617,59 @@ impl Client res, + Err(api::ClientError::LogNotFoundWithHint(log_id, header)) => { + let hint_reg = header.to_str().unwrap(); + let mut terms = hint_reg.split('='); + let namespace = terms.next(); + let registry = terms.next(); + let resp = if let (Some(namespace), Some(registry)) = (namespace, registry) { + print!( + "One of the packages you're requesting does not exist in the registry you're using. + However, the package namespace `{namespace}` does exist in the registry at {registry}.\nWould you like to configure your warg cli to use this registry for packages with this namespace in the future? y/N\n", + ); + std::io::Write::flush(&mut std::io::stdout()).expect("flush failed!"); + let mut buf = String::new(); + std::io::stdin().read_line(&mut buf).unwrap(); + let lowered = buf.to_lowercase(); + if lowered == "y" || lowered == "yes" { + self.store_namespace(namespace.to_string(), registry.to_string()) + .await?; + Some( + self.api + .fetch_logs(FetchLogsRequest { + log_length: checkpoint.log_length, + operator: operator + .head_fetch_token + .as_ref() + .map(|t| Cow::Borrowed(t.as_str())), + limit: None, + packages: Cow::Borrowed(&last_known), + }) + .await?, + ) + } else { + None + } + } else { + None + }; + if let Some(resp) = resp { + resp + } else { + return Err(ClientError::translate_log_not_found( + api::ClientError::Fetch(FetchError::LogNotFound(log_id)), + |id| packages.get(id).map(|p| p.name.clone()), + )); + } + } + Err(e) => { + return Err(ClientError::translate_log_not_found(e, |id| { packages.get(id).map(|p| p.name.clone()) - }) - })?; + })) + } + }; for record in response.operator { let proto_envelope: PublishedProtoEnvelope = @@ -714,7 +789,7 @@ impl Client Client) { - self.api.map_namespace(namespace.clone()); + pub fn map_warg_header(&mut self, namespace: &Option) { + self.api.map_warg_header(namespace.clone()); } async fn update_checkpoints<'a>( @@ -782,10 +857,10 @@ impl Client>, ) -> Result<(), ClientError> { for (name, ts_checkpoint) in ts_checkpoints { - if self.home_url().safe_label() != name { - self.map_namespace(&Some(name.clone())); + if self.url().safe_label() != name { + self.map_warg_header(&Some(HeaderValue::from_str(&name).unwrap())); } else { - self.map_namespace(&None) + self.map_warg_header(&None) } let mut packages = packages.get_mut(&name.clone()); if let Some(pkgs) = &mut packages { @@ -800,7 +875,7 @@ impl Client Result { match self .registry - .load_package(self.namespace_registry(), name) + .load_package(self.get_warg_header(), name) .await? { Some(info) => { diff --git a/crates/client/src/storage.rs b/crates/client/src/storage.rs index 0c4ebf6c..1b962d7f 100644 --- a/crates/client/src/storage.rs +++ b/crates/client/src/storage.rs @@ -4,6 +4,7 @@ use anyhow::Result; use async_trait::async_trait; use bytes::Bytes; use futures_util::Stream; +use reqwest::header::HeaderValue; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, path::PathBuf, pin::Pin, time::SystemTime}; use warg_crypto::{ @@ -37,13 +38,13 @@ pub trait RegistryStorage: Send + Sync { /// Loads most recent checkpoint async fn load_checkpoint( &self, - namespace_registry: &Option, + namespace_registry: &Option, ) -> Result>>; /// Stores most recent checkpoint async fn store_checkpoint( &self, - namespace_registry: &Option, + namespace_registry: &Option, ts_checkpoint: &SerdeEnvelope, ) -> Result<()>; @@ -52,13 +53,13 @@ pub trait RegistryStorage: Send + Sync { /// Returns `Ok(None)` if the information is not present. async fn load_operator( &self, - namespace_registry: &Option, + namespace_registry: &Option, ) -> Result>; /// Stores the operator information in the storage. async fn store_operator( &self, - namespace_registry: &Option, + namespace_registry: &Option, operator: OperatorInfo, ) -> Result<()>; @@ -73,14 +74,14 @@ pub trait RegistryStorage: Send + Sync { /// Returns `Ok(None)` if the information is not present. async fn load_package( &self, - namespace_registry: &Option, + namespace_registry: &Option, package: &PackageName, ) -> Result>; /// Stores the package information in the storage. async fn store_package( &self, - namespace_registry: &Option, + namespace_registry: &Option, info: &PackageInfo, ) -> Result<()>; @@ -140,6 +141,10 @@ pub trait ContentStorage: Send + Sync { pub trait NamespaceMapStorage: Send + Sync { /// Loads namespace map async fn load_namespace_map(&self) -> Result>>; + /// Reset namespace mappings + async fn reset_namespaces(&self) -> Result<()>; + /// Store namespace mapping + async fn store_namespace(&self, namespace: String, registry_domain: String) -> Result<()>; } /// Represents information about a registry operator. diff --git a/crates/client/src/storage/fs.rs b/crates/client/src/storage/fs.rs index fadf7cf9..5fe010cf 100644 --- a/crates/client/src/storage/fs.rs +++ b/crates/client/src/storage/fs.rs @@ -8,6 +8,7 @@ use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; use bytes::Bytes; use futures_util::{Stream, StreamExt, TryStreamExt}; +use reqwest::header::HeaderValue; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -76,20 +77,31 @@ impl FileSystemRegistryStorage { }) } - fn operator_path(&self, namespace_registry: &Option) -> PathBuf { + fn operator_path(&self, namespace_registry: &Option) -> PathBuf { if let Some(nm) = namespace_registry { - return self.registries_dir.join(nm).join("operator.log"); + return self + .registries_dir + .join(nm.to_str().unwrap()) + .join("operator.log"); } self.base_dir.join("operator.log") } - fn package_path(&self, namespace_registry: &Option, name: &PackageName) -> PathBuf { + fn package_path( + &self, + namespace_registry: &Option, + name: &PackageName, + ) -> PathBuf { if let Some(nm) = namespace_registry { - return self.registries_dir.join(nm).join(PACKAGE_LOGS_DIR).join( - LogId::package_log::(name) - .to_string() - .replace(':', "/"), - ); + return self + .registries_dir + .join(nm.to_str().unwrap()) + .join(PACKAGE_LOGS_DIR) + .join( + LogId::package_log::(name) + .to_string() + .replace(':', "/"), + ); } self.base_dir.join(PACKAGE_LOGS_DIR).join( LogId::package_log::(name) @@ -115,22 +127,31 @@ impl RegistryStorage for FileSystemRegistryStorage { async fn load_checkpoint( &self, - namespace_registry: &Option, + namespace_registry: &Option, ) -> Result>> { if let Some(nm) = namespace_registry { - return load(&self.registries_dir.join(nm).join("checkpoint")).await; + return load( + &self + .registries_dir + .join(nm.to_str().unwrap()) + .join("checkpoint"), + ) + .await; } load(&self.base_dir.join("checkpoint")).await } async fn store_checkpoint( &self, - namespace_registry: &Option, + namespace_registry: &Option, ts_checkpoint: &SerdeEnvelope, ) -> Result<()> { if let Some(nm) = namespace_registry { return store( - &self.registries_dir.join(nm).join("checkpoint"), + &self + .registries_dir + .join(nm.to_str().unwrap()) + .join("checkpoint"), ts_checkpoint, ) .await; @@ -215,14 +236,14 @@ impl RegistryStorage for FileSystemRegistryStorage { async fn load_operator( &self, - namespace_registry: &Option, + namespace_registry: &Option, ) -> Result> { Ok(load(&self.operator_path(namespace_registry)).await?) } async fn store_operator( &self, - namespace_registry: &Option, + namespace_registry: &Option, info: OperatorInfo, ) -> Result<()> { store(&self.operator_path(namespace_registry), info).await @@ -230,7 +251,7 @@ impl RegistryStorage for FileSystemRegistryStorage { async fn load_package( &self, - namespace_registry: &Option, + namespace_registry: &Option, package: &PackageName, ) -> Result> { Ok(load(&self.package_path(namespace_registry, package)).await?) @@ -238,7 +259,7 @@ impl RegistryStorage for FileSystemRegistryStorage { async fn store_package( &self, - namespace_registry: &Option, + namespace_registry: &Option, info: &PackageInfo, ) -> Result<()> { store(&self.package_path(namespace_registry, &info.name), info).await @@ -435,6 +456,26 @@ impl NamespaceMapStorage for FileSystemNamespaceMapStorage { let namespace_map = load(namespace_path).await?; Ok(namespace_map) } + + async fn reset_namespaces(&self) -> Result<()> { + remove(&self.base_dir).await?; + Ok(()) + } + + async fn store_namespace(&self, namespace: String, registry_domain: String) -> Result<()> { + let mapping = self.load_namespace_map().await?; + if let Some(mut mapping) = mapping { + mapping.insert(namespace, registry_domain); + let json = serde_json::to_string(&mapping)?; + fs::write(&self.base_dir, json)?; + } else { + let mut mapping = HashMap::new(); + mapping.insert(namespace, registry_domain); + let json = serde_json::to_string(&mapping)?; + fs::write(&self.base_dir, json)?; + } + Ok(()) + } } async fn remove(path: &Path) -> Result<()> { diff --git a/crates/protocol/src/operator/state.rs b/crates/protocol/src/operator/state.rs index 88debc8e..11545d26 100644 --- a/crates/protocol/src/operator/state.rs +++ b/crates/protocol/src/operator/state.rs @@ -4,7 +4,6 @@ use crate::registry::RecordId; use crate::ProtoEnvelope; use indexmap::{IndexMap, IndexSet}; use serde::{Deserialize, Serialize}; -use std::hash::RandomState; use std::time::SystemTime; use thiserror::Error; use warg_crypto::hash::{HashAlgorithm, Sha256}; @@ -73,23 +72,13 @@ pub enum ValidationError { /// The namespace definition. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct NamespaceDefinition { +struct NamespaceDefinition { /// Case sensitive namespace name. namespace: String, /// Namespace state. state: NamespaceState, } -impl NamespaceDefinition { - pub fn namespace(&self) -> &String { - &self.namespace - } - - pub fn state(&self) -> &NamespaceState { - &self.state - } -} - /// The namespace state for defining or importing from other registries. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -174,10 +163,6 @@ impl LogState { self.keys.get(key_id) } - pub fn namespaces(&self) -> &IndexMap { - &self.namespaces - } - /// Gets the namespace state. pub fn namespace_state(&self, namespace: &str) -> Result, &str> { if let Some(def) = self.namespaces.get(&namespace.to_ascii_lowercase()) { diff --git a/src/commands/bundle.rs b/src/commands/bundle.rs index 1b7aa5e8..03d0aca8 100644 --- a/src/commands/bundle.rs +++ b/src/commands/bundle.rs @@ -20,13 +20,22 @@ impl BundleCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); - if let Some(info) = client.registry().load_package(&self.package).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &self.package) + .await? + { client.bundle_component(&info).await?; } else { client.download(&self.package, &VersionReq::STAR).await?; - if let Some(info) = client.registry().load_package(&self.package).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &self.package) + .await? + { client.bundle_component(&info).await?; } else { bail!("Unable to find package {}", self.package.name()) diff --git a/src/commands/config.rs b/src/commands/config.rs index f7eaa6cf..ba6ed2d1 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,31 +1,8 @@ use anyhow::{bail, Context, Result}; use clap::Args; -use std::{path::PathBuf, str::FromStr}; +use std::path::PathBuf; use warg_client::{Config, RegistryUrl}; -#[derive(Clone)] -struct Namespace { - namespace: String, - domain: String, -} - -impl FromStr for Namespace { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::prelude::v1::Result { - let mut split = s.split('='); - let namespace = split.next(); - let domain = split.next(); - if let (Some(namespace), Some(domain)) = (namespace, domain) { - Ok(Namespace { - namespace: namespace.to_owned(), - domain: domain.to_owned(), - }) - } else { - bail!("expected namesape argument to be of form ="); - } - } -} /// Creates a new warg configuration file. #[derive(Args)] pub struct ConfigCommand { @@ -51,10 +28,6 @@ pub struct ConfigCommand { #[clap(value_name = "PATH")] pub path: Option, - /// The namespace and domain to map - #[clap(long, long, value_name = "NAMESPACE")] - namespace: Option, - /// The path to the namespace map #[clap(long, value_name = "NAMESPACE_PATH")] pub namespace_path: Option, @@ -68,14 +41,14 @@ impl ConfigCommand { .map(Ok) .unwrap_or_else(Config::default_config_path)?; - if !self.overwrite && path.is_file() && self.namespace.is_none() { + if !self.overwrite && path.is_file() { bail!( "configuration file `{path}` already exists; use `--overwrite` to overwrite it", path = path.display() ); } - let default_url = self + let home_url = self .registry .map(RegistryUrl::new) .transpose()? @@ -88,7 +61,7 @@ impl ConfigCommand { // the configuration file's directory. let cwd = std::env::current_dir().context("failed to determine current directory")?; let config = Config { - default_url, + home_url, registries_dir: self.registries_dir.map(|p| cwd.join(p)), content_dir: self.content_dir.map(|p| cwd.join(p)), namespace_map_path: self.namespace_path.map(|p| cwd.join(p)), diff --git a/src/commands/dependencies.rs b/src/commands/dependencies.rs index 4dbbb7fb..5dde5349 100644 --- a/src/commands/dependencies.rs +++ b/src/commands/dependencies.rs @@ -30,9 +30,14 @@ impl DependenciesCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.package.namespace()).await?; - if let Some(info) = client.registry().load_package(&self.package).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &self.package) + .await? + { Self::print_package_info(&client, &info).await?; } @@ -49,7 +54,10 @@ impl DependenciesCommand { ) -> Result<()> { client.download(id, &version).await?; - let package = client.registry().load_package(id).await?; + let package = client + .registry() + .load_package(client.get_warg_header(), id) + .await?; if let Some(pkg) = package { let latest = pkg.state.releases().last(); if let Some(l) = latest { @@ -85,7 +93,10 @@ impl DependenciesCommand { async fn print_package_info(client: &FileSystemClient, info: &PackageInfo) -> Result<()> { let mut parser = DepsParser::new(); - let root_package = client.registry().load_package(&info.name).await?; + let root_package = client + .registry() + .load_package(client.get_warg_header(), &info.name) + .await?; if let Some(rp) = root_package { let latest = rp.state.releases().last(); if let Some(l) = latest { diff --git a/src/commands/download.rs b/src/commands/download.rs index 005b840b..68a85900 100644 --- a/src/commands/download.rs +++ b/src/commands/download.rs @@ -1,7 +1,6 @@ use super::CommonOptions; use anyhow::{anyhow, Result}; use clap::Args; -use warg_client::storage::NamespaceMapStorage; use warg_protocol::{registry::PackageName, VersionReq}; /// Download a warg registry package. @@ -25,14 +24,6 @@ impl DownloadCommand { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - if client.namespace_registry().is_none() { - let map = client.namespace_map().load_namespace_map().await?; - if let Some(map) = map { - let namespace = map.get(self.name.namespace()); - client.map_namespace(&namespace.cloned()); - dbg!(&client.namespace_registry()); - } - } println!("downloading package `{name}`...", name = self.name); diff --git a/src/commands/info.rs b/src/commands/info.rs index 41539ef9..597bf872 100644 --- a/src/commands/info.rs +++ b/src/commands/info.rs @@ -30,14 +30,14 @@ impl InfoCommand { let config = self.common.read_config()?; let client = self.common.create_client(&config)?; - println!("registry: {url}", url = client.home_url()); + println!("registry: {url}", url = client.url()); println!("\npackages in client storage:"); if !self.namespaces { match self.package { Some(package) => { if let Some(info) = client .registry() - .load_package(client.namespace_registry(), &package) + .load_package(client.get_warg_header(), &package) .await? { Self::print_package_info(&info); diff --git a/src/commands/lock.rs b/src/commands/lock.rs index 57d6aa7c..4a6c256c 100644 --- a/src/commands/lock.rs +++ b/src/commands/lock.rs @@ -24,13 +24,22 @@ impl LockCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; - let client = self.common.create_client(&config)?; + let mut client = self.common.create_client(&config)?; + client.fetch_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); - if let Some(info) = client.registry().load_package(&self.package).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &self.package) + .await? + { Self::lock(client, &info).await?; } else { client.download(&self.package, &VersionReq::STAR).await?; - if let Some(info) = client.registry().load_package(&self.package).await? { + if let Some(info) = client + .registry() + .load_package(client.get_warg_header(), &self.package) + .await? + { Self::lock(client, &info).await?; } } diff --git a/src/commands/publish.rs b/src/commands/publish.rs index ac09af97..5cb18b09 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -121,10 +121,10 @@ impl PublishInitCommand { let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - let signing_key = if let Some(nm) = client.namespace_registry() { - self.common.signing_key(&RegistryUrl::new(nm)?)? + let signing_key = if let Some(nm) = client.get_warg_header() { + self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { - self.common.signing_key(client.home_url())? + self.common.signing_key(client.url())? }; match enqueue(&client, &self.name, |_| { std::future::ready(Ok(PublishEntry::Init)) @@ -195,10 +195,10 @@ impl PublishReleaseCommand { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - let signing_key = if let Some(nm) = client.namespace_registry() { - self.common.signing_key(&RegistryUrl::new(nm)?)? + let signing_key = if let Some(nm) = client.get_warg_header() { + self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { - self.common.signing_key(client.home_url())? + self.common.signing_key(client.url())? }; let path = self.path.clone(); @@ -286,10 +286,10 @@ impl PublishYankCommand { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - let signing_key = if let Some(nm) = client.namespace_registry() { - self.common.signing_key(&RegistryUrl::new(nm)?)? + let signing_key = if let Some(nm) = client.get_warg_header() { + self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { - self.common.signing_key(client.home_url())? + self.common.signing_key(client.url())? }; let version = self.version.clone(); @@ -368,10 +368,10 @@ impl PublishGrantCommand { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - let signing_key = if let Some(nm) = client.namespace_registry() { - self.common.signing_key(&RegistryUrl::new(nm)?)? + let signing_key = if let Some(nm) = client.get_warg_header() { + self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { - self.common.signing_key(client.home_url())? + self.common.signing_key(client.url())? }; match enqueue(&client, &self.name, |_| async { @@ -454,10 +454,10 @@ impl PublishRevokeCommand { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; client.fetch_namespace(self.name.namespace()).await?; - let signing_key = if let Some(nm) = client.namespace_registry() { - self.common.signing_key(&RegistryUrl::new(nm)?)? + let signing_key = if let Some(nm) = client.get_warg_header() { + self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { - self.common.signing_key(client.home_url())? + self.common.signing_key(client.url())? }; match enqueue(&client, &self.name, |_| async { @@ -657,7 +657,7 @@ impl PublishSubmitCommand { name = info.name ); - let signing_key = self.common.signing_key(client.home_url())?; + let signing_key = self.common.signing_key(client.url())?; let record_id = client.publish_with_info(&signing_key, info.clone()).await?; client.registry().store_publish(None).await?; diff --git a/src/commands/reset.rs b/src/commands/reset.rs index e36ffd92..b3fdb4a8 100644 --- a/src/commands/reset.rs +++ b/src/commands/reset.rs @@ -11,6 +11,9 @@ pub struct ResetCommand { /// Whether to reset all registries. #[clap(long)] pub all: bool, + /// Whether to reset namespace mappings + #[clap(long)] + pub namespaces: bool, } impl ResetCommand { @@ -23,13 +26,14 @@ impl ResetCommand { println!("resetting local data for all registries..."); client.reset_registry(true).await?; } else { - println!( - "resetting local data for registry `{}`...", - client.home_url() - ); + println!("resetting local data for registry `{}`...", client.url()); client.reset_registry(false).await?; } + if self.namespaces { + client.reset_namespaces().await?; + } + Ok(()) } } diff --git a/tests/client.rs b/tests/client.rs index fe7f9011..37cef51e 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -93,7 +93,7 @@ async fn client_incrementally_fetches() -> Result<()> { // Ensure the package log exists and has releases with all with the same digest let package = client .registry() - .load_package(client.namespace_registry(), &name) + .load_package(client.get_warg_header(), &name) .await? .context("package does not exist in client storage")?; diff --git a/tests/depsolve.rs b/tests/depsolve.rs index 0628a2d7..40ed4047 100644 --- a/tests/depsolve.rs +++ b/tests/depsolve.rs @@ -3,8 +3,8 @@ use anyhow::{Context, Result}; use std::time::Duration; use warg_client::{ storage::{ - ContentStorage, FileSystemContentStorage, FileSystemRegistryStorage, PublishEntry, - PublishInfo, RegistryStorage, + ContentStorage, FileSystemContentStorage, FileSystemNamespaceMapStorage, + FileSystemRegistryStorage, PublishEntry, PublishInfo, RegistryStorage, }, Client, }; @@ -89,7 +89,7 @@ async fn depsolve() -> Result<()> { let info = client .registry() - .load_package(&PackageName::new("test:meet")?) + .load_package(client.get_warg_header(), &PackageName::new("test:meet")?) .await? .context("package does not exist in client storage")?; @@ -109,7 +109,11 @@ async fn depsolve() -> Result<()> { } async fn publish_package( - client: &Client, + client: &Client< + FileSystemRegistryStorage, + FileSystemContentStorage, + FileSystemNamespaceMapStorage, + >, signing_key: &PrivateKey, name: &str, path: &str, diff --git a/tests/memory/mod.rs b/tests/memory/mod.rs index bb79580c..5a7368c3 100644 --- a/tests/memory/mod.rs +++ b/tests/memory/mod.rs @@ -16,7 +16,7 @@ async fn it_publishes_a_component() -> Result<()> { test_component_publishing(&config).await?; // There should be two log entries in the registry - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, @@ -35,7 +35,7 @@ async fn it_yanks_a_package() -> Result<()> { test_package_yanking(&config).await?; // There should be three entries in the registry - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, @@ -52,7 +52,7 @@ async fn it_publishes_a_wit_package() -> Result<()> { test_wit_publishing(&config).await?; // There should be two log entries in the registry - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, diff --git a/tests/postgres/mod.rs b/tests/postgres/mod.rs index dab03752..f7c08ad2 100644 --- a/tests/postgres/mod.rs +++ b/tests/postgres/mod.rs @@ -66,7 +66,7 @@ async fn it_works_with_postgres() -> TestResult { ]; // There should be two log entries in the registry - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, @@ -84,7 +84,7 @@ async fn it_works_with_postgres() -> TestResult { packages.push(PackageName::new("test:unknown-key")?); - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, diff --git a/tests/server.rs b/tests/server.rs index 3c468883..e354b93f 100644 --- a/tests/server.rs +++ b/tests/server.rs @@ -39,7 +39,7 @@ mod memory; mod postgres; async fn test_initial_checkpoint(config: &Config) -> Result<()> { - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; let checkpoint = &ts_checkpoint.as_ref().checkpoint; @@ -371,7 +371,7 @@ async fn test_invalid_signature(config: &Config) -> Result<()> { // Use a reqwest client directly here as we're going to be sending an invalid signature let name = PackageName::new(PACKAGE_NAME)?; let log_id = LogId::package_log::(&name); - let url = Url::parse(config.default_url.as_ref().unwrap())? + let url = Url::parse(config.home_url.as_ref().unwrap())? .join(&paths::publish_package_record(&log_id)) .unwrap(); @@ -441,7 +441,7 @@ async fn test_custom_content_url(config: &Config) -> Result<()> { client.upsert([&name]).await?; let package = client .registry() - .load_package(client.namespace_registry(), &name) + .load_package(client.get_warg_header(), &name) .await? .expect("expected the package to exist"); package @@ -450,7 +450,7 @@ async fn test_custom_content_url(config: &Config) -> Result<()> { .expect("expected the package version to exist"); // Look up the content URL for the record - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ContentSourcesResponse { content_sources } = client.content_sources(&digest).await?; assert_eq!(content_sources.len(), 1); let sources = content_sources @@ -476,7 +476,7 @@ async fn test_fetch_package_names(config: &Config) -> Result<()> { let name_1 = PackageName::new("test:component")?; let log_id_1 = LogId::package_log::(&name_1); - let url = Url::parse(config.default_url.as_ref().unwrap())? + let url = Url::parse(config.home_url.as_ref().unwrap())? .join(paths::fetch_package_names()) .unwrap(); @@ -511,12 +511,12 @@ async fn test_fetch_package_names(config: &Config) -> Result<()> { } async fn test_get_ledger(config: &Config) -> Result<()> { - let client = api::Client::new(config.default_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap())?; let ts_checkpoint = client.latest_checkpoint().await?; let checkpoint = &ts_checkpoint.as_ref().checkpoint; - let url = Url::parse(config.default_url.as_ref().unwrap())? + let url = Url::parse(config.home_url.as_ref().unwrap())? .join(paths::ledger_sources()) .unwrap(); @@ -565,7 +565,7 @@ async fn test_get_ledger(config: &Config) -> Result<()> { "unexpected ledger source last registry index: {last_registry_index}", ); - let url = Url::parse(config.default_url.as_ref().unwrap())? + let url = Url::parse(config.home_url.as_ref().unwrap())? .join(url) .unwrap(); diff --git a/tests/support/mod.rs b/tests/support/mod.rs index fe5a952d..1d54f2be 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -160,7 +160,7 @@ pub async fn spawn_server( }; let config = warg_client::Config { - default_url: Some(format!("http://{addr}")), + home_url: Some(format!("http://{addr}")), registries_dir: Some(root.join("registries")), content_dir: Some(root.join("content")), namespace_map_path: Some(root.join("namespaces")), From fbe1a4af0d7c9ec4e98c971526a7b81a3f3c5515 Mon Sep 17 00:00:00 2001 From: Calvin Prewitt Date: Fri, 16 Feb 2024 12:26:52 -0600 Subject: [PATCH 3/9] renamed mentions of `default` registry to `home` registry; removed some `&mut self`; updated the `fetch_namespace` to only check local namespace map if operator log does not map the namespace --- README.md | 4 ++-- crates/client/src/api.rs | 4 ++-- crates/client/src/config.rs | 4 ++-- crates/client/src/depsolve.rs | 2 +- crates/client/src/lib.rs | 34 ++++++++++++++++++---------------- src/bin/warg.rs | 4 ++-- src/commands/config.rs | 2 +- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0158c50c..93d1e86f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ warg config --registry http://127.0.0.1:8090 ``` This creates a [`$CONFIG_DIR/warg/config.json`][config_dir] configuration file; -the configuration file will specify the default registry URL to use so that the +the configuration file will specify the home registry URL to use so that the `--registry` option does not need to be specified for every command. Data downloaded by the client is stored in [`$CACHE_DIR/warg`][cache_dir] by @@ -156,7 +156,7 @@ warg publish revoke --name example:hello sha256:abc... ### Resetting and clearing local data -To reset local data for the default registry: +To reset local data for the home registry: ``` warg reset ``` diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 0be98ca3..c57cad87 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -215,7 +215,7 @@ impl Client { /// Gets the latest checkpoints from registries. pub async fn latest_checkpoints( - &mut self, + &self, registries: impl Iterator, ) -> Result>> { let mut timestamps = HashMap::new(); @@ -434,7 +434,7 @@ impl Client { self.client .post(url) .json(&request) - .warg_header(&self.get_warg_header()) + .warg_header(self.get_warg_header()) .send() .await?, ) diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 3a44dca6..e4b9529f 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -75,7 +75,7 @@ pub struct StoragePaths { #[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Config { - /// The default Warg registry server URL. + /// The home Warg registry server URL. #[serde(default, skip_serializing_if = "Option::is_none")] pub home_url: Option, @@ -273,7 +273,7 @@ impl Config { ) -> Result { let registry_url = RegistryUrl::new( url.or(self.home_url.as_deref()) - .ok_or(ClientError::NoDefaultUrl)?, + .ok_or(ClientError::NoHomeRegistryUrl)?, )?; let label = registry_url.safe_label(); diff --git a/crates/client/src/depsolve.rs b/crates/client/src/depsolve.rs index c1d98e67..8528a07f 100644 --- a/crates/client/src/depsolve.rs +++ b/crates/client/src/depsolve.rs @@ -213,7 +213,7 @@ where if let Some(info) = self .client .registry() - .load_package(&self.client.get_warg_header(), &pkg_id) + .load_package(self.client.get_warg_header(), &pkg_id) .await? { let release = if parsed_imp.req != VersionReq::STAR { diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 495e5da4..fde7ed97 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -100,11 +100,6 @@ impl Client &mut N { - // &mut self.namespace_map - // } - /// Gets the namespace map pub async fn reset_namespaces(&self) -> Result<()> { self.namespace_map.reset_namespaces().await?; @@ -134,19 +129,25 @@ impl Client {} + warg_protocol::operator::NamespaceState::Defined => true, warg_protocol::operator::NamespaceState::Imported { registry } => { self.api .map_warg_header(Some(HeaderValue::from_str(registry).unwrap())); + + true } } + } else { + false } - } - if self.api.get_warg_header().is_none() { + } else { + false + }; + if !operator_log_maps_namespace { let map = self.namespace_map().load_namespace_map().await?; if let Some(map) = map { let namespace = map.get(namespace); @@ -282,7 +283,7 @@ impl Client ClientResult { + pub async fn publish(&self, signing_key: &signing::PrivateKey) -> ClientResult { let info = self .registry .load_publish() @@ -452,6 +453,7 @@ impl Client ClientResult<()> { tracing::info!("updating all packages to latest checkpoint"); @@ -538,7 +540,7 @@ impl Client Result { @@ -957,7 +959,7 @@ pub enum StorageLockResult { impl FileSystemClient { /// Attempts to create a client for the given registry URL. /// - /// If the URL is `None`, the default URL is used; if there is no default + /// If the URL is `None`, the home registry URL is used; if there is no home registry /// URL, an error is returned. /// /// If a lock cannot be acquired for a storage directory, then @@ -994,7 +996,7 @@ impl FileSystemClient { /// Creates a client for the given registry URL. /// - /// If the URL is `None`, the default URL is used; if there is no default + /// If the URL is `None`, the home registry URL is used; if there is no home registry /// URL, an error is returned. /// /// This method blocks if storage locks cannot be acquired. @@ -1028,9 +1030,9 @@ pub struct PackageDownload { /// Represents an error returned by Warg registry clients. #[derive(Debug, Error)] pub enum ClientError { - /// No default registry server URL is configured. - #[error("no default registry server URL is configured")] - NoDefaultUrl, + /// No home registry registry server URL is configured. + #[error("no home registry registry server URL is configured")] + NoHomeRegistryUrl, /// Reset registry local state. #[error("reset registry state failed")] diff --git a/src/bin/warg.rs b/src/bin/warg.rs index 33f68f62..b6273777 100644 --- a/src/bin/warg.rs +++ b/src/bin/warg.rs @@ -68,8 +68,8 @@ async fn main() -> Result<()> { fn describe_client_error(e: &ClientError) { match e { - ClientError::NoDefaultUrl => { - eprintln!("error: {e}; use the `config` subcommand to set a default URL"); + ClientError::NoHomeRegistryUrl => { + eprintln!("error: {e}; use the `config` subcommand to set a home registry URL"); } ClientError::PackageValidationFailed { name, inner } => { eprintln!("error: the log for package `{name}` is invalid: {inner}") diff --git a/src/commands/config.rs b/src/commands/config.rs index ba6ed2d1..3f0a3a9b 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -6,7 +6,7 @@ use warg_client::{Config, RegistryUrl}; /// Creates a new warg configuration file. #[derive(Args)] pub struct ConfigCommand { - /// The default registry URL to use. + /// The home registry URL to use. #[clap(long, value_name = "URL")] pub registry: Option, From 010fefff44f38f34e366e10d8f47210639110810 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Fri, 16 Feb 2024 15:00:00 -0600 Subject: [PATCH 4/9] auth token --- crates/client/src/api.rs | 26 ++++ crates/client/src/depsolve.rs | 10 +- crates/client/src/lib.rs | 150 ++++++++++++++------- src/bin/warg.rs | 4 +- src/commands.rs | 24 ++++ src/commands/bundle.rs | 13 +- src/commands/dependencies.rs | 32 ++++- src/commands/download.rs | 6 +- src/commands/lock.rs | 21 ++- src/commands/login.rs | 239 ++++++++++++++++++++++++++++++++++ src/commands/publish.rs | 45 +++++-- src/commands/update.rs | 5 +- src/keyring.rs | 40 ++++++ 13 files changed, 536 insertions(+), 79 deletions(-) create mode 100644 src/commands/login.rs diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 0be98ca3..e842253d 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -173,6 +173,22 @@ impl WithWargHeader for RequestBuilder { } } +trait WithAuth { + type Client; + fn auth(self, auth_token: &Option) -> RequestBuilder; +} + +impl WithAuth for RequestBuilder { + type Client = Client; + fn auth(self, auth_token: &Option) -> reqwest::RequestBuilder { + if let Some(tok) = auth_token { + self.bearer_auth(tok) + } else { + self + } + } +} + /// Represents a Warg API client for communicating with /// a Warg registry server. pub struct Client { @@ -258,6 +274,7 @@ impl Client { pub async fn fetch_logs( &self, request: FetchLogsRequest<'_>, + auth_token: &Option, ) -> Result { let url = self.url.join(paths::fetch_logs()); tracing::debug!("fetching logs at `{url}`"); @@ -266,6 +283,7 @@ impl Client { .post(&url) .json(&request) .warg_header(self.get_warg_header()) + .auth(auth_token) .send() .await?; @@ -283,6 +301,7 @@ impl Client { /// Fetches package names from the registry. pub async fn fetch_package_names( &self, + auth_token: &Option, request: FetchPackageNamesRequest<'_>, ) -> Result { let url = self.url.join(paths::fetch_package_names()); @@ -292,6 +311,7 @@ impl Client { .client .post(url) .warg_header(self.get_warg_header()) + .auth(auth_token) .json(&request) .send() .await?; @@ -375,6 +395,7 @@ impl Client { /// Downloads the content associated with a given record. pub async fn download_content( &self, + auth_token: &Option, digest: &AnyHash, ) -> Result>, ClientError> { tracing::debug!("requesting content download for digest `{digest}`"); @@ -394,6 +415,7 @@ impl Client { .client .get(url) .warg_header(self.get_warg_header()) + .auth(auth_token) .send() .await?; if !response.status().is_success() { @@ -423,6 +445,7 @@ impl Client { /// Proves the inclusion of the given package log heads in the registry. pub async fn prove_inclusion( &self, + auth_token: &Option, request: InclusionRequest, checkpoint: &Checkpoint, leafs: &[LogLeaf], @@ -435,6 +458,7 @@ impl Client { .post(url) .json(&request) .warg_header(&self.get_warg_header()) + .auth(auth_token) .send() .await?, ) @@ -446,6 +470,7 @@ impl Client { /// Proves consistency between two log roots. pub async fn prove_log_consistency( &self, + auth_token: &Option, request: ConsistencyRequest, from_log_root: Cow<'_, AnyHash>, to_log_root: Cow<'_, AnyHash>, @@ -456,6 +481,7 @@ impl Client { .post(url) .json(&request) .warg_header(self.get_warg_header()) + .auth(auth_token) .send() .await?, ) diff --git a/crates/client/src/depsolve.rs b/crates/client/src/depsolve.rs index c1d98e67..db383e07 100644 --- a/crates/client/src/depsolve.rs +++ b/crates/client/src/depsolve.rs @@ -47,6 +47,7 @@ impl LockListBuilder { #[async_recursion] async fn parse_package( &mut self, + auth_token: &Option, client: &Client, mut bytes: &[u8], ) -> Result<()> { @@ -109,12 +110,12 @@ impl LockListBuilder { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { - self.parse_package(client, &bytes).await?; + self.parse_package(auth_token, client, &bytes).await?; } } self.lock_list.insert(import); } else { - client.download(&id, &VersionReq::STAR).await?; + client.download(auth_token, &id, &VersionReq::STAR).await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &id) @@ -123,7 +124,7 @@ impl LockListBuilder { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { - self.parse_package(client, &bytes).await?; + self.parse_package(auth_token, client, &bytes).await?; } } self.lock_list.insert(import); @@ -155,6 +156,7 @@ impl LockListBuilder { #[async_recursion] pub async fn build_list( &mut self, + auth_token: &Option, client: &Client, info: &PackageInfo, ) -> Result<()> { @@ -165,7 +167,7 @@ impl LockListBuilder { let path = client.content().content_location(content); if let Some(p) = path { let bytes = fs::read(p)?; - self.parse_package(client, &bytes).await?; + self.parse_package(auth_token, client, &bytes).await?; } } } diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 495e5da4..a440da54 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -130,8 +130,12 @@ impl Client ClientResult<()> { - self.update_checkpoint(&self.api.latest_checkpoint().await?, vec![]) + pub async fn fetch_namespace( + &mut self, + auth_token: &Option, + namespace: &str, + ) -> ClientResult<()> { + self.update_checkpoint(auth_token, &self.api.latest_checkpoint().await?, vec![]) .await?; let operator = self.registry().load_operator(&None).await.unwrap(); if let Some(op) = operator { @@ -167,9 +171,13 @@ impl Client ClientResult> { + pub async fn lock_component( + &self, + auth_token: &Option, + info: &PackageInfo, + ) -> ClientResult> { let mut builder = LockListBuilder::default(); - builder.build_list(self, info).await?; + builder.build_list(auth_token, self, info).await?; let top = Import { name: format!("{}:{}", info.name.namespace(), info.name.name()), req: VersionReq::STAR, @@ -261,11 +269,15 @@ impl Client ClientResult> { + pub async fn bundle_component( + &self, + auth_token: &Option, + info: &PackageInfo, + ) -> ClientResult> { let mut bundler = Bundler::new(self); let path = PathBuf::from("./locked.wasm"); let locked = if !path.is_file() { - self.lock_component(info).await? + self.lock_component(auth_token, info).await? } else { fs::read("./locked.wasm").map_err(|e| ClientError::Other(e.into()))? }; @@ -282,14 +294,18 @@ impl Client ClientResult { + pub async fn publish( + &mut self, + auth_token: &Option, + signing_key: &signing::PrivateKey, + ) -> ClientResult { let info = self .registry .load_publish() .await? .ok_or(ClientError::NotPublishing)?; - let res = self.publish_with_info(signing_key, info).await; + let res = self.publish_with_info(auth_token, signing_key, info).await; self.registry.store_publish(None).await?; res } @@ -303,6 +319,7 @@ impl Client, signing_key: &signing::PrivateKey, mut info: PublishInfo, ) -> ClientResult { @@ -330,8 +347,12 @@ impl Client Client ClientResult<()> { + pub async fn update_all(&mut self, auth_token: &Option) -> ClientResult<()> { let packages = self.registry.load_all_packages().await?; let checkpoints = self.api.latest_checkpoints(packages.keys()).await?; - self.update_checkpoints(checkpoints, packages).await?; + self.update_checkpoints(auth_token, checkpoints, packages) + .await?; Ok(()) } /// Updates every package log in client storage to the latest registry checkpoint. - pub async fn update(&self) -> ClientResult<()> { + pub async fn update(&self, auth_token: &Option) -> ClientResult<()> { tracing::info!("updating all packages to latest checkpoint"); let mut updating = self.registry.load_packages().await?; - self.update_checkpoint(&self.api.latest_checkpoint().await?, &mut updating) - .await?; + self.update_checkpoint( + auth_token, + &self.api.latest_checkpoint().await?, + &mut updating, + ) + .await?; Ok(()) } /// Inserts or updates the logs of the specified packages in client storage to /// the latest registry checkpoint. - pub async fn upsert<'a, I>(&self, packages: I) -> Result<(), ClientError> + pub async fn upsert<'a, I>( + &self, + auth_token: &Option, + packages: I, + ) -> Result<(), ClientError> where I: IntoIterator, I::IntoIter: ExactSizeIterator, @@ -484,8 +514,12 @@ impl Client Client, name: &PackageName, requirement: &VersionReq, ) -> Result, ClientError> { tracing::info!("downloading package `{name}` with requirement `{requirement}`"); - let info = self.fetch_package(name).await?; + let info = self.fetch_package(auth_token, name).await?; match info.state.find_latest_release(requirement) { Some(release) => { @@ -517,7 +552,7 @@ impl Client Client, package: &PackageName, version: &Version, ) -> Result { tracing::info!("downloading version {version} of package `{package}`"); - let info = self.fetch_package(package).await?; + let info = self.fetch_package(auth_token, package).await?; let release = info.state @@ -563,13 +599,14 @@ impl Client( &self, + auth_token: &Option, ts_checkpoint: &SerdeEnvelope, packages: impl IntoIterator, ) -> Result<(), ClientError> { @@ -607,15 +644,18 @@ impl Client res, @@ -638,15 +678,18 @@ impl Client Client Client { self.api .prove_log_consistency( + auth_token, ConsistencyRequest { from: from_log_length, to: to_log_length, @@ -853,6 +898,7 @@ impl Client( &mut self, + auth_token: &Option, ts_checkpoints: HashMap>, mut packages: HashMap>, ) -> Result<(), ClientError> { @@ -864,7 +910,7 @@ impl Client Client Result { + async fn fetch_package( + &self, + auth_token: &Option, + name: &PackageName, + ) -> Result { match self .registry .load_package(self.get_warg_header(), name) @@ -884,8 +934,12 @@ impl Client { let mut info = PackageInfo::new(name.clone()); - self.update_checkpoint(&self.api.latest_checkpoint().await?, [&mut info]) - .await?; + self.update_checkpoint( + auth_token, + &self.api.latest_checkpoint().await?, + [&mut info], + ) + .await?; Ok(info) } @@ -918,7 +972,11 @@ impl Client Result { + pub async fn download_content( + &self, + auth_token: &Option, + digest: &AnyHash, + ) -> Result { match self.content.content_location(digest) { Some(path) => { tracing::info!("content for digest `{digest}` already exists in storage"); @@ -927,7 +985,7 @@ impl Client { self.content .store_content( - Box::pin(self.api.download_content(digest).await?), + Box::pin(self.api.download_content(auth_token, digest).await?), Some(digest), ) .await?; diff --git a/src/bin/warg.rs b/src/bin/warg.rs index 33f68f62..8c21bb65 100644 --- a/src/bin/warg.rs +++ b/src/bin/warg.rs @@ -4,7 +4,7 @@ use std::process::exit; use tracing_subscriber::EnvFilter; use warg_cli::commands::{ BundleCommand, ClearCommand, ConfigCommand, DependenciesCommand, DownloadCommand, InfoCommand, - KeyCommand, LockCommand, PublishCommand, ResetCommand, UpdateCommand, + KeyCommand, LockCommand, LoginCommand, PublishCommand, ResetCommand, UpdateCommand, }; use warg_client::ClientError; @@ -34,6 +34,7 @@ enum WargCli { Publish(PublishCommand), Reset(ResetCommand), Clear(ClearCommand), + Login(LoginCommand), } #[tokio::main] @@ -54,6 +55,7 @@ async fn main() -> Result<()> { WargCli::Publish(cmd) => cmd.exec().await, WargCli::Reset(cmd) => cmd.exec().await, WargCli::Clear(cmd) => cmd.exec().await, + WargCli::Login(cmd) => cmd.exec().await, } { if let Some(e) = e.downcast_ref::() { describe_client_error(e); diff --git a/src/commands.rs b/src/commands.rs index 0bfaee46..417625e0 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -16,10 +16,12 @@ mod download; mod info; mod key; mod lock; +mod login; mod publish; mod reset; mod update; +use crate::keyring::get_auth_token; use crate::keyring::get_signing_key; pub use self::bundle::*; @@ -30,6 +32,7 @@ pub use self::download::*; pub use self::info::*; pub use self::key::*; pub use self::lock::*; +pub use self::login::*; pub use self::publish::*; pub use self::reset::*; pub use self::update::*; @@ -41,6 +44,12 @@ pub struct CommonOptions { #[clap(long, value_name = "URL")] pub registry: Option, /// The name to use for the signing key. + #[clap(long, short, value_name = "TOKEN_NAME", default_value = "default")] + pub token_name: String, + /// The path to the signing key file. + #[clap(long, value_name = "TOKEN_FILE", env = "WARG_AUTH_TOKEN_FILE")] + pub token_file: Option, + /// The name to use for the signing key. #[clap(long, short, value_name = "KEY_NAME", default_value = "default")] pub key_name: String, /// The path to the signing key file. @@ -97,4 +106,19 @@ impl CommonOptions { get_signing_key(registry_url, &self.key_name) } } + /// Gets the auth token for the given registry URL. + pub fn auth_token(&self) -> Result> { + if let Some(file) = &self.token_file { + Ok(Some( + std::fs::read_to_string(file) + .with_context(|| format!("failed to read key from {file:?}"))? + .trim_end() + .to_string(), + )) + } else if let Some(reg) = &self.registry { + get_auth_token(&RegistryUrl::new(reg)?, &self.token_name).map(|tok| Some(tok)) + } else { + Ok(None) + } + } } diff --git a/src/commands/bundle.rs b/src/commands/bundle.rs index 03d0aca8..5821903d 100644 --- a/src/commands/bundle.rs +++ b/src/commands/bundle.rs @@ -21,22 +21,27 @@ impl BundleCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.package.namespace()) + .await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - client.bundle_component(&info).await?; + client.bundle_component(&auth_token, &info).await?; } else { - client.download(&self.package, &VersionReq::STAR).await?; + client + .download(&auth_token, &self.package, &VersionReq::STAR) + .await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - client.bundle_component(&info).await?; + client.bundle_component(&auth_token, &info).await?; } else { bail!("Unable to find package {}", self.package.name()) } diff --git a/src/commands/dependencies.rs b/src/commands/dependencies.rs index 5dde5349..8f5c81c4 100644 --- a/src/commands/dependencies.rs +++ b/src/commands/dependencies.rs @@ -31,14 +31,17 @@ impl DependenciesCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.package.namespace()) + .await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::print_package_info(&client, &info).await?; + Self::print_package_info(&auth_token, &client, &info).await?; } Ok(()) @@ -46,13 +49,14 @@ impl DependenciesCommand { #[async_recursion] async fn parse_deps<'a>( + auth_token: &Option, id: &'a PackageName, version: VersionReq, client: &FileSystemClient, node: &mut TreeBuilder, parser: &mut DepsParser, ) -> Result<()> { - client.download(id, &version).await?; + client.download(auth_token, id, &version).await?; let package = client .registry() @@ -77,8 +81,15 @@ impl DependenciesCommand { match dep.kind { ImportKind::Locked(_) | ImportKind::Unlocked => { let id = PackageName::new(dep.name)?; - Self::parse_deps(&id, dep.req, client, grand_child, parser) - .await?; + Self::parse_deps( + auth_token, + &id, + dep.req, + client, + grand_child, + parser, + ) + .await?; } ImportKind::Interface(_) => {} } @@ -91,7 +102,11 @@ impl DependenciesCommand { Ok(()) } - async fn print_package_info(client: &FileSystemClient, info: &PackageInfo) -> Result<()> { + async fn print_package_info( + auth_token: &Option, + client: &FileSystemClient, + info: &PackageInfo, + ) -> Result<()> { let mut parser = DepsParser::new(); let root_package = client .registry() @@ -100,7 +115,9 @@ impl DependenciesCommand { if let Some(rp) = root_package { let latest = rp.state.releases().last(); if let Some(l) = latest { - client.download(&info.name, &VersionReq::STAR).await?; + client + .download(auth_token, &info.name, &VersionReq::STAR) + .await?; let mut tree = new_tree(info.name.namespace(), info.name.name(), &l.version); if let ReleaseState::Released { content } = &l.state { let path = client.content().content_location(content); @@ -118,6 +135,7 @@ impl DependenciesCommand { match dep.kind { ImportKind::Locked(_) | ImportKind::Unlocked => { Self::parse_deps( + auth_token, &PackageName::new(dep.name)?, dep.req, client, diff --git a/src/commands/download.rs b/src/commands/download.rs index 68a85900..3448790a 100644 --- a/src/commands/download.rs +++ b/src/commands/download.rs @@ -23,12 +23,16 @@ impl DownloadCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; println!("downloading package `{name}`...", name = self.name); let res = client .download( + &auth_token, &self.name, self.version.as_ref().unwrap_or(&VersionReq::STAR), ) diff --git a/src/commands/lock.rs b/src/commands/lock.rs index 4a6c256c..adc86537 100644 --- a/src/commands/lock.rs +++ b/src/commands/lock.rs @@ -25,29 +25,38 @@ impl LockCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.package.namespace()) + .await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::lock(client, &info).await?; + Self::lock(&auth_token, client, &info).await?; } else { - client.download(&self.package, &VersionReq::STAR).await?; + client + .download(&auth_token, &self.package, &VersionReq::STAR) + .await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::lock(client, &info).await?; + Self::lock(&auth_token, client, &info).await?; } } Ok(()) } - async fn lock(client: FileSystemClient, info: &PackageInfo) -> Result<()> { - client.lock_component(info).await?; + async fn lock( + auth_token: &Option, + client: FileSystemClient, + info: &PackageInfo, + ) -> Result<()> { + client.lock_component(&auth_token, info).await?; Ok(()) } } diff --git a/src/commands/login.rs b/src/commands/login.rs new file mode 100644 index 00000000..f7c6de83 --- /dev/null +++ b/src/commands/login.rs @@ -0,0 +1,239 @@ +use anyhow::{Context, Result}; +use clap::Args; +use warg_client::RegistryUrl; + +use crate::keyring::set_auth_token; + +/// Manage signing keys for interacting with a registry. +#[derive(Args)] +pub struct LoginCommand { + /// The subcommand to execute. + // #[clap(subcommand)] + // pub command: LogInSubcommand, + #[clap(flatten)] + keyring_entry: KeyringEntryArgs, + // token: String, +} + +#[derive(Args)] +struct KeyringEntryArgs { + /// The name to use for the signing key. + #[clap(long, short, value_name = "TOKEN_NAME", default_value = "default")] + pub name: String, + /// The URL of the registry to store an auth token for. + #[clap(value_name = "URL")] + pub url: RegistryUrl, +} + +impl std::fmt::Display for KeyringEntryArgs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "`{name}` for registry `{url}`", + name = self.name, + url = self.url + ) + } +} + +impl KeyringEntryArgs { + // fn get_entry(&self) -> Result { + // get_auth_token(&self.token) + // } + + // fn get_key(&self) -> Result { + // get_signing_key(&self.url, &self.name) + // } + + fn set_entry(&self, key: &str) -> Result<()> { + set_auth_token(&self.url, &self.name, key) + } + + // fn delete_entry(&self) -> Result<()> { + // delete_signing_key(&self.url, &self.name) + // } +} + +impl LoginCommand { + /// Executes the command. + pub async fn exec(self) -> Result<()> { + let token = rpassword::prompt_password("Enter auth token:\n") + .context("failed to read auth token")?; + self.keyring_entry.set_entry(&token)?; + println!( + "signing key {keyring} was set successfully", + keyring = self.keyring_entry + ); + Ok(()) + } +} + +// The subcommand to execute. +// #[derive(Subcommand)] +// pub enum KeySubcommand { +// /// Creates a new signing key for a registry in the local keyring. +// New(KeyNewCommand), +// /// Shows information about the signing key for a registry in the local keyring. +// Info(KeyInfoCommand), +// /// Sets the signing key for a registry in the local keyring. +// Set(KeySetCommand), +// /// Deletes the signing key for a registry from the local keyring. +// Delete(KeyDeleteCommand), +// } + +// #[derive(Args)] +// struct KeyringEntryArgs { +// /// The name to use for the signing key. +// #[clap(long, short, value_name = "KEY_NAME", default_value = "default")] +// pub name: String, +// /// The URL of the registry to create a signing key for. +// #[clap(value_name = "URL")] +// pub url: RegistryUrl, +// } + +// impl KeyringEntryArgs { +// fn get_entry(&self) -> Result { +// get_signing_key_entry(&self.url, &self.name) +// } + +// fn get_key(&self) -> Result { +// get_signing_key(&self.url, &self.name) +// } + +// fn set_entry(&self, key: &PrivateKey) -> Result<()> { +// set_signing_key(&self.url, &self.name, key) +// } + +// fn delete_entry(&self) -> Result<()> { +// delete_signing_key(&self.url, &self.name) +// } +// } + +// impl std::fmt::Display for KeyringEntryArgs { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// write!( +// f, +// "`{name}` for registry `{url}`", +// name = self.name, +// url = self.url +// ) +// } +// } + +// Creates a new signing key for a registry in the local keyring. +// #[derive(Args)] +// pub struct KeyNewCommand { +// #[clap(flatten)] +// keyring_entry: KeyringEntryArgs, +// } + +// impl KeyNewCommand { +// /// Executes the command. +// pub async fn exec(self) -> Result<()> { +// let entry = self.keyring_entry.get_entry()?; + +// match entry.get_password() { +// Err(KeyringError::NoEntry) => { +// // no entry exists, so we can continue +// } +// Ok(_) | Err(KeyringError::Ambiguous(_)) => { +// bail!( +// "a signing key `{name}` already exists for registry `{url}`", +// name = self.keyring_entry.name, +// url = self.keyring_entry.url +// ); +// } +// Err(e) => { +// bail!( +// "failed to get signing key {entry}: {e}", +// entry = self.keyring_entry +// ); +// } +// } + +// let key = SigningKey::random(&mut OsRng).into(); +// self.keyring_entry.set_entry(&key)?; + +// Ok(()) +// } +// } + +// /// Shows information about the signing key for a registry in the local keyring. +// #[derive(Args)] +// pub struct KeyInfoCommand { +// #[clap(flatten)] +// keyring_entry: KeyringEntryArgs, +// } + +// impl KeyInfoCommand { +// /// Executes the command. +// pub async fn exec(self) -> Result<()> { +// let private_key = self.keyring_entry.get_key()?; +// let public_key = private_key.public_key(); +// println!("Key ID: {}", public_key.fingerprint()); +// println!("Public Key: {public_key}"); +// Ok(()) +// } +// } + +// /// Sets the signing key for a registry in the local keyring. +// #[derive(Args)] +// pub struct KeySetCommand { +// #[clap(flatten)] +// keyring_entry: KeyringEntryArgs, +// } + +// impl KeySetCommand { +// /// Executes the command. +// pub async fn exec(self) -> Result<()> { +// let key_str = +// rpassword::prompt_password("input signing key (expected format is `:`): ") +// .context("failed to read signing key")?; +// let key = +// PrivateKey::decode(key_str).context("signing key is not in the correct format")?; + +// self.keyring_entry.set_entry(&key)?; + +// println!( +// "signing key {keyring} was set successfully", +// keyring = self.keyring_entry +// ); + +// Ok(()) +// } +// } + +// /// Deletes the signing key for a registry from the local keyring. +// #[derive(Args)] +// pub struct KeyDeleteCommand { +// #[clap(flatten)] +// keyring_entry: KeyringEntryArgs, +// } + +// impl KeyDeleteCommand { +// /// Executes the command. +// pub async fn exec(self) -> Result<()> { +// let prompt = format!( +// "are you sure you want to delete the signing key {entry}? ", +// entry = self.keyring_entry +// ); + +// if Confirm::with_theme(&ColorfulTheme::default()) +// .with_prompt(prompt) +// .interact()? +// { +// self.keyring_entry.delete_entry()?; +// println!( +// "signing key {entry} was deleted successfully", +// entry = self.keyring_entry +// ); +// } else { +// println!( +// "skipping deletion of signing key for registry `{url}`", +// url = self.keyring_entry.url, +// ); +// } + +// Ok(()) +// } +// } diff --git a/src/commands/publish.rs b/src/commands/publish.rs index 5cb18b09..0b887c1c 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -119,7 +119,10 @@ impl PublishInitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? @@ -134,6 +137,7 @@ impl PublishInitCommand { Some(entry) => { let record_id = client .publish_with_info( + &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -194,7 +198,10 @@ impl PublishReleaseCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -226,6 +233,7 @@ impl PublishReleaseCommand { Some(entry) => { let record_id = client .publish_with_info( + &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -285,7 +293,10 @@ impl PublishYankCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -301,6 +312,7 @@ impl PublishYankCommand { Some(entry) => { let record_id = client .publish_with_info( + &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -367,7 +379,10 @@ impl PublishGrantCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -385,6 +400,7 @@ impl PublishGrantCommand { Some(entry) => { let record_id = client .publish_with_info( + &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -453,7 +469,10 @@ impl PublishRevokeCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -471,6 +490,7 @@ impl PublishRevokeCommand { Some(entry) => { let record_id = client .publish_with_info( + &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -526,7 +546,10 @@ impl PublishStartCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; match client.registry().load_publish().await? { Some(info) => bail!("a publish is already in progress for package `{name}`; use `publish abort` to abort the current publish", name = info.name), @@ -649,6 +672,7 @@ impl PublishSubmitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let client = self.common.create_client(&config)?; + let auth_token = self.common.auth_token()?; match client.registry().load_publish().await? { Some(info) => { @@ -658,7 +682,9 @@ impl PublishSubmitCommand { ); let signing_key = self.common.signing_key(client.url())?; - let record_id = client.publish_with_info(&signing_key, info.clone()).await?; + let record_id = client + .publish_with_info(&auth_token, &signing_key, info.clone()) + .await?; client.registry().store_publish(None).await?; @@ -727,7 +753,10 @@ impl PublishWaitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + let auth_token = self.common.auth_token()?; + client + .fetch_namespace(&auth_token, self.name.namespace()) + .await?; let record_id = RecordId::from(self.record_id); println!( diff --git a/src/commands/update.rs b/src/commands/update.rs index bb918eac..38fe591c 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -19,12 +19,13 @@ impl UpdateCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; + let auth_token = self.common.auth_token()?; println!("updating package logs to the latest available versions..."); if self.all { - client.update_all().await?; + client.update_all(&auth_token).await?; } else { - client.update().await?; + client.update(&auth_token).await?; } Ok(()) diff --git a/src/keyring.rs b/src/keyring.rs index 2e2e2f09..2cc6528d 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -5,6 +5,46 @@ use keyring::Entry; use warg_client::RegistryUrl; use warg_crypto::signing::PrivateKey; +/// Gets the auth token entry for the given registry and key name. +pub fn get_auth_token_entry(registry_url: &RegistryUrl, token_name: &str) -> Result { + let label = format!("warg-auth-token:{}", registry_url.safe_label()); + Entry::new(&label, token_name).context("failed to get keyring entry") +} + +/// Gets the auth token +pub fn get_auth_token(registry_url: &RegistryUrl, token_name: &str) -> Result { + let entry = get_signing_key_entry(registry_url, token_name)?; + match entry.get_password() { + Ok(secret) => Ok(secret), + Err(keyring::Error::NoEntry) => { + bail!("no signing key found with name `{token_name}` of registry `{registry_url}`"); + } + Err(keyring::Error::Ambiguous(_)) => { + bail!("more than one signing key found with name `{token_name}` of registry `{registry_url}`"); + } + Err(e) => { + bail!("failed to get signing key with name `{token_name}` of registry `{registry_url}`: {e}"); + } + } +} + +/// Sets the auth token +pub fn set_auth_token(registry_url: &RegistryUrl, token_name: &str, token: &str) -> Result<()> { + let entry = get_auth_token_entry(registry_url, token_name)?; + match entry.set_password(token) { + Ok(()) => Ok(()), + Err(keyring::Error::NoEntry) => { + bail!("no auth token found with name `{token_name}` of registry `{registry_url}`"); + } + Err(keyring::Error::Ambiguous(_)) => { + bail!("more than one auth token with name `{token_name}` of registry `{registry_url}`"); + } + Err(e) => { + bail!("failed to set auth token with name `{token_name}` of registry `{registry_url}`: {e}"); + } + } +} + /// Gets the signing key entry for the given registry and key name. pub fn get_signing_key_entry(registry_url: &RegistryUrl, key_name: &str) -> Result { let label = format!("warg-signing-key:{}", registry_url.safe_label()); From b2bea99347328dcb1f111d20c6d3e2e87c45a9e5 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Sat, 17 Feb 2024 14:15:46 -0600 Subject: [PATCH 5/9] refactor retry logic to occur at command level --- crates/client/src/api.rs | 3 +- crates/client/src/lib.rs | 80 ++++++++---------------- src/bin/warg.rs | 117 ++++++++++++++++++++++++++++++++--- src/commands.rs | 31 ++++++++++ src/commands/bundle.rs | 9 ++- src/commands/dependencies.rs | 9 ++- src/commands/download.rs | 9 ++- src/commands/lock.rs | 9 ++- src/commands/publish.rs | 53 ++++++++++------ src/commands/update.rs | 7 ++- 10 files changed, 227 insertions(+), 100 deletions(-) diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index c57cad87..6ec5e351 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -165,8 +165,7 @@ impl WithWargHeader for RequestBuilder { type Client = Client; fn warg_header(self, registry_header: &Option) -> reqwest::RequestBuilder { if let Some(reg) = registry_header { - let registry_header = HeaderName::try_from(REGISTRY_HEADER_NAME).unwrap(); - self.header(registry_header, reg) + self.header(REGISTRY_HEADER_NAME, reg) } else { self } diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index fde7ed97..e3b16b97 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -125,10 +125,10 @@ impl Client ClientResult<()> { + pub async fn refresh_namespace(&mut self, namespace: &str) -> ClientResult<()> { self.update_checkpoint(&self.api.latest_checkpoint().await?, vec![]) .await?; - let operator = self.registry().load_operator(&None).await.unwrap(); + let operator = self.registry().load_operator(&None).await?; let operator_log_maps_namespace = if let Some(op) = operator { let namespace_state = op.state.namespace_state(namespace); if let Ok(Some(nm)) = namespace_state { @@ -607,7 +607,8 @@ impl Client>(); loop { - let response: FetchLogsResponse = match self + // let response: FetchLogsResponse = match self + let response: FetchLogsResponse = self .api .fetch_logs(FetchLogsRequest { log_length: checkpoint.log_length, @@ -619,59 +620,11 @@ impl Client res, - Err(api::ClientError::LogNotFoundWithHint(log_id, header)) => { - let hint_reg = header.to_str().unwrap(); - let mut terms = hint_reg.split('='); - let namespace = terms.next(); - let registry = terms.next(); - let resp = if let (Some(namespace), Some(registry)) = (namespace, registry) { - print!( - "One of the packages you're requesting does not exist in the registry you're using. - However, the package namespace `{namespace}` does exist in the registry at {registry}.\nWould you like to configure your warg cli to use this registry for packages with this namespace in the future? y/N\n", - ); - std::io::Write::flush(&mut std::io::stdout()).expect("flush failed!"); - let mut buf = String::new(); - std::io::stdin().read_line(&mut buf).unwrap(); - let lowered = buf.to_lowercase(); - if lowered == "y" || lowered == "yes" { - self.store_namespace(namespace.to_string(), registry.to_string()) - .await?; - Some( - self.api - .fetch_logs(FetchLogsRequest { - log_length: checkpoint.log_length, - operator: operator - .head_fetch_token - .as_ref() - .map(|t| Cow::Borrowed(t.as_str())), - limit: None, - packages: Cow::Borrowed(&last_known), - }) - .await?, - ) - } else { - None - } - } else { - None - }; - if let Some(resp) = resp { - resp - } else { - return Err(ClientError::translate_log_not_found( - api::ClientError::Fetch(FetchError::LogNotFound(log_id)), - |id| packages.get(id).map(|p| p.name.clone()), - )); - } - } - Err(e) => { - return Err(ClientError::translate_log_not_found(e, |id| { + .map_err(|e| { + ClientError::translate_log_not_found(e, |id| { packages.get(id).map(|p| p.name.clone()) - })) - } - }; + }) + })?; for record in response.operator { let proto_envelope: PublishedProtoEnvelope = @@ -1096,6 +1049,15 @@ pub enum ClientError { name: PackageName, }, + /// The package does not exist with hint. + #[error("package `{name}` does not exist")] + PackageDoesNotExistWithHint { + /// The missing package. + name: PackageName, + /// The registry hint + hint: HeaderValue, + }, + /// The package version does not exist. #[error("version `{version}` of package `{name}` does not exist")] PackageVersionDoesNotExist { @@ -1191,6 +1153,14 @@ impl ClientError { return Self::PackageDoesNotExist { name }; } } + api::ClientError::LogNotFoundWithHint(log_id, hint) => { + if let Some(name) = lookup(log_id) { + return Self::PackageDoesNotExistWithHint { + name, + hint: hint.clone(), + }; + } + } _ => {} } diff --git a/src/bin/warg.rs b/src/bin/warg.rs index b6273777..b23e9d24 100644 --- a/src/bin/warg.rs +++ b/src/bin/warg.rs @@ -1,10 +1,11 @@ use anyhow::Result; use clap::Parser; +use dialoguer::{theme::ColorfulTheme, Confirm}; use std::process::exit; use tracing_subscriber::EnvFilter; use warg_cli::commands::{ BundleCommand, ClearCommand, ConfigCommand, DependenciesCommand, DownloadCommand, InfoCommand, - KeyCommand, LockCommand, PublishCommand, ResetCommand, UpdateCommand, + KeyCommand, LockCommand, PublishCommand, ResetCommand, Retry, UpdateCommand, }; use warg_client::ClientError; @@ -46,17 +47,17 @@ async fn main() -> Result<()> { WargCli::Config(cmd) => cmd.exec().await, WargCli::Info(cmd) => cmd.exec().await, WargCli::Key(cmd) => cmd.exec().await, - WargCli::Lock(cmd) => cmd.exec().await, - WargCli::Bundle(cmd) => cmd.exec().await, - WargCli::Dependencies(cmd) => cmd.exec().await, - WargCli::Download(cmd) => cmd.exec().await, - WargCli::Update(cmd) => cmd.exec().await, - WargCli::Publish(cmd) => cmd.exec().await, + WargCli::Lock(cmd) => cmd.exec(None).await, + WargCli::Bundle(cmd) => cmd.exec(None).await, + WargCli::Dependencies(cmd) => cmd.exec(None).await, + WargCli::Download(cmd) => cmd.exec(None).await, + WargCli::Update(cmd) => cmd.exec(None).await, + WargCli::Publish(cmd) => cmd.exec(None).await, WargCli::Reset(cmd) => cmd.exec().await, WargCli::Clear(cmd) => cmd.exec().await, } { if let Some(e) = e.downcast_ref::() { - describe_client_error(e); + describe_client_error_or_retry(e).await?; } else { eprintln!("error: {e:?}"); } @@ -66,7 +67,7 @@ async fn main() -> Result<()> { Ok(()) } -fn describe_client_error(e: &ClientError) { +async fn describe_client_error_or_retry(e: &ClientError) -> Result<()> { match e { ClientError::NoHomeRegistryUrl => { eprintln!("error: {e}; use the `config` subcommand to set a home registry URL"); @@ -78,6 +79,102 @@ fn describe_client_error(e: &ClientError) { eprintln!("error: the log for package `{name}` is empty (the registry could be lying)"); eprintln!("see issue https://github.com/bytecodealliance/registry/issues/66"); } - _ => eprintln!("error: {e}"), + ClientError::PackageDoesNotExistWithHint { name, hint } => { + let hint_reg = hint.to_str().unwrap(); + let mut terms = hint_reg.split('='); + let namespace = terms.next(); + let registry = terms.next(); + if let (Some(namespace), Some(registry)) = (namespace, registry) { + let prompt = format!( + "The package `{}`, does not exist in the registry you're using.\nHowever, the package namespace `{namespace}` does exist in the registry at {registry}.\nWould you like to configure your warg cli to use this registry for packages with this namespace in the future? y/N\n", + name.name() + ); + if Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt(prompt) + .interact() + .unwrap() + { + if let Err(e) = match WargCli::parse() { + WargCli::Config(cmd) => cmd.exec().await, + WargCli::Info(cmd) => cmd.exec().await, + WargCli::Key(cmd) => cmd.exec().await, + WargCli::Lock(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Bundle(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Dependencies(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Download(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Update(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Publish(cmd) => { + cmd.exec(Some(Retry::new( + namespace.to_string(), + registry.to_string(), + ))) + .await + } + WargCli::Reset(cmd) => cmd.exec().await, + WargCli::Clear(cmd) => cmd.exec().await, + } { + if let Some(e) = e.downcast_ref::() { + describe_client_error(e).await?; + } else { + eprintln!("error: {e:?}"); + } + exit(1); + } + } + } + } + _ => { + eprintln!("error: {e}") + } + } + Ok(()) +} + +async fn describe_client_error(e: &ClientError) -> Result<()> { + match e { + ClientError::NoHomeRegistryUrl => { + eprintln!("error: {e}; use the `config` subcommand to set a default URL"); + } + ClientError::PackageValidationFailed { name, inner } => { + eprintln!("error: the log for package `{name}` is invalid: {inner}") + } + ClientError::PackageLogEmpty { name } => { + eprintln!("error: the log for package `{name}` is empty (the registry could be lying)"); + eprintln!("see issue https://github.com/bytecodealliance/registry/issues/66"); + } + _ => { + eprintln!("error: {e}") + } } + Ok(()) } diff --git a/src/commands.rs b/src/commands.rs index 0bfaee46..9f8fb1cd 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -4,6 +4,10 @@ use anyhow::Context; use anyhow::Result; use clap::Args; use std::path::PathBuf; +use warg_client::storage::ContentStorage; +use warg_client::storage::NamespaceMapStorage; +use warg_client::storage::RegistryStorage; +use warg_client::Client; use warg_client::RegistryUrl; use warg_client::{ClientError, Config, FileSystemClient, StorageLockResult}; use warg_crypto::signing::PrivateKey; @@ -98,3 +102,30 @@ impl CommonOptions { } } } + +/// Namespace mapping to store when retrying a command after receiving a hint header +pub struct Retry { + namespace: String, + registry: String, +} + +impl Retry { + /// New Retry + pub fn new(namespace: String, registry: String) -> Self { + Self { + namespace, + registry, + } + } + + /// Map namespace using Retry information + pub async fn store_namespace( + &self, + client: &Client, + ) -> Result<()> { + client + .store_namespace(self.namespace.clone(), self.registry.clone()) + .await?; + Ok(()) + } +} diff --git a/src/commands/bundle.rs b/src/commands/bundle.rs index 03d0aca8..0217c0c7 100644 --- a/src/commands/bundle.rs +++ b/src/commands/bundle.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::{bail, Result}; use clap::Args; use semver::VersionReq; @@ -18,10 +18,13 @@ pub struct BundleCommand { impl BundleCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() diff --git a/src/commands/dependencies.rs b/src/commands/dependencies.rs index 5dde5349..c0a43f01 100644 --- a/src/commands/dependencies.rs +++ b/src/commands/dependencies.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::{bail, Result}; use async_recursion::async_recursion; use clap::Args; @@ -28,10 +28,13 @@ pub struct DependenciesCommand { impl DependenciesCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.package.namespace()).await?; if let Some(info) = client .registry() diff --git a/src/commands/download.rs b/src/commands/download.rs index 68a85900..3ec9d99d 100644 --- a/src/commands/download.rs +++ b/src/commands/download.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::{anyhow, Result}; use clap::Args; use warg_protocol::{registry::PackageName, VersionReq}; @@ -20,10 +20,13 @@ pub struct DownloadCommand { impl DownloadCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; println!("downloading package `{name}`...", name = self.name); diff --git a/src/commands/lock.rs b/src/commands/lock.rs index 4a6c256c..18e68ce2 100644 --- a/src/commands/lock.rs +++ b/src/commands/lock.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::Result; use clap::Args; use semver::VersionReq; @@ -22,10 +22,13 @@ pub struct LockCommand { impl LockCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.package.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() diff --git a/src/commands/publish.rs b/src/commands/publish.rs index 5cb18b09..72f7d4c1 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::{anyhow, bail, Context, Result}; use clap::{Args, Subcommand}; use futures::TryStreamExt; @@ -83,13 +83,13 @@ pub enum PublishCommand { impl PublishCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { match self { - Self::Init(cmd) => cmd.exec().await, - Self::Release(cmd) => cmd.exec().await, - Self::Yank(cmd) => cmd.exec().await, - Self::Grant(cmd) => cmd.exec().await, - Self::Revoke(cmd) => cmd.exec().await, + Self::Init(cmd) => cmd.exec(retry).await, + Self::Release(cmd) => cmd.exec(retry).await, + Self::Yank(cmd) => cmd.exec(retry).await, + Self::Grant(cmd) => cmd.exec(retry).await, + Self::Revoke(cmd) => cmd.exec(retry).await, Self::Start(cmd) => cmd.exec().await, Self::List(cmd) => cmd.exec().await, Self::Abort(cmd) => cmd.exec().await, @@ -116,10 +116,13 @@ pub struct PublishInitCommand { impl PublishInitCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? @@ -191,10 +194,13 @@ pub struct PublishReleaseCommand { impl PublishReleaseCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -282,10 +288,13 @@ pub struct PublishYankCommand { impl PublishYankCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -364,10 +373,13 @@ pub struct PublishGrantCommand { impl PublishGrantCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -450,10 +462,13 @@ pub struct PublishRevokeCommand { impl PublishRevokeCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -526,7 +541,7 @@ impl PublishStartCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + client.refresh_namespace(self.name.namespace()).await?; match client.registry().load_publish().await? { Some(info) => bail!("a publish is already in progress for package `{name}`; use `publish abort` to abort the current publish", name = info.name), @@ -727,7 +742,7 @@ impl PublishWaitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - client.fetch_namespace(self.name.namespace()).await?; + client.refresh_namespace(self.name.namespace()).await?; let record_id = RecordId::from(self.record_id); println!( diff --git a/src/commands/update.rs b/src/commands/update.rs index bb918eac..ef699d33 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -1,4 +1,4 @@ -use super::CommonOptions; +use super::{CommonOptions, Retry}; use anyhow::Result; use clap::{ArgAction, Args}; @@ -16,9 +16,12 @@ pub struct UpdateCommand { impl UpdateCommand { /// Executes the command. - pub async fn exec(self) -> Result<()> { + pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; + if let Some(retry) = retry { + retry.store_namespace(&client).await? + } println!("updating package logs to the latest available versions..."); if self.all { From e3178bccfa1afe511c7146d3fb96046adeae08a9 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Sat, 17 Feb 2024 19:25:58 -0600 Subject: [PATCH 6/9] store auth in client as secret --- Cargo.lock | 2 + Cargo.toml | 1 + crates/client/Cargo.toml | 1 + crates/client/src/api.rs | 41 ++++---- crates/client/src/depsolve.rs | 10 +- crates/client/src/lib.rs | 156 +++++++++++----------------- src/commands.rs | 32 ++++-- src/commands/bundle.rs | 13 +-- src/commands/dependencies.rs | 32 ++---- src/commands/download.rs | 6 +- src/commands/lock.rs | 21 ++-- src/commands/login.rs | 188 +--------------------------------- src/commands/publish.rs | 45 ++------ src/commands/update.rs | 5 +- src/keyring.rs | 7 +- tests/client.rs | 8 +- tests/depsolve.rs | 23 ++--- tests/memory/mod.rs | 6 +- tests/server.rs | 33 +++--- tests/support/mod.rs | 3 +- 20 files changed, 176 insertions(+), 457 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed1d872d..642773b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4664,6 +4664,7 @@ dependencies = [ "rand_core", "reqwest", "rpassword", + "secrecy", "semver", "serde_json", "testresult", @@ -4708,6 +4709,7 @@ dependencies = [ "pathdiff", "ptree", "reqwest", + "secrecy", "semver", "serde 1.0.196", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 76444e47..82534e32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ keyring = "2.3.0" dialoguer = "0.11.0" rpassword = "7.3.1" itertools = "0.12.1" +secrecy= { workspace = true } [dev-dependencies] reqwest = { workspace = true } diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index 3374e148..db0d0480 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -44,6 +44,7 @@ wasm-encoder.workspace = true wasmprinter = "0.2.75" sha256 = "1.4.0" ptree = { workspace = true } +secrecy= { workspace = true } [target.'cfg(windows)'.dependencies.windows-sys] version = "0.52" diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 0701b428..1e985512 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -7,6 +7,7 @@ use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, Body, IntoUrl, Method, RequestBuilder, Response, StatusCode, }; +use secrecy::{ExposeSecret, Secret}; use serde::de::DeserializeOwned; use std::{borrow::Cow, collections::HashMap}; use thiserror::Error; @@ -157,12 +158,10 @@ async fn into_result) -> RequestBuilder; } impl WithWargHeader for RequestBuilder { - type Client = Client; fn warg_header(self, registry_header: &Option) -> reqwest::RequestBuilder { if let Some(reg) = registry_header { self.header(REGISTRY_HEADER_NAME, reg) @@ -173,15 +172,13 @@ impl WithWargHeader for RequestBuilder { } trait WithAuth { - type Client; - fn auth(self, auth_token: &Option) -> RequestBuilder; + fn auth(self, auth_token: &Option>) -> RequestBuilder; } impl WithAuth for RequestBuilder { - type Client = Client; - fn auth(self, auth_token: &Option) -> reqwest::RequestBuilder { + fn auth(self, auth_token: &Option>) -> reqwest::RequestBuilder { if let Some(tok) = auth_token { - self.bearer_auth(tok) + self.bearer_auth(tok.expose_secret()) } else { self } @@ -194,19 +191,26 @@ pub struct Client { url: RegistryUrl, client: reqwest::Client, warg_header: Option, + auth_token: Option>, } impl Client { /// Creates a new API client with the given URL. - pub fn new(url: impl IntoUrl) -> Result { + pub fn new(url: impl IntoUrl, auth_token: Option>) -> Result { let url = RegistryUrl::new(url)?; Ok(Self { url, client: reqwest::Client::new(), warg_header: None, + auth_token, }) } + /// Gets auth token + pub fn auth_token(&self) -> &Option> { + &self.auth_token + } + /// Gets the URL of the API client. pub fn url(&self) -> &RegistryUrl { &self.url @@ -222,6 +226,7 @@ impl Client { self.client .get(url) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?, ) @@ -264,6 +269,7 @@ impl Client { .post(url) .json(&request) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?; into_result::<_, MonitorError>(response).await @@ -273,7 +279,6 @@ impl Client { pub async fn fetch_logs( &self, request: FetchLogsRequest<'_>, - auth_token: &Option, ) -> Result { let url = self.url.join(paths::fetch_logs()); tracing::debug!("fetching logs at `{url}`"); @@ -282,7 +287,7 @@ impl Client { .post(&url) .json(&request) .warg_header(self.get_warg_header()) - .auth(auth_token) + .auth(self.auth_token()) .send() .await?; @@ -300,7 +305,6 @@ impl Client { /// Fetches package names from the registry. pub async fn fetch_package_names( &self, - auth_token: &Option, request: FetchPackageNamesRequest<'_>, ) -> Result { let url = self.url.join(paths::fetch_package_names()); @@ -310,7 +314,7 @@ impl Client { .client .post(url) .warg_header(self.get_warg_header()) - .auth(auth_token) + .auth(self.auth_token()) .json(&request) .send() .await?; @@ -326,6 +330,7 @@ impl Client { self.client .get(url) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?, ) @@ -349,6 +354,7 @@ impl Client { .post(url) .json(&request) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?; into_result::<_, PackageError>(response).await @@ -367,6 +373,7 @@ impl Client { self.client .get(url) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?, ) @@ -385,6 +392,7 @@ impl Client { self.client .get(url) .warg_header(self.get_warg_header()) + .auth(self.auth_token()) .send() .await?, ) @@ -394,7 +402,6 @@ impl Client { /// Downloads the content associated with a given record. pub async fn download_content( &self, - auth_token: &Option, digest: &AnyHash, ) -> Result>, ClientError> { tracing::debug!("requesting content download for digest `{digest}`"); @@ -414,7 +421,7 @@ impl Client { .client .get(url) .warg_header(self.get_warg_header()) - .auth(auth_token) + .auth(self.auth_token()) .send() .await?; if !response.status().is_success() { @@ -444,7 +451,6 @@ impl Client { /// Proves the inclusion of the given package log heads in the registry. pub async fn prove_inclusion( &self, - auth_token: &Option, request: InclusionRequest, checkpoint: &Checkpoint, leafs: &[LogLeaf], @@ -457,7 +463,7 @@ impl Client { .post(url) .json(&request) .warg_header(self.get_warg_header()) - .auth(auth_token) + .auth(self.auth_token()) .send() .await?, ) @@ -469,7 +475,6 @@ impl Client { /// Proves consistency between two log roots. pub async fn prove_log_consistency( &self, - auth_token: &Option, request: ConsistencyRequest, from_log_root: Cow<'_, AnyHash>, to_log_root: Cow<'_, AnyHash>, @@ -480,7 +485,7 @@ impl Client { .post(url) .json(&request) .warg_header(self.get_warg_header()) - .auth(auth_token) + .auth(self.auth_token()) .send() .await?, ) diff --git a/crates/client/src/depsolve.rs b/crates/client/src/depsolve.rs index 1f6fa5cd..8528a07f 100644 --- a/crates/client/src/depsolve.rs +++ b/crates/client/src/depsolve.rs @@ -47,7 +47,6 @@ impl LockListBuilder { #[async_recursion] async fn parse_package( &mut self, - auth_token: &Option, client: &Client, mut bytes: &[u8], ) -> Result<()> { @@ -110,12 +109,12 @@ impl LockListBuilder { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { - self.parse_package(auth_token, client, &bytes).await?; + self.parse_package(client, &bytes).await?; } } self.lock_list.insert(import); } else { - client.download(auth_token, &id, &VersionReq::STAR).await?; + client.download(&id, &VersionReq::STAR).await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &id) @@ -124,7 +123,7 @@ impl LockListBuilder { let release = info.state.releases().last(); if let Some(r) = release { if let Some(bytes) = self.release_bytes(r, client)? { - self.parse_package(auth_token, client, &bytes).await?; + self.parse_package(client, &bytes).await?; } } self.lock_list.insert(import); @@ -156,7 +155,6 @@ impl LockListBuilder { #[async_recursion] pub async fn build_list( &mut self, - auth_token: &Option, client: &Client, info: &PackageInfo, ) -> Result<()> { @@ -167,7 +165,7 @@ impl LockListBuilder { let path = client.content().content_location(content); if let Some(p) = path { let bytes = fs::read(p)?; - self.parse_package(auth_token, client, &bytes).await?; + self.parse_package(client, &bytes).await?; } } } diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 5aec4e07..ce3b968c 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -5,6 +5,7 @@ use crate::storage::PackageInfo; use anyhow::{anyhow, Context, Result}; use reqwest::header::HeaderValue; use reqwest::{Body, IntoUrl}; +use secrecy::Secret; use semver::{Version, VersionReq}; use std::cmp::Ordering; use std::fs; @@ -63,15 +64,27 @@ where impl Client { /// Creates a new client for the given URL, registry storage, and /// content storage. - pub fn new(url: impl IntoUrl, registry: R, content: C, namespace_map: N) -> ClientResult { + pub fn new( + url: impl IntoUrl, + registry: R, + content: C, + namespace_map: N, + auth_token: Option>, + ) -> ClientResult { + let api = api::Client::new(url, auth_token)?; Ok(Self { registry, content, namespace_map, - api: api::Client::new(url)?, + api, }) } + /// Gets auth token + pub fn auth_token(&self) -> &Option> { + self.api.auth_token() + } + /// Gets the URL of the client. pub fn url(&self) -> &RegistryUrl { self.api.url() @@ -125,12 +138,8 @@ impl Client, - namespace: &str, - ) -> ClientResult<()> { - self.update_checkpoint(auth_token, &self.api.latest_checkpoint().await?, vec![]) + pub async fn refresh_namespace(&mut self, namespace: &str) -> ClientResult<()> { + self.update_checkpoint(&self.api.latest_checkpoint().await?, vec![]) .await?; let operator = self.registry().load_operator(&None).await?; let operator_log_maps_namespace = if let Some(op) = operator { @@ -172,13 +181,9 @@ impl Client, - info: &PackageInfo, - ) -> ClientResult> { + pub async fn lock_component(&self, info: &PackageInfo) -> ClientResult> { let mut builder = LockListBuilder::default(); - builder.build_list(auth_token, self, info).await?; + builder.build_list(self, info).await?; let top = Import { name: format!("{}:{}", info.name.namespace(), info.name.name()), req: VersionReq::STAR, @@ -270,15 +275,11 @@ impl Client, - info: &PackageInfo, - ) -> ClientResult> { + pub async fn bundle_component(&self, info: &PackageInfo) -> ClientResult> { let mut bundler = Bundler::new(self); let path = PathBuf::from("./locked.wasm"); let locked = if !path.is_file() { - self.lock_component(auth_token, info).await? + self.lock_component(info).await? } else { fs::read("./locked.wasm").map_err(|e| ClientError::Other(e.into()))? }; @@ -295,18 +296,14 @@ impl Client, - signing_key: &signing::PrivateKey, - ) -> ClientResult { + pub async fn publish(&self, signing_key: &signing::PrivateKey) -> ClientResult { let info = self .registry .load_publish() .await? .ok_or(ClientError::NotPublishing)?; - let res = self.publish_with_info(auth_token, signing_key, info).await; + let res = self.publish_with_info(signing_key, info).await; self.registry.store_publish(None).await?; res } @@ -320,7 +317,6 @@ impl Client, signing_key: &signing::PrivateKey, mut info: PublishInfo, ) -> ClientResult { @@ -348,12 +344,8 @@ impl Client Client) -> ClientResult<()> { + pub async fn update_all(&mut self) -> ClientResult<()> { let packages = self.registry.load_all_packages().await?; let checkpoints = self.api.latest_checkpoints(packages.keys()).await?; - self.update_checkpoints(auth_token, checkpoints, packages) - .await?; + self.update_checkpoints(checkpoints, packages).await?; Ok(()) } /// Updates every package log in client storage to the latest registry checkpoint. - pub async fn update(&self, auth_token: &Option) -> ClientResult<()> { + pub async fn update(&self) -> ClientResult<()> { tracing::info!("updating all packages to latest checkpoint"); let mut updating = self.registry.load_packages().await?; - self.update_checkpoint( - auth_token, - &self.api.latest_checkpoint().await?, - &mut updating, - ) - .await?; + self.update_checkpoint(&self.api.latest_checkpoint().await?, &mut updating) + .await?; Ok(()) } /// Inserts or updates the logs of the specified packages in client storage to /// the latest registry checkpoint. - pub async fn upsert<'a, I>( - &self, - auth_token: &Option, - packages: I, - ) -> Result<(), ClientError> + pub async fn upsert<'a, I>(&self, packages: I) -> Result<(), ClientError> where I: IntoIterator, I::IntoIter: ExactSizeIterator, @@ -516,12 +499,8 @@ impl Client Client, name: &PackageName, requirement: &VersionReq, ) -> Result, ClientError> { tracing::info!("downloading package `{name}` with requirement `{requirement}`"); - let info = self.fetch_package(auth_token, name).await?; + let info = self.fetch_package(name).await?; match info.state.find_latest_release(requirement) { Some(release) => { @@ -554,7 +532,7 @@ impl Client Client, package: &PackageName, version: &Version, ) -> Result { tracing::info!("downloading version {version} of package `{package}`"); - let info = self.fetch_package(auth_token, package).await?; + let info = self.fetch_package(package).await?; let release = info.state @@ -601,14 +578,13 @@ impl Client( &self, - auth_token: &Option, ts_checkpoint: &SerdeEnvelope, packages: impl IntoIterator, ) -> Result<(), ClientError> { @@ -644,21 +620,17 @@ impl Client>(); loop { - // let response: FetchLogsResponse = match self let response: FetchLogsResponse = self .api - .fetch_logs( - FetchLogsRequest { - log_length: checkpoint.log_length, - operator: operator - .head_fetch_token - .as_ref() - .map(|t| Cow::Borrowed(t.as_str())), - limit: None, - packages: Cow::Borrowed(&last_known), - }, - auth_token, - ) + .fetch_logs(FetchLogsRequest { + log_length: checkpoint.log_length, + operator: operator + .head_fetch_token + .as_ref() + .map(|t| Cow::Borrowed(t.as_str())), + limit: None, + packages: Cow::Borrowed(&last_known), + }) .await .map_err(|e| { ClientError::translate_log_not_found(e, |id| { @@ -772,7 +744,6 @@ impl Client Client { self.api .prove_log_consistency( - auth_token, ConsistencyRequest { from: from_log_length, to: to_log_length, @@ -850,7 +820,6 @@ impl Client( &mut self, - auth_token: &Option, ts_checkpoints: HashMap>, mut packages: HashMap>, ) -> Result<(), ClientError> { @@ -862,7 +831,7 @@ impl Client Client, - name: &PackageName, - ) -> Result { + async fn fetch_package(&self, name: &PackageName) -> Result { match self .registry .load_package(self.get_warg_header(), name) @@ -886,12 +851,8 @@ impl Client { let mut info = PackageInfo::new(name.clone()); - self.update_checkpoint( - auth_token, - &self.api.latest_checkpoint().await?, - [&mut info], - ) - .await?; + self.update_checkpoint(&self.api.latest_checkpoint().await?, [&mut info]) + .await?; Ok(info) } @@ -924,11 +885,7 @@ impl Client, - digest: &AnyHash, - ) -> Result { + pub async fn download_content(&self, digest: &AnyHash) -> Result { match self.content.content_location(digest) { Some(path) => { tracing::info!("content for digest `{digest}` already exists in storage"); @@ -937,7 +894,7 @@ impl Client { self.content .store_content( - Box::pin(self.api.download_content(auth_token, digest).await?), + Box::pin(self.api.download_content(digest).await?), Some(digest), ) .await?; @@ -976,6 +933,7 @@ impl FileSystemClient { pub fn try_new_with_config( url: Option<&str>, config: &Config, + auth_token: Option>, ) -> Result, ClientError> { let StoragePaths { registry_url: url, @@ -999,6 +957,7 @@ impl FileSystemClient { packages, content, namespace_map, + auth_token, )?)) } @@ -1008,7 +967,11 @@ impl FileSystemClient { /// URL, an error is returned. /// /// This method blocks if storage locks cannot be acquired. - pub fn new_with_config(url: Option<&str>, config: &Config) -> Result { + pub fn new_with_config( + url: Option<&str>, + config: &Config, + auth_token: Option>, + ) -> Result { let StoragePaths { registry_url, registries_dir, @@ -1020,6 +983,7 @@ impl FileSystemClient { FileSystemRegistryStorage::lock(registries_dir)?, FileSystemContentStorage::lock(content_dir)?, FileSystemNamespaceMapStorage::new(namespace_map_path), + auth_token, ) } } diff --git a/src/commands.rs b/src/commands.rs index a7101435..ff31e8a1 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -3,6 +3,7 @@ use anyhow::Context; use anyhow::Result; use clap::Args; +use secrecy::Secret; use std::path::PathBuf; use warg_client::storage::ContentStorage; use warg_client::storage::NamespaceMapStorage; @@ -47,10 +48,10 @@ pub struct CommonOptions { /// The URL of the registry to use. #[clap(long, value_name = "URL")] pub registry: Option, - /// The name to use for the signing key. + /// The name to use for the auth token. #[clap(long, short, value_name = "TOKEN_NAME", default_value = "default")] pub token_name: String, - /// The path to the signing key file. + /// The path to the auth token file. #[clap(long, value_name = "TOKEN_FILE", env = "WARG_AUTH_TOKEN_FILE")] pub token_file: Option, /// The name to use for the signing key. @@ -84,7 +85,11 @@ impl CommonOptions { /// Creates the warg client to use. pub fn create_client(&self, config: &Config) -> Result { - match FileSystemClient::try_new_with_config(self.registry.as_deref(), config)? { + match FileSystemClient::try_new_with_config( + self.registry.as_deref(), + config, + self.auth_token(config)?.map(|tok| Secret::from(tok)), + )? { StorageLockResult::Acquired(client) => Ok(client), StorageLockResult::NotAcquired(path) => { println!( @@ -92,7 +97,11 @@ impl CommonOptions { path = path.display() ); - FileSystemClient::new_with_config(self.registry.as_deref(), config) + FileSystemClient::new_with_config( + self.registry.as_deref(), + config, + self.auth_token(config)?.map(|tok| Secret::from(tok)), + ) } } } @@ -111,18 +120,21 @@ impl CommonOptions { } } /// Gets the auth token for the given registry URL. - pub fn auth_token(&self) -> Result> { + pub fn auth_token(&self, config: &Config) -> Result>> { if let Some(file) = &self.token_file { - Ok(Some( + Ok(Some(Secret::from( std::fs::read_to_string(file) .with_context(|| format!("failed to read key from {file:?}"))? .trim_end() .to_string(), - )) - } else if let Some(reg) = &self.registry { - get_auth_token(&RegistryUrl::new(reg)?, &self.token_name).map(|tok| Some(tok)) + ))) } else { - Ok(None) + let tok = get_auth_token( + &RegistryUrl::new(config.home_url.as_ref().unwrap())?, + &self.token_name, + ) + .map(Some)?; + Ok(tok) } } } diff --git a/src/commands/bundle.rs b/src/commands/bundle.rs index 3516e084..0217c0c7 100644 --- a/src/commands/bundle.rs +++ b/src/commands/bundle.rs @@ -21,30 +21,25 @@ impl BundleCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.package.namespace()) - .await?; + client.refresh_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - client.bundle_component(&auth_token, &info).await?; + client.bundle_component(&info).await?; } else { - client - .download(&auth_token, &self.package, &VersionReq::STAR) - .await?; + client.download(&self.package, &VersionReq::STAR).await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - client.bundle_component(&auth_token, &info).await?; + client.bundle_component(&info).await?; } else { bail!("Unable to find package {}", self.package.name()) } diff --git a/src/commands/dependencies.rs b/src/commands/dependencies.rs index ab54e97f..c0a43f01 100644 --- a/src/commands/dependencies.rs +++ b/src/commands/dependencies.rs @@ -31,20 +31,17 @@ impl DependenciesCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.package.namespace()) - .await?; + client.refresh_namespace(self.package.namespace()).await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::print_package_info(&auth_token, &client, &info).await?; + Self::print_package_info(&client, &info).await?; } Ok(()) @@ -52,14 +49,13 @@ impl DependenciesCommand { #[async_recursion] async fn parse_deps<'a>( - auth_token: &Option, id: &'a PackageName, version: VersionReq, client: &FileSystemClient, node: &mut TreeBuilder, parser: &mut DepsParser, ) -> Result<()> { - client.download(auth_token, id, &version).await?; + client.download(id, &version).await?; let package = client .registry() @@ -84,15 +80,8 @@ impl DependenciesCommand { match dep.kind { ImportKind::Locked(_) | ImportKind::Unlocked => { let id = PackageName::new(dep.name)?; - Self::parse_deps( - auth_token, - &id, - dep.req, - client, - grand_child, - parser, - ) - .await?; + Self::parse_deps(&id, dep.req, client, grand_child, parser) + .await?; } ImportKind::Interface(_) => {} } @@ -105,11 +94,7 @@ impl DependenciesCommand { Ok(()) } - async fn print_package_info( - auth_token: &Option, - client: &FileSystemClient, - info: &PackageInfo, - ) -> Result<()> { + async fn print_package_info(client: &FileSystemClient, info: &PackageInfo) -> Result<()> { let mut parser = DepsParser::new(); let root_package = client .registry() @@ -118,9 +103,7 @@ impl DependenciesCommand { if let Some(rp) = root_package { let latest = rp.state.releases().last(); if let Some(l) = latest { - client - .download(auth_token, &info.name, &VersionReq::STAR) - .await?; + client.download(&info.name, &VersionReq::STAR).await?; let mut tree = new_tree(info.name.namespace(), info.name.name(), &l.version); if let ReleaseState::Released { content } = &l.state { let path = client.content().content_location(content); @@ -138,7 +121,6 @@ impl DependenciesCommand { match dep.kind { ImportKind::Locked(_) | ImportKind::Unlocked => { Self::parse_deps( - auth_token, &PackageName::new(dep.name)?, dep.req, client, diff --git a/src/commands/download.rs b/src/commands/download.rs index a2d026e2..3ec9d99d 100644 --- a/src/commands/download.rs +++ b/src/commands/download.rs @@ -23,19 +23,15 @@ impl DownloadCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; println!("downloading package `{name}`...", name = self.name); let res = client .download( - &auth_token, &self.name, self.version.as_ref().unwrap_or(&VersionReq::STAR), ) diff --git a/src/commands/lock.rs b/src/commands/lock.rs index 55ebca78..18e68ce2 100644 --- a/src/commands/lock.rs +++ b/src/commands/lock.rs @@ -25,41 +25,32 @@ impl LockCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.package.namespace()) - .await?; + client.refresh_namespace(self.package.namespace()).await?; println!("registry: {url}", url = client.url()); if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::lock(&auth_token, client, &info).await?; + Self::lock(client, &info).await?; } else { - client - .download(&auth_token, &self.package, &VersionReq::STAR) - .await?; + client.download(&self.package, &VersionReq::STAR).await?; if let Some(info) = client .registry() .load_package(client.get_warg_header(), &self.package) .await? { - Self::lock(&auth_token, client, &info).await?; + Self::lock(client, &info).await?; } } Ok(()) } - async fn lock( - auth_token: &Option, - client: FileSystemClient, - info: &PackageInfo, - ) -> Result<()> { - client.lock_component(&auth_token, info).await?; + async fn lock(client: FileSystemClient, info: &PackageInfo) -> Result<()> { + client.lock_component(info).await?; Ok(()) } } diff --git a/src/commands/login.rs b/src/commands/login.rs index f7c6de83..a1a0e096 100644 --- a/src/commands/login.rs +++ b/src/commands/login.rs @@ -37,21 +37,9 @@ impl std::fmt::Display for KeyringEntryArgs { } impl KeyringEntryArgs { - // fn get_entry(&self) -> Result { - // get_auth_token(&self.token) - // } - - // fn get_key(&self) -> Result { - // get_signing_key(&self.url, &self.name) - // } - - fn set_entry(&self, key: &str) -> Result<()> { - set_auth_token(&self.url, &self.name, key) + fn set_entry(&self, token: &str) -> Result<()> { + set_auth_token(&self.url, &self.name, token) } - - // fn delete_entry(&self) -> Result<()> { - // delete_signing_key(&self.url, &self.name) - // } } impl LoginCommand { @@ -61,179 +49,9 @@ impl LoginCommand { .context("failed to read auth token")?; self.keyring_entry.set_entry(&token)?; println!( - "signing key {keyring} was set successfully", + "auth token {keyring} was set successfully", keyring = self.keyring_entry ); Ok(()) } } - -// The subcommand to execute. -// #[derive(Subcommand)] -// pub enum KeySubcommand { -// /// Creates a new signing key for a registry in the local keyring. -// New(KeyNewCommand), -// /// Shows information about the signing key for a registry in the local keyring. -// Info(KeyInfoCommand), -// /// Sets the signing key for a registry in the local keyring. -// Set(KeySetCommand), -// /// Deletes the signing key for a registry from the local keyring. -// Delete(KeyDeleteCommand), -// } - -// #[derive(Args)] -// struct KeyringEntryArgs { -// /// The name to use for the signing key. -// #[clap(long, short, value_name = "KEY_NAME", default_value = "default")] -// pub name: String, -// /// The URL of the registry to create a signing key for. -// #[clap(value_name = "URL")] -// pub url: RegistryUrl, -// } - -// impl KeyringEntryArgs { -// fn get_entry(&self) -> Result { -// get_signing_key_entry(&self.url, &self.name) -// } - -// fn get_key(&self) -> Result { -// get_signing_key(&self.url, &self.name) -// } - -// fn set_entry(&self, key: &PrivateKey) -> Result<()> { -// set_signing_key(&self.url, &self.name, key) -// } - -// fn delete_entry(&self) -> Result<()> { -// delete_signing_key(&self.url, &self.name) -// } -// } - -// impl std::fmt::Display for KeyringEntryArgs { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// write!( -// f, -// "`{name}` for registry `{url}`", -// name = self.name, -// url = self.url -// ) -// } -// } - -// Creates a new signing key for a registry in the local keyring. -// #[derive(Args)] -// pub struct KeyNewCommand { -// #[clap(flatten)] -// keyring_entry: KeyringEntryArgs, -// } - -// impl KeyNewCommand { -// /// Executes the command. -// pub async fn exec(self) -> Result<()> { -// let entry = self.keyring_entry.get_entry()?; - -// match entry.get_password() { -// Err(KeyringError::NoEntry) => { -// // no entry exists, so we can continue -// } -// Ok(_) | Err(KeyringError::Ambiguous(_)) => { -// bail!( -// "a signing key `{name}` already exists for registry `{url}`", -// name = self.keyring_entry.name, -// url = self.keyring_entry.url -// ); -// } -// Err(e) => { -// bail!( -// "failed to get signing key {entry}: {e}", -// entry = self.keyring_entry -// ); -// } -// } - -// let key = SigningKey::random(&mut OsRng).into(); -// self.keyring_entry.set_entry(&key)?; - -// Ok(()) -// } -// } - -// /// Shows information about the signing key for a registry in the local keyring. -// #[derive(Args)] -// pub struct KeyInfoCommand { -// #[clap(flatten)] -// keyring_entry: KeyringEntryArgs, -// } - -// impl KeyInfoCommand { -// /// Executes the command. -// pub async fn exec(self) -> Result<()> { -// let private_key = self.keyring_entry.get_key()?; -// let public_key = private_key.public_key(); -// println!("Key ID: {}", public_key.fingerprint()); -// println!("Public Key: {public_key}"); -// Ok(()) -// } -// } - -// /// Sets the signing key for a registry in the local keyring. -// #[derive(Args)] -// pub struct KeySetCommand { -// #[clap(flatten)] -// keyring_entry: KeyringEntryArgs, -// } - -// impl KeySetCommand { -// /// Executes the command. -// pub async fn exec(self) -> Result<()> { -// let key_str = -// rpassword::prompt_password("input signing key (expected format is `:`): ") -// .context("failed to read signing key")?; -// let key = -// PrivateKey::decode(key_str).context("signing key is not in the correct format")?; - -// self.keyring_entry.set_entry(&key)?; - -// println!( -// "signing key {keyring} was set successfully", -// keyring = self.keyring_entry -// ); - -// Ok(()) -// } -// } - -// /// Deletes the signing key for a registry from the local keyring. -// #[derive(Args)] -// pub struct KeyDeleteCommand { -// #[clap(flatten)] -// keyring_entry: KeyringEntryArgs, -// } - -// impl KeyDeleteCommand { -// /// Executes the command. -// pub async fn exec(self) -> Result<()> { -// let prompt = format!( -// "are you sure you want to delete the signing key {entry}? ", -// entry = self.keyring_entry -// ); - -// if Confirm::with_theme(&ColorfulTheme::default()) -// .with_prompt(prompt) -// .interact()? -// { -// self.keyring_entry.delete_entry()?; -// println!( -// "signing key {entry} was deleted successfully", -// entry = self.keyring_entry -// ); -// } else { -// println!( -// "skipping deletion of signing key for registry `{url}`", -// url = self.keyring_entry.url, -// ); -// } - -// Ok(()) -// } -// } diff --git a/src/commands/publish.rs b/src/commands/publish.rs index cc1b0cde..72f7d4c1 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -119,13 +119,10 @@ impl PublishInitCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? @@ -140,7 +137,6 @@ impl PublishInitCommand { Some(entry) => { let record_id = client .publish_with_info( - &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -201,13 +197,10 @@ impl PublishReleaseCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -239,7 +232,6 @@ impl PublishReleaseCommand { Some(entry) => { let record_id = client .publish_with_info( - &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -299,13 +291,10 @@ impl PublishYankCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -321,7 +310,6 @@ impl PublishYankCommand { Some(entry) => { let record_id = client .publish_with_info( - &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -388,13 +376,10 @@ impl PublishGrantCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -412,7 +397,6 @@ impl PublishGrantCommand { Some(entry) => { let record_id = client .publish_with_info( - &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -481,13 +465,10 @@ impl PublishRevokeCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? } else { @@ -505,7 +486,6 @@ impl PublishRevokeCommand { Some(entry) => { let record_id = client .publish_with_info( - &auth_token, &signing_key, PublishInfo { name: self.name.clone(), @@ -561,10 +541,7 @@ impl PublishStartCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; match client.registry().load_publish().await? { Some(info) => bail!("a publish is already in progress for package `{name}`; use `publish abort` to abort the current publish", name = info.name), @@ -687,7 +664,6 @@ impl PublishSubmitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; match client.registry().load_publish().await? { Some(info) => { @@ -697,9 +673,7 @@ impl PublishSubmitCommand { ); let signing_key = self.common.signing_key(client.url())?; - let record_id = client - .publish_with_info(&auth_token, &signing_key, info.clone()) - .await?; + let record_id = client.publish_with_info(&signing_key, info.clone()).await?; client.registry().store_publish(None).await?; @@ -768,10 +742,7 @@ impl PublishWaitCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; - client - .refresh_namespace(&auth_token, self.name.namespace()) - .await?; + client.refresh_namespace(self.name.namespace()).await?; let record_id = RecordId::from(self.record_id); println!( diff --git a/src/commands/update.rs b/src/commands/update.rs index 330006ec..ef699d33 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -19,16 +19,15 @@ impl UpdateCommand { pub async fn exec(self, retry: Option) -> Result<()> { let config = self.common.read_config()?; let mut client = self.common.create_client(&config)?; - let auth_token = self.common.auth_token()?; if let Some(retry) = retry { retry.store_namespace(&client).await? } println!("updating package logs to the latest available versions..."); if self.all { - client.update_all(&auth_token).await?; + client.update_all().await?; } else { - client.update(&auth_token).await?; + client.update().await?; } Ok(()) diff --git a/src/keyring.rs b/src/keyring.rs index 2cc6528d..51e0da2a 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Context, Result}; use keyring::Entry; +use secrecy::{self, Secret}; use warg_client::RegistryUrl; use warg_crypto::signing::PrivateKey; @@ -12,10 +13,10 @@ pub fn get_auth_token_entry(registry_url: &RegistryUrl, token_name: &str) -> Res } /// Gets the auth token -pub fn get_auth_token(registry_url: &RegistryUrl, token_name: &str) -> Result { - let entry = get_signing_key_entry(registry_url, token_name)?; +pub fn get_auth_token(registry_url: &RegistryUrl, token_name: &str) -> Result> { + let entry = get_auth_token_entry(registry_url, token_name)?; match entry.get_password() { - Ok(secret) => Ok(secret), + Ok(secret) => Ok(Secret::from(secret)), Err(keyring::Error::NoEntry) => { bail!("no signing key found with name `{token_name}` of registry `{registry_url}`"); } diff --git a/tests/client.rs b/tests/client.rs index e9a8ed76..0f9bf3d6 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -10,7 +10,7 @@ use warg_protocol::registry::PackageName; pub mod support; fn create_client(config: &Config) -> Result { - match FileSystemClient::try_new_with_config(None, config)? { + match FileSystemClient::try_new_with_config(None, config, None)? { StorageLockResult::Acquired(client) => Ok(client), _ => bail!("failed to acquire storage lock"), } @@ -42,7 +42,6 @@ async fn client_incrementally_fetches() -> Result<()> { let name = PackageName::new(PACKAGE_NAME)?; let mut head = client .publish_with_info( - &None, &signing_key, PublishInfo { name: name.clone(), @@ -55,7 +54,6 @@ async fn client_incrementally_fetches() -> Result<()> { for i in 1..=RELEASE_COUNT { head = client .publish_with_info( - &None, &signing_key, PublishInfo { name: name.clone(), @@ -83,10 +81,10 @@ async fn client_incrementally_fetches() -> Result<()> { let client = create_client(&config)?; // Regression test: update on empty registry storage - client.update(&None).await?; + client.update().await?; // Fetch the package log - client.upsert(&None, [&name]).await?; + client.upsert([&name]).await?; // Ensure that the package is in the packages list let packages = client.registry().load_packages().await?; diff --git a/tests/depsolve.rs b/tests/depsolve.rs index 50befcc0..40ed4047 100644 --- a/tests/depsolve.rs +++ b/tests/depsolve.rs @@ -77,17 +77,14 @@ async fn depsolve() -> Result<()> { ) .await?; - client.update(&None).await?; + client.update().await?; client - .upsert( - &None, - [ - &PackageName::new("test:add")?, - &PackageName::new("test:inc")?, - &PackageName::new("test:five")?, - &PackageName::new("test:meet")?, - ], - ) + .upsert([ + &PackageName::new("test:add")?, + &PackageName::new("test:inc")?, + &PackageName::new("test:five")?, + &PackageName::new("test:meet")?, + ]) .await?; let info = client @@ -96,13 +93,13 @@ async fn depsolve() -> Result<()> { .await? .context("package does not exist in client storage")?; - let locked_bytes = client.lock_component(&None, &info).await?; + let locked_bytes = client.lock_component(&info).await?; let expected_locked = wat::parse_file("tests/components/meet_locked.wat")?; assert_eq!( wasmprinter::print_bytes(&locked_bytes)?, wasmprinter::print_bytes(expected_locked)? ); - let bundled_bytes = client.bundle_component(&None, &info).await?; + let bundled_bytes = client.bundle_component(&info).await?; let expected_bundled = wat::parse_file("tests/components/meet_bundled.wat")?; assert_eq!( wasmprinter::print_bytes(bundled_bytes)?, @@ -132,7 +129,6 @@ async fn publish_package( .await?; let mut head = client .publish_with_info( - &None, signing_key, PublishInfo { name: name.clone(), @@ -146,7 +142,6 @@ async fn publish_package( .await?; head = client .publish_with_info( - &None, signing_key, PublishInfo { name: name.clone(), diff --git a/tests/memory/mod.rs b/tests/memory/mod.rs index 5a7368c3..daa15ab0 100644 --- a/tests/memory/mod.rs +++ b/tests/memory/mod.rs @@ -16,7 +16,7 @@ async fn it_publishes_a_component() -> Result<()> { test_component_publishing(&config).await?; // There should be two log entries in the registry - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, @@ -35,7 +35,7 @@ async fn it_yanks_a_package() -> Result<()> { test_package_yanking(&config).await?; // There should be three entries in the registry - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, @@ -52,7 +52,7 @@ async fn it_publishes_a_wit_package() -> Result<()> { test_wit_publishing(&config).await?; // There should be two log entries in the registry - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ts_checkpoint = client.latest_checkpoint().await?; assert_eq!( ts_checkpoint.as_ref().checkpoint.log_length, diff --git a/tests/server.rs b/tests/server.rs index 0677db74..13fb2926 100644 --- a/tests/server.rs +++ b/tests/server.rs @@ -39,7 +39,7 @@ mod memory; mod postgres; async fn test_initial_checkpoint(config: &Config) -> Result<()> { - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ts_checkpoint = client.latest_checkpoint().await?; let checkpoint = &ts_checkpoint.as_ref().checkpoint; @@ -84,9 +84,9 @@ async fn test_component_publishing(config: &Config) -> Result<()> { .await?; // Assert that the package can be downloaded - client.upsert(&None, [&name]).await?; + client.upsert([&name]).await?; let download = client - .download(&None, &name, &PACKAGE_VERSION.parse()?) + .download(&name, &PACKAGE_VERSION.parse()?) .await? .context("failed to resolve package")?; @@ -109,10 +109,7 @@ async fn test_component_publishing(config: &Config) -> Result<()> { } // Assert that a different version can't be downloaded - assert!(client - .download(&None, &name, &"0.2.0".parse()?) - .await? - .is_none()); + assert!(client.download(&name, &"0.2.0".parse()?).await?.is_none()); Ok(()) } @@ -138,7 +135,6 @@ async fn test_package_yanking(config: &Config) -> Result<()> { // Yank release let record_id = client .publish_with_info( - &None, &signing_key, PublishInfo { name: name.clone(), @@ -154,10 +150,8 @@ async fn test_package_yanking(config: &Config) -> Result<()> { .await?; // Assert that the package is yanked - client.upsert(&None, [&name]).await?; - let opt = client - .download(&None, &name, &PACKAGE_VERSION.parse()?) - .await?; + client.upsert([&name]).await?; + let opt = client.download(&name, &PACKAGE_VERSION.parse()?).await?; assert!(opt.is_none(), "expected no download, got {opt:?}"); Ok(()) } @@ -180,9 +174,9 @@ async fn test_wit_publishing(config: &Config) -> Result<()> { .await?; // Assert that the package can be downloaded - client.upsert(&None, [&name]).await?; + client.upsert([&name]).await?; let download = client - .download(&None, &name, &PACKAGE_VERSION.parse()?) + .download(&name, &PACKAGE_VERSION.parse()?) .await? .context("failed to resolve package")?; @@ -205,10 +199,7 @@ async fn test_wit_publishing(config: &Config) -> Result<()> { } // Assert that a different version can't be downloaded - assert!(client - .download(&None, &name, &"0.2.0".parse()?) - .await? - .is_none()); + assert!(client.download(&name, &"0.2.0".parse()?).await?.is_none()); Ok(()) } @@ -447,7 +438,7 @@ async fn test_custom_content_url(config: &Config) -> Result<()> { ) .await?; - client.upsert(&None, [&name]).await?; + client.upsert([&name]).await?; let package = client .registry() .load_package(client.get_warg_header(), &name) @@ -459,7 +450,7 @@ async fn test_custom_content_url(config: &Config) -> Result<()> { .expect("expected the package version to exist"); // Look up the content URL for the record - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ContentSourcesResponse { content_sources } = client.content_sources(&digest).await?; assert_eq!(content_sources.len(), 1); let sources = content_sources @@ -520,7 +511,7 @@ async fn test_fetch_package_names(config: &Config) -> Result<()> { } async fn test_get_ledger(config: &Config) -> Result<()> { - let client = api::Client::new(config.home_url.as_ref().unwrap())?; + let client = api::Client::new(config.home_url.as_ref().unwrap(), None)?; let ts_checkpoint = client.latest_checkpoint().await?; let checkpoint = &ts_checkpoint.as_ref().checkpoint; diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 6825f752..9307b4ac 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -43,7 +43,7 @@ pub fn test_signing_key() -> PrivateKey { } pub fn create_client(config: &warg_client::Config) -> Result { - match FileSystemClient::try_new_with_config(None, config)? { + match FileSystemClient::try_new_with_config(None, config, None)? { StorageLockResult::Acquired(client) => Ok(client), _ => bail!("failed to acquire storage lock"), } @@ -196,7 +196,6 @@ pub async fn publish( let record_id = client .publish_with_info( - &None, signing_key, PublishInfo { name: name.clone(), From 59fd300c9424764ea0ad188aaf7c9bfba65a5e28 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Mon, 19 Feb 2024 09:42:55 -0600 Subject: [PATCH 7/9] iterating --- config.json | 21 +++++ crates/client/src/config.rs | 15 ++++ new | 6 ++ src/commands.rs | 4 +- src/commands/config.rs | 15 +++- src/commands/key.rs | 82 +++++++++++------- src/commands/publish.rs | 28 ++++--- src/keyring.rs | 163 ++++++++++++++++++++++++++++++++---- 8 files changed, 273 insertions(+), 61 deletions(-) create mode 100644 config.json create mode 100644 new diff --git a/config.json b/config.json new file mode 100644 index 00000000..1a2f0736 --- /dev/null +++ b/config.json @@ -0,0 +1,21 @@ +{ + "homeUrl": "https://dannyjaf.preview.wa.dev/", + "creds": { + "keys": { + "wasi.preview.wa.dev": { + "fooey": "fooey", + "default": "default" + }, + "default": { + "fooey": "fooey", + "default": "default", + "dannyjaf.preview.wa.dev": "dannyjaf.preview.wa.dev" + }, + "dannyjaf.preview.wa.dev": { + "dannyjaf.preview.wa.dev": "dannyjaf.preview.wa.dev", + "default": "default" + } + }, + "tokens": [] + } +} \ No newline at end of file diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index e4b9529f..08d56970 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -6,6 +6,7 @@ use normpath::PathExt; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use std::{ + collections::HashMap, env::current_dir, fs::{self, File}, path::{Component, Path, PathBuf}, @@ -71,6 +72,15 @@ pub struct StoragePaths { pub namespace_map_path: PathBuf, } +/// List of creds available in keyring +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct CredList { + /// Keys + pub keys: HashMap>, + /// Access tokens + pub tokens: Vec, +} + /// Represents the Warg client configuration. #[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -105,6 +115,10 @@ pub struct Config { /// `$CACHE_DIR` is the platform-specific cache directory. #[serde(default, skip_serializing_if = "Option::is_none")] pub namespace_map_path: Option, + + /// List of creds availabe in keyring + #[serde(default, skip_serializing_if = "Option::is_none")] + pub creds: Option, } impl Config { @@ -181,6 +195,7 @@ impl Config { assert!(p.is_absolute()); pathdiff::diff_paths(&p, &parent).unwrap() }), + creds: self.creds.clone(), }; serde_json::to_writer_pretty( diff --git a/new b/new new file mode 100644 index 00000000..67c63f3d --- /dev/null +++ b/new @@ -0,0 +1,6 @@ +{ + "creds": { + "keys": {}, + "tokens": [] + } +} \ No newline at end of file diff --git a/src/commands.rs b/src/commands.rs index ff31e8a1..cc7c0584 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -107,7 +107,7 @@ impl CommonOptions { } /// Gets the signing key for the given registry URL. - pub fn signing_key(&self, registry_url: &RegistryUrl) -> Result { + pub fn signing_key(&self, registry_url: &RegistryUrl, config: &Config) -> Result { if let Some(file) = &self.key_file { let key_str = std::fs::read_to_string(file) .with_context(|| format!("failed to read key from {file:?}"))? @@ -116,7 +116,7 @@ impl CommonOptions { PrivateKey::decode(key_str) .with_context(|| format!("failed to parse key from {file:?}")) } else { - get_signing_key(registry_url, &self.key_name) + get_signing_key(&Some(registry_url.clone()), &self.key_name, config) } } /// Gets the auth token for the given registry URL. diff --git a/src/commands/config.rs b/src/commands/config.rs index 3f0a3a9b..0a79cdcc 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,3 +1,4 @@ +use super::CommonOptions; use anyhow::{bail, Context, Result}; use clap::Args; use std::path::PathBuf; @@ -6,10 +7,13 @@ use warg_client::{Config, RegistryUrl}; /// Creates a new warg configuration file. #[derive(Args)] pub struct ConfigCommand { + /// The common command options. + #[clap(flatten)] + pub common: CommonOptions, + /// The home registry URL to use. #[clap(long, value_name = "URL")] - pub registry: Option, - + pub registry_url: Option, /// The path to the registries directory to use. #[clap(long, value_name = "REGISTRIES")] pub registries_dir: Option, @@ -48,8 +52,10 @@ impl ConfigCommand { ); } - let home_url = self + let home_url = &self + .common .registry + .clone() .map(RegistryUrl::new) .transpose()? .map(|u| u.to_string()); @@ -61,10 +67,11 @@ impl ConfigCommand { // the configuration file's directory. let cwd = std::env::current_dir().context("failed to determine current directory")?; let config = Config { - home_url, + home_url: home_url.clone(), registries_dir: self.registries_dir.map(|p| cwd.join(p)), content_dir: self.content_dir.map(|p| cwd.join(p)), namespace_map_path: self.namespace_path.map(|p| cwd.join(p)), + creds: self.common.read_config()?.creds, }; config.write_to_file(&path)?; diff --git a/src/commands/key.rs b/src/commands/key.rs index de76665f..98679988 100644 --- a/src/commands/key.rs +++ b/src/commands/key.rs @@ -5,9 +5,11 @@ use dialoguer::{theme::ColorfulTheme, Confirm}; use keyring::{Entry, Error as KeyringError}; use p256::ecdsa::SigningKey; use rand_core::OsRng; -use warg_client::RegistryUrl; +use warg_client::{Config, RegistryUrl}; use warg_crypto::signing::PrivateKey; +use super::CommonOptions; + /// Manage signing keys for interacting with a registry. #[derive(Args)] pub struct KeyCommand { @@ -48,35 +50,34 @@ struct KeyringEntryArgs { pub name: String, /// The URL of the registry to create a signing key for. #[clap(value_name = "URL")] - pub url: RegistryUrl, + pub url: Option, } impl KeyringEntryArgs { - fn get_entry(&self) -> Result { - get_signing_key_entry(&self.url, &self.name) + fn get_entry(&self, config: &Config) -> Result { + get_signing_key_entry(&self.url, &self.name, config) } - fn get_key(&self) -> Result { - get_signing_key(&self.url, &self.name) + fn get_key(&self, config: &Config) -> Result { + get_signing_key(&self.url, &self.name, config) } - fn set_entry(&self, key: &PrivateKey) -> Result<()> { - set_signing_key(&self.url, &self.name, key) + fn set_entry(&self, key: &PrivateKey, config: &mut Config) -> Result<()> { + set_signing_key(&self.url, &self.name, key, config) } - fn delete_entry(&self) -> Result<()> { - delete_signing_key(&self.url, &self.name) + fn delete_entry(&self, config: &Config) -> Result<()> { + delete_signing_key(&self.url, &self.name, config) } } impl std::fmt::Display for KeyringEntryArgs { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "`{name}` for registry `{url}`", - name = self.name, - url = self.url - ) + if let Some(url) = &self.url { + write!(f, "`{name}` for registry `{url}`", name = self.name,) + } else { + write!(f, "{name}", name = self.name) + } } } @@ -85,23 +86,34 @@ impl std::fmt::Display for KeyringEntryArgs { pub struct KeyNewCommand { #[clap(flatten)] keyring_entry: KeyringEntryArgs, + /// The common command options. + #[clap(flatten)] + pub common: CommonOptions, } impl KeyNewCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { - let entry = self.keyring_entry.get_entry()?; + let config = &mut self.common.read_config()?; + + let entry = self.keyring_entry.get_entry(config)?; match entry.get_password() { Err(KeyringError::NoEntry) => { // no entry exists, so we can continue } Ok(_) | Err(KeyringError::Ambiguous(_)) => { - bail!( - "a signing key `{name}` already exists for registry `{url}`", - name = self.keyring_entry.name, - url = self.keyring_entry.url - ); + if let Some(url) = self.keyring_entry.url { + bail!( + "a signing key `{name}` already exists for registry `{url}`", + name = self.keyring_entry.name, + ); + } else { + bail!( + "a signing key `{name}` already exists", + name = self.keyring_entry.name, + ); + } } Err(e) => { bail!( @@ -112,7 +124,7 @@ impl KeyNewCommand { } let key = SigningKey::random(&mut OsRng).into(); - self.keyring_entry.set_entry(&key)?; + self.keyring_entry.set_entry(&key, config)?; Ok(()) } @@ -123,12 +135,16 @@ impl KeyNewCommand { pub struct KeyInfoCommand { #[clap(flatten)] keyring_entry: KeyringEntryArgs, + /// The common command options. + #[clap(flatten)] + pub common: CommonOptions, } impl KeyInfoCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { - let private_key = self.keyring_entry.get_key()?; + let config = &self.common.read_config()?; + let private_key = self.keyring_entry.get_key(config)?; let public_key = private_key.public_key(); println!("Key ID: {}", public_key.fingerprint()); println!("Public Key: {public_key}"); @@ -141,18 +157,22 @@ impl KeyInfoCommand { pub struct KeySetCommand { #[clap(flatten)] keyring_entry: KeyringEntryArgs, + /// The common command options. + #[clap(flatten)] + pub common: CommonOptions, } impl KeySetCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { + let config = &mut self.common.read_config()?; let key_str = rpassword::prompt_password("input signing key (expected format is `:`): ") .context("failed to read signing key")?; let key = PrivateKey::decode(key_str).context("signing key is not in the correct format")?; - self.keyring_entry.set_entry(&key)?; + self.keyring_entry.set_entry(&key, config)?; println!( "signing key {keyring} was set successfully", @@ -168,11 +188,15 @@ impl KeySetCommand { pub struct KeyDeleteCommand { #[clap(flatten)] keyring_entry: KeyringEntryArgs, + /// The common command options. + #[clap(flatten)] + pub common: CommonOptions, } impl KeyDeleteCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { + let config = &self.common.read_config()?; let prompt = format!( "are you sure you want to delete the signing key {entry}? ", entry = self.keyring_entry @@ -182,16 +206,18 @@ impl KeyDeleteCommand { .with_prompt(prompt) .interact()? { - self.keyring_entry.delete_entry()?; + self.keyring_entry.delete_entry(&config)?; println!( "signing key {entry} was deleted successfully", entry = self.keyring_entry ); - } else { + } else if let Some(url) = self.keyring_entry.url { println!( "skipping deletion of signing key for registry `{url}`", - url = self.keyring_entry.url, + url = url ); + } else { + println!("skipping deletion of signing key"); } Ok(()) diff --git a/src/commands/publish.rs b/src/commands/publish.rs index 72f7d4c1..7b36c70a 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -125,9 +125,10 @@ impl PublishInitCommand { client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { - self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? + self.common + .signing_key(&RegistryUrl::new(nm.to_str()?)?, &config)? } else { - self.common.signing_key(client.url())? + self.common.signing_key(client.url(), &config)? }; match enqueue(&client, &self.name, |_| { std::future::ready(Ok(PublishEntry::Init)) @@ -202,9 +203,11 @@ impl PublishReleaseCommand { } client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { - self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? + self.common + .signing_key(&RegistryUrl::new(nm.to_str()?)?, &config)? } else { - self.common.signing_key(client.url())? + dbg!(&client.url()); + self.common.signing_key(client.url(), &config)? }; let path = self.path.clone(); @@ -296,9 +299,10 @@ impl PublishYankCommand { } client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { - self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? + self.common + .signing_key(&RegistryUrl::new(nm.to_str()?)?, &config)? } else { - self.common.signing_key(client.url())? + self.common.signing_key(client.url(), &config)? }; let version = self.version.clone(); @@ -381,9 +385,10 @@ impl PublishGrantCommand { } client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { - self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? + self.common + .signing_key(&RegistryUrl::new(nm.to_str()?)?, &config)? } else { - self.common.signing_key(client.url())? + self.common.signing_key(client.url(), &config)? }; match enqueue(&client, &self.name, |_| async { @@ -470,9 +475,10 @@ impl PublishRevokeCommand { } client.refresh_namespace(self.name.namespace()).await?; let signing_key = if let Some(nm) = client.get_warg_header() { - self.common.signing_key(&RegistryUrl::new(nm.to_str()?)?)? + self.common + .signing_key(&RegistryUrl::new(nm.to_str()?)?, &config)? } else { - self.common.signing_key(client.url())? + self.common.signing_key(client.url(), &config)? }; match enqueue(&client, &self.name, |_| async { @@ -672,7 +678,7 @@ impl PublishSubmitCommand { name = info.name ); - let signing_key = self.common.signing_key(client.url())?; + let signing_key = self.common.signing_key(client.url(), &config)?; let record_id = client.publish_with_info(&signing_key, info.clone()).await?; client.registry().store_publish(None).await?; diff --git a/src/keyring.rs b/src/keyring.rs index 51e0da2a..5f65b950 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -1,9 +1,11 @@ //! Utilities for interacting with keyring and performing signing operations. +use std::collections::HashMap; + use anyhow::{bail, Context, Result}; use keyring::Entry; use secrecy::{self, Secret}; -use warg_client::RegistryUrl; +use warg_client::{Config, CredList, RegistryUrl}; use warg_crypto::signing::PrivateKey; /// Gets the auth token entry for the given registry and key name. @@ -47,14 +49,56 @@ pub fn set_auth_token(registry_url: &RegistryUrl, token_name: &str, token: &str) } /// Gets the signing key entry for the given registry and key name. -pub fn get_signing_key_entry(registry_url: &RegistryUrl, key_name: &str) -> Result { - let label = format!("warg-signing-key:{}", registry_url.safe_label()); - Entry::new(&label, key_name).context("failed to get keyring entry") +pub fn get_signing_key_entry( + registry_url: RegistryUrl, + key_name: &str, + config: &Config, +) -> Result { + dbg!(®istry_url, key_name); + if let Some(creds) = config.creds { + if let Some(reg_keys) = creds.keys.get(®istry_url.safe_label()) { + if let Some(key_name) = reg_keys.get(key_name) { + Entry::new( + &format!("warg-signing-key:{}", registry_url.safe_label()), + key_name, + ) + .context("failed to get keyring entry") + } else { + Entry::new( + &format!("warg-signing-key:{}", registry_url.safe_label()), + "default", + ) + .context("failed to get keyring entry") + } + } else if let Some(default_keys) = creds.keys.get(®istry_url.safe_label()) { + if let Some(key_name) = default_keys.get(key_name) { + Entry::new( + &format!("warg-signing-key:{}", registry_url.safe_label()), + key_name, + ) + .context("failed to get keyring entry") + } else { + Entry::new( + &format!("warg-signing-key:{}", registry_url.safe_label()), + "default", + ) + .context("failed to get keyring entry") + } + } else { + Entry::new("warg-signing-key:default", key_name).context("failed to get keyring entry") + } + } else { + Entry::new("warg-signing-key:default", key_name).context("failed to get keyring entry") + } } /// Gets the signing key for the given registry registry_label and key name. -pub fn get_signing_key(registry_url: &RegistryUrl, key_name: &str) -> Result { - let entry = get_signing_key_entry(registry_url, key_name)?; +pub fn get_signing_key( + registry_url: RegistryUrl, + key_name: &str, + config: &Config, +) -> Result { + let entry = get_signing_key_entry(registry_url, key_name, config)?; match entry.get_password() { Ok(secret) => PrivateKey::decode(secret).context("failed to parse signing key"), @@ -71,35 +115,122 @@ pub fn get_signing_key(registry_url: &RegistryUrl, key_name: &str) -> Result Result<()> { - let entry = get_signing_key_entry(registry_url, key_name)?; +pub fn set_signing_key( + registry_url: &Option, + key_name: &str, + key: &PrivateKey, + config: &mut Config, +) -> Result<()> { + match (registry_url, &mut config.creds) { + (None, None) => { + let mut registries = HashMap::new(); + let mut keys = HashMap::new(); + + keys.insert("default".to_string(), key_name.to_string()); + registries.insert("default".to_string(), keys); + config.creds = Some(CredList { + keys: registries, + tokens: vec![], + }); + config.write_to_file(&Config::default_config_path()?)?; + } + (None, Some(creds)) => { + if let Some(default_keys) = creds.clone().keys.get_mut("default") { + default_keys.insert(key_name.to_string(), key_name.to_string()); + creds + .keys + .insert("default".to_string(), default_keys.clone()); + } else { + let mut default_keys = HashMap::new(); + default_keys.insert(key_name.to_string(), key_name.to_string()); + creds + .keys + .insert("default".to_string(), default_keys.clone()); + }; + config.write_to_file(&Config::default_config_path()?)?; + } + (Some(reg), None) => { + let mut registries = HashMap::new(); + let mut keys = HashMap::new(); + + keys.insert(reg.safe_label().to_string(), key_name.to_string()); + registries.insert(reg.safe_label().to_string(), keys); + config.creds = Some(CredList { + keys: registries, + tokens: vec![], + }); + config.write_to_file(&Config::default_config_path()?)?; + } + (Some(reg), Some(creds)) => { + if let Some(reg_keys) = creds.clone().keys.get_mut(®.safe_label()) { + reg_keys.insert(key_name.to_string(), key_name.to_string()); + creds + .keys + .insert(reg.safe_label().to_string(), reg_keys.clone()); + } else { + let mut reg_keys = HashMap::new(); + reg_keys.insert(key_name.to_string(), key_name.to_string()); + creds.keys.insert(reg.safe_label().to_string(), reg_keys); + }; + config.write_to_file(&Config::default_config_path()?)?; + } + } + let entry = get_signing_key_entry(registry_url, key_name, config)?; match entry.set_password(&key.encode()) { Ok(()) => Ok(()), Err(keyring::Error::NoEntry) => { - bail!("no signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(reg) = registry_url { + bail!("no signing key found with name `{key_name}` of registry `{reg}`"); + } else { + bail!("no signing key found with name `{key_name}`"); + } } Err(keyring::Error::Ambiguous(_)) => { - bail!("more than one signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(reg) = registry_url { + bail!("more than one signing key found with name `{key_name}` of registry `{reg}`"); + } else { + bail!("more than one signing key found with name `{key_name}`"); + } } Err(e) => { - bail!("failed to set signing key with name `{key_name}` of registry `{registry_url}`: {e}"); + if let Some(reg) = registry_url { + bail!("failed to get signing key with name `{key_name}` of registry `{reg}`: {e}"); + } else { + bail!("failed to get signing key with name `{key_name}`: {e}"); + } } } } /// Deletes the signing key for the given registry host and key name. -pub fn delete_signing_key(registry_url: &RegistryUrl, key_name: &str) -> Result<()> { - let entry = get_signing_key_entry(registry_url, key_name)?; +pub fn delete_signing_key( + registry_url: &Option, + key_name: &str, + config: &Config, +) -> Result<()> { + let entry = get_signing_key_entry(registry_url, key_name, config)?; match entry.delete_password() { Ok(()) => Ok(()), Err(keyring::Error::NoEntry) => { - bail!("no signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(reg) = registry_url { + bail!("no signing key found with name `{key_name}` of registry `{reg}`"); + } else { + bail!("no signing key found with name `{key_name}`"); + } } Err(keyring::Error::Ambiguous(_)) => { - bail!("more than one signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(reg) = registry_url { + bail!("more than one signing key found with name `{key_name}` of registry `{reg}`"); + } else { + bail!("more than one signing key found with name `{key_name}`"); + } } Err(e) => { - bail!("failed to set signing key with name `{key_name}` of registry `{registry_url}`: {e}"); + if let Some(reg) = registry_url { + bail!("failed to get signing key with name `{key_name}` of registry `{reg}`: {e}"); + } else { + bail!("failed to get signing key with name `{key_name}`: {e}"); + } } } } From 90ef09705cdd35858966e38027ae80d680356380 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Thu, 22 Feb 2024 11:14:00 -0600 Subject: [PATCH 8/9] throw --- Cargo.lock | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 96bcc1c0..b09e8da1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2774,6 +2774,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.2.3+3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.99" @@ -2782,6 +2791,7 @@ checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] From d723e218a8cca78219eb17b420bf7b6e8849057e Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Thu, 22 Feb 2024 21:47:02 -0600 Subject: [PATCH 9/9] refactor key names and wrap up default --- config.json | 21 ---- crates/client/src/api.rs | 8 +- crates/client/src/config.rs | 15 +-- crates/client/src/lib.rs | 7 +- crates/client/src/storage/fs.rs | 1 - new | 6 - src/commands.rs | 45 +++----- src/commands/config.rs | 5 +- src/commands/info.rs | 10 +- src/commands/key.rs | 159 ++++++++++---------------- src/commands/publish.rs | 2 +- src/keyring.rs | 190 ++++++++++++-------------------- tests/support/mod.rs | 1 + 13 files changed, 157 insertions(+), 313 deletions(-) delete mode 100644 config.json delete mode 100644 new diff --git a/config.json b/config.json deleted file mode 100644 index 1a2f0736..00000000 --- a/config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "homeUrl": "https://dannyjaf.preview.wa.dev/", - "creds": { - "keys": { - "wasi.preview.wa.dev": { - "fooey": "fooey", - "default": "default" - }, - "default": { - "fooey": "fooey", - "default": "default", - "dannyjaf.preview.wa.dev": "dannyjaf.preview.wa.dev" - }, - "dannyjaf.preview.wa.dev": { - "dannyjaf.preview.wa.dev": "dannyjaf.preview.wa.dev", - "default": "default" - } - }, - "tokens": [] - } -} \ No newline at end of file diff --git a/crates/client/src/api.rs b/crates/client/src/api.rs index 263054fe..a1c86d27 100644 --- a/crates/client/src/api.rs +++ b/crates/client/src/api.rs @@ -418,13 +418,7 @@ impl Client { tracing::debug!("downloading content `{digest}` from `{url}`"); - let response = self - .client - .get(url) - .warg_header(self.get_warg_header()) - .auth(self.auth_token()) - .send() - .await?; + let response = self.client.get(url).send().await?; if !response.status().is_success() { tracing::debug!( "failed to download content `{digest}` from `{url}`: {status}", diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 08d56970..55116ed8 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -6,7 +6,7 @@ use normpath::PathExt; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::HashSet, env::current_dir, fs::{self, File}, path::{Component, Path, PathBuf}, @@ -72,15 +72,6 @@ pub struct StoragePaths { pub namespace_map_path: PathBuf, } -/// List of creds available in keyring -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -pub struct CredList { - /// Keys - pub keys: HashMap>, - /// Access tokens - pub tokens: Vec, -} - /// Represents the Warg client configuration. #[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -118,7 +109,7 @@ pub struct Config { /// List of creds availabe in keyring #[serde(default, skip_serializing_if = "Option::is_none")] - pub creds: Option, + pub keys: Option>, } impl Config { @@ -195,7 +186,7 @@ impl Config { assert!(p.is_absolute()); pathdiff::diff_paths(&p, &parent).unwrap() }), - creds: self.creds.clone(), + keys: self.keys.clone(), }; serde_json::to_writer_pretty( diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 643324d2..46930a4d 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -13,7 +13,7 @@ use std::str::FromStr; use std::{borrow::Cow, collections::HashMap, path::PathBuf, time::Duration}; use storage::{ ContentStorage, FileSystemContentStorage, FileSystemNamespaceMapStorage, - FileSystemRegistryStorage, NamespaceMapStorage, PublishInfo, RegistryStorage, + FileSystemRegistryStorage, NamespaceMapStorage, PublishInfo, RegistryDomain, RegistryStorage, }; use thiserror::Error; use warg_api::v1::{ @@ -80,11 +80,6 @@ impl Client &Option> { - self.api.auth_token() - } - /// Gets the URL of the client. pub fn url(&self) -> &RegistryUrl { self.api.url() diff --git a/crates/client/src/storage/fs.rs b/crates/client/src/storage/fs.rs index c0c3db7e..6444a0ce 100644 --- a/crates/client/src/storage/fs.rs +++ b/crates/client/src/storage/fs.rs @@ -9,7 +9,6 @@ use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; use bytes::Bytes; use futures_util::{Stream, StreamExt, TryStreamExt}; -use reqwest::header::HeaderValue; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, diff --git a/new b/new deleted file mode 100644 index 67c63f3d..00000000 --- a/new +++ /dev/null @@ -1,6 +0,0 @@ -{ - "creds": { - "keys": {}, - "tokens": [] - } -} \ No newline at end of file diff --git a/src/commands.rs b/src/commands.rs index b5d63b66..1b1b561d 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -1,6 +1,5 @@ //! Commands for the `warg` tool. -use anyhow::Context; use anyhow::Result; use clap::Args; use secrecy::Secret; @@ -52,15 +51,6 @@ pub struct CommonOptions { /// The URL of the registry to use. #[clap(long, value_name = "URL")] pub registry: Option, - /// The path to the auth token file. - #[clap(long, value_name = "TOKEN_FILE", env = "WARG_AUTH_TOKEN_FILE")] - pub token_file: Option, - /// The name to use for the signing key. - #[clap(long, short, value_name = "KEY_NAME", default_value = "default")] - pub key_name: String, - /// The path to the signing key file. - #[clap(long, value_name = "KEY_FILE", env = "WARG_SIGNING_KEY_FILE")] - pub key_file: Option, /// The path to the client configuration file to use. /// /// If not specified, the following locations are searched in order: `./warg-config.json`, `/warg/config.json`. @@ -121,34 +111,25 @@ impl CommonOptions { client: &Client, ) -> Result { let registry_url = if let Some(nm) = &client.get_warg_registry() { - RegistryUrl::new(nm.to_string())? + Some(RegistryUrl::new(nm.to_string())?) } else { - client.url().clone() + None }; - if let Some(file) = &self.key_file { - let key_str = std::fs::read_to_string(file) - .with_context(|| format!("failed to read key from {file:?}"))? - .trim_end() - .to_string(); - PrivateKey::decode(key_str) - .with_context(|| format!("failed to parse key from {file:?}")) - } else { - get_signing_key(®istry_url, &self.key_name) - } + let config = self.read_config()?; + get_signing_key( + ®istry_url.map(|reg| reg.safe_label()), + config.keys.expect("Please set a default signing key by typing `warg key set ` or `warg key new"), + config.home_url, + ) } /// Gets the auth token for the given registry URL. pub fn auth_token(&self, config: &Config) -> Result>> { - if let Some(file) = &self.token_file { - Ok(Some(Secret::from( - std::fs::read_to_string(file) - .with_context(|| format!("failed to read auth token from {file:?}"))? - .trim_end() - .to_string(), - ))) + if let Some(reg_url) = &self.registry { + Ok(get_auth_token(&RegistryUrl::new(reg_url)?)?) + } else if let Some(url) = config.home_url.as_ref() { + Ok(get_auth_token(&RegistryUrl::new(url)?)?) } else { - Ok(get_auth_token(&RegistryUrl::new( - config.home_url.as_ref().unwrap(), - )?)?) + Ok(None) } } } diff --git a/src/commands/config.rs b/src/commands/config.rs index 0a79cdcc..e67be5be 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -11,9 +11,6 @@ pub struct ConfigCommand { #[clap(flatten)] pub common: CommonOptions, - /// The home registry URL to use. - #[clap(long, value_name = "URL")] - pub registry_url: Option, /// The path to the registries directory to use. #[clap(long, value_name = "REGISTRIES")] pub registries_dir: Option, @@ -71,7 +68,7 @@ impl ConfigCommand { registries_dir: self.registries_dir.map(|p| cwd.join(p)), content_dir: self.content_dir.map(|p| cwd.join(p)), namespace_map_path: self.namespace_path.map(|p| cwd.join(p)), - creds: self.common.read_config()?.creds, + keys: self.common.read_config()?.keys, }; config.write_to_file(&path)?; diff --git a/src/commands/info.rs b/src/commands/info.rs index d51c1a24..2a4921a9 100644 --- a/src/commands/info.rs +++ b/src/commands/info.rs @@ -43,10 +43,18 @@ impl InfoCommand { Self::print_package_info(&info); } } + None => { + client + .registry() + .load_packages() + .await? + .iter() + .for_each(Self::print_package_info); + } } - Self::print_namespace_map(&client).await?; if self.namespaces { + println!("\nnamespace mappings in client storage"); Self::print_namespace_map(&client).await?; return Ok(()); } diff --git a/src/commands/key.rs b/src/commands/key.rs index cedca01c..990d13c5 100644 --- a/src/commands/key.rs +++ b/src/commands/key.rs @@ -1,11 +1,11 @@ -use crate::keyring::{delete_signing_key, get_signing_key, get_signing_key_entry, set_signing_key}; +use crate::keyring::{delete_signing_key, get_signing_key, set_signing_key}; use anyhow::{bail, Context, Result}; use clap::{Args, Subcommand}; use dialoguer::{theme::ColorfulTheme, Confirm, Password}; -use keyring::{Entry, Error as KeyringError}; use p256::ecdsa::SigningKey; use rand_core::OsRng; -use warg_client::{Config, RegistryUrl}; +use std::collections::HashSet; +use warg_client::Config; use warg_crypto::signing::PrivateKey; use super::CommonOptions; @@ -43,49 +43,9 @@ pub enum KeySubcommand { Delete(KeyDeleteCommand), } -#[derive(Args)] -struct KeyringEntryArgs { - /// The name to use for the signing key. - #[clap(long, short, value_name = "KEY_NAME", default_value = "default")] - pub name: String, - /// The URL of the registry to create a signing key for. - #[clap(value_name = "URL")] - pub url: Option, -} - -impl KeyringEntryArgs { - fn get_entry(&self, config: &Config) -> Result { - get_signing_key_entry(&self.url, &self.name, config) - } - - fn get_key(&self, config: &Config) -> Result { - get_signing_key(&self.url, &self.name, config) - } - - fn set_entry(&self, key: &PrivateKey, config: &mut Config) -> Result<()> { - set_signing_key(&self.url, &self.name, key, config) - } - - fn delete_entry(&self, config: &Config) -> Result<()> { - delete_signing_key(&self.url, &self.name, config) - } -} - -impl std::fmt::Display for KeyringEntryArgs { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(url) = &self.url { - write!(f, "`{name}` for registry `{url}`", name = self.name,) - } else { - write!(f, "{name}", name = self.name) - } - } -} - /// Creates a new signing key for a registry in the local keyring. #[derive(Args)] pub struct KeyNewCommand { - #[clap(flatten)] - keyring_entry: KeyringEntryArgs, /// The common command options. #[clap(flatten)] pub common: CommonOptions, @@ -95,37 +55,25 @@ impl KeyNewCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = &mut self.common.read_config()?; - - let entry = self.keyring_entry.get_entry(config)?; - - match entry.get_password() { - Err(KeyringError::NoEntry) => { - // no entry exists, so we can continue - } - Ok(_) | Err(KeyringError::Ambiguous(_)) => { - if let Some(url) = self.keyring_entry.url { - bail!( - "a signing key `{name}` already exists for registry `{url}`", - name = self.keyring_entry.name, - ); - } else { - bail!( - "a signing key `{name}` already exists", - name = self.keyring_entry.name, - ); - } - } - Err(e) => { - bail!( - "failed to get signing key {entry}: {e}", - entry = self.keyring_entry - ); - } - } - let key = SigningKey::random(&mut OsRng).into(); - self.keyring_entry.set_entry(&key, config)?; - + if config.keys.is_some() { + if let Some(ref reg) = self.common.registry { + config.keys.as_mut().unwrap().insert(reg.to_string()); + } else { + config.keys.as_mut().unwrap().insert("default".to_string()); + } + } else { + let mut keys = HashSet::new(); + keys.insert("default".to_string()); + config.keys = Some(keys); + }; + set_signing_key( + &self.common.registry.clone(), + &key, + config.keys.as_mut().unwrap(), + config.home_url.clone(), + )?; + config.write_to_file(&Config::default_config_path()?)?; Ok(()) } } @@ -133,8 +81,6 @@ impl KeyNewCommand { /// Shows information about the signing key for a registry in the local keyring. #[derive(Args)] pub struct KeyInfoCommand { - #[clap(flatten)] - keyring_entry: KeyringEntryArgs, /// The common command options. #[clap(flatten)] pub common: CommonOptions, @@ -144,19 +90,22 @@ impl KeyInfoCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { let config = &self.common.read_config()?; - let private_key = self.keyring_entry.get_key(config)?; - let public_key = private_key.public_key(); - println!("Key ID: {}", public_key.fingerprint()); - println!("Public Key: {public_key}"); - Ok(()) + if let Some(keys) = &config.keys { + let private_key = + get_signing_key(&self.common.registry, keys.clone(), config.home_url.clone())?; + let public_key = private_key.public_key(); + println!("Key ID: {}", public_key.fingerprint()); + println!("Public Key: {public_key}"); + Ok(()) + } else { + bail!("error: Please set a default signing key by typing `warg key set ` or `warg key new`") + } } } /// Sets the signing key for a registry in the local keyring. #[derive(Args)] pub struct KeySetCommand { - #[clap(flatten)] - keyring_entry: KeyringEntryArgs, /// The common command options. #[clap(flatten)] pub common: CommonOptions, @@ -171,13 +120,20 @@ impl KeySetCommand { .context("failed to read signing key")?; let key = PrivateKey::decode(key_str).context("signing key is not in the correct format")?; + let config = &mut self.common.read_config()?; - self.keyring_entry.set_entry(&key, config)?; + if config.keys.is_none() { + config.keys = Some(HashSet::new()); + } + set_signing_key( + &self.common.registry.clone(), + &key, + config.keys.as_mut().unwrap(), + config.home_url.clone(), + )?; + config.write_to_file(&Config::default_config_path()?)?; - println!( - "signing key {keyring} was set successfully", - keyring = self.keyring_entry - ); + println!("signing key was set successfully"); Ok(()) } @@ -186,8 +142,6 @@ impl KeySetCommand { /// Deletes the signing key for a registry from the local keyring. #[derive(Args)] pub struct KeyDeleteCommand { - #[clap(flatten)] - keyring_entry: KeyringEntryArgs, /// The common command options. #[clap(flatten)] pub common: CommonOptions, @@ -196,22 +150,25 @@ pub struct KeyDeleteCommand { impl KeyDeleteCommand { /// Executes the command. pub async fn exec(self) -> Result<()> { - let config = &self.common.read_config()?; - let prompt = format!( - "are you sure you want to delete the signing key {entry}? ", - entry = self.keyring_entry - ); + let config = &mut self.common.read_config()?; if Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(prompt) + .with_prompt("are you sure you want to delete your signing key") .interact()? { - self.keyring_entry.delete_entry(&config)?; - println!( - "signing key {entry} was deleted successfully", - entry = self.keyring_entry - ); - } else if let Some(url) = self.keyring_entry.url { + // delete_signing_key(&self.common.registry, config)?; + delete_signing_key(&self.common.registry, config.keys.clone().expect("Please set a default signing key by typing `warg key set ` or `warg key new"), config.home_url.clone())?; + let keys = &mut config.keys; + if let Some(keys) = keys { + if let Some(registry_url) = self.common.registry { + keys.remove(®istry_url); + } else { + keys.remove("default"); + } + } + config.write_to_file(&Config::default_config_path()?)?; + println!("signing key was deleted successfully",); + } else if let Some(url) = self.common.registry { println!( "skipping deletion of signing key for registry `{url}`", url = url diff --git a/src/commands/publish.rs b/src/commands/publish.rs index 6fb5e654..553adf15 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -8,7 +8,7 @@ use tokio::io::BufReader; use tokio_util::io::ReaderStream; use warg_client::{ storage::{ContentStorage as _, PublishEntry, PublishInfo, RegistryStorage as _}, - FileSystemClient, RegistryUrl, + FileSystemClient, }; use warg_crypto::{ hash::AnyHash, diff --git a/src/keyring.rs b/src/keyring.rs index 5c36411a..425c967b 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -1,6 +1,6 @@ //! Utilities for interacting with keyring and performing signing operations. -use std::collections::HashMap; +use std::collections::HashSet; use anyhow::{bail, Context, Result}; use keyring::Entry; @@ -65,153 +65,96 @@ pub fn set_auth_token(registry_url: &RegistryUrl, token: &str) -> Result<()> { /// Gets the signing key entry for the given registry and key name. pub fn get_signing_key_entry( - registry_url: RegistryUrl, - key_name: &str, - config: &Config, + registry_url: &Option, + keys: HashSet, + home_url: Option, ) -> Result { - dbg!(®istry_url, key_name); - if let Some(creds) = config.creds { - if let Some(reg_keys) = creds.keys.get(®istry_url.safe_label()) { - if let Some(key_name) = reg_keys.get(key_name) { - Entry::new( - &format!("warg-signing-key:{}", registry_url.safe_label()), - key_name, - ) - .context("failed to get keyring entry") - } else { - Entry::new( - &format!("warg-signing-key:{}", registry_url.safe_label()), - "default", - ) - .context("failed to get keyring entry") - } - } else if let Some(default_keys) = creds.keys.get(®istry_url.safe_label()) { - if let Some(key_name) = default_keys.get(key_name) { - Entry::new( - &format!("warg-signing-key:{}", registry_url.safe_label()), - key_name, - ) - .context("failed to get keyring entry") - } else { - Entry::new( - &format!("warg-signing-key:{}", registry_url.safe_label()), - "default", - ) - .context("failed to get keyring entry") - } + if let Some(registry_url) = registry_url { + if keys.contains(®istry_url.clone()) { + Entry::new("warg-signing-key", registry_url).context("failed to get keyring entry") } else { - Entry::new("warg-signing-key:default", key_name).context("failed to get keyring entry") + Entry::new("warg-signing-key", "default").context("failed to get keyring entry") } } else { - Entry::new("warg-signing-key:default", key_name).context("failed to get keyring entry") + if let Some(url) = &home_url { + return Entry::new("warg-signing-key", &RegistryUrl::new(url)?.safe_label()) + .context("failed to get keyring entry"); + } + if keys.contains("default") { + Entry::new("warg-signing-key", "default").context("failed to get keyring entry") + } else { + bail!( + "error: Please set a default signing key by typing `warg key set ` or `warg key new`" + ) + } } } /// Gets the signing key for the given registry registry_label and key name. pub fn get_signing_key( - registry_url: RegistryUrl, - key_name: &str, - config: &Config, + // If being called by a cli key command, this will always be a cli flag + // If being called by a client publish command, this could also be supplied by namespace map config + registry_url: &Option, + keys: HashSet, + home_url: Option, ) -> Result { - let entry = get_signing_key_entry(registry_url, key_name, config)?; + let entry = get_signing_key_entry(registry_url, keys, home_url)?; match entry.get_password() { Ok(secret) => PrivateKey::decode(secret).context("failed to parse signing key"), Err(keyring::Error::NoEntry) => { - bail!("no signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(registry_url) = registry_url { + bail!("no signing key found for registry `{registry_url}`"); + } else { + bail!("no signing key found"); + } } Err(keyring::Error::Ambiguous(_)) => { - bail!("more than one signing key found with name `{key_name}` of registry `{registry_url}`"); + if let Some(registry_url) = registry_url { + bail!("more than one signing key found for registry `{registry_url}`"); + } else { + bail!("more than one signing key found"); + } } Err(e) => { - bail!("failed to get signing key with name `{key_name}` of registry `{registry_url}`: {e}"); + if let Some(registry_url) = registry_url { + bail!("failed to get signing key for registry `{registry_url}`: {e}"); + } else { + bail!("failed to get signing key`"); + } } } } /// Sets the signing key for the given registry host and key name. pub fn set_signing_key( - registry_url: &Option, - key_name: &str, + registry_url: &Option, key: &PrivateKey, - config: &mut Config, + keys: &mut HashSet, + home_url: Option, ) -> Result<()> { - match (registry_url, &mut config.creds) { - (None, None) => { - let mut registries = HashMap::new(); - let mut keys = HashMap::new(); - - keys.insert("default".to_string(), key_name.to_string()); - registries.insert("default".to_string(), keys); - config.creds = Some(CredList { - keys: registries, - tokens: vec![], - }); - config.write_to_file(&Config::default_config_path()?)?; - } - (None, Some(creds)) => { - if let Some(default_keys) = creds.clone().keys.get_mut("default") { - default_keys.insert(key_name.to_string(), key_name.to_string()); - creds - .keys - .insert("default".to_string(), default_keys.clone()); - } else { - let mut default_keys = HashMap::new(); - default_keys.insert(key_name.to_string(), key_name.to_string()); - creds - .keys - .insert("default".to_string(), default_keys.clone()); - }; - config.write_to_file(&Config::default_config_path()?)?; - } - (Some(reg), None) => { - let mut registries = HashMap::new(); - let mut keys = HashMap::new(); - - keys.insert(reg.safe_label().to_string(), key_name.to_string()); - registries.insert(reg.safe_label().to_string(), keys); - config.creds = Some(CredList { - keys: registries, - tokens: vec![], - }); - config.write_to_file(&Config::default_config_path()?)?; - } - (Some(reg), Some(creds)) => { - if let Some(reg_keys) = creds.clone().keys.get_mut(®.safe_label()) { - reg_keys.insert(key_name.to_string(), key_name.to_string()); - creds - .keys - .insert(reg.safe_label().to_string(), reg_keys.clone()); - } else { - let mut reg_keys = HashMap::new(); - reg_keys.insert(key_name.to_string(), key_name.to_string()); - creds.keys.insert(reg.safe_label().to_string(), reg_keys); - }; - config.write_to_file(&Config::default_config_path()?)?; - } - } - let entry = get_signing_key_entry(registry_url, key_name, config)?; + let entry = get_signing_key_entry(registry_url, keys.clone(), home_url.clone())?; match entry.set_password(&key.encode()) { Ok(()) => Ok(()), Err(keyring::Error::NoEntry) => { - if let Some(reg) = registry_url { - bail!("no signing key found with name `{key_name}` of registry `{reg}`"); + if let Some(registry_url) = registry_url { + bail!("no signing key found for registry `{registry_url}`"); } else { - bail!("no signing key found with name `{key_name}`"); + bail!("no signing key found`"); } } Err(keyring::Error::Ambiguous(_)) => { - if let Some(reg) = registry_url { - bail!("more than one signing key found with name `{key_name}` of registry `{reg}`"); + if let Some(registry_url) = registry_url { + bail!("more than one signing key found for registry `{registry_url}`"); } else { - bail!("more than one signing key found with name `{key_name}`"); + bail!("more than one signing key found"); } } Err(e) => { - if let Some(reg) = registry_url { - bail!("failed to get signing key with name `{key_name}` of registry `{reg}`: {e}"); + if let Some(registry_url) = registry_url { + bail!("failed to get signing key for registry `{registry_url}`: {e}"); } else { - bail!("failed to get signing key with name `{key_name}`: {e}"); + bail!("failed to get signing: {e}"); } } } @@ -219,29 +162,34 @@ pub fn set_signing_key( /// Deletes the signing key for the given registry host and key name. pub fn delete_signing_key( - registry_url: &Option, - key_name: &str, - config: &Config, + registry_url: &Option, + keys: HashSet, + home_url: Option, ) -> Result<()> { - let entry = get_signing_key_entry(registry_url, key_name, config)?; + let entry = get_signing_key_entry(registry_url, keys, home_url.clone())?; + match entry.delete_password() { Ok(()) => Ok(()), Err(keyring::Error::NoEntry) => { - if let Some(reg) = registry_url { - bail!("no signing key found with name `{key_name}` of registry `{reg}`"); + if let Some(registry_url) = registry_url { + bail!("no signing key found for registry `{registry_url}`"); } else { - bail!("no signing key found with name `{key_name}`"); + bail!("no signing key found"); } } Err(keyring::Error::Ambiguous(_)) => { - if let Some(reg) = registry_url { - bail!("more than one signing key found with name `{key_name}` of registry `{reg}`"); + if let Some(registry_url) = registry_url { + bail!("more than one signing key found for registry `{registry_url}`"); } else { - bail!("more than one signing key found with name `{key_name}`"); + bail!("more than one signing key found`"); } } Err(e) => { - bail!("failed to delete signing key with name `{key_name}` of registry `{registry_url}`: {e}"); + if let Some(registry_url) = registry_url { + bail!("failed to delete signing key for registry `{registry_url}`: {e}"); + } else { + bail!("failed to delete signing key"); + } } } } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 9307b4ac..838b5957 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -164,6 +164,7 @@ pub async fn spawn_server( registries_dir: Some(root.join("registries")), content_dir: Some(root.join("content")), namespace_map_path: Some(root.join("namespaces")), + keys: None, }; Ok((instance, config))