From 3db95d9f33c4411edcceef13b2bd18ee6fd59fb3 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 23 May 2025 21:19:02 +0800 Subject: [PATCH] demo: an easy to use catalog loader Signed-off-by: Xuanwo --- Cargo.lock | 38 +++++++----------------------- crates/catalog/rest/src/catalog.rs | 21 +++++++++++++++-- crates/iceberg/src/catalog/mod.rs | 15 +++++++++++- crates/iceberg/src/lib.rs | 5 ++-- 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a2987a89..8ec4146bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -637,7 +637,7 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -699,7 +699,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -723,7 +723,7 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -745,7 +745,7 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -767,7 +767,7 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -790,7 +790,7 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -813,7 +813,7 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", @@ -835,7 +835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" dependencies = [ "aws-credential-types", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -861,26 +861,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "aws-smithy-http" -version = "0.60.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - [[package]] name = "aws-smithy-http" version = "0.62.1" @@ -964,7 +944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.62.1", + "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 851819069..203a56a9a 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -19,13 +19,14 @@ use std::collections::HashMap; use std::str::FromStr; +use std::sync::Arc; use async_trait::async_trait; use iceberg::io::FileIO; use iceberg::table::Table; use iceberg::{ - Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, TableCreation, - TableIdent, + Catalog, CatalogLoader, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, + TableCreation, TableIdent, }; use itertools::Itertools; use reqwest::header::{ @@ -320,6 +321,22 @@ impl RestCatalog { } } +#[async_trait] +impl CatalogLoader for RestCatalog { + async fn load(mut properties: HashMap) -> Result> { + let uri = properties + .remove("uri") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "Missing required property `uri`"))?; + + let config = RestCatalogConfig::builder() + .uri(uri) + .props(properties) + .build(); + + Ok(Arc::new(RestCatalog::new(config))) + } +} + /// All requests and expected responses are derived from the REST catalog API spec: /// https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml #[async_trait] diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index 3457f8361..01dcfa710 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -21,6 +21,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::mem::take; use std::ops::Deref; +use std::sync::Arc; use _serde::deserialize_snapshot; use async_trait::async_trait; @@ -36,7 +37,19 @@ use crate::spec::{ use crate::table::Table; use crate::{Error, ErrorKind, Result}; +/// The CatalogLoader trait is used to load a catalog from a given name and properties. +#[async_trait] +pub trait CatalogLoader: Debug + Send + Sync { + /// Load a catalog from the given name and properties. + async fn load(properties: HashMap) -> Result>; +} + /// The catalog API for Iceberg Rust. +/// +/// Users will have two ways to construct a catalog: +/// +/// - Use `CatalogLoeader` to load a catalog from a name and properties. +/// - Use `CatalogBuilder` provided by the catalog implementer to build a catalog in a strong typed way. #[async_trait] pub trait Catalog: Debug + Sync + Send { /// List namespaces inside the catalog. @@ -2091,7 +2104,7 @@ mod tests { { "action": "remove-schemas", "schema-ids": [1, 2] - } + } "#, TableUpdate::RemoveSchemas { schema_ids: vec![1, 2], diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 556ff3e02..d2d9850af 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -61,10 +61,9 @@ mod error; pub use error::{Error, ErrorKind, Result}; mod catalog; - pub use catalog::{ - Catalog, Namespace, NamespaceIdent, TableCommit, TableCreation, TableIdent, TableRequirement, - TableUpdate, ViewCreation, + Catalog, CatalogLoader, Namespace, NamespaceIdent, TableCommit, TableCreation, TableIdent, + TableRequirement, TableUpdate, ViewCreation, }; pub mod table;