Skip to content

Commit 49dcf84

Browse files
authored
Merge branch 'main' into fix/schema-evolution-name-conflict
2 parents 07c6883 + 3a21ce9 commit 49dcf84

File tree

8 files changed

+1523
-919
lines changed

8 files changed

+1523
-919
lines changed

Cargo.lock

Lines changed: 975 additions & 875 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ async-std = "1.12"
5555
async-trait = "0.1.88"
5656
aws-config = "1.6.1"
5757
aws-sdk-glue = "1.39"
58+
aws-sdk-s3tables = "1.28.0"
5859
backon = "1.5.1"
5960
base64 = "0.22.1"
6061
bimap = "0.6"
@@ -73,11 +74,12 @@ faststr = "0.2.31"
7374
fnv = "1.0.7"
7475
fs-err = "3.1.0"
7576
futures = "0.3"
76-
hive_metastore = "0.1"
77+
hive_metastore = "0.2.0"
7778
http = "1.2"
7879
iceberg = { version = "0.6.0", path = "./crates/iceberg" }
7980
iceberg-catalog-rest = { version = "0.6.0", path = "./crates/catalog/rest" }
8081
iceberg-catalog-glue = { version = "0.6.0", path = "./crates/catalog/glue" }
82+
iceberg-catalog-s3tables = { version = "0.6.0", path = "./crates/catalog/s3tables" }
8183
iceberg-datafusion = { version = "0.6.0", path = "./crates/integrations/datafusion" }
8284
indicatif = "0.17"
8385
itertools = "0.13"
@@ -93,7 +95,7 @@ once_cell = "1.20"
9395
opendal = "0.54.0"
9496
ordered-float = "4"
9597
parquet = "55"
96-
pilota = "0.11.2"
98+
pilota = "0.11.10"
9799
port_scanner = "0.1.5"
98100
pretty_assertions = "1.4"
99101
rand = "0.8.5"
@@ -121,5 +123,5 @@ typed-builder = "0.20"
121123
url = "2.5.4"
122124
uuid = { version = "1.16", features = ["v7"] }
123125
volo = "0.10.6"
124-
volo-thrift = "0.10.6"
126+
volo-thrift = "0.10.8"
125127
zstd = "0.13.2"

crates/catalog/loader/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,6 @@ repository = { workspace = true }
3232
iceberg = { workspace = true }
3333
iceberg-catalog-rest = { workspace = true }
3434
iceberg-catalog-glue = { workspace = true }
35+
iceberg-catalog-s3tables = { workspace = true }
3536
tokio = { workspace = true }
3637
async-trait = { workspace = true }

crates/catalog/loader/src/lib.rs

Lines changed: 119 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,22 @@ use async_trait::async_trait;
2222
use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result};
2323
use iceberg_catalog_glue::GlueCatalogBuilder;
2424
use iceberg_catalog_rest::RestCatalogBuilder;
25+
use iceberg_catalog_s3tables::S3TablesCatalogBuilder;
26+
27+
/// A CatalogBuilderFactory creating a new catalog builder.
28+
type CatalogBuilderFactory = fn() -> Box<dyn BoxedCatalogBuilder>;
29+
30+
/// A registry of catalog builders.
31+
static CATALOG_REGISTRY: &[(&str, CatalogBuilderFactory)] = &[
32+
("rest", || Box::new(RestCatalogBuilder::default())),
33+
("glue", || Box::new(GlueCatalogBuilder::default())),
34+
("s3tables", || Box::new(S3TablesCatalogBuilder::default())),
35+
];
36+
37+
/// Return the list of supported catalog types.
38+
pub fn supported_types() -> Vec<&'static str> {
39+
CATALOG_REGISTRY.iter().map(|(k, _)| *k).collect()
40+
}
2541

2642
#[async_trait]
2743
pub trait BoxedCatalogBuilder {
@@ -44,22 +60,74 @@ impl<T: CatalogBuilder + 'static> BoxedCatalogBuilder for T {
4460
}
4561
}
4662

63+
/// Load a catalog from a string.
4764
pub fn load(r#type: &str) -> Result<Box<dyn BoxedCatalogBuilder>> {
48-
match r#type {
49-
"rest" => Ok(Box::new(RestCatalogBuilder::default()) as Box<dyn BoxedCatalogBuilder>),
50-
"glue" => Ok(Box::new(GlueCatalogBuilder::default()) as Box<dyn BoxedCatalogBuilder>),
51-
_ => Err(Error::new(
65+
let key = r#type.trim();
66+
if let Some((_, factory)) = CATALOG_REGISTRY
67+
.iter()
68+
.find(|(k, _)| k.eq_ignore_ascii_case(key))
69+
{
70+
Ok(factory())
71+
} else {
72+
Err(Error::new(
5273
ErrorKind::FeatureUnsupported,
53-
format!("Unsupported catalog type: {}", r#type),
54-
)),
74+
format!(
75+
"Unsupported catalog type: {}. Supported types: {}",
76+
r#type,
77+
supported_types().join(", ")
78+
),
79+
))
80+
}
81+
}
82+
83+
/// Ergonomic catalog loader builder pattern.
84+
pub struct CatalogLoader<'a> {
85+
catalog_type: &'a str,
86+
}
87+
88+
impl<'a> From<&'a str> for CatalogLoader<'a> {
89+
fn from(s: &'a str) -> Self {
90+
Self { catalog_type: s }
91+
}
92+
}
93+
94+
impl CatalogLoader<'_> {
95+
pub async fn load(
96+
self,
97+
name: String,
98+
props: HashMap<String, String>,
99+
) -> Result<Arc<dyn Catalog>> {
100+
let builder = load(self.catalog_type)?;
101+
builder.load(name, props).await
55102
}
56103
}
57104

58105
#[cfg(test)]
59106
mod tests {
60107
use std::collections::HashMap;
61108

62-
use crate::load;
109+
use crate::{CatalogLoader, load};
110+
111+
#[tokio::test]
112+
async fn test_load_glue_catalog() {
113+
use iceberg_catalog_glue::GLUE_CATALOG_PROP_WAREHOUSE;
114+
115+
let catalog_loader = load("glue").unwrap();
116+
let catalog = catalog_loader
117+
.load(
118+
"glue".to_string(),
119+
HashMap::from([
120+
(
121+
GLUE_CATALOG_PROP_WAREHOUSE.to_string(),
122+
"s3://test".to_string(),
123+
),
124+
("key".to_string(), "value".to_string()),
125+
]),
126+
)
127+
.await;
128+
129+
assert!(catalog.is_ok());
130+
}
63131

64132
#[tokio::test]
65133
async fn test_load_rest_catalog() {
@@ -83,17 +151,22 @@ mod tests {
83151
}
84152

85153
#[tokio::test]
86-
async fn test_load_glue_catalog() {
87-
use iceberg_catalog_glue::GLUE_CATALOG_PROP_WAREHOUSE;
154+
async fn test_load_unsupported_catalog() {
155+
let result = load("unsupported");
156+
assert!(result.is_err());
157+
}
88158

89-
let catalog_loader = load("glue").unwrap();
90-
let catalog = catalog_loader
159+
#[tokio::test]
160+
async fn test_catalog_loader_pattern() {
161+
use iceberg_catalog_rest::REST_CATALOG_PROP_URI;
162+
163+
let catalog = CatalogLoader::from("rest")
91164
.load(
92-
"glue".to_string(),
165+
"rest".to_string(),
93166
HashMap::from([
94167
(
95-
GLUE_CATALOG_PROP_WAREHOUSE.to_string(),
96-
"s3://test".to_string(),
168+
REST_CATALOG_PROP_URI.to_string(),
169+
"http://localhost:8080".to_string(),
97170
),
98171
("key".to_string(), "value".to_string()),
99172
]),
@@ -102,4 +175,36 @@ mod tests {
102175

103176
assert!(catalog.is_ok());
104177
}
178+
179+
#[tokio::test]
180+
async fn test_catalog_loader_pattern_s3tables() {
181+
use iceberg_catalog_s3tables::S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN;
182+
183+
let catalog = CatalogLoader::from("s3tables")
184+
.load(
185+
"s3tables".to_string(),
186+
HashMap::from([
187+
(
188+
S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(),
189+
"arn:aws:s3tables:us-east-1:123456789012:bucket/test".to_string(),
190+
),
191+
("key".to_string(), "value".to_string()),
192+
]),
193+
)
194+
.await;
195+
196+
assert!(catalog.is_ok());
197+
}
198+
199+
#[tokio::test]
200+
async fn test_error_message_includes_supported_types() {
201+
let err = match load("does-not-exist") {
202+
Ok(_) => panic!("expected error for unsupported type"),
203+
Err(e) => e,
204+
};
205+
let msg = err.message().to_string();
206+
assert!(msg.contains("Supported types:"));
207+
// Should include at least the built-in type
208+
assert!(msg.contains("rest"));
209+
}
105210
}

crates/catalog/rest/src/catalog.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -699,7 +699,7 @@ impl Catalog for RestCatalog {
699699
.config
700700
.unwrap_or_default()
701701
.into_iter()
702-
.chain(self.user_config.props.clone().into_iter())
702+
.chain(self.user_config.props.clone())
703703
.collect();
704704

705705
let file_io = self
@@ -750,7 +750,7 @@ impl Catalog for RestCatalog {
750750
.config
751751
.unwrap_or_default()
752752
.into_iter()
753-
.chain(self.user_config.props.clone().into_iter())
753+
.chain(self.user_config.props.clone())
754754
.collect();
755755

756756
let file_io = self

crates/catalog/s3tables/Cargo.toml

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,11 @@ license = { workspace = true }
2929
repository = { workspace = true }
3030

3131
[dependencies]
32-
anyhow = { workspace = true }
3332
async-trait = { workspace = true }
3433
aws-config = { workspace = true }
35-
aws-sdk-s3tables = "1.10.0"
34+
aws-sdk-s3tables = { workspace = true }
3635
iceberg = { workspace = true }
37-
typed-builder = { workspace = true }
36+
3837

3938
[dev-dependencies]
4039
iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }

0 commit comments

Comments
 (0)