Merge branch 'main' of github.com:apache/iceberg-rust into fd-avro-pyiceberg

Author: Fokko

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg Rust version are you using?
       multiple: false
       options:
-        - 0.4.0 (latest version)
+        - 0.5.1 (latest version)
+        - 0.4.0
         - 0.3.0
         - 0.2.0
     validations:

.github/workflows/ci_typos.yml

@@ -42,4 +42,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.32.0
+        uses: crate-ci/typos@v1.33.1

Cargo.lock

@@ -87,7 +87,7 @@ motore-macros = "0.4.3"
 murmur3 = "0.5.2"
 num-bigint = "0.4.6"
 once_cell = "1.20"
-opendal = "0.53.0"
+opendal = "0.53.3"
 ordered-float = "4"
 parquet = "55"
 pilota = "0.11.2"

Cargo.toml

@@ -101,6 +101,7 @@ The Apache Iceberg community is built on the principles described in the [Apache
 
 - [Databend](https://github.com/datafuselabs/databend/): An open-source cloud data warehouse that serves as a cost-effective alternative to Snowflake.
 - [Lakekeeper](https://github.com/lakekeeper/lakekeeper/): An Apache-licensed Iceberg REST Catalog with data access controls.
+- [Moonlink](https://github.com/Mooncake-Labs/moonlink): A Rust library that enables sub-second mirroring (CDC) of Postgres tables into Iceberg.
 - [RisingWave](https://github.com/risingwavelabs/risingwave): A Postgres-compatible SQL database designed for real-time event streaming data processing, analysis, and management.
 
 ## License

README.md

@@ -118,7 +118,7 @@ impl SchemaVisitor for GlueSchemaBuilder {
             (ICEBERG_FIELD_ID.to_string(), format!("{}", field.id)),
             (
                 ICEBERG_FIELD_OPTIONAL.to_string(),
-                format!("{}", field.required).to_lowercase(),
+                format!("{}", !field.required).to_lowercase(),
             ),
             (
                 ICEBERG_FIELD_CURRENT.to_string(),
@@ -209,10 +209,11 @@ mod tests {
         name: impl Into<String>,
         r#type: impl Into<String>,
         id: impl Into<String>,
+        optional: bool,
     ) -> Result<Column> {
         let parameters = HashMap::from([
             (ICEBERG_FIELD_ID.to_string(), id.into()),
-            (ICEBERG_FIELD_OPTIONAL.to_string(), "true".to_string()),
+            (ICEBERG_FIELD_OPTIONAL.to_string(), optional.to_string()),
             (ICEBERG_FIELD_CURRENT.to_string(), "true".to_string()),
         ]);
 
@@ -318,19 +319,19 @@ mod tests {
         let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build();
 
         let expected = vec![
-            create_column("c1", "boolean", "1")?,
-            create_column("c2", "int", "2")?,
-            create_column("c3", "bigint", "3")?,
-            create_column("c4", "float", "4")?,
-            create_column("c5", "double", "5")?,
-            create_column("c6", "decimal(2,2)", "6")?,
-            create_column("c7", "date", "7")?,
-            create_column("c8", "string", "8")?,
-            create_column("c9", "timestamp", "9")?,
-            create_column("c10", "string", "10")?,
-            create_column("c11", "string", "11")?,
-            create_column("c12", "binary", "12")?,
-            create_column("c13", "binary", "13")?,
+            create_column("c1", "boolean", "1", false)?,
+            create_column("c2", "int", "2", false)?,
+            create_column("c3", "bigint", "3", false)?,
+            create_column("c4", "float", "4", false)?,
+            create_column("c5", "double", "5", false)?,
+            create_column("c6", "decimal(2,2)", "6", false)?,
+            create_column("c7", "date", "7", false)?,
+            create_column("c8", "string", "8", false)?,
+            create_column("c9", "timestamp", "9", false)?,
+            create_column("c10", "string", "10", false)?,
+            create_column("c11", "string", "11", false)?,
+            create_column("c12", "binary", "12", false)?,
+            create_column("c13", "binary", "13", false)?,
         ];
 
         assert_eq!(result, expected);
@@ -378,6 +379,7 @@ mod tests {
             "person",
             "struct<name:string, age:int>",
             "1",
+            false,
         )?];
 
         assert_eq!(result, expected);
@@ -432,6 +434,7 @@ mod tests {
             "location",
             "array<struct<latitude:float, longitude:float>>",
             "1",
+            false,
         )?];
 
         assert_eq!(result, expected);
@@ -475,10 +478,50 @@ mod tests {
 
         let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build();
 
-        let expected = vec![create_column("quux", "map<string,map<string,int>>", "1")?];
+        let expected = vec![create_column(
+            "quux",
+            "map<string,map<string,int>>",
+            "1",
+            false,
+        )?];
 
         assert_eq!(result, expected);
 
         Ok(())
     }
+
+    #[test]
+    fn test_schema_with_optional_fields() -> Result<()> {
+        let record = r#"{
+            "type": "struct",
+            "schema-id": 1,
+            "fields": [
+                {
+                    "id": 1,
+                    "name": "required_field",
+                    "required": true,
+                    "type": "string"
+                },
+                {
+                    "id": 2,
+                    "name": "optional_field",
+                    "required": false,
+                    "type": "int"
+                }
+            ]
+        }"#;
+
+        let schema = serde_json::from_str::<Schema>(record)?;
+        let metadata = create_metadata(schema)?;
+
+        let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build();
+
+        let expected = vec![
+            create_column("required_field", "string", "1", false)?,
+            create_column("optional_field", "int", "2", true)?,
+        ];
+
+        assert_eq!(result, expected);
+        Ok(())
+    }
 }

crates/catalog/glue/src/schema.rs

@@ -151,7 +151,7 @@ pub(crate) fn convert_to_glue_table(
 
     let storage_descriptor = StorageDescriptor::builder()
         .set_columns(Some(glue_schema))
-        .location(&metadata_location)
+        .location(metadata.location().to_string())
         .build();
 
     let mut parameters = HashMap::from([
@@ -345,7 +345,7 @@ mod tests {
 
         let parameters = HashMap::from([
             (ICEBERG_FIELD_ID.to_string(), "1".to_string()),
-            (ICEBERG_FIELD_OPTIONAL.to_string(), "true".to_string()),
+            (ICEBERG_FIELD_OPTIONAL.to_string(), "false".to_string()),
             (ICEBERG_FIELD_CURRENT.to_string(), "true".to_string()),
         ]);
 
@@ -359,7 +359,7 @@ mod tests {
 
         let storage_descriptor = StorageDescriptor::builder()
             .set_columns(Some(vec![column]))
-            .location(&metadata_location)
+            .location(metadata.location())
             .build();
 
         let result =