Skip to content

Commit 582c554

Browse files
maismailmaismail
authored
[CLOUD-636] Add support for ArrowFlight server with DuckDB (#266)
* [CLOUD-636] Add support for ArrowFlight server with DuckDB * fix changelog * [CLOUD-636] fixing tests * [CLOUD-636] adding more tests * update changelog
1 parent fb1d640 commit 582c554

13 files changed

+537
-64
lines changed

‎CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ NOTES:
55
BREAKING CHANGES:
66

77
ENHANCEMENTS:
8+
* resource/hopsworksai_cluster: Add a new attribute `rondb/mysql_nodes/arrow_flight_with_duckdb` to enable or disable ArrowFlight server with DuckDB
89

910
FEATURES:
1011

@@ -21,6 +22,7 @@ ENHANCEMENTS:
2122
* resource/hopsworksai_cluster: Set Default `version` to 3.3.0
2223
* Bump minimum Go version to 1.19
2324

25+
2426
FEATURES:
2527

2628
BUG FIXES:

‎docs/data-sources/cluster.md

+1
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ Read-Only:
325325

326326
Read-Only:
327327

328+
- `arrow_flight_with_duckdb` (Boolean)
328329
- `count` (Number)
329330
- `disk_size` (Number)
330331
- `instance_type` (String)

‎docs/data-sources/clusters.md

+1
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ Read-Only:
349349

350350
Read-Only:
351351

352+
- `arrow_flight_with_duckdb` (Boolean)
352353
- `count` (Number)
353354
- `disk_size` (Number)
354355
- `instance_type` (String)

‎docs/resources/cluster.md

+1
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ Required:
261261

262262
Optional:
263263

264+
- `arrow_flight_with_duckdb` (Boolean) Enable or disable ArrowFight server with DuckDB to speed up different feature store operations for external python clients. Defaults to `false`.
264265
- `count` (Number) The number of MySQL nodes. Defaults to `1`.
265266
- `disk_size` (Number) The disk size of MySQL nodes in units of GB Defaults to `128`.
266267

‎docs/resources/cluster_from_backup.md

+1
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,7 @@ Read-Only:
389389

390390
Read-Only:
391391

392+
- `arrow_flight_with_duckdb` (Boolean)
392393
- `count` (Number)
393394
- `disk_size` (Number)
394395
- `instance_type` (String)

‎hopsworksai/data_source_aws_instance_profile_policy.go

+2
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ func dataSourceAWSInstanceProfilePolicyRead(ctx context.Context, d *schema.Resou
286286
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s/airflow", region, userEcrAccount, clusterId),
287287
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s/git", region, userEcrAccount, clusterId),
288288
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s/testconnector", region, userEcrAccount, clusterId),
289+
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s/flyingduck", region, userEcrAccount, clusterId),
289290
}
290291
var allowPullImagesFromHopsworkAiResource = []string{
291292
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/filebeat", region, hopsworksaiEcrAccount),
@@ -294,6 +295,7 @@ func dataSourceAWSInstanceProfilePolicyRead(ctx context.Context, d *schema.Resou
294295
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/airflow", region, hopsworksaiEcrAccount),
295296
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/git", region, hopsworksaiEcrAccount),
296297
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/testconnector", region, hopsworksaiEcrAccount),
298+
fmt.Sprintf("arn:aws:ecr:%s:%s:repository/flyingduck", region, hopsworksaiEcrAccount),
297299
}
298300
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
299301
}

‎hopsworksai/data_source_aws_instance_profile_policy_test.go

+16
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ func TestAccAWSInstanceProfilePolicy_basic(t *testing.T) {
2525
"arn:aws:ecr:*:*:repository/*/airflow",
2626
"arn:aws:ecr:*:*:repository/*/git",
2727
"arn:aws:ecr:*:*:repository/*/testconnector",
28+
"arn:aws:ecr:*:*:repository/*/flyingduck",
2829
}
2930
var allowPullImagesFromHopsworkAiResource = []string{
3031
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -33,6 +34,7 @@ func TestAccAWSInstanceProfilePolicy_basic(t *testing.T) {
3334
"arn:aws:ecr:*:822623301872:repository/airflow",
3435
"arn:aws:ecr:*:822623301872:repository/git",
3536
"arn:aws:ecr:*:822623301872:repository/testconnector",
37+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
3638
}
3739
policy.Statements = append(policy.Statements, awsEKSPermissions(allowDescribeEKSResource)...)
3840
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
@@ -68,6 +70,7 @@ func TestAccAWSInstanceProfilePolicy_eks_restriction(t *testing.T) {
6870
"arn:aws:ecr:*:*:repository/*/airflow",
6971
"arn:aws:ecr:*:*:repository/*/git",
7072
"arn:aws:ecr:*:*:repository/*/testconnector",
73+
"arn:aws:ecr:*:*:repository/*/flyingduck",
7174
}
7275
var allowPullImagesFromHopsworkAiResource = []string{
7376
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -76,6 +79,7 @@ func TestAccAWSInstanceProfilePolicy_eks_restriction(t *testing.T) {
7679
"arn:aws:ecr:*:822623301872:repository/airflow",
7780
"arn:aws:ecr:*:822623301872:repository/git",
7881
"arn:aws:ecr:*:822623301872:repository/testconnector",
82+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
7983
}
8084
policy.Statements = append(policy.Statements, awsEKSPermissions(allowDescribeEKSResource)...)
8185
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
@@ -111,6 +115,7 @@ func TestAccAWSInstanceProfilePolicy_cluster_id(t *testing.T) {
111115
"arn:aws:ecr:*:*:repository/cluster_id/airflow",
112116
"arn:aws:ecr:*:*:repository/cluster_id/git",
113117
"arn:aws:ecr:*:*:repository/cluster_id/testconnector",
118+
"arn:aws:ecr:*:*:repository/cluster_id/flyingduck",
114119
}
115120
var allowPullImagesFromHopsworkAiResource = []string{
116121
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -119,6 +124,7 @@ func TestAccAWSInstanceProfilePolicy_cluster_id(t *testing.T) {
119124
"arn:aws:ecr:*:822623301872:repository/airflow",
120125
"arn:aws:ecr:*:822623301872:repository/git",
121126
"arn:aws:ecr:*:822623301872:repository/testconnector",
127+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
122128
}
123129
policy.Statements = append(policy.Statements, awsEKSPermissions(allowDescribeEKSResource)...)
124130
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
@@ -154,6 +160,7 @@ func TestAccAWSInstanceProfilePolicy_singleBucket(t *testing.T) {
154160
"arn:aws:ecr:*:*:repository/*/airflow",
155161
"arn:aws:ecr:*:*:repository/*/git",
156162
"arn:aws:ecr:*:*:repository/*/testconnector",
163+
"arn:aws:ecr:*:*:repository/*/flyingduck",
157164
}
158165
var allowPullImagesFromHopsworkAiResource = []string{
159166
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -162,6 +169,7 @@ func TestAccAWSInstanceProfilePolicy_singleBucket(t *testing.T) {
162169
"arn:aws:ecr:*:822623301872:repository/airflow",
163170
"arn:aws:ecr:*:822623301872:repository/git",
164171
"arn:aws:ecr:*:822623301872:repository/testconnector",
172+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
165173
}
166174
policy.Statements = append(policy.Statements, awsEKSPermissions(allowDescribeEKSResource)...)
167175
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
@@ -220,6 +228,7 @@ func TestAccAWSInstanceProfilePolicy_disableEKS(t *testing.T) {
220228
"arn:aws:ecr:*:*:repository/*/airflow",
221229
"arn:aws:ecr:*:*:repository/*/git",
222230
"arn:aws:ecr:*:*:repository/*/testconnector",
231+
"arn:aws:ecr:*:*:repository/*/flyingduck",
223232
}
224233
var allowPullImagesFromHopsworkAiResource = []string{
225234
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -228,6 +237,7 @@ func TestAccAWSInstanceProfilePolicy_disableEKS(t *testing.T) {
228237
"arn:aws:ecr:*:822623301872:repository/airflow",
229238
"arn:aws:ecr:*:822623301872:repository/git",
230239
"arn:aws:ecr:*:822623301872:repository/testconnector",
240+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
231241
}
232242
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
233243

@@ -261,6 +271,7 @@ func TestAccAWSInstanceProfilePolicy_limitRegion(t *testing.T) {
261271
"arn:aws:ecr:us-east-2:*:repository/*/airflow",
262272
"arn:aws:ecr:us-east-2:*:repository/*/git",
263273
"arn:aws:ecr:us-east-2:*:repository/*/testconnector",
274+
"arn:aws:ecr:us-east-2:*:repository/*/flyingduck",
264275
}
265276
var allowPullImagesFromHopsworkAiResource = []string{
266277
"arn:aws:ecr:us-east-2:822623301872:repository/filebeat",
@@ -269,6 +280,7 @@ func TestAccAWSInstanceProfilePolicy_limitRegion(t *testing.T) {
269280
"arn:aws:ecr:us-east-2:822623301872:repository/airflow",
270281
"arn:aws:ecr:us-east-2:822623301872:repository/git",
271282
"arn:aws:ecr:us-east-2:822623301872:repository/testconnector",
283+
"arn:aws:ecr:us-east-2:822623301872:repository/flyingduck",
272284
}
273285
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
274286

@@ -302,6 +314,7 @@ func TestAccAWSInstanceProfilePolicy_limitUserAccount(t *testing.T) {
302314
"arn:aws:ecr:*:user:repository/*/airflow",
303315
"arn:aws:ecr:*:user:repository/*/git",
304316
"arn:aws:ecr:*:user:repository/*/testconnector",
317+
"arn:aws:ecr:*:user:repository/*/flyingduck",
305318
}
306319
var allowPullImagesFromHopsworkAiResource = []string{
307320
"arn:aws:ecr:*:822623301872:repository/filebeat",
@@ -310,6 +323,7 @@ func TestAccAWSInstanceProfilePolicy_limitUserAccount(t *testing.T) {
310323
"arn:aws:ecr:*:822623301872:repository/airflow",
311324
"arn:aws:ecr:*:822623301872:repository/git",
312325
"arn:aws:ecr:*:822623301872:repository/testconnector",
326+
"arn:aws:ecr:*:822623301872:repository/flyingduck",
313327
}
314328
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
315329

@@ -343,6 +357,7 @@ func TestAccAWSInstanceProfilePolicy_limitHopsworksAccount(t *testing.T) {
343357
"arn:aws:ecr:*:*:repository/*/airflow",
344358
"arn:aws:ecr:*:*:repository/*/git",
345359
"arn:aws:ecr:*:*:repository/*/testconnector",
360+
"arn:aws:ecr:*:*:repository/*/flyingduck",
346361
}
347362
var allowPullImagesFromHopsworkAiResource = []string{
348363
"arn:aws:ecr:*:hopsworks:repository/filebeat",
@@ -351,6 +366,7 @@ func TestAccAWSInstanceProfilePolicy_limitHopsworksAccount(t *testing.T) {
351366
"arn:aws:ecr:*:hopsworks:repository/airflow",
352367
"arn:aws:ecr:*:hopsworks:repository/git",
353368
"arn:aws:ecr:*:hopsworks:repository/testconnector",
369+
"arn:aws:ecr:*:hopsworks:repository/flyingduck",
354370
}
355371
policy.Statements = append(policy.Statements, awsECRPermissions(allowPullImagesFromHopsworkAiResource, allowPushandPullImagesResource)...)
356372

0 commit comments

Comments
 (0)