Skip to content

Commit 7ae5f36

Browse files
buhtrbuhtr
authored
Use HiveMetastore instead of Hive (#17483)
1 parent af01dce commit 7ae5f36

File tree

9 files changed

+67
-67
lines changed

9 files changed

+67
-67
lines changed

‎ydb/core/external_sources/external_source_factory.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -69,16 +69,16 @@ IExternalSource::TPtr BuildIcebergSource(const std::vector<TRegExMatch>& hostnam
6969
GetRequiredValidator(),
7070
GetHasSettingCondition(WAREHOUSE_TYPE, VALUE_S3)
7171
)
72-
// Catalog type is a required field and can be equal only to "hive" or "hadoop"
72+
// Catalog type is a required field and can be equal only to "hive_metastore" or "hadoop"
7373
.Property(
7474
CATALOG_TYPE,
75-
GetIsInListValidator({VALUE_HIVE, VALUE_HADOOP}, true)
75+
GetIsInListValidator({VALUE_HIVE_METASTORE, VALUE_HADOOP}, true)
7676
)
77-
// If catalog type is equal to "hive" the field "hive_uri" is required
77+
// If catalog type is equal to "hive_metastore" the field "catalog_hive_metastore_uri" is required
7878
.Property(
79-
CATALOG_HIVE_URI,
79+
CATALOG_HIVE_METASTORE_URI,
8080
GetRequiredValidator(),
81-
GetHasSettingCondition(CATALOG_TYPE,VALUE_HIVE)
81+
GetHasSettingCondition(CATALOG_TYPE, VALUE_HIVE_METASTORE)
8282
)
8383
.HostnamePatterns(hostnamePatternsRegEx)
8484
.Build();

‎ydb/core/external_sources/iceberg_ddl_ut.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ Y_UNIT_TEST_SUITE(IcebergDdlTest) {
5050
Y_UNIT_TEST_F(HiveCatalogWithS3Test, TTestFixture) {
5151
using namespace NKikimr::NExternalSource::NIceberg;
5252

53-
Props[CATALOG_TYPE] = VALUE_HIVE;
54-
Props[CATALOG_HIVE_URI] = "hive_uri";
53+
Props[CATALOG_TYPE] = VALUE_HIVE_METASTORE;
54+
Props[CATALOG_HIVE_METASTORE_URI] = "hive_metastore_uri";
5555

5656
UNIT_ASSERT_NO_EXCEPTION(Source->ValidateExternalDataSource(Proto.SerializeAsString()));
5757

‎ydb/core/external_sources/iceberg_fields.h

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ constexpr char WAREHOUSE_TLS[] = "use_tls";
1414
constexpr char WAREHOUSE_DB[] = "database_name";
1515

1616
// Fields that belongs to a catalog
17-
constexpr char CATALOG_TYPE[] = "catalog_type";
18-
constexpr char CATALOG_HIVE_URI[] = "catalog_hive_uri";
17+
constexpr char CATALOG_TYPE[] = "catalog_type";
18+
constexpr char CATALOG_HIVE_METASTORE_URI[] = "catalog_hive_metastore_uri";
1919

2020
// Some values
21-
constexpr char VALUE_S3[] = "s3";
22-
constexpr char VALUE_HIVE[] = "hive";
23-
constexpr char VALUE_HADOOP[] = "hadoop";
21+
constexpr char VALUE_S3[] = "s3";
22+
constexpr char VALUE_HIVE_METASTORE[] = "hive_metastore";
23+
constexpr char VALUE_HADOOP[] = "hadoop";
2424

2525
// List of fields which is pass to a connector
2626
constexpr std::array<const char*, 6> FieldsToConnector = {
@@ -29,7 +29,7 @@ constexpr std::array<const char*, 6> FieldsToConnector = {
2929
WAREHOUSE_S3_REGION,
3030
WAREHOUSE_S3_URI,
3131
CATALOG_TYPE,
32-
CATALOG_HIVE_URI
32+
CATALOG_HIVE_METASTORE_URI
3333
};
3434

3535
} // NKikimr::NExternalSource::NIceberg

‎ydb/core/fq/libs/common/iceberg_processor.cpp

+8-8
Original file line numberDiff line numberDiff line change
@@ -176,14 +176,14 @@ TString MakeIcebergCreateExternalDataSourceProperties(const NConfig::TCommonConf
176176
catalogSection = fmt::format(
177177
R"(
178178
{catalog_type}={catalog_type_value},
179-
{catalog_hive_uri}={catalog_hive_uri_value},
179+
{catalog_hive_metastore_uri}={catalog_hive_metastore_uri_value},
180180
database_name={database_name}
181181
)",
182-
"catalog_type"_a = CATALOG_TYPE,
183-
"catalog_type_value"_a = EncloseAndEscapeString(VALUE_HIVE, '"'),
184-
"catalog_hive_uri"_a = CATALOG_HIVE_URI,
185-
"catalog_hive_uri_value"_a = EncloseAndEscapeString(hiveMetastore.uri(), '"'),
186-
"database_name"_a = EncloseAndEscapeString(hiveMetastore.database_name(), '"')
182+
"catalog_type"_a = CATALOG_TYPE,
183+
"catalog_type_value"_a = EncloseAndEscapeString(VALUE_HIVE_METASTORE, '"'),
184+
"catalog_hive_metastore_uri"_a = CATALOG_HIVE_METASTORE_URI,
185+
"catalog_hive_metastore_uri_value"_a = EncloseAndEscapeString(hiveMetastore.uri(), '"'),
186+
"database_name"_a = EncloseAndEscapeString(hiveMetastore.database_name(), '"')
187187
);
188188
});
189189

@@ -241,8 +241,8 @@ void FillIcebergGenericClusterConfig(const NConfig::TCommonConfig& yqConfig, con
241241
});
242242

243243
processor.SetDoOnCatalogHive([&options, &cluster](const FederatedQuery::IcebergCatalog_HiveMetastore& hiveMetastore) {
244-
options[CATALOG_TYPE] = VALUE_HIVE;
245-
options[CATALOG_HIVE_URI] = hiveMetastore.uri();
244+
options[CATALOG_TYPE] = VALUE_HIVE_METASTORE;
245+
options[CATALOG_HIVE_METASTORE_URI] = hiveMetastore.uri();
246246

247247
cluster.SetDatabaseName(hiveMetastore.database_name());
248248
});

‎ydb/core/fq/libs/common/iceberg_processor_ut.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ Y_UNIT_TEST_SUITE(IcebergClusterProcessor) {
169169
SubstGlobal(ddlProperties, " ", "");
170170
SubstGlobal(ddlProperties, "\n", "");
171171

172-
UNIT_ASSERT_VALUES_EQUAL(ddlProperties, "source_type=\"Iceberg\",use_tls=\"true\",warehouse_type=\"s3\",warehouse_s3_region=\"ru-central1\",warehouse_s3_endpoint=\"s3endpoint\",warehouse_s3_uri=\"s3a://iceberg-bucket/storage\",catalog_type=\"hive\",catalog_hive_uri=\"hive_metastore_uri\",database_name=\"hive_metastore_db\"");
172+
UNIT_ASSERT_VALUES_EQUAL(ddlProperties, "source_type=\"Iceberg\",use_tls=\"true\",warehouse_type=\"s3\",warehouse_s3_region=\"ru-central1\",warehouse_s3_endpoint=\"s3endpoint\",warehouse_s3_uri=\"s3a://iceberg-bucket/storage\",catalog_type=\"hive_metastore\",catalog_hive_metastore_uri=\"hive_metastore_uri\",database_name=\"hive_metastore_db\"");
173173
}
174174

175175
// Test parsing for FederatedQuery::IcebergCluster without warehouse

‎ydb/core/kqp/ut/federated_query/generic_ut/iceberg_ut_data.cpp

+13-13
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55

66
namespace NTestUtils {
77

8-
constexpr char VALUE_HIVE_URI[] = "hive_uri";
9-
constexpr char VALUE_S3_URI[] = "s3_uri";
10-
constexpr char VALUE_S3_ENDPOINT[] = "s3_endpoint";
11-
constexpr char VALUE_S3_REGION[] = "s3_region";
8+
constexpr char VALUE_HIVE_METASTORE_URI[] = "hive_metastore_uri";
9+
constexpr char VALUE_S3_URI[] = "s3_uri";
10+
constexpr char VALUE_S3_ENDPOINT[] = "s3_endpoint";
11+
constexpr char VALUE_S3_REGION[] = "s3_region";
1212

1313
constexpr char VALUE_IAM[] = "IAM";
1414

@@ -74,10 +74,10 @@ NYql::TGenericDataSourceInstance TIcebergTestData::CreateDataSourceForHadoop() {
7474
return data.Result_;
7575
}
7676

77-
NYql::TGenericDataSourceInstance TIcebergTestData::CreateDataSourceForHive() {
77+
NYql::TGenericDataSourceInstance TIcebergTestData::CreateDataSourceForHiveMetastore() {
7878
TTestData data(this);
79-
auto& hive = *data.Catalog_.mutable_hive();
80-
hive.set_uri(VALUE_HIVE_URI);
79+
auto& h = *data.Catalog_.mutable_hive_metastore();
80+
h.set_uri(VALUE_HIVE_METASTORE_URI);
8181
return data.Result_;
8282
}
8383

@@ -156,18 +156,18 @@ void TIcebergTestData::ExecuteQuery(const std::shared_ptr<NKikimr::NKqp::TKikimr
156156
}
157157

158158

159-
void TIcebergTestData::ExecuteCreateHiveExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr) {
159+
void TIcebergTestData::ExecuteCreateHiveMetastoreExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr) {
160160
using namespace fmt::literals;
161161

162-
TString hiveCatalog = fmt::format(R"(
162+
TString hiveMetastoreCatalog = fmt::format(R"(
163163
CATALOG_TYPE="{type}",
164-
CATALOG_HIVE_URI="{uri}"
164+
CATALOG_HIVE_METASTORE_URI="{uri}"
165165
)",
166-
"type"_a = NKikimr::NExternalSource::NIceberg::VALUE_HIVE,
167-
"uri"_a = VALUE_HIVE_URI
166+
"type"_a = NKikimr::NExternalSource::NIceberg::VALUE_HIVE_METASTORE,
167+
"uri"_a = VALUE_HIVE_METASTORE_URI
168168
);
169169

170-
ExecuteQuery(kikimr, CreateQuery(hiveCatalog));
170+
ExecuteQuery(kikimr, CreateQuery(hiveMetastoreCatalog));
171171
}
172172

173173
void TIcebergTestData::ExecuteCreateHadoopExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr) {

‎ydb/core/kqp/ut/federated_query/generic_ut/iceberg_ut_data.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ class TIcebergTestData final {
2828

2929
NYql::TGenericDataSourceInstance CreateDataSourceForHadoop();
3030

31-
NYql::TGenericDataSourceInstance CreateDataSourceForHive();
31+
NYql::TGenericDataSourceInstance CreateDataSourceForHiveMetastore();
3232

33-
void ExecuteCreateHiveExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr);
33+
void ExecuteCreateHiveMetastoreExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr);
3434

3535
void ExecuteCreateHadoopExternalDataSource(const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr);
3636

‎ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp

+27-27
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ namespace NKikimr::NKqp {
3838
PostgreSQL,
3939
ClickHouse,
4040
Ydb,
41-
IcebergHiveBasic,
42-
IcebergHiveSa,
43-
IcebergHiveToken,
41+
IcebergHiveMetastoreBasic,
42+
IcebergHiveMetastoreSa,
43+
IcebergHiveMetastoreToken,
4444
IcebergHadoopBasic,
4545
IcebergHadoopSa,
4646
IcebergHadoopToken,
@@ -54,12 +54,12 @@ namespace NKikimr::NKqp {
5454
return TConnectorClientMock::TClickHouseDataSourceInstanceBuilder<>().GetResult();
5555
case EProviderType::Ydb:
5656
return TConnectorClientMock::TYdbDataSourceInstanceBuilder<>().GetResult();
57-
case EProviderType::IcebergHiveBasic:
58-
return NTestUtils::CreateIcebergBasic().CreateDataSourceForHive();
59-
case EProviderType::IcebergHiveSa:
60-
return NTestUtils::CreateIcebergSa().CreateDataSourceForHive();
61-
case EProviderType::IcebergHiveToken:
62-
return NTestUtils::CreateIcebergToken().CreateDataSourceForHive();
57+
case EProviderType::IcebergHiveMetastoreBasic:
58+
return NTestUtils::CreateIcebergBasic().CreateDataSourceForHiveMetastore();
59+
case EProviderType::IcebergHiveMetastoreSa:
60+
return NTestUtils::CreateIcebergSa().CreateDataSourceForHiveMetastore();
61+
case EProviderType::IcebergHiveMetastoreToken:
62+
return NTestUtils::CreateIcebergToken().CreateDataSourceForHiveMetastore();
6363
case EProviderType::IcebergHadoopBasic:
6464
return NTestUtils::CreateIcebergBasic().CreateDataSourceForHadoop();
6565
case EProviderType::IcebergHadoopSa:
@@ -77,15 +77,15 @@ namespace NKikimr::NKqp {
7777
return CreateClickHouseExternalDataSource(kikimr);
7878
case EProviderType::Ydb:
7979
return CreateYdbExternalDataSource(kikimr);
80-
case EProviderType::IcebergHiveBasic:
80+
case EProviderType::IcebergHiveMetastoreBasic:
8181
return NTestUtils::CreateIcebergBasic()
82-
.ExecuteCreateHiveExternalDataSource(kikimr);
83-
case EProviderType::IcebergHiveSa:
82+
.ExecuteCreateHiveMetastoreExternalDataSource(kikimr);
83+
case EProviderType::IcebergHiveMetastoreSa:
8484
return NTestUtils::CreateIcebergSa()
85-
.ExecuteCreateHiveExternalDataSource(kikimr);
86-
case EProviderType::IcebergHiveToken:
85+
.ExecuteCreateHiveMetastoreExternalDataSource(kikimr);
86+
case EProviderType::IcebergHiveMetastoreToken:
8787
return NTestUtils::CreateIcebergToken()
88-
.ExecuteCreateHiveExternalDataSource(kikimr);
88+
.ExecuteCreateHiveMetastoreExternalDataSource(kikimr);
8989
case EProviderType::IcebergHadoopBasic:
9090
return NTestUtils::CreateIcebergBasic()
9191
.ExecuteCreateHadoopExternalDataSource(kikimr);
@@ -238,15 +238,15 @@ namespace NKikimr::NKqp {
238238
}
239239

240240
Y_UNIT_TEST(IcebergHiveBasicSelectAll) {
241-
TestSelectAllFields(EProviderType::IcebergHiveBasic);
241+
TestSelectAllFields(EProviderType::IcebergHiveMetastoreBasic);
242242
}
243243

244244
Y_UNIT_TEST(IcebergHiveSaSelectAll) {
245-
TestSelectAllFields(EProviderType::IcebergHiveSa);
245+
TestSelectAllFields(EProviderType::IcebergHiveMetastoreSa);
246246
}
247247

248248
Y_UNIT_TEST(IcebergHiveTokenSelectAll) {
249-
TestSelectAllFields(EProviderType::IcebergHiveToken);
249+
TestSelectAllFields(EProviderType::IcebergHiveMetastoreToken);
250250
}
251251

252252
Y_UNIT_TEST(IcebergHadoopBasicSelectAll) {
@@ -353,15 +353,15 @@ namespace NKikimr::NKqp {
353353
}
354354

355355
Y_UNIT_TEST(IcebergHiveBasicSelectConstant) {
356-
TestSelectConstant(EProviderType::IcebergHiveBasic);
356+
TestSelectConstant(EProviderType::IcebergHiveMetastoreBasic);
357357
}
358358

359359
Y_UNIT_TEST(IcebergHiveSaSelectConstant) {
360-
TestSelectConstant(EProviderType::IcebergHiveSa);
360+
TestSelectConstant(EProviderType::IcebergHiveMetastoreSa);
361361
}
362362

363363
Y_UNIT_TEST(IcebergHiveTokenSelectConstant) {
364-
TestSelectConstant(EProviderType::IcebergHiveToken);
364+
TestSelectConstant(EProviderType::IcebergHiveMetastoreToken);
365365
}
366366

367367
Y_UNIT_TEST(IcebergHadoopBasicSelectConstant) {
@@ -464,15 +464,15 @@ namespace NKikimr::NKqp {
464464
}
465465

466466
Y_UNIT_TEST(IcebergHiveBasicSelectCount) {
467-
TestSelectCount(EProviderType::IcebergHiveBasic);
467+
TestSelectCount(EProviderType::IcebergHiveMetastoreBasic);
468468
}
469469

470470
Y_UNIT_TEST(IcebergHiveSaSelectCount) {
471-
TestSelectCount(EProviderType::IcebergHiveSa);
471+
TestSelectCount(EProviderType::IcebergHiveMetastoreSa);
472472
}
473473

474474
Y_UNIT_TEST(IcebergHiveTokenSelectCount) {
475-
TestSelectCount(EProviderType::IcebergHiveToken);
475+
TestSelectCount(EProviderType::IcebergHiveMetastoreToken);
476476
}
477477

478478
Y_UNIT_TEST(IcebergHadoopBasicSelectCount) {
@@ -600,15 +600,15 @@ namespace NKikimr::NKqp {
600600
}
601601

602602
Y_UNIT_TEST(IcebergHiveBasicFilterPushdown) {
603-
TestFilterPushdown(EProviderType::IcebergHiveBasic);
603+
TestFilterPushdown(EProviderType::IcebergHiveMetastoreBasic);
604604
}
605605

606606
Y_UNIT_TEST(IcebergHiveSaFilterPushdown) {
607-
TestFilterPushdown(EProviderType::IcebergHiveSa);
607+
TestFilterPushdown(EProviderType::IcebergHiveMetastoreSa);
608608
}
609609

610610
Y_UNIT_TEST(IcebergHiveTokenFilterPushdown) {
611-
TestFilterPushdown(EProviderType::IcebergHiveToken);
611+
TestFilterPushdown(EProviderType::IcebergHiveMetastoreToken);
612612
}
613613

614614
Y_UNIT_TEST(IcebergHadoopBasicFilterPushdown) {

‎ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -517,10 +517,10 @@ namespace NYql {
517517
if (VALUE_HADOOP == catalogType) {
518518
// hadoop nothing yet
519519
catalog.mutable_hadoop();
520-
} else if (VALUE_HIVE == catalogType) {
521-
auto hiveUri = GetOptionValue(clusterOptions, CATALOG_HIVE_URI);
520+
} else if (VALUE_HIVE_METASTORE == catalogType) {
521+
auto uri = GetOptionValue(clusterOptions, CATALOG_HIVE_METASTORE_URI);
522522

523-
catalog.mutable_hive()->set_uri(hiveUri);
523+
catalog.mutable_hive_metastore()->set_uri(uri);
524524
} else {
525525
throw yexception() << "Unexpected catalog type: " << catalogType;
526526
}

0 commit comments

Comments
 (0)