Merge branch 'main' into CsPushdown2

Author: pavelvelikhov

.github/CODEOWNERS

@@ -1,3 +1,4 @@
+/CHANGELOG.md @ydb-platform/changelog-reviewers
 /*.md @ydb-platform/docs
 
 /build @ydb-platform/ci

.github/config/muted_ya.txt

@@ -14,7 +14,6 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadHuge
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadSmall
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteHuge
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall
-ydb/core/kqp/ut/batch_operations KqpBatchUpdate.Large_3
 ydb/core/kqp/ut/batch_operations [*/*] chunk chunk
 ydb/core/kqp/ut/cost KqpCost.OlapWriteRow
 ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.ExecuteScriptWithLargeFile
@@ -23,6 +22,8 @@ ydb/core/kqp/ut/federated_query/s3 sole chunk chunk
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
 ydb/core/kqp/ut/olap KqpOlapDelete.DeleteWithDiffrentTypesPKColumns-isStream
+ydb/core/kqp/ut/olap KqpOlapJson.CompactionVariants
+ydb/core/kqp/ut/olap KqpOlapJson.DuplicationCompactionVariants
 ydb/core/kqp/ut/olap KqpOlapJson.SwitchAccessorCompactionVariants
 ydb/core/kqp/ut/olap KqpOlapOptimizer.OptimizationByTime
 ydb/core/kqp/ut/olap KqpOlapSysView.StatsSysViewBytesColumnActualization
@@ -49,6 +50,7 @@ ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithData
 ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithDataAndPersistentPartitionStats
 ydb/core/tx/tiering/ut ColumnShardTiers.TTLUsage
 ydb/core/viewer/ut Viewer.TabletMerging
+ydb/library/actors/interconnect/ut_fat InterconnectZcLocalOp.ZcDisabledAfterHiddenCopy
 ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
 ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk
 ydb/library/yaml_config/ut_transform test_transform.py.TestYamlConfigTransformations.test_basic[args1-dump_ds_init]
@@ -76,9 +78,16 @@ ydb/tests/fq/yds test_recovery.py.TestRecovery.test_ic_disconnection
 ydb/tests/fq/yds test_select_limit_db_id.py.TestSelectLimitWithDbId.test_select_same_with_id[v1-mvp_external_ydb_endpoint0]
 ydb/tests/fq/yds test_yds_bindings.py.TestBindings.test_yds_insert[v1]
 ydb/tests/functional/compatibility sole chunk chunk
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_simple[last_stable_to_current_mixed-column]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[current_to_last_stable-column]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[current_to_last_stable-row]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[last_stable_to_current-column]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[last_stable_to_current-row]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[last_stable_to_current_mixed-column]
+ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_tpch1[last_stable_to_current_mixed-row]
 ydb/tests/functional/compatibility test_followers.py.TestFollowersCompatibility.test_followers_compatability
+ydb/tests/functional/compatibility test_stress.py.TestStress.test_kv[mixed-column]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[current-column]
-ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[last_stable-row]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[mixed-column]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[mixed-row]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_tpch1[current-column]
@@ -119,14 +128,12 @@ ydb/tests/olap/oom overlapping_portions.py.TestOverlappingPortions.test
 ydb/tests/olap/scenario sole chunk chunk
 ydb/tests/olap/scenario test_alter_compression.py.TestAlterCompression.test[alter_compression]
 ydb/tests/olap/scenario test_alter_tiering.py.TestAlterTiering.test[many_tables]
-ydb/tests/olap/scenario test_insert.py.TestInsert.test[read_data_during_bulk_upsert]
 ydb/tests/olap/scenario test_read_update_write_load.py.TestReadUpdateWriteLoad.test[read_update_write_load]
 ydb/tests/olap/ttl_tiering [data_migration_when_alter_ttl.py] chunk chunk
 ydb/tests/olap/ttl_tiering [ttl_delete_s3.py] chunk chunk
 ydb/tests/olap/ttl_tiering data_migration_when_alter_ttl.py.TestDataMigrationWhenAlterTtl.test
 ydb/tests/olap/ttl_tiering sole chunk chunk
 ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_delete_s3_tiering
-ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_ttl_delete
 ydb/tests/sql/large sole chunk chunk
 ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[column]
 ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[row]

.github/workflows/update_muted_ya.yml

@@ -8,7 +8,7 @@ on:
       branches:
         description: 'Comma-separated list of branches to process'
         required: false
-        default: 'main,stable-25-1,stable-25-1-1'
+        default: 'main,stable-25-1,stable-25-1-analytics'
 
 env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -29,7 +29,7 @@ jobs:
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo "branches=$(echo '${{ github.event.inputs.branches }}' | jq -R -s -c 'split(",")')" >> $GITHUB_OUTPUT
           else
-            echo "branches=$(echo '["main","stable-25-1","stable-25-1-1"]' | jq -c .)" >> $GITHUB_OUTPUT
+            echo "branches=$(echo '["main","stable-25-1","stable-25-1-analytics"]' | jq -c .)" >> $GITHUB_OUTPUT
           fi
 
   update-muted-tests:

ydb/apps/etcd_proxy/service/etcd_watch.cpp

@@ -602,7 +602,7 @@ class TWatchtower : public TActorBootstrapped<TWatchtower> {
         sql << "$Leases = select 0L as `lease` union all select `id` as `lease` from `leases` where unwrap(interval('PT1S') * `ttl` + `updated`) > CurrentUtcDatetime(`id`);" << std::endl;
         sql << "$Victims = select `key`, `value`, `created`, `modified`, `version`, `lease` from `current` as h left only join $Leases as l using(`lease`);" << std::endl;
 
-        sql << "insert into `content`" << std::endl;
+        sql << "insert into `history`" << std::endl;
         sql << "select `key`, `created`, " << revName << " as `modified`, 0L as `version`, `value`, `lease` from $Victims;" << std::endl;
         sql << "delete from `current` on select `key` from $Victims;" << std::endl;
 

ydb/apps/ydb/CHANGELOG.md

@@ -1,3 +1,4 @@
+* Improved `ydb import file` commands to support files with BOM (Byte Order Mark).
 * Brackets are now inserted in pairs in YDB CLI interactive mode
 * Added `--scale` option to `ydb workload tpch init` and `ydb workload tpcds init` commands. Sets the percentage of the benchmark's data size and workload to use, relative to full scale.
 * Added "--no-discovery" option. It allows to skip discovery and use user provided endpoint to connect to YDB cluster.

ydb/core/base/appdata.cpp

@@ -73,6 +73,7 @@ struct TAppData::TImpl {
     NKikimrConfig::TDataErasureConfig DataErasureConfig;
     NKikimrConfig::THealthCheckConfig HealthCheckConfig;
     NKikimrConfig::TWorkloadManagerConfig WorkloadManagerConfig;
+    NKikimrConfig::TQueryServiceConfig QueryServiceConfig;
 };
 
 TAppData::TAppData(
@@ -132,6 +133,7 @@ TAppData::TAppData(
     , DataErasureConfig(Impl->DataErasureConfig)
     , HealthCheckConfig(Impl->HealthCheckConfig)
     , WorkloadManagerConfig(Impl->WorkloadManagerConfig)
+    , QueryServiceConfig(Impl->QueryServiceConfig)
     , KikimrShouldContinue(kikimrShouldContinue)
     , TracingConfigurator(MakeIntrusive<NJaegerTracing::TSamplingThrottlingConfigurator>(TimeProvider, RandomProvider))
 {}

ydb/core/base/appdata_fwd.h

@@ -75,6 +75,7 @@ namespace NKikimrConfig {
     class TFeatureFlags;
     class THealthCheckConfig;
     class TWorkloadManagerConfig;
+    class TQueryServiceConfig;
 }
 
 namespace NKikimrReplication {
@@ -251,6 +252,7 @@ struct TAppData {
     NKikimrConfig::TDataErasureConfig& DataErasureConfig;
     NKikimrConfig::THealthCheckConfig& HealthCheckConfig;
     NKikimrConfig::TWorkloadManagerConfig& WorkloadManagerConfig;
+    NKikimrConfig::TQueryServiceConfig& QueryServiceConfig;
     bool EnforceUserTokenRequirement = false;
     bool EnforceUserTokenCheckRequirement = false; // check token if it was specified
     bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request

ydb/core/driver_lib/run/run.cpp

@@ -1253,6 +1253,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
         AppData->WorkloadManagerConfig = runConfig.AppConfig.GetWorkloadManagerConfig();
     }
 
+    if (runConfig.AppConfig.HasQueryServiceConfig()) {
+        AppData->QueryServiceConfig = runConfig.AppConfig.GetQueryServiceConfig();
+    }
+
     // setup resource profiles
     AppData->ResourceProfiles = new TResourceProfiles;
     if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize())

ydb/core/fq/libs/quota_manager/quota_manager.cpp

@@ -141,6 +141,7 @@ TString ToString(const THashMap<TString, TQuotaCachedUsage>& usageMap) {
 }
 
 class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagementService> {
+    using TLimits = decltype(TEvQuotaService::TQuotaSetRequest::Limits);
 public:
     TQuotaManagementService(
         const NConfig::TQuotasManagerConfig& config,
@@ -261,23 +262,23 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
 
         if (it == subjectMap.end()) {
             ReadQuota(subjectType, subjectId,
-                [this, ev=ev](TReadQuotaExecuter& executer) {
+                [this, allowStaleUsage=ev->Get()->AllowStaleUsage, sender=ev->Sender, cookie=ev->Cookie](TReadQuotaExecuter& executer) {
                     // This block is executed in correct self-context, no locks/syncs required
                     auto& subjectMap = this->QuotaCacheMap[executer.State.SubjectType];
                     auto& cache = subjectMap[executer.State.SubjectId];
                     LOG_D(executer.State.SubjectType << "." << executer.State.SubjectId << ToString(cache.UsageMap) << " LOADED");
-                    CheckUsageMaybeReply(executer.State.SubjectType, executer.State.SubjectId, cache, ev);
+                    CheckUsageMaybeReply(executer.State.SubjectType, executer.State.SubjectId, cache, allowStaleUsage, sender, cookie);
                 }
             );
         } else {
-            CheckUsageMaybeReply(subjectType, subjectId, it->second, ev);
+            CheckUsageMaybeReply(subjectType, subjectId, it->second, ev->Get()->AllowStaleUsage, ev->Sender, ev->Cookie);
         }
     }
 
-    void CheckUsageMaybeReply(const TString& subjectType, const TString& subjectId, TQuotaCache& cache, const TEvQuotaService::TQuotaGetRequest::TPtr& ev) {
+    void CheckUsageMaybeReply(const TString& subjectType, const TString& subjectId, TQuotaCache& cache, bool allowStaleUsage, const TActorId& sender, ui64 cookie) {
         bool pended = false;
         auto& infoMap = QuotaInfoMap[subjectType];
-        if (!ev->Get()->AllowStaleUsage) {
+        if (!allowStaleUsage) {
             // Refresh usage
             for (auto& itUsage : cache.UsageMap) {
                 auto metricName = itUsage.first;
@@ -293,7 +294,7 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
                                 cachedUsage.RequestedAt = Now();
                             }
                             if (!pended) {
-                                cache.PendingUsageRequests.emplace(ev->Sender, ev->Cookie);
+                                cache.PendingUsageRequests.emplace(sender, cookie);
                                 pended = true;
                             }
                         }
@@ -303,16 +304,16 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
         }
 
         if (!pended) {
-            SendQuota(ev->Sender, ev->Cookie, subjectType, subjectId, cache);
-            cache.PendingUsageRequests.erase(ev->Sender);
+            SendQuota(sender, cookie, subjectType, subjectId, cache);
+            cache.PendingUsageRequests.erase(sender);
         }
     }
 
-    void ChangeLimitsAndReply(const TString& subjectType, const TString& subjectId, TQuotaCache& cache, const TEvQuotaService::TQuotaSetRequest::TPtr& ev) {
+    void ChangeLimitsAndReply(const TString& subjectType, const TString& subjectId, TQuotaCache& cache, const TLimits& limits, const TActorId& sender, ui64 cookie) {
 
         auto pended = false;
         auto& infoMap = QuotaInfoMap[subjectType];
-        for (auto metricLimit : ev->Get()->Limits) {
+        for (auto metricLimit : limits) {
             auto& metricName = metricLimit.first;
 
             auto it = cache.UsageMap.find(metricName);
@@ -332,8 +333,8 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
 
                     if (info.QuotaController != NActors::TActorId{}) {
                         pended = true;
-                        cache.PendingLimitRequest = ev->Sender;
-                        cache.PendingLimitCookie = ev->Cookie;
+                        cache.PendingLimitRequest = sender;
+                        cache.PendingLimitCookie = cookie;
                         cache.PendingLimit.insert(metricName);
                         Send(info.QuotaController, new TEvQuotaService::TQuotaLimitChangeRequest(subjectType, subjectId, metricName, cached.Usage.Limit.Value, limit));
                         continue;
@@ -354,7 +355,7 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
             for (auto it : cache.UsageMap) {
                 response->Limits.emplace(it.first, it.second.Usage.Limit.Value);
             }
-            Send(ev->Sender, response.Release());
+            Send(sender, response.Release());
         }
     }
 
@@ -723,16 +724,16 @@ class TQuotaManagementService : public NActors::TActorBootstrapped<TQuotaManagem
         auto it = subjectMap.find(subjectId);
         if (it == subjectMap.end()) {
             ReadQuota(subjectType, subjectId,
-                [this, ev=ev](TReadQuotaExecuter& executer) {
+                [this, limits=std::move(ev->Get()->Limits), sender=ev->Sender, cookie=ev->Cookie](TReadQuotaExecuter& executer) {
                     // This block is executed in correct self-context, no locks/syncs required
                     auto& subjectMap = this->QuotaCacheMap[executer.State.SubjectType];
                     auto& cache = subjectMap[executer.State.SubjectId];
                     LOG_D(executer.State.SubjectType << "." << executer.State.SubjectId << ToString(cache.UsageMap) << " LOADED");
-                    ChangeLimitsAndReply(executer.State.SubjectType, executer.State.SubjectId, cache, ev);
+                    ChangeLimitsAndReply(executer.State.SubjectType, executer.State.SubjectId, cache, limits, sender, cookie);
                 }
             );
         } else {
-            ChangeLimitsAndReply(subjectType, subjectId, it->second, ev);
+            ChangeLimitsAndReply(subjectType, subjectId, it->second, ev->Get()->Limits, ev->Sender, ev->Cookie);
         }
     }
 

ydb/core/grpc_services/rpc_describe_external_data_source.cpp

@@ -5,6 +5,9 @@
 #include <ydb/core/ydb_convert/ydb_convert.h>
 #include <ydb/public/api/protos/ydb_table.pb.h>
 
+#include <library/cpp/json/json_writer.h>
+#include <library/cpp/json/writer/json_value.h>
+
 namespace NKikimr::NGRpcService {
 
 using namespace NActors;
@@ -74,6 +77,21 @@ void Convert(const TAuth& in, TProperties& out) {
     }
 }
 
+void Convert(const TExternalTableReferences& in, TProperties& out) {
+    auto& references = out["REFERENCES"];
+    NJson::TJsonValue json(NJson::EJsonValueType::JSON_ARRAY);
+
+    for (const auto& reference : in.GetReferences()) {
+        if (reference.HasPath()) {
+            json.AppendValue(reference.GetPath());
+        } else if (reference.HasPathId()) {
+            // this should never happen, but in case it does, it is better to return something meaningful
+            json.AppendValue(TPathId::FromProto(reference.GetPathId()).ToString());
+        }
+    }
+    references = WriteJson(json, false);
+}
+
 Ydb::Table::DescribeExternalDataSourceResult Convert(const TDirEntry& inSelf, const TExternalDataSourceDescription& inDesc) {
     Ydb::Table::DescribeExternalDataSourceResult out;
     ConvertDirectoryEntry(inSelf, out.mutable_self(), true);
@@ -85,6 +103,7 @@ Ydb::Table::DescribeExternalDataSourceResult Convert(const TDirEntry& inSelf, co
         properties[to_upper(key)] = value;
     }
     Convert(inDesc.GetAuth(), properties);
+    Convert(inDesc.GetReferences(), properties);
     return out;
 }
 

ydb/core/keyvalue/keyvalue_events.h

@@ -13,7 +13,7 @@ namespace NKeyValue {
     struct TIntermediate;
 };
 
-struct TEvKeyValue {
+namespace TEvKeyValue {
     enum EEv {
         EvRequest = EventSpaceBegin(TKikimrEvents::ES_KEYVALUE),
         EvIntermediate,
@@ -208,7 +208,7 @@ struct TEvKeyValue {
         using TResponse = TEvCleanUpDataResponse;
 
         TEvCleanUpDataRequest() = default;
-        
+
         TEvCleanUpDataRequest(ui64 generation, bool reset=false) {
             Record.set_generation(generation);
             Record.set_reset_actual_generation(reset);
@@ -221,27 +221,28 @@ struct TEvKeyValue {
 
         TEvCleanUpDataResponse() = default;
 
-        TEvCleanUpDataResponse(ui64 generation, NKikimrKeyValue::CleanUpDataResponse::Status status, const TString& errorReason, ui64 actualGeneration) {
+        TEvCleanUpDataResponse(ui64 generation, NKikimrKeyValue::CleanUpDataResponse::Status status, const TString& errorReason, ui64 actualGeneration, ui64 tabletId) {
             Record.set_generation(generation);
             Record.set_status(status);
             Record.set_error_reason(errorReason);
             Record.set_actual_generation(actualGeneration);
+            Record.set_tablet_id(tabletId);
         }
 
-        static std::unique_ptr<TEvCleanUpDataResponse> MakeSuccess(ui64 generation) {
-            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_SUCCESS, "", generation);
+        static std::unique_ptr<TEvCleanUpDataResponse> MakeSuccess(ui64 generation, ui64 tabletId) {
+            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_SUCCESS, "", generation, tabletId);
         }
 
-        static std::unique_ptr<TEvCleanUpDataResponse> MakeAborted(ui64 generation, const TString& errorReason, ui64 actualGeneration) {
-            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ABORTED, errorReason, actualGeneration);
+        static std::unique_ptr<TEvCleanUpDataResponse> MakeAborted(ui64 generation, const TString& errorReason, ui64 actualGeneration, ui64 tabletId) {
+            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ABORTED, errorReason, actualGeneration, tabletId);
         }
 
-        static std::unique_ptr<TEvCleanUpDataResponse> MakeAlreadyCompleted(ui64 generation, ui64 actualGeneration) {
-            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ALREADY_COMPLETED, "", actualGeneration);
+        static std::unique_ptr<TEvCleanUpDataResponse> MakeAlreadyCompleted(ui64 generation, ui64 actualGeneration, ui64 tabletId) {
+            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ALREADY_COMPLETED, "", actualGeneration, tabletId);
         }
 
-        static std::unique_ptr<TEvCleanUpDataResponse> MakeError(ui64 generation, const TString& errorReason, ui64 actualGeneration) {
-            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ERROR, errorReason, actualGeneration);
+        static std::unique_ptr<TEvCleanUpDataResponse> MakeError(ui64 generation, const TString& errorReason, ui64 actualGeneration, ui64 tabletId) {
+            return std::make_unique<TEvCleanUpDataResponse>(generation, NKikimrKeyValue::CleanUpDataResponse::STATUS_ERROR, errorReason, actualGeneration, tabletId);
         }
     };
 
@@ -253,6 +254,6 @@ struct TEvKeyValue {
         {}
     };
 
-};
+}
 
 } // NKikimr