Intermediate changes

x-stable-origin-commit: 8b96eef194d7fb3b315816b97322e8dd90bf3d94

Author: dcherednik

.github/workflows/docs_release.yaml

@@ -491,8 +491,11 @@ namespace NActors {
                     s << ", errorReason# " << errorReason;
                     errorCallback(s.Str());
                 }
-            } else {
+            } else if (proto.HasVersionTag()) {
                 ValidateVersionTag(proto, std::forward<TCallback>(errorCallback));
+            } else {
+                LOG_LOG_IC_X(NActorsServices::INTERCONNECT, "ICH09", NLog::PRI_WARN,
+                    "Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default");
             }
         }
 

library/cpp/actors/interconnect/interconnect_handshake.cpp

@@ -915,8 +915,6 @@ class TRopeArena {
 
     TAllocateCallback Allocator;
     TRope Arena;
-    size_t Size = 0;
-    THashSet<const void*> AccountedBuffers;
 
 public:
     TRopeArena(TAllocateCallback&& allocator)
@@ -949,16 +947,6 @@ class TRopeArena {
 
         return res;
     }
-
-    size_t GetSize() const {
-        return Size;
-    }
-
-    void AccountChunk(const TRcBuf& chunk) {
-        if (AccountedBuffers.insert(chunk.Backend.UniqueId()).second) {
-            Size += chunk.GetOccupiedMemorySize();
-        }
-    }
 };
 
 struct TRopeUtils {
@@ -1131,9 +1119,6 @@ inline TRope TRope::CopySpaceOptimized(TRope&& origin, size_t worstRatioPer1k, T
     }
     res.Size = origin.Size;
     origin = TRope();
-    for (const TRcBuf& chunk : res.Chain) {
-        arena.AccountChunk(chunk);
-    }
     return res;
 }
 

library/cpp/actors/util/rope.h

@@ -131,6 +131,7 @@ struct TGrpcStatus {
     TString Details;
     int GRpcStatusCode;
     bool InternalError;
+    std::multimap<TString, TString> ServerTrailingMetadata;
 
     TGrpcStatus()
         : GRpcStatusCode(grpc::StatusCode::OK)
@@ -800,6 +801,11 @@ class TStreamRequestReadProcessor
         } else if (readCallback) {
             if (status.Ok()) {
                 status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF");
+                for (const auto& [name, value] : Context.GetServerTrailingMetadata()) {
+                    status.ServerTrailingMetadata.emplace(
+                        TString(name.begin(), name.end()),
+                        TString(value.begin(), value.end()));
+                }
             }
             readCallback(std::move(status));
         } else if (finishCallback) {
@@ -1196,6 +1202,11 @@ class TStreamRequestReadWriteProcessor
         } else if (readCallback) {
             if (status.Ok()) {
                 status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF");
+                for (const auto& [name, value] : Context.GetServerTrailingMetadata()) {
+                    status.ServerTrailingMetadata.emplace(
+                        TString(name.begin(), name.end()),
+                        TString(value.begin(), value.end()));
+                }
             }
             readCallback(std::move(status));
         } else if (finishCallback) {

library/cpp/grpc/client/grpc_client_low.h

@@ -258,6 +258,10 @@ Json2SingleField(const NJson::TJsonValue& json,
 
     const NJson::TJsonValue& fieldJson = name ? json[name] : json;
 
+    if (name && config.UnknownFieldsCollector) {
+        config.UnknownFieldsCollector->OnEnterMapItem(name);
+    }
+
     switch (field.cpp_type()) {
         JSON_TO_FIELD(CPPTYPE_INT32, field.name(), fieldJson, IsInteger, SetInt32, GetInteger);
         JSON_TO_FIELD(CPPTYPE_INT64, field.name(), fieldJson, IsInteger, SetInt64, GetInteger);
@@ -295,6 +299,10 @@ Json2SingleField(const NJson::TJsonValue& json,
             ythrow yexception() << "Unknown protobuf field type: "
                                 << static_cast<int>(field.cpp_type()) << ".";
     }
+
+    if (name && config.UnknownFieldsCollector) {
+        config.UnknownFieldsCollector->OnLeaveMapItem();
+    }
 }
 
 static void
@@ -397,6 +405,10 @@ Json2RepeatedField(const NJson::TJsonValue& json,
     if (fieldJson.GetType() == NJson::JSON_UNDEFINED || fieldJson.GetType() == NJson::JSON_NULL)
         return;
 
+    if (config.UnknownFieldsCollector) {
+        config.UnknownFieldsCollector->OnEnterMapItem(name);
+    }
+
     bool isMap = fieldJson.GetType() == NJson::JSON_MAP;
     if (isMap) {
         if (!config.MapAsObject) {
@@ -421,25 +433,51 @@ Json2RepeatedField(const NJson::TJsonValue& json,
         for (const auto& x : jsonMap) {
             const TString& key = x.first;
             const NJson::TJsonValue& jsonValue = x.second;
+            if (config.UnknownFieldsCollector) {
+                config.UnknownFieldsCollector->OnEnterMapItem(key);
+            }
             Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection, key);
+            if (config.UnknownFieldsCollector) {
+                config.UnknownFieldsCollector->OnLeaveMapItem();
+            }
         }
     } else {
         if (config.ReplaceRepeatedFields) {
             reflection->ClearField(&proto, &field);
         }
         if (fieldJson.GetType() == NJson::JSON_ARRAY) {
             const NJson::TJsonValue::TArray& jsonArray = fieldJson.GetArray();
+            ui64 id = 0;
             for (const NJson::TJsonValue& jsonValue : jsonArray) {
+                if (config.UnknownFieldsCollector) {
+                    config.UnknownFieldsCollector->OnEnterArrayItem(id);
+                }
                 Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection);
+                if (config.UnknownFieldsCollector) {
+                    config.UnknownFieldsCollector->OnLeaveArrayItem();
+                }
+                ++id;
             }
         } else if (config.ValueVectorizer) {
+            ui64 id = 0;
             for (const NJson::TJsonValue& jsonValue : config.ValueVectorizer(fieldJson)) {
+                if (config.UnknownFieldsCollector) {
+                    config.UnknownFieldsCollector->OnEnterArrayItem(id);
+                }
                 Json2RepeatedFieldValue(jsonValue, proto, field, config, reflection);
+                if (config.UnknownFieldsCollector) {
+                    config.UnknownFieldsCollector->OnLeaveArrayItem();
+                }
+                ++id;
             }
         } else if (config.VectorizeScalars) {
             Json2RepeatedFieldValue(fieldJson, proto, field, config, reflection);
         }
     }
+
+    if (config.UnknownFieldsCollector) {
+        config.UnknownFieldsCollector->OnLeaveMapItem();
+    }
 }
 
 namespace NProtobufJson {
@@ -463,14 +501,18 @@ namespace NProtobufJson {
             }
         }
 
-        if (!config.AllowUnknownFields) {
+        if (!config.AllowUnknownFields || config.UnknownFieldsCollector) {
             THashMap<TString, bool> knownFields;
             for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) {
                 const google::protobuf::FieldDescriptor* field = descriptor->field(f);
                 knownFields[GetFieldName(*field, config)] = 1;
             }
             for (const auto& f : json.GetMap()) {
-                Y_ENSURE(knownFields.contains(f.first), "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\"");
+                const bool isFieldKnown = knownFields.contains(f.first);
+                Y_ENSURE(config.AllowUnknownFields || isFieldKnown, "unknown field \"" << f.first << "\" for \"" << descriptor->full_name() << "\"");
+                if (!isFieldKnown) {
+                    config.UnknownFieldsCollector->OnUnknownField(f.first, *descriptor);
+                }
             }
         }
     }

library/cpp/protobuf/json/json2proto.cpp

@@ -2,10 +2,12 @@
 
 #include "string_transform.h"
 #include "name_generator.h"
+#include "unknown_fields_collector.h"
 
 #include <library/cpp/json/json_reader.h>
 #include <library/cpp/json/json_value.h>
 
+#include <util/generic/ptr.h>
 #include <util/stream/input.h>
 #include <util/stream/str.h>
 #include <util/stream/mem.h>
@@ -108,6 +110,11 @@ namespace NProtobufJson {
             return *this;
         }
 
+        TSelf& SetUnknownFieldsCollector(TSimpleSharedPtr<IUnknownFieldsCollector> value) {
+            UnknownFieldsCollector = std::move(value);
+            return *this;
+        }
+
         FldNameMode FieldNameMode = FieldNameOriginalCase;
         bool AllowUnknownFields = true;
 
@@ -152,6 +159,9 @@ namespace NProtobufJson {
 
         /// Allow nonstandard conversions, e.g. google.protobuf.Duration from String
         bool AllowString2TimeConversion = false;
+
+        /// Stores information about unknown fields
+        TSimpleSharedPtr<IUnknownFieldsCollector> UnknownFieldsCollector = nullptr;
     };
 
     /// @throw yexception

library/cpp/protobuf/json/json2proto.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+namespace google {
+    namespace protobuf {
+        class FieldDescriptor;
+        class Descriptor;
+    }
+}
+
+namespace NProtobufJson {
+    /*  Methods OnEnter.../OnLeave... are called on every field of structure
+     *  during traverse and should be used to build context
+     *  Method OnUnknownField are called every time when field which can't
+     *  be mapped
+     */
+    struct IUnknownFieldsCollector {
+        virtual ~IUnknownFieldsCollector() = default;
+
+        virtual void OnEnterMapItem(const TString& key) = 0;
+        virtual void OnLeaveMapItem() = 0;
+
+        virtual void OnEnterArrayItem(ui64 id) = 0;
+        virtual void OnLeaveArrayItem() = 0;
+
+        virtual void OnUnknownField(const TString& key, const google::protobuf::Descriptor& value) = 0;
+    };
+}