Cleaner/Faster Partial Reading (#1502)

* Cleaner/Faster Partial Reading

* Updating documentation

* non-null buffer support for partial reading with missing fields

* Update partial-read.md

* nested array partial read test

Author: stephenberry

docs/partial-read.md

@@ -2,18 +2,17 @@
 
 At times it is not necessary to read the entire JSON document, but rather just a header or some of the initial fields. Glaze provides multiple solutions:
 
--  `partial_read` in `glz::opts`
 -  `partial_read` in `glz::meta`
-- Partial reading via [JSON Pointer syntax](./json-pointer-syntax.md)
-
-> [!NOTE]
->
-> If you wish partial reading to work on nested objects, you must also turn on `.partial_read_nested = true` in `glz::opts`.
+-  Partial reading via [JSON Pointer syntax](./json-pointer-syntax.md)
 
 # Partial reading with glz::opts
 
 `partial_read` is a compile time flag in `glz::opts` that indicates only existing array and object elements should be read into, and once the memory has been read, parsing returns without iterating through the rest of the document.
 
+> [!NOTE]
+>
+> Once any sub-object is read, the parsing will finish. This ensures high performance with short circuiting.
+
 > A [wrapper](./wrappers.md) by the same name also exists.
 
 Example: read only the first two elements into a `std::tuple`
@@ -47,7 +46,7 @@ expect(obj.size() == 1);
 expect(obj.at("2") = 2);
 ```
 
-Example: read only the fields present in a struct
+Example: read only the fields present in a struct and then short circuit the parse
 
 ```c++
 struct partial_struct
@@ -65,51 +64,6 @@ expect(obj.string == "ha!");
 expect(obj.integer == 400);
 ```
 
-# Partial reading with glz::meta
-
-Glaze allows a `partial_read` flag that can be set to `true` within the glaze metadata.
-
-```json
-{
-  "id": "187d3cb2-942d-484c-8271-4e2141bbadb1",
-  "type": "message_type"
-            .....
-  // the rest of the large JSON
-}
-```
-
-When `partial_read` is `true`, parsing will end once all the keys defined in the struct have been parsed.
-
-## partial_read_nested
-
-If your object that you wish to only read part of is nested within other objects, set `partial_read_nested = true` so that Glaze will properly parse the parent objects by skipping to the end of the partially read object.
-
-## Example
-
-```c++
-struct Header
-{
-   std::string id{};
-   std::string type{};
-};
-
-template <>
-struct glz::meta<Header>
-{
-   static constexpr auto partial_read = true;
-};
-```
-
-```c++
-Header h{};
-std::string buf = R"({"id":"51e2affb","type":"message_type","unknown key":"value"})";
-
-expect(glz::read_json(h, buf) == glz::error_code::none);
-// Glaze will stop reading after "type" is parsed
-expect(h.id == "51e2affb");
-expect(h.type == "message_type");
-```
-
 ## Unit Test Examples
 
 ```c++
@@ -119,10 +73,10 @@ struct Header
    std::string type{};
 };
 
-template <>
-struct glz::meta<Header>
+struct HeaderFlipped
 {
-   static constexpr auto partial_read = true;
+   std::string type{};
+   std::string id{};
 };
 
 struct NestedPartialRead
@@ -133,23 +87,25 @@ struct NestedPartialRead
 };
 
 suite partial_read_tests = [] {
-   using namespace boost::ut;
+   using namespace ut;
+   
+   static constexpr glz::opts partial_read{.partial_read = true};
 
    "partial read"_test = [] {
       Header h{};
       std::string buf = R"({"id":"51e2affb","type":"message_type","unknown key":"value"})";
 
-      expect(glz::read_json(h, buf) == glz::error_code::none);
+      expect(!glz::read<partial_read>(h, buf));
       expect(h.id == "51e2affb");
       expect(h.type == "message_type");
    };
 
    "partial read 2"_test = [] {
       Header h{};
       // closing curly bracket is missing
-      std::string buf = R"({"id":"51e2affb","type":"message_type","unknown key":"value"})";
+      std::string buf = R"({"id":"51e2affb","type":"message_type","unknown key":"value")";
 
-      expect(glz::read_json(h, buf) == glz::error_code::none);
+      expect(!glz::read<partial_read>(h, buf));
       expect(h.id == "51e2affb");
       expect(h.type == "message_type");
    };
@@ -158,7 +114,7 @@ suite partial_read_tests = [] {
       Header h{};
       std::string buf = R"({"id":"51e2affb","unknown key":"value","type":"message_type"})";
 
-      expect(glz::read_json(h, buf) == glz::error_code::unknown_key);
+      expect(glz::read<partial_read>(h, buf) == glz::error_code::unknown_key);
       expect(h.id == "51e2affb");
       expect(h.type.empty());
    };
@@ -167,7 +123,16 @@ suite partial_read_tests = [] {
       Header h{};
       std::string buf = R"({"id":"51e2affb","unknown key":"value","type":"message_type"})";
 
-      expect(glz::read<glz::opts{.error_on_unknown_keys = false}>(h, buf) == glz::error_code::none);
+      expect(!glz::read<glz::opts{.error_on_unknown_keys = false, .partial_read = true}>(h, buf));
+      expect(h.id == "51e2affb");
+      expect(h.type == "message_type");
+   };
+
+   "partial read don't read garbage"_test = [] {
+      Header h{};
+      std::string buf = R"({"id":"51e2affb","unknown key":"value","type":"message_type"garbage})";
+
+      expect(!glz::read<glz::opts{.error_on_unknown_keys = false, .partial_read = true}>(h, buf));
       expect(h.id == "51e2affb");
       expect(h.type == "message_type");
    };
@@ -176,46 +141,75 @@ suite partial_read_tests = [] {
       Header h{};
       std::string buf = R"({"id":"51e2affb","unknown key":"value"})";
 
-      expect(glz::read<glz::opts{.error_on_unknown_keys = false}>(h, buf) != glz::error_code::missing_key);
+      expect(glz::read<glz::opts{.error_on_unknown_keys = false, .partial_read = true}>(h, buf) != glz::error_code::missing_key);
       expect(h.id == "51e2affb");
       expect(h.type.empty());
    };
 
    "partial read missing key 2"_test = [] {
       Header h{};
-      std::string buf = R"({"id":"51e2affb",""unknown key":"value"})";
+      std::string buf = R"({"id":"51e2affb","unknown key":"value"})";
 
-      expect(glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = false}>(h, buf) !=
-             glz::error_code::none);
+      expect(!glz::read<glz::opts{.error_on_unknown_keys = false, .partial_read = true}>(h, buf));
       expect(h.id == "51e2affb");
       expect(h.type.empty());
    };
+
+   "partial read HeaderFlipped"_test = [] {
+      HeaderFlipped h{};
+      std::string buf = R"({"id":"51e2affb","type":"message_type","unknown key":"value"})";
+
+      expect(not glz::read<partial_read>(h, buf));
+      expect(h.id == "51e2affb");
+      expect(h.type == "message_type");
+   };
+
+   "partial read HeaderFlipped unknown key"_test = [] {
+      HeaderFlipped h{};
+      std::string buf = R"({"id":"51e2affb","unknown key":"value","type":"message_type"})";
+
+      expect(glz::read<partial_read>(h, buf) == glz::error_code::unknown_key);
+      expect(h.id == "51e2affb");
+      expect(h.type.empty());
+   };
+
+   "partial read unknown key 2 HeaderFlipped"_test = [] {
+      HeaderFlipped h{};
+      std::string buf = R"({"id":"51e2affb","unknown key":"value","type":"message_type","another_field":409845})";
+
+      expect(glz::read<glz::opts{.error_on_unknown_keys = false, .partial_read = true}>(h, buf) == glz::error_code::none);
+      expect(h.id == "51e2affb");
+      expect(h.type == "message_type");
+   };
 };
 
 suite nested_partial_read_tests = [] {
-   using namespace boost::ut;
+   using namespace ut;
+   
+   static constexpr glz::opts partial_read{.partial_read = true};
 
    "nested object partial read"_test = [] {
       NestedPartialRead n{};
       std::string buf =
          R"({"method":"m1","header":{"id":"51e2affb","type":"message_type","unknown key":"value"},"number":51})";
 
-      expect(glz::read_json(n, buf) == glz::error_code::unknown_key);
+      expect(not glz::read<partial_read>(n, buf));
       expect(n.method == "m1");
       expect(n.header.id == "51e2affb");
       expect(n.header.type == "message_type");
       expect(n.number == 0);
    };
 
-   "nested object partial read 2"_test = [] {
+   "nested object partial read, don't read garbage"_test = [] {
       NestedPartialRead n{};
       std::string buf =
-         R"({"method":"m1","header":{"id":"51e2affb","type":"message_type","unknown key":"value"},"number":51})";
+         R"({"method":"m1","header":{"id":"51e2affb","type":"message_type","unknown key":"value",garbage},"number":51})";
 
-      expect(glz::read<glz::opts{.partial_read_nested = true}>(n, buf) == glz::error_code::none);
+      expect(not glz::read<partial_read>(n, buf));
       expect(n.method == "m1");
       expect(n.header.id == "51e2affb");
       expect(n.header.type == "message_type");
+      expect(n.number == 0);
    };
 };
 ```

include/glaze/beve/read.hpp

@@ -1290,7 +1290,7 @@ namespace glz
             }();
 
             decltype(auto) fields = [&]() -> decltype(auto) {
-               if constexpr (is_partial_read<T> || Opts.partial_read) {
+               if constexpr (Opts.partial_read) {
                   return bit_array<N>{};
                }
                else {
@@ -1304,7 +1304,7 @@ namespace glz
             }
 
             for (size_t i = 0; i < n_keys; ++i) {
-               if constexpr (is_partial_read<T> || Opts.partial_read) {
+               if constexpr (Opts.partial_read) {
                   if ((all_fields & fields) == all_fields) {
                      return;
                   }
@@ -1325,7 +1325,7 @@ namespace glz
                   const auto index = decode_hash_with_size<BEVE, T, HashInfo, HashInfo.type>::op(it, end, n);
 
                   if (index < N) [[likely]] {
-                     if constexpr (is_partial_read<T> || Opts.partial_read) {
+                     if constexpr (Opts.partial_read) {
                         fields[index] = true;
                      }
 

include/glaze/core/common.hpp

@@ -587,6 +587,7 @@ struct glz::meta<glz::error_code>
                                     "send_error",
                                     "connection_failure",
                                     "end_reached",
+                                    "partial_read_complete",
                                     "no_read_input",
                                     "data_must_be_null_terminated",
                                     "parse_number_failure",
@@ -650,6 +651,7 @@ struct glz::meta<glz::error_code>
                                      send_error, //
                                      connection_failure, //
                                      end_reached, // A non-error code for non-null terminated input buffers
+                                     partial_read_complete,
                                      no_read_input, //
                                      data_must_be_null_terminated, //
                                      parse_number_failure, //

include/glaze/core/context.hpp

@@ -26,6 +26,7 @@ namespace glz
       connection_failure,
       // Other errors
       end_reached, // A non-error code for non-null terminated input buffers
+      partial_read_complete, // A non-error code for short circuiting partial reads
       no_read_input, //
       data_must_be_null_terminated, //
       parse_number_failure, //

include/glaze/core/meta.hpp

@@ -317,7 +317,4 @@ namespace glz
 
    template <class T>
    concept custom_write = requires { requires meta<T>::custom_write == true; };
-
-   template <class T>
-   concept is_partial_read = requires { requires meta<T>::partial_read == true; };
 }

include/glaze/core/opts.hpp

@@ -93,7 +93,6 @@ namespace glz
          false; // Reads into only existing fields and elements and then exits without parsing the rest of the input
 
       // glaze_object_t concepts
-      bool_t partial_read_nested = false; // Advance the partially read struct to the end of the struct
       bool_t concatenate = true; // Concatenates ranges of std::pair into single objects when writing
 
       bool_t hide_non_invocable =

include/glaze/core/read.hpp

@@ -83,10 +83,12 @@ namespace glz
       }
 
    finish:
-      if constexpr (Opts.partial_read_nested || Opts.partial_read) {
-         // We don't do depth validation for partial reading
-         // This end_reached condition is set for valid end points in parsing
-         if (ctx.error == error_code::end_reached) {
+      // We don't do depth validation for partial reading
+      if constexpr (Opts.partial_read) {
+         if (ctx.error == error_code::partial_read_complete) [[likely]] {
+            ctx.error = error_code::none;
+         }
+         else if (ctx.error == error_code::end_reached && ctx.indentation_level == 0) {
             ctx.error = error_code::none;
          }
       }