Merge pull request #669 from userlocalhost/enhancement/search_entry_chain/enable_backward_search_entries

Added API handler that searches Entries across multiple referral structure (/api/v1/search_chain)

Author: hinashi

CHANGELOG.md

@@ -3,6 +3,9 @@
 ## In development
 
 ### Added
+* Added API handler that searchs Entries across multiple referral structure
+  (/api/v1/search_chain)
+  Contributed by @userlocalhost, @hinashi
 
 ### Changed
 * Improved processing to get referred Entry from elasticsearch

api_v1/entry/serializer.py

@@ -4,6 +4,8 @@
 from entity.models import Entity, EntityAttr
 from entry.models import Entry
 
+SEARCH_ENTRY_LIMIT = 50
+
 
 class ReferSerializer(serializers.Serializer):
     entity = serializers.CharField(max_length=200)
@@ -12,12 +14,24 @@ class ReferSerializer(serializers.Serializer):
     attrs = serializers.ListField(required=False)
     refers = serializers.ListField(required=False)
 
+    def validate_entity(self, entity_name):
+        if not Entity.objects.filter(name=entity_name, is_active=True).exists():
+            raise ValidationError("There is no specified Entity (%s)" % entity_name)
+
+        return entity_name
+
+    def validate(self, data):
+        entity = Entity.objects.filter(name=data["entity"], is_active=True).first()
+        data["entity_id"] = entity.id
+        return data
+
 
 class AttrSerializer(serializers.Serializer):
     name = serializers.CharField(max_length=200)
     value = serializers.CharField(max_length=200, required=False, allow_blank=True)
     is_any = serializers.BooleanField(default=False)
     attrs = serializers.ListField(required=False)
+    refers = serializers.ListField(required=False)
 
     def validate_name(self, name):
         if not EntityAttr.objects.filter(name=name, is_active=True).exists():
@@ -31,8 +45,8 @@ def validate_is_any(self, value):
 
 class EntrySearchChainSerializer(serializers.Serializer):
     entities = serializers.ListField(child=serializers.CharField(max_length=200))
-    # conditions = serializers.ListField(child=SearchConditionSerializer())
-    conditions = serializers.ListField()
+    attrs = serializers.ListField(child=AttrSerializer(), required=False)
+    refers = serializers.ListField(child=ReferSerializer(), required=False)
     is_any = serializers.BooleanField(default=False)
 
     def validate_is_any(self, value):
@@ -95,14 +109,10 @@ def _get_serializer(condition):
             else:
                 return AttrSerializer
 
-        def _may_validate_and_complement_condition(condition, entities, serializer_hint=None):
-            serializer_class = serializer_hint
-            if serializer_hint is None:
-                serializer_class = _get_serializer(condition)
-
+        def _may_validate_and_complement_condition(condition, entities, serializer_class):
             serializer = serializer_class(data=condition)
             if not serializer.is_valid():
-                raise ValidationError("Invalid condition was specified" % str(condition))
+                raise ValidationError("Invalid condition(%s) was specified" % str(condition))
 
             if not entities:
                 raise ValidationError("Condition(%s) couldn't find valid Entities" % str(condition))
@@ -111,8 +121,12 @@ def _may_validate_and_complement_condition(condition, entities, serializer_hint=
             if "name" in validated_data:
                 _validate_attribute(validated_data["name"], entities)
 
-            # complement "entities" parameter at this condition
-            _complement_entities(validated_data, entities)
+            if isinstance(serializer, AttrSerializer):
+                # complement "entities" parameter at this condition
+                _complement_entities(validated_data, entities)
+
+            if isinstance(serializer, ReferSerializer):
+                validated_data["entities"] = [validated_data["entity_id"]]
 
             # call this method recursively to validate and complement value for each conditions
             if "attrs" in validated_data:
@@ -123,19 +137,32 @@ def _may_validate_and_complement_condition(condition, entities, serializer_hint=
                     for x in validated_data["attrs"]
                 ]
 
+            elif "refers" in validated_data:
+                validated_data["refers"] = [
+                    _may_validate_and_complement_condition(
+                        x, validated_data["entities"], ReferSerializer
+                    )
+                    for x in validated_data["refers"]
+                ]
+
             return validated_data
 
-        # validate and compelment "conditions" parameter context
-        data["conditions"] = [
-            _may_validate_and_complement_condition(x, data["entities"]) for x in data["conditions"]
-        ]
+        # validate parameter context
+        if data.get("attrs"):
+            data["attrs"] = [
+                _may_validate_and_complement_condition(x, data["entities"], AttrSerializer)
+                for x in data["attrs"]
+            ]
 
-        return data
+        if data.get("refers"):
+            data["refers"] = [
+                _may_validate_and_complement_condition(x, data["entities"], ReferSerializer)
+                for x in data["refers"]
+            ]
 
-    def search_entries(self, user, query=None):
-        if query is None:
-            query = self.validated_data
+        return data
 
+    def merge_search_result(self, stored_list, result_data, is_any):
         def _deduplication(item_list):
             """
             This removes duplication items, that have same Entry-ID with other ones,from item_list
@@ -147,92 +174,190 @@ def _deduplication(item_list):
 
             return returned_items
 
-        def _merge_search_result(stored_list, result_data, is_any):
-            if is_any:
-                # This is OR condition processing
-                result = result_data + stored_list
+        if is_any:
+            # This is OR condition processing
+            result = result_data + stored_list
 
-            else:
-                # This is AND condition processing
-                # The "stored_id_list" is an explanatory variable that only has Entry-ID
-                # of stored_list Entry information
+        else:
+            # This is AND condition processing
+            # The "stored_id_list" is an explanatory variable that only has Entry-ID
+            # of stored_list Entry information
+            if stored_list:
                 stored_id_list = [x["id"] for x in stored_list]
                 result = [x for x in result_data if x["id"] in stored_id_list]
+            else:
+                result = result_data
+
+        return _deduplication(result)
 
-            return _deduplication(result)
+    def backward_search_entries(self, user, queries, entity_id_list, is_any):
+        # digging into the condition tree to get to leaf condition by depth-first search
+        accumulated_result = []
 
-        sub_queries = None
-        if "conditions" in query:
-            sub_queries = query["conditions"]
+        def _do_backward_search(sub_query, sub_query_result):
+            # make query to search Entries using Entry.search_entries()
+            search_keyword = "|".join([x["name"] for x in sub_query_result])
+            if isinstance(sub_query.get("entry"), str) and len(sub_query["entry"]) > 0:
+                search_keyword = sub_query.get("entry")
+
+            # Query for forward search
+            query_params = {
+                "user": user,
+                "hint_entity_ids": entity_id_list,
+                "hint_referral": search_keyword,
+                "hint_referral_entity_id": sub_query["entity_id"],
+                "limit": 99999,
+            }
+
+            # get Entry informations from result
+            search_result = Entry.search_entries(**query_params)
+
+            return [x["entry"] for x in search_result["ret_values"]]
+
+        # This expects only AttrSerialized sub-query
+        for sub_query in queries:
+            (is_leaf, sub_query_result) = self.search_entries(user, sub_query)
+            if not is_leaf and not sub_query_result:
+                # In this case, it's useless to continue to search processing because
+                # there is no possiblity to find out data that user wants to.
+                return (False, [])
+
+            # This divides results into small chunks, that will be sent to the elasticsearch again
+            # when it has large amount of data. The size of each chunks is SEARCH_ENTRY_LIMIT
+            # at most.
+            dividing_index = 0
+            if len(sub_query_result) > 0:
+                search_results = []
+                while (dividing_index * SEARCH_ENTRY_LIMIT) < len(sub_query_result):
+                    chunk_result = sub_query_result[
+                        dividing_index
+                        * SEARCH_ENTRY_LIMIT : (dividing_index + 1)
+                        * SEARCH_ENTRY_LIMIT
+                    ]
+                    dividing_index += 1
+
+                    search_results += _do_backward_search(sub_query, chunk_result)
+
+            else:
+                # This search Entries with hint values from sub_query and sub_query_result
+                search_results = _do_backward_search(sub_query, sub_query_result)
 
-        elif "attrs" in query:
-            sub_queries = query["attrs"]
+            # merge result to the accumulated ones considering is_any value
+            accumulated_result = self.merge_search_result(
+                accumulated_result, search_results, is_any
+            )
 
-        elif "refers" in query:
-            raise RuntimeError("This is not impelemnted yet")
+        # The first return value (False) describe this result returned by NO-leaf-node
+        return (False, accumulated_result)
 
-        # digging into the conditions tree to get to leaf condition by depth-first search
+    def forward_search_entries(self, user, queries, entity_id_list, is_any):
+        # digging into the condition tree to get to leaf condition by depth-first search
         accumulated_result = []
-        if sub_queries:
-            # This expects only AttrSerialized sub-query
-            for sub_query in sub_queries:
-                (is_leaf, sub_query_result) = self.search_entries(user, sub_query)
-
-                if not is_leaf and not sub_query_result:
-                    # In this case, it's useless to continue to search processing because
-                    # there is no possiblity to find out data that user wants to.
-                    return (False, [])
-
-                # make query to search Entries using Entry.search_entries()
-                search_keyword = "|".join([x["name"] for x in sub_query_result])
-                if isinstance(sub_query.get("value"), str) and len(sub_query["value"]) > 0:
-                    search_keyword = sub_query.get("value")
-
-                elif sub_query.get("value") == "":
-                    # When value has empty string, this specify special character "\",
-                    # which will match Entries that refers nothing Entry at specified Attribute.
-                    search_keyword = "\\"
-
-                search_query = [
-                    {
-                        "name": sub_query["name"],
-                        "keyword": search_keyword,
-                    }
-                ]
 
-                # get Entry informations from result
-                search_result = Entry.search_entries(
-                    user, query["entities"], search_query, limit=99999
-                )
+        def _do_forward_search(sub_query, sub_query_result):
+            # make query to search Entries using Entry.search_entries()
+            search_keyword = "|".join([x["name"] for x in sub_query_result])
+            if isinstance(sub_query.get("value"), str) and len(sub_query["value"]) > 0:
+                search_keyword = sub_query.get("value")
+
+            elif sub_query.get("value") == "":
+                # When value has empty string, this specify special character "\",
+                # which will match Entries that refers nothing Entry at specified Attribute.
+                search_keyword = "\\"
+
+            # Query for forward search
+            hint_attrs = [
+                {
+                    "name": sub_query["name"],
+                    "keyword": search_keyword,
+                }
+            ]
+
+            # get Entry informations from result
+            search_result = Entry.search_entries(user, entity_id_list, hint_attrs, limit=99999)
+
+            return [x["entry"] for x in search_result["ret_values"]]
+
+        # This expects only AttrSerialized sub-query
+        for sub_query in queries:
+            (is_leaf, sub_query_result) = self.search_entries(user, sub_query)
+            if not is_leaf and not sub_query_result:
+                # In this case, it's useless to continue to search processing because
+                # there is no possiblity to find out data that user wants to.
+                return (False, [])
+
+            # This divides results into small chunks, that will be sent to the elasticsearch again
+            # when it has large amount of data. The size of each chunks is SEARCH_ENTRY_LIMIT
+            # at most.
+            dividing_index = 0
+            if len(sub_query_result) > 0:
+                search_results = []
+                while (dividing_index * SEARCH_ENTRY_LIMIT) < len(sub_query_result):
+                    chunk_result = sub_query_result[
+                        dividing_index
+                        * SEARCH_ENTRY_LIMIT : (dividing_index + 1)
+                        * SEARCH_ENTRY_LIMIT
+                    ]
+                    dividing_index += 1
+
+                    search_results += _do_forward_search(sub_query, chunk_result)
 
-                result_entry_info = [x["entry"] for x in search_result["ret_values"]]
-                if not accumulated_result:
-                    accumulated_result = result_entry_info
-                else:
-                    accumulated_result = _merge_search_result(
-                        accumulated_result, result_entry_info, query["is_any"]
-                    )
+            else:
+                # This search Entries with hint values from sub_query and sub_query_result
+                search_results = _do_forward_search(sub_query, sub_query_result)
 
-        else:
-            # In the leaf condition return nothing
-            # The first return value describe whethere this is leaf condition or not.
-            # (True means result is returned by leaf-node)
-            #
-            # Empty result of second returned value has different meaning depends on
-            # whethere that is leaf condition or intermediate one.
-            # * Leaf condition:
-            #   - it must return empty whatever condition.
-            #     it should continue processing
-            #
-            # * Intermediate one:
-            #   - it returns empty when there is no result.
-            #     it's useless to continue this processing because there is no possibility
-            #     to find out any data, which user wants to
-            return (True, [])
+            # merge current result to the accumulated ones considering is_any value
+            accumulated_result = self.merge_search_result(
+                accumulated_result, search_results, is_any
+            )
 
         # The first return value (False) describe this result returned by NO-leaf-node
         return (False, accumulated_result)
 
+    def search_entries(self, user, query=None):
+        if query is None:
+            query = self.validated_data
+
+        accumulated_result = []
+        is_leaf = True
+        if len(query.get("attrs", [])) > 0:
+            is_leaf = False
+            sub_query = query.get("attrs", [])
+
+            (_, results) = self.forward_search_entries(
+                user, sub_query, query["entities"], query["is_any"]
+            )
+            accumulated_result = self.merge_search_result(
+                accumulated_result, results, query["is_any"]
+            )
+
+        if len(query.get("refers", [])) > 0:
+            is_leaf = False
+            sub_query = query.get("refers", [])
+
+            (_, results) = self.backward_search_entries(
+                user, sub_query, query["entities"], query["is_any"]
+            )
+            accumulated_result = self.merge_search_result(
+                accumulated_result, results, query["is_any"]
+            )
+
+        # In the leaf condition return nothing
+        # The first return value describe whethere this is leaf condition or not.
+        # (True means result is returned by leaf-node)
+        #
+        # Empty result of second returned value has different meaning depends on
+        # whethere that is leaf condition or intermediate one.
+        # * Leaf condition:
+        #   - it must return empty whatever condition.
+        #     it should continue processing
+        #
+        # * Intermediate one:
+        #   - it returns empty when there is no result.
+        #     it's useless to continue this processing because there is no possibility
+        #     to find out any data, which user wants to
+        return (is_leaf, accumulated_result)
+
     def is_attr_chained(self, entry, attrs=None, is_any=False):
         if not attrs:
             attrs = self.validated_data["attrs"]