Merge pull request #488 from maykinmedia/feature/472-data-attr-contains-comma

support commas in data_attrs query param

Author: annashamray

.github/workflows/oas-check.yml

@@ -12,13 +12,8 @@ on:
 jobs:
   open-api-workflow-check-oas:
     uses: maykinmedia/open-api-workflows/.github/workflows/oas-check.yml@v1
-    strategy:
-      matrix:
-        version:
-          - v2
     with:
-      schema-path: 'src/objects/api/${{ matrix.version }}/openapi.yaml'
-      schema-options: "--api-version ${{ matrix.version }}"
+      schema-path: 'src/objects/api/v2/openapi.yaml'
       python-version: '3.11'
       django-settings-module: 'objects.conf.ci'
       apt-packages: 'libgdal-dev gdal-bin'

src/objects/api/v2/filters.py

@@ -1,18 +1,99 @@
 from datetime import date as date_
 
 from django import forms
+from django.db.models import QuerySet
 from django.utils.translation import gettext_lazy as _
 
 from django_filters import filters
 from rest_framework import serializers
 from vng_api_common.filtersets import FilterSet
 
 from objects.core.models import ObjectRecord, ObjectType
-from objects.utils.filters import ObjectTypeFilter
+from objects.utils.filters import ManyCharFilter, ObjectTypeFilter
 
 from ..constants import Operators
 from ..utils import display_choice_values_for_help_text, string_to_value
-from ..validators import validate_data_attrs
+from ..validators import validate_data_attr, validate_data_attrs
+
+DATA_ATTR_VALUE_HELP_TEXT = f"""A valid parameter value has the form `key__operator__value`.
+`key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+Valid operator values are:
+{display_choice_values_for_help_text(Operators)}
+
+`value` may not contain double underscore or comma characters.
+`key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+"""
+
+DATA_ATTRS_HELP_TEXT = (
+    _(
+        """**DEPRECATED: Use 'data_attr' instead**.
+Only include objects that have attributes with certain values.
+Data filtering expressions are comma-separated and are structured as follows:
+
+%(value_part_help_text)s
+
+Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
+should be used. If `height` is nested inside `dimensions` attribute, query should look like
+`data_attrs=dimensions__height__exact__100`
+
+`value` may not contain comma, since commas are used as separator between filtering expressions.
+If you want to use commas in `value` you can use `data_attr` query parameter.
+"""
+    )
+    % {"value_part_help_text": DATA_ATTR_VALUE_HELP_TEXT}
+)
+
+DATA_ATTR_HELP_TEXT = (
+    _(
+        """Only include objects that have attributes with certain values.
+
+%(value_part_help_text)s
+
+Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+should be used. If `height` is nested inside `dimensions` attribute, query should look like
+`data_attr=dimensions__height__exact__100`
+
+This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+* `value` may contain commas
+* only one filtering expression is allowed
+
+If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
+"""
+    )
+    % {"value_part_help_text": DATA_ATTR_VALUE_HELP_TEXT}
+)
+
+
+def filter_data_attr_value_part(value_part: str, queryset: QuerySet) -> QuerySet:
+    """
+    filter one value part for data_attr and data_attrs filters
+    """
+    variable, operator, str_value = value_part.rsplit("__", 2)
+    real_value = string_to_value(str_value)
+
+    if operator == "exact":
+        #  for exact operator try to filter on string and numeric values
+        in_vals = [str_value]
+        if real_value != str_value:
+            in_vals.append(real_value)
+        queryset = queryset.filter(**{f"data__{variable}__in": in_vals})
+    elif operator == "icontains":
+        # icontains treats everything like strings
+        queryset = queryset.filter(**{f"data__{variable}__icontains": str_value})
+    elif operator == "in":
+        # in must be a list
+        values = str_value.split("|")
+        queryset = queryset.filter(**{f"data__{variable}__in": values})
+
+    else:
+        # gt, gte, lt, lte operators
+        queryset = queryset.filter(**{f"data__{variable}__{operator}": real_value})
+    return queryset
 
 
 class ObjectRecordFilterForm(forms.Form):
@@ -58,67 +139,40 @@ class ObjectRecordFilterSet(FilterSet):
             "date would be between `registrationAt` attributes of different records"
         ),
     )
+
     data_attrs = filters.CharFilter(
         method="filter_data_attrs",
         validators=[validate_data_attrs],
-        help_text=_(
-            """Only include objects that have attributes with certain values.
-Data filtering expressions are comma-separated and are structured as follows:
-A valid parameter value has the form `key__operator__value`.
-`key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
-Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
-
-Valid operator values are:
-%(operator_choices)s
-
-`value` may not contain double underscore or comma characters.
-`key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+        help_text=DATA_ATTRS_HELP_TEXT,
+    )
 
-Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
-should be used. If `height` is nested inside `dimensions` attribute, query should look like
-`data_attrs=dimensions__height__exact__100`
-"""
-        )
-        % {"operator_choices": display_choice_values_for_help_text(Operators)},
+    data_attr = ManyCharFilter(
+        method="filter_data_attr",
+        validators=[validate_data_attr],
+        help_text=DATA_ATTR_HELP_TEXT,
     )
+
     data_icontains = filters.CharFilter(
         method="filter_data_icontains",
         help_text=_("Search in all `data` values of string properties."),
     )
 
     class Meta:
         model = ObjectRecord
-        fields = ("type", "data_attrs", "date", "registrationDate")
+        fields = ("type", "data_attrs", "data_attr", "date", "registrationDate")
         form = ObjectRecordFilterForm
 
     def filter_data_attrs(self, queryset, name, value: str):
         parts = value.split(",")
 
         for value_part in parts:
-            variable, operator, str_value = value_part.rsplit("__", 2)
-            real_value = string_to_value(str_value)
-
-            if operator == "exact":
-                #  for exact operator try to filter on string and numeric values
-                in_vals = [str_value]
-                if real_value != value:
-                    in_vals.append(real_value)
-                queryset = queryset.filter(**{f"data__{variable}__in": in_vals})
-            elif operator == "icontains":
-                # icontains treats everything like strings
-                queryset = queryset.filter(
-                    **{f"data__{variable}__icontains": str_value}
-                )
-            elif operator == "in":
-                # in must be a list
-                values = str_value.split("|")
-                queryset = queryset.filter(**{f"data__{variable}__in": values})
-
-            else:
-                # gt, gte, lt, lte operators
-                queryset = queryset.filter(
-                    **{f"data__{variable}__{operator}": real_value}
-                )
+            queryset = filter_data_attr_value_part(value_part, queryset)
+
+        return queryset
+
+    def filter_data_attr(self, queryset, name, value: list):
+        for value_part in value:
+            queryset = filter_data_attr_value_part(value_part, queryset)
 
         return queryset
 

src/objects/api/v2/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: Objects API
-  version: 2.4.3 (v2)
+  version: 2.4.3
   description: |
     An API to manage Objects.
 
@@ -88,13 +88,52 @@ paths:
         description: 'The desired ''Coordinate Reference System'' (CRS) of the response
           data. According to the GeoJSON spec, WGS84 is the default (EPSG: 4326 is
           the same as WGS84).'
+      - in: query
+        name: data_attr
+        schema:
+          type: string
+        description: |
+          Only include objects that have attributes with certain values.
+
+          A valid parameter value has the form `key__operator__value`.
+          `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+          Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+          Valid operator values are:
+          * `exact` - equal to
+          * `gt` - greater than
+          * `gte` - greater than or equal to
+          * `lt` - lower than
+          * `lte` - lower than or equal to
+          * `icontains` - case-insensitive partial match
+          * `in` - in a list of values separated by `|`
+
+          `value` may not contain double underscore or comma characters.
+          `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+
+
+          Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+          should be used. If `height` is nested inside `dimensions` attribute, query should look like
+          `data_attr=dimensions__height__exact__100`
+
+          This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+          * `value` may contain commas
+          * only one filtering expression is allowed
+
+          If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+          Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
+        explode: true
       - in: query
         name: data_attrs
         schema:
           type: string
         description: |
+          **DEPRECATED: Use 'data_attr' instead**.
           Only include objects that have attributes with certain values.
           Data filtering expressions are comma-separated and are structured as follows:
+
           A valid parameter value has the form `key__operator__value`.
           `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
           Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
@@ -111,9 +150,15 @@ paths:
           `value` may not contain double underscore or comma characters.
           `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
 
+
+
           Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
           should be used. If `height` is nested inside `dimensions` attribute, query should look like
           `data_attrs=dimensions__height__exact__100`
+
+          `value` may not contain comma, since commas are used as separator between filtering expressions.
+          If you want to use commas in `value` you can use `data_attr` query parameter.
+        deprecated: true
       - in: query
         name: data_icontains
         schema:
@@ -620,8 +665,10 @@ paths:
                   data_attrs:
                     type: string
                     description: |
+                      **DEPRECATED: Use 'data_attr' instead**.
                       Only include objects that have attributes with certain values.
                       Data filtering expressions are comma-separated and are structured as follows:
+
                       A valid parameter value has the form `key__operator__value`.
                       `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
                       Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
@@ -638,9 +685,48 @@ paths:
                       `value` may not contain double underscore or comma characters.
                       `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
 
+
+
                       Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
                       should be used. If `height` is nested inside `dimensions` attribute, query should look like
                       `data_attrs=dimensions__height__exact__100`
+
+                      `value` may not contain comma, since commas are used as separator between filtering expressions.
+                      If you want to use commas in `value` you can use `data_attr` query parameter.
+                  data_attr:
+                    type: string
+                    description: |
+                      Only include objects that have attributes with certain values.
+
+                      A valid parameter value has the form `key__operator__value`.
+                      `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+                      Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+                      Valid operator values are:
+                      * `exact` - equal to
+                      * `gt` - greater than
+                      * `gte` - greater than or equal to
+                      * `lt` - lower than
+                      * `lte` - lower than or equal to
+                      * `icontains` - case-insensitive partial match
+                      * `in` - in a list of values separated by `|`
+
+                      `value` may not contain double underscore or comma characters.
+                      `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+
+
+                      Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+                      should be used. If `height` is nested inside `dimensions` attribute, query should look like
+                      `data_attr=dimensions__height__exact__100`
+
+                      This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+                      * `value` may contain commas
+                      * only one filtering expression is allowed
+
+                      If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+                      Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
                   date:
                     type: string
                     format: date

src/objects/api/v2/views.py

@@ -4,8 +4,12 @@
 from django.db import models
 from django.utils.dateparse import parse_date
 
-from drf_spectacular.types import OpenApiTypes
-from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
+from drf_spectacular.utils import (
+    OpenApiParameter,
+    OpenApiTypes,
+    extend_schema,
+    extend_schema_view,
+)
 from rest_framework import mixins, viewsets
 from rest_framework.decorators import action
 from rest_framework.generics import get_object_or_404
@@ -28,13 +32,32 @@
     PermissionSerializer,
 )
 from ..utils import is_date
-from .filters import ObjectRecordFilterSet
+from .filters import DATA_ATTR_HELP_TEXT, DATA_ATTRS_HELP_TEXT, ObjectRecordFilterSet
+
+# manually override OAS because of "deprecated" attribute
+data_attrs_parameter = OpenApiParameter(
+    name="data_attrs",
+    type=OpenApiTypes.STR,
+    location=OpenApiParameter.QUERY,
+    description=DATA_ATTRS_HELP_TEXT,
+    deprecated=True,
+)
+
+# manually override OAS because of "explode" attribute
+data_attr_parameter = OpenApiParameter(
+    name="data_attr",
+    location=OpenApiParameter.QUERY,
+    type=OpenApiTypes.STR,
+    description=DATA_ATTR_HELP_TEXT,
+    explode=True,
+)
 
 
 @extend_schema_view(
     list=extend_schema(
         description="Retrieve a list of OBJECTs and their actual RECORD. "
-        "The actual record is defined as if the query parameter `date=<today>` was given."
+        "The actual record is defined as if the query parameter `date=<today>` was given.",
+        parameters=[data_attrs_parameter, data_attr_parameter],
     ),
     retrieve=extend_schema(
         description="Retrieve a single OBJECT and its actual RECORD. "