Merge pull request #557 from syucream/feature/entry-export-apiv2

Implement entry export API V2

airone/lib/test.py

@@ -135,8 +135,8 @@ def _do_login(self, uname, is_superuser=False):
     def admin_login(self):
         return self._do_login("admin", True)
 
-    def guest_login(self):
-        return self._do_login("guest")
+    def guest_login(self, uname="guest"):
+        return self._do_login(uname)
 
     def open_fixture_file(self, fname):
         test_file_path = inspect.getfile(self.__class__)

entry/api_v2/serializers.py

@@ -458,3 +458,12 @@ class GetEntrySimpleSerializer(serializers.ModelSerializer):
     class Meta:
         model = Entry
         fields = ("id", "name")
+
+
+class EntryExportSerializer(serializers.Serializer):
+    format = serializers.CharField(default="yaml")
+
+    def validate_format(self, data: str) -> str:
+        if data.lower() == "csv":
+            return "csv"
+        return "yaml"

entry/api_v2/urls.py

@@ -45,5 +45,9 @@
             }
         ),
     ),
+    path(
+        "<int:entity_id>/export/",
+        views.EntryExportAPI.as_view(),
+    ),
     path("advanced_search/", views.AdvancedSearchAPI.as_view()),
 ]

entry/api_v2/views.py

@@ -3,7 +3,7 @@
 from django.db.models import Q
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import OpenApiParameter, extend_schema
-from rest_framework import status, viewsets
+from rest_framework import generics, status, viewsets
 from rest_framework.exceptions import ValidationError
 from rest_framework.permissions import BasePermission, IsAuthenticated
 from rest_framework.response import Response
@@ -17,6 +17,7 @@
 from entry.api_v2.serializers import (
     EntryBaseSerializer,
     EntryCopySerializer,
+    EntryExportSerializer,
     EntryRetrieveSerializer,
     EntryUpdateSerializer,
     GetEntrySimpleSerializer,
@@ -304,3 +305,57 @@ def get_queryset(self):
             query &= Q(name__iregex=r"%s" % keyword)
 
         return Entry.objects.filter(query)
+
+
+class EntryExportAPI(generics.GenericAPIView):
+    serializer_class = EntryExportSerializer
+
+    def post(self, request, entity_id: int):
+        if not Entity.objects.filter(id=entity_id).exists():
+            return Response(
+                "Failed to get entity of specified id", status=status.HTTP_400_BAD_REQUEST
+            )
+
+        serializer = EntryExportSerializer(data=request.data)
+        if not serializer.is_valid():
+            return Response(
+                "Parameters in post body is invalid", status=status.HTTP_400_BAD_REQUEST
+            )
+
+        job_params = {
+            "export_format": serializer.validated_data["format"],
+            "target_id": entity_id,
+        }
+
+        # check whether same job is sent
+        job_status_not_finished = [Job.STATUS["PREPARING"], Job.STATUS["PROCESSING"]]
+        if (
+            Job.get_job_with_params(request.user, job_params)
+            .filter(status__in=job_status_not_finished)
+            .exists()
+        ):
+            return Response(
+                "Same export processing is under execution", status=status.HTTP_400_BAD_REQUEST
+            )
+
+        entity = Entity.objects.get(id=entity_id)
+        if not request.user.has_permission(entity, ACLType.Readable):
+            return Response(
+                'Permission denied to _value "%s"' % entity.name, status=status.HTTP_400_BAD_REQUEST
+            )
+
+        # create a job to export search result and run it
+        job = Job.new_export(
+            request.user,
+            **{
+                "text": "entry_%s.%s" % (entity.name, job_params["export_format"]),
+                "target": entity,
+                "params": job_params,
+            },
+        )
+        job.run()
+
+        return Response(
+            {"result": "Succeed in registering export processing. " + "Please check Job list."},
+            status=status.HTTP_200_OK,
+        )

entry/tests/test_api_v2.py

@@ -1,13 +1,27 @@
 import datetime
+import errno
 import json
 from unittest import mock
+from unittest.mock import Mock, patch
+
+import yaml
 
 from airone.lib.test import AironeViewTest
-from airone.lib.types import AttrTypeStr, AttrTypeValue
+from airone.lib.types import (
+    AttrTypeArrNamedObj,
+    AttrTypeArrObj,
+    AttrTypeArrStr,
+    AttrTypeNamedObj,
+    AttrTypeObj,
+    AttrTypeStr,
+    AttrTypeText,
+    AttrTypeValue,
+)
 from entity.models import Entity, EntityAttr
 from entry import tasks
-from entry.models import Entry
+from entry.models import Attribute, AttributeValue, Entry
 from group.models import Group
+from job.models import Job, JobOperation
 from role.models import Role
 from user.models import User
 
@@ -1414,3 +1428,219 @@ def test_referral_unrelated_to_entry(self):
         resp_data = resp.json()
         results = resp_data["results"]
         self.assertEqual(len(results), 0)
+
+    @patch("entry.tasks.export_entries.delay", Mock(side_effect=tasks.export_entries))
+    def test_post_export(self):
+        user = self.admin_login()
+
+        entity = Entity.objects.create(name="ほげ", created_user=user)
+        for name in ["foo", "bar"]:
+            entity.attrs.add(
+                EntityAttr.objects.create(
+                    **{
+                        "name": name,
+                        "type": AttrTypeValue["string"],
+                        "created_user": user,
+                        "parent_entity": entity,
+                    }
+                )
+            )
+
+        entry = Entry.objects.create(name="fuga", schema=entity, created_user=user)
+        entry.complement_attrs(user)
+        for attr in entry.attrs.all():
+            [attr.add_value(user, x) for x in ["hoge", "fuga"]]
+
+        resp = self.client.post(
+            "/entry/api/v2/%d/export/" % entity.id,
+            json.dumps({}),
+            "application/json",
+        )
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp.json(),
+            {"result": "Succeed in registering export processing. Please check Job list."},
+        )
+
+        job = Job.objects.last()
+        self.assertEqual(job.operation, JobOperation.EXPORT_ENTRY.value)
+        self.assertEqual(job.status, Job.STATUS["DONE"])
+        self.assertEqual(job.text, "entry_ほげ.yaml")
+
+        obj = yaml.load(job.get_cache(), Loader=yaml.SafeLoader)
+        self.assertTrue(entity.name in obj)
+
+        self.assertEqual(len(obj[entity.name]), 1)
+        entry_data = obj[entity.name][0]
+        self.assertTrue(all(["name" in entry_data and "attrs" in entry_data]))
+
+        self.assertEqual(entry_data["name"], entry.name)
+        self.assertEqual(len(entry_data["attrs"]), entry.attrs.count())
+        self.assertEqual(entry_data["attrs"]["foo"], "fuga")
+        self.assertEqual(entry_data["attrs"]["bar"], "fuga")
+
+        resp = self.client.post(
+            "/entry/api/v2/%d/export/" % entity.id,
+            json.dumps({"format": "CSV"}),
+            "application/json",
+        )
+        self.assertEqual(resp.status_code, 200)
+
+        # append an unpermitted Attribute
+        entity.attrs.add(
+            EntityAttr.objects.create(
+                **{
+                    "name": "new_attr",
+                    "type": AttrTypeValue["string"],
+                    "created_user": user,
+                    "parent_entity": entity,
+                    "is_public": False,
+                }
+            )
+        )
+
+        # re-login with guest user
+        self.guest_login("guest2")
+
+        resp = self.client.post(
+            "/entry/api/v2/%d/export/" % entity.id,
+            json.dumps({}),
+            "application/json",
+        )
+        self.assertEqual(resp.status_code, 200)
+        obj = yaml.load(Job.objects.last().get_cache(), Loader=yaml.SafeLoader)
+
+        # check permitted attributes exist in the result
+        self.assertTrue(all([x in obj["ほげ"][0]["attrs"] for x in ["foo", "bar"]]))
+
+        # check unpermitted attribute doesn't exist in the result
+        self.assertFalse("new_attr" in obj["ほげ"][0]["attrs"])
+
+        ###
+        # Check the case of canceling job
+        ###
+        with patch.object(Job, "is_canceled", return_value=True):
+            resp = self.client.post(
+                "/entry/api/v2/%d/export/" % entity.id,
+                json.dumps({}),
+                "application/json",
+            )
+
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp.json(),
+            {"result": "Succeed in registering export processing. Please check Job list."},
+        )
+
+        job = Job.objects.last()
+        self.assertEqual(job.operation, JobOperation.EXPORT_ENTRY.value)
+        self.assertEqual(job.text, "entry_ほげ.yaml")
+        with self.assertRaises(OSError) as e:
+            raise OSError
+
+        if e.exception.errno == errno.ENOENT:
+            job.get_cache()
+
+    @patch("entry.tasks.export_entries.delay", Mock(side_effect=tasks.export_entries))
+    def test_get_export_csv_escape(self):
+        user = self.admin_login()
+
+        dummy_entity = Entity.objects.create(name="Dummy", created_user=user)
+        dummy_entry = Entry(name='D,U"MM"Y', schema=dummy_entity, created_user=user)
+        dummy_entry.save()
+
+        CASES = [
+            [AttrTypeStr, 'raison,de"tre', '"raison,de""tre"'],
+            [AttrTypeObj, dummy_entry, '"D,U""MM""Y"'],
+            [AttrTypeText, "1st line\r\n2nd line", '"1st line' + "\r\n" + '2nd line"'],
+            [AttrTypeNamedObj, {"key": dummy_entry}, '"{\'key\': \'D,U""MM""Y\'}"'],
+            [AttrTypeArrStr, ["one", "two", "three"], "\"['one', 'two', 'three']\""],
+            [AttrTypeArrObj, [dummy_entry], '"[\'D,U""MM""Y\']"'],
+            [
+                AttrTypeArrNamedObj,
+                [{"key1": dummy_entry}],
+                '"[{\'key1\': \'D,U""MM""Y\'}]"',
+            ],
+        ]
+
+        for case in CASES:
+            type_name = case[0].__name__  # AttrTypeStr -> 'AttrTypeStr'
+            attr_name = type_name + ',"ATTR"'
+
+            test_entity = Entity.objects.create(name="TestEntity_" + type_name, created_user=user)
+
+            test_entity_attr = EntityAttr.objects.create(
+                name=attr_name,
+                type=case[0],
+                created_user=user,
+                parent_entity=test_entity,
+            )
+
+            test_entity.attrs.add(test_entity_attr)
+            test_entity.save()
+
+            test_entry = Entry.objects.create(
+                name=type_name + ',"ENTRY"', schema=test_entity, created_user=user
+            )
+            test_entry.save()
+
+            test_attr = Attribute.objects.create(
+                name=attr_name,
+                schema=test_entity_attr,
+                created_user=user,
+                parent_entry=test_entry,
+            )
+
+            test_attr.save()
+            test_entry.attrs.add(test_attr)
+            test_entry.save()
+
+            test_val = None
+
+            if case[0].TYPE & AttrTypeValue["array"] == 0:
+                if case[0] == AttrTypeStr:
+                    test_val = AttributeValue.create(user=user, attr=test_attr, value=case[1])
+                elif case[0] == AttrTypeObj:
+                    test_val = AttributeValue.create(user=user, attr=test_attr, referral=case[1])
+                elif case[0] == AttrTypeText:
+                    test_val = AttributeValue.create(user=user, attr=test_attr, value=case[1])
+                elif case[0] == AttrTypeNamedObj:
+                    [(k, v)] = case[1].items()
+                    test_val = AttributeValue.create(user=user, attr=test_attr, value=k, referral=v)
+            else:
+                test_val = AttributeValue.create(user=user, attr=test_attr)
+                test_val.set_status(AttributeValue.STATUS_DATA_ARRAY_PARENT)
+                for child in case[1]:
+                    test_val_child = None
+                    if case[0] == AttrTypeArrStr:
+                        test_val_child = AttributeValue.create(
+                            user=user, attr=test_attr, value=child
+                        )
+                    elif case[0] == AttrTypeArrObj:
+                        test_val_child = AttributeValue.create(
+                            user=user, attr=test_attr, referral=child
+                        )
+                    elif case[0] == AttrTypeArrNamedObj:
+                        [(k, v)] = child.items()
+                        test_val_child = AttributeValue.create(
+                            user=user, attr=test_attr, value=k, referral=v
+                        )
+                    test_val.data_array.add(test_val_child)
+
+            test_val.save()
+            test_attr.values.add(test_val)
+            test_attr.save()
+
+            resp = self.client.post(
+                "/entry/api/v2/%d/export/" % test_entity.id,
+                json.dumps({"format": "CSV"}),
+                "application/json",
+            )
+            self.assertEqual(resp.status_code, 200)
+
+            content = Job.objects.last().get_cache()
+            header = content.splitlines()[0]
+            self.assertEqual(header, 'Name,"%s,""ATTR"""' % type_name)
+
+            data = content.replace(header, "", 1).strip()
+            self.assertEqual(data, '"%s,""ENTRY""",' % type_name + case[2])

frontend/src/apiclient/AironeApiClientV2.ts

@@ -315,6 +315,23 @@ class AironeApiClientV2 {
     });
   }
 
+  async exportEntries(entityId: number, format: string): Promise<void> {
+    await this.entry.entryApiV2ExportCreate(
+      {
+        entityId,
+        entryExport: {
+          format,
+        },
+      },
+      {
+        headers: {
+          "Content-Type": "application/json;charset=utf-8",
+          "X-CSRFToken": getCsrfToken(),
+        },
+      }
+    );
+  }
+
   // FIXME replace with auto-generated client code
   async advancedSearchEntries(
     entityIds: number[] = [],

frontend/src/apiclient/autogenerated/.openapi-generator/FILES

@@ -18,6 +18,7 @@ models/EntityUpdate.ts
 models/EntryBase.ts
 models/EntryCopy.ts
 models/EntryCreate.ts
+models/EntryExport.ts
 models/EntryRetrieve.ts
 models/EntryRetrieveAttrs.ts
 models/EntryRetrieveValue.ts