added management comand for easy local provider setup

addon_service/management/commands/fill_external_services.py

@@ -0,0 +1,70 @@
+import csv
+from pathlib import Path
+from typing import Any
+
+from django.core.management import BaseCommand
+
+from addon_service.external_service.citation.models import ExternalCitationService
+from addon_service.external_service.computing.models import ExternalComputingService
+from addon_service.external_service.storage.models import ExternalStorageService
+from addon_service.oauth1 import OAuth1ClientConfig
+from addon_service.oauth2 import OAuth2ClientConfig
+
+
+csv_base_path = Path(__file__).parent / "providers"
+services_csv = csv_base_path / "providers.csv"
+oauth2_csv = csv_base_path / "oauth2.csv"
+oauth1_csv = csv_base_path / "oauth1.csv"
+
+service_type_map = {
+    "storage": ExternalStorageService,
+    "citation": ExternalCitationService,
+    "computing": ExternalComputingService,
+}
+
+icons_path = Path(__file__).parent.parent.parent / "static/provider_icons"
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs):
+        services = self.load_csv_data(services_csv)
+        oauth2_configs = self.load_csv_data(oauth2_csv)
+        oauth1_configs = self.load_csv_data(oauth1_csv)
+
+        for service in services.values():
+            model = service_type_map[service.pop("service_type")]
+            service["icon_name"] = str(icons_path / service["icon_name"])
+            if oauth2_id := service.pop("oauth2_client_config_id", None):
+                oauth2_config = oauth2_configs[oauth2_id]
+                oauth2_config = OAuth2ClientConfig.objects.create(**oauth2_config)
+                service["oauth2_client_config"] = oauth2_config
+            if oauth1_id := service.pop("oauth1_client_config_id", None):
+                oauth1_config = oauth1_configs[oauth1_id]
+                oauth1_config = OAuth1ClientConfig.objects.create(**oauth1_config)
+                service["oauth1_client_config"] = oauth1_config
+            model.objects.create(**service)
+
+    def load_csv_data(self, path: Path) -> dict[str, dict[str, Any]]:
+        with path.open() as services_file:
+            item = csv.reader(services_file)
+            field_names = next(item)
+            data_list = [dict(zip(field_names, service)) for service in item]
+            return {item.pop("id"): self.fix_values(item) for item in data_list}
+
+    def fix_values(self, item):
+        raw_values = {key: self.fix_value(value) for key, value in item.items()}
+        return {key: value for key, value in raw_values.items() if value is not None}
+
+    def fix_value(self, data):
+        if data == "":
+            return None
+        if data.startswith("{") and data.endswith("}"):
+            values = data.removeprefix("{").removesuffix("}").split(",")
+            if not values or not any(values):
+                return None
+            return values
+        try:
+            return int(data)
+        except ValueError:
+            pass
+        return data

addon_service/management/commands/providers/oauth1.csv

@@ -0,0 +1,2 @@
+id,request_token_url,auth_url,auth_callback_url,access_token_url,client_key,client_secret
+e3663c71-3520-4fce-b0ff-e56c81a73d54,https://www.zotero.org/oauth/request,https://www.zotero.org/oauth/authorize,http://localhost:8004/v1/oauth1/callback,https://www.zotero.org/oauth/access,client_key,client_secret

addon_service/management/commands/providers/oauth2.csv

@@ -0,0 +1,10 @@
+id,auth_uri,auth_callback_url,token_endpoint_url,client_id,client_secret,quirks
+5c161353-7444-478e-881f-3b8aec3a002a,https://figshare.com/account/applications/authorize,http://localhost:8004/v1/oauth2/callback,https://api.figshare.com/v2/token,client_id,client_secret,
+90b44f3d-66e3-444c-9261-534b9e37b21a,https://www.dropbox.com/oauth2/authorize?token_access_type=offline,http://localhost:8004/v1/oauth2/callback,https://www.dropbox.com/oauth2/token,client_id,client_secret,
+5cf93c06-c27f-4446-b8ac-ffac209ba465,https://accounts.google.com/o/oauth2/auth?access_type=offline&approval_prompt=force,http://localhost:8004/v1/oauth2/callback,https://www.googleapis.com/oauth2/v3/token?access_type=offline&prompt=consent,client_id,client_secret,
+a64e222c-a37b-4f34-9f27-83611d851f5c,https://login.microsoftonline.com/common/oauth2/v2.0/authorize,http://localhost:8004/v1/oauth2/callback,https://login.microsoftonline.com/common/oauth2/v2.0/token,client_id,client_secret,
+1b2d71cc-425e-4aa6-886c-e2ff247c4f2c,https://bitbucket.org/site/oauth2/authorize,http://localhost:5000/oauth/callback/bitbucket,https://bitbucket.org/site/oauth2/access_token,client_id,client_secret,
+05328f70-6634-4256-8218-6d562b49fc07,https://github.com/login/oauth/authorize,http://localhost:8004/v1/oauth2/callback,https://github.com/login/oauth/access_token,client_id,client_secret,
+4258d732-db77-42aa-b275-8da841adf77f,https://www.box.com/api/oauth2/authorize,http://localhost:8004/v1/oauth2/callback,https://www.box.com/api/oauth2/token,client_id,client_secret,
+b3fd76fc-3368-47b4-9053-ff4b7ca4baa2,https://api.mendeley.com/oauth/authorize,http://localhost:5000/oauth/callback/mendeley/,https://api.mendeley.com/oauth/token,client_id,client_secret,
+285f28ee-15b5-429e-b7ba-3371a19678a8,https://github.com/login/oauth/authorize,http://localhost:5000/oauth/callback/github,https://github.com/login/oauth/access_token,client_id,client_secret,1

addon_service/management/commands/providers/providers.csv

@@ -0,0 +1,15 @@
+id,display_name,int_credentials_format,int_service_type,int_addon_imp,supported_scopes,api_base_url,wb_key,oauth1_client_config_id,oauth2_client_config_id,icon_name,api_base_url_options,externalservice_ptr_id,max_concurrent_downloads,max_upload_mb,int_supported_features,service_type
+d424bec8-811b-4d23-bf1c-f4e8219e0454,Mendeley,1,1,1004,{all},https://api.mendeley.com,mendeley,,b3fd76fc-3368-47b4-9053-ff4b7ca4baa2,mendeley.svg,,,,,3,citation
+fce6ada4-9a6e-4986-94c8-9a378170f5fb,Zotero,5,1,1002,{all},https://api.zotero.org,zotero_org,e3663c71-3520-4fce-b0ff-e56c81a73d54,,zotero.svg,,,,,3,citation
+455cfb5f-6ca0-4d60-8fc8-279f7b229da2,Boa,3,1,1020,{all},https://boa.com,boa,,,boa.png,,,,,1,computing
+d5ad0101-b732-4a08-8306-384d66beef0e,Gitlab,4,1,1011,{},https://gitlab.com,gitlab,,,gitlab.svg,{},d5ad0101-b732-4a08-8306-384d66beef0e,10,10,511,storage
+e3cb1bc9-83ec-49fa-86de-1d04d3f94834,Dataverse,6,3,1010,{},https://demo.dataverse.org,dataverse,,,dataverse.svg,,e3cb1bc9-83ec-49fa-86de-1d04d3f94834,10,10,511,storage
+4a2aa5c7-c5be-4e3f-abd8-04fb34928bd3,s3,2,1,1003,{},https://api.github.com,s3,,,s3.svg,,4a2aa5c7-c5be-4e3f-abd8-04fb34928bd3,10,10,511,storage
+4854b243-593a-4c6f-96e8-5bb2a2df9134,BOX,1,1,1001,{root_readwrite},https://api.box.com/2.0/,box,,4258d732-db77-42aa-b275-8da841adf77f,box.svg,,4854b243-593a-4c6f-96e8-5bb2a2df9134,10,10,2047,storage
+9e7148ae-4919-4676-bb44-a7a6662c6dd0,Dropbox,1,1,1006,{files.metadata.write files.content.write files.content.read files.metadata.read},https://api.dropboxapi.com/2/,dropbox,,90b44f3d-66e3-444c-9261-534b9e37b21a,dropbox.svg,,9e7148ae-4919-4676-bb44-a7a6662c6dd0,10,10,511,storage
+15b66b0c-4db9-4425-b2e0-dfd701a55b05,OneDrive,1,1,1008,{User.Read offline_access Files.Read Files.Read.All Files.ReadWrite},https://graph.microsoft.com/v1.0/,onedrive,,a64e222c-a37b-4f34-9f27-83611d851f5c,onedrive.svg,,15b66b0c-4db9-4425-b2e0-dfd701a55b05,10,10,2047,storage
+babfcfcc-0e47-4c8d-9ca1-3b335bb8cf55,Github,1,1,1013,{repo contents},https://api.github.com,github,,285f28ee-15b5-429e-b7ba-3371a19678a8,github.svg,,babfcfcc-0e47-4c8d-9ca1-3b335bb8cf55,10,10,2047,storage
+b670cf7a-752f-49ed-8367-8238cf8a24d1,Figshare,1,1,1007,{all},https://api.figshare.com/v2/,figshare,,5c161353-7444-478e-881f-3b8aec3a002a,figshare.svg,,b670cf7a-752f-49ed-8367-8238cf8a24d1,10,10,511,storage
+2acd3e09-5e27-4e5b-a3d1-579f16663cfc,Bitbucket,1,1,1012,{account repository team},https://api.bitbucket.org/2.0/,bitbucket,,1b2d71cc-425e-4aa6-886c-e2ff247c4f2c,bitbucket.svg,,2acd3e09-5e27-4e5b-a3d1-579f16663cfc,10,10,511,storage
+102c6356-e31a-4316-a9fe-8b2bcb20fdd6,Google Drive,1,1,1005,{https://www.googleapis.com/auth/drive},https://www.googleapis.com/,googledrive,,5cf93c06-c27f-4446-b8ac-ffac209ba465,google_drive.svg,,102c6356-e31a-4316-a9fe-8b2bcb20fdd6,10,10,511,storage
+5bc440c8-a03a-4aa6-bd9e-856aee9883e9,Owncloud,3,2,1009,{},,owncloud,,,owncloud.svg,,5bc440c8-a03a-4aa6-bd9e-856aee9883e9,10,10,511,storage