Merge pull request #33 from dataiku/feature/handle-xml-n-csv

alexbourret · web-flow · commit 7306d3766a18 · 2024-04-19T12:05:14.000+02:00
Feature/ Adds xml and csv parsing
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## [Version 1.2.2](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.2) - Feature release - 2024-02-14
+
+- Handle XML and CSV endpoints
+- Add secure SSO preset
+- Add secure username / password preset
+
 ## [Version 1.2.1](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.1) - Bugfix release - 2023-12-13
 
 - Fix the `Add an error column` error behaviour on the recipe
@@ -15,6 +21,8 @@
 - Add Brotli compression
 - Faster recurring calls
 - dku_error column kept at all time in API-Connect recipe output schema
+- Add XML to JSON conversion
+- Add CSV decoding
 - Updated code-env descriptor for DSS 12
 
 ## [Version 1.1.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.1.4) - Feature and bugfix release - 2023-02-28
diff --git a/code-env/python/spec/requirements.txt b/code-env/python/spec/requirements.txt
@@ -2,3 +2,4 @@ jsonpath-ng==1.5.3
 requests_ntlm==1.1.0
 requests==2.26.0
 Brotli==1.0.9
+xmltodict==0.13.0
diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json
@@ -32,11 +32,38 @@
             "type": "SEPARATOR",
             "label": "Authentication"
         },
+        {
+            "name": "auth_type",
+            "label": "Authentication type",
+            "description": "",
+            "type": "SELECT",
+            "defaultValue": null,
+            "selectChoices":[
+                {"value": "secure_oauth", "label": "SSO"},
+                {"value": "secure_basic", "label": "Secure username / password"},
+                {"value": null, "label": "Other"}
+            ]
+        },
         {
             "name": "credential",
             "label": "Credential preset",
             "type": "PRESET",
-            "parameterSetId": "credential"
+            "parameterSetId": "credential",
+            "visibilityCondition": "model.auth_type == null"
+        },
+        {
+            "name": "secure_oauth",
+            "label": "SSO preset",
+            "type": "PRESET",
+            "parameterSetId": "secure-oauth",
+            "visibilityCondition": "model.auth_type == 'secure_oauth'"
+        },
+        {
+            "name": "secure_basic",
+            "label": "Credential preset",
+            "type": "PRESET",
+            "parameterSetId": "secure-basic",
+            "visibilityCondition": "model.auth_type == 'secure_basic'"
         },
         {
             "type": "SEPARATOR",
@@ -254,6 +281,7 @@
             "name": "ignore_ssl_check",
             "label": "Ignore SSL check",
             "type": "BOOLEAN",
+            "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
             "defaultValue": false
         },
         {
diff --git a/custom-recipes/api-connect/recipe.py b/custom-recipes/api-connect/recipe.py
@@ -3,7 +3,7 @@
 from dataiku.customrecipe import get_input_names_for_role, get_recipe_config, get_output_names_for_role
 import pandas as pd
 from safe_logger import SafeLogger
-from dku_utils import get_dku_key_values, get_endpoint_parameters
+from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials
 from rest_api_recipe_session import RestApiRecipeSession
 from dku_constants import DKUConstants
 
@@ -37,6 +37,7 @@ def get_partitioning_keys(id_list, dku_flow_variables):
 credential_parameters = config.get("credential", {})
 behaviour_when_error = config.get("behaviour_when_error", "add-error-column")
 endpoint_parameters = get_endpoint_parameters(config)
+secure_credentials = get_secure_credentials(config)
 extraction_key = endpoint_parameters.get("extraction_key", "")
 is_raw_output = endpoint_parameters.get("raw_output", True)
 parameter_columns = [column for column in config.get("parameter_columns", []) if column]
@@ -54,6 +55,7 @@ def get_partitioning_keys(id_list, dku_flow_variables):
 recipe_session = RestApiRecipeSession(
     custom_key_values,
     credential_parameters,
+    secure_credentials,
     endpoint_parameters,
     extraction_key,
     parameter_columns,
diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json
@@ -0,0 +1,45 @@
+{
+    "meta" : {
+        "label": "Secure basic",
+        "description": "Secured preset to be used solely on one domain. Username / password have to be set by individual users from their own profile's credential tab.",
+        "icon": "icon-puzzle-piece"
+    },
+    "defaultDefinableInline": false,
+    "defaultDefinableAtProjectLevel": false,
+    "pluginParams": [
+    ],
+    "params": [
+        {
+            "name": "secure_token",
+            "type": "CREDENTIAL_REQUEST",
+            "label": "Azure Single Sign On",
+            "credentialRequestSettings": {
+                "type": "BASIC"
+            },
+            "mandatory": true
+        },
+        {
+            "name": "secure_domain",
+            "label": "Domain",
+            "description": "",
+            "type": "STRING",
+            "mandatory": true
+        },
+        {
+            "name": "login_type",
+            "label": "Login type",
+            "type": "SELECT",
+            "defaultValue": "basic_login",
+            "selectChoices": [
+                {
+                    "value": "basic_login",
+                    "label": "Basic auth"
+                },
+                {
+                    "value": "ntlm",
+                    "label": "NTLM"
+                }
+            ]
+        }
+    ]
+}
diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json
@@ -0,0 +1,54 @@
+{
+    "meta" : {
+        "label": "Secure SSO credentials",
+        "description": "",
+        "icon": "icon-puzzle-piece"
+    },
+    "defaultDefinableInline": false,
+    "defaultDefinableAtProjectLevel": false,
+    "pluginParams": [
+    ],
+
+    "params": [
+        {
+            "name": "secure_token",
+            "type": "CREDENTIAL_REQUEST",
+            "label": "Single Sign On",
+            "credentialRequestSettings": {
+                "type": "OAUTH2",
+                "authorizationEndpoint": " ",
+                "tokenEndpoint": " ",
+                "scope": " "
+            },
+            "mandatory": true
+        },
+        {
+            "name": "authorizationEndpoint",
+            "label": "Authorization endpoint",
+            "type": "STRING",
+            "description": "",
+            "mandatory": false
+        },
+        {
+            "name": "tokenEndpoint",
+            "label": "Token endpoint",
+            "type": "STRING",
+            "description": "",
+            "mandatory": false
+        },
+        {
+            "name": "scope",
+            "label": "Scope",
+            "type": "STRING",
+            "description": "",
+            "mandatory": false
+        },
+        {
+            "name": "secure_domain",
+            "label": "Domain",
+            "description": "",
+            "type": "STRING",
+            "mandatory": true
+        }
+    ]
+}
diff --git a/plugin.json b/plugin.json
@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.2.1",
+    "version": "1.2.2",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",
diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json
@@ -11,11 +11,38 @@
             "type": "SEPARATOR",
             "label": "Authentication"
         },
+        {
+            "name": "auth_type",
+            "label": "Authentication type",
+            "description": "",
+            "type": "SELECT",
+            "defaultValue": null,
+            "selectChoices":[
+                {"value": "secure_oauth", "label": "SSO"},
+                {"value": "secure_basic", "label": "Secure username / password"},
+                {"value": null, "label": "Other"}
+            ]
+        },
         {
             "name": "credential",
             "label": "Credential preset",
             "type": "PRESET",
-            "parameterSetId": "credential"
+            "parameterSetId": "credential",
+            "visibilityCondition": "model.auth_type == null"
+        },
+        {
+            "name": "secure_oauth",
+            "label": "SSO preset",
+            "type": "PRESET",
+            "parameterSetId": "secure-oauth",
+            "visibilityCondition": "model.auth_type == 'secure_oauth'"
+        },
+        {
+            "name": "secure_basic",
+            "label": "Credential preset",
+            "type": "PRESET",
+            "parameterSetId": "secure-basic",
+            "visibilityCondition": "model.auth_type == 'secure_basic'"
         },
         {
             "type": "SEPARATOR",
@@ -117,8 +144,8 @@
         },
         {
             "name": "raw_output",
-            "label": "Raw JSON output",
-            "description": "",
+            "label": " ",
+            "description": "Raw JSON output",
             "defaultValue": true,
             "type": "BOOLEAN"
         },
@@ -198,20 +225,23 @@
         },
         {
             "name": "ignore_ssl_check",
-            "label": "Ignore SSL check",
+            "label": " ",
+            "description": "Ignore SSL check",
             "type": "BOOLEAN",
+            "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
             "defaultValue": false
         },
         {
             "name": "redirect_auth_header",
-            "label": "Redirect authorization header",
+            "label": " ",
+            "description": "Redirect authorization header",
             "type": "BOOLEAN",
             "defaultValue": false
         },
         {
             "name": "display_metadata",
-            "label": "Display metadata",
-            "description": "Status code, request time...",
+            "label": " ",
+            "description": "Display metadata",
             "type": "BOOLEAN",
             "defaultValue": false
         },
diff --git a/python-connectors/api-connect_dataset/connector.py b/python-connectors/api-connect_dataset/connector.py
@@ -2,7 +2,10 @@
 from dataikuapi.utils import DataikuException
 from safe_logger import SafeLogger
 from rest_api_client import RestAPIClient
-from dku_utils import get_dku_key_values, get_endpoint_parameters, parse_keys_for_json, get_value_from_path
+from dku_utils import (
+    get_dku_key_values, get_endpoint_parameters,
+    parse_keys_for_json, get_value_from_path, get_secure_credentials, decode_csv_data
+)
 from dku_constants import DKUConstants
 import json
 
@@ -17,9 +20,10 @@ def __init__(self, config, plugin_config):
         logger.info('API-Connect plugin connector v{}'.format(DKUConstants.PLUGIN_VERSION))
         logger.info("config={}".format(logger.filter_secrets(config)))
         endpoint_parameters = get_endpoint_parameters(config)
+        secure_credentials = get_secure_credentials(config)
         credential = config.get("credential", {})
         custom_key_values = get_dku_key_values(config.get("custom_key_values", {}))
-        self.client = RestAPIClient(credential, endpoint_parameters, custom_key_values)
+        self.client = RestAPIClient(credential, secure_credentials, endpoint_parameters, custom_key_values)
         extraction_key = endpoint_parameters.get("extraction_key", None)
         self.extraction_key = extraction_key or ''
         self.extraction_path = self.extraction_key.split('.')
@@ -49,9 +53,14 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                 record_count += len(data)
                 for row in data:
                     yield self.format_output(row, metadata)
-            else:
+            elif isinstance(data, dict):
                 record_count += 1
                 yield self.format_output(data, metadata)
+            else:
+                data = decode_csv_data(data)
+                record_count += len(data)
+                for row in data:
+                    yield self.format_output(row, metadata)
             if is_records_limit and record_count >= records_limit:
                 break
 
diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py
@@ -1,7 +1,7 @@
 class DKUConstants(object):
     API_RESPONSE_KEY = "api_response"
-    FORBIDDEN_KEYS = ["token", "password", "api_key_value"]
+    FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
-    PLUGIN_VERSION = "1.2.1"
+    PLUGIN_VERSION = "1.2.2"
     RAW_BODY_FORMAT = "RAW"
     REPONSE_ERROR_KEY = "dku_error"
diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py
diff --git a/python-lib/rest_api_auth.py b/python-lib/rest_api_auth.py
diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py
diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py
diff --git a/tests/python/integration/test_scenario.py b/tests/python/integration/test_scenario.py

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "api-connect",`
`3`		`- "version": "1.2.1",`
	`3`	`+ "version": "1.2.2",`
`4`	`4`	`"meta": {`
`5`	`5`	`"label": "API Connect",`
`6`	`6`	`"description": "Retrieve data from any REST API",`