From fd016429cde0a90161f9942c5a5046c74874f09e Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Wed, 28 Aug 2024 11:36:54 -0700 Subject: [PATCH 1/6] wip --- maap/Secrets.py | 129 ++++++++++++++++++++++++++++++++++++++++ maap/config_reader.py | 1 + maap/maap.py | 4 +- maap/utils/endpoints.py | 2 + 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 maap/Secrets.py diff --git a/maap/Secrets.py b/maap/Secrets.py new file mode 100644 index 0000000..548ca30 --- /dev/null +++ b/maap/Secrets.py @@ -0,0 +1,129 @@ +import requests +import logging +import json +from maap.utils import endpoints +from maap.utils import requests_utils + + +class Secrets: + """ + Functions used for Member secrets API interfacing + """ + def __init__(self, config, api_header): + self._api_header = api_header + # self._members_endpoint = members_endpoint + self._members_endpoint = "https://api.dit.maap-project.org/api/members/self" + self._logger = logging.getLogger(__name__) + + + def get_secrets(self): + """ + Returns a list of secrets for a given user. + + Returns: + list: Secret names for a given user. + """ + try: + response = requests.get( + url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}", + headers=self._api_header + ) + + return json.loads(response.text) + + except Exception as ex: + raise(f"Error retrieving secrets: {ex}") + + + def get_secret(self, secret_name=None): + """ + Returns secret value for provided secret name. + + Args: + secret_name (str, required): Secret name. + secret_value (str, optional): Secret value. + + Returns: + dict: Secret name and value. + + Raises: + ValueError: If secret name is not provided. + """ + try: + if secret_name is None: + raise ValueError("Failed to get secret value. Please provide secret name.") + + response = requests.get( + url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}/{secret_name}", + headers=self._api_header + ) + + return json.loads(response.text) + + except Exception as ex: + raise(f"Error retrieving secret: {ex}") + + + def add_secret(self, secret_name=None, secret_value=None): + """ + Adds a secret. Secret name must be provided. Secret value may be null. + + Args: + secret_name (str, required): Secret name. + secret_value (str, optional): Secret value. + + Returns: + dict: Containing name and value of secret that was just added. + + Raises: + ValueError: If secret name is not provided. + """ + try: + if secret_name is None: + raise ValueError("Failed to add secret. Please provide secret name.") + + response = requests.post( + url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}", + headers=self._api_header, + data=json.dumps({"secret_name": secret_name, "secret_value": secret_value}) + ) + + return json.loads(response.text) + + except Exception as ex: + raise(f"Error adding secret: {ex}") + + + + def delete_secret(self, secret_name=None): + """ + Deletes a secret. + + Args: + secret_name (str, required): Secret name. + + Returns: + dict: Containing response code and message indicating whether or not deletion was successful. + + Raises: + ValueError: If secret name is not provided. + """ + try: + if secret_name is None: + raise ValueError("Failed to delete secret. Please provide secret name.") + + response = requests.delete( + url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}/{secret_name}", + headers=self._api_header + ) + + return json.loads(response.text) + + except Exception as ex: + raise(f"Error deleting secret: {ex}") + + + + + + diff --git a/maap/config_reader.py b/maap/config_reader.py index e8a2fbd..5171573 100644 --- a/maap/config_reader.py +++ b/maap/config_reader.py @@ -81,6 +81,7 @@ def __init__(self, maap_host): self.algorithm_build = self._get_api_endpoint("algorithm_build") self.mas_algo = self._get_api_endpoint("mas_algo") self.dps_job = self._get_api_endpoint("dps_job") + self.member_secrets = self._get_api_endpoint("member_secrets") self.member_dps_token = self._get_api_endpoint("member_dps_token") self.requester_pays = self._get_api_endpoint("requester_pays") self.edc_credentials = self._get_api_endpoint("edc_credentials") diff --git a/maap/maap.py b/maap/maap.py index 8b91695..40c87ca 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -17,6 +17,7 @@ from maap.utils import algorithm_utils from maap.Profile import Profile from maap.AWS import AWS +from maap.Secrets import Secrets from maap.dps.DpsHelper import DpsHelper from maap.utils import endpoints @@ -40,10 +41,11 @@ def __init__(self, maap_host=os.getenv('MAAP_API_HOST', 'api.maap-project.org')) self.config.workspace_bucket_credentials, self._get_api_header() ) + self.secrets = Secrets(self.config, self._get_api_header(content_type="application/json")) def _get_api_header(self, content_type=None): - api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token} + api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token, 'Content-Type': content_type if content_type else self.config.content_type} if os.environ.get("MAAP_PGT"): api_header['proxy-ticket'] = os.environ.get("MAAP_PGT") diff --git a/maap/utils/endpoints.py b/maap/utils/endpoints.py index 33bc816..8937f8e 100644 --- a/maap/utils/endpoints.py +++ b/maap/utils/endpoints.py @@ -3,3 +3,5 @@ DPS_JOB_DISMISS = "cancel" DPS_JOB_LIST = "list" CMR_ALGORITHM_DATA = "data" + +MEMBERS_SECRETS = "secrets" From 5e4cf5688ca0deea73712104c219116c1f917e98 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Wed, 28 Aug 2024 16:39:04 -0700 Subject: [PATCH 2/6] cleanup --- maap/Secrets.py | 17 ++++++++--------- maap/config_reader.py | 1 - maap/maap.py | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/maap/Secrets.py b/maap/Secrets.py index 548ca30..881a9f3 100644 --- a/maap/Secrets.py +++ b/maap/Secrets.py @@ -3,16 +3,16 @@ import json from maap.utils import endpoints from maap.utils import requests_utils +from maap.utils import endpoints class Secrets: """ - Functions used for Member secrets API interfacing + Functions used for member secrets API interfacing """ - def __init__(self, config, api_header): + def __init__(self, member_endpoint, api_header): self._api_header = api_header - # self._members_endpoint = members_endpoint - self._members_endpoint = "https://api.dit.maap-project.org/api/members/self" + self._members_endpoint = f"{member_endpoint}/{endpoints.MEMBERS_SECRETS}" self._logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ def get_secrets(self): """ try: response = requests.get( - url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}", + url = self._members_endpoint, headers=self._api_header ) @@ -41,7 +41,6 @@ def get_secret(self, secret_name=None): Args: secret_name (str, required): Secret name. - secret_value (str, optional): Secret value. Returns: dict: Secret name and value. @@ -54,7 +53,7 @@ def get_secret(self, secret_name=None): raise ValueError("Failed to get secret value. Please provide secret name.") response = requests.get( - url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}/{secret_name}", + url = f"{self._members_endpoint}/{secret_name}", headers=self._api_header ) @@ -83,7 +82,7 @@ def add_secret(self, secret_name=None, secret_value=None): raise ValueError("Failed to add secret. Please provide secret name.") response = requests.post( - url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}", + url = self._members_endpoint, headers=self._api_header, data=json.dumps({"secret_name": secret_name, "secret_value": secret_value}) ) @@ -113,7 +112,7 @@ def delete_secret(self, secret_name=None): raise ValueError("Failed to delete secret. Please provide secret name.") response = requests.delete( - url = f"{self._members_endpoint}/{endpoints.MEMBERS_SECRETS}/{secret_name}", + url = f"{self._members_endpoint}/{secret_name}", headers=self._api_header ) diff --git a/maap/config_reader.py b/maap/config_reader.py index 5171573..e8a2fbd 100644 --- a/maap/config_reader.py +++ b/maap/config_reader.py @@ -81,7 +81,6 @@ def __init__(self, maap_host): self.algorithm_build = self._get_api_endpoint("algorithm_build") self.mas_algo = self._get_api_endpoint("mas_algo") self.dps_job = self._get_api_endpoint("dps_job") - self.member_secrets = self._get_api_endpoint("member_secrets") self.member_dps_token = self._get_api_endpoint("member_dps_token") self.requester_pays = self._get_api_endpoint("requester_pays") self.edc_credentials = self._get_api_endpoint("edc_credentials") diff --git a/maap/maap.py b/maap/maap.py index 40c87ca..679ed39 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -41,7 +41,7 @@ def __init__(self, maap_host=os.getenv('MAAP_API_HOST', 'api.maap-project.org')) self.config.workspace_bucket_credentials, self._get_api_header() ) - self.secrets = Secrets(self.config, self._get_api_header(content_type="application/json")) + self.secrets = Secrets(self.config.member, self._get_api_header(content_type="application/json")) def _get_api_header(self, content_type=None): From dfc6cd78d13377c3842d9e567cd00eb1ac197aab Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Wed, 28 Aug 2024 17:12:09 -0700 Subject: [PATCH 3/6] updated get_secret endpoint to return value --- maap/Secrets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/maap/Secrets.py b/maap/Secrets.py index 881a9f3..0a9e0a3 100644 --- a/maap/Secrets.py +++ b/maap/Secrets.py @@ -43,7 +43,7 @@ def get_secret(self, secret_name=None): secret_name (str, required): Secret name. Returns: - dict: Secret name and value. + string: Secret value. Raises: ValueError: If secret name is not provided. @@ -57,6 +57,10 @@ def get_secret(self, secret_name=None): headers=self._api_header ) + if response.ok: + response = response.json() + return response["secret_value"] + return json.loads(response.text) except Exception as ex: From c797c896350638997b8eed35ce7dff615a487996 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Thu, 29 Aug 2024 14:09:06 -0700 Subject: [PATCH 4/6] added tests for secrets management --- test/functional_test.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/functional_test.py b/test/functional_test.py index 64a3580..4a79e8b 100644 --- a/test/functional_test.py +++ b/test/functional_test.py @@ -108,6 +108,30 @@ def cancel_job(maap: MAAP, job_id): assert resp is not None assert 'Accepted' in str(resp) +@log_decorator +def add_secret(maap: MAAP, secret_name=None, secret_value=None): + resp = maap.secrets.add_secret(secret_name, secret_value) + print(resp) + assert resp is not None + +@log_decorator +def get_secrets(maap: MAAP): + resp = maap.secrets.get_secrets() + print(resp) + assert resp is not None + +@log_decorator +def get_secret(maap: MAAP, secret_name=None): + resp = maap.secrets.get_secret(secret_name) + print(resp) + assert resp is not None + +@log_decorator +def delete_secret(maap: MAAP, secret_name=None): + resp = maap.secrets.delete_secret(secret_name) + print(resp) + assert resp is not None + def main(): if os.environ.get('MAAP_PGT') is None: @@ -118,9 +142,19 @@ def main(): # list_algorithms(maap) job = submit_job(maap, queue="maap-dps-sandbox") cancel_job(maap, job.id) + + # Test secrets management + secret_name = "test_secret" + secret_value = "test_value" + get_secrets(maap) + add_secret(maap, secret_name, secret_value) + get_secret(maap, secret_name) + delete_secret(maap, secret_name) + # submit_job(maap, wait_for_completion=True) # delete_algorithm(maap, "maap_functional_test_algo:main") + if __name__ == '__main__': main() From b5d833d0011974ac2ec6baf4037522d76e27f84e Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Thu, 29 Aug 2024 16:26:16 -0700 Subject: [PATCH 5/6] updated error handling | added logging --- maap/Secrets.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/maap/Secrets.py b/maap/Secrets.py index 0a9e0a3..46dd6bc 100644 --- a/maap/Secrets.py +++ b/maap/Secrets.py @@ -5,6 +5,7 @@ from maap.utils import requests_utils from maap.utils import endpoints +logger = logging.getLogger(__name__) class Secrets: """ @@ -13,7 +14,6 @@ class Secrets: def __init__(self, member_endpoint, api_header): self._api_header = api_header self._members_endpoint = f"{member_endpoint}/{endpoints.MEMBERS_SECRETS}" - self._logger = logging.getLogger(__name__) def get_secrets(self): @@ -28,11 +28,11 @@ def get_secrets(self): url = self._members_endpoint, headers=self._api_header ) - + logger.debug(response.text) return json.loads(response.text) - except Exception as ex: - raise(f"Error retrieving secrets: {ex}") + except Exception as e: + raise(f"Error retrieving secrets: {e}") def get_secret(self, secret_name=None): @@ -48,10 +48,10 @@ def get_secret(self, secret_name=None): Raises: ValueError: If secret name is not provided. """ - try: - if secret_name is None: - raise ValueError("Failed to get secret value. Please provide secret name.") + if secret_name is None: + raise ValueError("Failed to get secret value. Please provide secret name.") + try: response = requests.get( url = f"{self._members_endpoint}/{secret_name}", headers=self._api_header @@ -61,10 +61,11 @@ def get_secret(self, secret_name=None): response = response.json() return response["secret_value"] + logger.debug(response.text) return json.loads(response.text) - except Exception as ex: - raise(f"Error retrieving secret: {ex}") + except Exception as e: + raise(f"Error retrieving secret: {e}") def add_secret(self, secret_name=None, secret_value=None): @@ -79,23 +80,23 @@ def add_secret(self, secret_name=None, secret_value=None): dict: Containing name and value of secret that was just added. Raises: - ValueError: If secret name is not provided. + ValueError: If secret name or secret value is not provided. """ - try: - if secret_name is None: - raise ValueError("Failed to add secret. Please provide secret name.") + if secret_name is None or secret_value is None: + raise ValueError("Failed to add secret. Secret name and secret value must not be 'None'.") + try: response = requests.post( url = self._members_endpoint, headers=self._api_header, data=json.dumps({"secret_name": secret_name, "secret_value": secret_value}) ) + logger.debug(response.text) return json.loads(response.text) - except Exception as ex: - raise(f"Error adding secret: {ex}") - + except Exception as e: + raise(f"Error adding secret: {e}") def delete_secret(self, secret_name=None): @@ -111,19 +112,20 @@ def delete_secret(self, secret_name=None): Raises: ValueError: If secret name is not provided. """ - try: - if secret_name is None: - raise ValueError("Failed to delete secret. Please provide secret name.") + if secret_name is None: + raise ValueError("Failed to delete secret. Please provide secret name.") + try: response = requests.delete( url = f"{self._members_endpoint}/{secret_name}", headers=self._api_header ) + logger.debug(response.text) return json.loads(response.text) - except Exception as ex: - raise(f"Error deleting secret: {ex}") + except Exception as e: + raise(f"Error deleting secret: {e}") From b001b4e4d88278be29f4ef79792e84e6d44c6ab6 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Wed, 4 Sep 2024 14:27:49 -0700 Subject: [PATCH 6/6] review comments --- maap/Secrets.py | 19 ++++++++----------- test/functional_test.py | 5 +++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/maap/Secrets.py b/maap/Secrets.py index 46dd6bc..51f75cf 100644 --- a/maap/Secrets.py +++ b/maap/Secrets.py @@ -21,21 +21,20 @@ def get_secrets(self): Returns a list of secrets for a given user. Returns: - list: Secret names for a given user. + list: Returns a list of dicts containing secret names e.g. [{'secret_name': 'secret1'}, {'secret_name': 'secret2'}]. """ try: response = requests.get( url = self._members_endpoint, headers=self._api_header ) - logger.debug(response.text) + logger.debug(f"Response from get_secrets request: {response.text}") return json.loads(response.text) - except Exception as e: raise(f"Error retrieving secrets: {e}") - def get_secret(self, secret_name=None): + def get_secret(self, secret_name): """ Returns secret value for provided secret name. @@ -49,7 +48,7 @@ def get_secret(self, secret_name=None): ValueError: If secret name is not provided. """ if secret_name is None: - raise ValueError("Failed to get secret value. Please provide secret name.") + raise ValueError("Secret name parameter cannot be None.") try: response = requests.get( @@ -57,13 +56,13 @@ def get_secret(self, secret_name=None): headers=self._api_header ) + # Return secret value directly for user ease-of-use if response.ok: response = response.json() return response["secret_value"] - logger.debug(response.text) + logger.debug(f"Response from get_secret request: {response.text}") return json.loads(response.text) - except Exception as e: raise(f"Error retrieving secret: {e}") @@ -92,9 +91,8 @@ def add_secret(self, secret_name=None, secret_value=None): data=json.dumps({"secret_name": secret_name, "secret_value": secret_value}) ) - logger.debug(response.text) + logger.debug(f"Response from add_secret: {response.text}") return json.loads(response.text) - except Exception as e: raise(f"Error adding secret: {e}") @@ -121,9 +119,8 @@ def delete_secret(self, secret_name=None): headers=self._api_header ) - logger.debug(response.text) + logger.debug(f"Response from delete_secret: {response.text}") return json.loads(response.text) - except Exception as e: raise(f"Error deleting secret: {e}") diff --git a/test/functional_test.py b/test/functional_test.py index 4a79e8b..de55154 100644 --- a/test/functional_test.py +++ b/test/functional_test.py @@ -121,10 +121,11 @@ def get_secrets(maap: MAAP): assert resp is not None @log_decorator -def get_secret(maap: MAAP, secret_name=None): +def get_secret(maap: MAAP, secret_name=None, secret_value=None): resp = maap.secrets.get_secret(secret_name) print(resp) assert resp is not None + assert resp == secret_value @log_decorator def delete_secret(maap: MAAP, secret_name=None): @@ -148,7 +149,7 @@ def main(): secret_value = "test_value" get_secrets(maap) add_secret(maap, secret_name, secret_value) - get_secret(maap, secret_name) + get_secret(maap, secret_name, secret_value) delete_secret(maap, secret_name) # submit_job(maap, wait_for_completion=True)