From 0e32a0f2cd202a4b1cae04c09af2ab0bd1003837 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 27 Mar 2024 11:02:47 -0700 Subject: [PATCH 1/2] adds s3urls to jsonlite and jsonlite2 endpoints --- SearchAPI/CMR/Output/jsonlite.py | 6 +++++- SearchAPI/CMR/Output/jsonlite2.py | 3 +++ SearchAPI/CMR/Translate/parse_cmr_response.py | 9 ++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index 99795463..e6b56117 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -42,7 +42,8 @@ def req_fields_jsonlite(): 'subswath', 'pgeVersion', 'operaBurstID', - 'additionalUrls' + 'additionalUrls', + 's3Urls' ] return fields @@ -186,4 +187,7 @@ def getItem(self, p): if p.get('validityStartDate'): result['opera']['validityStartDate'] = p.get('validityStartDate') + if p.get('platform') == 'NISAR': + result['s3Urls'] = p.get('s3Urls', []) + return result diff --git a/SearchAPI/CMR/Output/jsonlite2.py b/SearchAPI/CMR/Output/jsonlite2.py index b403e566..2758f963 100644 --- a/SearchAPI/CMR/Output/jsonlite2.py +++ b/SearchAPI/CMR/Output/jsonlite2.py @@ -61,4 +61,7 @@ def getItem(self, p): if p.get('opera') is not None: result['s1o'] = p['opera'] + if p.get('platform') == 'NISAR': + result['s3du'] = p.get('s3Urls', []) + return result diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 29b2c0e9..c4345cf9 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -217,7 +217,14 @@ def float_or_none(a): if 'STATIC' in result['processingLevel']: result['validityStartDate'] = get_val('./Temporal/SingleDateTime') - + if result.get('platform', '') == 'NISAR': + accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] + OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] + result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) + result['s3Urls'] = list(set( + *[url for url in get_all_vals('./OnlineResources/OnlineAccessURL/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], + *[url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + )) return result From ba3bd35e7098d72cb3f05b90e3eeeacd7413be0e Mon Sep 17 00:00:00 2001 From: kim Date: Mon, 1 Apr 2024 08:32:42 -0800 Subject: [PATCH 2/2] reveals s3urls and additionalUrls for nisar products --- SearchAPI/CMR/Output/jsonlite.py | 5 ++++- SearchAPI/CMR/Output/jsonlite2.py | 4 ++-- SearchAPI/CMR/Translate/fields.py | 1 + SearchAPI/CMR/Translate/parse_cmr_response.py | 13 +++++++++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index e6b56117..660440a7 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -188,6 +188,9 @@ def getItem(self, p): result['opera']['validityStartDate'] = p.get('validityStartDate') if p.get('platform') == 'NISAR': - result['s3Urls'] = p.get('s3Urls', []) + result['nisar'] = { + 'additionalUrls': p.get('additionalUrls', []), + 's3Urls': p.get('s3Urls', []) + } return result diff --git a/SearchAPI/CMR/Output/jsonlite2.py b/SearchAPI/CMR/Output/jsonlite2.py index 2758f963..5cdff707 100644 --- a/SearchAPI/CMR/Output/jsonlite2.py +++ b/SearchAPI/CMR/Output/jsonlite2.py @@ -61,7 +61,7 @@ def getItem(self, p): if p.get('opera') is not None: result['s1o'] = p['opera'] - if p.get('platform') == 'NISAR': - result['s3du'] = p.get('s3Urls', []) + if p.get('nisar') is not None: + result['nsr'] = p['nisar'] return result diff --git a/SearchAPI/CMR/Translate/fields.py b/SearchAPI/CMR/Translate/fields.py index b9f5bebe..f639ff28 100644 --- a/SearchAPI/CMR/Translate/fields.py +++ b/SearchAPI/CMR/Translate/fields.py @@ -61,6 +61,7 @@ def get_field_paths(): 'track': attr_path('PATH_NUMBER'), 'pgeVersion': "./PGEVersionClass/PGEVersion", 'additionalUrls': "./OnlineAccessURLs", + 's3Urls': "./OnlineAccessURLs", # BURST FIELDS 'absoluteBurstID': attr_path('BURST_ID_ABSOLUTE'), diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index c4345cf9..08528328 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -221,9 +221,18 @@ def float_or_none(a): accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url] result['additionalUrls'] = list(set([*accessUrls, *OnlineResources])) + + accessUrls = get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') + if accessUrls is None: + accessUrls = [] + resourceUrls = get_all_vals('./OnlineResources/OnlineResource/URL') + if resourceUrls is None: + resourceUrls = [] + result['s3Urls'] = list(set( - *[url for url in get_all_vals('./OnlineResources/OnlineAccessURL/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], - *[url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + [*[url for url in accessUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)], + *[url for url in resourceUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] + ] )) return result