Merge pull request #747 from asfadmin/feature-s3-urls

NISAR S3 & Additional Urls
asfadmin · Apr 1, 2024 · 5664c24 · 5664c24
2 parents e520313 + ba3bd35
commit 5664c24
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 1 deletion.
diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py
@@ -42,7 +42,8 @@ def req_fields_jsonlite():
         'subswath',
         'pgeVersion',
         'operaBurstID',
-        'additionalUrls'
+        'additionalUrls',
+        's3Urls'
     ]
     return fields
 
@@ -186,4 +187,10 @@ def getItem(self, p):
             if p.get('validityStartDate'):
                 result['opera']['validityStartDate'] = p.get('validityStartDate')
 
+        if p.get('platform') == 'NISAR':
+            result['nisar'] = {
+                'additionalUrls': p.get('additionalUrls', []),
+                's3Urls': p.get('s3Urls', [])
+            }
+
         return result
diff --git a/SearchAPI/CMR/Output/jsonlite2.py b/SearchAPI/CMR/Output/jsonlite2.py
@@ -61,4 +61,7 @@ def getItem(self, p):
         if p.get('opera') is not None:
             result['s1o'] = p['opera']
 
+        if p.get('nisar') is not None:
+            result['nsr'] = p['nisar']
+
         return result
diff --git a/SearchAPI/CMR/Translate/fields.py b/SearchAPI/CMR/Translate/fields.py
@@ -61,6 +61,7 @@ def get_field_paths():
         'track':                    attr_path('PATH_NUMBER'),
         'pgeVersion':               "./PGEVersionClass/PGEVersion",
         'additionalUrls':           "./OnlineAccessURLs",
+        's3Urls':                   "./OnlineAccessURLs",
 
         # BURST FIELDS
         'absoluteBurstID':          attr_path('BURST_ID_ABSOLUTE'),

diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py
@@ -217,7 +217,23 @@ def float_or_none(a):
 
         if 'STATIC' in result['processingLevel']:
             result['validityStartDate'] = get_val('./Temporal/SingleDateTime')
+    if result.get('platform', '') == 'NISAR':
+        accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
+        OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
+        result['additionalUrls'] = list(set([*accessUrls, *OnlineResources]))
 
+        accessUrls = get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL')
+        if accessUrls is None:
+            accessUrls = []
+        resourceUrls = get_all_vals('./OnlineResources/OnlineResource/URL')
+        if resourceUrls is None:
+            resourceUrls = []
+
+        result['s3Urls'] = list(set(
+            [*[url for url in accessUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)],
+            *[url for url in resourceUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)]
+            ]
+            ))
     return result