Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactors some url parsing logic #749

Merged
merged 1 commit into from
Apr 2, 2024
Merged

refactors some url parsing logic #749

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 25 additions & 19 deletions SearchAPI/CMR/Translate/parse_cmr_response.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,11 +205,31 @@ def float_or_none(a):
result['downloadUrl'] = urls[0]
result['fileName'] = result['granuleName'] + '.' + urls[0].split('.')[-1]



def get_all_urls():
accessPath = './OnlineAccessURLs/OnlineAccessURL/URL'
resourcesPath = './OnlineResources/OnlineResource/URL'

access_urls = get_all_vals(accessPath)
if access_urls is None:
access_urls = []

resource_urls = get_all_vals(resourcesPath)
if resource_urls is None:
resource_urls = []

return list(set([*access_urls, *resource_urls]))

def get_http_urls():
return [url for url in get_all_urls() if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]

def get_s3_urls():
return [url for url in get_all_urls() if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)]

if result.get('product_file_id', '').startswith('OPERA'):
result['beamMode'] = get_val(attr_path('BEAM_MODE'))
accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
result['additionalUrls'] = list(set([*accessUrls, *OnlineResources]))
result['additionalUrls'] = get_http_urls()
result['configurationName'] = "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses."

if (providerbrowseUrls := get_all_vals('./AssociatedBrowseImageUrls/ProviderBrowseUrl/URL')):
Expand All @@ -218,22 +238,8 @@ def float_or_none(a):
if 'STATIC' in result['processingLevel']:
result['validityStartDate'] = get_val('./Temporal/SingleDateTime')
if result.get('platform', '') == 'NISAR':
accessUrls = [url for url in get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
OnlineResources = [url for url in get_all_vals('./OnlineResources/OnlineResource/URL') if not url.endswith('.md5') and not url.startswith('s3://') and not 's3credentials' in url]
result['additionalUrls'] = list(set([*accessUrls, *OnlineResources]))

accessUrls = get_all_vals('./OnlineAccessURLs/OnlineAccessURL/URL')
if accessUrls is None:
accessUrls = []
resourceUrls = get_all_vals('./OnlineResources/OnlineResource/URL')
if resourceUrls is None:
resourceUrls = []

result['s3Urls'] = list(set(
[*[url for url in accessUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)],
*[url for url in resourceUrls if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)]
]
))
result['additionalUrls'] = get_http_urls()
result['s3Urls'] = get_s3_urls()
return result


Expand Down
Loading