From e39bda5dd53d26544aee7dbd343e7c5407c74a91 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 2 Nov 2023 11:14:54 -0800 Subject: [PATCH] adds dataset constants and 'datasets' searchable parameter, for use by Vertex's dataset based searches --- SearchAPI/CMR/Translate/__init__.py | 1 + SearchAPI/CMR/Translate/datasets.py | 365 +++++++++++++++++++++++++ SearchAPI/CMR/Translate/input_fixer.py | 8 + SearchAPI/CMR/Translate/input_map.py | 3 +- 4 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 SearchAPI/CMR/Translate/datasets.py diff --git a/SearchAPI/CMR/Translate/__init__.py b/SearchAPI/CMR/Translate/__init__.py index d9f9b1fd..0a0e2d65 100644 --- a/SearchAPI/CMR/Translate/__init__.py +++ b/SearchAPI/CMR/Translate/__init__.py @@ -3,3 +3,4 @@ from .translate_params import translate_params from .input_fixer import input_fixer from .fields import get_field_paths +from .datasets import platform_datasets \ No newline at end of file diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py new file mode 100644 index 00000000..fef03061 --- /dev/null +++ b/SearchAPI/CMR/Translate/datasets.py @@ -0,0 +1,365 @@ +platform_datasets = { + "SENTINEL-1": [ + "C1214470488-ASF", + "C1327985661-ASF", + "C1214470533-ASF", + "C1214470576-ASF", + "C1327985645-ASF", + "C1214470496-ASF", + "C1214470532-ASF", + "C1214472977-ASF", + "C1214472336-ASF", + "C1266376001-ASF", + "C1214470732-ASF", + "C1327985660-ASF", + "C1327985741-ASF", + "C1327985578-ASF", + "C1327985650-ASF", + "C1327985617-ASF", + "C1327985579-ASF", + "C1327985619-ASF", + "C1214472994-ASF", + "C1214473170-ASF", + "C1327985646-ASF", + "C1327985740-ASF", + "C1327985739-ASF", + "C1214470561-ASF", + "C1214471521-ASF", + "C1214470682-ASF", + "C1327985647-ASF", + "C1214471197-ASF", + "C1214471960-ASF", + "C1214472978-ASF", + "C1214473165-ASF", + "C1327985697-ASF", + "C1327985651-ASF", + "C1327985644-ASF", + "C1327985571-ASF", + "C1327985674-ASF", + "C2450786986-ASF", + "C1205428742-ASF", + "C1216244348-ASF", + "C1212201032-ASF", + "C1216244589-ASF", + "C1212212560-ASF", + "C1205264459-ASF", + "C1216244594-ASF", + "C1216244593-ASF", + "C1208117434-ASF", + "C1212209035-ASF", + "C1212209226-ASF", + "C1216244585-ASF", + "C1216244592-ASF", + "C1208115009-ASF", + "C1216244595-ASF", + "C1212158327-ASF", + "C1215704763-ASF", + "C1212158318-ASF", + "C1216244590-ASF", + "C1216244601-ASF", + "C1216244600-ASF", + "C1212212493-ASF", + "C1212158326-ASF", + "C1212233976-ASF", + "C1216244591-ASF", + "C1216244587-ASF", + "C1216244598-ASF", + "C1216244586-ASF", + "C1212200781-ASF", + "C1212209075-ASF", + "C1216244597-ASF", + "C1216244596-ASF", + "C1216244588-ASF", + "C1216244599-ASF", + "C1245953394-ASF", + "C1234413245-ASFDEV", + "C1234413263-ASFDEV", + "C1234413229-ASFDEV", + "C1234413247-ASFDEV", + "C1234413237-ASFDEV", + "C1234413238-ASFDEV", + "C1234413248-ASFDEV", + "C1234413255-ASFDEV", + "C1234413236-ASFDEV", + "C1234413230-ASFDEV", + "C1234413232-ASFDEV", + "C1234413254-ASFDEV", + "C1234413256-ASFDEV", + "C1234413235-ASFDEV", + "C1234413253-ASFDEV", + "C1234413240-ASFDEV", + "C1234413234-ASFDEV", + "C1234413241-ASFDEV", + "C1234413252-ASFDEV", + "C1234413250-ASFDEV", + "C1234413259-ASFDEV", + "C1234413233-ASFDEV", + "C1234413243-ASFDEV", + "C1234413244-ASFDEV", + "C1234413251-ASFDEV", + "C1234413261-ASFDEV", + "C1234413262-ASFDEV", + "C1234413258-ASFDEV", + "C1244552887-ASFDEV", + "C1234413228-ASFDEV", + "C1234413231-ASFDEV", + "C1234413239-ASFDEV", + "C1234413242-ASFDEV", + "C1234413246-ASFDEV", + "C1234413249-ASFDEV", + "C1234413257-ASFDEV", + "C1234413260-ASFDEV", + "C1257175154-ASFDEV", + "C1244598379-ASFDEV", + "C1240784657-ASFDEV", + ], + "OPERA-S1": [ + "C2777443834-ASF", + "C2777436413-ASF", + "C1258354200-ASF", # maybe extra? + "C1259982010-ASF", + "C1257995185-ASF", + "C1259976861-ASF", + "C1258354201-ASF", + "C1259981910-ASF", + "C1257995186-ASF", + "C1259974840-ASF", + ], + "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], + "ALOS PALSAR": [ + "C1206487504-ASF", + "C1206485940-ASF", + "C1206487217-ASF", + "C1206156901-ASF", + "C1206485320-ASF", + "C1206485527-ASF", + "C2011599335-ASF", + "C1207181535-ASF", + "C1207710476-ASF", + "C1239927797-ASF", + "C1205261223-ASF", + "C1208013295-ASF", + "C1207019609-ASF", + "C1229740239-ASF", + "C1239611505-ASFDEV", + "C1238733834-ASFDEV", + ], + "ALOS AVNIR-2": [ + "C1808440897-ASF", + "C1234712303-ASF", + "C1233629671-ASF", + "C1234413224-ASFDEV", + ], + "SIR-C": [ + "C1661710578-ASF", + "C1661710581-ASF", + "C1661710583-ASF", + "C1661710586-ASF", + "C1661710588-ASF", + "C1661710590-ASF", + "C1661710593-ASF", + "C1661710596-ASF", + "C1661710597-ASF", + "C1661710600-ASF", + "C1661710603-ASF", + "C1661710604-ASF", + "C1226557813-ASF", + "C1226557814-ASF", + "C1226557819-ASF", + "C1226557809-ASF", + "C1226557808-ASF", + "C1226557810-ASF", + "C1226557811-ASF", + "C1226557812-ASF", + "C1226557815-ASF", + "C1226557816-ASF", + "C1226557817-ASF", + "C1226557818-ASF", + "C1234413270-ASFDEV", + "C1234413271-ASFDEV", + "C1234413264-ASFDEV", + "C1234413265-ASFDEV", + "C1234413266-ASFDEV", + "C1234413267-ASFDEV", + "C1234413268-ASFDEV", + "C1234413269-ASFDEV", + "C1234413272-ASFDEV", + "C1234413273-ASFDEV", + "C1234413274-ASFDEV", + "C1234413275-ASFDEV", + ], + "ARIA S1 GUNW": [ + "C1595422627-ASF", + "C1596065640-ASF", + "C1596065639-ASF", + "C1596065641-ASF", + "C1595765183-ASF", + "C1225776654-ASF", + "C1225776655-ASF", + "C1225776657-ASF", + "C1225776658-ASF", + "C1225776659-ASF", + ], + "SMAP": [ + "C1243122884-ASF", + "C1243124139-ASF", + "C1214473171-ASF", + "C1243149604-ASF", + "C1214473426-ASF", + "C1243119801-ASF", + "C1214473839-ASF", + "C1243133204-ASF", + "C1243197402-ASF", + "C1243215430-ASF", + "C1243124754-ASF", + "C1243141638-ASF", + "C1243162394-ASF", + "C1243168733-ASF", + "C1243168866-ASF", + "C1214473308-ASF", + "C1243253631-ASF", + "C1243133445-ASF", + "C1214473550-ASF", + "C1243197502-ASF", + "C1243126328-ASF", + "C1214474243-ASF", + "C1243216659-ASF", + "C1243129847-ASF", + "C1214473367-ASF", + "C1243268956-ASF", + "C1243144528-ASF", + "C1214473624-ASF", + "C1243228612-ASF", + "C1243136142-ASF", + "C1214474435-ASF", + "C1243255360-ASF", + "C1243140611-ASF", + "C1233103964-ASF", + "C1216074923-ASF", + "C1212243761-ASF", + "C1213091807-ASF", + "C1216074922-ASF", + "C1212243437-ASF", + "C1213096699-ASF", + "C1216074750-ASF", + "C1212249653-ASF", + "C1213101573-ASF", + "C1216074751-ASF", + "C1213136240-ASF", + "C1216074755-ASF", + "C1213136752-ASF", + "C1213136799-ASF", + "C1213136709-ASF", + "C1213136844-ASF", + "C1212249811-ASF", + "C1213125007-ASF", + "C1216074919-ASF", + "C1212196951-ASF", + "C1213115690-ASF", + "C1216074758-ASF", + "C1212243666-ASF", + "C1213115896-ASF", + "C1216074761-ASF", + "C1212250364-ASF", + "C1213134622-ASF", + "C1216074770-ASF", + "C1212246173-ASF", + "C1213125156-ASF", + "C1216074764-ASF", + "C1212249773-ASF", + "C1213134486-ASF", + "C1233101609-ASF", + ], + "UAVSAR": [ + "C1214353986-ASF", + "C1214336717-ASF", + "C1214336045-ASF", + "C1214335430-ASF", + "C1214335471-ASF", + "C1214335903-ASF", + "C1214336154-ASF", + "C1214336554-ASF", + "C1214353593-ASF", + "C1214353754-ASF", + "C1214353859-ASF", + "C1214337770-ASF", + "C1214354144-ASF", + "C1214354235-ASF", + "C1214343609-ASF", + "C1214354031-ASF", + "C1214408428-ASF", + "C1214419355-ASF", + "C1210487703-ASF", + "C1212030772-ASF", + "C1206116665-ASF", + "C1206132445-ASF", + "C1211962154-ASF", + "C1212001698-ASF", + "C1212005594-ASF", + "C1212019993-ASF", + "C1207638502-ASF", + "C1210025872-ASF", + "C1210485039-ASF", + "C1207188317-ASF", + "C1210546638-ASF", + "C1206122195-ASF", + "C1209970710-ASF", + "C1207038647-ASF", + "C1210599503-ASF", + "C1210599673-ASF", + ], + "RADARSAT-1": [ + "C1206897141-ASF", + "C1206936391-ASF", + "C1205181982-ASF", + "C1215670813-ASF", + "C1215709884-ASF", + "C1215709880-ASF", + "C1215714443-ASF", + "C1215775284-ASF", + "C1215614037-ASF", + ], + "ERS": [ + "C1210197768-ASF", + "C1211627521-ASF", + "C1208794942-ASF", + "C1209373626-ASF", + "C1207143701-ASF", + "C1205261222-ASF", + "C1205302527-ASF", + "C1207144966-ASF", + ], + "JERS-1": [ + "C1208662092-ASF", + "C1207933168-ASF", + "C1207175327-ASF", + "C1207177736-ASF", + ], + "AIRSAR": [ + "C1213921661-ASF", + "C1213921626-ASF", + "C1213928843-ASF", + "C1213927035-ASF", + "C179001730-ASF", + "C1213925022-ASF", + "C1213926419-ASF", + "C1213926777-ASF", + "C1213927939-ASF", + "C1213928209-ASF", + "C1205256880-ASF", + "C1208652494-ASF", + "C1000000306-ASF", + "C1208713702-ASF", + "C1208707768-ASF", + "C1208655639-ASF", + "C1208680681-ASF", + "C1208691361-ASF", + "C1208703384-ASF", + ], + "SEASAT": [ + "C1206500826-ASF", + "C1206500991-ASF", + "C1206752770-ASF", + "C1206144699-ASF", + ], +} diff --git a/SearchAPI/CMR/Translate/input_fixer.py b/SearchAPI/CMR/Translate/input_fixer.py index a7b56196..0a751532 100644 --- a/SearchAPI/CMR/Translate/input_fixer.py +++ b/SearchAPI/CMR/Translate/input_fixer.py @@ -5,6 +5,7 @@ from SearchAPI.asf_env import get_config from .collections_by_platform import collections_by_platform, collections_by_platform_uat, collections_by_platform_uat_asfdev +from .datasets import platform_datasets def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): """ @@ -107,6 +108,13 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): if any_processing_level: fixed_params['collections'] = collection_list + elif k == 'datasets': + fixed_params['collections'] = [] + for dataset in params[k]: + logging.warn(dataset) + logging.warn(platform_datasets.get(dataset)) + fixed_params['collections'].extend(platform_datasets.get(dataset)) + logging.warn(fixed_params) elif k == 'beammode': beammap = { 'STD': 'Standard' diff --git a/SearchAPI/CMR/Translate/input_map.py b/SearchAPI/CMR/Translate/input_map.py index 9fce56a0..4b4a50e9 100644 --- a/SearchAPI/CMR/Translate/input_map.py +++ b/SearchAPI/CMR/Translate/input_map.py @@ -56,7 +56,8 @@ def input_map(): 'relativeburstid': ['attribute[]', 'int,BURST_ID_RELATIVE,{0}', parse_int_list], 'absoluteburstid': ['attribute[]', 'int,BURST_ID_ABSOLUTE,{0}', parse_int_list], 'fullburstid': ['attribute[]', 'string,BURST_ID_FULL,{0}', parse_string_list], - 'operaburstid': ['attribute[]', 'string,OPERA_BURST_ID,{0}', parse_string_list] + 'operaburstid': ['attribute[]', 'string,OPERA_BURST_ID,{0}', parse_string_list], + 'datasets': [None, '{0}', parse_string_list] } return parameter_map