diff --git a/astroquery/nrao/__init__.py b/astroquery/nrao/__init__.py new file mode 100644 index 0000000000..fd6eb14fa7 --- /dev/null +++ b/astroquery/nrao/__init__.py @@ -0,0 +1,44 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +NRAO Archive service. +""" +from astropy import config as _config + + +# list the URLs here separately so they can be used in tests. +_url_list = ['https://data.nrao.edu' + ] + +tap_urls = ['https://data-query.nrao.edu/'] + +auth_urls = ['data.nrao.edu'] + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.nrao`. + """ + + timeout = _config.ConfigItem(60, "Timeout in seconds.") + + archive_url = _config.ConfigItem( + _url_list, + 'The NRAO Archive mirror to use.') + + auth_url = _config.ConfigItem( + auth_urls, + 'NRAO Central Authentication Service URLs' + ) + + username = _config.ConfigItem( + "", + 'Optional default username for NRAO archive.') + + +conf = Conf() + +from .core import Nrao, NraoClass, NRAO_BANDS + +__all__ = ['Nrao', 'NraoClass', + 'Conf', 'conf', + ] diff --git a/astroquery/nrao/core.py b/astroquery/nrao/core.py new file mode 100644 index 0000000000..b1729e9fb9 --- /dev/null +++ b/astroquery/nrao/core.py @@ -0,0 +1,454 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import os.path +import keyring +import numpy as np +import re +import tarfile +import string +import requests +import warnings +import json +import time + +from pkg_resources import resource_filename +from bs4 import BeautifulSoup +import pyvo +from urllib.parse import urljoin + +from astropy.table import Table, Column, vstack +from astroquery import log +from astropy.utils.console import ProgressBar +from astropy import units as u +from astropy.time import Time + +try: + from pyvo.dal.sia2 import SIA2_PARAMETERS_DESC, SIA2Service +except ImportError: + # Can be removed once min version of pyvo is 1.5 + from pyvo.dal.sia2 import SIA_PARAMETERS_DESC as SIA2_PARAMETERS_DESC + from pyvo.dal.sia2 import SIAService as SIA2Service + +from ..exceptions import LoginError +from ..utils import commons +from ..query import BaseQuery, QueryWithLogin, BaseVOQuery +from . import conf, auth_urls, tap_urls +from astroquery.exceptions import CorruptDataWarning +from ..alma.tapsql import (_gen_str_sql, _gen_numeric_sql, + _gen_band_list_sql, _gen_datetime_sql, _gen_pol_sql, _gen_pub_sql, + _gen_science_sql, _gen_spec_res_sql, ALMA_DATE_FORMAT) +from .tapsql import (_gen_pos_sql) + +__all__ = {'NraoClass',} + +__doctest_skip__ = ['NraoClass.*'] + +NRAO_BANDS = { + 'L': (1*u.GHz, 2*u.GHz), + 'S': (2*u.GHz, 4*u.GHz), + 'C': (4*u.GHz, 8*u.GHz), + 'X': (8*u.GHz, 12*u.GHz), + 'U': (12*u.GHz, 18*u.GHz), + 'K': (18*u.GHz, 26*u.GHz), + 'A': (26*u.GHz, 39*u.GHz), + 'Q': (39*u.GHz, 50*u.GHz), + 'W': (80*u.GHz, 115*u.GHz) +} + +TAP_SERVICE_PATH = 'tap' + +NRAO_FORM_KEYS = { + 'Position': { + 'Source name (astropy Resolver)': ['source_name_resolver', + 'SkyCoord.from_name', _gen_pos_sql], + 'Source name (NRAO)': ['source_name', 'target_name', _gen_str_sql], + 'RA Dec (Sexagesimal)': ['ra_dec', 's_ra, s_dec', _gen_pos_sql], + 'Galactic (Degrees)': ['galactic', 'gal_longitude, gal_latitude', + _gen_pos_sql], + 'Angular resolution (arcsec)': ['spatial_resolution', + 'spatial_resolution', _gen_numeric_sql], + 'Field of view (arcsec)': ['fov', 's_fov', _gen_numeric_sql], + 'Configuration': ['configuration', 'configuration', _gen_numeric_sql], + 'Maximum UV Distance (meters)': ['max_uv_dist', 'max_uv_dist', _gen_numeric_sql] + + + }, + 'Project': { + 'Project code': ['project_code', 'project_code', _gen_str_sql], + 'Telescope': ['instrument', 'instrument_name', _gen_str_sql], + 'Number of Antennas': ['n_ants', 'num_antennas', _gen_str_sql], + + }, + 'Time': { + 'Observation start': ['start_date', 't_min', _gen_datetime_sql], + 'Observation end': ['end_date', 't_max', _gen_datetime_sql], + 'Integration time (s)': ['integration_time', 't_exptime', + _gen_numeric_sql] + }, + 'Polarization': { + 'Polarisation type (Single, Dual, Full)': ['polarisation_type', + 'pol_states', _gen_pol_sql] + }, + 'Energy': { + 'Frequency (GHz)': ['frequency', 'center_frequencies', _gen_numeric_sql], + 'Bandwidth (Hz)': ['bandwidth', 'aggregate_bandwidth', _gen_numeric_sql], + 'Spectral resolution (KHz)': ['spectral_resolution', + 'em_resolution', _gen_spec_res_sql], + 'Band': ['band_list', 'band_list', _gen_band_list_sql] + }, + +} + +_OBSCORE_TO_NRAORESULT = { + 's_ra': 'RA', + 's_dec': 'Dec', +} + + +def _gen_sql(payload): + sql = 'select * from tap_schema.obscore' + where = '' + unused_payload = payload.copy() + if payload: + for constraint in payload: + for attrib_category in NRAO_FORM_KEYS.values(): + for attrib in attrib_category.values(): + if constraint in attrib: + # use the value and the second entry in attrib which + # is the new name of the column + val = payload[constraint] + if constraint == 'em_resolution': + # em_resolution does not require any transformation + attrib_where = _gen_numeric_sql(constraint, val) + else: + attrib_where = attrib[2](attrib[1], val) + if attrib_where: + if where: + where += ' AND ' + else: + where = ' WHERE ' + where += attrib_where + + # Delete this key to see what's left over afterward + # + # Use pop to avoid the slight possibility of trying to remove + # an already removed key + unused_payload.pop(constraint) + + if unused_payload: + # Left over (unused) constraints passed. Let the user know. + remaining = [f'{p} -> {unused_payload[p]}' for p in unused_payload] + raise TypeError(f'Unsupported arguments were passed:\n{remaining}') + + return sql + where + + +# class NraoAuth(BaseVOQuery, BaseQuery): +# """ +# TODO: this needs to be implemented +# """ +# pass + + +class NraoClass(BaseQuery): + TIMEOUT = conf.timeout + archive_url = conf.archive_url + USERNAME = conf.username + + def __init__(self): + # sia service does not need disambiguation but tap does + super().__init__() + self._sia = None + self._tap = None + self._datalink = None + self._sia_url = None + self._tap_url = None + self._datalink_url = None + # TODO self._auth = NraoAuth() + + @property + def auth(self): + return self._auth + + @property + def datalink(self): + if not self._datalink: + self._datalink = pyvo.dal.adhoc.DatalinkService(self.datalink_url) + return self._datalink + + @property + def datalink_url(self): + if not self._datalink_url: + try: + self._datalink_url = urljoin(self._get_dataarchive_url(), DATALINK_SERVICE_PATH) + except requests.exceptions.HTTPError as err: + log.debug( + f"ERROR getting the NRAO Archive URL: {str(err)}") + raise err + return self._datalink_url + + @property + def sia(self): + if not self._sia: + self._sia = SIA2Service(baseurl=self.sia_url) + return self._sia + + @property + def sia_url(self): + if not self._sia_url: + try: + self._sia_url = urljoin(self._get_dataarchive_url(), SIA_SERVICE_PATH) + except requests.exceptions.HTTPError as err: + log.debug( + f"ERROR getting the NRAO Archive URL: {str(err)}") + raise err + return self._sia_url + + @property + def tap(self): + if not self._tap: + self._tap = pyvo.dal.tap.TAPService(baseurl=self.tap_url, session=self._session) + return self._tap + + @property + def tap_url(self): + if not self._tap_url: + try: + self._tap_url = urljoin(self._get_dataarchive_url(), TAP_SERVICE_PATH) + except requests.exceptions.HTTPError as err: + log.debug( + f"ERROR getting the NRAO Archive URL: {str(err)}") + raise err + return self._tap_url + + def query_tap(self, query, maxrec=None): + """ + Send query to the NRAO TAP. Results in pyvo.dal.TapResult format. + result.table in Astropy table format + + Parameters + ---------- + maxrec : int + maximum number of records to return + + """ + log.debug('TAP query: {}'.format(query)) + return self.tap.search(query, language='ADQL', maxrec=maxrec) + + def _get_dataarchive_url(self): + return tap_urls[0] + + def query_object(self, object_name, *, payload=None, **kwargs): + """ + Query the archive for a source name. + + Parameters + ---------- + object_name : str + The object name. Will be resolved by astropy.coord.SkyCoord + payload : dict + Dictionary of additional keywords. See `help`. + """ + if payload is not None: + payload['source_name_resolver'] = object_name + else: + payload = {'source_name_resolver': object_name} + return self.query(payload=payload, **kwargs) + + def query_region(self, coordinate, radius, *, + get_query_payload=False, + payload=None, **kwargs): + """ + Query the NRAO archive with a source name and radius + + Parameters + ---------- + coordinates : str / `astropy.coordinates` + the identifier or coordinates around which to query. + radius : str / `~astropy.units.Quantity`, optional + the radius of the region + payload : dict + Dictionary of additional keywords. See `help`. + """ + rad = radius + if not isinstance(radius, u.Quantity): + rad = radius*u.deg + obj_coord = commons.parse_coordinates(coordinate).icrs + ra_dec = '{}, {}'.format(obj_coord.to_string(), rad.to(u.deg).value) + if payload is None: + payload = {} + if 'ra_dec' in payload: + payload['ra_dec'] += ' | {}'.format(ra_dec) + else: + payload['ra_dec'] = ra_dec + + if get_query_payload: + return payload + + return self.query(payload=payload, **kwargs) + + def query(self, payload, *, get_query_payload=False, + maxrec=None, **kwargs): + """ + Perform a generic query with user-specified payload + + Parameters + ---------- + payload : dictionary + Please consult the `help` method + legacy_columns : bool + True to return the columns from the obsolete NRAO advanced query, + otherwise return the current columns based on ObsCore model. + get_query_payload : bool + Flag to indicate whether to simply return the payload. + maxrec : integer + Cap on the amount of records returned. Default is no limit. + [ we don't know for sure that this is implemented for NRAO ] + + Returns + ------- + + Table with results. + """ + + if payload is None: + payload = {} + for arg in kwargs: + value = kwargs[arg] + if arg in payload: + payload[arg] = '{} {}'.format(payload[arg], value) + else: + payload[arg] = value + query = _gen_sql(payload) + if get_query_payload: + # Return the TAP query payload that goes out to the server rather + # than the unprocessed payload dict from the python side + return query + + result = self.query_tap(query, maxrec=maxrec) + + if result is not None: + result = result.to_table() + else: + # Should not happen + raise RuntimeError('BUG: Unexpected result None') + + return result + + + def _get_data(self, solr_id, email=None, workflow='runBasicMsWorkflow', + apply_flags=True + ): + """ + This private function can, under a very limited set of circumstances, + be used to retrieve the data download page from the NRAO data handler. + Because the data handler is run through a fairly complex, multi-step, + private API, we are not yet ready to make this service public. + + Parameters + ---------- + workflow : 'runBasicMsWorkflow', "runDownloadWorkflow" + """ + url = f'{self.archive_url}/portal/#/subscanViewer/{solr_id}' + + #self._session.headers['User-Agent'] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + + resp = self._request('GET', url, cache=False) + resp.raise_for_status() + + eb_deets = self._request('GET', + f'{self.archive_url}/archive-service/restapi_get_full_exec_block_details', + params={'solr_id': solr_id}, + cache=False + ) + eb_deets.raise_for_status() + assert len(self._session.cookies) > 0 + + resp1b = self._request('GET', + f'{self.archive_url}/archive-service/restapi_spw_details_view', + params={'exec_block_id': solr_id.split(":")[-1]}, + cache=False + ) + resp1b.raise_for_status() + + # returned data is doubly json-encoded + jd = json.loads(eb_deets.json()) + locator = jd['curr_eb']['sci_prod_locator'] + project_code = jd['curr_eb']['project_code'] + + instrument = ('VLBA' if 'vlba' in solr_id.lower() else + 'VLA' if 'vla' in solr_id.lower() else + 'EVLA' if 'nrao' in solr_id.lower() else + 'GBT' if 'gbt' in solr_id.lower() else None) + if instrument is None: + raise ValueError("Invalid instrument") + + if instrument == 'VLBA': + downloadDataFormat = "VLBARaw" + elif instrument in ('VLA', 'EVLA'): + # there are other options! + downloadDataFormat = 'MS' + + post_data = { + "emailNotification": email, + "requestDescription": f"{instrument} Download Request", + "archive": "VLA", + "p_telescope": instrument, + "p_project": project_code, + "productLocator": locator, + "requestCommand": "startVlaPPIWorkflow", + "p_workflowEventName": workflow, + "p_downloadDataFormat": downloadDataFormat, + "p_intentsFileName": "intents_hifv.xml", + "p_proceduresFileName": "procedure_hifv.xml" + } + + if instrument in ('VLA', 'EVLA'): + post_data['p_applyTelescopeFlags'] = apply_flags + casareq = self._request('GET', + f'{self.archive_url}/archive-service/restapi_get_casa_version_list', + cache=False + ) + casareq.raise_for_status() + casavdata = json.loads(casareq.json()) + for casav in casavdata['casa_version_list']: + if 'recommended' in casav['version']: + post_data['p_casaHome'] = casav['path'] + + presp = self._request('POST', + f'{self.archive_url}/rh/submission', + data=post_data, + cache=False + ) + presp.raise_for_status() + + resp2 = self._request('GET', presp.url, cache=False) + resp2.raise_for_status() + + for row in resp2.text.split(): + if 'window.location.href=' in row: + subrespurl = row.split("'")[1] + + nextresp = self._request('GET', subrespurl, cache=False) + wait_url = nextresp.url + nextresp.raise_for_status() + + if f'{self.archive_url}/rh/requests/' not in wait_url: + raise ValueError(f"Got wrong URL from post request: {wait_url}") + + # to get the right format of response, you need to specify this: + # accept = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"} + + while True: + time.sleep(1) + print(".", end='', flush=True) + resp = self._request('GET', wait_url + "/state", cache=False) + resp.raise_for_status() + if resp.text == 'COMPLETE': + break + + return wait_url + + + +Nrao = NraoClass() diff --git a/astroquery/nrao/tapsql.py b/astroquery/nrao/tapsql.py new file mode 100644 index 0000000000..5a0d4bfe98 --- /dev/null +++ b/astroquery/nrao/tapsql.py @@ -0,0 +1,265 @@ +""" +Utilities for generating ADQL for ALMA TAP service +""" +from datetime import datetime + +from astropy import units as u +import astropy.coordinates as coord +from astropy.time import Time + +ALMA_DATE_FORMAT = '%d-%m-%Y' + + +def _gen_pos_sql(field, value): + result = '' + if field == 'SkyCoord.from_name': + # resolve the source first + if value: + obj_coord = coord.SkyCoord.from_name(value) + frame = 'icrs' + ras = [str(obj_coord.icrs.ra.to(u.deg).value)] + decs = [str(obj_coord.icrs.dec.to(u.deg).value)] + radius = 10 * u.arcmin + else: + raise ValueError('Object name missing') + else: + if field == 's_ra, s_dec': + frame = 'icrs' + else: + frame = 'galactic' + radius = 10*u.arcmin + if ',' in value: + center_coord, rad = value.split(',') + try: + radius = float(rad.strip())*u.degree + except ValueError: + raise ValueError('Cannot parse radius in ' + value) + else: + center_coord = value.strip() + try: + ra, dec = center_coord.split(' ') + except ValueError: + raise ValueError('Cannot find ra/dec in ' + value) + ras = _val_parse(ra, val_type=str) + decs = _val_parse(dec, val_type=str) + + for ra in ras: + for dec in decs: + if result: + result += ' OR ' + if isinstance(ra, str) and isinstance(dec, str): + # circle + center = coord.SkyCoord(ra, dec, + unit=(u.deg, u.deg), + frame=frame) + + result += \ + "CONTAINS(POINT('ICRS',s_ra,s_dec),CIRCLE('ICRS',{},{},{}))=1".\ + format(center.icrs.ra.to(u.deg).value, + center.icrs.dec.to(u.deg).value, + radius.to(u.deg).value) + else: + raise ValueError('Cannot interpret ra({}), dec({}'. + format(ra, dec)) + if ' OR ' in result: + # use brackets for multiple ORs + return '(' + result + ')' + else: + return result + + +def _gen_numeric_sql(field, value): + result = '' + for interval in _val_parse(value, float): + if result: + result += ' OR ' + if isinstance(interval, tuple): + int_min, int_max = interval + if int_min is None: + if int_max is None: + # no constraints on bandwith + pass + else: + result += '{}<={}'.format(field, int_max) + elif int_max is None: + result += '{}>={}'.format(field, int_min) + else: + result += '({1}<={0} AND {0}<={2})'.format(field, int_min, + int_max) + else: + result += '{}={}'.format(field, interval) + if ' OR ' in result: + # use brakets for multiple ORs + return '(' + result + ')' + else: + return result + + +def _gen_str_sql(field, value): + result = '' + for interval in _val_parse(value, str): + if result: + result += ' OR ' + if '*' in interval: + # use LIKE + # escape wildcards if they exists in the value + interval = interval.replace('%', r'\%') # noqa + interval = interval.replace('_', r'\_') # noqa + # ADQL wild cards are % and _ + interval = interval.replace('*', '%') + interval = interval.replace('?', '_') + result += "{} LIKE '{}'".format(field, interval) + else: + result += "{}='{}'".format(field, interval) + if ' OR ' in result: + # use brackets for multiple ORs + return '(' + result + ')' + else: + return result + + +def _gen_datetime_sql(field, value): + result = '' + for interval in _val_parse(value, str): + if result: + result += ' OR ' + if isinstance(interval, tuple): + min_datetime, max_datetime = interval + if max_datetime is None: + result += "{}>={}".format( + field, Time(datetime.strptime(min_datetime, ALMA_DATE_FORMAT)).mjd) + elif min_datetime is None: + result += "{}<={}".format( + field, Time(datetime.strptime(max_datetime, ALMA_DATE_FORMAT)).mjd) + else: + result += "({1}<={0} AND {0}<={2})".format( + field, Time(datetime.strptime(min_datetime, ALMA_DATE_FORMAT)).mjd, + Time(datetime.strptime(max_datetime, ALMA_DATE_FORMAT)).mjd) + else: + # TODO is it just a value (midnight) or the entire day? + result += "{}={}".format( + field, Time(datetime.strptime(interval, ALMA_DATE_FORMAT)).mjd) + if ' OR ' in result: + # use brackets for multiple ORs + return '(' + result + ')' + else: + return result + + +def _gen_spec_res_sql(field, value): + # This needs special treatment because spectral_resolution in AQ is in + # KHz while corresponding em_resolution is in m + result = '' + for interval in _val_parse(value): + if result: + result += ' OR ' + if isinstance(interval, tuple): + min_val, max_val = interval + if max_val is None: + result += "{}<={}".format( + field, + min_val*u.kHz.to(u.m, equivalencies=u.spectral())) + elif min_val is None: + result += "{}>={}".format( + field, + max_val*u.kHz.to(u.m, equivalencies=u.spectral())) + else: + result += "({1}<={0} AND {0}<={2})".format( + field, + max_val*u.kHz.to(u.m, equivalencies=u.spectral()), + min_val*u.kHz.to(u.m, equivalencies=u.spectral())) + else: + result += "{}={}".format( + field, interval*u.kHz.to(u.m, equivalencies=u.spectral())) + if ' OR ' in result: + # use brackets for multiple ORs + return '(' + result + ')' + else: + return result + + +def _gen_pub_sql(field, value): + if value is True: + return "{}='Public'".format(field) + elif value is False: + return "{}='Proprietary'".format(field) + else: + return None + + +def _gen_science_sql(field, value): + if value is True: + return "{}='T'".format(field) + elif value is False: + return "{}='F'".format(field) + else: + return None + + +def _gen_band_list_sql(field, value): + # band list value is expected to be space separated list of bands + if isinstance(value, list): + val = value + else: + val = value.split(' ') + return _gen_str_sql(field, '|'.join( + ['*{}*'.format(_) for _ in val])) + + +def _gen_pol_sql(field, value): + # band list value is expected to be space separated list of bands + val = '' + states_map = {'Stokes I': '*I*', + 'Single': '/LL/', + 'Dual': '/LL/RR/', + 'Full': '/LL/LR/RL/RR/'} + for state in states_map: + if state in value: + if val: + val += '|' + val += states_map[state] + return _gen_str_sql(field, val) + + +def _val_parse(value, val_type=float): + # parses an ALMA query field and returns a list of values (of type + # val_type) or tuples representing parsed values or intervals. Open + # intervals have None at one of the ends + def _one_val_parse(value, val_type=float): + # parses the value and returns corresponding interval for + # sia to work with. E.g <2 => (None, 2) + if value.startswith('<'): + return (None, val_type(value[1:])) + elif value.startswith('>'): + return (val_type(value[1:]), None) + else: + return val_type(value) + result = [] + if isinstance(value, str): + try: + if value.startswith('!'): + start, end = _val_parse(value[2:-1].strip(), val_type=val_type)[0] + result.append((None, start)) + result.append((end, None)) + elif value.startswith('('): + result += _val_parse(value[1:-1], val_type=val_type) + elif '|' in value: + for vv in value.split('|'): + result += _val_parse(vv.strip(), val_type=val_type) + elif '..' in value: + start, end = value.split('..') + if not start or not end: + raise ValueError('start or end interval missing in {}'. + format(value)) + result.append((_one_val_parse(start.strip(), val_type=val_type), + _one_val_parse(end.strip(), val_type=val_type))) + else: + result.append(_one_val_parse(value, val_type=val_type)) + except Exception as e: + raise ValueError( + 'Error parsing {}. Details: {}'.format(value, str(e))) + elif isinstance(value, list): + result = value + else: + result.append(value) + return result diff --git a/astroquery/nrao/tests/__init__.py b/astroquery/nrao/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/nrao/tests/data/nrao-empty.txt b/astroquery/nrao/tests/data/nrao-empty.txt new file mode 100644 index 0000000000..eaa6ed04c1 --- /dev/null +++ b/astroquery/nrao/tests/data/nrao-empty.txt @@ -0,0 +1 @@ +dataproduct_type calib_level obs_collection obs_id s_ra s_dec s_fov obs_publisher_did access_url access_format target_name s_region s_resolution t_min t_max t_exptime t_resolution freq_min freq_max em_min em_max em_res_power em_xel o_ucd facility_name instrument_name pol_states configuration access_estsize num_antennas max_uv_dist spw_names center_frequencies bandwidths nums_channels spectral_resolutions aggregate_bandwidth diff --git a/astroquery/nrao/tests/test_nrao.py b/astroquery/nrao/tests/test_nrao.py new file mode 100644 index 0000000000..9d036b5471 --- /dev/null +++ b/astroquery/nrao/tests/test_nrao.py @@ -0,0 +1,297 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +from io import StringIO +import os + +import pytest +from unittest.mock import patch, Mock + +from astropy import units as u +from astropy import coordinates as coord +from astropy.table import Table +from astropy.coordinates import SkyCoord +from astropy.time import Time + +from astroquery.nrao import Nrao +from astroquery.nrao.core import _gen_sql, _OBSCORE_TO_nraoRESULT +from astroquery.nrao.tapsql import _val_parse + + +DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') + + +def data_path(filename): + return os.path.join(DATA_DIR, filename) + + +def assert_called_with(mock, band=None, calib_level=None, collection=None, + data_rights=None, data_type=None, exptime=None, + facility=None, + field_of_view=None, instrument=None, maxrec=None, + pol=None, pos=None, publisher_did=None, res_format=None, + spatial_resolution=None, spectral_resolving_power=None, + target_name=None, time=None, timeres=None): + mock.assert_called_once_with( + band=band, calib_level=calib_level, + collection=collection, data_rights=data_rights, data_type=data_type, + exptime=exptime, facility=facility, + field_of_view=field_of_view, instrument=instrument, + maxrec=maxrec, pol=pol, pos=pos, publisher_did=publisher_did, + res_format=res_format, spatial_resolution=spatial_resolution, + spectral_resolving_power=spectral_resolving_power, + target_name=target_name, time=time, timeres=timeres) + +def test_gen_pos_sql(): + # test circle + # radius defaults to 1.0arcmin + common_select = 'select * from ivoa.obscore WHERE ' + assert _gen_sql({'ra_dec': '1 2'}) == common_select + "(INTERSECTS(" \ + "CIRCLE('ICRS',1.0,2.0,0.16666666666666666), s_region) = 1)" + assert _gen_sql({'ra_dec': '1 2, 3'}) == common_select + \ + "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,3.0), s_region) = 1)" + assert _gen_sql({'ra_dec': '12:13:14.0 -00:01:02.1, 3'}) == \ + common_select + \ + "(INTERSECTS(CIRCLE('ICRS',12.220555555555556,-0.01725,3.0), " \ + "s_region) = 1)" + # multiple circles + assert _gen_sql({'ra_dec': '1 20|40, 3'}) == common_select + \ + "((INTERSECTS(CIRCLE('ICRS',1.0,20.0,3.0), s_region) = 1) OR " \ + "(INTERSECTS(CIRCLE('ICRS',1.0,40.0,3.0), s_region) = 1))" + assert _gen_sql({'ra_dec': '1|10 20|40, 1'}) == common_select + \ + "((INTERSECTS(CIRCLE('ICRS',1.0,20.0,1.0), s_region) = 1) OR " \ + "(INTERSECTS(CIRCLE('ICRS',1.0,40.0,1.0), s_region) = 1) OR " \ + "(INTERSECTS(CIRCLE('ICRS',10.0,20.0,1.0), s_region) = 1) OR " \ + "(INTERSECTS(CIRCLE('ICRS',10.0,40.0,1.0), s_region) = 1))" + + # test range + assert _gen_sql({'ra_dec': '0.0..20.0 >20'}) == common_select + \ + "(INTERSECTS(RANGE_S2D(0.0,20.0,20.0,90.0), s_region) = 1)" + assert _gen_sql({'ra_dec': '12:13:14..12:13:20 <4:20:20'}) == \ + common_select +\ + "(INTERSECTS(RANGE_S2D(12.220555555555556,12.222222222222223," \ + "-90.0,4.338888888888889), s_region) = 1)" + assert _gen_sql({'ra_dec': '!(10..20) >60'}) == common_select + \ + "((INTERSECTS(RANGE_S2D(0.0,10.0,60.0,90.0), s_region) = 1) OR " \ + "(INTERSECTS(RANGE_S2D(20.0,0.0,60.0,90.0), s_region) = 1))" + assert _gen_sql({'ra_dec': '0..20|40..60 <-50|>50'}) == common_select + \ + "((INTERSECTS(RANGE_S2D(0.0,20.0,-90.0,-50.0), s_region) = 1) OR " \ + "(INTERSECTS(RANGE_S2D(0.0,20.0,50.0,90.0), s_region) = 1) OR " \ + "(INTERSECTS(RANGE_S2D(40.0,60.0,-90.0,-50.0), s_region) = 1) OR " \ + "(INTERSECTS(RANGE_S2D(40.0,60.0,50.0,90.0), s_region) = 1))" + + # galactic frame + center = coord.SkyCoord(1, 2, unit=u.deg, frame='galactic') + assert _gen_sql({'galactic': '1 2, 3'}) == common_select + "(INTERSECTS(" \ + "CIRCLE('ICRS',{},{},3.0), s_region) = 1)".format( + center.icrs.ra.to(u.deg).value, center.icrs.dec.to(u.deg).value) + min_point = coord.SkyCoord('12:13:14.0', '-00:01:02.1', unit=u.deg, + frame='galactic') + max_point = coord.SkyCoord('12:14:14.0', '-00:00:02.1', unit=(u.deg, u.deg), + frame='galactic') + assert _gen_sql( + {'galactic': '12:13:14.0..12:14:14.0 -00:01:02.1..-00:00:02.1'}) == \ + common_select +\ + "(INTERSECTS(RANGE_S2D({},{},{},{}), s_region) = 1)".format( + min_point.icrs.ra.to(u.deg).value, + max_point.icrs.ra.to(u.deg).value, + min_point.icrs.dec.to(u.deg).value, + max_point.icrs.dec.to(u.deg).value) + + # combination of frames + center = coord.SkyCoord(1, 2, unit=u.deg, frame='galactic') + assert _gen_sql({'ra_dec': '1 2, 3', 'galactic': '1 2, 3'}) == \ + "select * from ivoa.obscore WHERE " \ + "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,3.0), s_region) = 1) AND " \ + "(INTERSECTS(CIRCLE('ICRS',{},{},3.0), s_region) = 1)".format( + center.icrs.ra.to(u.deg).value, center.icrs.dec.to(u.deg).value) + + +def test_gen_numeric_sql(): + common_select = 'select * from ivoa.obscore WHERE ' + assert _gen_sql({'bandwidth': '23'}) == common_select + 'bandwidth=23.0' + assert _gen_sql({'bandwidth': '22 .. 23'}) == common_select +\ + '(22.0<=bandwidth AND bandwidth<=23.0)' + assert _gen_sql( + {'bandwidth': '<100'}) == common_select + 'bandwidth<=100.0' + assert _gen_sql( + {'bandwidth': '>100'}) == common_select + 'bandwidth>=100.0' + assert _gen_sql({'bandwidth': '!(20 .. 30)'}) == common_select + \ + '(bandwidth<=20.0 OR bandwidth>=30.0)' + assert _gen_sql({'bandwidth': '<10 | >20'}) == common_select + \ + '(bandwidth<=10.0 OR bandwidth>=20.0)' + assert _gen_sql({'bandwidth': 100, 'frequency': '>3'}) == common_select +\ + "bandwidth=100 AND frequency>=3.0" + + +def test_gen_str_sql(): + common_select = 'select * from ivoa.obscore WHERE ' + assert _gen_sql({'pub_title': '*Cosmic*'}) == common_select + \ + "pub_title LIKE '%Cosmic%'" + assert _gen_sql({'pub_title': 'Galaxy'}) == common_select + \ + "pub_title='Galaxy'" + assert _gen_sql({'pub_abstract': '*50% of the mass*'}) == common_select + \ + r"pub_abstract LIKE '%50\% of the mass%'" + assert _gen_sql({'project_code': '2012.* | 2013.?3*'}) == common_select + \ + "(proposal_id LIKE '2012.%' OR proposal_id LIKE '2013._3%')" + # test with brackets like the form example + assert _gen_sql({'project_code': '(2012.* | 2013.?3*)'}) == common_select + \ + "(proposal_id LIKE '2012.%' OR proposal_id LIKE '2013._3%')" + + +def test_gen_array_sql(): + # test string array input (regression in #2094) + # string arrays should be OR'd together + common_select = "select * from ivoa.obscore WHERE " + test_keywords = ["High-mass star formation", "Disks around high-mass stars"] + assert (_gen_sql({"spatial_resolution": "<0.1", "science_keyword": test_keywords}) + == common_select + ("spatial_resolution<=0.1 AND (science_keyword='High-mass star formation' " + "OR science_keyword='Disks around high-mass stars')")) + + +def test_gen_datetime_sql(): + common_select = 'select * from ivoa.obscore WHERE ' + assert _gen_sql({'start_date': '01-01-2020'}) == common_select + \ + "t_min=58849.0" + assert _gen_sql({'start_date': '>01-01-2020'}) == common_select + \ + "t_min>=58849.0" + assert _gen_sql({'start_date': '<01-01-2020'}) == common_select + \ + "t_min<=58849.0" + assert _gen_sql({'start_date': '(01-01-2020 .. 01-02-2020)'}) == \ + common_select + "(58849.0<=t_min AND t_min<=58880.0)" + + +def test_gen_spec_res_sql(): + common_select = 'select * from ivoa.obscore WHERE ' + assert _gen_sql({'spectral_resolution': 70}) == common_select + "em_resolution=20985472.06" + assert _gen_sql({'spectral_resolution': '<70'}) == common_select + "em_resolution>=20985472.06" + assert _gen_sql({'spectral_resolution': '>70'}) == common_select + "em_resolution<=20985472.06" + assert _gen_sql({'spectral_resolution': '(70 .. 80)'}) == common_select + \ + "(23983396.64<=em_resolution AND em_resolution<=20985472.06)" + assert _gen_sql({'spectral_resolution': '(70|80)'}) == common_select + \ + "(em_resolution=20985472.06 OR em_resolution=23983396.64)" + + +def test_gen_public_sql(): + common_select = 'select * from ivoa.obscore' + assert _gen_sql({'public_data': None}) == common_select + assert _gen_sql({'public_data': True}) == common_select +\ + " WHERE data_rights='Public'" + assert _gen_sql({'public_data': False}) == common_select + \ + " WHERE data_rights='Proprietary'" + + +def test_gen_science_sql(): + common_select = 'select * from ivoa.obscore' + assert _gen_sql({'science_observation': None}) == common_select + assert _gen_sql({'science_observation': True}) == common_select +\ + " WHERE science_observation='T'" + assert _gen_sql({'science_observation': False}) == common_select +\ + " WHERE science_observation='F'" + + +def test_pol_sql(): + common_select = 'select * from ivoa.obscore' + assert _gen_sql({'polarisation_type': 'Stokes I'}) == common_select +\ + " WHERE pol_states LIKE '%I%'" + assert _gen_sql({'polarisation_type': 'Single'}) == common_select + \ + " WHERE pol_states='/XX/'" + assert _gen_sql({'polarisation_type': 'Dual'}) == common_select + \ + " WHERE pol_states='/XX/YY/'" + assert _gen_sql({'polarisation_type': 'Full'}) == common_select + \ + " WHERE pol_states='/XX/XY/YX/YY/'" + assert _gen_sql({'polarisation_type': ['Single', 'Dual']}) == \ + common_select + " WHERE (pol_states='/XX/' OR pol_states='/XX/YY/')" + assert _gen_sql({'polarisation_type': 'Single, Dual'}) == \ + common_select + " WHERE (pol_states='/XX/' OR pol_states='/XX/YY/')" + + +def test_unused_args(): + nrao = Nrao() + nrao._get_dataarchive_url = Mock() + # with patch('astroquery.nrao.tapsql.coord.SkyCoord.from_name') as name_mock, pytest.raises(TypeError) as typeError: + with patch('astroquery.nrao.tapsql.coord.SkyCoord.from_name') as name_mock: + with pytest.raises(TypeError) as typeError: + name_mock.return_value = SkyCoord(1, 2, unit='deg') + nrao.query_object('M13', public=False, bogus=True, nope=False, band_list=[3]) + + assert "['bogus -> True', 'nope -> False']" in str(typeError.value) + + +def test_query(): + # Tests the query and return values + tap_mock = Mock() + empty_result = Table.read(os.path.join(DATA_DIR, 'nrao-empty.txt'), + format='ascii') + mock_result = Mock() + mock_result.to_table.return_value = empty_result + tap_mock.search.return_value = mock_result + nrao = Nrao() + nrao._get_dataarchive_url = Mock() + nrao._tap = tap_mock + result = nrao.query_region(SkyCoord(1*u.deg, 2*u.deg, frame='icrs'), + radius=0.001*u.deg) + assert len(result) == 0 + assert 'proposal_id' in result.columns + tap_mock.search.assert_called_once_with( + "select * from tap_schema.obscore WHERE CONTAINS(POINT('ICRS',s_ra,s_dec),CIRCLE('ICRS',1.0,2.0,1.0))=1", + language='ADQL', maxrec=None) + + # one row result + tap_mock = Mock() + onerow_result = Table.read(os.path.join(DATA_DIR, 'nrao-onerow.txt'), + format='ascii') + mock_result = Mock() + mock_result.to_table.return_value = onerow_result + tap_mock.search.return_value = mock_result + nrao = Nrao() + nrao._tap = tap_mock + with patch('astroquery.nrao.tapsql.coord.SkyCoord.from_name') as name_mock: + name_mock.return_value = SkyCoord(1, 2, unit='deg') + # mock data generated by running this query w/maxrec=5 + result = nrao.query_object('M83') + assert len(result) == 1 + + tap_mock.search.assert_called_once_with( + "select * from tap_schema.obscore WHERE CONTAINS(POINT('ICRS',s_ra,s_dec),CIRCLE('ICRS',204.25383,-29.865761111,0.16666666666666666))=1", + language='ADQL', maxrec=None) + + + +def test_tap(): + tap_mock = Mock() + empty_result = Table.read(os.path.join(DATA_DIR, 'nrao-empty.txt'), + format='ascii') + tap_mock.search.return_value = Mock(table=empty_result) + nrao = Nrao() + nrao._get_dataarchive_url = Mock() + nrao._tap = tap_mock + result = nrao.query_tap('select * from tap_scheme.ObsCore') + assert len(result.table) == 0 + + tap_mock.search.assert_called_once_with('select * from tap_scheme.ObsCore', + language='ADQL', maxrec=None) + + +def _test_tap_url(data_archive_url): + nrao = Nrao() + nrao._get_dataarchive_url = Mock(return_value=data_archive_url) + nrao._get_dataarchive_url.reset_mock() + assert nrao.tap_url == f"{data_archive_url}/tap" + + +def test_galactic_query(): + """ + regression test for 1867 + """ + tap_mock = Mock() + empty_result = Table.read(os.path.join(DATA_DIR, 'nrao-empty.txt'), + format='ascii') + mock_result = Mock() + mock_result.to_table.return_value = empty_result + tap_mock.search.return_value = mock_result + nrao = Nrao() + nrao._get_dataarchive_url = Mock() + nrao._tap = tap_mock + result = nrao.query_region(SkyCoord(0*u.deg, 0*u.deg, frame='galactic'), + radius=1*u.deg, get_query_payload=True) + + assert "'ICRS',266.405,-28.9362,1.0" in result diff --git a/astroquery/nrao/tests/test_nrao_remote.py b/astroquery/nrao/tests/test_nrao_remote.py new file mode 100644 index 0000000000..2bb03286bf --- /dev/null +++ b/astroquery/nrao/tests/test_nrao_remote.py @@ -0,0 +1,34 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +from datetime import datetime, timezone +import os +from pathlib import Path +from urllib.parse import urlparse +import re +from unittest.mock import Mock, MagicMock, patch + +from astropy import coordinates +from astropy import units as u +import numpy as np +import pytest + +from pyvo.dal.exceptions import DALOverflowWarning + +from astroquery.exceptions import CorruptDataWarning +from .. import Nrao + + +@pytest.mark.remote_data +class TestNrao: + def test_SgrAstar(self, tmp_path, nrao): + nrao.cache_location = tmp_path + result_s = nrao.query_object('Sgr A*', maxrec=5) + + def test_ra_dec(self, nrao): + payload = {'ra_dec': '181.0192d -0.01928d'} + result = nrao.query(payload) + assert len(result) > 0 + + def test_query(self, tmp_path, nrao): + nrao.cache_location = tmp_path + + result = nrao.query_object('M83', maxrec=5)