diff --git a/CHANGES.rst b/CHANGES.rst index d951b9a5ad..2569ee3741 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -83,6 +83,11 @@ gaia With this change the epoch photometry service returns all data associated to a given source. [#2376] +sdss +^^^^ + +- ``query_region()`` now does a cone search around the specified + coordinates. [#2477] Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- diff --git a/astroquery/sdss/core.py b/astroquery/sdss/core.py index de0e91e451..32f7605a02 100644 --- a/astroquery/sdss/core.py +++ b/astroquery/sdss/core.py @@ -7,7 +7,7 @@ import numpy as np from astropy import units as u -import astropy.coordinates as coord +from astropy.coordinates import Angle from astropy.table import Table, Column from ..query import BaseQuery @@ -27,6 +27,7 @@ @async_to_sync class SDSSClass(BaseQuery): TIMEOUT = conf.timeout + MAX_CROSSID_RADIUS = 3.0 * u.arcmin QUERY_URL_SUFFIX_DR_OLD = '/dr{dr}/en/tools/search/x_sql.asp' QUERY_URL_SUFFIX_DR_10 = '/dr{dr}/en/tools/search/x_sql.aspx' QUERY_URL_SUFFIX_DR_NEW = '/dr{dr}/en/tools/search/x_results.aspx' @@ -55,19 +56,21 @@ class SDSSClass(BaseQuery): 'qso_bal': [30, 31], 'qso_bright': 32 } - def query_crossid_async(self, coordinates, obj_names=None, - photoobj_fields=None, specobj_fields=None, - get_query_payload=False, timeout=TIMEOUT, - radius=5. * u.arcsec, + def query_crossid_async(self, coordinates, *, radius=5. * u.arcsec, timeout=TIMEOUT, + fields=None, photoobj_fields=None, specobj_fields=None, obj_names=None, + spectro=False, region=False, field_help=False, get_query_payload=False, data_release=conf.default_release, cache=True): """ Query using the cross-identification web interface. + This query returns the nearest `primary object`_. + + .. _`primary object`: https://www.sdss.org/dr17/help/glossary/#surveyprimary + Parameters ---------- - coordinates : str or `astropy.coordinates` object or list of - coordinates or `~astropy.table.Column` of coordinates The - target(s) around which to search. It may be specified as a + coordinates : str or `astropy.coordinates` object or list of coordinates or `~astropy.table.Column` of coordinates + The target(s) around which to search. It may be specified as a string in which case it is resolved using online services or as the appropriate `astropy.coordinates` object. ICRS coordinates may also be entered as strings as specified in the @@ -77,13 +80,19 @@ def query_crossid_async(self, coordinates, obj_names=None, ra = np.array([220.064728084,220.064728467,220.06473483]) dec = np.array([0.870131920218,0.87013210119,0.870138329659]) coordinates = SkyCoord(ra, dec, frame='icrs', unit='deg') - radius : str or `~astropy.units.Quantity` object, optional The - string must be parsable by `~astropy.coordinates.Angle`. The + radius : str or `~astropy.units.Quantity` object or `~astropy.coordinates.Angle` object, optional + The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from - `astropy.units` may also be used. Defaults to 5 arcsec. + `astropy.units` or `~astropy.coordinates.Angle` object from + `astropy.coordinates` may also be used. Defaults to 5 arcsec. + The maximum allowed value is 3 arcmin. timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. + fields : list, optional + SDSS PhotoObj or SpecObj quantities to return. If None, defaults + to quantities required to find corresponding spectra and images + of matched objects (e.g. plate, fiberID, mjd, etc.). photoobj_fields : list, optional PhotoObj quantities to return. If photoobj_fields is None and specobj_fields is None then the value of fields is used @@ -94,76 +103,86 @@ def query_crossid_async(self, coordinates, obj_names=None, Target names. If given, every coordinate should have a corresponding name, and it gets repeated in the query result. It generates unique object names by default. - get_query_payload : bool + spectro : bool, optional + Look for spectroscopic match in addition to photometric match? If + True, objects will only count as a match if photometry *and* + spectroscopy exist. If False, will look for photometric matches + only. + region : bool, optional + Normally cross-id only returns the closest primary object. + Setting this to ``True`` will return all objects. + field_help: str or bool, optional + Field name to check whether a valid PhotoObjAll or SpecObjAll + field name. If `True` or it is an invalid field name all the valid + field names are returned as a dict. + get_query_payload : bool, optional If True, this will return the data the query would have sent out, but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. - """ + cache : bool, optional + If ``True`` use the request caching mechanism. - if (not isinstance(coordinates, list) and - not isinstance(coordinates, Column) and - not (isinstance(coordinates, commons.CoordClasses) and - not coordinates.isscalar)): - coordinates = [coordinates] + Returns + ------- + result : `~astropy.table.Table` + The result of the query as a `~astropy.table.Table` object. - if obj_names is None: - obj_names = ['obj_{0}'.format(i) for i in range(len(coordinates))] - elif len(obj_names) != len(coordinates): - raise ValueError("Number of coordinates and obj_names should " - "be equal") + """ - if isinstance(radius, u.Quantity): - radius = radius.to(u.arcmin).value + if isinstance(radius, Angle): + radius = radius.to_value(u.arcmin) else: try: - float(radius) - except TypeError: + radius = Angle(radius).to_value(u.arcmin) + except ValueError: raise TypeError("radius should be either Quantity or " "convertible to float.") + if radius > self.MAX_CROSSID_RADIUS.value: + raise ValueError(f"radius must be less than {self.MAX_CROSSID_RADIUS}.") - sql_query = 'SELECT\r\n' # Older versions expect the CRLF to be there. - - if specobj_fields is None: - if photoobj_fields is None: - photoobj_fields = crossid_defs - photobj_fields = ['p.{0}'.format(i) for i in photoobj_fields] - photobj_fields.append('p.objID as obj_id') - specobj_fields = [] + if (not isinstance(coordinates, list) and + not isinstance(coordinates, Column) and + not (isinstance(coordinates, commons.CoordClasses) and + not coordinates.isscalar)): + coordinates = [coordinates] + if obj_names is None: + obj_names = [f'obj_{i:d}' for i in range(len(coordinates))] + elif len(obj_names) != len(coordinates): + raise ValueError("Number of coordinates and obj_names should " + "be equal") + if region: + data = "ra dec \n" + data_format = '{ra} {dec}' else: - specobj_fields = ['s.{0}'.format(i) for i in specobj_fields] - if photoobj_fields is not None: - photobj_fields = ['p.{0}'.format(i) for i in photoobj_fields] - photobj_fields.append('p.objID as obj_id') - else: - photobj_fields = [] - specobj_fields.append('s.SpecObjID as obj_id') - - sql_query += ', '.join(photobj_fields + specobj_fields) - - sql_query += ',dbo.fPhotoTypeN(p.type) as type \ - FROM #upload u JOIN #x x ON x.up_id = u.up_id \ - JOIN PhotoObjAll p ON p.objID = x.objID ' - if specobj_fields: - sql_query += 'JOIN SpecObjAll s ON p.objID = s.bestObjID ' - sql_query += 'ORDER BY x.up_id' - - data = "obj_id ra dec \n" - data += " \n ".join(['{0} {1} {2}'.format(obj_names[i], - coordinates[i].ra.deg, - coordinates[i].dec.deg) + data = "name ra dec \n" # SDSS's own examples default to 'name'. 'obj_id' is too easy to confuse with 'objID' + data_format = '{obj} {ra} {dec}' + data += " \n ".join([data_format.format(obj=obj_names[i], + ra=coordinates[i].ra.deg, + dec=coordinates[i].dec.deg) for i in range(len(coordinates))]) # firstcol is hardwired, as obj_names is always passed files = {'upload': ('astroquery', data)} - request_payload = dict(uquery=sql_query, - firstcol=1, - format='csv', photoScope='nearPrim', - radius=radius, - photoUpType='ra-dec', searchType='photo') - if data_release > 11: - request_payload['searchtool'] = 'CrossID' + request_payload = self._args_to_payload(coordinates=coordinates, + fields=fields, + spectro=spectro, region=region, + photoobj_fields=photoobj_fields, + specobj_fields=specobj_fields, field_help=field_help, + data_release=data_release) + if field_help: + return request_payload, files + + request_payload['radius'] = radius + if region: + request_payload['firstcol'] = 0 # First column is RA. + request_payload['photoScope'] = 'allObj' # All nearby objects, i.e. PhotoObjAll + else: + request_payload['firstcol'] = 1 # Skip one column, which contains the object name. + request_payload['photoScope'] = 'nearPrim' # Nearest primary object + request_payload['photoUpType'] = 'ra-dec' # Input data payload has RA, Dec coordinates + request_payload['searchType'] = 'photo' if get_query_payload: return request_payload, files @@ -174,21 +193,22 @@ def query_crossid_async(self, coordinates, obj_names=None, timeout=timeout, cache=cache) return response - def query_region_async(self, coordinates, radius=2. * u.arcsec, - fields=None, spectro=False, timeout=TIMEOUT, - get_query_payload=False, photoobj_fields=None, - specobj_fields=None, field_help=False, - obj_names=None, data_release=conf.default_release, - cache=True): + def query_region_async(self, coordinates, *, radius=2. * u.arcsec, timeout=TIMEOUT, + fields=None, photoobj_fields=None, specobj_fields=None, obj_names=None, + spectro=False, field_help=False, get_query_payload=False, + data_release=conf.default_release, cache=True): """ - Used to query a region around given coordinates. Equivalent to - the object cross-ID from the web interface. + Used to query a circular region (a "cone search") around given coordinates. + + This function is equivalent to the object cross-ID (`query_crossid`), + with slightly different parameters. It returns all objects within the + search radius; this could potentially include duplicate observations + of the same object. Parameters ---------- - coordinates : str or `astropy.coordinates` object or list of - coordinates or `~astropy.table.Column` of coordinates The - target(s) around which to search. It may be specified as a + coordinates : str or `astropy.coordinates` object or list of coordinates or `~astropy.table.Column` of coordinates + The target(s) around which to search. It may be specified as a string in which case it is resolved using online services or as the appropriate `astropy.coordinates` object. ICRS coordinates may also be entered as strings as specified in the @@ -198,40 +218,43 @@ def query_region_async(self, coordinates, radius=2. * u.arcsec, ra = np.array([220.064728084,220.064728467,220.06473483]) dec = np.array([0.870131920218,0.87013210119,0.870138329659]) coordinates = SkyCoord(ra, dec, frame='icrs', unit='deg') - radius : str or `~astropy.units.Quantity` object, optional The - string must be parsable by `~astropy.coordinates.Angle`. The + radius : str or `~astropy.units.Quantity` object, optional + The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `astropy.units` may also be used. Defaults to 2 arcsec. + The maximum allowed value is 3 arcmin. + timeout : float, optional + Time limit (in seconds) for establishing successful connection with + remote server. Defaults to `SDSSClass.TIMEOUT`. fields : list, optional SDSS PhotoObj or SpecObj quantities to return. If None, defaults to quantities required to find corresponding spectra and images of matched objects (e.g. plate, fiberID, mjd, etc.). - spectro : bool, optional - Look for spectroscopic match in addition to photometric match? If - True, objects will only count as a match if photometry *and* - spectroscopy exist. If False, will look for photometric matches - only. - timeout : float, optional - Time limit (in seconds) for establishing successful connection with - remote server. Defaults to `SDSSClass.TIMEOUT`. photoobj_fields : list, optional PhotoObj quantities to return. If photoobj_fields is None and specobj_fields is None then the value of fields is used specobj_fields : list, optional SpecObj quantities to return. If photoobj_fields is None and specobj_fields is None then the value of fields is used + obj_names : str, or list or `~astropy.table.Column`, optional + Target names. If given, every coordinate should have a + corresponding name, and it gets repeated in the query result. + spectro : bool, optional + Look for spectroscopic match in addition to photometric match? If + True, objects will only count as a match if photometry *and* + spectroscopy exist. If False, will look for photometric matches + only. field_help: str or bool, optional Field name to check whether a valid PhotoObjAll or SpecObjAll field name. If `True` or it is an invalid field name all the valid field names are returned as a dict. - obj_names : str, or list or `~astropy.table.Column`, optional - Target names. If given, every coordinate should have a - corresponding name, and it gets repeated in the query result. - get_query_payload : bool + get_query_payload : bool, optional If True, this will return the data the query would have sent out, but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. + cache : bool, optional + If ``True`` use the request caching mechanism. Examples -------- @@ -254,20 +277,27 @@ def query_region_async(self, coordinates, radius=2. * u.arcsec, The result of the query as a `~astropy.table.Table` object. """ - request_payload = self._args_to_payload( - coordinates=coordinates, radius=radius, fields=fields, - spectro=spectro, photoobj_fields=photoobj_fields, - specobj_fields=specobj_fields, field_help=field_help, - obj_names=obj_names, data_release=data_release) + request_payload, files = self.query_crossid_async(coordinates=coordinates, + radius=radius, fields=fields, + photoobj_fields=photoobj_fields, + specobj_fields=specobj_fields, + obj_names=obj_names, + spectro=spectro, + region=True, + field_help=field_help, + get_query_payload=True, + data_release=data_release) + if get_query_payload or field_help: return request_payload - url = self._get_query_url(data_release) - response = self._request("GET", url, params=request_payload, + url = self._get_crossid_url(data_release) + response = self._request("POST", url, data=request_payload, + files=files, timeout=timeout, cache=cache) return response - def query_specobj_async(self, plate=None, mjd=None, fiberID=None, + def query_specobj_async(self, *, plate=None, mjd=None, fiberID=None, fields=None, timeout=TIMEOUT, get_query_payload=False, field_help=False, data_release=conf.default_release, cache=True): @@ -297,11 +327,13 @@ def query_specobj_async(self, plate=None, mjd=None, fiberID=None, Field name to check whether a valid PhotoObjAll or SpecObjAll field name. If `True` or it is an invalid field name all the valid field names are returned as a dict. - get_query_payload : bool + get_query_payload : bool, optional If True, this will return the data the query would have sent out, but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. + cache : bool, optional + If ``True`` use the request caching mechanism. Examples -------- @@ -341,7 +373,7 @@ def query_specobj_async(self, plate=None, mjd=None, fiberID=None, timeout=timeout, cache=cache) return response - def query_photoobj_async(self, run=None, rerun=301, camcol=None, + def query_photoobj_async(self, *, run=None, rerun=301, camcol=None, field=None, fields=None, timeout=TIMEOUT, get_query_payload=False, field_help=False, data_release=conf.default_release, cache=True): @@ -375,11 +407,13 @@ def query_photoobj_async(self, run=None, rerun=301, camcol=None, Field name to check whether a valid PhotoObjAll or SpecObjAll field name. If `True` or it is an invalid field name all the valid field names are returned as a dict. - get_query_payload : bool + get_query_payload : bool, optional If True, this will return the data the query would have sent out, but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. + cache : bool, optional + If ``True`` use the request caching mechanism. Examples -------- @@ -425,7 +459,7 @@ def __sanitize_query(self, stmt): fsql += ' ' + line.split('--')[0] return fsql - def query_sql_async(self, sql_query, timeout=TIMEOUT, + def query_sql_async(self, sql_query, *, timeout=TIMEOUT, data_release=conf.default_release, cache=True, **kwargs): """ @@ -438,8 +472,10 @@ def query_sql_async(self, sql_query, timeout=TIMEOUT, timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. - data_release : int + data_release : int, optional The data release of the SDSS to use. + cache : bool, optional + If ``True`` use the request caching mechanism. Examples -------- @@ -484,7 +520,7 @@ class = 'galaxy' \ def get_spectra_async(self, coordinates=None, radius=2. * u.arcsec, matches=None, plate=None, fiberID=None, mjd=None, - timeout=TIMEOUT, get_query_payload=False, + timeout=TIMEOUT, data_release=conf.default_release, cache=True, show_progress=True): """ @@ -515,20 +551,21 @@ def get_spectra_async(self, coordinates=None, radius=2. * u.arcsec, Result of `query_region`. plate : integer, optional Plate number. + fiberID : integer, optional + Fiber number. mjd : integer, optional Modified Julian Date indicating the date a given piece of SDSS data was taken. - fiberID : integer, optional - Fiber number. timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. - get_query_payload : bool - If True, this will return the data the query would have sent out, - but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. With the default server, this only supports DR8 or later. + cache : bool, optional + Cache the spectra using astropy's caching system + show_progress : bool, optional + If False, do not display download progress. Returns ------- @@ -559,20 +596,15 @@ def get_spectra_async(self, coordinates=None, radius=2. * u.arcsec, """ if not matches: - request_payload = self._args_to_payload( - specobj_fields=['run2d', 'plate', - 'mjd', 'fiberID'], - coordinates=coordinates, radius=radius, spectro=True, - plate=plate, mjd=mjd, fiberID=fiberID, - data_release=data_release) - if get_query_payload: - return request_payload - - url = self._get_query_url(data_release) - result = self._request("GET", url, params=request_payload, - timeout=timeout, cache=cache) - - matches = self._parse_result(result) + if coordinates is None: + matches = self.query_specobj(plate=plate, mjd=mjd, fiberID=fiberID, + fields=['run2d', 'plate', 'mjd', 'fiberID'], + timeout=timeout, data_release=data_release, cache=cache) + else: + matches = self.query_crossid(coordinates, radius=radius, + specobj_fields=['run2d', 'plate', 'mjd', 'fiberID'], + spectro=True, + timeout=timeout, data_release=data_release, cache=cache) if matches is None: warnings.warn("Query returned no results.", NoResultsWarning) return @@ -634,7 +666,7 @@ def get_spectra(self, coordinates=None, radius=2. * u.arcsec, def get_images_async(self, coordinates=None, radius=2. * u.arcsec, matches=None, run=None, rerun=301, camcol=None, field=None, band='g', timeout=TIMEOUT, - get_query_payload=False, cache=True, + cache=True, data_release=conf.default_release, show_progress=True): """ @@ -676,19 +708,18 @@ def get_images_async(self, coordinates=None, radius=2. * u.arcsec, Output of one camera column of CCDs. field : integer, optional Part of a camcol of size 2048 by 1489 pixels. - band : str, list + band : str or list Could be individual band, or list of bands. Options: ``'u'``, ``'g'``, ``'r'``, ``'i'``, or ``'z'``. timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. - cache : bool + cache : bool, optional Cache the images using astropy's caching system - get_query_payload : bool - If True, this will return the data the query would have sent out, - but does not actually do the query. - data_release : int + data_release : int, optional The data release of the SDSS to use. + show_progress : bool, optional + If False, do not display download progress. Returns ------- @@ -718,21 +749,20 @@ def get_images_async(self, coordinates=None, radius=2. * u.arcsec, """ if not matches: - request_payload = self._args_to_payload( - fields=['run', 'rerun', 'camcol', 'field'], - coordinates=coordinates, radius=radius, spectro=False, run=run, - rerun=rerun, camcol=camcol, field=field, - data_release=data_release) - if get_query_payload: - return request_payload - - url = self._get_query_url(data_release) - result = self._request("GET", url, params=request_payload, - timeout=timeout, cache=cache) - matches = self._parse_result(result) + if coordinates is None: + matches = self.query_photoobj(run=run, rerun=rerun, + camcol=camcol, field=field, + fields=['run', 'rerun', 'camcol', 'field'], + timeout=timeout, + data_release=data_release, cache=cache) + else: + matches = self.query_crossid(coordinates, radius=radius, + fields=['run', 'rerun', 'camcol', 'field'], + timeout=timeout, data_release=data_release, cache=cache) if matches is None: warnings.warn("Query returned no results.", NoResultsWarning) return + if not isinstance(matches, Table): raise ValueError("'matches' must be an astropy Table") @@ -771,8 +801,7 @@ def get_images(self, coordinates=None, radius=2. * u.arcsec, readable_objs = self.get_images_async( coordinates=coordinates, radius=radius, matches=matches, run=run, rerun=rerun, data_release=data_release, camcol=camcol, field=field, - band=band, timeout=timeout, get_query_payload=get_query_payload, - show_progress=show_progress) + band=band, timeout=timeout, show_progress=show_progress) if readable_objs is not None: if isinstance(readable_objs, dict): @@ -780,7 +809,7 @@ def get_images(self, coordinates=None, radius=2. * u.arcsec, else: return [obj.get_fits() for obj in readable_objs] - def get_spectral_template_async(self, kind='qso', timeout=TIMEOUT, + def get_spectral_template_async(self, kind='qso', *, timeout=TIMEOUT, show_progress=True): """ Download spectral templates from SDSS DR-2. @@ -795,12 +824,14 @@ def get_spectral_template_async(self, kind='qso', timeout=TIMEOUT, Parameters ---------- - kind : str, list + kind : str or list Which spectral template to download? Options are stored in the dictionary astroquery.sdss.SDSS.AVAILABLE_TEMPLATES timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. + show_progress : bool, optional + If False, do not display download progress. Examples -------- @@ -833,7 +864,7 @@ def get_spectral_template_async(self, kind='qso', timeout=TIMEOUT, return results @prepend_docstr_nosections(get_spectral_template_async.__doc__) - def get_spectral_template(self, kind='qso', timeout=TIMEOUT, + def get_spectral_template(self, kind='qso', *, timeout=TIMEOUT, show_progress=True): """ Returns @@ -857,6 +888,8 @@ def _parse_result(self, response, verbose=False): ---------- response : `requests.Response` Result of requests -> np.atleast_1d. + verbose : bool, optional + Not currently used. Returns ------- @@ -879,29 +912,24 @@ def _parse_result(self, response, verbose=False): else: return Table(arr) - def _args_to_payload(self, coordinates=None, radius=2. * u.arcsec, - fields=None, spectro=False, + def _args_to_payload(self, coordinates=None, + fields=None, spectro=False, region=False, plate=None, mjd=None, fiberID=None, run=None, rerun=301, camcol=None, field=None, photoobj_fields=None, specobj_fields=None, - field_help=None, obj_names=None, + field_help=None, data_release=conf.default_release): """ Construct the SQL query from the arguments. Parameters ---------- - coordinates : str or `astropy.coordinates` object or list of - coordinates or `~astropy.table.Column` or coordinates + coordinates : str or `astropy.coordinates` object or list of coordinates or `~astropy.table.Column` or coordinates The target around which to search. It may be specified as a string in which case it is resolved using online services or as the appropriate `astropy.coordinates` object. ICRS coordinates may also be entered as strings as specified in the `astropy.coordinates` module. - radius : str or `~astropy.units.Quantity` object, optional - The string must be parsable by `~astropy.coordinates.Angle`. The - appropriate `~astropy.units.Quantity` object from `astropy.units` - may also be used. Defaults to 2 arcsec. fields : list, optional SDSS PhotoObj or SpecObj quantities to return. If None, defaults to quantities required to find corresponding spectra and images @@ -913,6 +941,8 @@ def _args_to_payload(self, coordinates=None, radius=2. * u.arcsec, only. If ``spectro`` is True, it is possible to let coordinates undefined and set at least one of ``plate``, ``mjd`` or ``fiberID`` to search using these fields. + region : bool, optional + Used internally to distinguish certain types of queries. plate : integer, optional Plate number. mjd : integer, optional @@ -940,10 +970,7 @@ def _args_to_payload(self, coordinates=None, radius=2. * u.arcsec, Field name to check whether it is a valid PhotoObjAll or SpecObjAll field name. If `True` or it is an invalid field name all the valid field names are returned as a dict. - obj_names : str, or list or `~astropy.table.Column`, optional - Target names. If given, every coordinate should have a - corresponding name, and it gets repeated in the query result - data_release : int + data_release : int, optional The data release of the SDSS to use. Returns @@ -965,72 +992,68 @@ def _args_to_payload(self, coordinates=None, radius=2. * u.arcsec, elif field_help: ret = 0 if field_help in photoobj_all: - print("{0} is a valid 'photoobj_field'".format(field_help)) + print(f"{field_help} is a valid 'photoobj_field'") ret += 1 if field_help in specobj_all: - print("{0} is a valid 'specobj_field'".format(field_help)) + print(f"{field_help} is a valid 'specobj_field'") ret += 1 if ret > 0: return else: if field_help is not True: - warnings.warn("{0} isn't a valid 'photobj_field' or " + warnings.warn(f"{field_help} isn't a valid 'photobj_field' or " "'specobj_field' field, valid fields are" - "returned.".format(field_help)) + "returned.") return {'photoobj_all': photoobj_all, 'specobj_all': specobj_all} # Construct SQL query q_select = 'SELECT DISTINCT ' + crossid = coordinates is not None and not region # crossid queries have different default fields + if coordinates is not None: + q_select = 'SELECT\r\n' # Older versions expect the CRLF to be there. q_select_field = [] + fields_spectro = False if photoobj_fields is None and specobj_fields is None: # Fields to return if fields is None: - photoobj_fields = photoobj_defs + if crossid: + photoobj_fields = crossid_defs + else: + photoobj_fields = photoobj_defs if spectro: specobj_fields = specobj_defs else: for sql_field in fields: if (sql_field in photoobj_all - or sql_field.lower() in photoobj_all): - q_select_field.append('p.{0}'.format(sql_field)) + or sql_field.lower() in photoobj_all): + q_select_field.append(f'p.{sql_field}') elif (sql_field in specobj_all - or sql_field.lower() in specobj_all): - q_select_field.append('s.{0}'.format(sql_field)) + or sql_field.lower() in specobj_all): + fields_spectro = True + q_select_field.append(f's.{sql_field}') if photoobj_fields is not None: for sql_field in photoobj_fields: - q_select_field.append('p.{0}'.format(sql_field)) + q_select_field.append(f'p.{sql_field}') if specobj_fields is not None: for sql_field in specobj_fields: - q_select_field.append('s.{0}'.format(sql_field)) + q_select_field.append(f's.{sql_field}') + if crossid and fields is None: + q_select_field.append('s.SpecObjID AS obj_id') + if crossid: + q_select_field.append('dbo.fPhotoTypeN(p.type) AS type') q_select += ', '.join(q_select_field) - q_from = 'FROM PhotoObjAll AS p ' - if spectro: - q_join = 'JOIN SpecObjAll s ON p.objID = s.bestObjID ' - else: - q_join = '' + q_from = 'FROM PhotoObjAll AS p' + if coordinates is not None: + q_from = 'FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID' + if spectro or specobj_fields or fields_spectro: + q_from += ' JOIN SpecObjAll AS s ON p.objID = s.bestObjID' - q_where = 'WHERE ' + q_where = None if coordinates is not None: - if (not isinstance(coordinates, list) and - not isinstance(coordinates, Column) and - not (isinstance(coordinates, commons.CoordClasses) and - not coordinates.isscalar)): - coordinates = [coordinates] - for n, target in enumerate(coordinates): - # Query for a region - target = commons.parse_coordinates(target).transform_to('fk5') - - ra = target.ra.degree - dec = target.dec.degree - dr = coord.Angle(radius).to('degree').value - if n > 0: - q_where += ' or ' - q_where += ('((p.ra between %g and %g) and ' - '(p.dec between %g and %g))' - % (ra - dr, ra + dr, dec - dr, dec + dr)) + q_where = 'ORDER BY x.up_id' elif spectro: # Spectra: query for specified plate, mjd, fiberid s_fields = ['s.%s=%d' % (key, val) for (key, val) in @@ -1055,16 +1078,26 @@ def _args_to_payload(self, coordinates=None, radius=2. * u.arcsec, raise ValueError('must specify at least one of `coordinates`, ' '`run`, `camcol` or `field`') - sql = "{0} {1} {2} {3}".format(q_select, q_from, q_join, q_where) - - request_payload = dict(cmd=sql, format='csv') + sql = f"{q_select} {q_from} {q_where}" - if data_release > 11: - request_payload['searchtool'] = 'SQL' + # In DR 8 & DR9 the format parameter is case-sensitive, but in later + # releases that does not appear to be the case. In principle 'csv' + # should work for all. + request_payload = dict(format='csv') + if coordinates is not None: + request_payload['uquery'] = sql + if data_release > 11: + request_payload['searchtool'] = 'CrossID' + else: + request_payload['cmd'] = sql + if data_release > 11: + request_payload['searchtool'] = 'SQL' return request_payload def _get_query_url(self, data_release): + """Generate URL for generic SQL queries. + """ if data_release < 10: suffix = self.QUERY_URL_SUFFIX_DR_OLD elif data_release == 10: @@ -1077,6 +1110,8 @@ def _get_query_url(self, data_release): return url def _get_crossid_url(self, data_release): + """Generate URL for CrossID queries. + """ if data_release < 10: suffix = self.XID_URL_SUFFIX_OLD elif data_release == 10: diff --git a/astroquery/sdss/tests/test_sdss.py b/astroquery/sdss/tests/test_sdss.py index 38b7d1d4cc..4077a56391 100644 --- a/astroquery/sdss/tests/test_sdss.py +++ b/astroquery/sdss/tests/test_sdss.py @@ -6,7 +6,9 @@ import numpy as np from numpy.testing import assert_allclose +import astropy.units as u from astropy.io import fits +from astropy.coordinates import Angle from astropy.table import Column, Table import pytest @@ -25,26 +27,33 @@ @pytest.fixture -def patch_get(request): - mp = request.getfixturevalue("monkeypatch") - - mp.setattr(sdss.SDSS, '_request', get_mockreturn) - return mp - +def patch_request(request): + def mockreturn(method, url, **kwargs): + if 'data' in kwargs: + cmd = kwargs['data']['uquery'] + else: + cmd = kwargs['params']['cmd'] + if 'SpecObjAll' in cmd: + filename = data_path(DATA_FILES['spectra_id']) + else: + filename = data_path(DATA_FILES['images_id']) + content = open(filename, 'rb').read() + return MockResponse(content, url) -@pytest.fixture -def patch_post(request): mp = request.getfixturevalue("monkeypatch") - mp.setattr(sdss.SDSS, '_request', post_mockreturn) + mp.setattr(sdss.SDSS, '_request', mockreturn) return mp @pytest.fixture -def patch_get_slow(request): +def patch_request_slow(request): + def mockreturn_slow(method, url, **kwargs): + raise TimeoutError + mp = request.getfixturevalue("monkeypatch") - mp.setattr(sdss.SDSS, '_request', get_mockreturn_slow) + mp.setattr(sdss.SDSS, '_request', mockreturn_slow) return mp @@ -82,25 +91,6 @@ def get_readable_fileobj_mockreturn(filename, **kwargs): return mp -def get_mockreturn(method, url, params=None, timeout=10, cache=True, **kwargs): - if 'SpecObjAll' in params['cmd']: - filename = data_path(DATA_FILES['spectra_id']) - else: - filename = data_path(DATA_FILES['images_id']) - content = open(filename, 'rb').read() - return MockResponse(content, **kwargs) - - -def get_mockreturn_slow(method, url, params=None, timeout=0, **kwargs): - raise TimeoutError - - -def post_mockreturn(method, url, params=None, timeout=0, **kwargs): - filename = data_path(DATA_FILES['images_id']) - content = open(filename, 'rb').read() - return MockResponse(content) - - def data_path(filename): data_dir = os.path.join(os.path.dirname(__file__), 'data') return os.path.join(data_dir, filename) @@ -116,7 +106,7 @@ def data_path(filename): coords_column = Column(coords_list, name='coordinates') # List of all data releases. -dr_list = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) +dr_list = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) # We are not testing queries for DR11 because it is not easily available to @@ -164,30 +154,30 @@ def image_tester(images, filetype): @pytest.mark.parametrize("dr", dr_list) -def test_sdss_spectrum(patch_get, patch_get_readable_fileobj, dr, +def test_sdss_spectrum(patch_request, patch_get_readable_fileobj, dr, coords=coords): xid = sdss.SDSS.query_region(coords, data_release=dr, spectro=True) - url_tester(dr) + url_tester_crossid(dr) sp = sdss.SDSS.get_spectra(matches=xid, data_release=dr) image_tester(sp, 'spectra') - url_tester(dr) + # url_tester(dr) @pytest.mark.parametrize("dr", dr_list) -def test_sdss_spectrum_mjd(patch_get, patch_get_readable_fileobj, dr): +def test_sdss_spectrum_mjd(patch_request, patch_get_readable_fileobj, dr): sp = sdss.SDSS.get_spectra(plate=2345, fiberID=572, data_release=dr) image_tester(sp, 'spectra') @pytest.mark.parametrize("dr", dr_list) -def test_sdss_spectrum_coords(patch_get, patch_get_readable_fileobj, dr, +def test_sdss_spectrum_coords(patch_request, patch_get_readable_fileobj, dr, coords=coords): sp = sdss.SDSS.get_spectra(coords, data_release=dr) image_tester(sp, 'spectra') @pytest.mark.parametrize("dr", dr_list) -def test_sdss_sql(patch_get, patch_get_readable_fileobj, dr): +def test_sdss_sql(patch_request, patch_get_readable_fileobj, dr): query = """ select top 10 z, ra, dec, bestObjID @@ -210,35 +200,35 @@ class = 'galaxy' @pytest.mark.parametrize("dr", dr_list) -def test_sdss_image_from_query_region(patch_get, patch_get_readable_fileobj, +def test_sdss_image_from_query_region(patch_request, patch_get_readable_fileobj, dr, coords=coords): xid = sdss.SDSS.query_region(coords, data_release=dr) + url_tester_crossid(dr) # TODO test what img is img = sdss.SDSS.get_images(matches=xid) image_tester(img, 'images') - url_tester(dr) @pytest.mark.parametrize("dr", dr_list) -def test_sdss_image_run(patch_get, patch_get_readable_fileobj, dr): +def test_sdss_image_run(patch_request, patch_get_readable_fileobj, dr): img = sdss.SDSS.get_images(run=1904, camcol=3, field=164, data_release=dr) image_tester(img, 'images') @pytest.mark.parametrize("dr", dr_list) -def test_sdss_image_coord(patch_get, patch_get_readable_fileobj, dr, +def test_sdss_image_coord(patch_request, patch_get_readable_fileobj, dr, coord=coords): img = sdss.SDSS.get_images(coords, data_release=dr) image_tester(img, 'images') -def test_sdss_template(patch_get, patch_get_readable_fileobj): +def test_sdss_template(patch_request, patch_get_readable_fileobj): template = sdss.SDSS.get_spectral_template('qso') image_tester(template, 'spectra') @pytest.mark.parametrize("dr", dr_list) -def test_sdss_specobj(patch_get, dr): +def test_sdss_specobj(patch_request, dr): xid = sdss.SDSS.query_specobj(plate=2340, data_release=dr) data = Table.read(data_path(DATA_FILES['spectra_id']), format='ascii.csv', comment='#') @@ -252,7 +242,7 @@ def test_sdss_specobj(patch_get, dr): @pytest.mark.parametrize("dr", dr_list) -def test_sdss_photoobj(patch_get, dr): +def test_sdss_photoobj(patch_request, dr): xid = sdss.SDSS.query_photoobj( run=1904, camcol=3, field=164, data_release=dr) data = Table.read(data_path(DATA_FILES['images_id']), @@ -266,7 +256,7 @@ def test_sdss_photoobj(patch_get, dr): @pytest.mark.parametrize("dr", dr_list) -def test_list_coordinates(patch_get, dr): +def test_list_coordinates(patch_request, dr): xid = sdss.SDSS.query_region(coords_list, data_release=dr) data = Table.read(data_path(DATA_FILES['images_id']), format='ascii.csv', comment='#') @@ -275,10 +265,11 @@ def test_list_coordinates(patch_get, dr): # test fail. data['objid'] = data['objid'].astype(np.int64) compare_xid_data(xid, data) + url_tester_crossid(dr) @pytest.mark.parametrize("dr", dr_list) -def test_column_coordinates(patch_get, dr): +def test_column_coordinates(patch_request, dr): xid = sdss.SDSS.query_region(coords_column, data_release=dr) data = Table.read(data_path(DATA_FILES['images_id']), format='ascii.csv', comment='#') @@ -287,26 +278,26 @@ def test_column_coordinates(patch_get, dr): # test fail. data['objid'] = data['objid'].astype(np.int64) compare_xid_data(xid, data) - url_tester(dr) + url_tester_crossid(dr) -def test_query_timeout(patch_get_slow, coord=coords): +def test_query_timeout(patch_request_slow, coord=coords): with pytest.raises(TimeoutError): sdss.SDSS.query_region(coords, timeout=1) -def test_spectra_timeout(patch_get, patch_get_readable_fileobj_slow): +def test_spectra_timeout(patch_request, patch_get_readable_fileobj_slow): with pytest.raises(TimeoutError): sdss.SDSS.get_spectra(plate=2345, fiberID=572) -def test_images_timeout(patch_get, patch_get_readable_fileobj_slow): +def test_images_timeout(patch_request, patch_get_readable_fileobj_slow): with pytest.raises(TimeoutError): sdss.SDSS.get_images(run=1904, camcol=3, field=164) @pytest.mark.parametrize("dr", dr_list) -def test_query_crossid(patch_post, dr): +def test_query_crossid(patch_request, dr): xid = sdss.SDSS.query_crossid(coords_column, data_release=dr) data = Table.read(data_path(DATA_FILES['images_id']), format='ascii.csv', comment='#') @@ -318,42 +309,186 @@ def test_query_crossid(patch_post, dr): url_tester_crossid(dr) +def test_query_crossid_large_radius(patch_request): + """Test raising an exception if too large a search radius. + """ + with pytest.raises(ValueError, match="radius must be less than"): + xid = sdss.SDSS.query_crossid(coords_column, radius=5.0 * u.arcmin) + + +def test_query_crossid_invalid_radius(patch_request): + """Test raising an exception if search radius can't be parsed. + """ + with pytest.raises(TypeError, match="radius should be either Quantity"): + xid = sdss.SDSS.query_crossid(coords_column, radius='2.0 * u.arcmin') + + +def test_query_crossid_invalid_names(patch_request): + """Test raising an exception if user-supplied object names are invalid. + """ + with pytest.raises(ValueError, match="Number of coordinates and obj_names"): + xid = sdss.SDSS.query_crossid(coords_column, obj_names=['A1']) + + +def test_query_crossid_parse_angle_value(patch_request): + """Test parsing angles with astropy.coordinates.Angle. + """ + query_payload, files = sdss.SDSS.query_crossid(coords_column, + radius='3 arcsec', + get_query_payload=True) + + assert query_payload['radius'] == 0.05 + + +def test_query_crossid_explicit_angle_value(patch_request): + """Test parsing angles with astropy.coordinates.Angle. + """ + query_payload, files = sdss.SDSS.query_crossid(coords_column, + radius=Angle('3 arcsec'), + get_query_payload=True) + + assert query_payload['radius'] == 0.05 + + # =========== # Payload tests @pytest.mark.parametrize("dr", dr_list) -def test_list_coordinates_payload(patch_get, dr): - expect = ("SELECT DISTINCT " +def test_list_coordinates_region_payload(patch_request, dr): + expect = ("SELECT\r\n" "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field " - "FROM PhotoObjAll AS p WHERE " - "((p.ra between 2.02291 and 2.02402) and " - "(p.dec between 14.8393 and 14.8404)) or " - "((p.ra between 2.02291 and 2.02402) and " - "(p.dec between 14.8393 and 14.8404))") + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") query_payload = sdss.SDSS.query_region(coords_list, get_query_payload=True, data_release=dr) - assert query_payload['cmd'] == expect + assert query_payload['uquery'] == expect assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'allObj' @pytest.mark.parametrize("dr", dr_list) -def test_column_coordinates_payload(patch_get, dr): - expect = ("SELECT DISTINCT " +def test_column_coordinates_region_payload(patch_request, dr): + expect = ("SELECT\r\n" "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field " - "FROM PhotoObjAll AS p WHERE " - "((p.ra between 2.02291 and 2.02402) and " - "(p.dec between 14.8393 and 14.8404)) or " - "((p.ra between 2.02291 and 2.02402) and " - "(p.dec between 14.8393 and 14.8404))") + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") query_payload = sdss.SDSS.query_region(coords_column, get_query_payload=True, data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'allObj' + + +@pytest.mark.parametrize("dr", dr_list) +def test_column_coordinates_region_spectro_payload(patch_request, dr): + expect = ("SELECT\r\n" + "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field, " + "s.z, s.plate, s.mjd, s.fiberID, s.specobjid, s.run2d " + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "JOIN SpecObjAll AS s ON p.objID = s.bestObjID " + "ORDER BY x.up_id") + query_payload = sdss.SDSS.query_region(coords_column, spectro=True, + get_query_payload=True, + data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'allObj' + + +@pytest.mark.parametrize("dr", dr_list) +def test_column_coordinates_region_payload_custom_fields(patch_request, dr): + expect = ("SELECT\r\n" + "p.r, p.psfMag_r " + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") + query_payload = sdss.SDSS.query_region(coords_column, + get_query_payload=True, + fields=['r', 'psfMag_r'], + data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'allObj' + + +@pytest.mark.parametrize("dr", dr_list) +def test_list_coordinates_cross_id_payload(patch_request, dr): + expect = ("SELECT\r\n" + "p.ra, p.dec, p.psfMag_u, p.psfMagerr_u, p.psfMag_g, p.psfMagerr_g, " + "p.psfMag_r, p.psfMagerr_r, p.psfMag_i, p.psfMagerr_i, p.psfMag_z, p.psfMagerr_z, " + "dbo.fPhotoTypeN(p.type) AS type " + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") + query_payload, files = sdss.SDSS.query_crossid(coords_list, + get_query_payload=True, + data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'nearPrim' + + +@pytest.mark.parametrize("dr", dr_list) +def test_column_coordinates_cross_id_payload(patch_request, dr): + expect = ("SELECT\r\n" + "p.ra, p.dec, p.psfMag_u, p.psfMagerr_u, p.psfMag_g, p.psfMagerr_g, " + "p.psfMag_r, p.psfMagerr_r, p.psfMag_i, p.psfMagerr_i, p.psfMag_z, p.psfMagerr_z, " + "dbo.fPhotoTypeN(p.type) AS type " + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") + query_payload, files = sdss.SDSS.query_crossid(coords_column, + get_query_payload=True, + data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'nearPrim' + + +@pytest.mark.parametrize("dr", dr_list) +def test_column_coordinates_cross_id_payload_custom_fields(patch_request, dr): + expect = ("SELECT\r\n" + "p.r, p.psfMag_r, dbo.fPhotoTypeN(p.type) AS type " + "FROM #upload u JOIN #x x ON x.up_id = u.up_id JOIN PhotoObjAll AS p ON p.objID = x.objID " + "ORDER BY x.up_id") + query_payload, files = sdss.SDSS.query_crossid(coords_column, + get_query_payload=True, + fields=['r', 'psfMag_r'], + data_release=dr) + assert query_payload['uquery'] == expect + assert query_payload['format'] == 'csv' + assert query_payload['photoScope'] == 'nearPrim' + + +@pytest.mark.parametrize("dr", dr_list) +def test_photoobj_run_camcol_field_payload(patch_request, dr): + expect = ("SELECT DISTINCT " + "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field " + "FROM PhotoObjAll AS p WHERE " + "(p.run=5714 AND p.camcol=6 AND p.rerun=301)") + query_payload = sdss.SDSS.query_photoobj_async(run=5714, camcol=6, + get_query_payload=True, + data_release=dr) + assert query_payload['cmd'] == expect + assert query_payload['format'] == 'csv' + + +@pytest.mark.parametrize("dr", dr_list) +def test_spectra_plate_mjd_payload(patch_request, dr): + expect = ("SELECT DISTINCT " + "p.ra, p.dec, p.objid, p.run, p.rerun, p.camcol, p.field, " + "s.z, s.plate, s.mjd, s.fiberID, s.specobjid, s.run2d " + "FROM PhotoObjAll AS p " + "JOIN SpecObjAll AS s ON p.objID = s.bestObjID " + "WHERE " + "(s.plate=751 AND s.mjd=52251)") + query_payload = sdss.SDSS.query_specobj_async(plate=751, mjd=52251, + get_query_payload=True, + data_release=dr) assert query_payload['cmd'] == expect assert query_payload['format'] == 'csv' -def test_field_help_region(patch_get): +def test_field_help_region(patch_request): valid_field = sdss.SDSS.query_region(coords, field_help=True) assert isinstance(valid_field, dict) assert 'photoobj_all' in valid_field diff --git a/astroquery/sdss/tests/test_sdss_remote.py b/astroquery/sdss/tests/test_sdss_remote.py index 26251eaf6c..8e066facef 100644 --- a/astroquery/sdss/tests/test_sdss_remote.py +++ b/astroquery/sdss/tests/test_sdss_remote.py @@ -171,13 +171,13 @@ def test_query_non_default_field(self): assert isinstance(query1, Table) assert isinstance(query2, Table) - assert query1.colnames == ['r', 'psfMag_r'] - assert query2.colnames == ['ra', 'dec', 'r'] + assert query1.colnames == ['objID', 'r', 'psfMag_r'] + assert query2.colnames == ['objID', 'ra', 'dec', 'r'] @pytest.mark.parametrize("dr", dr_list) def test_query_crossid(self, dr): query1 = sdss.SDSS.query_crossid(self.coords, data_release=dr) - query2 = sdss.SDSS.query_crossid([self.coords, self.coords]) + query2 = sdss.SDSS.query_crossid([self.coords, self.coords], data_release=dr) assert isinstance(query1, Table) assert query1['objID'][0] == 1237652943176138868