From a5ca2e223f31394bd5df4527749561cfe1e96db3 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 18 Nov 2025 12:49:02 -0500 Subject: [PATCH 01/21] CatalogCollection class Updating docstrings, clarity --- astroquery/mast/catalog_collection.py | 311 ++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 astroquery/mast/catalog_collection.py diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py new file mode 100644 index 0000000000..78a8269ae2 --- /dev/null +++ b/astroquery/mast/catalog_collection.py @@ -0,0 +1,311 @@ +import difflib +from dataclasses import dataclass +from typing import Dict, Optional + +from pyvo.dal import TAPService, DALQueryError +from astropy.table import Table + +from astroquery import log +from astroquery.exceptions import InvalidQueryError + +__all__ = ['CatalogCollection'] + + +@dataclass +class CatalogMetadata: + """ + Data class to hold metadata about a catalog, including column metadata, + RA/Dec column names, and spatial query support. + """ + column_metadata: Table + ra_column: Optional[str] + dec_column: Optional[str] + supports_spatial_queries: bool + + +class CatalogCollection: + """ + This class provides an interface to interact with MAST catalog collections via TAP service. + """ + + TAP_BASE_URL = 'https://masttest.stsci.edu/vo-tap/api/v0.1/' + + def __init__(self, collection): + """ + Initialize a CatalogCollection object for interacting with a MAST catalog collection. + + Parameters + ---------- + collection : str + The name of the MAST catalog collection to interact with. + """ + self.name = collection.strip().lower() + # Initialize TAP service for the specified collection + self.tap_service = TAPService(self.TAP_BASE_URL + self.name) + + # Get catalogs within this collection + self.catalogs = self._get_catalogs() + self.catalog_names = self.catalogs['catalog_name'].tolist() + + # ADQL functions supported by this collection's TAP service + self.supported_adql_functions = self._get_adql_supported_functions() + + # Determine the default catalog for this collection + self.default_catalog = self.get_default_catalog() + + # Cache for catalog metadata to avoid redundant queries + self._catalog_metadata_cache: Dict[str, CatalogMetadata] = dict() + + def get_catalog_metadata(self, catalog): + """ + For a given catalog, cache and return metadata about its columns and capabilties. + + Parameters + ---------- + catalog : str + The catalog within the collection to get metadata for. + + Returns + ------- + CatalogMetadata + A CatalogMetadata object containing metadata about the specified catalog, including column metadata, + RA/Dec column names, and spatial query support. + """ + # Check cache first + if catalog in self._catalog_metadata_cache: + return self._catalog_metadata_cache[catalog] + + # Verify catalog validity for this collection + self._verify_catalog(catalog) + + # Get column metadata + metadata = self._get_column_metadata(catalog) + + # Get RA/Dec column names + ra_col, dec_col = self._get_ra_dec_column_names(metadata) + + # Determine if spatial queries are supported + supports_spatial_queries = (ra_col is not None and dec_col is not None) + if supports_spatial_queries: + # If an ra and dec column exist, test spatial query support + spatial_query = (f'SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT(\'ICRS\', {ra_col}, {dec_col}), ' + 'CIRCLE(\'ICRS\', 0, 0, 0.1)) = 1') + try: + self.tap_service.search(spatial_query) + except DALQueryError: + supports_spatial_queries = False + + meta = CatalogMetadata( + column_metadata=metadata, + ra_column=ra_col, + dec_column=dec_col, + supports_spatial_queries=supports_spatial_queries, + ) + + # Cache and return + self._catalog_metadata_cache[catalog] = meta + return meta + + def get_default_catalog(self): + """ + Get the default catalog for this collection. This is the first catalog that does not start with "tap_schema.". + + Returns + ------- + str + The default catalog name. + """ + # Pick default catalog = first one that does NOT start with "tap_schema." + default_catalog = next((c for c in self.catalog_names if not c.startswith("tap_schema.")), None) + + # If no valid catalog found, fallback to the first one + if default_catalog is None: + default_catalog = self.catalog_names[0] if self.catalog_names else None + + return default_catalog + + def run_tap_query(self, adql): + """ + Run a TAP query against the specified catalog. + + Parameters + ---------- + adql : str + The ADQL query string. + + Returns + ------- + response : `~astropy.table.Table` + The result of the TAP query as an Astropy Table. + """ + log.debug(f"Running TAP query on collection '{self.name}': {adql}") + result = self.tap_service.search(adql) + return result.to_table() + + def _get_catalogs(self): + """ + Retrieve the list of catalogs in this collection. + + Returns + ------- + list of str + List of catalog names. + """ + log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") + tables = self.tap_service.tables + names = [t.name for t in tables] + descriptions = [t.description for t in tables] + + # Create an Astropy Table to hold the results + result_table = Table([names, descriptions], names=('catalog_name', 'description')) + return result_table + + def _get_adql_supported_functions(self): + """ + Retrieve the ADQL supported functions of the TAP service. + + Returns + ------- + list + A list of supported ADQL functions. + """ + adql_functions = ['CIRCLE', 'POLYGON', 'POINT', 'CONTAINS', 'INTERSECTS'] + supported = [] + feature_id = 'ivo://ivoa.net/std/TAPRegExt#features-adqlgeo' + for capability in self.tap_service.capabilities: + if capability.standardid != 'ivo://ivoa.net/std/TAP': + continue + + for lang in capability.languages: + if lang.name != 'ADQL': + continue + + for func in adql_functions: + if lang.get_feature(feature_id, func): + supported.append(func) + + return supported + + def _verify_catalog(self, catalog): + """ + Verify that the specified catalog is valid for the given collection. + + Parameters + ---------- + catalog : str + The catalog to be verified. + + Raises + ------ + InvalidQueryError + If the specified catalog is not valid for the given collection. + """ + lower_map = {name.lower(): name for name in self.catalog_names} + if catalog.lower() not in lower_map: + closest_match = difflib.get_close_matches(catalog, self.catalog_names, n=1) + error_msg = ( + f"Catalog '{catalog}' is not recognized for collection '{self.name}'. " + f"Did you mean '{closest_match[0]}'?" + if closest_match + else f"Catalog '{catalog}' is not recognized for collection '{self.name}'." + ) + error_msg += " Available catalogs are: " + ", ".join(self.catalog_names) + raise InvalidQueryError(error_msg) + + def _get_column_metadata(self, catalog): + """ + For a given catalog, return metadata about its columns. + + Parameters + ---------- + catalog : str + The catalog within the collection to get metadata for. + + Returns + ------- + response : `~astropy.table.Table` + A table containing metadata about the specified table, including column names, data types, and descriptions. + """ + log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") + + # Case-insensitive match to find the table + tap_table = next((t for name, t in self.tap_service.tables.items() if name.lower() == catalog.lower()), None) + + # Extract column metadata + col_names = [col.name for col in tap_table.columns] + col_datatypes = [] + for col in tap_table.columns: + try: + col_datatypes.append(col.datatype._content) + except AttributeError: + # Some pyvo versions store datatype differently; fall back gracefully + # Fallback: str(col.datatype) or None + col_datatypes.append(getattr(col.datatype, '_content', str(col.datatype))) + col_units = [col.unit for col in tap_table.columns] + col_ucds = [col.ucd for col in tap_table.columns] + col_descriptions = [col.description for col in tap_table.columns] + + # Create an Astropy Table to hold the metadata + column_metadata = Table([col_names, col_datatypes, col_units, col_ucds, col_descriptions], + names=('name', 'data_type', 'unit', 'ucd', 'description')) + return column_metadata + + def _get_ra_dec_column_names(self, column_metadata): + """ + Return the RA and Dec column names for a given catalog and table. + + Parameters + ---------- + column_metadata : `~astropy.table.Table` + The column metadata table for a catalog. + + Returns + ------- + tuple + A tuple containing the (ra_column, dec_column) names. + """ + # Look for a column with UCD 'pos.eq.ra;meta.main' and 'pos.eq.dec;meta.main' + ra_col = None + dec_col = None + for name, ucd in zip(column_metadata['name'], column_metadata['ucd']): + if ucd and 'pos.eq.ra;meta.main' in ucd: + # TODO: ps1_dr2.mean_object and ps1_dr2.stacked_object has a column that can be used, + # but is not labeled with "meta.main" + ra_col = name + elif ucd and 'pos.eq.dec;meta.main' in ucd: + dec_col = name + return ra_col, dec_col + + def _verify_criteria(self, catalog, **criteria): + """ + Check that criteria keyword arguments are valid column names for the specified collection and catalog. + + Parameters + ---------- + catalog : str + The catalog within the collection to query. + **criteria + Keyword arguments representing criteria filters to apply. + + Raises + ------ + InvalidQueryError + If a keyword does not match any valid column names, an error is raised that suggests the closest + matching column name, if available. + """ + if not criteria: + return + self._verify_catalog(catalog) + col_names = list(self.get_catalog_metadata(catalog)['column_metadata']['name']) + + # Check each criteria argument for validity + for kwd in criteria.keys(): + if kwd not in col_names: + closest_match = difflib.get_close_matches(kwd, col_names, n=1) + error_msg = ( + f"Filter '{kwd}' is not recognized for collection '{self.name}' and catalog '{catalog}'. " + f"Did you mean '{closest_match[0]}'?" + if closest_match + else f"Filter '{kwd}' is not recognized for collection '{self.name}' and catalog '{catalog}'." + ) + raise InvalidQueryError(error_msg) From bdaace5086cca99268265e8752089e0ba8e19464 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 25 Nov 2025 09:39:42 -0500 Subject: [PATCH 02/21] Discovery methods TAP URL constant, add comments Updating docstrings, don't set `available_collections` in method --- astroquery/mast/collections.py | 710 ++++++--------------------------- 1 file changed, 131 insertions(+), 579 deletions(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 62cc12d9a7..9c0ad76bf7 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -3,673 +3,225 @@ MAST Collections ================ -This module contains various methods for querying MAST collections such as catalogs. +This module contains methods for discovering and querying MAST catalog collections. """ - import difflib -from json import JSONDecodeError -import warnings -import os -import time - -from requests import HTTPError, RequestException -import astropy.units as u -import astropy.coordinates as coord -from astropy.table import Table, Row -from astropy.utils.decorators import deprecated_renamed_argument +from astropy.table import Table -from ..utils import commons, async_to_sync +from ..utils import async_to_sync from ..utils.class_or_instance import class_or_instance -from ..exceptions import InvalidQueryError, MaxResultsWarning, InputWarning - -from . import utils, conf +from .catalog_collection import CatalogCollection from .core import MastQueryWithLogin - __all__ = ['Catalogs', 'CatalogsClass'] @async_to_sync class CatalogsClass(MastQueryWithLogin): """ - MAST catalog query class. - - Class for querying MAST catalog data. + Class for discovering and querying MAST catalog collections. """ - def __init__(self): - - super().__init__() + TAP_BASE_URL = 'https://masttest.stsci.edu/vo-tap/api/v0.1/' - services = {"panstarrs": {"path": "panstarrs/{data_release}/{table}.json", - "args": {"data_release": "dr2", "table": "mean"}}} - self._catalogs_mast_search_options = ['columns', 'sort_by', 'table', 'data_release'] + def __init__(self, collection='hsc', catalog=None): - self._service_api_connection.set_service_params(services, "catalogs", True) - - self.catalog_limit = None - self._current_connection = None - self._service_columns = dict() # Info about columns for Catalogs.MAST services - - def _parse_result(self, response, *, verbose=False): + super().__init__() - results_table = self._current_connection._parse_result(response, verbose=verbose) + self.available_collections = self.get_collections()['collection_name'].tolist() + self._no_longer_supported_collections = ['ctl', 'diskdetective', 'galex', 'plato'] + self._renamed_collections = {'panstarrs': 'ps1_dr2', 'gaia': 'gaiadr3'} + self._collections_cache = dict() - if len(results_table) == self.catalog_limit: - warnings.warn("Maximum catalog results returned, may not include all sources within radius.", - MaxResultsWarning) + self.collection = collection + if catalog: + self.catalog = catalog - return results_table + @property + def collection(self): + """ + The current MAST collection to be queried. + """ + return self._collection - def _get_service_col_config(self, catalog, release='dr2', table='mean'): + @collection.setter + def collection(self, collection): + """ + Setter that creates a CatalogCollection object when the collection is changed and updates + the catalog accordingly. """ - For a given Catalogs.MAST catalog, return a list of all searchable columns and their descriptions. - As of now, this function is exclusive to the Pan-STARRS catalog. + collection_obj = self._get_collection_obj(collection) + self._collection = collection_obj - Parameters - ---------- - catalog : str - The catalog to be queried. - release : str, optional - Catalog data release to query from. - table : str, optional - Catalog table to query from. + # Only change catalog if not set yet or invalid for this collection + if not hasattr(self, '_catalog') or self._catalog not in collection_obj.catalog_names: + self._catalog = collection_obj.default_catalog - Returns - ------- - response : `~astropy.table.Table` that contains columns names, types, and descriptions + @property + def catalog(self): """ - # Only supported for PanSTARRS currently - if catalog != 'panstarrs': - return - - service_key = (catalog, release, table) - if service_key not in self._service_columns: - try: - # Send server request to get column list for given parameters - request_url = f'{conf.catalogs_server}/api/v0.1/{catalog}/{release}/{table}/metadata.json' - resp = utils._simple_request(request_url) - - # Parse JSON and extract necessary info - results = resp.json() - - # Prepare data for Table creation - rows = [] - for result in results: - colname = result.get('column_name') or result.get('name') - dtype = result.get('db_type') - desc = result.get('description', '') - - if colname is None or dtype is None: - continue # Skip invalid entries - - rows.append((colname, dtype, desc)) - - # Create Table with parsed data - col_table = Table(rows=rows, names=('name', 'data_type', 'description')) - self._service_columns[service_key] = col_table - - except JSONDecodeError as ex: - raise JSONDecodeError(f'Failed to decode JSON response while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') - except RequestException as ex: - raise ConnectionError(f'Failed to connect to the server while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') - except Exception as ex: - raise RuntimeError(f'An unexpected error occurred while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') - - return self._service_columns[service_key] - - def _validate_service_criteria(self, catalog, **criteria): + The current catalog within the MAST collection. """ - Check that criteria keyword arguments are valid column names for the service. - Raises InvalidQueryError if a criteria argument is invalid. - - Parameters - ---------- - catalog : str - The catalog to be queried. - **criteria - Keyword arguments representing criteria filters to apply. + return self._catalog - Raises - ------- - InvalidQueryError - If a keyword does not match any valid column names, an error is raised that suggests the closest - matching column name, if available. + @catalog.setter + def catalog(self, catalog): """ - # Ensure that self._service_columns is populated - release = criteria.get('data_release', 'dr2') - table = criteria.get('table', 'mean') - col_config = self._get_service_col_config(catalog, release, table) - - if col_config: - # Check each criteria argument for validity - valid_cols = list(col_config['name']) + self._catalogs_mast_search_options - for kwd in criteria.keys(): - col = next((name for name in valid_cols if name.lower() == kwd.lower()), None) - if not col: - closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) - error_msg = ( - f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}. " - f"Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}." - ) - raise InvalidQueryError(error_msg) + Setter that verifies that the catalog is valid for the current collection. + """ + self.collection._verify_catalog(catalog) + self._catalog = catalog @class_or_instance - def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", - version=None, pagesize=None, page=None, **criteria): + def get_collections(self): """ - Given a sky position and radius, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. - - Parameters - ---------- - coordinates : str or `~astropy.coordinates` object - The target around which to search. It may be specified as a - string or as the appropriate `~astropy.coordinates` object. - radius : str or `~astropy.units.Quantity` object, optional - Default 0.2 degrees. - The string must be parsable by `~astropy.coordinates.Angle`. The - appropriate `~astropy.units.Quantity` object from - `~astropy.units` may also be used. Defaults to 0.2 deg. - catalog : str, optional - Default HSC. - The catalog to be queried. - version : int, optional - Version number for catalogs that have versions. Default is highest version. - pagesize : int, optional - Default None. - Can be used to override the default pagesize for (set in configs) this query only. - E.g. when using a slow internet connection. - page : int, optional - Default None. - Can be used to override the default behavior of all results being returned to obtain a - specific page of results. - **criteria - Other catalog-specific keyword args. - These can be found in the (service documentation)[https://mast.stsci.edu/api/v0/_services.html] - for specific catalogs. For example, one can specify the magtype for an HSC search. - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + Return a list of available collections from MAST. Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing the available MAST collections. """ + # If already cached, use it directly + if getattr(self, "available_collections", None): + return Table([self.available_collections], names=('collection_name',)) - # Put coordinates and radius into consistent format - coordinates = commons.parse_coordinates(coordinates, return_frame='icrs') - - # if radius is just a number we assume degrees - radius = coord.Angle(radius, u.deg) - - # basic params - params = {'ra': coordinates.ra.deg, - 'dec': coordinates.dec.deg, - 'radius': radius.deg} - - # Determine API connection and service name - if catalog.lower() in self._service_api_connection.SERVICES: - self._current_connection = self._service_api_connection - service = catalog - - # validate user criteria - self._validate_service_criteria(catalog.lower(), **criteria) - - # adding additional user specified parameters - for prop, value in criteria.items(): - params[prop] = value - - else: - self._current_connection = self._portal_api_connection - - # valid criteria keywords - valid_criteria = [] - - # Sorting out the non-standard portal service names - if catalog.lower() == "hsc": - if version == 2: - service = "Mast.Hsc.Db.v2" - else: - if version not in (3, None): - warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning) - service = "Mast.Hsc.Db.v3" - - # Hsc specific parameters (can be overridden by user) - self.catalog_limit = criteria.pop('nr', 50000) - valid_criteria = ['nr', 'ni', 'magtype'] - params['nr'] = self.catalog_limit - params['ni'] = criteria.pop('ni', 1) - params['magtype'] = criteria.pop('magtype', 1) - - elif catalog.lower() == "galex": - service = "Mast.Galex.Catalog" - self.catalog_limit = criteria.get('maxrecords', 50000) - - # galex specific parameters (can be overridden by user) - valid_criteria = ['maxrecords'] - params['maxrecords'] = criteria.pop('maxrecords', 50000) - - elif catalog.lower() == "gaia": - if version == 1: - service = "Mast.Catalogs.GaiaDR1.Cone" - elif version == 2: - service = "Mast.Catalogs.GaiaDR2.Cone" - else: - if version not in (None, 3): - warnings.warn("Invalid Gaia version number, defaulting to DR3.", InputWarning) - service = "Mast.Catalogs.GaiaDR3.Cone" - - elif catalog.lower() == 'plato': - if version in (None, 1): - service = "Mast.Catalogs.Plato.Cone" - else: - warnings.warn("Invalid PLATO catalog version number, defaulting to DR1.", InputWarning) - service = "Mast.Catalogs.Plato.Cone" - - else: - service = "Mast.Catalogs." + catalog + ".Cone" - self.catalog_limit = None - - # additional user-specified parameters are not valid - if criteria: - key = next(iter(criteria)) - closest_match = difflib.get_close_matches(key, valid_criteria, n=1) - error_msg = ( - f"Filter '{key}' does not exist for catalog {catalog}. Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{key}' does not exist for catalog {catalog}." - ) - raise InvalidQueryError(error_msg) + # Otherwise, fetch from the TAP service + log.debug("Fetching available collections from MAST TAP service.") + url = self.TAP_BASE_URL + "openapi.json" + response = requests.get(url) + response.raise_for_status() + data = response.json() - # Parameters will be passed as JSON objects only when accessing the PANSTARRS API - use_json = catalog.lower() == 'panstarrs' + # Extract collection enumeration + collection_enum = data["components"]["schemas"]["CatalogName"]["enum"] - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page, - use_json=use_json) + # Build an Astropy Table to hold the results + collection_table = Table([collection_enum], names=('collection_name',)) + return collection_table @class_or_instance - @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') - def query_object_async(self, object_name, *, radius=0.2*u.deg, catalog="Hsc", - pagesize=None, page=None, version=None, resolver=None, **criteria): + def get_catalogs(self, collection=None): """ - Given an object name, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. + For a given collection, return a list of available catalogs. Parameters ---------- - object_name : str - The name of the target around which to search. - radius : str or `~astropy.units.Quantity` object, optional - Default 0.2 degrees. - The string must be parsable by `~astropy.coordinates.Angle`. - The appropriate `~astropy.units.Quantity` object from - `~astropy.units` may also be used. Defaults to 0.2 deg. - catalog : str, optional - Default HSC. - The catalog to be queried. - pagesize : int, optional - Default None. - Can be used to override the default pagesize for (set in configs) this query only. - E.g. when using a slow internet connection. - page : int, optional - Default None. - Can be used to override the default behavior of all results being returned - to obtain a specific page of results. - version : int, optional - Version number for catalogs that have versions. Default is highest version. - resolver : str, optional - The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED". - If not specified, the default resolver order will be used. Please see the - `STScI Archive Name Translation Application (SANTA) `__ - for more information. Default is None. - **criteria - Catalog-specific keyword args. - These can be found in the `service documentation `__. - for specific catalogs. For example, one can specify the magtype for an HSC search. - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + collection : str, optional + The collection to be queried. Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing the available catalogs within the specified collection. """ - - coordinates = utils.resolve_object(object_name, resolver=resolver) - - return self.query_region_async(coordinates, - radius=radius, - catalog=catalog, - version=version, - pagesize=pagesize, - page=page, - **criteria) + # If no collection specified, use the class attribute + collection_obj = self._get_collection_obj(collection) if collection else self.collection + return collection_obj.catalogs @class_or_instance - def query_criteria_async(self, catalog, *, pagesize=None, page=None, resolver=None, **criteria): + def get_catalog_metadata(self, collection=None, catalog=None): """ - Given an set of filters, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. + For a given collection and catalog, return metadata about the catalog. Parameters ---------- - catalog : str - The catalog to be queried. - pagesize : int, optional - Can be used to override the default pagesize. - E.g. when using a slow internet connection. - page : int, optional - Can be used to override the default behavior of all results being returned to obtain - one specific page of results. - resolver : str, optional - The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED". - If not specified, the default resolver order will be used. Please see the - `STScI Archive Name Translation Application (SANTA) `__ - for more information. Default is None. - **criteria - Criteria to apply. At least one non-positional criteria must be supplied. - Valid criteria are coordinates, object_name, radius (as in `query_region` and `query_object`), - and all fields listed in the column documentation for the catalog being queried. - The Column Name is the keyword, with the argument being one or more acceptable values for that parameter, - except for fields with a float datatype where the argument should be in the form [minVal, maxVal]. - For non-float type criteria wildcards maybe used (both * and % are considered wildcards), however - only one wildcarded value can be processed per criterion. - RA and Dec must be given in decimal degrees, and datetimes in MJD. - For example: filters=["FUV","NUV"],proposal_pi="Ost*",t_max=[52264.4586,54452.8914] - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + collection : str, optional + The collection to be queried. + catalog : str, optional + The catalog within the collection to get metadata for. Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing metadata about the specified catalog, including column names, data types, + and descriptions. """ + collection_obj, catalog = self._parse_inputs(collection, catalog) + return collection_obj.get_catalog_metadata(catalog).column_metadata - # Separating any position info from the rest of the filters - coordinates = criteria.pop('coordinates', None) - object_name = criteria.pop('object_name', None) - radius = criteria.pop('radius', 0.2*u.deg) - - if object_name or coordinates: - coordinates = utils.parse_input_location(coordinates=coordinates, - object_name=object_name, - resolver=resolver) - - # if radius is just a number we assume degrees - radius = coord.Angle(radius, u.deg) - - # build query - params = {} - if coordinates: - params["ra"] = coordinates.ra.deg - params["dec"] = coordinates.dec.deg - params["radius"] = radius.deg - - # Determine API connection, service name, and build filter set - filters = None - if catalog.lower() in self._service_api_connection.SERVICES: - self._current_connection = self._service_api_connection - service = catalog - - # validate user criteria - self._validate_service_criteria(catalog.lower(), **criteria) - - if not self._current_connection.check_catalogs_criteria_params(criteria): - raise InvalidQueryError("At least one non-positional criterion must be supplied.") - - for prop, value in criteria.items(): - params[prop] = value - - else: - self._current_connection = self._portal_api_connection - - if catalog.lower() == "tic": - service = "Mast.Catalogs.Filtered.Tic" - if coordinates or object_name: - service += ".Position" - service += ".Rows" # Using the rowstore version of the query for speed - column_config_name = "Mast.Catalogs.Tess.Cone" - params["columns"] = "*" - elif catalog.lower() == "ctl": - service = "Mast.Catalogs.Filtered.Ctl" - if coordinates or object_name: - service += ".Position" - service += ".Rows" # Using the rowstore version of the query for speed - column_config_name = "Mast.Catalogs.Tess.Cone" - params["columns"] = "*" - elif catalog.lower() == "diskdetective": - service = "Mast.Catalogs.Filtered.DiskDetective" - if coordinates or object_name: - service += ".Position" - column_config_name = "Mast.Catalogs.Dd.Cone" - else: - raise InvalidQueryError("Criteria query not available for {}".format(catalog)) - - filters = self._current_connection.build_filter_set(column_config_name, service, **criteria) - - if not filters: - raise InvalidQueryError("At least one non-positional criterion must be supplied.") - params["filters"] = filters - - # Parameters will be passed as JSON objects only when accessing the PANSTARRS API - use_json = catalog.lower() == 'panstarrs' - - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page, - use_json=use_json) - - @class_or_instance - def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None): + def _verify_collection(self, collection): """ - Returns all the matches for a given Hubble Source Catalog MatchID. + Verify that the specified collection is valid. Parameters ---------- - match : int or `~astropy.table.Row` - The matchID or HSC entry to return matches for. - version : int, optional - The HSC version to match against. Default is v3. - pagesize : int, optional - Can be used to override the default pagesize. - E.g. when using a slow internet connection. - page : int, optional - Can be used to override the default behavior of all results being returned to obtain - one specific page of results. + collection : str + The collection to be verified. - Returns - ------- - response : list of `~requests.Response` + Raises + ------ + InvalidQueryError + If the specified collection is not valid. """ + if collection.lower() not in self.available_collections: + if collection in self._no_longer_supported_collections: + error_msg = (f"Collection '{collection}' is no longer supported. To query from this catalog, " + f"please use a version of Astroquery older than 0.4.12.") + else: + closest_match = difflib.get_close_matches(collection, self.available_collections, n=1) + error_msg = ( + f"Collection '{collection}' is not recognized. Did you mean '{closest_match[0]}'?" + if closest_match + else f"Collection '{collection}' is not recognized." + ) + error_msg += " Available collections are: " + ", ".join(self.available_collections) + raise InvalidQueryError(error_msg) - self._current_connection = self._portal_api_connection - - if isinstance(match, Row): - match = match["MatchID"] - match = str(match) # np.int64 gives json serializer problems, so stringify right here - - if version == 2: - service = "Mast.HscMatches.Db.v2" - else: - if version not in (3, None): - warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning) - service = "Mast.HscMatches.Db.v3" - - params = {"input": match} - - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page) - - @class_or_instance - def get_hsc_spectra_async(self, *, pagesize=None, page=None): + def _get_collection_obj(self, collection_name): """ - Returns all Hubble Source Catalog spectra. + Given a collection name, find or create the corresponding CatalogCollection object. Parameters ---------- - pagesize : int, optional - Can be used to override the default pagesize. - E.g. when using a slow internet connection. - page : int, optional - Can be used to override the default behavior of all results being returned to obtain - one specific page of results. + collection_name : str + The name of the collection. Returns ------- - response : list of `~requests.Response` + CatalogCollection + The corresponding CatalogCollection object. """ + collection_name = collection_name.lower().strip() + if collection_name in self._collections_cache: + return self._collections_cache[collection_name] - self._current_connection = self._portal_api_connection - - service = "Mast.HscSpectra.Db.All" - params = {} - - return self._current_connection.service_request_async(service, params, pagesize, page) + self._verify_collection(collection_name) + collection_obj = CatalogCollection(collection_name) + self._collections_cache[collection_name] = collection_obj + return collection_obj - def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_flag=False): + def _parse_inputs(self, collection=None, catalog=None): """ - Download one or more Hubble Source Catalog spectra. + Return (collection, catalog) applying default attributes, validation, and normalization. Parameters ---------- - spectra : `~astropy.table.Table` or `~astropy.table.Row` - One or more HSC spectra to be downloaded. - download_dir : str, optional - Specify the base directory to download spectra into. - Spectra will be saved in the subdirectory download_dir/mastDownload/HSC. - If download_dir is not specified the base directory will be '.'. - cache : bool, optional - Default is True. If file is found on disc it will not be downloaded again. - Note: has no affect when downloading curl script. - curl_flag : bool, optional - Default is False. If true instead of downloading files directly, a curl script - will be downloaded that can be used to download the data files at a later time. + collection : str, optional + The collection to be queried. If None, uses the instance's default collection. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's default catalog. Returns ------- - response : list of `~requests.Response` + tuple + A tuple containing the (collection, catalog) to be queried. """ + collection_obj = self._get_collection_obj(collection) if collection else self.collection - # if spectra is not a Table, put it in a list - if isinstance(spectra, Row): - spectra = [spectra] - - # set up the download directory and paths - if not download_dir: - download_dir = '.' - - if curl_flag: # don't want to download the files now, just the curl script - - download_file = "mastDownload_" + time.strftime("%Y%m%d%H%M%S") - - url_list = [] - path_list = [] - for spec in spectra: - if spec['SpectrumType'] < 2: - url_list.append('https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={0}' - .format(spec['DatasetName'])) - - else: - url_list.append('https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={0}' - .format(spec['DatasetName']) + '.fits') - - path_list.append(download_file + "/HSC/" + spec['DatasetName'] + '.fits') - - description_list = [""]*len(spectra) - producttype_list = ['spectrum']*len(spectra) - - service = "Mast.Bundle.Request" - params = {"urlList": ",".join(url_list), - "filename": download_file, - "pathList": ",".join(path_list), - "descriptionList": list(description_list), - "productTypeList": list(producttype_list), - "extension": 'curl'} - - response = self._portal_api_connection.service_request_async(service, params) - bundler_response = response[0].json() - - local_path = os.path.join(download_dir, "{}.sh".format(download_file)) - self._download_file(bundler_response['url'], local_path, head_safe=True) - - status = "COMPLETE" - msg = None - url = None - - if not os.path.isfile(local_path): - status = "ERROR" - msg = "Curl could not be downloaded" - url = bundler_response['url'] + if not catalog: + # If the class attribute catalog is valid for the collection, use it + # Otherwise, use the default catalog for the collection + if self.catalog in collection_obj.catalog_names: + catalog = self.catalog else: - missing_files = [x for x in bundler_response['statusList'].keys() - if bundler_response['statusList'][x] != 'COMPLETE'] - if len(missing_files): - msg = "{} files could not be added to the curl script".format(len(missing_files)) - url = ",".join(missing_files) - - manifest = Table({'Local Path': [local_path], - 'Status': [status], - 'Message': [msg], - "URL": [url]}) - + catalog = collection_obj.default_catalog else: - base_dir = download_dir.rstrip('/') + "/mastDownload/HSC" - - if not os.path.exists(base_dir): - os.makedirs(base_dir) - - manifest_array = [] - for spec in spectra: - - if spec['SpectrumType'] < 2: - data_url = f'https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={spec["DatasetName"]}' - else: - data_url = f'https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={spec["DatasetName"]}.fits' - - local_path = os.path.join(base_dir, f'{spec["DatasetName"]}.fits') - - status = "COMPLETE" - msg = None - url = None - - try: - self._download_file(data_url, local_path, cache=cache, head_safe=True) - - # check file size also this is where would perform md5 - if not os.path.isfile(local_path): - status = "ERROR" - msg = "File was not downloaded" - url = data_url - - except HTTPError as err: - status = "ERROR" - msg = "HTTPError: {0}".format(err) - url = data_url - - manifest_array.append([local_path, status, msg, url]) - - manifest = Table(rows=manifest_array, names=('Local Path', 'Status', 'Message', "URL")) + catalog = catalog.lower() + collection_obj._verify_catalog(catalog) - return manifest + return collection_obj, catalog Catalogs = CatalogsClass() From cf964828222ffc7ee05d31ab588966600527f4b3 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 3 Dec 2025 23:35:40 -0500 Subject: [PATCH 03/21] Query methods and helpers Minor refactoring, more comments style Update default catalogs mapping name copilot comments Fixing collection names Update docstrings to be more explicit about region types fix style Add TODO about sky region classes --- astroquery/mast/catalog_collection.py | 101 +- astroquery/mast/collections.py | 1441 ++++++++++++++++++++++++- 2 files changed, 1479 insertions(+), 63 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 78a8269ae2..bea517f6b0 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -10,6 +10,25 @@ __all__ = ['CatalogCollection'] +DEFAULT_CATALOGS = { + 'caom': 'dbo.obspointing', + 'gaiadr3': 'dbo.gaia_source', + 'hsc': 'dbo.SumMagAper2CatView', + 'hscv2': 'dbo.SumMagAper2CatView', + 'missionmast': 'dbo.hst_science_missionmast', + 'ps1dr1': 'dbo.MeanObjectView', + 'ps1dr2': 'dbo.MeanObjectView', + 'ps1_dr2': 'ps1_dr2.forced_mean_object', + 'skymapperdr4': 'dr4.master', + 'tic': 'dbo.CatalogRecord', + 'classy': 'dbo.targets', + 'ullyses': 'dbo.sciencemetadata', + 'goods': 'dbo.goods_master_view', + '3dhst': 'dbo.HLSP_3DHST_summary', + 'candels': 'dbo.candels_master_view', + 'deepspace': 'dbo.DeepSpace_Summary' +} + @dataclass class CatalogMetadata: @@ -76,7 +95,7 @@ def get_catalog_metadata(self, catalog): return self._catalog_metadata_cache[catalog] # Verify catalog validity for this collection - self._verify_catalog(catalog) + catalog = self._verify_catalog(catalog) # Get column metadata metadata = self._get_column_metadata(catalog) @@ -115,6 +134,10 @@ def get_default_catalog(self): str The default catalog name. """ + # Check if collection has a known default catalog + if self.name in DEFAULT_CATALOGS: + return DEFAULT_CATALOGS[self.name] + # Pick default catalog = first one that does NOT start with "tap_schema." default_catalog = next((c for c in self.catalog_names if not c.startswith("tap_schema.")), None) @@ -188,29 +211,63 @@ def _get_adql_supported_functions(self): def _verify_catalog(self, catalog): """ - Verify that the specified catalog is valid for the given collection. + Verify that the specified catalog is valid for this collection and return the correct catalog name. + Raises an error if the catalog is not valid. Parameters ---------- catalog : str The catalog to be verified. + Returns + ------- + str + The validated catalog name. + Raises ------ InvalidQueryError If the specified catalog is not valid for the given collection. """ - lower_map = {name.lower(): name for name in self.catalog_names} - if catalog.lower() not in lower_map: - closest_match = difflib.get_close_matches(catalog, self.catalog_names, n=1) - error_msg = ( - f"Catalog '{catalog}' is not recognized for collection '{self.name}'. " - f"Did you mean '{closest_match[0]}'?" - if closest_match - else f"Catalog '{catalog}' is not recognized for collection '{self.name}'." + catalog = catalog.lower() + + # Build a mapping for case-insensitive and no-prefix lookup + lookup = {} + no_prefix_map = {} + for cat in self.catalog_names: + cat_lower = cat.lower() + lookup[cat_lower] = cat # case-insensitive match + no_prefix = cat_lower.split('.')[-1] + if no_prefix not in no_prefix_map: + no_prefix_map[no_prefix] = [cat] # no-prefix match (first occurrence) + else: + no_prefix_map[no_prefix].append(cat) + + # Add unambiguous no-prefix matches to lookup + for no_prefix, cats in no_prefix_map.items(): + if len(cats) == 1: + lookup[no_prefix] = cats[0] + + # Direct or unambiguous no-prefix match + if catalog in lookup: + return lookup[catalog] + + # Check for ambiguous no-prefix matches + if catalog in no_prefix_map and len(no_prefix_map[catalog]) > 1: + matches = ', '.join(no_prefix_map[catalog]) + raise InvalidQueryError( + f"Catalog '{catalog}' is ambiguous for collection '{self.name}'. " + f"It matches multiple catalogs: {matches}. Please specify the full catalog name." ) - error_msg += " Available catalogs are: " + ", ".join(self.catalog_names) - raise InvalidQueryError(error_msg) + + # Suggest closest match (based on full catalog names) + closest = difflib.get_close_matches(catalog, self.catalog_names, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + + raise InvalidQueryError( + f"Catalog '{catalog}' is not recognized for collection '{self.name}'." + f"{suggestion} Available catalogs are: {', '.join(self.catalog_names)}" + ) def _get_column_metadata(self, catalog): """ @@ -229,7 +286,9 @@ def _get_column_metadata(self, catalog): log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") # Case-insensitive match to find the table - tap_table = next((t for name, t in self.tap_service.tables.items() if name.lower() == catalog.lower()), None) + tap_table = next((t for name, t in self.tap_service.tables.items() if name == catalog), None) + if tap_table is None: + raise InvalidQueryError(f"Catalog '{catalog}' not found in collection '{self.name}'.") # Extract column metadata col_names = [col.name for col in tap_table.columns] @@ -295,17 +354,15 @@ def _verify_criteria(self, catalog, **criteria): """ if not criteria: return - self._verify_catalog(catalog) - col_names = list(self.get_catalog_metadata(catalog)['column_metadata']['name']) + col_names = list(self.get_catalog_metadata(catalog).column_metadata['name']) # Check each criteria argument for validity for kwd in criteria.keys(): if kwd not in col_names: - closest_match = difflib.get_close_matches(kwd, col_names, n=1) - error_msg = ( - f"Filter '{kwd}' is not recognized for collection '{self.name}' and catalog '{catalog}'. " - f"Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{kwd}' is not recognized for collection '{self.name}' and catalog '{catalog}'." + # Suggest closest match for invalid keyword + closest = difflib.get_close_matches(kwd, col_names, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + raise InvalidQueryError( + f"Filter '{kwd}' is not recognized for collection '{self.name}' and " + f"catalog '{catalog}'.{suggestion}" ) - raise InvalidQueryError(error_msg) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 9c0ad76bf7..c32ee64d74 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -5,16 +5,31 @@ This module contains methods for discovering and querying MAST catalog collections. """ + import difflib +import os +import re +import time +import warnings +from collections.abc import Iterable -from astropy.table import Table +import astropy.coordinates as coord +import astropy.units as u +import requests +from astropy.table import Row, Table +from astropy.time import Time +from astropy.utils.decorators import deprecated, deprecated_renamed_argument +from regions import CircleSkyRegion, PolygonSkyRegion +from .. import log +from ..exceptions import InputWarning, InvalidQueryError, NoResultsWarning from ..utils import async_to_sync from ..utils.class_or_instance import class_or_instance +from . import conf, utils from .catalog_collection import CatalogCollection from .core import MastQueryWithLogin -__all__ = ['Catalogs', 'CatalogsClass'] +__all__ = ["Catalogs", "CatalogsClass"] @async_to_sync @@ -23,20 +38,29 @@ class CatalogsClass(MastQueryWithLogin): Class for discovering and querying MAST catalog collections. """ - TAP_BASE_URL = 'https://masttest.stsci.edu/vo-tap/api/v0.1/' + TAP_BASE_URL = conf.server + "/vo-tap/api/v0.1/" - def __init__(self, collection='hsc', catalog=None): + def __init__(self, collection=None, catalog=None): super().__init__() - self.available_collections = self.get_collections()['collection_name'].tolist() - self._no_longer_supported_collections = ['ctl', 'diskdetective', 'galex', 'plato'] - self._renamed_collections = {'panstarrs': 'ps1_dr2', 'gaia': 'gaiadr3'} + self._available_collections = None # Lazy load on first request + self._no_longer_supported_collections = ["ctl", "diskdetective", "galex", "plato"] + self._renamed_collections = {"panstarrs": "ps1_dr2", "gaia": "gaiadr3"} self._collections_cache = dict() - self.collection = collection - if catalog: - self.catalog = catalog + # Default initialization of this class should not trigger network requests + # Only set collection and catalog if explicitly provided, otherwise defer to property setters + # which will handle defaults without network calls + if not collection: + self._collection = CatalogCollection("hsc") # default collection + else: + self.collection = collection # Use the setter for validation if collection is provided + + if not catalog: + self._catalog = self._collection.default_catalog + else: + self.catalog = catalog # Use the setter for validation if catalog is provided @property def collection(self): @@ -55,7 +79,7 @@ def collection(self, collection): self._collection = collection_obj # Only change catalog if not set yet or invalid for this collection - if not hasattr(self, '_catalog') or self._catalog not in collection_obj.catalog_names: + if not hasattr(self, "_catalog") or self._catalog not in collection_obj.catalog_names: self._catalog = collection_obj.default_catalog @property @@ -70,9 +94,19 @@ def catalog(self, catalog): """ Setter that verifies that the catalog is valid for the current collection. """ - self.collection._verify_catalog(catalog) + catalog = self.collection._verify_catalog(catalog) self._catalog = catalog + @property + def available_collections(self): + """ + The list of available MAST catalog collections. + """ + if self._available_collections is None: + table = self.get_collections() + self._available_collections = table["collection_name"].tolist() + return self._available_collections + @class_or_instance def get_collections(self): """ @@ -84,21 +118,12 @@ def get_collections(self): A table containing the available MAST collections. """ # If already cached, use it directly - if getattr(self, "available_collections", None): - return Table([self.available_collections], names=('collection_name',)) + if getattr(self, "_available_collections", None): + return Table([self._available_collections], names=("collection_name",)) - # Otherwise, fetch from the TAP service + # Otherwise, fetch from TAP service discovery, including grouped collections. log.debug("Fetching available collections from MAST TAP service.") - url = self.TAP_BASE_URL + "openapi.json" - response = requests.get(url) - response.raise_for_status() - data = response.json() - - # Extract collection enumeration - collection_enum = data["components"]["schemas"]["CatalogName"]["enum"] - - # Build an Astropy Table to hold the results - collection_table = Table([collection_enum], names=('collection_name',)) + collection_table = CatalogCollection.discover_collections()[["collection_name"]] return collection_table @class_or_instance @@ -121,9 +146,9 @@ def get_catalogs(self, collection=None): return collection_obj.catalogs @class_or_instance - def get_catalog_metadata(self, collection=None, catalog=None): + def get_column_metadata(self, collection=None, catalog=None): """ - For a given collection and catalog, return metadata about the catalog. + For a given collection and catalog, return metadata about the catalog's columns. Parameters ---------- @@ -139,11 +164,685 @@ def get_catalog_metadata(self, collection=None, catalog=None): and descriptions. """ collection_obj, catalog = self._parse_inputs(collection, catalog) - return collection_obj.get_catalog_metadata(catalog).column_metadata + return collection_obj._get_column_metadata(catalog) + + def supports_spatial_queries(self, collection=None, catalog=None): + """ + Check if a given collection and catalog support spatial queries. + + Parameters + ---------- + collection : str, optional + The collection to be queried. + catalog : str, optional + The catalog within the collection to check. + + Returns + ------- + bool + True if the specified catalog supports spatial queries, False otherwise. + """ + collection_obj, catalog = self._parse_inputs(collection, catalog) + return collection_obj.get_catalog_metadata(catalog).supports_spatial_queries + + @class_or_instance + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_criteria( + self, + collection=None, + *, + catalog=None, + coordinates=None, + region=None, + object_name=None, + radius=0.2 * u.deg, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): + """ + Query a MAST catalog from a given collection using criteria filters. To return columns for a given + collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + + Parameters + ---------- + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + coordinates : str or `~astropy.coordinates` object, optional + The target around which to search. It may be specified as a string (e.g., '350 -80') or as an + Astropy coordinates object. + region : str | iterable | `~astropy.regions.CircleSkyRegion` | `~astropy.regions.PolygonSkyRegion`, optional + The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string + (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an + `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion`. + object_name : str, optional + The name of the object to resolve and search around. + radius : str or `~astropy.units.Quantity` object, optional + The search radius around the target coordinates or object. Default 0.2 degrees. + resolver : str, optional + The name resolver service to use when resolving ``object_name``. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, + applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + pagesize : int, optional + Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + page : int, optional + Deprecated. The page argument is no longer used. Please use `offset` instead. + **criteria + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + + Returns + ------- + response : `~astropy.table.Table` + A table containing the query results. + """ + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + # Should not specify both region and coordinates + if coordinates and region: + raise InvalidQueryError("Specify either `region` or `coordinates`, not both.") + + # Should not specify both region and object_name + if object_name and region: + raise InvalidQueryError("Specify either `region` or `object_name`, not both.") + + collection_obj, catalog = self._parse_inputs(collection, catalog) + + # Check for conflicts between named parameters and filters dict + if criteria and filters: + overlap = set(k.lower() for k in criteria) & set(k.lower() for k in filters) + if overlap: + raise InvalidQueryError( + f"Criteria specified both as keyword arguments and in 'filters' for columns: " + f"{', '.join(sorted(overlap))}" + ) + + # Merge criteria from named parameters and filters dict + search_criteria = {} + search_criteria.update(criteria) + if filters: + search_criteria.update(filters) + + # Validate sort_by columns together with criteria keys so all column checks go through one path + validation_criteria = dict(search_criteria) + if sort_by: + sort_by = [sort_by] if isinstance(sort_by, str) else list(sort_by) + validation_criteria.update({col: True for col in sort_by}) + + collection_obj._verify_criteria(catalog, **validation_criteria) + catalog_metadata = collection_obj.get_catalog_metadata(catalog) + column_metadata = catalog_metadata.column_metadata + columns = "*" if not select_cols else self._parse_select_cols(select_cols, column_metadata) + + adql = ( + f"SELECT TOP {limit} {columns} FROM {catalog.lower()} " + if not count_only + else f"SELECT TOP 1 COUNT(*) AS count_all FROM {catalog.lower()} " + ) + has_where = False + if region or coordinates or object_name: + # Check if the catalog supports spatial queries + if not catalog_metadata.supports_spatial_queries: + raise InvalidQueryError( + f"Catalog '{catalog}' in collection '{collection_obj.name}' does not support spatial queries." + ) + + # Positional query + adql_region = "" + if region: + adql_region = self._create_adql_region(region) + if object_name or coordinates: # Cone search + coordinates = utils.parse_input_location( + coordinates=coordinates, object_name=object_name, resolver=resolver + ) + radius = coord.Angle(radius, u.deg) # If radius is just a number we assume degrees + adql_region = f"CIRCLE('ICRS', {coordinates.ra.deg}, {coordinates.dec.deg}, {radius.to(u.deg).value})" + + region_types = ["POLYGON", "CIRCLE"] + for region_type in region_types: + if region_type in adql_region and region_type not in collection_obj.supported_adql_functions: + raise InvalidQueryError( + f"Catalog '{catalog}' in collection '{collection_obj.name}' " + f"does not support ADQL region type '{region_type}'." + ) + + # Get RA/Dec column names + ra_col = catalog_metadata.ra_column + dec_col = catalog_metadata.dec_column + adql += f"WHERE CONTAINS(POINT('ICRS', {ra_col}, {dec_col}), {adql_region}) = 1 " + has_where = True + + # Add additional constraints + if search_criteria: + conditions = self._format_criteria_conditions(collection_obj, catalog, search_criteria) + if has_where: + adql += "AND " + " AND ".join(conditions) + else: + adql += "WHERE " + " AND ".join(conditions) + + # Add sorting if specified + if sort_by: + # Add ORDER BY clause + if isinstance(sort_desc, bool): + sort_desc = [sort_desc] + + if len(sort_desc) not in (1, len(sort_by)): + raise InvalidQueryError("Length of 'sort_desc' must be 1 or equal to length of 'sort_by'.") + + if len(sort_desc) == 1: + sort_desc = sort_desc * len(sort_by) + + order_parts = [ + f"{col} {'DESC' if desc else 'ASC'}" + for col, desc in zip(sort_by, sort_desc) + ] + + adql += " ORDER BY " + ", ".join(order_parts) + + # Add offset + if offset: + adql += f" OFFSET {offset}" + + # Execute the query + result_table = collection_obj.run_tap_query(adql) + + if len(result_table) == 0: + warnings.warn("The query returned no results.", NoResultsWarning) + + if count_only: + return result_table["count_all"][0] + else: + # TODO: Add schema browser URL to the result table metadata when available + result_table.meta["collection"] = collection_obj.name + result_table.meta["catalog"] = catalog + return result_table + + @class_or_instance + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + def query_region( + self, + coordinates=None, + *, + radius=0.2 * u.deg, + region=None, + collection=None, + catalog=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): + """ + Query for MAST catalog entries within a specified region using criteria filters. To return columns for a given + collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates` object, optional + The target around which to search. It may be specified as a string (e.g., '350 -80') or as an + Astropy coordinates object. + radius : str or `~astropy.units.Quantity` object, optional + The search radius around the target coordinates or object. Default 0.2 degrees. + region : str | iterable | `~astropy.regions.CircleSkyRegion` | `~astropy.regions.PolygonSkyRegion`, optional + The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string + (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an + `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion`. + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, + applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + pagesize : int, optional + Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + page : int, optional + Deprecated. The page argument is no longer used. Please use `offset` instead. + **criteria + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + + Returns + ------- + response : `~astropy.table.Table` + A table containing the query results. + """ + # Must specify one of region or coordinates + if region is None and coordinates is None: + raise InvalidQueryError( + "Must specify either `region` or `coordinates`. For non-positional queries, " + "use `Catalogs.query_criteria`." + ) + + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + return self.query_criteria( + collection=collection, + catalog=catalog, + coordinates=coordinates, + region=region, + radius=radius, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) + + @class_or_instance + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_object( + self, + object_name, + *, + radius=0.2 * u.deg, + collection=None, + catalog=None, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): + """ + Query for MAST catalog entries around a specified object name using criteria filters. To return columns + for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + + Parameters + ---------- + object_name : str, optional + The name of the object to resolve and search around. + radius : str or `~astropy.units.Quantity` object, optional + The search radius around the target coordinates or object. Default 0.2 degrees. + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + resolver : str, optional + The name resolver service to use when resolving ``object_name``. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, + applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + pagesize : int, optional + Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + page : int, optional + Deprecated. The page argument is no longer used. Please use `offset` instead. + **criteria + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + + Returns + ------- + response : `~astropy.table.Table` + A table containing the query results. + """ + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + return self.query_criteria( + collection=collection, + catalog=catalog, + object_name=object_name, + radius=radius, + resolver=resolver, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) + + @class_or_instance + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None): + """ + Returns all the matches for a given Hubble Source Catalog MatchID. + + Parameters + ---------- + match : int or `~astropy.table.Row` + The matchID or HSC entry to return matches for. + version : int, optional + The HSC version to match against. Default is v3. + pagesize : int, optional + Can be used to override the default pagesize. + E.g. when using a slow internet connection. + page : int, optional + Can be used to override the default behavior of all results being returned to obtain + one specific page of results. + + Returns + ------- + response : list of `~requests.Response` + """ + self._current_connection = self._portal_api_connection + + if isinstance(match, Row): + match = match["MatchID"] if "MatchID" in match.colnames else match["matchid"] + match = str(match) # np.int64 gives json serializer problems, so stringify right here + + if version == 2: + service = "Mast.HscMatches.Db.v2" + else: + if version not in (3, None): + warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning) + service = "Mast.HscMatches.Db.v3" + + params = {"input": match} + + return self._current_connection.service_request_async(service, params, pagesize, page) + + @class_or_instance + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + def get_hsc_spectra_async(self, *, pagesize=None, page=None): + """ + Returns all Hubble Source Catalog spectra. + + Parameters + ---------- + pagesize : int, optional + Can be used to override the default pagesize. + E.g. when using a slow internet connection. + page : int, optional + Can be used to override the default behavior of all results being returned to obtain + one specific page of results. + + Returns + ------- + response : list of `~requests.Response` + """ + self._current_connection = self._portal_api_connection + return self._current_connection.service_request_async("Mast.HscSpectra.Db.All", {}, pagesize, page) + + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_flag=False): + """ + Download one or more Hubble Source Catalog spectra. + + Parameters + ---------- + spectra : `~astropy.table.Table` or `~astropy.table.Row` + One or more HSC spectra to be downloaded. + download_dir : str, optional + Specify the base directory to download spectra into. + Spectra will be saved in the subdirectory download_dir/mastDownload/HSC. + If download_dir is not specified the base directory will be '.'. + cache : bool, optional + Default is True. If file is found on disc it will not be downloaded again. + Note: has no affect when downloading curl script. + curl_flag : bool, optional + Default is False. If true instead of downloading files directly, a curl script + will be downloaded that can be used to download the data files at a later time. + + Returns + ------- + response : list of `~requests.Response` + """ + # Normalize spectra input to a list + if isinstance(spectra, Row): + spectra = [spectra] + + # Ensure download directory is set + download_dir = download_dir or "." + + if curl_flag: + timestamp = time.strftime("%Y%m%d%H%M%S") + bundle_name = "mastDownload_" + timestamp + url_list = [self._make_data_url(spec) for spec in spectra] + path_list = [f"{bundle_name}/HSC/{spec['DatasetName']}.fits" for spec in spectra] + + params = dict( + urlList=",".join(url_list), + filename=bundle_name, + pathList=",".join(path_list), + descriptionList=[""] * len(spectra), + productTypeList=["spectrum"] * len(spectra), + extension="curl", + ) + + service = "Mast.Bundle.Request" + response = self._portal_api_connection.service_request_async(service, params) + bundle_info = response[0].json() + local_script = os.path.join(download_dir, f"{bundle_name}.sh") + self._download_file(bundle_info["url"], local_script, head_safe=True) + + # Build manifest row + exists = os.path.isfile(local_script) + missing = [k for k, v in bundle_info.get("statusList", {}).items() if v != "COMPLETE"] + manifest = Table( + { + "Local Path": [local_script], + "Status": ["COMPLETE" if exists else "ERROR"], + "Message": [ + None + if exists and not missing + else ( + f"{len(missing)} files could not be added to curl script" + if exists + else "Curl script could not be downloaded" + ) + ], + "URL": [None if exists and not missing else (",".join(missing) if missing else bundle_info["url"])], + } + ) + return manifest + + base_dir = os.path.join(download_dir, "mastDownload", "HSC") + os.makedirs(base_dir, exist_ok=True) + manifest_rows = [] + + for row in spectra: + dataset = row["DatasetName"] + url = self._make_data_url(row) + local_path = os.path.join(base_dir, f"{dataset}.fits") + status = "COMPLETE" + message = None + + try: + self._download_file(url, local_path, cache=cache, head_safe=True) + + if not os.path.exists(local_path): + status = "ERROR" + message = "File was not downloaded" + except requests.HTTPError as err: + status = "ERROR" + message = f"HTTPError: {err}" + + manifest_rows.append([local_path, status, message, url]) + + return Table(rows=manifest_rows, names=("Local Path", "Status", "Message", "URL")) + + def _parse_result(self, response, *, verbose=False): + """Parse the async responses from HSC queries.""" + return self._current_connection._parse_result(response, verbose=verbose) def _verify_collection(self, collection): """ - Verify that the specified collection is valid. + Verify that the specified collection is valid and return the correct collection name. + Warns the user if the collection has been renamed and raises an error if the collection is not valid. Parameters ---------- @@ -155,17 +854,26 @@ def _verify_collection(self, collection): InvalidQueryError If the specified collection is not valid. """ - if collection.lower() not in self.available_collections: + collection = collection.lower().strip() + if collection in self.available_collections: + return collection + else: + if collection in self._renamed_collections: + new_name = self._renamed_collections[collection] + warn_msg = f"Collection '{collection}' has been renamed. Please use '{new_name}' instead." + warnings.warn(warn_msg, InputWarning) + return new_name + + error_msg = "" if collection in self._no_longer_supported_collections: - error_msg = (f"Collection '{collection}' is no longer supported. To query from this catalog, " - f"please use a version of Astroquery older than 0.4.12.") - else: - closest_match = difflib.get_close_matches(collection, self.available_collections, n=1) error_msg = ( - f"Collection '{collection}' is not recognized. Did you mean '{closest_match[0]}'?" - if closest_match - else f"Collection '{collection}' is not recognized." + f"Collection '{collection}' is no longer supported. To query from this catalog, " + f"please use a version of Astroquery older than 0.4.12." ) + else: + closest = difflib.get_close_matches(collection, self.available_collections, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + error_msg = f"Collection '{collection}' is not recognized.{suggestion}" error_msg += " Available collections are: " + ", ".join(self.available_collections) raise InvalidQueryError(error_msg) @@ -183,18 +891,21 @@ def _get_collection_obj(self, collection_name): CatalogCollection The corresponding CatalogCollection object. """ + if not isinstance(collection_name, str): + raise InvalidQueryError("Collection name must be a string.") + collection_name = collection_name.lower().strip() if collection_name in self._collections_cache: return self._collections_cache[collection_name] - self._verify_collection(collection_name) + collection_name = self._verify_collection(collection_name) collection_obj = CatalogCollection(collection_name) self._collections_cache[collection_name] = collection_obj return collection_obj def _parse_inputs(self, collection=None, catalog=None): """ - Return (collection, catalog) applying default attributes, validation, and normalization. + Parse and validate the collection and catalog inputs. Parameters ---------- @@ -219,9 +930,657 @@ def _parse_inputs(self, collection=None, catalog=None): catalog = collection_obj.default_catalog else: catalog = catalog.lower() - collection_obj._verify_catalog(catalog) + # For backwards compatibility, check if the user is trying to specify a collection via catalog + if ( + catalog in self.available_collections + or catalog in self._no_longer_supported_collections + or catalog in self._renamed_collections + ) and not collection: + warnings.warn( + f"Specifying collection '{catalog}' via the `catalog` parameter is deprecated. " + f"Please use the `collection` parameter instead.", + DeprecationWarning, + ) + # As a convenience to the user, set the collection accordingly and use its default catalog + collection_obj = self._get_collection_obj(catalog) + catalog = collection_obj.default_catalog + else: + catalog = collection_obj._verify_catalog(catalog) return collection_obj, catalog + def _parse_select_cols(self, select_cols, column_metadata): + """ + Validate and parse the select_cols parameter. + + Parameters + ---------- + select_cols : list of str + List of column names to include in the result. + column_metadata : `~astropy.table.Table` + The catalog's column metadata table. + + Returns + ------- + str + Comma-separated string of valid column names for ADQL SELECT clause. + + Raises + ------ + InvalidQueryError + If any specified column is not found in the catalog metadata. + """ + valid_columns = column_metadata["column_name"].tolist() + valid_selected = [] + for col in select_cols: + if col not in valid_columns: + closest = difflib.get_close_matches(col, valid_columns, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + warnings.warn(f"Column '{col}' not found in catalog.{suggestion}", InputWarning) + else: + valid_selected.append(col) + if not valid_selected: + raise InvalidQueryError("No valid columns specified in `select_cols`.") + return ", ".join(valid_selected) + + def _parse_legacy_pagination(self, limit, offset, pagesize, page): + """ + Parse legacy pagesize and page parameters to determine limit and offset. + + Parameters + ---------- + limit : int + The maximum number of results to return. + offset : int + The number of rows to skip before starting to return rows. + pagesize : int, optional + The number of results per page (legacy parameter). + page : int, optional + The page number to return (legacy parameter). + + Returns + ------- + tuple + A tuple containing the (limit, offset) values. + """ + # If limit and offset are default, check for legacy pagination params + if limit == 5000 and offset == 0: + if pagesize is not None: + if page is None: + page = 1 # Default to first page if not specified + limit = pagesize + offset = (page - 1) * pagesize + elif page is not None: + warnings.warn( + "The 'page' parameter is ignored without 'pagesize'. " + "Please use `limit` and `offset` to specify pagination.", + InputWarning, + ) + return limit, offset + + def _create_adql_region(self, region): + """ + Returns the ADQL description of the given polygon or circle region. + + Parameters + ---------- + region : str | iterable | astropy.regions.Region + - Iterable of RA/Dec pairs as lists/sequences + - STC-S POLYGON or CIRCLE string + - `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion` + + Returns + ------- + adql_region : str + ADQL representation of the region (POLYGON or CIRCLE) + """ + # Case 1: region is a string (e.g. STC-S syntax) + if isinstance(region, str): + parts = region.strip().lower().split() + shape = parts[0] + + if shape == "polygon": + # POLYGON lon1 lat1 lon2 lat2 ... + # Optional format: POLYGON ICRS lon1 lat1 ... + # parts = ["POLYGON", maybe_frame?, ...coords...] + + # Determine if parts[1] is a coord or a frame name + if len(parts) < 3: + raise InvalidQueryError(f"Invalid POLYGON region string: {region}") + try: + float(parts[1]) # numeric → no frame name + point_parts = parts[1:] + except ValueError: + point_parts = parts[2:] # skip optional frame name + + if len(point_parts) < 6 or len(point_parts) % 2 != 0: + # polygon requires at least 3 points (6 numbers), and must be pairs + raise InvalidQueryError(f"Invalid POLYGON region string: {region}") + + point_string = ",".join(point_parts) + return f"POLYGON('ICRS',{point_string})" + + elif shape == "circle": + # CIRCLE ra dec radius (or CIRCLE ICRS ra dec radius) + if len(parts) < 4: + raise InvalidQueryError(f"Invalid CIRCLE region string: {region}") + + # Try interpreting parts[1] as RA. If not numeric, assume it's a frame + try: + float(parts[1]) + # Format: CIRCLE ra dec radius + ra, dec, radius = parts[1], parts[2], parts[3] + except ValueError: + # Format: CIRCLE FRAME ra dec radius + if len(parts) < 5: + raise InvalidQueryError(f"Invalid CIRCLE region string: {region}") + ra, dec, radius = parts[2], parts[3], parts[4] + + return f"CIRCLE('ICRS',{ra},{dec},{radius})" + + else: + raise InvalidQueryError(f"Unrecognized region string: {region}") + + # Case 2: region is an astropy region object + # TODO: When released, change these to use `CircleSphericalSkyRegion` and `PolygonSphericalSkyRegion` + elif isinstance(region, CircleSkyRegion): + center = region.center.icrs + radius = region.radius.to(u.deg).value + return f"CIRCLE('ICRS',{center.ra.deg},{center.dec.deg},{radius})" + elif isinstance(region, PolygonSkyRegion): + verts = region.vertices.icrs + point_string = ",".join(f"{v.ra.deg},{v.dec.deg}" for v in verts) + return f"POLYGON('ICRS',{point_string})" + + # Case 3: region is an iterable of coordinate pairs + elif isinstance(region, Iterable): + # Expect something like [(ra1, dec1), (ra2, dec2), ...] + try: + points = [float(x) for point in region for x in point] + except Exception as e: + raise InvalidQueryError(f"Invalid iterable region format: {region}") from e + return f"POLYGON('ICRS',{','.join(str(x) for x in points)})" + + else: + raise TypeError(f"Unsupported region type: {type(region)}") + + def _classify_columns(self, meta): + """ + Classify columns as numeric or temporal based on their datatype and UCD metadata. + + Parameters + ---------- + meta : `~astropy.table.Table` + The catalog's column metadata table, which must include 'column_name', 'datatype', and 'ucd' columns. + + Returns + ------- + tuple + A tuple containing two sets: (numeric_columns, temporal_columns), where each set contains the names of the + columns classified as numeric or temporal, respectively. + """ + num_types = {"int", "long", "short", "float", "double", "floatcomplex", "doublecomplex"} + temporal_tokens = {"date", "time", "timestamp", "datetime"} + + numeric = set() + temporal = set() + + for name, dtype, ucd in zip(meta["column_name"], meta["datatype"], meta["ucd"]): + dtype_str = dtype.lower() if isinstance(dtype, str) else "" + ucd_str = ucd.lower() if isinstance(ucd, str) else "" + + # Classify as numeric if the datatype matches known numeric types + if dtype_str in num_types: + numeric.add(name) + + # Classify as temporal if the datatype contains temporal tokens or if the + # UCD indicates a time-related quantity (but exclude numeric types that may have time-related UCDs) + has_temporal_dtype = any(token in dtype_str for token in temporal_tokens) + has_temporal_ucd = ucd_str.startswith("time.") or ";time." in ucd_str + not_numeric = dtype_str not in num_types + + if has_temporal_dtype or (has_temporal_ucd and not_numeric): + temporal.add(name) + + return numeric, temporal + + def _quote_adql_string(self, adql_str): + """Escape single quotes in ADQL query strings by doubling them.""" + return adql_str.replace("'", "''") + + def _parse_range_cmp_expr(self, col, expr, value_formatter): + expr = expr.strip() + + range_match = re.fullmatch(r"(.+?)\s*\.\.\s*(.+)", expr) + if range_match: + low = value_formatter(range_match.group(1).strip(), col) + high = value_formatter(range_match.group(2).strip(), col) + return f"{col} BETWEEN {low} AND {high}" + + cmp_match = re.fullmatch(r"(<=|>=|<|>)\s*(.+)", expr) + if cmp_match: + rhs = value_formatter(cmp_match.group(2).strip(), col) + return f"{col} {cmp_match.group(1)} {rhs}" + + def _parse_numeric_expr(self, col, expr): + """ + Parse a numeric expression for a column and return the corresponding ADQL predicate. + + Parameters + ---------- + col : str + The column name. + expr : str + The numeric expression (e.g., "5", "<10", "5..10"). + + Returns + ------- + str + The ADQL predicate for the numeric expression. + """ + parsed = self._parse_range_cmp_expr(col, expr, lambda x, _: x) + if parsed: + return parsed + + try: + return f"{col} = {float(expr)}" + except ValueError: + raise InvalidQueryError( + f"Column '{col}' is numeric; unsupported value '{expr}'. Use numbers, comparisons like '<10', or " + "ranges like '5..10'." + ) + + def _normalize_time(self, value): + """ + Normalize a datetime value to a string format suitable for ADQL queries. + + Parameters + ---------- + value : str or `~astropy.time.Time` or `~datetime.datetime` + The datetime value to normalize. + + Returns + ------- + str + The normalized datetime string in the format 'YYYY-MM-DD HH:MM:SS' or the original + value if it cannot be parsed as a datetime. + + Raises + ------ + ValueError + If the value cannot be parsed as a datetime. + """ + t = Time(value) + dt = t.to_datetime() + + # Drop microseconds to avoid ADQL parsing issues + if dt.microsecond: + dt = dt.replace(microsecond=0) + + return dt.strftime("%Y-%m-%d %H:%M:%S") + + def _to_temporal_literal(self, value, col): + """ + Convert a datetime value to an ADQL literal string, ensuring it is properly formatted and quoted. + + Parameters + ---------- + value : str or `~astropy.time.Time` or `~datetime.datetime` + The datetime value to convert. + col : str + The column name (used for error messages). + + Returns + ------- + str + The ADQL literal string representing the datetime value. + """ + try: + normalized = self._normalize_time(value) + except ValueError: + # If it can't be parsed as a time, send the value as-is for the backend to handle + # (which may raise an error if it's invalid) + normalized = value if isinstance(value, str) else str(value) + escaped = self._quote_adql_string(normalized) + return f"'{escaped}'" + + def _parse_temporal_expr(self, col, expr): + """ + Parse a datetime expression for a column and return the corresponding ADQL predicate. + + Parameters + ---------- + col : str + The column name. + expr : str + Datetime expression (e.g., "2024-01-01", ">=2024-01-01", "2024-01-01..2024-12-31"). + + Returns + ------- + str + The ADQL predicate for the datetime expression. + """ + if isinstance(expr, str): + parsed = self._parse_range_cmp_expr(col, expr, self._to_temporal_literal) + if parsed: + return parsed + + # For simple equality, expand to a small range + try: + t0 = Time(self._normalize_time(expr)) + t1 = t0 + 1 * u.s + low = self._quote_adql_string(t0.strftime("%Y-%m-%d %H:%M:%S")) + high = self._quote_adql_string(t1.strftime("%Y-%m-%d %H:%M:%S")) + return f"{col} BETWEEN '{low}' AND '{high}'" + except ValueError: + # If it can't be parsed as a time, send the value as-is for the backend to handle + # (which may raise an error if it's invalid) + return f"{col} = '{expr}'" + + def _format_scalar_predicate(self, col, val, is_numeric=False, is_temporal=False): + """ + Build predicate for a scalar value, aware of column type. + + Parameters + ---------- + col : str + The column name. + val : scalar + The value to build the predicate for. + is_numeric : bool + Whether the column is numeric. + is_temporal : bool + Whether the column is temporal. + + Returns + ------- + str + The ADQL predicate for the scalar value. + """ + if isinstance(val, bool): + # Booleans stored as integers + return f"{col} = {int(val)}" + + if is_temporal: + is_neg = isinstance(val, str) and val.startswith("!") + sval = val[1:].strip() if is_neg else val + parsed = self._parse_temporal_expr(col, sval) + return f"NOT ({parsed})" if is_neg else parsed + + if isinstance(val, str): + # Check for negation + is_neg = val.startswith("!") + sval = val[1:].strip() if is_neg else val + + # Strings for numeric columns + if is_numeric: + parsed = self._parse_numeric_expr(col, sval) + return f"NOT ({parsed})" if is_neg else parsed + + # Non-numeric strings + has_wild = ("*" in sval) or ("%" in sval) + pattern = self._quote_adql_string(sval.replace("*", "%")) + expr = f"{col} LIKE '{pattern}'" if has_wild else f"{col} = '{pattern}'" + return f"NOT ({expr})" if is_neg else expr + + # Numerics or others + return f"{col} = {val}" + + def _combine_predicates(self, pos_parts, neg_parts): + """ + Combine positive and negative predicate parts into a single ADQL expression. + + Parameters + ---------- + pos_parts : list of str + List of positive predicate strings. + neg_parts : list of str + List of negative predicate strings. + + Returns + ------- + str + The combined ADQL predicate. + """ + pos_expr = "" + if len(pos_parts) == 1: + pos_expr = pos_parts[0] + elif len(pos_parts) > 1: + pos_expr = "(" + " OR ".join(pos_parts) + ")" + + if neg_parts and pos_expr: + return "(" + " AND ".join(neg_parts) + ") AND " + pos_expr + if neg_parts: + return " AND ".join(neg_parts) + return pos_expr + + def _build_list_predicate(self, col, pos_items, neg_items, pos_builder, neg_builder): + """ + General helper to build predicates for list values with separate positive and negative handling. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + pos_builder : function + Function to build positive predicates, called as pos_builder(col, pos_items). + neg_builder : function + Function to build negative predicates, called as neg_builder(col, neg_items). + + Returns + ------- + str + The combined ADQL predicate for the list values. + """ + pos_parts = pos_builder(col, pos_items) if pos_items else [] + neg_parts = [neg_builder(col, v) for v in neg_items] if neg_items else [] + return self._combine_predicates(pos_parts, neg_parts) + + def _build_numeric_list_predicate(self, col, pos_items, neg_items): + """ + Build predicate for multiple values passed into a numeric column with separated positives and negatives. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the numeric list. + """ + def build_positive(col, items): + # Separate simple numeric values from complex expressions (comparisons, ranges) + simple_numbers = [] + complex_parts = [] + for val in items: + if isinstance(val, bool): + simple_numbers.append(int(val)) + elif isinstance(val, (int, float)): + simple_numbers.append(val) + elif isinstance(val, str): + try: + simple_numbers.append(float(val)) + except ValueError: + complex_parts.append(self._parse_numeric_expr(col, val)) + else: + try: + simple_numbers.append(float(val)) + except (ValueError, TypeError): + raise InvalidQueryError(f"Unsupported numeric value type: {type(val)}") + + parts = [] + if simple_numbers: + vals = [str(v) for v in simple_numbers] + parts.append(f"{col} IN (" + ", ".join(vals) + ")") + parts.extend(complex_parts) + return parts + + def build_negative(col, v): + # For negatives, we can't use NOT IN or NOT (complex), so we build each as a separate predicate + # and AND them together + return self._format_scalar_predicate(col, f"!{v}", is_numeric=True) + + return self._build_list_predicate( + col, + pos_items, + neg_items, + build_positive, + build_negative, + ) + + def _build_string_list_predicate(self, col, pos_items, neg_items): + """ + Build predicate for multiple values passed into a string column with separated positives and negatives. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the string list. + """ + def build_positive(col, items): + # Separate simple strings (no wildcards) from those that require LIKE + simple_strings = [] + pattern_parts = [] + for v in items: + if isinstance(v, bool): + simple_strings.append(str(int(v))) + elif isinstance(v, str): + if ("*" in v) or ("%" in v): + patt = self._quote_adql_string(v.replace("*", "%")) + pattern_parts.append(f"{col} LIKE '{patt}'") + else: + simple_strings.append("'" + self._quote_adql_string(v) + "'") + else: + simple_strings.append(str(v)) + + parts = [] + if simple_strings: + parts.append(f"{col} IN (" + ", ".join(simple_strings) + ")") + parts.extend(pattern_parts) + return parts + + # Negative predicates → use helper + def build_negative(col, v): + # For negatives, we can't use NOT IN or NOT (LIKE), so we build each as a separate predicate + # and AND them together + return self._format_scalar_predicate(col, f"!{v}") + + return self._build_list_predicate( + col, + pos_items, + neg_items, + build_positive, + build_negative, + ) + + def _build_temporal_list_predicate(self, col, pos_items, neg_items): + """ + Build temporal list predicates using column datatype to choose CAST vs string comparison. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the temporal list. + """ + return self._build_list_predicate( + col, + pos_items, + neg_items, + pos_builder=lambda c, vals: [self._parse_temporal_expr(c, v) for v in vals], + neg_builder=lambda c, v: self._format_scalar_predicate(c, f"!{v}", is_temporal=True), + ) + + def _format_criteria_conditions(self, collection_obj, catalog, criteria): + """ + Turn a criteria dict into ADQL WHERE clause expressions, aware of column types. + + - Scalars: equality (strings quoted; booleans -> 0/1; numerics raw). + - Strings with wildcards '*' or '%': uses LIKE (converting '*' to '%'). + - Lists/Tuples: if any string contains wildcard, build OR of LIKEs; otherwise use IN (...). + - Numeric columns: support comparison strings ('<10', '>= 5') and ranges ('5..10', inclusive). + Empty lists yield a false predicate (1=0). + - Negation: a value prefixed with '!' is treated as a negated predicate. For list values, all negations are + AND'ed together and combined with the OR of positives: (neg1 AND neg2) AND (pos1 OR pos2 ...). + + Parameters + ---------- + criteria : dict + Mapping of column name -> scalar or list of scalars. + + Returns + ------- + list of str + ADQL predicate strings (without leading WHERE/AND), suitable for joining with ' AND '. + """ + column_meta = collection_obj.get_catalog_metadata(catalog).column_metadata + numeric_cols, temporal_cols = self._classify_columns(column_meta) + conditions = [] + for key, value in criteria.items(): + # Handle list-like values => IN or OR(LIKE ...) + if isinstance(value, (list, tuple)): + values = list(value) + if len(values) == 0: + conditions.append("1=0") + continue + # Separate negatives (prefixed with '!') and positives + neg_items = [] + pos_items = [] + for v in values: + if isinstance(v, str) and v.startswith("!"): + neg_items.append(v[1:].strip()) + else: + pos_items.append(v) + + if key in temporal_cols: + expr = self._build_temporal_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + elif key in numeric_cols: + expr = self._build_numeric_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + else: + expr = self._build_string_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + else: + conditions.append(self._format_scalar_predicate(key, value, key in numeric_cols, key in temporal_cols)) + return conditions + + def _make_data_url(self, row): + """Return the correct data URL for a given spectrum row.""" + dataset = row["DatasetName"] + if row["SpectrumType"] < 2: + return f"https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={dataset}" + return f"https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={dataset}.fits" + -Catalogs = CatalogsClass() +Catalogs = CatalogsClass() \ No newline at end of file From 2de8db6b1139c3f2a242214d25b56bcb03672d23 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 8 Jan 2026 12:40:59 -0500 Subject: [PATCH 04/21] lazy loading, use queries instead of tables attribute --- astroquery/mast/catalog_collection.py | 79 +++++++++++++++------------ 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index bea517f6b0..d40a9dbbd6 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -63,11 +63,10 @@ def __init__(self, collection): self.tap_service = TAPService(self.TAP_BASE_URL + self.name) # Get catalogs within this collection - self.catalogs = self._get_catalogs() - self.catalog_names = self.catalogs['catalog_name'].tolist() + self._catalogs = None # Lazy-loaded property # ADQL functions supported by this collection's TAP service - self.supported_adql_functions = self._get_adql_supported_functions() + self._supported_adql_functions = None # Lazy-loaded property # Determine the default catalog for this collection self.default_catalog = self.get_default_catalog() @@ -75,6 +74,22 @@ def __init__(self, collection): # Cache for catalog metadata to avoid redundant queries self._catalog_metadata_cache: Dict[str, CatalogMetadata] = dict() + @property + def catalogs(self): + if self._catalogs is None: + self._catalogs = self._fetch_catalogs() + return self._catalogs + + @property + def catalog_names(self): + return self.catalogs['catalog_name'].tolist() + + @property + def supported_adql_functions(self): + if self._supported_adql_functions is None: + self._supported_adql_functions = self._fetch_adql_supported_functions() + return self._supported_adql_functions + def get_catalog_metadata(self, catalog): """ For a given catalog, cache and return metadata about its columns and capabilties. @@ -90,13 +105,13 @@ def get_catalog_metadata(self, catalog): A CatalogMetadata object containing metadata about the specified catalog, including column metadata, RA/Dec column names, and spatial query support. """ + # Verify catalog validity for this collection + catalog = self._verify_catalog(catalog) + # Check cache first if catalog in self._catalog_metadata_cache: return self._catalog_metadata_cache[catalog] - # Verify catalog validity for this collection - catalog = self._verify_catalog(catalog) - # Get column metadata metadata = self._get_column_metadata(catalog) @@ -165,7 +180,7 @@ def run_tap_query(self, adql): result = self.tap_service.search(adql) return result.to_table() - def _get_catalogs(self): + def _fetch_catalogs(self): """ Retrieve the list of catalogs in this collection. @@ -175,15 +190,15 @@ def _get_catalogs(self): List of catalog names. """ log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") - tables = self.tap_service.tables - names = [t.name for t in tables] - descriptions = [t.description for t in tables] + query = 'SELECT table_name, description FROM tap_schema.tables' + result = self.tap_service.run_sync(query) - # Create an Astropy Table to hold the results - result_table = Table([names, descriptions], names=('catalog_name', 'description')) + # Rename table_name to catalog_name for clarity + result_table = result.to_table() + result_table.rename_column('table_name', 'catalog_name') return result_table - def _get_adql_supported_functions(self): + def _fetch_adql_supported_functions(self): """ Retrieve the ADQL supported functions of the TAP service. @@ -285,28 +300,22 @@ def _get_column_metadata(self, catalog): """ log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") - # Case-insensitive match to find the table - tap_table = next((t for name, t in self.tap_service.tables.items() if name == catalog), None) - if tap_table is None: + query = f""" + SELECT + column_name, + datatype, + unit, + ucd, + description + FROM tap_schema.columns + WHERE table_name = '{catalog}' + """ + result = self.tap_service.run_sync(query) + + if len(result) == 0: raise InvalidQueryError(f"Catalog '{catalog}' not found in collection '{self.name}'.") - # Extract column metadata - col_names = [col.name for col in tap_table.columns] - col_datatypes = [] - for col in tap_table.columns: - try: - col_datatypes.append(col.datatype._content) - except AttributeError: - # Some pyvo versions store datatype differently; fall back gracefully - # Fallback: str(col.datatype) or None - col_datatypes.append(getattr(col.datatype, '_content', str(col.datatype))) - col_units = [col.unit for col in tap_table.columns] - col_ucds = [col.ucd for col in tap_table.columns] - col_descriptions = [col.description for col in tap_table.columns] - - # Create an Astropy Table to hold the metadata - column_metadata = Table([col_names, col_datatypes, col_units, col_ucds, col_descriptions], - names=('name', 'data_type', 'unit', 'ucd', 'description')) + column_metadata = result.to_table() return column_metadata def _get_ra_dec_column_names(self, column_metadata): @@ -326,7 +335,7 @@ def _get_ra_dec_column_names(self, column_metadata): # Look for a column with UCD 'pos.eq.ra;meta.main' and 'pos.eq.dec;meta.main' ra_col = None dec_col = None - for name, ucd in zip(column_metadata['name'], column_metadata['ucd']): + for name, ucd in zip(column_metadata['column_name'], column_metadata['ucd']): if ucd and 'pos.eq.ra;meta.main' in ucd: # TODO: ps1_dr2.mean_object and ps1_dr2.stacked_object has a column that can be used, # but is not labeled with "meta.main" @@ -354,7 +363,7 @@ def _verify_criteria(self, catalog, **criteria): """ if not criteria: return - col_names = list(self.get_catalog_metadata(catalog).column_metadata['name']) + col_names = list(self.get_catalog_metadata(catalog).column_metadata['column_name']) # Check each criteria argument for validity for kwd in criteria.keys(): From d01a461aef9877c1e7d93f88be7eb67dd86cddb2 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Sun, 11 Jan 2026 21:04:32 -0500 Subject: [PATCH 05/21] docs --- docs/mast/mast_catalog.rst | 631 +++++++++++++++++++++---------------- 1 file changed, 357 insertions(+), 274 deletions(-) diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index fe8c05d857..c2ab0ec7b3 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -1,332 +1,415 @@ - *************** Catalog Queries *************** -The Catalogs class provides access to a subset of the astronomical catalogs stored at MAST. -The catalogs currently available through this interface are: +The `~astroquery.mast.Catalogs` interface provides toold for discovering and querying the wide +range of astronomical catalogs hosted by MAST. These catalogs span multiple missions and surveys +and are organized into **collections**, each of which may contain one or more **catalogs** with +distinct schemas and capabilties. This interface is designed for **flexible, SQL-like querying** of +catalog data, including spatial searches and column-based filtering. + +At a high level, querying MAST catalogs follows three steps: + 1. Discover available collections and catalogs. + 2. Inspect catalog metadata to understand available fields and data types. + 3. Query the catalog using positional and/or criteria-based filters. -- The Hubble Source Catalog (HSC) -- The GALEX Catalog (V2 and V3) -- The Gaia (DR1 and DR2) and TGAS Catalogs -- The TESS Input Catalog (TIC) -- The TESS Candidate Target List (CTL) -- The Disk Detective Catalog -- The PanSTARRS Catalog (DR1 and DR2) -- The All-Sky PLATO Input Catalog (DR1) +Collections and Catalogs +======================== -Catalog Positional Queries -========================== +MAST catalogs are organized into **collections**, where each collection represents a related set of catalogs +with a shared scientific or mission context (for example, Hubble source catalogs, Gaia data releases, etc). +Within a collection, one or more **catalogs** may be available, each with its own schema and data. -Positional queries can be based on a sky position or a target name. -The returned fields vary by catalog, find the field documentation for specific catalogs -`here `__. -If no catalog is specified, the Hubble Source Catalog will be queried. +`~astroquery.mast.CatalogsClass` maintains a current collection and catalog as attributes. If no collection or catalog +is specified in a query, these attributes will be used as defaults. The ``collection`` attribute is a +`~astroquery.mast.catalog_collection.CatalogCollection` instance representing the current collection, +and the ``catalog`` attribute is a string representing the name of the current catalog within that collection. - .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", catalog="Galex") - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - distance_arcmin objID survey ... fuv_flux_aper_7 fuv_artifact - ------------------ ------------------- ------ ... --------------- ------------ - 0.3493802506329695 6382034098673685038 AIS ... 0.047751952 0 - 0.7615422488595471 6382034098672634783 AIS ... -- 0 - 0.9243329366166956 6382034098672634656 AIS ... -- 0 - 1.162615739258038 6382034098672634662 AIS ... -- 0 - 1.2670891287503308 6382034098672634735 AIS ... -- 0 - 1.492173395497916 6382034098674731780 AIS ... 0.0611195639 0 - 1.6051235757244107 6382034098672634645 AIS ... -- 0 - 1.705418541388336 6382034098672634716 AIS ... -- 0 - 1.7463721100195875 6382034098672634619 AIS ... -- 0 - 1.7524423152919317 6382034098672634846 AIS ... -- 0 - - -Some catalogs have a maximum number of results they will return. -If a query results in this maximum number of results a warning will be displayed to alert -the user that they might be getting a subset of the true result set. + >>> print("Default collection:", Catalogs.collection.name) + Default collection: hsc + >>> print("Default catalog:", Catalogs.catalog) + Default catalog: dbo.SumMagAper2CatView + +These attributes may be changed at any time to set new defaults. Both ``collection`` and ``catalog`` will be validated +when set. When changing the collection, the catalog will be reset to the default for the new collection. .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_region("322.49324 12.16683", - ... catalog="HSC", - ... magtype=2) # doctest: +SHOW_WARNINGS - InputWarning: Coordinate string is being interpreted as an ICRS coordinate provided in degrees. - MaxResultsWarning: Maximum catalog results returned, may not include all sources within radius. - >>> print(catalog_data[:10]) - MatchID Distance MatchRA ... W3_F160W_MAD W3_F160W_N - --------- -------------------- ------------------ ... ------------ ---------- - 50180585 0.003984902849540913 322.4931746094701 ... nan 0 - 8150896 0.006357935819940561 322.49334740450234 ... nan 0 - 100906349 0.00808206428937523 322.4932839715549 ... nan 0 - 105434103 0.011947078376104195 322.49324000530777 ... nan 0 - 103116183 0.01274757103013683 322.4934207202404 ... nan 0 - 45593349 0.013026569623011767 322.4933878707698 ... nan 0 - 103700905 0.01306760650244682 322.4932769229944 ... nan 0 - 102470085 0.014611879195009472 322.49311034430366 ... nan 0 - 93722307 0.01476438046135455 322.49348351134466 ... nan 0 - 24781941 0.015234351867433582 322.49300148743345 ... nan 0 - -Radius is an optional parameter and the default is 0.2 degrees. + >>> Catalogs.collection = "TIC" # set collection to TESS Input Catalog + >>> print("New collection:", Catalogs.collection.name) + New collection: tic + >>> print("New catalog:", Catalogs.catalog) + New catalog: dbo.CatalogRecord -.. doctest-remote-data:: +Discovering Available Collections +--------------------------------- - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="TIC") - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - ID ra dec ... wdflag dstArcSec - ---------- ---------------- ----------------- ... ------ ------------------ - 510188144 254.287449269816 -4.09954224264168 ... -1 0.7650443624931581 - 510188143 254.28717785824 -4.09908635292493 ... -1 1.3400566638148848 - 189844423 254.287799703996 -4.0994998249247 ... 0 1.3644407138867785 - 1305764031 254.287147439535 -4.09866105132406 ... -1 2.656905409847388 - 1305763882 254.286696117371 -4.09925522448626 ... -1 2.7561196688252894 - 510188145 254.287431890823 -4.10017293344746 ... -1 3.036238557555728 - 1305763844 254.286675148545 -4.09971617257086 ... 0 3.1424781549696217 - 1305764030 254.287249718516 -4.09841883152995 ... -1 3.365991083435227 - 1305764097 254.287599269103 -4.09837925361712 ... -1 3.4590276863989 - 1305764215 254.28820865799 -4.09859677020253 ... -1 3.7675526728257034 - - -The Hubble Source Catalog, the Gaia Catalog, and the PanSTARRS Catalog have multiple versions. -An optional version parameter allows you to select which version you want, the default is the highest version. +To list all available catalog collections hosted at MAST, use the +`~astroquery.mast.CatalogsClass.get_collections` method. -.. doctest-remote-data:: +This returns an `~astropy.table.Table` containing the names of all available collections. - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, - ... catalog="Gaia", version=2) - >>> print("Number of results:",len(catalog_data)) - Number of results: 111 - >>> print(catalog_data[:4]) - solution_id designation ... distance - ------------------- ---------------------------- ... ------------------ - 1635721458409799680 Gaia DR2 3774902350511581696 ... 0.6326770410972467 - 1635721458409799680 Gaia DR2 3774901427093274112 ... 0.8440033390947586 - 1635721458409799680 Gaia DR2 3774902148648277248 ... 0.9199206487344911 - 1635721458409799680 Gaia DR2 3774902453590798208 ... 1.3578181104319944 - -The PanSTARRS Catalog has multiple data releases as well as multiple queryable tables. -An optional data release parameter allows you to select which data release is desired, with the default being the latest version (dr2). -The table to query is a required parameter. +Some historical collections are no longer supported for querying, and will not appear in this list. If a collection +has been renamed or deprecated, Astroquery will issue a warning and suggest the appropriate replacement where possible. .. doctest-remote-data:: - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", - ... radius=0.01, - ... catalog="Panstarrs", - ... data_release="dr1", - ... table="mean") - >>> print("Number of results:",len(catalog_data)) # doctest: +IGNORE_OUTPUT - Number of results: 42 - >>> print(catalog_data[:5]) # doctest: +IGNORE_OUTPUT - objName objAltName1 ... yFlags distance - -------------------------- ----------- ... ------ --------------------- - PSO J103357.027-071828.380 -999 ... 0 0.008463641060218161 - PSO J103357.130-071834.314 -999 ... 4120 0.008734008139467626 - PSO J103357.065-071832.617 -999 ... 4120 0.008480972171475138 - PSO J103355.542-071833.037 -999 ... 16416 0.0022151507196657037 - PSO J103356.363-071839.939 -999 ... 0 0.005754569818470991 - -Catalog Criteria Queries -======================== + >>> collections = Catalogs.get_collections() + >>> print(collections) # doctest: +IGNORE_OUTPUT + collection_name + --------------- + caom + classy + gaiadr3 + hsc + hscv2 + missionmast + ps1dr1 + ps1dr2 + ps1_dr2 + registry + skymapperdr4 + tic + ullyses + goods + candels + 3dhst + deepspace + +Discovering Catalogs in a Collection +------------------------------------- + +Once a collection is selected, you can list the catalogs available within it using the +`~astroquery.mast.catalog_collection.CatalogCollection.get_catalogs` method. + +To query catalogs for a specific collection without changing the class state, you can pass the +collection name as an argument to the method. + +.. doctest-remote-data:: -The TESS Input Catalog (TIC), Disk Detective Catalog, and PanSTARRS Catalog can also be queried based on non-positional criteria. + >>> catalogs = Catalogs.get_catalogs('hsc') + >>> catalogs.pprint(max_width=-1) # doctest: +IGNORE_OUTPUT + catalog_name description + -------------------------- ------------------------------------------------------- + tap_schema.schemas description of schemas in this dataset + tap_schema.tables description of tables in this dataset + tap_schema.columns description of columns in this dataset + tap_schema.keys description of foreign keys in this dataset + tap_schema.key_columns description of foreign key columns in this dataset + dbo.detailedcatalog Detailed list of source catalog parameters + dbo.hcvdetailedview Detailed list of Hubble Catalog of Variables parameters + dbo.hcvsummaryview Summary list of Hubble Catalog of Variables parameters + dbo.propermotionsview List of proper motion information + dbo.sourcepositionsview List of source position information + dbo.summagaper2catview Summary list of source catalog with Aper2 magnitudes + dbo.summagautocatview Summary list of source catalog with MagAuto magnitudes + dbo.catalog_image_metadata Summary list of Image processing metadata + +Inspecting Catalog Metadata +---------------------------- + +Before querying a catalog, it is often useful to inspect its metadata to understand the available columns, +data types, and supported query capabilties. The `~astroquery.mast.CatalogsClass.get_catalog_metadata` method returns +an `~astropy.table.Table` describing the catalog schema, including column names, data types, units, and descriptions. +This metadata can help you construct valid queries, select columns of interest, and understand which fields support +numeric comparions, string matching, or spatial queries. + +Again, you can specify a collection and catalog explicitly as inputs to the function, or use the current defaults. +If you only specify a collection, the default catalog for that collection will be used. .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_criteria(catalog="Tic", Bmag=[30,50], objType="STAR") - >>> print(catalog_data) # doctest: +IGNORE_OUTPUT - ID version HIP TYC ... e_Dec_orig raddflag wdflag objID - --------- -------- --- --- ... ------------------ -------- ------ ---------- - 125413929 20190415 -- -- ... 0.293682765259495 1 0 579825059 - 261459129 20190415 -- -- ... 0.200397148604244 1 0 1701625107 - 64575709 20190415 -- -- ... 0.21969663115091 1 0 595775997 - 94322581 20190415 -- -- ... 0.205286802302475 1 0 606092549 - 125414201 20190415 -- -- ... 0.22398993783274 1 0 579825329 - 463721073 20190415 -- -- ... 0.489828592248652 -1 1 710312391 - 81609218 20190415 -- -- ... 0.146788572369267 1 0 630541794 - 282024596 20190415 -- -- ... 0.548806522539047 1 0 573765450 - 23868624 20190415 -- -- ... 355.949 -- 0 916384285 - 282391528 20190415 -- -- ... 0.47766300834538 0 0 574723760 - 123585000 20190415 -- -- ... 0.618316068787371 0 0 574511442 - 260216294 20190415 -- -- ... 0.187170498094167 1 0 683390717 - 406300991 20190415 -- -- ... 0.0518318978617112 0 0 1411465651 + >>> catalog_metadata = Catalogs.get_catalog_metadata('gaiadr3', 'dbo.gaia_source') + >>> catalog_metadata[:5].pprint(max_width=-1) + column_name datatype unit ucd description + ------------ -------- ---- ----------------- ---------------------------- + solution_id long meta.version catalog version + designation char meta.id;meta.main designation + source_id long meta.id source id + random_index long meta.code random index + ref_epoch double yr time.epoch reference epoch julian years -.. doctest-remote-data:: +Querying Catalogs +================== - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_criteria(catalog="Ctl", - ... object_name='M101', - ... radius=1, - ... Tmag=[10.75,11]) - >>> print(catalog_data) - ID version HIP TYC ... raddflag wdflag objID - --------- -------- --- ------------ ... -------- ------ --------- - 233458861 20190415 -- 3852-01407-1 ... 1 0 150390757 - 441662028 20190415 -- 3855-00941-1 ... 1 0 150395533 - 441658008 20190415 -- 3852-00116-1 ... 1 0 150246361 - 441639577 20190415 -- 3852-00429-1 ... 1 0 150070672 - 441658179 20190415 -- 3855-00816-1 ... 1 0 150246482 - 154258521 20190415 -- 3852-01403-1 ... 1 0 150281963 - 441659970 20190415 -- 3852-00505-1 ... 1 0 150296707 - 441660006 20190415 -- 3852-00341-1 ... 1 0 150296738 +The Catalogs interface provides three closely related query methods. All three methods return results from a MAST +catalog as an `~astropy.table.Table` (or a scalar count, if requested), and all three support column-based filtering, +sorting, and result limiting. The primary difference between them is how positional constraints are specified. + +At a high level: + - `~astroquery.mast.CatalogsClass.query_criteria` is the most flexible method. It supports purely column-based queries, + purely positional queries, or a combination of both. + + - `~astroquery.mast.CatalogsClass.query_region` is a convenience wrapper for positional queries using coordinates or an explicit region. + + - `~astroquery.mast.CatalogsClass.query_object` is a convenience wrapper for cone searches centered on a resolved object name. + +All three methods ultimately construct and execute an ADQL query against the MAST TAP service. +Shared Query Parameters +------------------------ + +The following parameters are supported by all three query methods: + - ``collection`` : The catalog collection to query. If not specified, the value of the instance's ``collection`` attribute is used. + + - ``catalog`` : The catalog within the collection to query. If not specified, the value of the instance's ``catalog`` attribute is used, or if ``collection`` is specified, the default catalog for that collection. + + - ``limit`` : Maximum number of rows to return (default: 5000). + + - ``offset`` : Number of rows to skip before returning results (default: 0). + + - ``count_only`` : If True, return only the number of matching records instead of the records themselves. + + - ``select_cols`` : A list of column names to include in the result. If omitted, all columns are returned. + + - ``sort_by``: One or more column names to sort by. + + - ``sort_desc`` : Whether to sort in descending order (either a single boolean applied to all ``sort_by`` columns or one per column). + +These parameters allow users to control the scope and format of their queries consistently across all three methods. + +Writing Queries +---------------- + +The `~astroquery.mast.CatalogsClass.query_criteria` method supports both positional parameters and column-based filters. +Positional constraints are optional. + +Supported positional parameters include: + - ``coordinates`` : Sky coordinates around which to perform a cone search. + - ``objectname`` : Name of the object around which to perform a cone search. + - ``resolver`` : Resolver service to use for object name resolution. + - ``radius`` : Radius of the cone search around the specified coordinates or object name. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. + - ``region`` : Explicit region specification for the search. See :ref:`specifying-spatial-regions` below for more details. + +If no positional parameters are supplied, the query is purely criteria-based. If positional parameters are supplied, +they are combined with any column-based criteria using logical **AND**. .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_criteria(catalog="DiskDetective", - ... object_name="M10", - ... radius=2, - ... state="complete") - >>> print(catalog_data) # doctest: +IGNORE_OUTPUT - designation ... ZooniverseURL - ------------------- ... ---------------------------------------------------- - J165628.40-054630.8 ... https://talk.diskdetective.org/#/subjects/AWI0005cka - J165748.96-054915.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckd - J165427.11-022700.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ck5 - J165749.79-040315.1 ... https://talk.diskdetective.org/#/subjects/AWI0005cke - J165327.01-042546.2 ... https://talk.diskdetective.org/#/subjects/AWI0005ck3 - J165949.90-054300.7 ... https://talk.diskdetective.org/#/subjects/AWI0005ckk - J170314.11-035210.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckv - - -The `~astroquery.mast.CatalogsClass.query_criteria` function requires at least one non-positional parameter. -These parameters are the column names listed in the `field descriptions `__ -of the catalog being queried. They do not include object_name, coordinates, or radius. Running a query with only positional -parameters will result in an error. + >>> result = Catalogs.query_criteria(collection='hsc', + ... coordinates="322.49324 12.16683", + ... radius='2 arcsec', + ... sort_by=['numimages', 'starttime'], + ... sort_desc=[False, True], + ... limit=5, + ... select_cols=['matchid', 'matchra', 'matchdec', 'numimages', 'starttime']) + >>> result.pprint(max_width=-1) + matchid matchra matchdec numimages starttime + deg deg + --------- ------------------ ------------------ --------- -------------------------- + 100906349 322.4932839715549 12.166957658789572 1 2013-09-01 14:42:57.487000 + 37053748 322.49333237602906 12.167170257824768 1 2013-09-01 14:09:53.487000 + 61895629 322.493715383149 12.166629788750484 1 2011-10-22 08:10:21.217000 + 19779150 322.49277386636714 12.166728768957904 2 2013-09-01 14:09:53.487000 + 11562863 322.49294957070185 12.166668540816076 2 2006-05-02 01:13:43.920000 + +To filter results based on column values, users may specify criteria as keyword arguments, +where the keyword is the column name and the value is the desired filter. Multiple criteria are combined +using logical **AND**. + +Criteria syntax supports a variety of operations, including: + +- Exact matches by specifying a value for a column. + +- To filter by multiple values for a single column, use a list of values. This performs an **OR** operation between the values. + +- A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results. + When a negated value is present in a list of filters, any positive values in that set are combined with **OR** logic, and any negated + values are combined with **AND** logic against the positives. + +- For columns with a numeric data type, filter using comparison values (``<``, ``>``, ``<=``, ``>=``). + + - ``<``: Return values less than or before the given number/date + + - ``>``: Return values greater than or after the given number/date + + - ``<=``: Return values less than or equal to the given number/date + + - ``>=``: Return values greater than or equal to the given number/date + +- For columns with a numeric data type, select an inclusive range with the syntax ``'#..#'``. + +- Wildcards are special characters used in search patterns to represent one or more unknown characters, + allowing for flexible matching of strings. The wildcard characters are ``*`` and ``%`` and they replace any number + of characters preceding, following, or in between existing characters, depending on their placement. .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_criteria(catalog="Tic", - ... object_name='M101', radius=1) - Traceback (most recent call last): - ... - astroquery.exceptions.InvalidQueryError: At least one non-positional criterion must be supplied. + >>> result = Catalogs.query_criteria(collection='ullyses', + ... catalog='sciencemetadata', + ... target_name_ullyses='NGC*', + ... target_classification='!Galaxy', + ... known_binary=False, + ... sp_class=['O', 'B'], + ... gaia_parallax=['<-0.1', '>=0'], + ... star_teff='30000..50000', + ... select_cols=['target_name_ullyses', 'target_classification', 'known_binary', 'sp_class', 'gaia_parallax', 'star_teff']) + >>> result.pprint(max_width=-1) + target_name_ullyses target_classification known_binary sp_class gaia_parallax star_teff + mas K + ------------------- --------------------- ------------ -------- ------------- --------- + NGC346 ELS 043 Early B Dwarf False B -0.111579 33000.0 + NGC346 MPG 487 Late O Dwarf False O -0.389724 35800.0 + NGC 3109 EBU 20 Mid O Supergiant False O 0.876327 31150.0 + NGC 3109 EBU 34 Mid O Supergiant False O -0.126069 33050.0 -The PanSTARRS catalog also accepts additional parameters to allow for query refinement. These options include column selection, -sorting, column criteria, page size and page number. Additional information on PanSTARRS queries may be found -`here `__. +The `~astroquery.mast.CatalogsClass.query_region` and `~astroquery.mast.CatalogsClass.query_object` methods are +convenience wrappers around `~astroquery.mast.CatalogsClass.query_criteria`: -Columns returned from the query may be submitted with the columns parameter as a list of column names. + - `~astroquery.mast.CatalogsClass.query_region` requires a positional constraint (``coordinates`` or ``region``). -The query may be sorted with the sort_by parameter composed of either a single column name (to sort ascending), -or a list of multiple column names and/or tuples of direction and column name (ASC/DESC, column name). + - `~astroquery.mast.CatalogsClass.query_object` requires an ``objectname`` and performs a cone search. -To filter the query, criteria per column name are accepted. The 'AND' operation is performed between all -column name criteria, and the 'OR' operation is performed within column name criteria. Per each column name -parameter, criteria may consist of either a value or a list. The list may consist of a mix of values and -tuples of criteria decorator (min, gte, gt, max, lte, lt, like, contains) and value. +Both methods also accept column-based criteria, which are applied in the same way as in `~astroquery.mast.CatalogsClass.query_criteria`. .. doctest-remote-data:: - >>> catalog_data = Catalogs.query_criteria(coordinates="5.97754 32.53617", - ... radius=0.01, - ... catalog="PANSTARRS", - ... table="mean", - ... data_release="dr2", - ... nStackDetections=[("gte", 2)], - ... columns=["objName", "objID", "nStackDetections", "distance"], - ... sort_by=[("desc", "distance")], - ... pagesize=15) - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - objName objID nStackDetections distance - --------------------- ------------------ ---------------- --------------------- - PSO J005.9812+32.5270 147030059812483022 5 0.009651200148871086 - PSO J005.9726+32.5278 147030059727583992 2 0.0093857181370567 - PSO J005.9787+32.5453 147050059787164914 4 0.009179045509852305 - PSO J005.9722+32.5418 147050059721440704 4 0.007171813230776031 - PSO J005.9857+32.5377 147040059855825725 4 0.007058815429178634 - PSO J005.9810+32.5424 147050059809651427 2 0.006835678269917365 - PSO J005.9697+32.5368 147040059697224794 2 0.006654002479439699 - PSO J005.9712+32.5330 147040059711340087 4 0.006212461367287632 - PSO J005.9747+32.5413 147050059747400181 5 0.0056515210592035965 - PSO J005.9775+32.5314 147030059774678271 3 0.004739286624336443 - - -Hubble Source Catalog (HSC) specific queries -============================================ - -Given an HSC Match ID, return all catalog results. + >>> result = Catalogs.query_region(collection='skymapperdr4', + ... coordinates="158.47924 -7.30962", + ... radius='1 arcmin', + ... select_cols=['object_id', 'raj2000', 'dej2000']) + >>> result.pprint(max_width=-1) + object_id raj2000 dej2000 + deg deg + ---------- ---------- --------- + 1116262239 158.475216 -7.29985 + 92307496 158.483015 -7.323183 + 92307428 158.46783 -7.319955 + 2151499732 158.466411 -7.319867 .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_object("M10", - ... radius=.001, - ... catalog="HSC", - ... magtype=1) - >>> matchid = catalog_data[0]["MatchID"] - >>> print(matchid) - 7542452 - >>> matches = Catalogs.query_hsc_matchid(matchid) - >>> print(matches) - CatID MatchID ... cd_matrix - --------- ------- ... ------------------------------------------------------ - 419094794 7542452 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 - 419094795 7542452 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 - 401289578 7542452 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 - 401289577 7542452 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 - 257194049 7542452 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 - 257438887 7542452 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 - - -HSC spectra accessed through this class as well. `~astroquery.mast.CatalogsClass.get_hsc_spectra` -does not take any arguments, and simply loads all HSC spectra. + >>> result = Catalogs.query_object(collection='skymapperdr4', + ... objectname='M11', + ... radius=0.01, + ... catwise_id='2825m061*', + ... sort_by='mean_epoch', + ... select_cols=['object_id', 'raj2000', 'dej2000', 'catwise_id', 'mean_epoch']) + >>> result[:5].pprint(max_width=-1) + object_id raj2000 dej2000 catwise_id mean_epoch + deg deg d + ---------- ---------- --------- ------------------ ---------- + 277772662 282.75865 -6.273223 2825m061_b0-113841 56833.6242 + 277772668 282.762729 -6.274544 2825m061_b0-010099 56834.1188 + 2389914703 282.768791 -6.275556 2825m061_b0-045485 56834.6156 + 277772658 282.758508 -6.276218 2825m061_b0-049457 56834.6167 + 277772511 282.764593 -6.27897 2825m061_b0-046693 56895.2497 + +.. _specifying-spatial-regions: + +Specifying Spatial Regions +-------------------------- + +Catalogs that support spatial queries allow regions to be specified in several ways. + +Cone Searches +^^^^^^^^^^^^^ + +Cone searches are the most common type of spatial query, defined by a center position and a radius. They may be specifed using: + - ``coordinates`` and ``radius`` + - ``objectname`` and ``radius`` + - A `~astropy.regions.CircleSkyRegion` object as the ``region`` parameter + - A Space-Time Coordinate (STC) CIRCLE region string as the ``region`` parameter + +An STC-S CIRCLE region string has the following format: + +``CIRCLE [frame] `` + +For example: + +``CIRCLE ICRS 210.8 54.35 0.05`` + +This means: + - ICRS: Coordinate frame (optional; defaults to ICRS if omitted) + - 210.8: Right Ascension in degrees + - 54.35: Declination in degrees + - 0.05: Radius in degrees .. doctest-remote-data:: + >>> result = Catalogs.query_region(collection='ps1_dr2', + ... region='CIRCLE ICRS 18.895 -6.944 0.01', + ... sort_by='objID', + ... select_cols=['objName', 'objID', 'raMean', 'decMean']) + >>> result[:5].pprint(max_width=-1) + objName objID raMean decMean + deg deg + ----------------------- ----------------- ----------------- ------------------ + PSX J011534.22-065706.1 99650188926188340 18.89261844 -6.95171552 + PSX J011534.90-065704.8 99650188954378784 18.89543744 -6.95134445 + PSX J011533.84-065651.9 99660188909803092 18.89100927680362 -6.947762708324004 + PSX J011533.97-065643.6 99660188915395866 18.8915527 -6.9454449 + PSX J011534.30-065636.0 99660188931668392 18.89294147 -6.94333417 - >>> from astroquery.mast import Catalogs - ... - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> print(all_spectra[:10]) - ObjID DatasetName MatchID ... PropID HSCMatch - ----- -------------------------------------------- -------- ... ------ -------- - 20010 HAG_J072655.67+691648.9_J8HPAXAEQ_V01.SPEC1D 19657846 ... 9482 Y - 20011 HAG_J072655.69+691648.9_J8HPAOZMQ_V01.SPEC1D 19657846 ... 9482 Y - 20012 HAG_J072655.76+691729.7_J8HPAOZMQ_V01.SPEC1D 19659745 ... 9482 Y - 20013 HAG_J072655.82+691620.0_J8HPAOZMQ_V01.SPEC1D 19659417 ... 9482 Y - 20014 HAG_J072656.34+691704.7_J8HPAXAEQ_V01.SPEC1D 19660230 ... 9482 Y - 20015 HAG_J072656.36+691704.7_J8HPAOZMQ_V01.SPEC1D 19660230 ... 9482 Y - 20016 HAG_J072656.36+691744.9_J8HPAOZMQ_V01.SPEC1D 19658847 ... 9482 Y - 20017 HAG_J072656.37+691630.2_J8HPAXAEQ_V01.SPEC1D 19660827 ... 9482 Y - 20018 HAG_J072656.39+691630.2_J8HPAOZMQ_V01.SPEC1D 19660827 ... 9482 Y - 20019 HAG_J072656.41+691734.9_J8HPAOZMQ_V01.SPEC1D 19656620 ... 9482 Y - - -Individual or ranges of spectra can be downloaded using the -`~astroquery.mast.CatalogsClass.download_hsc_spectra` function. +Polygon Searches +^^^^^^^^^^^^^^^^^ + +Polygon searches allow for more complex spatial queries by defining a polygonal region on the sky. +They may be specified using any of the following as the ``region`` parameter: + + - An iterable of coordinate pairs + - A `~astropy.regions.PolygonSkyRegion` object + - A Space-Time Coordinate (STC) POLYGON region string + +An STC-S POLYGON string has the form: + +``POLYGON [frame] ...`` + +For example: + +``POLYGON ICRS 210.7 54.3 210.9 54.3 210.9 54.4 210.7 54.4`` + +This defines a four-vertex polygon with vertices given as (RA, Dec) pairs in degrees. At least three vertices (six numbers) are required. .. doctest-remote-data:: + >>> result = Catalogs.query_criteria(collection='caom', + ... region='POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94', + ... limit=5, + ... select_cols=['target_name', 'obs_id', 's_ra', 's_dec']) + >>> result.pprint(max_width=-1) + target_name obs_id s_ra s_dec + deg deg + ----------- ------------------------------------------------------------------------- ----------------- ------------------ + 408084461 hlsp_tglc_tess_ffi_gaiaid-2475555794352572672-s0003-cam1-ccd3_tess_v1_llc 18.85365901920825 -6.941643227687541 + 408084461 hlsp_t16_tess_ffi_s0003-cam1-ccd3-02475555794352572672_tess_v01 18.8536465402 -6.9416358523 + 408084461 hlsp_tasoc_tess_ffi_tic00408084461-s0003-cam1-ccd3-c1800_tess_v05 18.8532579321195 -6.94140657784404 + 408084461 hlsp_gsfc-eleanor-lite_tess_ffi_s0003-0000000408084461_tess_v1.0_lc 18.8532579321195 -6.941406577844042 + 408084461 hlsp_qlp_tess_ffi_s0097-0000000408084461_tess_v01_llc 18.8532579321 -6.94140657784 + + +Counting Results +----------------- + +All query methods support a ``count_only=True`` option, which returns only the number of matching records: - >>> from astroquery.mast import Catalogs - ... - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> manifest = Catalogs.download_hsc_spectra(all_spectra[100:104]) # doctest: +IGNORE_OUTPUT - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - ... - >>> print(manifest) # doctest: +IGNORE_OUTPUT - Local Path ... URL - -------------------------------------------------------------------- ... ---- - ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... None \ No newline at end of file +.. doctest-remote-data:: + + >>> count = Catalogs.query_criteria(collection='skymapperdr4', + ... region=[(20, -5), (20, -6), (21, -6), (21, -5)], + ... count_only=True) + >>> print('Number of matching records:', count) + Number of matching records: 5479 + +This is useful for estimating result sizes before executing large queries. + +Deprecated Interfaces +====================== + +Several legacy methods related to the Hubble Source Catalog (HSC) remain available but are deprecated and will be removed +in a future release. These methods include: + +- `~astroquery.mast.CatalogsClass.query_hsc_matchid` +- `~astroquery.mast.CatalogsClass.get_hsc_spectra` +- `~astroquery.mast.CatalogsClass.download_hsc_spectra` + +New workflows should use the general `~astroquery.mast.CatalogsClass` interface described above. From 7c9622ff37798a502d13847d79863e33130d8207 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Sun, 11 Jan 2026 21:28:01 -0500 Subject: [PATCH 06/21] Better handling for spatial query support --- astroquery/mast/catalog_collection.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index d40a9dbbd6..a9a5edcf01 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -119,15 +119,11 @@ def get_catalog_metadata(self, catalog): ra_col, dec_col = self._get_ra_dec_column_names(metadata) # Determine if spatial queries are supported - supports_spatial_queries = (ra_col is not None and dec_col is not None) - if supports_spatial_queries: - # If an ra and dec column exist, test spatial query support - spatial_query = (f'SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT(\'ICRS\', {ra_col}, {dec_col}), ' - 'CIRCLE(\'ICRS\', 0, 0, 0.1)) = 1') - try: - self.tap_service.search(spatial_query) - except DALQueryError: - supports_spatial_queries = False + supports_adql_geometry = all( + func in self.supported_adql_functions + for func in ('POINT', 'CIRCLE', 'CONTAINS') + ) + supports_spatial_queries = (supports_adql_geometry and ra_col is not None and dec_col is not None) meta = CatalogMetadata( column_metadata=metadata, @@ -177,7 +173,10 @@ def run_tap_query(self, adql): The result of the TAP query as an Astropy Table. """ log.debug(f"Running TAP query on collection '{self.name}': {adql}") - result = self.tap_service.search(adql) + try: + result = self.tap_service.search(adql) + except DALQueryError as e: + raise InvalidQueryError(f"TAP query failed for collection '{self.name}': {e}") return result.to_table() def _fetch_catalogs(self): From df5178cfe63b7101ba3ed0d031b3299a924f337a Mon Sep 17 00:00:00 2001 From: Celia Parts Date: Tue, 23 Dec 2025 13:41:38 -0500 Subject: [PATCH 07/21] initial catalog colleciton remote tests and bugfix addressing comments from Sam uncommenting out line adjusting to new naming convention addressing more comments Starting off non-remote testing More test examples Style fixes to test_mast_remote Notes for non-remote tests --- astroquery/mast/catalog_collection.py | 2 +- .../mast/tests/data/tap_capabilities.xml | 1 + astroquery/mast/tests/data/tap_catalogs.vot | 49 + .../mast/tests/data/tap_collections.json | 1955 +++++++++++++++++ astroquery/mast/tests/data/tap_columns.vot | 915 ++++++++ astroquery/mast/tests/data/tap_results.vot | 895 ++++++++ astroquery/mast/tests/test_mast.py | 80 +- astroquery/mast/tests/test_mast_remote.py | 84 + 8 files changed, 3972 insertions(+), 9 deletions(-) create mode 100644 astroquery/mast/tests/data/tap_capabilities.xml create mode 100644 astroquery/mast/tests/data/tap_catalogs.vot create mode 100644 astroquery/mast/tests/data/tap_collections.json create mode 100644 astroquery/mast/tests/data/tap_columns.vot create mode 100644 astroquery/mast/tests/data/tap_results.vot diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index a9a5edcf01..ab711860e1 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -174,7 +174,7 @@ def run_tap_query(self, adql): """ log.debug(f"Running TAP query on collection '{self.name}': {adql}") try: - result = self.tap_service.search(adql) + result = self.tap_service.run_sync(adql) except DALQueryError as e: raise InvalidQueryError(f"TAP query failed for collection '{self.name}': {e}") return result.to_table() diff --git a/astroquery/mast/tests/data/tap_capabilities.xml b/astroquery/mast/tests/data/tap_capabilities.xml new file mode 100644 index 0000000000..60911ad49c --- /dev/null +++ b/astroquery/mast/tests/data/tap_capabilities.xml @@ -0,0 +1 @@ +https://masttest.stsci.edu/vo-tap/api/v0.1/ticADQL2.0ADQL-2.0
CONTAINS
POINT
CIRCLE
application/jsonjsontext/csv;header=presentcsvapplication/xmlxmlapplication/x-votable+xmlvotable100000100000https://masttest.stsci.edu/vo-tap/api/v0.1/tic/capabilitieshttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/availabilityhttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/tableshttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/examples \ No newline at end of file diff --git a/astroquery/mast/tests/data/tap_catalogs.vot b/astroquery/mast/tests/data/tap_catalogs.vot new file mode 100644 index 0000000000..21dbeffe59 --- /dev/null +++ b/astroquery/mast/tests/data/tap_catalogs.vot @@ -0,0 +1,49 @@ + + + + + + + + + + Fully qualified table name + + + + + Brief description of the table + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
tap_schema.schemasdescription of schemas in this dataset
tap_schema.tablesdescription of tables in this dataset
tap_schema.columnsdescription of columns in this dataset
tap_schema.keysdescription of foreign keys in this dataset
tap_schema.key_columnsdescription of foreign key columns in this dataset
dbo.catalogrecordMain Catalog Record
+ + diff --git a/astroquery/mast/tests/data/tap_collections.json b/astroquery/mast/tests/data/tap_collections.json new file mode 100644 index 0000000000..c1a95bf879 --- /dev/null +++ b/astroquery/mast/tests/data/tap_collections.json @@ -0,0 +1,1955 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "tap_search", + "version": "0.1" + }, + "servers": [ + { + "url": "/vo-tap/api/v0.1" + } + ], + "paths": { + "/{catalog}/": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Get Site Page", + "description": "Returns the service query manager doc page", + "operationId": "TAPHandler_get_site_page__catalog___get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/sync": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Get", + "description": "VO TAP Search", + "operationId": "TAPHandler_get__catalog__sync_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "catalog to search" + }, + "description": "catalog to search" + }, + { + "name": "lang", + "in": "query", + "required": true, + "schema": { + "$ref": "#/components/schemas/LangName", + "description": "Query language to be used for this request" + }, + "description": "Query language to be used for this request" + }, + { + "name": "responseformat", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "description": "Content type of the response" + }, + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "description": "Content type of the response" + }, + { + "name": "query", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Query to perform", + "title": "Query" + }, + "description": "Query to perform" + }, + { + "name": "maxrec", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000, + "title": "Maxrec" + }, + "description": "Maximum number of records to be returned (max 100k)" + }, + { + "name": "runid", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "description": "RUNID for the request", + "title": "Runid" + }, + "description": "RUNID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Post", + "description": "VO TAP Search (FORM)", + "operationId": "TAPHandler_post__catalog__sync_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "catalog to search" + }, + "description": "catalog to search" + } + ], + "requestBody": { + "required": true, + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPHandler_post__catalog__sync_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/examples": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Exampleshandler.Get", + "description": "Page displaying example TAP queries.\n\nThese example pages conform to the DALI-examples spec and are used to provide both human and machine-readable\nexample queries to the user.\n\nhttps://www.ivoa.net/documents/DALI/20170517/REC-DALI-1.1.html#tth_sEc2.3\n\nDuring automated testing, the queries on this page are stripped from the XHTML and used as test queries.\nThe XHTML element tags of the queries are specific to this testing/spec and should not be changed without\nconsulting the above specification, then also changing the necessary tests.", + "operationId": "ExamplesHandler_get__catalog__examples_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/tables": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vositableshandler.Get All Tables", + "description": "Return information on all available tables.", + "operationId": "VOSITablesHandler_get_all_tables__catalog__tables_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI table info for." + }, + "description": "Catalog to view VOSI table info for." + }, + { + "name": "detail", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Amount of table detail to return.", + "default": "min", + "title": "Detail" + }, + "description": "Amount of table detail to return." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/tables/{table}": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vositableshandler.Get Table", + "description": "Return information for a specific table.", + "operationId": "VOSITablesHandler_get_table__catalog__tables__table__get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI table info for." + }, + "description": "Catalog to view VOSI table info for." + }, + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "The specific table to view info for.", + "title": "Table" + }, + "description": "The specific table to view info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/availability": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vosiavailabilityhandler.Get Availability", + "description": "Availability metadata defined in the VOSI 1.1 spec at:\nhttps://www.ivoa.net/documents/VOSI/20170524/REC-VOSI-1.1.html#tth_sEc3.2", + "operationId": "VOSIAvailabilityHandler_get_availability__catalog__availability_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI availability info for." + }, + "description": "Catalog to view VOSI availability info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/capabilities": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vosicapabilitieshandler.Get Capabilities", + "description": "TAP specific capability metadata as described in the service VOResource", + "operationId": "VOSICapabilitiesHandler_get_capabilities__catalog__capabilities_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "Catalog to view VOSI capability info for.", + "title": "Catalog" + }, + "description": "Catalog to view VOSI capability info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Async", + "description": "Returns Jobs list per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_async__catalog__async_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "phase", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExecutionPhase" + } + }, + { + "type": "null" + } + ], + "description": "The current execution phase to filter jobs by.", + "title": "Phase" + }, + "description": "The current execution phase to filter jobs by." + }, + { + "name": "after", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "description": "Return jobs with creation times after the given date.", + "title": "After" + }, + "description": "Return jobs with creation times after the given date." + }, + { + "name": "last", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ], + "description": "Return the given number of most recently created jobs.", + "title": "Last" + }, + "description": "Return the given number of most recently created jobs." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post", + "description": "VO TAP Search, asynchronous job setup via form", + "operationId": "TAPAsyncHandler_post__catalog__async_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to query." + }, + "description": "Catalog to query." + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post__catalog__async_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/results/result": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Results", + "description": "Returns job results if existent, per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_results__catalog__async__job_id__results_result_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/results": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Results", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_results__catalog__async__job_id__results_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/error": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Error", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding\nIn case of a valid job with no errors, an empty 200 OK response is returned.\n\nFor a TAP service, the response should match the requested format, if specified.\nhttps://www.ivoa.net/documents/TAP/20190927/REC-TAP-1.1.html#tth_sEc3.3", + "operationId": "TAPAsyncHandler_get_job_info_detail_error__catalog__async__job_id__error_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/phase": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Phase", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_phase__catalog__async__job_id__phase_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Phase", + "description": "Job control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog associated with this job." + }, + "description": "Catalog associated with this job." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "required": true, + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/executionduration": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Executionduration", + "description": "Execution Duration is a potentially client-negotiated timeout value in seconds.\nIf it is not explicitly set by the client, a server default is used.\nReturn job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_executionduration__catalog__async__job_id__executionduration_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Execution Duration", + "description": "Updates job execution timeout, in seconds.\nOnly has an effect before the job is set to Phase RUN.\nNot supported on all catalogs. Limited by a server side max.\nJob control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog associated with this job." + }, + "description": "Catalog associated with this job." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/destruction": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Destruction", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_destruction__catalog__async__job_id__destruction_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Destruction", + "description": "Job control for existing jobs via POST per UWS spec:\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#DestructionTime\nThe service may forbid changes, or may set limits on the allowed destruction time.\nDestruction datetime is expected in ISO 8601 UTC format with Z for UTC times:\nhttps://www.ivoa.net/documents/VOResource/20180625/REC-VOResource-1.1.html#tth_sEc2.2.4", + "operationId": "TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/quote": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Quote", + "description": "Job completion time estimate quote not provided by this service, returns empty value.\nReturn job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_quote__catalog__async__job_id__quote_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/parameters": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Parameters", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_parameters__catalog__async__job_id__parameters_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Update Job Info Parameters", + "description": "Update job parameters by submitting a POST of key-value pairs.", + "operationId": "TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/owner": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Owner", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_owner__catalog__async__job_id__owner_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying." + }, + "description": "Catalog for querying." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info", + "description": "Returns specific Job info per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info__catalog__async__job_id__get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + }, + { + "name": "wait", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Maximum time in seconds to block until a job status change", + "title": "Wait" + }, + "description": "Maximum time in seconds to block until a job status change" + }, + { + "name": "phase", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExecutionPhase" + }, + { + "type": "null" + } + ], + "description": "For blocking behavior, can supply the current execution phase to monitor for changes", + "title": "Phase" + }, + "description": "For blocking behavior, can supply the current execution phase to monitor for changes" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Info", + "description": "Job control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Delete Job Route", + "description": "Deletes specific Job info per UWS spec. Cancels if running. Note the /jobs path for TAP is just /async\nPer https://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#d1e1390:\nSending a HTTP DELETE to a Job resource destroys that job, with the meaning noted in the definition\nof the Job object, above. No other resource of the UWS may be directly deleted by the client.\nThe response to this request must have code 303 \u201cSee other\u201d and the Location header of the response\nmust point to the Job List at the /{jobs} URI.", + "operationId": "TAPAsyncHandler_delete_job_route__catalog__async__job_id__delete", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Body_TAPAsyncHandler_post__catalog__async_post": { + "properties": { + "lang": { + "anyOf": [ + { + "$ref": "#/components/schemas/LangName" + }, + { + "type": "string" + } + ], + "title": "Lang", + "description": "Query language to be used for this request", + "default": "ADQL" + }, + "responseformat": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "param_format": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "query": { + "type": "string", + "title": "Query", + "description": "Query to perform", + "default": "" + }, + "maxrec": { + "type": "integer", + "minimum": 0.0, + "title": "Maxrec", + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000 + }, + "runid": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "RUNID for the request", + "default": "" + }, + "phase": { + "anyOf": [ + { + "type": "string", + "enum": [ + "RUN" + ], + "const": "RUN" + }, + { + "type": "null" + } + ], + "title": "Phase", + "description": "Execution Phase (set to RUN for autorun)" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post__catalog__async_post" + }, + "Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post": { + "properties": { + "destruction": { + "type": "string", + "format": "date-time", + "title": "Destruction", + "description": "ISO 8601 UTC datetime for proposed job destruction" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post" + }, + "Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post": { + "properties": { + "execution_duration": { + "type": "integer", + "minimum": 0.0, + "title": "Execution Duration", + "description": "Requested execution duration in seconds", + "default": 600 + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post" + }, + "Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post": { + "properties": { + "phase": { + "anyOf": [ + { + "$ref": "#/components/schemas/PhaseAction" + }, + { + "type": "null" + } + ], + "description": "Execution Phase" + }, + "destruction": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "title": "Destruction", + "description": "ISO 8601 UTC datetime for proposed job destruction" + }, + "action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Action", + "description": "Action to apply to job: Only DELETE supported." + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post" + }, + "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post": { + "properties": { + "phase": { + "$ref": "#/components/schemas/PhaseAction", + "description": "Execution Phase" + } + }, + "type": "object", + "required": [ + "phase" + ], + "title": "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post" + }, + "Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post": { + "properties": { + "query": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Query", + "description": "Query to be executed" + }, + "lang": { + "anyOf": [ + { + "$ref": "#/components/schemas/LangName" + }, + { + "type": "null" + } + ], + "description": "Query langauge to be used." + }, + "responseformat": { + "anyOf": [ + { + "$ref": "#/components/schemas/EncodingEnum" + }, + { + "type": "null" + } + ], + "description": "Content type of the response" + }, + "param_format": { + "anyOf": [ + { + "$ref": "#/components/schemas/EncodingEnum" + }, + { + "type": "null" + } + ], + "description": "Content type of the response" + }, + "maxrec": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxrec", + "description": "Maximum number of records to return" + }, + "runid": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "Run ID to be used" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post" + }, + "Body_TAPHandler_post__catalog__sync_post": { + "properties": { + "lang": { + "$ref": "#/components/schemas/LangName", + "description": "Query language to be used for this request" + }, + "responseformat": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "param_format": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "query": { + "type": "string", + "title": "Query", + "description": "Query to perform" + }, + "maxrec": { + "type": "integer", + "minimum": 0.0, + "title": "Maxrec", + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000 + }, + "runid": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "RUNID for the request" + } + }, + "type": "object", + "required": [ + "lang", + "query" + ], + "title": "Body_TAPHandler_post__catalog__sync_post" + }, + "CatalogName": { + "type": "string", + "enum": [ + "caom", + "classy", + "gaiadr3", + "greenplum", + "hsc", + "hscv2", + "missionmast", + "ps1dr1", + "ps1dr2", + "ps1_dr2", + "registry", + "roman", + "romanspectra", + "skymapperdr4", + "tic", + "ullyses", + "goods", + "candels", + "3dhst", + "deepspace" + ], + "title": "CatalogName" + }, + "EncodingEnum": { + "type": "string", + "enum": [ + "json", + "csv", + "xml", + "votable" + ], + "title": "EncodingEnum", + "description": "Enumeration class for the format_param (format) query param" + }, + "ExecutionPhase": { + "type": "string", + "enum": [ + "PENDING", + "QUEUED", + "EXECUTING", + "COMPLETED", + "ERROR", + "UNKNOWN", + "HELD", + "SUSPENDED", + "ABORTED", + "ARCHIVED" + ], + "title": "ExecutionPhase", + "description": "Enumeration of possible phases of job execution." + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "LangName": { + "type": "string", + "enum": [ + "ADQL", + "ADQL-2.0" + ], + "title": "LangName", + "description": "Enum for language names as param" + }, + "PhaseAction": { + "type": "string", + "enum": [ + "ABORT", + "RUN" + ], + "title": "PhaseAction", + "description": "Enum for async job phase requests" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + } + } + } +} \ No newline at end of file diff --git a/astroquery/mast/tests/data/tap_columns.vot b/astroquery/mast/tests/data/tap_columns.vot new file mode 100644 index 0000000000..3a51181cdd --- /dev/null +++ b/astroquery/mast/tests/data/tap_columns.vot @@ -0,0 +1,915 @@ + + + + + + + + + + Column name + + + + + ADQL datatype + + + + + Unit in VO standard format + + + + + UCD of column if any + + + + + Brief description of column + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
idlong + meta.id;meta.maintess input catalog identifier
versionchar + meta.idcatalog version
hipint + meta.idhipparcos identifier
tycchar + meta.idtycho2 identifier
ucacchar + meta.iducac4 identifier
twomasschar + meta.id2mass identifier (hhmmssss+ddmmsss j2000)
sdsslong + meta.idsdss dr9 objid identifier
allwisechar + meta.idallwise identifier (jhhmmss.ss+ddmmss.s)
gaiachar + meta.idgaia dr2 identifier
apasschar + meta.idapass dr9 identifier
kicint + meta.idkic identifier
objtypechar + src.class.stargalaxyobject type (star or extended)
typesrcchar + meta.refthe source of the object in the tic
radoubledegpos.eq.ra;meta.mainright ascension (j2000)
decdoubledegpos.eq.dec;meta.maindeclination (j2000)
posflagchar + meta.ref;pos.framesource of the position
pmrafloatmas/yrpos.pm;pos.eq.raproper motion in right ascension
e_pmrafloatmas/yrstat.error;pos.pm;pos.eq.rauncertainty in pmra
pmdecfloatmas/yrpos.pm;pos.eq.decproper motion in declination
e_pmdecfloatmas/yrstat.error;pos.pm;pos.eq.decuncertainty in pmde
pmflagchar + meta.ref;pos.framesource of the proper motion
plxfloatmaspos.parallaxparallax
e_plxfloatmasstat.error;pos.parallaxerror in the parallax
parflagchar + meta.ref;pos.framesource of the parallax
gallongdoubledegpos.galactic.longalactic longitude
gallatdoubledegpos.galactic.latgalactic latitude
eclongdoubledegpos.ecliptic.lonecliptic longitude
eclatdoubledegpos.ecliptic.latecliptic latitude
bmagfloatmagphot.mag;em.opt.bjohnson b magnitude
e_bmagfloatmagstat.error;phot.maguncertainty in bmag
vmagfloatmagphot.mag;em.opt.vjohnson v magnitude
e_vmagfloatmagstat.error;phot.maguncertainty in vmag
umagfloatmagphot.mag;em.opt.usdss u-band (ab) magnitude
e_umagfloatmagstat.error;phot.maguncertainty in umag
gmagfloatmagphot.mag;em.opt.bsdss g-band (ab) magnitude
e_gmagfloatmagstat.error;phot.maguncertainty in gmag
rmagfloatmagphot.mag;em.opt.rsdss r-band (ab) magnitude
e_rmagfloatmagstat.error;phot.maguncertainty in rmag
imagfloatmagphot.mag;em.opt.isdss i-band (ab) magnitude
e_imagfloatmagstat.error;phot.maguncertainty in imag
zmagfloatmagphot.mag;em.opt.isdss z-band (ab) magnitude
e_zmagfloatmagstat.error;phot.maguncertainty in zmag
jmagfloatmagphot.mag;em.ir.j2mass johnson j-band magnitude
e_jmagfloatmagstat.error;phot.maguncertainty in jmag
hmagfloatmagphot.mag;em.ir.h2mass johnson h-band magnitude
e_hmagfloatmagstat.error;phot.maguncertainty in hmag
kmagfloatmagphot.mag;em.ir.k2mass johnson k-band magnitude
e_kmagfloatmagstat.error;phot.maguncertainty in kmag
twomflagchar + meta.code.qualquality flags for 2mass
proxfloatarcsecpos.angdistanceobject proximity
w1magfloatmagphot.mag;em.ir.3-4umallwise w1 (3.4um) magnitude
e_w1magfloatmagstat.error;phot.maguncertainty in w1mag
w2magfloatmagphot.mag;em.ir.4-8umallwise w2 (4.6um) magnitude
e_w2magfloatmagstat.error;phot.maguncertainty in w2mag
w3magfloatmagphot.mag;em.ir.8-15umallwise w3 (12um) magnitude
e_w3magfloatmagstat.error;phot.maguncertainty in w3mag
w4magfloatmagphot.mag;em.ir.15-30umallwise w4 (22um) magnitude
e_w4magfloatmagstat.error;phot.maguncertainty in w4mag
gaiamagfloatmagphot.mag;em.opt.vgaiadr2 g magnitude
e_gaiamagfloatmagstat.error;phot.maguncertainty in gmag
tmagfloatmagphot.mag;em.opttess magnitude
e_tmagfloatmagstat.error;phot.maguncertainty in tess mag
tessflagchar + meta.codetess magnitude flag
spflagchar + meta.codestellar properties flag
tefffloatKphys.temperature.effectiveeffective temperature
e_tefffloatKstat.error;phys.temperature.effectiveuncertainty in teff
loggfloatcm/s**2phys.gravitylog of the surface gravity
e_loggfloatcm/s**2stat.error;phys.gravityuncertainty in logg
mhfloat + phys.abund.zmetallicity [m/h]
e_mhfloat + stat.error;phys.abund.zuncertainty in [m/h]
radfloatsolRadphys.size.radiusradius
e_radfloatsolRadstat.error;phys.size.radiusuncertainty in radius
massfloatsolMassphys.massmass
e_massfloatsolMassstat.error;phys.massuncertainty in mass
rhofloat + phys.densitystellar density in solar units (mass/rad^3)
e_rhofloat + stat.error;phys.densityuncertainty in rho
lumclasschar + src.class.luminosityluminosity class
lumfloatsolLumphys.luminositystellar luminosity; rad^2*(teff/5772)^4
e_lumfloatsolLumstat.error;phys.luminosityuncertainty in luminosity
dfloatpcpos.distance;pos.heliocentricdistance
e_dfloatpcstat.error;pos.distanceuncertainty in distance
ebvfloatmagphot.color.excessapplied color excess
e_ebvfloatmagstat.error;phot.color.excessuncertainty in e(b-v)
numcontint + meta.numbernumber of contaminants found within 10arcsec of the star used in the calculation of the contamination ratio
contratiofloat + stat.fitcontamination ratio
dispositionchar + meta.code.classdisposition type
duplicate_idlong + meta.code.multiptic id of another object in duplicate or split set of stars
priorityfloat + meta.numberpriority (number from 0 to 1 = highest priority)
eneg_ebvfloatmagstat.error;phot.color.excess;stat.minnegative error for e(b-v)
epos_ebvfloatmagstat.error;phot.color.excess;stat.maxpositive error for e(b-v)
ebvflagchar + meta.ref;phot.color.excesssource of e(b-v)
eneg_massfloatsolMassstat.error;phys.mass;stat.minnegative error for mass
epos_massfloatsolMassstat.error;phys.mass;stat.maxpositive error for mass
eneg_radfloatsolRadstat.error;phys.size.radius;stat.minnegative error for radius
epos_radfloatsolRadstat.error;phys.size.radius;stat.maxpositive error for radius
eneg_rhofloat + stat.error;phys.density;stat.minnegative error for stellar density
epos_rhofloat + stat.error;phys.density;stat.maxpositive error for stellar density
eneg_loggfloatcm/s**2stat.error;phys.gravity;stat.minnegative error for surface gravity
epos_loggfloatcm/s**2stat.error;phys.gravity;stat.maxpositive error for surface gravity
eneg_lumfloatsolLumstat.error;phys.luminosity;stat.minnegative error for luminosity
epos_lumfloatsolLumstat.error;phys.luminosity;stat.maxpositive error for luminosity
eneg_distfloatpcstat.error;pos.distance;stat.minnegative error for distance
epos_distfloatpcstat.error;pos.distance;stat.maxpositive error for distance
distflagchar + meta.ref;pos.distancesource for distance
eneg_tefffloatKstat.error;phys.temperature.effective;stat.minnegative error for effective temperature
epos_tefffloatKstat.error;phys.temperature.effective;stat.maxpositive error for effective temperature
teffflagchar + meta.ref;phys.temperature.effectivesource for effective temperature
gaiabpfloatmagphot.mag;em.opt.bgaiadr2 bp magnitude
e_gaiabpfloatmagstat.error;phot.maguncertainty in bp magnitude
gaiarpfloatmagphot.mag;em.opt.rgaiadr2 rp magnitude
e_gaiarpfloatmagstat.error;phot.maguncertainty in rp magnitude
gaiaqflagint + meta.code.qualquality flags for gaia information
starchareflagchar + stat.errorasymmetric errors
vmagflagchar + meta.ref;phot.mag;em.opt.vsource of v magnitude
bmagflagchar + meta.ref;phot.mag;em.opt.bsource of b magnitude
splistschar + meta.codeidentifies if star is in a specially curated list
e_radoublemasstat.error;pos.eq.raerror in radeg
e_decdoublemasstat.error;pos.eq.decerror in decgeg
ra_origdoubledegpos.eq.rara from original catalog
dec_origdoubledegpos.eq.decdec from original catalog
e_ra_origdoublemasstat.error;pos.eq.raerror in ra_orig
e_dec_origdoublemasstat.error;pos.eq.decerror in dec_orig
raddflagint + meta.codedwarf by radius; 0: giant by radius; -1: insufficient information
wdflagint + meta.codestar in gaia photometric white dwarf region
objidlong + meta.recorddatabase internal identifier
+ + diff --git a/astroquery/mast/tests/data/tap_results.vot b/astroquery/mast/tests/data/tap_results.vot new file mode 100644 index 0000000000..9900f6c70f --- /dev/null +++ b/astroquery/mast/tests/data/tap_results.vot @@ -0,0 +1,895 @@ + + + + + + + + + + tess input catalog identifier + + + + + catalog version + + + + + hipparcos identifier + + + + + tycho2 identifier + + + + + ucac4 identifier + + + + + 2mass identifier (hhmmssss+ddmmsss j2000) + + + + + sdss dr9 objid identifier + + + + + allwise identifier (jhhmmss.ss+ddmmss.s) + + + + + gaia dr2 identifier + + + + + apass dr9 identifier + + + + + kic identifier + + + + + object type (star or extended) + + + + + the source of the object in the tic + + + + + right ascension (j2000) + + + + + declination (j2000) + + + + + source of the position + + + + + proper motion in right ascension + + + + + uncertainty in pmra + + + + + proper motion in declination + + + + + uncertainty in pmde + + + + + source of the proper motion + + + + + parallax + + + + + error in the parallax + + + + + source of the parallax + + + + + galactic longitude + + + + + galactic latitude + + + + + ecliptic longitude + + + + + ecliptic latitude + + + + + johnson b magnitude + + + + + uncertainty in bmag + + + + + johnson v magnitude + + + + + uncertainty in vmag + + + + + sdss u-band (ab) magnitude + + + + + uncertainty in umag + + + + + sdss g-band (ab) magnitude + + + + + uncertainty in gmag + + + + + sdss r-band (ab) magnitude + + + + + uncertainty in rmag + + + + + sdss i-band (ab) magnitude + + + + + uncertainty in imag + + + + + sdss z-band (ab) magnitude + + + + + uncertainty in zmag + + + + + 2mass johnson j-band magnitude + + + + + uncertainty in jmag + + + + + 2mass johnson h-band magnitude + + + + + uncertainty in hmag + + + + + 2mass johnson k-band magnitude + + + + + uncertainty in kmag + + + + + quality flags for 2mass + + + + + object proximity + + + + + allwise w1 (3.4um) magnitude + + + + + uncertainty in w1mag + + + + + allwise w2 (4.6um) magnitude + + + + + uncertainty in w2mag + + + + + allwise w3 (12um) magnitude + + + + + uncertainty in w3mag + + + + + allwise w4 (22um) magnitude + + + + + uncertainty in w4mag + + + + + gaiadr2 g magnitude + + + + + uncertainty in gmag + + + + + tess magnitude + + + + + uncertainty in tess mag + + + + + tess magnitude flag + + + + + stellar properties flag + + + + + effective temperature + + + + + uncertainty in teff + + + + + log of the surface gravity + + + + + uncertainty in logg + + + + + metallicity [m/h] + + + + + uncertainty in [m/h] + + + + + radius + + + + + uncertainty in radius + + + + + mass + + + + + uncertainty in mass + + + + + stellar density in solar units (mass/rad^3) + + + + + uncertainty in rho + + + + + luminosity class + + + + + stellar luminosity; rad^2*(teff/5772)^4 + + + + + uncertainty in luminosity + + + + + distance + + + + + uncertainty in distance + + + + + applied color excess + + + + + uncertainty in e(b-v) + + + + + number of contaminants found within 10arcsec of the star used in + the calculation of the contamination ratio + + + + + contamination ratio + + + + + disposition type + + + + + tic id of another object in duplicate or split set of stars + + + + + priority (number from 0 to 1 = highest priority) + + + + + negative error for e(b-v) + + + + + positive error for e(b-v) + + + + + source of e(b-v) + + + + + negative error for mass + + + + + positive error for mass + + + + + negative error for radius + + + + + positive error for radius + + + + + negative error for stellar density + + + + + positive error for stellar density + + + + + negative error for surface gravity + + + + + positive error for surface gravity + + + + + negative error for luminosity + + + + + positive error for luminosity + + + + + negative error for distance + + + + + positive error for distance + + + + + source for distance + + + + + negative error for effective temperature + + + + + positive error for effective temperature + + + + + source for effective temperature + + + + + gaiadr2 bp magnitude + + + + + uncertainty in bp magnitude + + + + + gaiadr2 rp magnitude + + + + + uncertainty in rp magnitude + + + + + quality flags for gaia information + + + + + asymmetric errors + + + + + source of v magnitude + + + + + source of b magnitude + + + + + identifies if star is in a specially curated list + + + + + error in radeg + + + + + error in decgeg + + + + + ra from original catalog + + + + + dec from original catalog + + + + + error in ra_orig + + + + + error in dec_orig + + + + + dwarf by radius; 0: giant by radius; -1: insufficient information + + + + + star in gaia photometric white dwarf region + + + + + database internal identifier + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
42770553920190415 + + 382-12368718475108-1342233 + J184751.04-134222.641051754443095892486823646 + STARgaia2281.9629072438-13.7065985389427gaia2-5.661380.0764805-1.146080.0688171gaia20.3980230.0440811gaia220.2937183862185-5.42345791126297281.7724783304729.2451314111868416.8560.18115.3230.149 + + + + + + + + + + 11.9420.04711.2480.06810.7520.035EEE-222-111-000-0-0 + 10.3340.02310.4480.02110.6720.0889.134 + 14.71930.00047613.67730.0106reredgaia24418123 + + + + + + + + + + + + + 2360.4257.6050.5037430.0162428 + + SPLIT + + 0.01257280.0199128panstarrs + + + + + + + + + + 230.46284.75bj2018 + + dered15.77360.00978613.67780.0077211 + ucac4bpbj + 1.194495727858711.06728505482206281.962882153899-13.70660347345510.03513726893589260.03637388286974680022205941
60699594820190415 + + + + + + 510528438770827136 + + STARgaia219.415533913500161.0286394857995gaia2-0.9694640.1876690.8473730.18527gaia20.3482330.134513gaia2126.103557179616-1.6815130899793647.338495783371947.6142191319478 + + 18.58020.0486 + + + + + + + + + + + + + + + + + + + + + + + + + + 18.16120.00123517.38710.0271reredgaia25722154 + + + + + + + + + + + + + 2527.82921.4750.5078120.03941955 + + DUPLICATE54375580 + 0.02864960.0501895panstarrs + + + + + + + + + + 665.591177.36bj2018 + + dered18.75960.02656117.25690.0143231 + gaia2 + + 3.378239895504092.8735447568218719.415525295995661.02864313421130.1081048093514390.1033669687738271021716702
+ + diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index ad817949d3..849d479d8c 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,25 +4,47 @@ import os import re import warnings +from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, patch -from pathlib import Path import astropy.units as u -import pytest import numpy as np -from astropy.table import Table, unique +import pytest from astropy.coordinates import SkyCoord from astropy.io import fits +from astropy.table import Table, unique from astropy.utils.exceptions import AstropyDeprecationWarning +from pyvo.dal import TAPResults +from pyvo.dal.exceptions import DALQueryError +from pyvo.io.vosi import parse_capabilities from requests import HTTPError, Response -from astroquery.mast import (Catalogs, MastMissions, Observations, Tesscut, Zcut, Mast, utils, services, - discovery_portal, auth, core, cloud) +from astroquery.exceptions import ( + BlankResponseWarning, + InputWarning, + InvalidQueryError, + MaxResultsWarning, + NoResultsWarning, + RemoteServiceError, + ResolverError, +) +from astroquery.mast import ( + Catalogs, + Mast, + MastMissions, + Observations, + Tesscut, + Zcut, + auth, + cloud, + core, + discovery_portal, + services, + utils, +) from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse -from astroquery.exceptions import (BlankResponseWarning, InvalidQueryError, InputWarning, MaxResultsWarning, - NoResultsWarning, RemoteServiceError, ResolverError) try: # Optional dependency import for cloud access functionality @@ -62,6 +84,11 @@ 'get_cloud_paths': 'mast_relative_path.json', 'panstarrs': 'panstarrs.json', 'panstarrs_columns': 'panstarrs_columns.json', + 'tap_collections': 'tap_collections.json', # Collections available + 'tap_catalogs': 'tap_catalogs.vot', # Catalogs for TIC + 'tap_columns': 'tap_columns.vot', # Column metadata + 'tap_capabilities': 'tap_capabilities.xml', # TAP service capabilities + 'tap_results': 'tap_results.vot', # Results of a TAP query 'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip', 'tess_sector': 'tess_sector.json', 'z_cutout_fit': 'astrocut_189.49206_62.20615_100x100px_f.zip', @@ -82,6 +109,10 @@ def patch_post(request): mp.setattr(discovery_portal.PortalAPI, '_request', post_mockreturn) mp.setattr(services.ServiceAPI, '_request', service_mockreturn) mp.setattr(auth.MastAuth, 'session_info', session_info_mockreturn) + mp.setattr( + 'astroquery.mast.catalog_collection.TAPService', + lambda *args, **kwargs: vo_tap_mock() + ) mp.setattr(Tesscut, '_download_file', tesscut_download_mockreturn) mp.setattr(Zcut, '_download_file', zcut_download_mockreturn) @@ -179,6 +210,8 @@ def request_mockreturn(url, params={}): filename = data_path(DATA_FILES['panstarrs_columns']) elif 'path_lookup' in url: filename = data_path(DATA_FILES['get_cloud_paths']) + elif 'vo-tap' in url: + filename = data_path(DATA_FILES['tap_collections']) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -237,12 +270,43 @@ def zcut_download_mockreturn(url, file_path): return +def vo_tap_mock(): + def run_sync_mock(query, **kwargs): + print(query) + if 'invalid' in query: + # Use this when wanting to simulate a DALQueryError + # Where it occurs will depend on where you pass it (collection, catalog, parameter, etc.) + raise DALQueryError('Simulated TAP query error for testing.') + elif 'tap_schema.tables' in query: + # Queries to get catalogs + filename = data_path(DATA_FILES['tap_catalogs']) + elif 'tap_schema.columns' in query: + # Queries to get column metadata + filename = data_path(DATA_FILES['tap_columns']) + elif 'WHERE' in query: + # Queries with results, keep in mind this is not meaninful and results won't match the query + filename = data_path(DATA_FILES['tap_results']) + votable = parse(filename) + return TAPResults(votable) + + # Mock TAPService + mock_tap = MagicMock() + mock_tap.run_sync.side_effect = run_sync_mock + + # Capabilities -> Not much to do here + filename = data_path(DATA_FILES['tap_capabilities']) + with open(filename, "rb") as f: + caps = parse_capabilities(f) + mock_tap.capabilities = caps + + return mock_tap + ########################### # MissionSearchClass Test # ########################### -def test_missions_query_region_async(): +def test_missions_query_region_async(patch_post): responses = MastMissions.query_region_async(regionCoords, radius=0.002, sci_pi_last_name='GORDON') assert isinstance(responses, MockResponse) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index a3c1d0405c..f350b10786 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -16,6 +16,7 @@ from astroquery.mast import (Catalogs, Hapcut, Mast, MastMissions, Observations, Tesscut, Zcut, utils) from ...exceptions import (InputWarning, InvalidQueryError, MaxResultsWarning, NoResultsWarning) +from ..catalog_collection import CatalogMetadata, CatalogCollection, DEFAULT_CATALOGS from ..utils import ResolverError @@ -1435,6 +1436,89 @@ def test_catalogs_download_hsc_spectra(self, tmpdir): assert isinstance(result, Table) assert os.path.isfile(result['Local Path'][0]) + ########################### + # CatalogCollection tests # + ########################### + + @pytest.mark.filterwarnings("ignore::pyvo.io.vosi.exceptions.W02") + @pytest.mark.parametrize("catalog", ["caom", "ullyses", "tic"]) + def test_catalog_collection_get_catalog_metadata(self, catalog): + cc = CatalogCollection(catalog) + default_catalog = cc.get_default_catalog() + default_metadata = cc.get_catalog_metadata(default_catalog) + assert isinstance(default_metadata, CatalogMetadata) + assert isinstance(default_metadata.column_metadata, Table) + assert isinstance(default_metadata.ra_column, str) + assert isinstance(default_metadata.dec_column, str) + assert isinstance(default_metadata.supports_spatial_queries, bool) + + assert len(default_metadata.column_metadata) > 1 + assert default_metadata.ra_column in default_metadata.column_metadata["column_name"] + assert default_metadata.dec_column in default_metadata.column_metadata["column_name"] + if catalog != "ullyses": + assert default_metadata.supports_spatial_queries + + metadata_cache = cc._catalog_metadata_cache + assert default_catalog.casefold() in metadata_cache + default_metadata_cached = cc.get_catalog_metadata(default_catalog) + assert default_metadata is default_metadata_cached + + def test_catalog_collection_invalid_get_catalog_metadata(self): + cc = CatalogCollection("TIC") + invalid_catalog = "invalid_catalog" + with pytest.raises(InvalidQueryError, match=f"Catalog '{invalid_catalog}' is not " + f"recognized for collection '{cc.name}'."): + cc.get_catalog_metadata(invalid_catalog) + + @pytest.mark.filterwarnings("ignore::pyvo.io.vosi.exceptions.W02") + @pytest.mark.parametrize("catalog", ["tic", "classy", "ullyses"]) + def test_catalog_collection_get_default_catalog(self, catalog): + cc = CatalogCollection(catalog) + catalogs = cc._fetch_catalogs() + default = cc.get_default_catalog() + + assert len(catalogs) > 1 + assert isinstance(catalogs, Table) + assert catalogs.colnames == ['catalog_name', 'description'] + + assert not default.startswith("tap_schema") + assert default.casefold() in [name.casefold() for name in catalogs["catalog_name"]] + assert DEFAULT_CATALOGS[catalog] == default + + def test_catalog_collection_run_tap_query(self): + cc = CatalogCollection("GAIADR3") + adql_str = "SELECT TOP 10 solution_id, designation, source_id, ra, dec FROM gaia_source WHERE " \ + "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + result = cc.run_tap_query(adql_str) + + assert isinstance(result, Table) + assert result.colnames == ['solution_id', 'designation', 'source_id', 'ra', 'dec'] + assert ((result["ra"] >= 10) & (result["ra"] <= 11)).all() + assert ((result["dec"] >= 12) & (result["dec"] <= 13)).all() + + def test_catalog_collection_verify_criteria(self): + cc = CatalogCollection("TIC") + default_catalog = cc.get_default_catalog() + + result = cc._verify_criteria(default_catalog) + assert result is None + + result = cc._verify_criteria(default_catalog, gaiabp=1) + assert result is None + + result = cc._verify_criteria(default_catalog, gaiabp=1, teff=1, e_gaiabp=1) + assert result is None + + close_match_col = "gaiagaiabp" + with pytest.raises(InvalidQueryError, match=f"Filter '{close_match_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?"): + cc._verify_criteria(default_catalog, gaiagaiabp=1) + + invalid_col = "fake_column" + with pytest.raises(InvalidQueryError, match=f"Filter '{invalid_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'."): + cc._verify_criteria(default_catalog, fake_column=1) + ###################### # TesscutClass tests # ###################### From bfbb68a78f2262807bffa6879ece6deb79d608a0 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 15 Jan 2026 15:18:27 -0500 Subject: [PATCH 08/21] Add back in spatial check, add filters parameter for conflicting columns --- astroquery/mast/catalog_collection.py | 10 ++ astroquery/mast/collections.py | 213 +++++++------------------- 2 files changed, 63 insertions(+), 160 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index ab711860e1..eb5683b481 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -123,7 +123,17 @@ def get_catalog_metadata(self, catalog): func in self.supported_adql_functions for func in ('POINT', 'CIRCLE', 'CONTAINS') ) + + # Try an inexpensive spatial query if RA/Dec columns are known supports_spatial_queries = (supports_adql_geometry and ra_col is not None and dec_col is not None) + if supports_spatial_queries: + # If an ra and dec column exist, test spatial query support + spatial_query = (f'SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT(\'ICRS\', {ra_col}, {dec_col}), ' + 'CIRCLE(\'ICRS\', 0, 0, 0.001)) = 1') + try: + self.tap_service.search(spatial_query) + except DALQueryError: + supports_spatial_queries = False meta = CatalogMetadata( column_metadata=metadata, diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index c32ee64d74..8f98509e20 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -186,50 +186,15 @@ def supports_spatial_queries(self, collection=None, catalog=None): return collection_obj.get_catalog_metadata(catalog).supports_spatial_queries @class_or_instance - @deprecated_renamed_argument( - "version", - None, - since="0.4.12", - message="The `version` argument is deprecated and " - "will be removed in a future release. Please use `collection` and `catalog` instead.", - ) - @deprecated_renamed_argument( - "pagesize", - None, - since="0.4.12", - message="The `pagesize` argument is deprecated " - "and will be removed in a future release. Please use `limit` instead.", - ) - @deprecated_renamed_argument( - "page", - None, - since="0.4.12", - message="The `page` argument is deprecated " - "and will be removed in a future release. Please use `offset` instead.", - ) - @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") - def query_criteria( - self, - collection=None, - *, - catalog=None, - coordinates=None, - region=None, - object_name=None, - radius=0.2 * u.deg, - resolver=None, - limit=5000, - offset=0, - count_only=False, - select_cols=None, - sort_by=None, - sort_desc=False, - filters=None, - version=None, - pagesize=None, - page=None, - **criteria, - ): + @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' + 'will be removed in a future release. Please use `collection` and `catalog` instead.') + @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' + 'and will be removed in a future release. Please use `limit` instead.') + @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' + 'and will be removed in a future release. Please use `offset` instead.') + def query_criteria(self, collection=None, *, catalog=None, coordinates=None, region=None, objectname=None, + radius=0.2*u.deg, resolver=None, limit=5000, offset=0, count_only=False, select_cols=None, + sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): """ Query a MAST catalog from a given collection using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -421,47 +386,15 @@ def query_criteria( return result_table @class_or_instance - @deprecated_renamed_argument( - "version", - None, - since="0.4.12", - message="The `version` argument is deprecated and " - "will be removed in a future release. Please use `collection` and `catalog` instead.", - ) - @deprecated_renamed_argument( - "pagesize", - None, - since="0.4.12", - message="The `pagesize` argument is deprecated " - "and will be removed in a future release. Please use `limit` instead.", - ) - @deprecated_renamed_argument( - "page", - None, - since="0.4.12", - message="The `page` argument is deprecated " - "and will be removed in a future release. Please use `offset` instead.", - ) - def query_region( - self, - coordinates=None, - *, - radius=0.2 * u.deg, - region=None, - collection=None, - catalog=None, - limit=5000, - offset=0, - count_only=False, - select_cols=None, - sort_by=None, - sort_desc=False, - filters=None, - version=None, - pagesize=None, - page=None, - **criteria, - ): + @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' + 'will be removed in a future release. Please use `collection` and `catalog` instead.') + @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' + 'and will be removed in a future release. Please use `limit` instead.') + @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' + 'and will be removed in a future release. Please use `offset` instead.') + def query_region(self, coordinates=None, *, radius=0.2*u.deg, region=None, collection=None, + catalog=None, limit=5000, offset=0, count_only=False, select_cols=None, + sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): """ Query for MAST catalog entries within a specified region using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -536,65 +469,30 @@ def query_region( # Parse pagination params limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) - return self.query_criteria( - collection=collection, - catalog=catalog, - coordinates=coordinates, - region=region, - radius=radius, - limit=limit, - offset=offset, - count_only=count_only, - select_cols=select_cols, - sort_by=sort_by, - sort_desc=sort_desc, - filters=filters, - **criteria, - ) + return self.query_criteria(collection=collection, + catalog=catalog, + coordinates=coordinates, + region=region, + radius=radius, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria) @class_or_instance - @deprecated_renamed_argument( - "version", - None, - since="0.4.12", - message="The `version` argument is deprecated and " - "will be removed in a future release. Please use `collection` and `catalog` instead.", - ) - @deprecated_renamed_argument( - "pagesize", - None, - since="0.4.12", - message="The `pagesize` argument is deprecated " - "and will be removed in a future release. Please use `limit` instead.", - ) - @deprecated_renamed_argument( - "page", - None, - since="0.4.12", - message="The `page` argument is deprecated " - "and will be removed in a future release. Please use `offset` instead.", - ) - @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") - def query_object( - self, - object_name, - *, - radius=0.2 * u.deg, - collection=None, - catalog=None, - resolver=None, - limit=5000, - offset=0, - count_only=False, - select_cols=None, - sort_by=None, - sort_desc=False, - filters=None, - version=None, - pagesize=None, - page=None, - **criteria, - ): + @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' + 'will be removed in a future release. Please use `collection` and `catalog` instead.') + @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' + 'and will be removed in a future release. Please use `limit` instead.') + @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' + 'and will be removed in a future release. Please use `offset` instead.') + def query_object(self, objectname, *, radius=0.2*u.deg, collection=None, catalog=None, resolver=None, + limit=5000, offset=0, count_only=False, select_cols=None, sort_by=None, sort_desc=False, + filters={}, version=None, pagesize=None, page=None, **criteria): """ Query for MAST catalog entries around a specified object name using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -656,24 +554,19 @@ def query_object( response : `~astropy.table.Table` A table containing the query results. """ - # Parse pagination params - limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) - - return self.query_criteria( - collection=collection, - catalog=catalog, - object_name=object_name, - radius=radius, - resolver=resolver, - limit=limit, - offset=offset, - count_only=count_only, - select_cols=select_cols, - sort_by=sort_by, - sort_desc=sort_desc, - filters=filters, - **criteria, - ) + return self.query_criteria(collection=collection, + catalog=catalog, + objectname=objectname, + radius=radius, + resolver=resolver, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria) @class_or_instance @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) From e75151e50cda657b6752eb470969bee87a328783 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 15 Jan 2026 15:28:22 -0500 Subject: [PATCH 09/21] Update test data README --- astroquery/mast/tests/data/README.rst | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 5e0a21f6d4..5f3b25bee8 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -62,3 +62,66 @@ To generate `~astroquery.mast.tests.data.resolver.json`, use the following: ... {'name': objects, 'outputFormat': 'json', 'resolveAll': 'true'}) >>> with open('resolver.json', 'w') as file: ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_collections.json`, use the following: + +.. doctest-remote-data:: + + >>> import json + >>> from astroquery.mast import utils + ... + >>> resp = utils._simple_request('https://mast.stsci.edu/vo-tap/api/v0.1/openapi.json') + ... + >>> with open('tap_collections.json', 'w') as file: + ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_catalogs.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = 'SELECT table_name, description FROM tap_schema.tables' + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_catalogs.vot') # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_columns.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = "SELECT column_name, datatype, unit, ucd, description FROM tap_schema.columns WHERE table_name = 'dbo.catalogrecord'" + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_columns.vot') # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_capabilities.vot`, use the following: + +.. doctest-remote-data:: + + >>> import requests + >>> import pyvo + ... + >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> caps_url = tap_service.baseurl.rstrip("/") + "/capabilities" + >>> resp = requests.get(caps_url) + >>> resp.raise_for_status() + ... + >>> with open("tap_capabilities.xml", "wb") as f: + ... f.write(resp.content) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_results.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = "SELECT TOP 10 * FROM dbo.catalogrecord WHERE CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 23.34086, 60.658, 0.002)) = 1" + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_results.vot') # doctest: +SKIP From 681778a49bd45d9ae4c779978526f6b94a9116a4 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 11 Feb 2026 16:13:15 -0500 Subject: [PATCH 10/21] Pagination Adjusting from rebase Clear cache in non-remote tests --- astroquery/mast/collections.py | 3 +++ astroquery/mast/tests/test_mast.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 8f98509e20..a46d70d1fb 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -554,6 +554,9 @@ def query_object(self, objectname, *, radius=0.2*u.deg, collection=None, catalog response : `~astropy.table.Table` A table containing the query results. """ + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + return self.query_criteria(collection=collection, catalog=catalog, objectname=objectname, diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 849d479d8c..5c8f74c447 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -147,6 +147,12 @@ def reset_cloud_state(): Observations._cloud_enabled_explicitly = None +@pytest.fixture +def reset_catalogs_cache(): + Catalogs._collections_cache.clear() + yield + + def post_mockreturn(self, method="POST", url=None, data=None, timeout=10, **kwargs): if "columnsconfig" in url: if "Mast.Catalogs.Tess.Cone" in data: @@ -272,7 +278,6 @@ def zcut_download_mockreturn(url, file_path): def vo_tap_mock(): def run_sync_mock(query, **kwargs): - print(query) if 'invalid' in query: # Use this when wanting to simulate a DALQueryError # Where it occurs will depend on where you pass it (collection, catalog, parameter, etc.) From c82bd1389d353802173e627ad3bcc89c363aa840 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 26 Feb 2026 17:00:34 -0500 Subject: [PATCH 11/21] Deprecate objectname Catch up rebase --- astroquery/mast/collections.py | 14 ++++++++------ astroquery/mast/tests/test_mast.py | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index a46d70d1fb..669ae9d273 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -192,7 +192,8 @@ def supports_spatial_queries(self, collection=None, catalog=None): 'and will be removed in a future release. Please use `limit` instead.') @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' 'and will be removed in a future release. Please use `offset` instead.') - def query_criteria(self, collection=None, *, catalog=None, coordinates=None, region=None, objectname=None, + @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') + def query_criteria(self, collection=None, *, catalog=None, coordinates=None, region=None, object_name=None, radius=0.2*u.deg, resolver=None, limit=5000, offset=0, count_only=False, select_cols=None, sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): """ @@ -320,9 +321,9 @@ def query_criteria(self, collection=None, *, catalog=None, coordinates=None, reg if region: adql_region = self._create_adql_region(region) if object_name or coordinates: # Cone search - coordinates = utils.parse_input_location( - coordinates=coordinates, object_name=object_name, resolver=resolver - ) + coordinates = utils.parse_input_location(coordinates=coordinates, + objectname=object_name, + resolver=resolver) radius = coord.Angle(radius, u.deg) # If radius is just a number we assume degrees adql_region = f"CIRCLE('ICRS', {coordinates.ra.deg}, {coordinates.dec.deg}, {radius.to(u.deg).value})" @@ -490,7 +491,8 @@ def query_region(self, coordinates=None, *, radius=0.2*u.deg, region=None, colle 'and will be removed in a future release. Please use `limit` instead.') @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' 'and will be removed in a future release. Please use `offset` instead.') - def query_object(self, objectname, *, radius=0.2*u.deg, collection=None, catalog=None, resolver=None, + @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') + def query_object(self, object_name, *, radius=0.2*u.deg, collection=None, catalog=None, resolver=None, limit=5000, offset=0, count_only=False, select_cols=None, sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): """ @@ -559,7 +561,7 @@ def query_object(self, objectname, *, radius=0.2*u.deg, collection=None, catalog return self.query_criteria(collection=collection, catalog=catalog, - objectname=objectname, + object_name=object_name, radius=radius, resolver=resolver, limit=limit, diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 5c8f74c447..44bc62473f 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -13,6 +13,7 @@ import pytest from astropy.coordinates import SkyCoord from astropy.io import fits +from astropy.io.votable import parse from astropy.table import Table, unique from astropy.utils.exceptions import AstropyDeprecationWarning from pyvo.dal import TAPResults From 5857c4c6f5e115a19be32f03ef1188ed263d2424 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 7 Apr 2026 13:58:11 -0400 Subject: [PATCH 12/21] Adjusting for new database structure --- astroquery/mast/__init__.py | 2 +- astroquery/mast/catalog_collection.py | 123 ++++- astroquery/mast/tests/test_mast_remote.py | 621 +++++++++------------- 3 files changed, 377 insertions(+), 369 deletions(-) diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index fa0b9e8ff8..b5c9311686 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -15,7 +15,7 @@ class Conf(_config.ConfigNamespace): """ server = _config.ConfigItem( - 'https://mast.stsci.edu', + 'https://masttest.stsci.edu', 'Name of the MAST server.') ssoserver = _config.ConfigItem( 'https://ssoportal.stsci.edu', diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index eb5683b481..1f71da875e 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -5,8 +5,9 @@ from pyvo.dal import TAPService, DALQueryError from astropy.table import Table -from astroquery import log -from astroquery.exceptions import InvalidQueryError +from . import conf, utils +from .. import log +from ..exceptions import InvalidQueryError __all__ = ['CatalogCollection'] @@ -26,9 +27,12 @@ 'goods': 'dbo.goods_master_view', '3dhst': 'dbo.HLSP_3DHST_summary', 'candels': 'dbo.candels_master_view', - 'deepspace': 'dbo.DeepSpace_Summary' + 'deepspace': 'dbo.DeepSpace_Summary', + 'tic_v82': 'tic_v82.source' } +GROUPED_COLLECTION_ENDPOINTS = ['mast_catalogs', 'roman_catalogs'] + @dataclass class CatalogMetadata: @@ -47,7 +51,86 @@ class CatalogCollection: This class provides an interface to interact with MAST catalog collections via TAP service. """ - TAP_BASE_URL = 'https://masttest.stsci.edu/vo-tap/api/v0.1/' + TAP_BASE_URL = conf.server + '/vo-tap/api/v0.1/' + _discovered_collections = None + _collection_parent_map = None + + @classmethod + def discover_collections(cls): + """ + Discover collection names available through TAP and track parent collections. + + Returns + ------- + `~astropy.table.Table` + A table containing collection_name and parent_collection columns. + """ + if cls._discovered_collections is not None: + return cls._discovered_collections + + # Query TAP service for collection names + url = cls.TAP_BASE_URL + 'openapi.json' + response = utils._simple_request(url) + response.raise_for_status() + data = response.json() + + collection_enum = data['components']['schemas']['CatalogName']['enum'] + collection_parent_map = {} + + # Discover collections stored under grouped TAP collections + for parent_collection in GROUPED_COLLECTION_ENDPOINTS: + if parent_collection not in collection_enum: + continue + + tap_service = TAPService(cls.TAP_BASE_URL + parent_collection) + result = tap_service.run_sync('SELECT table_name FROM tap_schema.tables') + tables = result.to_table() + + for table_name in tables['table_name']: + table_name = str(table_name) + if table_name.lower().startswith('tap_schema.'): + continue + + collection_name = table_name.split('.', 1)[0].lower() + collection_parent_map.setdefault(collection_name, parent_collection) + + # Include standalone collections in map + for collection_name in collection_enum: + normalized_name = collection_name.lower() + if normalized_name in GROUPED_COLLECTION_ENDPOINTS: + continue + collection_parent_map.setdefault(normalized_name, normalized_name) + + collection_names = sorted(collection_parent_map.keys()) + parent_names = [collection_parent_map[name] for name in collection_names] + cls._collection_parent_map = collection_parent_map + cls._discovered_collections = Table( + [collection_names, parent_names], + names=('collection_name', 'parent_collection'), + ) + + return cls._discovered_collections + + @classmethod + def get_parent_collection(cls, collection_name): + """ + Return the parent TAP collection for a user-facing collection name. + + Parameters + ---------- + collection_name : str + The user-facing collection name to get the parent collection for. + """ + if cls._collection_parent_map is None: + try: + cls.discover_collections() + except Exception as ex: + log.debug(f"Failed to discover TAP collections; falling back to direct endpoint routing: {ex}") + cls._collection_parent_map = {} + + normalized_name = collection_name.lower().strip() + parent_collection = cls._collection_parent_map.get(normalized_name, normalized_name) + return parent_collection def __init__(self, collection): """ @@ -59,8 +142,8 @@ def __init__(self, collection): The name of the MAST catalog collection to interact with. """ self.name = collection.strip().lower() - # Initialize TAP service for the specified collection - self.tap_service = TAPService(self.TAP_BASE_URL + self.name) + self._parent_collection = None + self._tap_service = None # Get catalogs within this collection self._catalogs = None # Lazy-loaded property @@ -68,12 +151,30 @@ def __init__(self, collection): # ADQL functions supported by this collection's TAP service self._supported_adql_functions = None # Lazy-loaded property - # Determine the default catalog for this collection - self.default_catalog = self.get_default_catalog() + # Determine the default catalog lazily to avoid requests during initialization + self._default_catalog = None # Cache for catalog metadata to avoid redundant queries self._catalog_metadata_cache: Dict[str, CatalogMetadata] = dict() + @property + def parent_collection(self): + if self._parent_collection is None: + self._parent_collection = self.get_parent_collection(self.name) + return self._parent_collection + + @property + def tap_service(self): + if self._tap_service is None: + self._tap_service = TAPService(self.TAP_BASE_URL + self.parent_collection) + return self._tap_service + + @property + def default_catalog(self): + if self._default_catalog is None: + self._default_catalog = self.get_default_catalog() + return self._default_catalog + @property def catalogs(self): if self._catalogs is None: @@ -200,11 +301,17 @@ def _fetch_catalogs(self): """ log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") query = 'SELECT table_name, description FROM tap_schema.tables' + + # If this catalog is within a grouped collection, filter to only tables that belong to this collection + if self.parent_collection in GROUPED_COLLECTION_ENDPOINTS: + query += f" WHERE table_name LIKE '{self.name}.%'" + result = self.tap_service.run_sync(query) # Rename table_name to catalog_name for clarity result_table = result.to_table() result_table.rename_column('table_name', 'catalog_name') + return result_table def _fetch_adql_supported_functions(self): diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index f350b10786..192be858b0 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -11,6 +11,7 @@ from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.table import Table, unique +from astropy.time import Time from requests.models import Response from astroquery.mast import (Catalogs, Hapcut, Mast, MastMissions, Observations, Tesscut, Zcut, utils) @@ -1053,397 +1054,289 @@ def test_observations_get_cloud_uris_no_duplicates(self, msa_product_table, rese # CatalogClass tests # ###################### - # query functions - def test_catalogs_query_region_async(self): - in_rad = 0.001 * u.deg - responses = Catalogs.query_region_async("158.47924 -7.30962", - radius=in_rad, - catalog="Galex") - assert isinstance(responses, list) - - # Default catalog is HSC - responses = Catalogs.query_region_async("322.49324 12.16683", - radius=in_rad) - assert isinstance(responses, list) - - responses = Catalogs.query_region_async("322.49324 12.16683", - radius=in_rad, - catalog="panstarrs", - table="mean") - assert isinstance(responses, Response) - - def test_catalogs_query_region(self): - def check_result(result, row, exp_values): - assert isinstance(result, Table) - for k, v in exp_values.items(): - assert result[row][k] == v - - in_radius = 0.1 * u.deg - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="Gaia") - row = np.where(result['source_id'] == '3774902350511581696') - check_result(result, row, {'solution_id': '1635721458409799680'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.001*u.deg, - catalog="HSC", - magtype=2) - row = np.where(result['MatchID'] == '8150896') - - with pytest.warns(MaxResultsWarning): - result = Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2, nr=5) - - check_result(result, row, {'NumImages': 14, 'TargetName': 'M15'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.001*u.deg, - catalog="HSC", - version=2, - magtype=2) - row = np.where(result['MatchID'] == '82361658') - check_result(result, row, {'NumImages': 11, 'TargetName': 'NGC7078'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=in_radius, - catalog="Gaia", - version=1) - row = np.where(result['source_id'] == '1745948323734098688') - check_result(result, row, {'solution_id': '1635378410781933568'}) - result = Catalogs.query_region("322.49324 12.16683", - radius=0.01*u.deg, - catalog="Gaia", - version=2) - - row = np.where(result['source_id'] == '1745947739618544000') - check_result(result, row, {'solution_id': '1635721458409799680'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.01*u.deg, catalog="panstarrs", - table="mean", - columns=['objName', 'objID', 'yFlags', 'distance']) - row = np.where((result['objName'] == 'PSO J322.4622+12.1920') & (result['yFlags'] == 16777496)) - assert isinstance(result, Table) - np.testing.assert_allclose(result[row]['distance'], 0.039381703406789904) + def test_catalogs_collection(self): + # Default collection should be HSC + c = Catalogs() + assert c.collection.name == "hsc" + assert c.catalog == "dbo.SumMagAper2CatView" + + # Initialize with a different collection + c = Catalogs(collection="gaiadr3") + assert c.collection.name == "gaiadr3" + assert c.catalog == "dbo.gaia_source" + + # Initialize with a different collection and catalog + c = Catalogs(collection="ullyses", catalog="publications") + assert c.collection.name == "ullyses" + assert c.catalog == "dbo.publications" + + # Set the collection + c.collection = "caom" + assert c.collection.name == "caom" + assert c.catalog == "dbo.obspointing" + + def test_catalogs_get_collections(self): + collections = Catalogs.get_collections() + assert isinstance(collections, Table) + assert "collection_name" in collections.colnames + + def test_catalogs_get_catalogs(self): + catalogs = Catalogs.get_catalogs() + assert isinstance(catalogs, Table) + assert "catalog_name" in catalogs.colnames + assert "description" in catalogs.colnames + + def test_catalogs_get_column_metadata(self): + metadata = Catalogs.get_column_metadata("hsc", "dbo.SumMagAper2CatView") + assert isinstance(metadata, Table) + assert len(metadata) > 0 + assert "column_name" in metadata.colnames + assert "datatype" in metadata.colnames + assert "unit" in metadata.colnames + assert "ucd" in metadata.colnames + assert "description" in metadata.colnames + + def test_catalogs_supports_spatial_queries(self): + assert Catalogs.supports_spatial_queries("hsc", "dbo.SumMagAper2CatView") + assert not Catalogs.supports_spatial_queries("ullyses", "dbo.publications") - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="Galex") - in_radius_arcmin = 0.1*u.deg.to(u.arcmin) - distances = list(result['distance_arcmin']) + def test_catalogs_query_criteria(self): + # Positional query with multiple filters + c = Catalogs() + search_coord = SkyCoord(322.49324, 12.16683, unit="deg") + select_cols = ["matchid", "matchra", "matchdec", "numimages", "starttime", "targetname"] + result = c.query_criteria( + coordinates=search_coord, + radius="2 arcsec", + sort_by=["numimages", "starttime"], + sort_desc=[False, True], + targetname=["M-15", "NGC*"], + starttime=">2010", + limit=5, + select_cols=select_cols, + ) assert isinstance(result, Table) - assert max(distances) <= in_radius_arcmin - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="tic") - row = np.where(result['ID'] == '841736289') - second_id = result[1]['ID'] - check_result(result, row, {'gaiaqflag': 1}) - np.testing.assert_allclose(result[row]['RA_orig'], 158.475246786483) - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="tic", - pagesize=1, - page=2) + assert len(result) == 5 + assert all(c in result.colnames for c in select_cols) + assert all(val == "M-15" or str(val).startswith("NGC") for val in result["targetname"]) + assert all(result["starttime"] > Time("2010-01-01T00:00:00")) + # Assert that all results are within 2 arcsec of the specified coordinates + coords = SkyCoord(result["matchra"], result["matchdec"], unit="deg") + separation = coords.separation(search_coord) + assert all(separation <= 2 * u.arcsec) + # Assert that results are sorted by numimages ascending and then starttime descending + assert all(result["numimages"][i] <= result["numimages"][i + 1] for i in range(len(result) - 1)) + assert all( + result["starttime"][i] >= result["starttime"][i + 1] + for i in range(len(result) - 1) + if result["numimages"][i] == result["numimages"][i + 1] + ) + + # Non-positional query with multiple filters and select_cols + select_cols = [ + "target_name_ullyses", + "target_classification", + "known_binary", + "sp_class", + "gaia_parallax", + "star_teff", + "coordinate_epoch", + "spectral_type_ref", + ] + result = c.query_criteria( + collection="ullyses", + catalog="sciencemetadata", + target_name_ullyses="NGC*", + target_classification=["!Galaxy", "!Late O Dwarf"], + known_binary=False, + sp_class=["O", "B"], + gaia_parallax=["<-0.01", ">=0", "!<-0.3"], + star_teff="30000..50000", + coordinate_epoch=2016, + spectral_type_ref=[51, 18, 59], + select_cols=select_cols, + ) assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="ctl") - row = np.where(result['ID'] == '56662064') - check_result(result, row, {'TYC': '4918-01335-1'}) - - result = Catalogs.query_region("210.80227 54.34895", - radius=1*u.deg, - catalog="diskdetective") - row = np.where(result['designation'] == 'J140544.95+535941.1') - check_result(result, row, {'ZooniverseID': 'AWI0000r57'}) - - def test_catalogs_query_object_async(self): - responses = Catalogs.query_object_async("M10", - radius=.02, - catalog="TIC") - assert isinstance(responses, list) - - def test_catalogs_query_object(self): - def check_result(result, exp_values): - assert isinstance(result, Table) - for k, v in exp_values.items(): - assert v in result[k] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="TIC") - check_result(result, {'ID': '1305764225'}) - second_id = result[1]['ID'] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="TIC", - pagesize=1, - page=2) - assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) - check_result(result, {'MatchID': '667727'}) - - result = Catalogs.query_object("M10", - radius=.001, - catalog="panstarrs", - table="mean", - columns=['objName', 'objID']) - check_result(result, {'objName': 'PSO J254.2873-04.1006'}) - - result = Catalogs.query_object("M10", - radius=0.18, - catalog="diskdetective") - check_result(result, {'designation': 'J165749.79-040315.1'}) - - result = Catalogs.query_object("M10", - radius=0.001, - catalog="Gaia", - version=1) - distances = list(result['distance']) - radius_arcmin = 0.01 * u.deg.to(u.arcmin) + assert len(result) > 0 + assert all(str(val).startswith("NGC") for val in result["target_name_ullyses"]) + assert all(result["target_classification"] != "Galaxy") + assert all(result["target_classification"] != "Late O Dwarf") + assert not any(result["known_binary"]) + assert all(np.isin(result["sp_class"], ["O", "B"])) + assert all( + (result["gaia_parallax"] < -0.01) | (result["gaia_parallax"] >= 0) | (result["gaia_parallax"] >= -0.3) + ) + assert all((result["star_teff"] >= 30000) & (result["star_teff"] <= 50000)) + assert all(result["coordinate_epoch"] == 2016) + assert all(np.isin(result["spectral_type_ref"], [51, 18, 59, 1])) + assert all(c in result.colnames for c in select_cols) + + # Test offset + result = c.query_criteria(collection="hsc", numimages=8, sort_by="matchid", limit=5) + result_offset = c.query_criteria(collection="hsc", numimages=8, sort_by="matchid", limit=5, offset=2) + assert result_offset["matchid"][0] == result["matchid"][2] + + # Count only + result_count = c.query_criteria(collection="goods", class_star=0.23, count_only=True) + assert isinstance(result_count, (int, np.integer)) + assert result_count > 0 + + # Temporal filters and filter passed in through filters argument + result = c.query_criteria(collection='caom', + catalog='caommembers', + limit=10, + recordcreated=['>2014-01-01T00:00:00', '<2000'], + recordmodified='2021-03-01..2021-03-31', + filters={'collection': 'GALEX'}) assert isinstance(result, Table) - assert max(distances) < radius_arcmin - - result = Catalogs.query_object("TIC 441662144", - radius=0.001, - catalog="ctl") - check_result(result, {'ID': '441662144'}) - - result = Catalogs.query_object('M10', - radius=0.08, - catalog='plato') - assert 'PICidDR1' in result.colnames - - def test_catalogs_query_criteria_async(self): - # without position - responses = Catalogs.query_criteria_async(catalog="Tic", - Bmag=[30, 50], - objType="STAR") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="ctl", - Bmag=[30, 50], - objType="STAR") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - state=["inactive", "disabled"], - oval=[8, 10], - multi=[3, 7]) - assert isinstance(responses, list) - - # with position - responses = Catalogs.query_criteria_async(catalog="Tic", - object_name="M10", - objType="EXTENDED") - assert isinstance(responses, list) + assert len(result) <= 10 + assert all(c in result.colnames for c in ['recordcreated', 'collection']) + assert all( + (result['recordcreated'] > Time('2014-01-01T00:00:00')) + | (result['recordcreated'] < Time('2000-01-01T00:00:00')) + ) + assert all((result['recordmodified'] >= Time('2021-03-01T00:00:00')) + & (result['recordmodified'] <= Time('2021-03-31T23:59:59'))) + assert all(result['collection'] == 'GALEX') + + def test_catalogs_query_criteria_error(self): + # No results should warn user + # with pytest.warns(NoResultsWarning): + # Catalogs.query_criteria(collection="ps1_dr2", skycellid=-1) - responses = Catalogs.query_criteria_async(catalog="CTL", - object_name="M10", - objType="EXTENDED") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - object_name="M10", - radius=2, - state="complete") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="panstarrs", - table="mean", - object_name="M10", - radius=.02, - qualityFlag=48) - assert isinstance(responses, Response) - - def test_catalogs_query_criteria(self): - def check_result(result, exp_vals): - assert isinstance(result, Table) - for k, v in exp_vals.items(): - assert v in result[k] + with pytest.warns(NoResultsWarning): + Catalogs.query_criteria(collection="classy", target=[]) - # without position - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], - objType="STAR") - check_result(result, {'ID': '81609218'}) - second_id = result[1]['ID'] - - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], - objType="STAR", - pagesize=1, - page=2) + def test_catalogs_query_region(self): + # Region search with polygon + select_cols = ["target_name", "obs_id", "s_ra", "s_dec", "s_region"] + result = Catalogs.query_region( + collection="caom", + region="POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94", + limit=5, + select_cols=select_cols, + ) assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_criteria(catalog="ctl", - Tmag=[10.5, 11], - POSflag="2mass") - check_result(result, {'ID': '291067184'}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - state=["inactive", "disabled"], - oval=[8, 10], - multi=[3, 7]) - check_result(result, {'designation': 'J003920.04-300132.4'}) - - # with position - result = Catalogs.query_criteria(catalog="Tic", - object_name="M10", objType="EXTENDED") - check_result(result, {'ID': '10000732589'}) - - result = Catalogs.query_criteria(object_name='TIC 291067184', - catalog="ctl", - Tmag=[10.5, 11], - POSflag="2mass") - check_result(result, {'Tmag': 10.893}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - object_name="M10", - radius=2, - state="complete") - check_result(result, {'designation': 'J165628.40-054630.8'}) - - result = Catalogs.query_criteria(catalog="panstarrs", - object_name="M10", - radius=.01, - qualityFlag=32, - zoneID=10306, - columns=['objName', 'objID']) - check_result(result, {'objName': 'PSO J254.2861-04.1091'}) - - result = Catalogs.query_criteria(coordinates="158.47924 -7.30962", - radius=0.01, - catalog="PANSTARRS", - table="mean", - data_release="dr2", - nStackDetections=[("gte", "1")], - columns=["objName", "distance"], - sort_by=[("asc", "distance")]) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified polygon + # We can't just check that the coordinates are within the polygon because + # they may intersect the polygon without the center being within it + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + polygon = SkyCoord(18.855, -6.945, unit="deg") + separation = coords.separation(polygon) + assert all(separation <= 0.1 * u.deg) + + # Region search with circle + result = Catalogs.query_region( + collection="caom", region="CIRCLE ICRS 18.85 -6.95 0.1", limit=5, select_cols=select_cols + ) assert isinstance(result, Table) - assert result['distance'][0] <= result['distance'][1] - - # with case-insensitive keyword arguments - result = Catalogs.query_criteria(catalog="Tic", - bMAG=[30, 50], - objtype="STAR") - check_result(result, {'ID': '81609218'}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - STATE=["inactive", "disabled"], - oVaL=[8, 10], - Multi=[3, 7]) - check_result(result, {'designation': 'J003920.04-300132.4'}) - - def test_catalogs_query_criteria_invalid_keyword(self): - # attempt to make a criteria query with invalid keyword - with pytest.raises(InvalidQueryError) as err_no_alt: - Catalogs.query_criteria(catalog='tic', not_a_keyword='TESS') - assert "Filter 'not_a_keyword' does not exist." in str(err_no_alt.value) - - # keyword is close enough for difflib to offer alternative - with pytest.raises(InvalidQueryError) as err_with_alt: - Catalogs.query_criteria(catalog='ctl', objectType="STAR") - assert 'objType' in str(err_with_alt.value) - - # region query with invalid keyword - with pytest.raises(InvalidQueryError) as err_region: - Catalogs.query_region('322.49324 12.16683', - radius=0.001*u.deg, - catalog='HSC', - invalid=2) - assert "Filter 'invalid' does not exist for catalog HSC." in str(err_region.value) - - # panstarrs criteria query with invalid keyword - with pytest.raises(InvalidQueryError) as err_ps_criteria: - Catalogs.query_criteria(coordinates="158.47924 -7.30962", - catalog="PANSTARRS", - table="mean", - data_release="dr2", - columns=["objName", "distance"], - sort_by=[("asc", "distance")], - obj_name='invalid') - assert 'objName' in str(err_ps_criteria.value) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified circle + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + center = SkyCoord(18.85, -6.95, unit="deg") + separation = coords.separation(center) + assert all(separation <= 0.1 * u.deg) + def test_catalogs_query_object(self): + # Object search + select_cols = ["target_name", "obs_id", "s_ra", "s_dec"] + result = Catalogs.query_object(collection="caom", object_name="M2", limit=5, select_cols=select_cols) + assert isinstance(result, Table) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified object + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + m2_coords = SkyCoord(323.36258, -0.82325, unit="deg") + separation = coords.separation(m2_coords) + assert all(separation <= 0.1 * u.deg) + + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_query_hsc_matchid_async(self): - catalogData = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) + catalogData = Catalogs.query_object("M10", radius=0.001, collection="HSC") responses = Catalogs.query_hsc_matchid_async(catalogData[0]) assert isinstance(responses, list) - responses = Catalogs.query_hsc_matchid_async(catalogData[0]["MatchID"]) + responses = Catalogs.query_hsc_matchid_async(catalogData[0]["matchid"]) assert isinstance(responses, list) + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_query_hsc_matchid(self): - catalogData = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) - matchid = catalogData[0]["MatchID"] + catalogData = Catalogs.query_object("M10", radius=0.001, collection="HSC") + matchid = str(catalogData[0]["matchid"]) result = Catalogs.query_hsc_matchid(catalogData[0]) assert isinstance(result, Table) - assert (result['MatchID'] == matchid).all() + assert (result["MatchID"].value == matchid).all() result2 = Catalogs.query_hsc_matchid(matchid) assert isinstance(result2, Table) assert len(result2) == len(result) - assert (result2['MatchID'] == matchid).all() + assert (result2["MatchID"] == matchid).all() + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_get_hsc_spectra_async(self): responses = Catalogs.get_hsc_spectra_async() assert isinstance(responses, list) + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_get_hsc_spectra(self): result = Catalogs.get_hsc_spectra() assert isinstance(result, Table) - assert result[np.where(result['MatchID'] == '19657846')] - assert result[np.where(result['DatasetName'] == 'HAG_J072657.06+691415.5_J8HPAXAEQ_V01.SPEC1D')] + assert result[np.where(result["MatchID"] == "19657846")] + assert result[np.where(result["DatasetName"] == "HAG_J072657.06+691415.5_J8HPAXAEQ_V01.SPEC1D")] + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_download_hsc_spectra(self, tmpdir): allSpectra = Catalogs.get_hsc_spectra() # actually download the products - result = Catalogs.download_hsc_spectra(allSpectra[10], - download_dir=str(tmpdir)) + result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) assert isinstance(result, Table) for row in result: - if row['Status'] == 'COMPLETE': - assert os.path.isfile(row['Local Path']) + if row["Status"] == "COMPLETE": + assert os.path.isfile(row["Local Path"]) # just get the curl script - result = Catalogs.download_hsc_spectra(allSpectra[20:24], - download_dir=str(tmpdir), curl_flag=True) + result = Catalogs.download_hsc_spectra(allSpectra[20:24], download_dir=str(tmpdir), curl_flag=True) assert isinstance(result, Table) - assert os.path.isfile(result['Local Path'][0]) + assert os.path.isfile(result["Local Path"][0]) ########################### # CatalogCollection tests # ########################### - @pytest.mark.filterwarnings("ignore::pyvo.io.vosi.exceptions.W02") - @pytest.mark.parametrize("catalog", ["caom", "ullyses", "tic"]) - def test_catalog_collection_get_catalog_metadata(self, catalog): - cc = CatalogCollection(catalog) + def test_catalog_collection_discover_collections(self): + collections = CatalogCollection.discover_collections() + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames + assert "parent_collection" in collections.colnames + assert CatalogCollection._discovered_collections is not None + + def test_catalog_collection_get_parent_collection(self): + parent = CatalogCollection.get_parent_collection("TIC") + assert parent == "tic" + + # parent = CatalogCollection.get_parent_collection("tic_v82") + # assert parent == "mast_catalogs" + + @pytest.mark.parametrize("collection", ["caom"]) + # @pytest.mark.parametrize("collection", ["caom", "tic_v82"]) + def test_catalog_collection_get_catalogs(self, collection): + cc = CatalogCollection(collection) + catalogs = cc._fetch_catalogs() + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + assert catalogs.colnames == ["catalog_name", "description"] + + @pytest.mark.parametrize("collection", ["caom", "ullyses", "tic"]) + def test_catalog_collection_get_catalog_metadata(self, collection): + cc = CatalogCollection(collection) default_catalog = cc.get_default_catalog() default_metadata = cc.get_catalog_metadata(default_catalog) assert isinstance(default_metadata, CatalogMetadata) @@ -1455,7 +1348,7 @@ def test_catalog_collection_get_catalog_metadata(self, catalog): assert len(default_metadata.column_metadata) > 1 assert default_metadata.ra_column in default_metadata.column_metadata["column_name"] assert default_metadata.dec_column in default_metadata.column_metadata["column_name"] - if catalog != "ullyses": + if collection != "ullyses": assert default_metadata.supports_spatial_queries metadata_cache = cc._catalog_metadata_cache @@ -1466,33 +1359,35 @@ def test_catalog_collection_get_catalog_metadata(self, catalog): def test_catalog_collection_invalid_get_catalog_metadata(self): cc = CatalogCollection("TIC") invalid_catalog = "invalid_catalog" - with pytest.raises(InvalidQueryError, match=f"Catalog '{invalid_catalog}' is not " - f"recognized for collection '{cc.name}'."): + with pytest.raises( + InvalidQueryError, match=f"Catalog '{invalid_catalog}' is not recognized for collection '{cc.name}'." + ): cc.get_catalog_metadata(invalid_catalog) - @pytest.mark.filterwarnings("ignore::pyvo.io.vosi.exceptions.W02") - @pytest.mark.parametrize("catalog", ["tic", "classy", "ullyses"]) - def test_catalog_collection_get_default_catalog(self, catalog): - cc = CatalogCollection(catalog) + @pytest.mark.parametrize("collection", ["tic", "classy", "ullyses"]) + def test_catalog_collection_get_default_catalog(self, collection): + cc = CatalogCollection(collection) catalogs = cc._fetch_catalogs() default = cc.get_default_catalog() assert len(catalogs) > 1 assert isinstance(catalogs, Table) - assert catalogs.colnames == ['catalog_name', 'description'] + assert catalogs.colnames == ["catalog_name", "description"] assert not default.startswith("tap_schema") assert default.casefold() in [name.casefold() for name in catalogs["catalog_name"]] - assert DEFAULT_CATALOGS[catalog] == default + assert DEFAULT_CATALOGS[collection] == default def test_catalog_collection_run_tap_query(self): cc = CatalogCollection("GAIADR3") - adql_str = "SELECT TOP 10 solution_id, designation, source_id, ra, dec FROM gaia_source WHERE " \ - "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + adql_str = ( + "SELECT TOP 10 solution_id, designation, source_id, ra, dec FROM gaia_source WHERE " + "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + ) result = cc.run_tap_query(adql_str) assert isinstance(result, Table) - assert result.colnames == ['solution_id', 'designation', 'source_id', 'ra', 'dec'] + assert result.colnames == ["solution_id", "designation", "source_id", "ra", "dec"] assert ((result["ra"] >= 10) & (result["ra"] <= 11)).all() assert ((result["dec"] >= 12) & (result["dec"] <= 13)).all() @@ -1510,13 +1405,19 @@ def test_catalog_collection_verify_criteria(self): assert result is None close_match_col = "gaiagaiabp" - with pytest.raises(InvalidQueryError, match=f"Filter '{close_match_col}' is not recognized for collection " - f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?"): + with pytest.raises( + InvalidQueryError, + match=f"Filter '{close_match_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?", + ): cc._verify_criteria(default_catalog, gaiagaiabp=1) invalid_col = "fake_column" - with pytest.raises(InvalidQueryError, match=f"Filter '{invalid_col}' is not recognized for collection " - f"'{cc.name}' and catalog '{default_catalog}'."): + with pytest.raises( + InvalidQueryError, + match=f"Filter '{invalid_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'.", + ): cc._verify_criteria(default_catalog, fake_column=1) ###################### @@ -1824,4 +1725,4 @@ def check_cutout_list(cutout_list, data_shape): with pytest.warns(NoResultsWarning, match='Missing HAP files for input target. Cutout not performed.'): cutout_list = Hapcut.get_cutouts(coordinates=bad_coord) assert isinstance(cutout_list, list) - assert len(cutout_list) == 0 + assert len(cutout_list) == 0 \ No newline at end of file From 5c43b3e34b61685b61230699ecf993fb1f9bbb5b Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 8 Apr 2026 14:16:32 -0400 Subject: [PATCH 13/21] case insensitive filters, remote tests do not call network for default initializations of Catalogs --- astroquery/mast/catalog_collection.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 1f71da875e..7de2f1b88f 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -122,11 +122,7 @@ def get_parent_collection(cls, collection_name): The user-facing collection name to get the parent collection for. """ if cls._collection_parent_map is None: - try: - cls.discover_collections() - except Exception as ex: - log.debug(f"Failed to discover TAP collections; falling back to direct endpoint routing: {ex}") - cls._collection_parent_map = {} + cls.discover_collections() normalized_name = collection_name.lower().strip() parent_collection = cls._collection_parent_map.get(normalized_name, normalized_name) @@ -480,13 +476,14 @@ def _verify_criteria(self, catalog, **criteria): if not criteria: return col_names = list(self.get_catalog_metadata(catalog).column_metadata['column_name']) + col_name_lookup = {col.lower(): col for col in col_names} # Check each criteria argument for validity for kwd in criteria.keys(): - if kwd not in col_names: + if kwd.lower() not in col_name_lookup: # Suggest closest match for invalid keyword - closest = difflib.get_close_matches(kwd, col_names, n=1) - suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + closest = difflib.get_close_matches(kwd.lower(), list(col_name_lookup.keys()), n=1) + suggestion = f" Did you mean '{col_name_lookup[closest[0]]}'?" if closest else "" raise InvalidQueryError( f"Filter '{kwd}' is not recognized for collection '{self.name}' and " f"catalog '{catalog}'.{suggestion}" From ac2ff1de8436bee5b695aba77675c54817b1f2b0 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 16 Apr 2026 16:20:30 -0400 Subject: [PATCH 14/21] Time queries, styling Avoid overflow errors updating tap_collections.json slight adjustments to parsing logic catalog tests typo correction addressing comments Some rebase fixes Rebase fix --- astroquery/mast/__init__.py | 36 +- astroquery/mast/catalog_collection.py | 173 +-- astroquery/mast/collections.py | 218 +++- .../mast/tests/data/tap_collections.json | 79 +- astroquery/mast/tests/test_mast.py | 1099 +++++++++++++++-- 5 files changed, 1309 insertions(+), 296 deletions(-) diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index b5c9311686..5bb93b4a7e 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -15,7 +15,7 @@ class Conf(_config.ConfigNamespace): """ server = _config.ConfigItem( - 'https://masttest.stsci.edu', + 'https://mast.stsci.edu', 'Name of the MAST server.') ssoserver = _config.ConfigItem( 'https://ssoportal.stsci.edu', @@ -37,18 +37,28 @@ class Conf(_config.ConfigNamespace): conf = Conf() -from .cutouts import TesscutClass, Tesscut, ZcutClass, Zcut, HapcutClass, Hapcut -from .observations import Observations, ObservationsClass, MastClass, Mast +from . import utils from .collections import Catalogs, CatalogsClass +from .cutouts import Hapcut, HapcutClass, Tesscut, TesscutClass, Zcut, ZcutClass from .missions import MastMissions, MastMissionsClass -from . import utils +from .observations import Mast, MastClass, Observations, ObservationsClass -__all__ = ['Observations', 'ObservationsClass', - 'Catalogs', 'CatalogsClass', - 'MastMissions', 'MastMissionsClass', - 'Mast', 'MastClass', - 'Tesscut', 'TesscutClass', - 'Zcut', 'ZcutClass', - 'Hapcut', 'HapcutClass', - 'Conf', 'conf', 'utils', - ] +__all__ = [ + "Observations", + "ObservationsClass", + "Catalogs", + "CatalogsClass", + "MastMissions", + "MastMissionsClass", + "Mast", + "MastClass", + "Tesscut", + "TesscutClass", + "Zcut", + "ZcutClass", + "Hapcut", + "HapcutClass", + "Conf", + "conf", + "utils", +] diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 7de2f1b88f..672e457aae 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -2,36 +2,36 @@ from dataclasses import dataclass from typing import Dict, Optional -from pyvo.dal import TAPService, DALQueryError from astropy.table import Table +from pyvo.dal import DALQueryError, TAPService -from . import conf, utils from .. import log from ..exceptions import InvalidQueryError +from . import conf, utils -__all__ = ['CatalogCollection'] +__all__ = ["CatalogCollection"] DEFAULT_CATALOGS = { - 'caom': 'dbo.obspointing', - 'gaiadr3': 'dbo.gaia_source', - 'hsc': 'dbo.SumMagAper2CatView', - 'hscv2': 'dbo.SumMagAper2CatView', - 'missionmast': 'dbo.hst_science_missionmast', - 'ps1dr1': 'dbo.MeanObjectView', - 'ps1dr2': 'dbo.MeanObjectView', - 'ps1_dr2': 'ps1_dr2.forced_mean_object', - 'skymapperdr4': 'dr4.master', - 'tic': 'dbo.CatalogRecord', - 'classy': 'dbo.targets', - 'ullyses': 'dbo.sciencemetadata', - 'goods': 'dbo.goods_master_view', - '3dhst': 'dbo.HLSP_3DHST_summary', - 'candels': 'dbo.candels_master_view', - 'deepspace': 'dbo.DeepSpace_Summary', - 'tic_v82': 'tic_v82.source' + "caom": "dbo.obspointing", + "gaiadr3": "dbo.gaia_source", + "hsc": "dbo.SumMagAper2CatView", + "hscv2": "dbo.SumMagAper2CatView", + "missionmast": "dbo.hst_science_missionmast", + "ps1dr1": "dbo.MeanObjectView", + "ps1dr2": "dbo.MeanObjectView", + "ps1_dr2": "ps1_dr2.forced_mean_object", + "skymapperdr4": "dr4.master", + "tic": "dbo.CatalogRecord", + "classy": "dbo.targets", + "ullyses": "dbo.sciencemetadata", + "goods": "dbo.goods_master_view", + "3dhst": "dbo.HLSP_3DHST_summary", + "candels": "dbo.candels_master_view", + "deepspace": "dbo.DeepSpace_Summary", + "tic_v82": "tic_v82.source", } -GROUPED_COLLECTION_ENDPOINTS = ['mast_catalogs', 'roman_catalogs'] +GROUPED_COLLECTION_ENDPOINTS = ["mast_catalogs", "roman_catalogs"] @dataclass @@ -40,6 +40,7 @@ class CatalogMetadata: Data class to hold metadata about a catalog, including column metadata, RA/Dec column names, and spatial query support. """ + column_metadata: Table ra_column: Optional[str] dec_column: Optional[str] @@ -51,7 +52,7 @@ class CatalogCollection: This class provides an interface to interact with MAST catalog collections via TAP service. """ - TAP_BASE_URL = conf.server + '/vo-tap/api/v0.1/' + TAP_BASE_URL = conf.server + "/vo-tap/api/v0.1/" _discovered_collections = None _collection_parent_map = None @@ -69,12 +70,16 @@ def discover_collections(cls): return cls._discovered_collections # Query TAP service for collection names - url = cls.TAP_BASE_URL + 'openapi.json' + url = cls.TAP_BASE_URL + "openapi.json" response = utils._simple_request(url) response.raise_for_status() data = response.json() - collection_enum = data['components']['schemas']['CatalogName']['enum'] + try: + collection_enum = data["components"]["schemas"]["CatalogName"]["enum"] + except KeyError: + raise RuntimeError("Failed to discover collections from TAP service: Unexpected response format") + collection_parent_map = {} # Discover collections stored under grouped TAP collections @@ -83,15 +88,15 @@ def discover_collections(cls): continue tap_service = TAPService(cls.TAP_BASE_URL + parent_collection) - result = tap_service.run_sync('SELECT table_name FROM tap_schema.tables') + result = tap_service.run_sync("SELECT TOP 5000 table_name FROM tap_schema.tables") tables = result.to_table() - for table_name in tables['table_name']: + for table_name in tables["table_name"]: table_name = str(table_name) - if table_name.lower().startswith('tap_schema.'): + if table_name.lower().startswith("tap_schema."): continue - collection_name = table_name.split('.', 1)[0].lower() + collection_name = table_name.split(".", 1)[0].lower() collection_parent_map.setdefault(collection_name, parent_collection) # Include standalone collections in map @@ -106,7 +111,7 @@ def discover_collections(cls): cls._collection_parent_map = collection_parent_map cls._discovered_collections = Table( [collection_names, parent_names], - names=('collection_name', 'parent_collection'), + names=("collection_name", "parent_collection"), ) return cls._discovered_collections @@ -121,11 +126,18 @@ def get_parent_collection(cls, collection_name): collection_name : str The user-facing collection name to get the parent collection for. """ + if not isinstance(collection_name, str): + raise InvalidQueryError(f"Collection name must be a string, got {type(collection_name)}") + if cls._collection_parent_map is None: cls.discover_collections() normalized_name = collection_name.lower().strip() - parent_collection = cls._collection_parent_map.get(normalized_name, normalized_name) + parent_collection = cls._collection_parent_map.get(normalized_name) + + if parent_collection is None: + raise InvalidQueryError(f"Collection '{collection_name}' not found") + return parent_collection def __init__(self, collection): @@ -137,6 +149,9 @@ def __init__(self, collection): collection : str The name of the MAST catalog collection to interact with. """ + if not isinstance(collection, str): + raise ValueError(f"Collection name must be a string, got {type(collection)}") + self.name = collection.strip().lower() self._parent_collection = None self._tap_service = None @@ -153,6 +168,10 @@ def __init__(self, collection): # Cache for catalog metadata to avoid redundant queries self._catalog_metadata_cache: Dict[str, CatalogMetadata] = dict() + # Cache the catalog lookup mapping for validating catalog names in queries + self._catalog_lookup = None + self._no_prefix_lookup = None + @property def parent_collection(self): if self._parent_collection is None: @@ -179,7 +198,7 @@ def catalogs(self): @property def catalog_names(self): - return self.catalogs['catalog_name'].tolist() + return self.catalogs["catalog_name"].tolist() @property def supported_adql_functions(self): @@ -216,17 +235,16 @@ def get_catalog_metadata(self, catalog): ra_col, dec_col = self._get_ra_dec_column_names(metadata) # Determine if spatial queries are supported - supports_adql_geometry = all( - func in self.supported_adql_functions - for func in ('POINT', 'CIRCLE', 'CONTAINS') - ) + supports_adql_geometry = all(func in self.supported_adql_functions for func in ("POINT", "CIRCLE", "CONTAINS")) # Try an inexpensive spatial query if RA/Dec columns are known - supports_spatial_queries = (supports_adql_geometry and ra_col is not None and dec_col is not None) + supports_spatial_queries = supports_adql_geometry and ra_col is not None and dec_col is not None if supports_spatial_queries: # If an ra and dec column exist, test spatial query support - spatial_query = (f'SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT(\'ICRS\', {ra_col}, {dec_col}), ' - 'CIRCLE(\'ICRS\', 0, 0, 0.001)) = 1') + spatial_query = ( + f"SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT('ICRS', {ra_col}, {dec_col}), " + "CIRCLE('ICRS', 0, 0, 0.001)) = 1" + ) try: self.tap_service.search(spatial_query) except DALQueryError: @@ -296,7 +314,7 @@ def _fetch_catalogs(self): List of catalog names. """ log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") - query = 'SELECT table_name, description FROM tap_schema.tables' + query = "SELECT TOP 5000 table_name, description FROM tap_schema.tables" # If this catalog is within a grouped collection, filter to only tables that belong to this collection if self.parent_collection in GROUPED_COLLECTION_ENDPOINTS: @@ -306,7 +324,7 @@ def _fetch_catalogs(self): # Rename table_name to catalog_name for clarity result_table = result.to_table() - result_table.rename_column('table_name', 'catalog_name') + result_table.rename_column("table_name", "catalog_name") return result_table @@ -319,20 +337,20 @@ def _fetch_adql_supported_functions(self): list A list of supported ADQL functions. """ - adql_functions = ['CIRCLE', 'POLYGON', 'POINT', 'CONTAINS', 'INTERSECTS'] - supported = [] - feature_id = 'ivo://ivoa.net/std/TAPRegExt#features-adqlgeo' + adql_functions = ["CIRCLE", "POLYGON", "POINT", "CONTAINS", "INTERSECTS"] + supported = set() + feature_id = "ivo://ivoa.net/std/TAPRegExt#features-adqlgeo" for capability in self.tap_service.capabilities: - if capability.standardid != 'ivo://ivoa.net/std/TAP': + if capability.standardid != "ivo://ivoa.net/std/TAP": continue for lang in capability.languages: - if lang.name != 'ADQL': + if lang.name != "ADQL": continue for func in adql_functions: if lang.get_feature(feature_id, func): - supported.append(func) + supported.add(func) return supported @@ -356,24 +374,32 @@ def _verify_catalog(self, catalog): InvalidQueryError If the specified catalog is not valid for the given collection. """ - catalog = catalog.lower() - - # Build a mapping for case-insensitive and no-prefix lookup - lookup = {} - no_prefix_map = {} - for cat in self.catalog_names: - cat_lower = cat.lower() - lookup[cat_lower] = cat # case-insensitive match - no_prefix = cat_lower.split('.')[-1] - if no_prefix not in no_prefix_map: - no_prefix_map[no_prefix] = [cat] # no-prefix match (first occurrence) - else: - no_prefix_map[no_prefix].append(cat) - - # Add unambiguous no-prefix matches to lookup - for no_prefix, cats in no_prefix_map.items(): - if len(cats) == 1: - lookup[no_prefix] = cats[0] + catalog = catalog.lower().strip() + + if self._catalog_lookup is not None and self._no_prefix_lookup is not None: + lookup = self._catalog_lookup + no_prefix_map = self._no_prefix_lookup + else: + # Build a mapping for case-insensitive and no-prefix lookup + lookup = {} + no_prefix_map = {} + for cat in self.catalog_names: + cat_lower = cat.lower() + lookup[cat_lower] = cat # case-insensitive match + no_prefix = cat_lower.split(".")[-1] + if no_prefix not in no_prefix_map: + no_prefix_map[no_prefix] = [cat] # no-prefix match (first occurrence) + else: + no_prefix_map[no_prefix].append(cat) + + # Add unambiguous no-prefix matches to lookup + for no_prefix, cats in no_prefix_map.items(): + if len(cats) == 1: + lookup[no_prefix] = cats[0] + + # Cache the lookup maps for future calls + self._catalog_lookup = lookup + self._no_prefix_lookup = no_prefix_map # Direct or unambiguous no-prefix match if catalog in lookup: @@ -381,7 +407,7 @@ def _verify_catalog(self, catalog): # Check for ambiguous no-prefix matches if catalog in no_prefix_map and len(no_prefix_map[catalog]) > 1: - matches = ', '.join(no_prefix_map[catalog]) + matches = ", ".join(no_prefix_map[catalog]) raise InvalidQueryError( f"Catalog '{catalog}' is ambiguous for collection '{self.name}'. " f"It matches multiple catalogs: {matches}. Please specify the full catalog name." @@ -413,7 +439,7 @@ def _get_column_metadata(self, catalog): log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") query = f""" - SELECT + SELECT TOP 5000 column_name, datatype, unit, @@ -427,8 +453,7 @@ def _get_column_metadata(self, catalog): if len(result) == 0: raise InvalidQueryError(f"Catalog '{catalog}' not found in collection '{self.name}'.") - column_metadata = result.to_table() - return column_metadata + return result.to_table() def _get_ra_dec_column_names(self, column_metadata): """ @@ -447,12 +472,12 @@ def _get_ra_dec_column_names(self, column_metadata): # Look for a column with UCD 'pos.eq.ra;meta.main' and 'pos.eq.dec;meta.main' ra_col = None dec_col = None - for name, ucd in zip(column_metadata['column_name'], column_metadata['ucd']): - if ucd and 'pos.eq.ra;meta.main' in ucd: + for name, ucd in zip(column_metadata["column_name"], column_metadata["ucd"]): + if ucd and "pos.eq.ra;meta.main" in ucd.lower(): # TODO: ps1_dr2.mean_object and ps1_dr2.stacked_object has a column that can be used, # but is not labeled with "meta.main" ra_col = name - elif ucd and 'pos.eq.dec;meta.main' in ucd: + elif ucd and "pos.eq.dec;meta.main" in ucd.lower(): dec_col = name return ra_col, dec_col @@ -475,11 +500,11 @@ def _verify_criteria(self, catalog, **criteria): """ if not criteria: return - col_names = list(self.get_catalog_metadata(catalog).column_metadata['column_name']) + col_names = list(self.get_catalog_metadata(catalog).column_metadata["column_name"]) col_name_lookup = {col.lower(): col for col in col_names} # Check each criteria argument for validity - for kwd in criteria.keys(): + for kwd in criteria: if kwd.lower() not in col_name_lookup: # Suggest closest match for invalid keyword closest = difflib.get_close_matches(kwd.lower(), list(col_name_lookup.keys()), n=1) @@ -487,4 +512,4 @@ def _verify_criteria(self, catalog, **criteria): raise InvalidQueryError( f"Filter '{kwd}' is not recognized for collection '{self.name}' and " f"catalog '{catalog}'.{suggestion}" - ) + ) \ No newline at end of file diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 669ae9d273..c32ee64d74 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -186,16 +186,50 @@ def supports_spatial_queries(self, collection=None, catalog=None): return collection_obj.get_catalog_metadata(catalog).supports_spatial_queries @class_or_instance - @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' - 'will be removed in a future release. Please use `collection` and `catalog` instead.') - @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' - 'and will be removed in a future release. Please use `limit` instead.') - @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' - 'and will be removed in a future release. Please use `offset` instead.') - @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') - def query_criteria(self, collection=None, *, catalog=None, coordinates=None, region=None, object_name=None, - radius=0.2*u.deg, resolver=None, limit=5000, offset=0, count_only=False, select_cols=None, - sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_criteria( + self, + collection=None, + *, + catalog=None, + coordinates=None, + region=None, + object_name=None, + radius=0.2 * u.deg, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ Query a MAST catalog from a given collection using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -321,9 +355,9 @@ def query_criteria(self, collection=None, *, catalog=None, coordinates=None, reg if region: adql_region = self._create_adql_region(region) if object_name or coordinates: # Cone search - coordinates = utils.parse_input_location(coordinates=coordinates, - objectname=object_name, - resolver=resolver) + coordinates = utils.parse_input_location( + coordinates=coordinates, object_name=object_name, resolver=resolver + ) radius = coord.Angle(radius, u.deg) # If radius is just a number we assume degrees adql_region = f"CIRCLE('ICRS', {coordinates.ra.deg}, {coordinates.dec.deg}, {radius.to(u.deg).value})" @@ -387,15 +421,47 @@ def query_criteria(self, collection=None, *, catalog=None, coordinates=None, reg return result_table @class_or_instance - @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' - 'will be removed in a future release. Please use `collection` and `catalog` instead.') - @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' - 'and will be removed in a future release. Please use `limit` instead.') - @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' - 'and will be removed in a future release. Please use `offset` instead.') - def query_region(self, coordinates=None, *, radius=0.2*u.deg, region=None, collection=None, - catalog=None, limit=5000, offset=0, count_only=False, select_cols=None, - sort_by=None, sort_desc=False, filters={}, version=None, pagesize=None, page=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + def query_region( + self, + coordinates=None, + *, + radius=0.2 * u.deg, + region=None, + collection=None, + catalog=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ Query for MAST catalog entries within a specified region using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -470,31 +536,65 @@ def query_region(self, coordinates=None, *, radius=0.2*u.deg, region=None, colle # Parse pagination params limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) - return self.query_criteria(collection=collection, - catalog=catalog, - coordinates=coordinates, - region=region, - radius=radius, - limit=limit, - offset=offset, - count_only=count_only, - select_cols=select_cols, - sort_by=sort_by, - sort_desc=sort_desc, - filters=filters, - **criteria) + return self.query_criteria( + collection=collection, + catalog=catalog, + coordinates=coordinates, + region=region, + radius=radius, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) @class_or_instance - @deprecated_renamed_argument('version', None, since='0.4.12', message='The `version` argument is deprecated and ' - 'will be removed in a future release. Please use `collection` and `catalog` instead.') - @deprecated_renamed_argument('pagesize', None, since='0.4.12', message='The `pagesize` argument is deprecated ' - 'and will be removed in a future release. Please use `limit` instead.') - @deprecated_renamed_argument('page', None, since='0.4.12', message='The `page` argument is deprecated ' - 'and will be removed in a future release. Please use `offset` instead.') - @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') - def query_object(self, object_name, *, radius=0.2*u.deg, collection=None, catalog=None, resolver=None, - limit=5000, offset=0, count_only=False, select_cols=None, sort_by=None, sort_desc=False, - filters={}, version=None, pagesize=None, page=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_object( + self, + object_name, + *, + radius=0.2 * u.deg, + collection=None, + catalog=None, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ Query for MAST catalog entries around a specified object name using criteria filters. To return columns for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. @@ -559,19 +659,21 @@ def query_object(self, object_name, *, radius=0.2*u.deg, collection=None, catalo # Parse pagination params limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) - return self.query_criteria(collection=collection, - catalog=catalog, - object_name=object_name, - radius=radius, - resolver=resolver, - limit=limit, - offset=offset, - count_only=count_only, - select_cols=select_cols, - sort_by=sort_by, - sort_desc=sort_desc, - filters=filters, - **criteria) + return self.query_criteria( + collection=collection, + catalog=catalog, + object_name=object_name, + radius=radius, + resolver=resolver, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) @class_or_instance @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) diff --git a/astroquery/mast/tests/data/tap_collections.json b/astroquery/mast/tests/data/tap_collections.json index c1a95bf879..73509c6a97 100644 --- a/astroquery/mast/tests/data/tap_collections.json +++ b/astroquery/mast/tests/data/tap_collections.json @@ -4,11 +4,6 @@ "title": "tap_search", "version": "0.1" }, - "servers": [ - { - "url": "/vo-tap/api/v0.1" - } - ], "paths": { "/{catalog}/": { "get": { @@ -1558,7 +1553,7 @@ "schemas": { "Body_TAPAsyncHandler_post__catalog__async_post": { "properties": { - "lang": { + "LANG": { "anyOf": [ { "$ref": "#/components/schemas/LangName" @@ -1576,25 +1571,25 @@ "description": "Content type of the response", "default": "votable" }, - "param_format": { + "FORMAT": { "$ref": "#/components/schemas/EncodingEnum", "description": "Content type of the response", "default": "votable" }, - "query": { + "QUERY": { "type": "string", "title": "Query", "description": "Query to perform", "default": "" }, - "maxrec": { + "MAXREC": { "type": "integer", "minimum": 0.0, "title": "Maxrec", "description": "Maximum number of records to be returned (max 100k)", "default": 100000 }, - "runid": { + "RUNID": { "anyOf": [ { "type": "string", @@ -1608,7 +1603,7 @@ "description": "RUNID for the request", "default": "" }, - "phase": { + "PHASE": { "anyOf": [ { "type": "string", @@ -1630,7 +1625,7 @@ }, "Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post": { "properties": { - "destruction": { + "DESTRUCTION": { "type": "string", "format": "date-time", "title": "Destruction", @@ -1642,10 +1637,10 @@ }, "Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post": { "properties": { - "execution_duration": { + "EXECUTIONDURATION": { "type": "integer", "minimum": 0.0, - "title": "Execution Duration", + "title": "Executionduration", "description": "Requested execution duration in seconds", "default": 600 } @@ -1655,7 +1650,7 @@ }, "Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post": { "properties": { - "phase": { + "PHASE": { "anyOf": [ { "$ref": "#/components/schemas/PhaseAction" @@ -1666,7 +1661,7 @@ ], "description": "Execution Phase" }, - "destruction": { + "DESTRUCTION": { "anyOf": [ { "type": "string", @@ -1679,7 +1674,7 @@ "title": "Destruction", "description": "ISO 8601 UTC datetime for proposed job destruction" }, - "action": { + "ACTION": { "anyOf": [ { "type": "string" @@ -1697,20 +1692,20 @@ }, "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post": { "properties": { - "phase": { + "PHASE": { "$ref": "#/components/schemas/PhaseAction", "description": "Execution Phase" } }, "type": "object", "required": [ - "phase" + "PHASE" ], "title": "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post" }, "Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post": { "properties": { - "query": { + "QUERY": { "anyOf": [ { "type": "string" @@ -1722,7 +1717,7 @@ "title": "Query", "description": "Query to be executed" }, - "lang": { + "LANG": { "anyOf": [ { "$ref": "#/components/schemas/LangName" @@ -1733,7 +1728,7 @@ ], "description": "Query langauge to be used." }, - "responseformat": { + "RESPONSEFORMAT": { "anyOf": [ { "$ref": "#/components/schemas/EncodingEnum" @@ -1744,7 +1739,7 @@ ], "description": "Content type of the response" }, - "param_format": { + "FORMAT": { "anyOf": [ { "$ref": "#/components/schemas/EncodingEnum" @@ -1755,7 +1750,7 @@ ], "description": "Content type of the response" }, - "maxrec": { + "MAXREC": { "anyOf": [ { "type": "integer" @@ -1767,7 +1762,7 @@ "title": "Maxrec", "description": "Maximum number of records to return" }, - "runid": { + "RUNID": { "anyOf": [ { "type": "string" @@ -1785,7 +1780,7 @@ }, "Body_TAPHandler_post__catalog__sync_post": { "properties": { - "lang": { + "LANG": { "$ref": "#/components/schemas/LangName", "description": "Query language to be used for this request" }, @@ -1794,17 +1789,17 @@ "description": "Content type of the response", "default": "votable" }, - "param_format": { + "format": { "$ref": "#/components/schemas/EncodingEnum", "description": "Content type of the response", "default": "votable" }, - "query": { + "QUERY": { "type": "string", "title": "Query", "description": "Query to perform" }, - "maxrec": { + "MAXREC": { "type": "integer", "minimum": 0.0, "title": "Maxrec", @@ -1827,8 +1822,8 @@ }, "type": "object", "required": [ - "lang", - "query" + "LANG", + "QUERY" ], "title": "Body_TAPHandler_post__catalog__sync_post" }, @@ -1838,16 +1833,14 @@ "caom", "classy", "gaiadr3", - "greenplum", "hsc", "hscv2", + "mast_catalogs", "missionmast", "ps1dr1", "ps1dr2", - "ps1_dr2", "registry", - "roman", - "romanspectra", + "roman_catalogs", "skymapperdr4", "tic", "ullyses", @@ -1903,7 +1896,7 @@ "type": "string", "enum": [ "ADQL", - "ADQL-2.0" + "ADQL-2.1" ], "title": "LangName", "description": "Enum for language names as param" @@ -1940,6 +1933,13 @@ "type": { "type": "string", "title": "Error Type" + }, + "input": { + "title": "Input" + }, + "ctx": { + "type": "object", + "title": "Context" } }, "type": "object", @@ -1951,5 +1951,10 @@ "title": "ValidationError" } } - } + }, + "servers": [ + { + "url": "/vo-tap/api/v0.1" + } + ] } \ No newline at end of file diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 44bc62473f..71c23971f6 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,6 +4,7 @@ import os import re import warnings +from datetime import datetime from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, patch @@ -11,14 +12,16 @@ import astropy.units as u import numpy as np import pytest -from astropy.coordinates import SkyCoord +from astropy.coordinates import Angle, SkyCoord from astropy.io import fits from astropy.io.votable import parse from astropy.table import Table, unique +from astropy.time import Time from astropy.utils.exceptions import AstropyDeprecationWarning from pyvo.dal import TAPResults from pyvo.dal.exceptions import DALQueryError from pyvo.io.vosi import parse_capabilities +from regions import CirclePixelRegion, CircleSkyRegion, PixCoord, PolygonSkyRegion from requests import HTTPError, Response from astroquery.exceptions import ( @@ -47,6 +50,8 @@ from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse +from ..catalog_collection import DEFAULT_CATALOGS, CatalogCollection, CatalogMetadata + try: # Optional dependency import for cloud access functionality from botocore.exceptions import ClientError @@ -122,6 +127,33 @@ def patch_post(request): return mp +@pytest.fixture +def patch_tap(request, reset_catalogs_cache): + mp = request.getfixturevalue("monkeypatch") + + mock_tap = vo_tap_mock() + mp.setattr( + 'astroquery.mast.catalog_collection.TAPService', + lambda *args, **kwargs: mock_tap + ) + # We have to set this because CatalogsClass uses a simple request to get collections + mp.setattr(utils, '_simple_request', request_mockreturn) + + return mock_tap + + +@pytest.fixture +def reset_catalogs_cache(): + Catalogs._collections_cache.clear() + yield + + +def get_patch_tap_query(patch_tap): + args, _ = patch_tap.run_sync.call_args + query = args[0] + return query + + @pytest.fixture() def patch_boto3(monkeypatch, reset_cloud_state): """Fixture to patch boto3 client and resource for cloud access tests.""" @@ -148,12 +180,6 @@ def reset_cloud_state(): Observations._cloud_enabled_explicitly = None -@pytest.fixture -def reset_catalogs_cache(): - Catalogs._collections_cache.clear() - yield - - def post_mockreturn(self, method="POST", url=None, data=None, timeout=10, **kwargs): if "columnsconfig" in url: if "Mast.Catalogs.Tess.Cone" in data: @@ -293,6 +319,13 @@ def run_sync_mock(query, **kwargs): # Queries with results, keep in mind this is not meaninful and results won't match the query filename = data_path(DATA_FILES['tap_results']) votable = parse(filename) + + if 'empty' in query: + # Simulate a query that returns no results by clearing the resources in the votable + for resource in votable.resources: + for table in resource.tables: + table.array = np.array([]) # Clear the data to simulate no results + return TAPResults(votable) # Mock TAPService @@ -1443,171 +1476,1009 @@ def test_observations_disable_cloud_dataset(patch_boto3): assert Observations._cloud_enabled_explicitly is False -###################### -# CatalogClass tests # -###################### +####################### +# CatalogsClass tests # +####################### -def test_catalogs_query_region_async(): - responses = Catalogs.query_region_async(regionCoords, radius=0.002) - assert isinstance(responses, list) +def test_catalogs_attributes(patch_tap): + Catalogs.query_criteria( + collection="tic", + region="Circle ICRS 202.4656816 +47.1999842 0.04", + radius=0.002 * u.deg, + offset=1, + sort_by="ra", + ) + assert isinstance(Catalogs.collection, CatalogCollection) + assert Catalogs.catalog == "dbo.SumMagAper2CatView" -def test_catalogs_fabric_query_region_async(): - responses = Catalogs.query_region_async(regionCoords, radius=0.002, catalog="panstarrs", table="mean") - assert isinstance(responses, MockResponse) +def test_catalogs_get_catalogs(patch_tap): + catalogs = Catalogs.get_catalogs("tic") + assert isinstance(catalogs, Table) -def test_catalogs_query_region(): - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg) - assert isinstance(result, Table) +def test_catalogs_get_column_metadata(patch_tap): + metadata = Catalogs.get_column_metadata(collection="tic") + assert isinstance(metadata, Table) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="hsc", version=2) - assert isinstance(result, Table) - with pytest.warns(InputWarning) as i_w: - Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="hsc", version=5) - assert "Invalid HSC version number" in str(i_w[0].message) +def test_catalogs_query_criteria(patch_tap): + # Coordinates + result = Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + limit=2, + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="galex") assert isinstance(result, Table) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "TOP 2" in query + + # Region query + result = Catalogs.query_criteria( + collection="tic", + region="Circle ICRS 202.4656816 +47.1999842 0.04", + radius=0.002 * u.deg, + offset=1, + sort_by="ra", + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=2) assert isinstance(result, Table) - - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=1) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "OFFSET" in query + assert "ORDER BY" in query + + # Non-positional query + result = Catalogs.query_criteria( + collection="tic", + filters={"pmra": [">-10", "<10"]}, + ) assert isinstance(result, Table) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "WHERE" in query + assert "pmra > -10" in query + assert "pmra < 10" in query + + +def test_catalogs_invalid_query_criteria(patch_tap): + # Specifying both region and coordinates + with pytest.raises(InvalidQueryError, match="Specify either `region` or `coordinates`"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + region="Circle ICRS 202.4656816 +47.1999842 0.04" + ) - with pytest.warns(InputWarning) as i_w: - Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=5) - assert "Invalid Gaia version number" in str(i_w[0].message) + # Specifying both region and object_name + with pytest.raises(InvalidQueryError, match="Specify either `region` or `object_name`"): + Catalogs.query_criteria( + collection="tic", + object_name="M31", + region="Circle ICRS 202.4656816 +47.1999842 0.04" + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="Sample") - assert isinstance(result, Table) + # Named parameters and filters dict specifying criteria + with pytest.raises(InvalidQueryError, match="Criteria specified both"): + Catalogs.query_criteria( + collection="tic", + object_name="M31", + file_suffix=['A', 'B', '!C'], + filters={"file_suffix": ['A', 'B', '!C']} + ) + # sort_by cols and sort_desc different lengths + with pytest.raises(InvalidQueryError, match="must be 1 or equal to length of 'sort_by'"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + sort_by=["ra", "dec"], + sort_desc=[True, False, True] + ) -def test_catalogs_fabric_query_region(): - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="panstarrs", table="mean") - assert isinstance(result, Table) + # Invalid sort col + with pytest.raises(InvalidQueryError, match="Filter 'fake' is not recognized"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + sort_by="fake", + ) + # Invalid filter + with pytest.raises(InvalidQueryError, match="Filter 'fake' is not recognized"): + Catalogs.query_criteria( + collection="tic", + fake=1 + ) -def test_catalogs_query_object_async(): - responses = Catalogs.query_object_async("M101", radius="0.002 deg") - assert isinstance(responses, list) + # Collection is not a string + with pytest.raises(InvalidQueryError, match="Collection name must be a string."): + Catalogs.query_criteria( + collection=123, + objtype="star" + ) + # Warn if result table is empty + with pytest.warns(NoResultsWarning, match="The query returned no results."): + Catalogs.query_criteria( + collection="tic", + objtype="empty" + ) -def test_catalogs_fabric_query_object_async(): - responses = Catalogs.query_object_async("M101", radius="0.002 deg", catalog="panstarrs", table="mean") - assert isinstance(responses, MockResponse) +def test_catalogs_query_region(patch_tap): + # Passing region coords and radius + result = Catalogs.query_region( + regionCoords, + radius=0.002 * u.deg, + collection="tic" + ) -def test_catalogs_query_object(): - result = Catalogs.query_object("M101", radius=".002 deg") assert isinstance(result, Table) + assert len(result) > 0 + assert "ra" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "POINT" in query + + +def test_catalogs_invalid_query_region(): + # Query without region or coordinates + with pytest.raises(InvalidQueryError, match="Must specify either `region` or `coordinates`"): + Catalogs.query_region( + collection="tic", + ) + + # Query with unsupported region type + with pytest.raises(InvalidQueryError, match="does not support ADQL region type 'POLYGON'"): + Catalogs.query_region( + region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842", + collection="tic" + ) + +def test_catalogs_query_object(patch_tap): + # Object and radius query + radius = .001 + result = Catalogs.query_object( + "M10", + radius=radius, + collection="TIC" + ) -def test_catalogs_fabric_query_object(): - result = Catalogs.query_object("M101", radius=".002 deg", catalog="panstarrs", table="mean") assert isinstance(result, Table) + assert len(result) > 0 + assert "ra" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "POINT" in query + assert str(radius) in query + + +def test_catalogs_init_with_catalog(patch_tap): + catalog = Catalogs( + collection="tic", + catalog="tap_schema.schemas" + ) + assert catalog.catalog == "tap_schema.schemas" -def test_catalogs_query_criteria_async(): - responses = Catalogs.query_criteria_async(catalog="Tic", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) +def test_catalogs_setting_catalog(patch_tap): + catalog = Catalogs( + collection="tic", + catalog="tap_schema.schemas" + ) + catalog.catalog = "tap_schema.key_columns" + assert catalog.catalog == "tap_schema.key_columns" - responses = Catalogs.query_criteria_async(catalog="Ctl", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) - responses = Catalogs.query_criteria_async(catalog="Tic", object_name="M10", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) +def test_catalogs_get_collections_cached(patch_tap): + catalog = Catalogs("tic") + collections = catalog.get_collections() - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - object_name="M10", radius=2, - state="complete") - assert isinstance(responses, list) + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames - responses = Catalogs.query_criteria_async(catalog="panstarrs", object_name="M10", radius=2, - table="mean", qualityFlag=48) - assert isinstance(responses, MockResponse) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="Tic") - assert "non-positional" in str(invalid_query.value) +def test_catalogs_supports_spatial_queries(patch_tap): + catalog = Catalogs() + result = catalog.supports_spatial_queries( + collection="tic", + catalog="tap_schema.schemas" + ) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="SampleFail") - assert "query not available" in str(invalid_query.value) + assert isinstance(result, bool) + assert result + + +def test_catalogs_verify_collection(patch_tap): + valid = Catalogs._verify_collection("tic") + assert valid.lower() == "tic" + + # Renamed collection + renamed = list(Catalogs._renamed_collections.keys())[0] + new_name = Catalogs._renamed_collections[renamed] + with pytest.warns(InputWarning, match="has been renamed"): + result = Catalogs._verify_collection(renamed) + assert result == new_name + + # Invalid collection + with pytest.raises(InvalidQueryError, match="is not recognized"): + Catalogs._verify_collection("FAKE") + + # No longer supported collection + if Catalogs._no_longer_supported_collections: + unsupported = list(Catalogs._no_longer_supported_collections)[0] + with pytest.raises(InvalidQueryError) as exc: + Catalogs._verify_collection(unsupported) + assert "no longer supported" in str(exc.value) + + +def test_catalogs_parse_inputs(patch_tap): + collection_name = Catalogs.available_collections[0] + collection_obj, catalog = Catalogs._parse_inputs(collection=collection_name, catalog=None) + assert isinstance(collection_obj, CatalogCollection) + assert collection_obj.name == collection_name + assert catalog == collection_obj.default_catalog + + # Catalog parameter warning + with pytest.warns(DeprecationWarning, match="via the `catalog` parameter is deprecated."): + collection_name = Catalogs.available_collections[0] + collection_obj, catalog = Catalogs._parse_inputs(collection=None, catalog=collection_name) + assert isinstance(collection_obj, CatalogCollection) + assert catalog == collection_obj.default_catalog + + # Use catalog attribute if valid for collection + Catalogs.catalog = "dbo.catalogrecord" + collection_obj, catalog = Catalogs._parse_inputs(collection="tic") + assert collection_obj.name == "tic" + assert catalog == "dbo.catalogrecord" + + +def test_catalogs_parse_select_cols(patch_tap): + catalog = Catalogs("tic") + column_metadata = catalog.get_column_metadata() + result = Catalogs._parse_select_cols( + ["ra", "dec"], + column_metadata) + assert result == "ra, dec" + + # Close match suggestion + close_match_col = "gaiagaiabp" + with pytest.warns(InputWarning, match=" not found in catalog. Did you mean"): + result = Catalogs._parse_select_cols( + ["ra", close_match_col], + column_metadata + ) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="panstarrs", object_name="M10", coordinates=regionCoords, - objType="STAR") - assert "one of object_name and coordinates" in str(invalid_query.value) + # Empty columns + with pytest.raises(InvalidQueryError, match="No valid columns specified in `select_cols`"): + result = Catalogs._parse_select_cols( + [], + column_metadata + ) -def test_catalogs_query_criteria(): - # without position - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], objType="STAR") +def test_catalogs_parse_legacy_pagination(patch_tap): + catalog = Catalogs("tic") + limit, offset = catalog._parse_legacy_pagination( + limit=5000, + offset=0, + pagesize=10, + page=None, + ) + assert limit == 10 + assert offset == 0 + + # Missing pagesize + with pytest.warns(InputWarning, match="The 'page' parameter is ignored without 'pagesize'."): + catalog._parse_legacy_pagination( + limit=5000, + offset=0, + pagesize=None, + page=2, + ) - assert isinstance(result, Table) - result = Catalogs.query_criteria(catalog="Ctl", - Bmag=[30, 50], objType="STAR") +def test_catalogs_create_adql_region(patch_tap): + # String regions + adql_region = Catalogs._create_adql_region( + region="Circle 202.4656816 +47.1999842 0.2" + ) + assert adql_region == "CIRCLE('ICRS',202.4656816,+47.1999842,0.2)" - assert isinstance(result, Table) + adql_region = Catalogs._create_adql_region( + region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842" + ) + assert adql_region == "POLYGON('ICRS',202.4656816,+47.1999842,202.5656816,+47.2999842,202.3656816,+47.0999842)" - # with position - result = Catalogs.query_criteria(catalog="DiskDetective", - object_name="M10", radius=2, - state="complete") - assert isinstance(result, Table) + adql_region = Catalogs._create_adql_region( + region="Polygon 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842" + ) + assert adql_region == "POLYGON('ICRS',202.4656816,+47.1999842,202.5656816,+47.2999842,202.3656816,+47.0999842)" - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria(catalog="Tic", object_name="M10") - assert "non-positional" in str(invalid_query.value) + # Astropy region objects + cone_region = CircleSkyRegion( + center=SkyCoord(10.8, 6.5, unit="deg"), + radius=Angle(0.5, unit="deg") + ) + adql_region = Catalogs._create_adql_region(region=cone_region) + assert adql_region == "CIRCLE('ICRS',10.8,6.5,0.5)" + + polygon_region = PolygonSkyRegion( + SkyCoord( + [57.376, 56.391, 56.025, 56.616], + [24.053, 24.622, 24.049, 24.291], + frame="icrs", + unit="deg", + ) + ) + adql_region = Catalogs._create_adql_region(region=polygon_region) + assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" + + # Iterable coord pairs + adql_region = Catalogs._create_adql_region( + region=[ + (57.376, 24.053), + (56.391, 24.622), + (56.025, 24.049), + (56.616, 24.291) + ] + ) + assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" -def test_catalogs_query_hsc_matchid_async(): - responses = Catalogs.query_hsc_matchid_async(82371983) - assert isinstance(responses, list) +def test_catalogs_invalid_create_adql_region(patch_tap): + # Polygon without points + with pytest.raises(InvalidQueryError, match="Invalid POLYGON region string"): + Catalogs._create_adql_region(region="Polygon ICRS") - responses = Catalogs.query_hsc_matchid_async(82371983, version=2) - assert isinstance(responses, list) + # Polygon without sufficient points + with pytest.raises(InvalidQueryError, match="Invalid POLYGON region string"): + Catalogs._create_adql_region(region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842") + + # Missing circle spec, frame not specified + with pytest.raises(InvalidQueryError, match="Invalid CIRCLE region string"): + Catalogs._create_adql_region(region="CIRCLE 202.4656816 +47.1999842") + + # Missing circle spec, frame specified + with pytest.raises(InvalidQueryError, match="Invalid CIRCLE region string"): + Catalogs._create_adql_region(region="CIRCLE ICRS 202.4656816 +47.1999842") + + # Invalid region str + with pytest.raises(InvalidQueryError, match="Unrecognized region string"): + Catalogs._create_adql_region(region="Badshape ICRS 202.4656816 +47.1999842 0.04") + + # Invalid list of coord pairs + with pytest.raises(InvalidQueryError, match="Invalid iterable region format"): + Catalogs._create_adql_region( + region=[57.376, 24.053, 56.391, 24.622, 56.025, 24.049, 56.616, 24.291] + ) + + # Invalid astropy region + with pytest.raises(TypeError, match="Unsupported region type"): + Catalogs._create_adql_region( + region=CirclePixelRegion(PixCoord(x=42, y=43), 4.2) + ) + + +def test_catalogs_parse_numeric_expression(patch_tap): + # Handling between + predicate = Catalogs._parse_numeric_expr("dec", "5..10") + assert predicate == "dec BETWEEN 5 AND 10" + + # Handling inequalities + predicate = Catalogs._parse_numeric_expr("teff", "<1") + assert predicate == "teff < 1" + + # Handling specific values + predicate = Catalogs._parse_numeric_expr("gaiabp", "1") + assert predicate == "gaiabp = 1.0" + + # Passing not numeric str + with pytest.raises(InvalidQueryError, match="is numeric; unsupported value"): + Catalogs._parse_numeric_expr("dec", "notnumeric") + + +def test_catalogs_parse_temporal_expression(patch_tap): + # Handling between + predicate = Catalogs._parse_temporal_expr("time", "2024-01-01..2024-12-31") + assert predicate == "time BETWEEN '2024-01-01 00:00:00' AND '2024-12-31 00:00:00'" + + # Handling inequalities + predicate = Catalogs._parse_temporal_expr("datetime", ">=2020-06-01") + assert predicate == "datetime >= '2020-06-01 00:00:00'" + + # Handling specific values + predicate = Catalogs._parse_temporal_expr("obs_time", "2025-04-01 12:00:00") + assert predicate == "obs_time BETWEEN '2025-04-01 12:00:00' AND '2025-04-01 12:00:01'" + + # Passing not datetime str + predicate = Catalogs._parse_temporal_expr("dec", "notdatetime") + assert predicate == "dec = 'notdatetime'" + + # Handling microseconds + predicate = Catalogs._parse_temporal_expr("datetime", ">=2020-06-01 10:00:00.0001") + assert predicate == "datetime >= '2020-06-01 10:00:00'" + + # Handling year-only str + predicate = Catalogs._parse_temporal_expr("time", "2025") + assert predicate == "time = '2025'" + + # Handling date str + predicate = Catalogs._parse_temporal_expr("time", "2020-08-01") + assert predicate == "time BETWEEN '2020-08-01 00:00:00' AND '2020-08-01 00:00:01'" + + # Handling astropy Time + predicate = Catalogs._parse_temporal_expr("obs_time", Time('2000-01-01 12:30:00')) + assert predicate == "obs_time BETWEEN '2000-01-01 12:30:00' AND '2000-01-01 12:30:01'" + + # Handling datetime + predicate = Catalogs._parse_temporal_expr("obs_time", datetime(2024, 6, 1, 8, 0, 1)) + assert predicate == "obs_time BETWEEN '2024-06-01 08:00:01' AND '2024-06-01 08:00:02'" + + +def test_catalogs_format_scalar_predicate(patch_tap): + # Handling bool + predicate = Catalogs._format_scalar_predicate( + "var", True + ) + assert predicate == 'var = 1' + + # Handling num + predicate = Catalogs._format_scalar_predicate( + "e_lum", 1, is_numeric=True + ) + assert predicate == 'e_lum = 1' - with pytest.warns(InputWarning) as i_w: + # Handling str(num) + predicate = Catalogs._format_scalar_predicate( + "e_lum", "1", is_numeric=True + ) + assert predicate == 'e_lum = 1.0' + + # Handling ! nots + predicate = Catalogs._format_scalar_predicate( + "e_lum", "!1", is_numeric=True + ) + assert predicate == 'NOT (e_lum = 1.0)' + + # Handling wildcard * + predicate = Catalogs._format_scalar_predicate( + "var", "WILDCARD*" + ) + assert predicate == "var LIKE 'WILDCARD%'" + + # Handling wildcard * and nots + predicate = Catalogs._format_scalar_predicate( + "var", "!WILDCARD*" + ) + assert predicate == "NOT (var LIKE 'WILDCARD%')" + + # Handling wildcard % + predicate = Catalogs._format_scalar_predicate( + "var", "WILDCARD%" + ) + assert predicate == "var LIKE 'WILDCARD%'" + + # Handling wildcard % and nots + predicate = Catalogs._format_scalar_predicate( + "var", "!WILDCARD%" + ) + assert predicate == "NOT (var LIKE 'WILDCARD%')" + + # Handling temporal + predicate = Catalogs._format_scalar_predicate( + "time", "!1", is_temporal=True + ) + assert predicate == "NOT (time = '1')" + + +def test_catalogs_combine_predicates(patch_tap): + # No predicates + result = Catalogs._combine_predicates([], []) + assert result == "" + + # One positive predicate + result = Catalogs._combine_predicates(["ra > 5"], []) + assert result == "ra > 5" + + # Multiple positive predicate + result = Catalogs._combine_predicates( + ["ra > 5", "dec < 0"], + [] + ) + assert result == "(ra > 5 OR dec < 0)" + + # Multiple negative predicates + result = Catalogs._combine_predicates( + [], + ["ra != 5", "dec != 0"] + ) + assert result == "ra != 5 AND dec != 0" + + # Multiple positive and negative predicates + result = Catalogs._combine_predicates( + ["ra > 5", "dec < 0"], + ["ra != 10"] + ) + assert result == "(ra != 10) AND (ra > 5 OR dec < 0)" + + +def test_catalogs_build_numeric_list_predicate(patch_tap): + # Multiple positive nums + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[1, 2, 3], + neg_items=[] + ) + assert result == "ra IN (1, 2, 3)" + + # Multiple positive bools + result = Catalogs._build_numeric_list_predicate( + "tessflag", + pos_items=[True, False], + neg_items=[] + ) + assert result == "tessflag IN (1, 0)" + + # Multiple positive inequalities and ranges + result = Catalogs._build_numeric_list_predicate( + "dec", + pos_items=["<5", "10..20"], + neg_items=[] + ) + assert "< 5" in result + assert "BETWEEN 10 AND 20" in result + + # Single positive num + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[np.int64(7)], + neg_items=[] + ) + assert "ra IN (7.0)" in result + + # Positive and negative nums + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[1, 2], + neg_items=[">5"] + ) + assert "ra IN (1, 2)" in result + assert "ra > 5" in result + assert "AND" in result + + # Unsupported numeric value type + with pytest.raises(InvalidQueryError, match="Unsupported numeric value type"): + Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[{"not": "a number"}], + neg_items=[] + ) + + +def test_catalogs_build_string_list_predicate(patch_tap): + # Multiple positive strs + result = Catalogs._build_string_list_predicate( + "var", + pos_items=["str", "str"], + neg_items=[] + ) + assert result == "var IN ('str', 'str')" + + # Multiple positive bools + result = Catalogs._build_string_list_predicate( + "var", + pos_items=[True, False], + neg_items=[] + ) + assert result == "var IN (1, 0)" + + # Multiple positive strs with wildcard + result = Catalogs._build_string_list_predicate( + "var", + pos_items=["WILDCARD%", "str"], + neg_items=[] + ) + assert "var IN ('str')" in result + assert "LIKE 'WILDCARD%" in result + assert "OR" in result + + # Multiple positive nums + result = Catalogs._build_string_list_predicate( + "var", + pos_items=[1, 0], + neg_items=[] + ) + assert result == "var IN (1, 0)" + + +def test_catalogs_build_temporal_list_predicate(patch_tap): + # Multiple positive temps + result = Catalogs._build_temporal_list_predicate( + "var", + pos_items=["<2020-06-01", ">2025-06-01"], + neg_items=[] + ) + assert result == "(var < '2020-06-01 00:00:00' OR var > '2025-06-01 00:00:00')" + + # Multiple positive inequalities and ranges + result = Catalogs._build_temporal_list_predicate( + "var", + pos_items=[">2025-06-01", "2020-06-01..2020-12-31"], + neg_items=[] + ) + assert "var > '2025-06-01 00:00:00'" in result + assert "BETWEEN '2020-06-01 00:00:00' AND '2020-12-31 00:00:00'" in result + + +def test_catalogs_format_criteria_conditions(patch_tap): + # Multiple numeric cols and singular criteria + criteria = {"ra": 5, "dec": 10} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["ra = 5", "dec = 10"] + + # Str cols and singular criteria + criteria = {"obj_type": "STAR"} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["obj_type = 'STAR'"] + + # Multiple cols and multiple criteria + criteria = {"ra": [1, 2, 3], "dec": [">5", "<10"]} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert any("ra IN" in r or "ra >" in r for r in result) + assert any("dec <" in r or "dec >" in r for r in result) + + # Str cols and multiple criteria + criteria = {"obj_type": ["STAR", "!GALAXY"]} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert any("NOT" in r for r in result) + assert any("STAR" in r for r in result) + + # Empty criteria + criteria = {"ra": []} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["1=0"] + + +def test_catalogs_invalid_tap_query(patch_tap): + # This will trigger a DALQueryError in the mock TAP service + # when 'invalid' is found in the query string + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + allwise='invalid' + ) + + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_region( + regionCoords, + radius=0.002 * u.deg, + collection="tic", + allwise='invalid' + ) + + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_object( + "M10", + radius=.001, + collection="TIC", + allwise='invalid' + ) + + +def test_catalogs_invalid_spatial_query(patch_tap): + # Force spatial query to fail + patch_tap.search = MagicMock(side_effect=DALQueryError("spatial failed")) + with pytest.raises(InvalidQueryError, match="does not support spatial queries"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + ) + + +def test_catalogs_query_hsc_matchid_async(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.query_hsc_matchid_async(82371983) + assert isinstance(responses, list) + + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.query_hsc_matchid_async(82371983, version=2) + assert isinstance(responses, list) + + with pytest.warns((AstropyDeprecationWarning, InputWarning)) as record: Catalogs.query_hsc_matchid_async(82371983, version=5) - assert "Invalid HSC version number" in str(i_w[0].message) + messages = [str(w.message) for w in record] + assert any("This function is deprecated" in m for m in messages) + assert any("Invalid HSC version number" in m for m in messages) -def test_catalogs_query_hsc_matchid(): - result = Catalogs.query_hsc_matchid(82371983) - assert isinstance(result, Table) +def test_catalogs_query_hsc_matchid(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + result = Catalogs.query_hsc_matchid(82371983) + assert isinstance(result, Table) -def test_catalogs_get_hsc_spectra_async(): - responses = Catalogs.get_hsc_spectra_async() - assert isinstance(responses, list) +def test_catalogs_get_hsc_spectra_async(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.get_hsc_spectra_async() + assert isinstance(responses, list) -def test_catalogs_get_hsc_spectra(): - result = Catalogs.get_hsc_spectra() - assert isinstance(result, Table) +def test_catalogs_get_hsc_spectra(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + result = Catalogs.get_hsc_spectra() + assert isinstance(result, Table) -def test_catalogs_download_hsc_spectra(tmpdir): - allSpectra = Catalogs.get_hsc_spectra() +def test_catalogs_download_hsc_spectra(patch_post, tmpdir): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + allSpectra = Catalogs.get_hsc_spectra() - # actually download the products - result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) - assert isinstance(result, Table) + # Actually download the products + result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) + assert isinstance(result, Table) + + # Just get the curl script + result = Catalogs.download_hsc_spectra(allSpectra[20:24], + download_dir=str(tmpdir), curl_flag=True) + assert isinstance(result, Table) + + +########################### +# CatalogCollection tests # +########################### + + +def test_catalog_collection_discover_collections(patch_tap): + collections = CatalogCollection.discover_collections() + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames + assert "parent_collection" in collections.colnames + + +def test_catalog_collection_get_parent_collection(patch_tap): + parent = CatalogCollection.get_parent_collection("tic") + assert parent == "tic" + + # Error if collection not a string + with pytest.raises(InvalidQueryError): + CatalogCollection.get_parent_collection(123) + + # Error if collection not found + with pytest.raises(InvalidQueryError, match="Collection 'fake' not found"): + CatalogCollection.get_parent_collection("fake") + + +def test_catalog_collection_tap_get_catalog_metadata(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + default_metadata = cc.get_catalog_metadata(default_catalog) + assert isinstance(default_metadata, CatalogMetadata) + assert isinstance(default_metadata.column_metadata, Table) + assert isinstance(default_metadata.ra_column, str) + assert isinstance(default_metadata.dec_column, str) + assert isinstance(default_metadata.supports_spatial_queries, bool) + + assert len(default_metadata.column_metadata) > 0 + assert default_metadata.ra_column in default_metadata.column_metadata["column_name"] + assert default_metadata.dec_column in default_metadata.column_metadata["column_name"] + + metadata_cached = cc.get_catalog_metadata(default_catalog) + assert default_metadata is metadata_cached + + +def test_catalog_collection_get_default_catalog(patch_tap): + cc = CatalogCollection("tic") + catalogs = cc._fetch_catalogs() + default = cc.get_default_catalog() + + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + assert catalogs.colnames == ['catalog_name', 'description'] + + assert not default.startswith("tap_schema") + assert default.casefold() in [name.casefold() for name in catalogs["catalog_name"]] + assert DEFAULT_CATALOGS["tic"] == default + + # First non-tap_schema + cc = CatalogCollection("tic") + cc.name = "fake" + fake_catalogs = Table({ + "catalog_name": [ + "tap_schema.tables", + "tap_schema.columns", + "real_catalog", + "real_catalog_2", + ], + "description": ["", "", "", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + + default = cc.get_default_catalog() + assert default == "real_catalog" + + # All are tap_schema + cc = CatalogCollection("tic") + cc.name = "fake" + fake_catalogs = Table({ + "catalog_name": [ + "tap_schema.tables", + "tap_schema.columns", + ], + "description": ["", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + + default = cc.get_default_catalog() + assert default == "tap_schema.tables" + + +def test_catalog_collection_run_tap_query(patch_tap): + cc = CatalogCollection("tic") + + adql_str = ( + "SELECT TOP 10 solution_id, designation, source_id, ra, dec " + "FROM gaia_source WHERE " + "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + ) + result = cc.run_tap_query(adql_str) - # just get the curl script - result = Catalogs.download_hsc_spectra(allSpectra[20:24], - download_dir=str(tmpdir), curl_flag=True) assert isinstance(result, Table) + assert len(result) > 0 + + query = get_patch_tap_query(patch_tap) + assert adql_str in query + + +def test_catalog_collection_invalid_run_tap_query(patch_tap): + cc = CatalogCollection("tic") + with pytest.raises(InvalidQueryError, match="TAP query failed for collection 'tic'"): + adql_str = "invalid" + cc.run_tap_query(adql_str) + + +def test_catalog_collection_grouped_fetch_catalogs(patch_tap): + name = "dbo" + cc = CatalogCollection(name) + _ = cc._fetch_catalogs() + query = get_patch_tap_query(patch_tap) + assert f"WHERE table_name LIKE '{name}" in query + + +def test_catalog_collection_verify_catalog(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + # Valid catalog + assert isinstance(cc._verify_catalog(default_catalog), str) + assert cc._verify_catalog(default_catalog) == 'dbo.catalogrecord' + + +def test_catalog_collection_invalid_verify_catalog(patch_tap): + cc = CatalogCollection("tic") + + # Ambiguous + fake_catalogs = Table({ + "catalog_name": [ + "mission1.catalogA", + "mission2.catalogA", + ], + "description": ["", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + + with pytest.raises(InvalidQueryError, match="is ambiguous for collection"): + cc._verify_catalog("catalogA") + + # Invalid catalog + with pytest.raises(InvalidQueryError, match="Catalog 'fake' is not recognized for collection 'tic'"): + cc._verify_catalog("fake") + + +def test_catalog_collection_invalid_get_column_metadata(patch_tap): + cc = CatalogCollection("tic") + + empty_result = Table( + names=["column_name", "datatype", "unit", "ucd", "description"] + ) + cc.tap_service.run_sync = MagicMock(return_value=empty_result) + + with pytest.raises( + InvalidQueryError, + match="Catalog 'fake_catalog' not found in collection 'tic'" + ): + cc._get_column_metadata("fake_catalog") + + +def test_catalog_collection_verify_criteria(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + # Valid filters + assert cc._verify_criteria(default_catalog) is None + assert cc._verify_criteria(default_catalog, gaiabp=1) is None + assert cc._verify_criteria(default_catalog, gaiabp=1, teff=1) is None + + +def test_catalog_collection_invalid_verify_criteria(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + close_match_col = "gaiagaiabp" + with pytest.raises(InvalidQueryError, match=f"Filter '{close_match_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?"): + cc._verify_criteria(default_catalog, gaiagaiabp=1) + + invalid_col = "fake_column" + with pytest.raises(InvalidQueryError, match=f"Filter '{invalid_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'."): + cc._verify_criteria(default_catalog, fake_column=1) + + +def test_catalog_collection_invalid_spatial_query(patch_tap): + cc = CatalogCollection("tic") + + # Force only spatial query to fail + patch_tap.search = MagicMock(side_effect=DALQueryError("spatial failed")) + default_catalog = cc.get_default_catalog() + metadata = cc.get_catalog_metadata(default_catalog) + + assert metadata.supports_spatial_queries is False + assert patch_tap.search.called + + +def test_catalog_collection_invalid_collection_type(patch_tap): + # Error if collection name is not a string + with pytest.raises(ValueError, match="Collection name must be a string, got "): + CatalogCollection(123) ###################### @@ -2031,4 +2902,4 @@ def test_download_file_from_cloud_verbose(patch_boto3): ) # Ensure callback was supplied _, kwargs = mock_resource.Bucket.return_value.download_file.call_args - assert "Callback" in kwargs + assert "Callback" in kwargs \ No newline at end of file From 137b123cf0fba5859182c5e2bb0ab94a849d9bf2 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 28 Apr 2026 13:31:52 -0400 Subject: [PATCH 15/21] Changelog Co-authored-by: Copilot --- CHANGES.rst | 19 +++++++++++++++++++ astroquery/mast/catalog_collection.py | 2 +- astroquery/mast/collections.py | 2 +- astroquery/mast/tests/data/README.rst | 8 ++++---- astroquery/mast/tests/test_mast.py | 2 +- astroquery/mast/tests/test_mast_remote.py | 2 +- 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c6158ea041..864d0598e1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,12 @@ esa.emds.einsteinprobe - New module to access the ESA Einstein Probe Science Archive. [#3511] +mast +^^^^ + +- Updated ``Catalogs`` interface for MAST TAP catalogs, including collection/catalog discovery helpers and a unified query workflow across + positional and non-positional searches. [#3582] + API changes @@ -82,6 +88,19 @@ mast - The ``objectname`` parameter in ``Catalogs``, ``Observations``, ``Tesscut``, and ``utils`` is deprecated in favor of ``object_name``. [#3567] - Gaia cone search now defaults to DR3. Previously, DR3 was not supported and the default was DR2. [#3622] +- ``Catalogs`` has been refactored around VO-TAP queries. The new workflow uses ``collection`` + ``catalog`` (instead of HSC/PanSTARRS-specific + assumptions), supports discovery helpers (``get_collections``, ``get_catalogs``, ``get_column_metadata``), and + adds ``supports_spatial_queries`` to inspect positional-query support before querying. [#3582] +- ``Catalogs.query_criteria`` now provides a unified query interface for positional and non-positional searches, with support for + cone searches around coordinates or an object, STC-S regions via the ``region`` parameter, column selection, sorting, count-only queries, + pagination via ``limit``/``offset``, and a ``filters`` dictionary for column names that conflict with method arguments. [#3582] +- ``Catalogs.query_region`` and ``Catalogs.query_object`` now follow the same unified backend as ``query_criteria``, including support + for ``collection``/``catalog``, ``limit``/``offset``, ``select_cols``, sorting, and advanced criteria filters. [#3582] +- The legacy ``version``, ``pagesize``, and ``page`` parameters in ``Catalogs`` query methods are now deprecated in favor + of ``collection``/``catalog`` and ``limit``/``offset``. [#3582] +- Passing a collection name via ``Catalogs(..., catalog=...)`` is deprecated; use the ``collection`` parameter instead. [#3582] +- ``Catalogs`` legacy HSC-only helper methods (``query_hsc_matchid[_async]``, ``get_hsc_spectra[_async]``, and + ``download_hsc_spectra``) are deprecated and will be removed in a future release. [#3582] vo_conesearch ^^^^^^^^^^^^^ diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 672e457aae..986295c494 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -512,4 +512,4 @@ def _verify_criteria(self, catalog, **criteria): raise InvalidQueryError( f"Filter '{kwd}' is not recognized for collection '{self.name}' and " f"catalog '{catalog}'.{suggestion}" - ) \ No newline at end of file + ) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index c32ee64d74..f36d78e41a 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -1583,4 +1583,4 @@ def _make_data_url(self, row): return f"https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={dataset}.fits" -Catalogs = CatalogsClass() \ No newline at end of file +Catalogs = CatalogsClass() diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 5f3b25bee8..3cc1155daf 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -82,7 +82,7 @@ To generate `~astroquery.mast.tests.data.tap_catalogs.vot`, use the following: >>> import pyvo >>> from astropy.io.votable import writeto ... - >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") >>> query = 'SELECT table_name, description FROM tap_schema.tables' >>> result = tap_service.run_sync(query) >>> writeto(result.votable, 'tap_catalogs.vot') # doctest: +SKIP @@ -94,7 +94,7 @@ To generate `~astroquery.mast.tests.data.tap_columns.vot`, use the following: >>> import pyvo >>> from astropy.io.votable import writeto ... - >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") >>> query = "SELECT column_name, datatype, unit, ucd, description FROM tap_schema.columns WHERE table_name = 'dbo.catalogrecord'" >>> result = tap_service.run_sync(query) >>> writeto(result.votable, 'tap_columns.vot') # doctest: +SKIP @@ -106,7 +106,7 @@ To generate `~astroquery.mast.tests.data.tap_capabilities.vot`, use the followin >>> import requests >>> import pyvo ... - >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") >>> caps_url = tap_service.baseurl.rstrip("/") + "/capabilities" >>> resp = requests.get(caps_url) >>> resp.raise_for_status() @@ -121,7 +121,7 @@ To generate `~astroquery.mast.tests.data.tap_results.vot`, use the following: >>> import pyvo >>> from astropy.io.votable import writeto ... - >>> tap_service = pyvo.dal.TAPService("https://masttest.stsci.edu/vo-tap/api/v0.1/tic") + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") >>> query = "SELECT TOP 10 * FROM dbo.catalogrecord WHERE CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 23.34086, 60.658, 0.002)) = 1" >>> result = tap_service.run_sync(query) >>> writeto(result.votable, 'tap_results.vot') # doctest: +SKIP diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 71c23971f6..8b6cbcf075 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -2902,4 +2902,4 @@ def test_download_file_from_cloud_verbose(patch_boto3): ) # Ensure callback was supplied _, kwargs = mock_resource.Bucket.return_value.download_file.call_args - assert "Callback" in kwargs \ No newline at end of file + assert "Callback" in kwargs diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 192be858b0..843cd1623b 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -1725,4 +1725,4 @@ def check_cutout_list(cutout_list, data_shape): with pytest.warns(NoResultsWarning, match='Missing HAP files for input target. Cutout not performed.'): cutout_list = Hapcut.get_cutouts(coordinates=bad_coord) assert isinstance(cutout_list, list) - assert len(cutout_list) == 0 \ No newline at end of file + assert len(cutout_list) == 0 From 12e23425f11617ce7b611a123009dd9d4759d6da Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 28 Apr 2026 13:57:41 -0400 Subject: [PATCH 16/21] Docs fixes, guards around regions import Co-authored-by: Copilot More doc fixes Co-authored-by: Copilot Typos Co-authored-by: Copilot --- astroquery/mast/__init__.py | 5 +- astroquery/mast/catalog_collection.py | 11 ++-- astroquery/mast/collections.py | 70 ++++++++++++----------- astroquery/mast/tests/data/README.rst | 2 +- astroquery/mast/tests/test_mast.py | 63 +++++++++++--------- astroquery/mast/tests/test_mast_remote.py | 5 +- docs/mast/mast_catalog.rst | 28 ++++----- 7 files changed, 100 insertions(+), 84 deletions(-) diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index 5bb93b4a7e..078c4ee74d 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -20,9 +20,6 @@ class Conf(_config.ConfigNamespace): ssoserver = _config.ConfigItem( 'https://ssoportal.stsci.edu', 'MAST SSO Portal server.') - catalogs_server = _config.ConfigItem( - 'https://catalogs.mast.stsci.edu', - 'Catalogs.MAST server.') timeout = _config.ConfigItem( 600, 'Time limit for requests from the STScI server.') @@ -38,6 +35,7 @@ class Conf(_config.ConfigNamespace): conf = Conf() from . import utils +from .catalog_collection import CatalogCollection from .collections import Catalogs, CatalogsClass from .cutouts import Hapcut, HapcutClass, Tesscut, TesscutClass, Zcut, ZcutClass from .missions import MastMissions, MastMissionsClass @@ -48,6 +46,7 @@ class Conf(_config.ConfigNamespace): "ObservationsClass", "Catalogs", "CatalogsClass", + "CatalogCollection", "MastMissions", "MastMissionsClass", "Mast", diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 986295c494..de452d6300 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -208,7 +208,7 @@ def supported_adql_functions(self): def get_catalog_metadata(self, catalog): """ - For a given catalog, cache and return metadata about its columns and capabilties. + For a given catalog, cache and return metadata about its columns and capabilities. Parameters ---------- @@ -310,8 +310,8 @@ def _fetch_catalogs(self): Returns ------- - list of str - List of catalog names. + `~astropy.table.Table` + A table containing the catalog names and descriptions for this collection. """ log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") query = "SELECT TOP 5000 table_name, description FROM tap_schema.tables" @@ -334,8 +334,9 @@ def _fetch_adql_supported_functions(self): Returns ------- - list - A list of supported ADQL functions. + set + A set of supported ADQL geometry functions (e.g. "POINT", "CIRCLE", "CONTAINS", etc.) for + this collection's TAP service. """ adql_functions = ["CIRCLE", "POLYGON", "POINT", "CONTAINS", "INTERSECTS"] supported = set() diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index f36d78e41a..8d8bde90a0 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -19,7 +19,6 @@ from astropy.table import Row, Table from astropy.time import Time from astropy.utils.decorators import deprecated, deprecated_renamed_argument -from regions import CircleSkyRegion, PolygonSkyRegion from .. import log from ..exceptions import InputWarning, InvalidQueryError, NoResultsWarning @@ -29,6 +28,12 @@ from .catalog_collection import CatalogCollection from .core import MastQueryWithLogin +try: + from regions import CircleSkyRegion, PolygonSkyRegion + HAS_REGIONS = True +except ImportError: + HAS_REGIONS = False + __all__ = ["Catalogs", "CatalogsClass"] @@ -232,7 +237,7 @@ def query_criteria( ): """ Query a MAST catalog from a given collection using criteria filters. To return columns for a given - collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- @@ -243,10 +248,10 @@ def query_criteria( coordinates : str or `~astropy.coordinates` object, optional The target around which to search. It may be specified as a string (e.g., '350 -80') or as an Astropy coordinates object. - region : str | iterable | `~astropy.regions.CircleSkyRegion` | `~astropy.regions.PolygonSkyRegion`, optional + region : str | iterable | `~regions.CircleSkyRegion` | `~regions.PolygonSkyRegion`, optional The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an - `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion`. + `~regions.CircleSkyRegion` or `~regions.PolygonSkyRegion`. object_name : str, optional The name of the object to resolve and search around. radius : str or `~astropy.units.Quantity` object, optional @@ -264,17 +269,17 @@ def query_criteria( sort_by : str or list of str, optional Column name(s) to sort the results by. sort_desc : bool or list of bool, optional - Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, - applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. version : str, optional - Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Deprecated. The page argument is no longer used. Please use `offset` instead. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria Keyword arguments representing criteria filters to apply. @@ -464,7 +469,7 @@ def query_region( ): """ Query for MAST catalog entries within a specified region using criteria filters. To return columns for a given - collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- @@ -473,10 +478,10 @@ def query_region( Astropy coordinates object. radius : str or `~astropy.units.Quantity` object, optional The search radius around the target coordinates or object. Default 0.2 degrees. - region : str | iterable | `~astropy.regions.CircleSkyRegion` | `~astropy.regions.PolygonSkyRegion`, optional + region : str | iterable | `~regions.CircleSkyRegion` | `~regions.PolygonSkyRegion`, optional The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an - `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion`. + `~regions.CircleSkyRegion` or `~regions.PolygonSkyRegion`. collection : str, optional The collection to be queried. If None, uses the instance's `collection` attribute. catalog : str, optional @@ -492,17 +497,17 @@ def query_region( sort_by : str or list of str, optional Column name(s) to sort the results by. sort_desc : bool or list of bool, optional - Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, - applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. version : str, optional - Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Deprecated. The page argument is no longer used. Please use `offset` instead. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria Keyword arguments representing criteria filters to apply. @@ -597,7 +602,7 @@ def query_object( ): """ Query for MAST catalog entries around a specified object name using criteria filters. To return columns - for a given collection and catalog, use `~astroquery.mast.collections.get_catalog_metadata`. + for a given collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- @@ -622,17 +627,17 @@ def query_object( sort_by : str or list of str, optional Column name(s) to sort the results by. sort_desc : bool or list of bool, optional - Indicates whether to sort in descending order for each column in `sort_by`. If a single bool, - applies to all columns. If a list, must match length of `sort_by`. Default is False (ascending order). + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. version : str, optional - Deprecated. The version argument is no longer used. Please use `collection` and `catalog` instead. + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Deprecated. The pagesize argument is no longer used. Please use `limit` instead. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Deprecated. The page argument is no longer used. Please use `offset` instead. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria Keyword arguments representing criteria filters to apply. @@ -1083,17 +1088,18 @@ def _create_adql_region(self, region): # Case 2: region is an astropy region object # TODO: When released, change these to use `CircleSphericalSkyRegion` and `PolygonSphericalSkyRegion` - elif isinstance(region, CircleSkyRegion): - center = region.center.icrs - radius = region.radius.to(u.deg).value - return f"CIRCLE('ICRS',{center.ra.deg},{center.dec.deg},{radius})" - elif isinstance(region, PolygonSkyRegion): - verts = region.vertices.icrs - point_string = ",".join(f"{v.ra.deg},{v.dec.deg}" for v in verts) - return f"POLYGON('ICRS',{point_string})" + if HAS_REGIONS: + if isinstance(region, CircleSkyRegion): + center = region.center.icrs + radius = region.radius.to(u.deg).value + return f"CIRCLE('ICRS',{center.ra.deg},{center.dec.deg},{radius})" + elif isinstance(region, PolygonSkyRegion): + verts = region.vertices.icrs + point_string = ",".join(f"{v.ra.deg},{v.dec.deg}" for v in verts) + return f"POLYGON('ICRS',{point_string})" # Case 3: region is an iterable of coordinate pairs - elif isinstance(region, Iterable): + if isinstance(region, Iterable): # Expect something like [(ra1, dec1), (ra2, dec2), ...] try: points = [float(x) for point in region for x in point] diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 3cc1155daf..84b82bc727 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -99,7 +99,7 @@ To generate `~astroquery.mast.tests.data.tap_columns.vot`, use the following: >>> result = tap_service.run_sync(query) >>> writeto(result.votable, 'tap_columns.vot') # doctest: +SKIP -To generate `~astroquery.mast.tests.data.tap_capabilities.vot`, use the following: +To generate `~astroquery.mast.tests.data.tap_capabilities.xml`, use the following: .. doctest-remote-data:: diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 8b6cbcf075..5660c4c48c 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -21,7 +21,6 @@ from pyvo.dal import TAPResults from pyvo.dal.exceptions import DALQueryError from pyvo.io.vosi import parse_capabilities -from regions import CirclePixelRegion, CircleSkyRegion, PixCoord, PolygonSkyRegion from requests import HTTPError, Response from astroquery.exceptions import ( @@ -34,6 +33,7 @@ ResolverError, ) from astroquery.mast import ( + CatalogCollection, Catalogs, Mast, MastMissions, @@ -50,7 +50,7 @@ from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse -from ..catalog_collection import DEFAULT_CATALOGS, CatalogCollection, CatalogMetadata +from ..catalog_collection import DEFAULT_CATALOGS, CatalogMetadata try: # Optional dependency import for cloud access functionality @@ -58,6 +58,13 @@ except ImportError: pass +try: + # Optional dependency import for region handling in collections queries + from regions import CirclePixelRegion, CircleSkyRegion, PixCoord, PolygonSkyRegion + HAS_REGIONS = True +except ImportError: + HAS_REGIONS = False + DATA_FILES = {'Mast.Caom.Cone': 'caom.json', 'Mast.Name.Lookup': 'resolver.json', 'mission_search_results': 'mission_results.json', @@ -316,7 +323,7 @@ def run_sync_mock(query, **kwargs): # Queries to get column metadata filename = data_path(DATA_FILES['tap_columns']) elif 'WHERE' in query: - # Queries with results, keep in mind this is not meaninful and results won't match the query + # Queries with results, keep in mind this is not meaningful and results won't match the query filename = data_path(DATA_FILES['tap_results']) votable = parse(filename) @@ -1815,25 +1822,6 @@ def test_catalogs_create_adql_region(patch_tap): ) assert adql_region == "POLYGON('ICRS',202.4656816,+47.1999842,202.5656816,+47.2999842,202.3656816,+47.0999842)" - # Astropy region objects - cone_region = CircleSkyRegion( - center=SkyCoord(10.8, 6.5, unit="deg"), - radius=Angle(0.5, unit="deg") - ) - adql_region = Catalogs._create_adql_region(region=cone_region) - assert adql_region == "CIRCLE('ICRS',10.8,6.5,0.5)" - - polygon_region = PolygonSkyRegion( - SkyCoord( - [57.376, 56.391, 56.025, 56.616], - [24.053, 24.622, 24.049, 24.291], - frame="icrs", - unit="deg", - ) - ) - adql_region = Catalogs._create_adql_region(region=polygon_region) - assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" - # Iterable coord pairs adql_region = Catalogs._create_adql_region( region=[ @@ -1845,6 +1833,26 @@ def test_catalogs_create_adql_region(patch_tap): ) assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" + if HAS_REGIONS: + # Astropy region objects + cone_region = CircleSkyRegion( + center=SkyCoord(10.8, 6.5, unit="deg"), + radius=Angle(0.5, unit="deg") + ) + adql_region = Catalogs._create_adql_region(region=cone_region) + assert adql_region == "CIRCLE('ICRS',10.8,6.5,0.5)" + + polygon_region = PolygonSkyRegion( + SkyCoord( + [57.376, 56.391, 56.025, 56.616], + [24.053, 24.622, 24.049, 24.291], + frame="icrs", + unit="deg", + ) + ) + adql_region = Catalogs._create_adql_region(region=polygon_region) + assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" + def test_catalogs_invalid_create_adql_region(patch_tap): # Polygon without points @@ -1873,11 +1881,12 @@ def test_catalogs_invalid_create_adql_region(patch_tap): region=[57.376, 24.053, 56.391, 24.622, 56.025, 24.049, 56.616, 24.291] ) - # Invalid astropy region - with pytest.raises(TypeError, match="Unsupported region type"): - Catalogs._create_adql_region( - region=CirclePixelRegion(PixCoord(x=42, y=43), 4.2) - ) + if HAS_REGIONS: + # Invalid astropy region + with pytest.raises(TypeError, match="Unsupported region type"): + Catalogs._create_adql_region( + region=CirclePixelRegion(PixCoord(x=42, y=43), 4.2) + ) def test_catalogs_parse_numeric_expression(patch_tap): diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 843cd1623b..889887f1b4 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -1164,11 +1164,12 @@ def test_catalogs_query_criteria(self): assert not any(result["known_binary"]) assert all(np.isin(result["sp_class"], ["O", "B"])) assert all( - (result["gaia_parallax"] < -0.01) | (result["gaia_parallax"] >= 0) | (result["gaia_parallax"] >= -0.3) + ((result["gaia_parallax"] < -0.01) | (result["gaia_parallax"] >= 0)) + & ~(result["gaia_parallax"] < -0.3) ) assert all((result["star_teff"] >= 30000) & (result["star_teff"] <= 50000)) assert all(result["coordinate_epoch"] == 2016) - assert all(np.isin(result["spectral_type_ref"], [51, 18, 59, 1])) + assert all(np.isin(result["spectral_type_ref"], [51, 18, 59])) assert all(c in result.colnames for c in select_cols) # Test offset diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index c2ab0ec7b3..c64f3c9419 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -3,10 +3,10 @@ Catalog Queries *************** -The `~astroquery.mast.Catalogs` interface provides toold for discovering and querying the wide +The `~astroquery.mast.CatalogsClass` interface provides tools for discovering and querying the wide range of astronomical catalogs hosted by MAST. These catalogs span multiple missions and surveys and are organized into **collections**, each of which may contain one or more **catalogs** with -distinct schemas and capabilties. This interface is designed for **flexible, SQL-like querying** of +distinct schemas and capabilities. This interface is designed for **flexible, SQL-like querying** of catalog data, including spatial searches and column-based filtering. At a high level, querying MAST catalogs follows three steps: @@ -85,7 +85,7 @@ Discovering Catalogs in a Collection ------------------------------------- Once a collection is selected, you can list the catalogs available within it using the -`~astroquery.mast.catalog_collection.CatalogCollection.get_catalogs` method. +`~astroquery.mast.CatalogsClass.get_catalogs` method. To query catalogs for a specific collection without changing the class state, you can pass the collection name as an argument to the method. @@ -114,10 +114,10 @@ Inspecting Catalog Metadata ---------------------------- Before querying a catalog, it is often useful to inspect its metadata to understand the available columns, -data types, and supported query capabilties. The `~astroquery.mast.CatalogsClass.get_catalog_metadata` method returns +data types, and supported query capabilities. The `~astroquery.mast.CatalogsClass.get_column_metadata` method returns an `~astropy.table.Table` describing the catalog schema, including column names, data types, units, and descriptions. This metadata can help you construct valid queries, select columns of interest, and understand which fields support -numeric comparions, string matching, or spatial queries. +numeric comparisons, string matching, or spatial queries. Again, you can specify a collection and catalog explicitly as inputs to the function, or use the current defaults. If you only specify a collection, the default catalog for that collection will be used. @@ -126,7 +126,7 @@ If you only specify a collection, the default catalog for that collection will b >>> from astroquery.mast import Catalogs ... - >>> catalog_metadata = Catalogs.get_catalog_metadata('gaiadr3', 'dbo.gaia_source') + >>> catalog_metadata = Catalogs.get_column_metadata('gaiadr3', 'dbo.gaia_source') >>> catalog_metadata[:5].pprint(max_width=-1) column_name datatype unit ucd description ------------ -------- ---- ----------------- ---------------------------- @@ -184,7 +184,7 @@ Positional constraints are optional. Supported positional parameters include: - ``coordinates`` : Sky coordinates around which to perform a cone search. - - ``objectname`` : Name of the object around which to perform a cone search. + - ``object_name`` : Name of the object around which to perform a cone search. - ``resolver`` : Resolver service to use for object name resolution. - ``radius`` : Radius of the cone search around the specified coordinates or object name. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. - ``region`` : Explicit region specification for the search. See :ref:`specifying-spatial-regions` below for more details. @@ -223,7 +223,7 @@ Criteria syntax supports a variety of operations, including: - To filter by multiple values for a single column, use a list of values. This performs an **OR** operation between the values. -- A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results. +- A filter value can be negated by prefixing it with ``!``, meaning that rows matching that value will be excluded from the results. When a negated value is present in a list of filters, any positive values in that set are combined with **OR** logic, and any negated values are combined with **AND** logic against the positives. @@ -269,7 +269,7 @@ convenience wrappers around `~astroquery.mast.CatalogsClass.query_criteria`: - `~astroquery.mast.CatalogsClass.query_region` requires a positional constraint (``coordinates`` or ``region``). - - `~astroquery.mast.CatalogsClass.query_object` requires an ``objectname`` and performs a cone search. + - `~astroquery.mast.CatalogsClass.query_object` requires an ``object_name`` and performs a cone search. Both methods also accept column-based criteria, which are applied in the same way as in `~astroquery.mast.CatalogsClass.query_criteria`. @@ -291,7 +291,7 @@ Both methods also accept column-based criteria, which are applied in the same wa .. doctest-remote-data:: >>> result = Catalogs.query_object(collection='skymapperdr4', - ... objectname='M11', + ... object_name='M11', ... radius=0.01, ... catwise_id='2825m061*', ... sort_by='mean_epoch', @@ -316,10 +316,10 @@ Catalogs that support spatial queries allow regions to be specified in several w Cone Searches ^^^^^^^^^^^^^ -Cone searches are the most common type of spatial query, defined by a center position and a radius. They may be specifed using: +Cone searches are the most common type of spatial query, defined by a center position and a radius. They may be specified using: - ``coordinates`` and ``radius`` - - ``objectname`` and ``radius`` - - A `~astropy.regions.CircleSkyRegion` object as the ``region`` parameter + - ``object_name`` and ``radius`` + - A `~regions.CircleSkyRegion` object as the ``region`` parameter - A Space-Time Coordinate (STC) CIRCLE region string as the ``region`` parameter An STC-S CIRCLE region string has the following format: @@ -358,7 +358,7 @@ Polygon searches allow for more complex spatial queries by defining a polygonal They may be specified using any of the following as the ``region`` parameter: - An iterable of coordinate pairs - - A `~astropy.regions.PolygonSkyRegion` object + - A `~regions.PolygonSkyRegion` object - A Space-Time Coordinate (STC) POLYGON region string An STC-S POLYGON string has the form: From e395f6f1a5f3a1e9a75876a0abf4343f1d17de89 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 28 Apr 2026 15:25:01 -0400 Subject: [PATCH 17/21] Make CatalogCollection private --- astroquery/mast/__init__.py | 2 -- astroquery/mast/tests/test_mast.py | 3 +-- docs/mast/mast_catalog.rst | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index 078c4ee74d..85c465724b 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -35,7 +35,6 @@ class Conf(_config.ConfigNamespace): conf = Conf() from . import utils -from .catalog_collection import CatalogCollection from .collections import Catalogs, CatalogsClass from .cutouts import Hapcut, HapcutClass, Tesscut, TesscutClass, Zcut, ZcutClass from .missions import MastMissions, MastMissionsClass @@ -46,7 +45,6 @@ class Conf(_config.ConfigNamespace): "ObservationsClass", "Catalogs", "CatalogsClass", - "CatalogCollection", "MastMissions", "MastMissionsClass", "Mast", diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 5660c4c48c..294efb27eb 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -33,7 +33,6 @@ ResolverError, ) from astroquery.mast import ( - CatalogCollection, Catalogs, Mast, MastMissions, @@ -50,7 +49,7 @@ from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse -from ..catalog_collection import DEFAULT_CATALOGS, CatalogMetadata +from ..catalog_collection import DEFAULT_CATALOGS, CatalogCollection, CatalogMetadata try: # Optional dependency import for cloud access functionality diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index c64f3c9419..e53b16e8bd 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -22,9 +22,9 @@ with a shared scientific or mission context (for example, Hubble source catalogs Within a collection, one or more **catalogs** may be available, each with its own schema and data. `~astroquery.mast.CatalogsClass` maintains a current collection and catalog as attributes. If no collection or catalog -is specified in a query, these attributes will be used as defaults. The ``collection`` attribute is a -`~astroquery.mast.catalog_collection.CatalogCollection` instance representing the current collection, -and the ``catalog`` attribute is a string representing the name of the current catalog within that collection. +is specified in a query, these attributes will be used as defaults. The ``collection`` attribute is an object +representing the current collection, and the ``catalog`` attribute is a string representing the name of the current +catalog within that collection. .. doctest-remote-data:: From 5fb697bc9d4bb85edfc196c00d8cde943cd770eb Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 7 May 2026 10:41:22 -0400 Subject: [PATCH 18/21] Hide CatalogCollection from user --- astroquery/mast/collections.py | 10 ++++++---- astroquery/mast/tests/test_mast.py | 3 ++- astroquery/mast/tests/test_mast_remote.py | 10 +++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 8d8bde90a0..f0a7b88dca 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -72,7 +72,9 @@ def collection(self): """ The current MAST collection to be queried. """ - return self._collection + # Return the collection name instead of the object for easier user interaction, + # but keep the object internally for API calls + return self._collection.name @collection.setter def collection(self, collection): @@ -99,7 +101,7 @@ def catalog(self, catalog): """ Setter that verifies that the catalog is valid for the current collection. """ - catalog = self.collection._verify_catalog(catalog) + catalog = self._collection._verify_catalog(catalog) self._catalog = catalog @property @@ -147,7 +149,7 @@ def get_catalogs(self, collection=None): A table containing the available catalogs within the specified collection. """ # If no collection specified, use the class attribute - collection_obj = self._get_collection_obj(collection) if collection else self.collection + collection_obj = self._get_collection_obj(collection) if collection else self._collection return collection_obj.catalogs @class_or_instance @@ -924,7 +926,7 @@ def _parse_inputs(self, collection=None, catalog=None): tuple A tuple containing the (collection, catalog) to be queried. """ - collection_obj = self._get_collection_obj(collection) if collection else self.collection + collection_obj = self._get_collection_obj(collection) if collection else self._collection if not catalog: # If the class attribute catalog is valid for the collection, use it diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 294efb27eb..6e0cd9a9d1 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -1495,7 +1495,8 @@ def test_catalogs_attributes(patch_tap): offset=1, sort_by="ra", ) - assert isinstance(Catalogs.collection, CatalogCollection) + # Should not change after query + assert Catalogs.collection == "hsc" assert Catalogs.catalog == "dbo.SumMagAper2CatView" diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 889887f1b4..e9e6333f02 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -1057,22 +1057,22 @@ def test_observations_get_cloud_uris_no_duplicates(self, msa_product_table, rese def test_catalogs_collection(self): # Default collection should be HSC c = Catalogs() - assert c.collection.name == "hsc" + assert c.collection == "hsc" assert c.catalog == "dbo.SumMagAper2CatView" # Initialize with a different collection c = Catalogs(collection="gaiadr3") - assert c.collection.name == "gaiadr3" + assert c.collection == "gaiadr3" assert c.catalog == "dbo.gaia_source" # Initialize with a different collection and catalog c = Catalogs(collection="ullyses", catalog="publications") - assert c.collection.name == "ullyses" + assert c.collection == "ullyses" assert c.catalog == "dbo.publications" # Set the collection c.collection = "caom" - assert c.collection.name == "caom" + assert c.collection == "caom" assert c.catalog == "dbo.obspointing" def test_catalogs_get_collections(self): @@ -1230,7 +1230,7 @@ def test_catalogs_query_region(self): # Region search with circle result = Catalogs.query_region( - collection="caom", region="CIRCLE ICRS 18.85 -6.95 0.1", limit=5, select_cols=select_cols + collection="caom", region="CIRCLE ICRS 18.85 -6.95 0.01", limit=5, select_cols=select_cols ) assert isinstance(result, Table) assert len(result) <= 5 From daf6e3c386f02efb11a46ec7f541100ff792552b Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 13 May 2026 13:20:48 -0400 Subject: [PATCH 19/21] Shared collections_cache --- astroquery/mast/catalog_collection.py | 8 +++++--- astroquery/mast/collections.py | 6 ++---- astroquery/mast/tests/test_mast.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index de452d6300..0786b3d543 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -69,6 +69,8 @@ def discover_collections(cls): if cls._discovered_collections is not None: return cls._discovered_collections + log.debug("Fetching available collections from MAST TAP service.") + # Query TAP service for collection names url = cls.TAP_BASE_URL + "openapi.json" response = utils._simple_request(url) @@ -228,6 +230,8 @@ def get_catalog_metadata(self, catalog): if catalog in self._catalog_metadata_cache: return self._catalog_metadata_cache[catalog] + log.debug(f"Fetching catalog metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") + # Get column metadata metadata = self._get_column_metadata(catalog) @@ -313,7 +317,7 @@ def _fetch_catalogs(self): `~astropy.table.Table` A table containing the catalog names and descriptions for this collection. """ - log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") + log.debug(f"Fetching available catalogs for collection '{self.name}' from MAST TAP service.") query = "SELECT TOP 5000 table_name, description FROM tap_schema.tables" # If this catalog is within a grouped collection, filter to only tables that belong to this collection @@ -437,8 +441,6 @@ def _get_column_metadata(self, catalog): response : `~astropy.table.Table` A table containing metadata about the specified table, including column names, data types, and descriptions. """ - log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") - query = f""" SELECT TOP 5000 column_name, diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index f0a7b88dca..69fd9d84ff 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -20,7 +20,6 @@ from astropy.time import Time from astropy.utils.decorators import deprecated, deprecated_renamed_argument -from .. import log from ..exceptions import InputWarning, InvalidQueryError, NoResultsWarning from ..utils import async_to_sync from ..utils.class_or_instance import class_or_instance @@ -44,6 +43,7 @@ class CatalogsClass(MastQueryWithLogin): """ TAP_BASE_URL = conf.server + "/vo-tap/api/v0.1/" + _collections_cache = dict() def __init__(self, collection=None, catalog=None): @@ -52,7 +52,6 @@ def __init__(self, collection=None, catalog=None): self._available_collections = None # Lazy load on first request self._no_longer_supported_collections = ["ctl", "diskdetective", "galex", "plato"] self._renamed_collections = {"panstarrs": "ps1_dr2", "gaia": "gaiadr3"} - self._collections_cache = dict() # Default initialization of this class should not trigger network requests # Only set collection and catalog if explicitly provided, otherwise defer to property setters @@ -129,7 +128,6 @@ def get_collections(self): return Table([self._available_collections], names=("collection_name",)) # Otherwise, fetch from TAP service discovery, including grouped collections. - log.debug("Fetching available collections from MAST TAP service.") collection_table = CatalogCollection.discover_collections()[["collection_name"]] return collection_table @@ -171,7 +169,7 @@ def get_column_metadata(self, collection=None, catalog=None): and descriptions. """ collection_obj, catalog = self._parse_inputs(collection, catalog) - return collection_obj._get_column_metadata(catalog) + return collection_obj.get_catalog_metadata(catalog).column_metadata def supports_spatial_queries(self, collection=None, catalog=None): """ diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 6e0cd9a9d1..6d3c11d22e 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -1705,6 +1705,17 @@ def test_catalogs_get_collections_cached(patch_tap): assert "collection_name" in collections.colnames +def test_catalogs_collection_cache_is_shared(patch_tap): + collection = "gaiadr3" + catalog_one = Catalogs(collection) + catalog_two = Catalogs(collection) + + collection_one = catalog_one._get_collection_obj(collection) + collection_two = catalog_two._get_collection_obj(collection) + + assert collection_one is collection_two + + def test_catalogs_supports_spatial_queries(patch_tap): catalog = Catalogs() result = catalog.supports_spatial_queries( From 4948df35cec9063b73bfc57e81417f2748783640 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Fri, 15 May 2026 15:03:27 -0400 Subject: [PATCH 20/21] async queries, return ADQL only, docs updates --- astroquery/mast/catalog_collection.py | 18 +- astroquery/mast/collections.py | 145 +++-- astroquery/mast/tests/test_mast.py | 81 ++- astroquery/mast/tests/test_mast_remote.py | 60 ++- docs/mast/mast_catalog.rst | 623 +++++++++++++--------- 5 files changed, 592 insertions(+), 335 deletions(-) diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py index 0786b3d543..6dd82ae530 100644 --- a/astroquery/mast/catalog_collection.py +++ b/astroquery/mast/catalog_collection.py @@ -3,7 +3,7 @@ from typing import Dict, Optional from astropy.table import Table -from pyvo.dal import DALQueryError, TAPService +from pyvo.dal import DALQueryError, DALServiceError, TAPService from .. import log from ..exceptions import InvalidQueryError @@ -287,7 +287,7 @@ def get_default_catalog(self): return default_catalog - def run_tap_query(self, adql): + def run_tap_query(self, adql, *, run_async=False): """ Run a TAP query against the specified catalog. @@ -295,6 +295,9 @@ def run_tap_query(self, adql): ---------- adql : str The ADQL query string. + run_async : bool, optional + If True, run the query in asynchronous mode. This mode is more robust and + preferable for long-running queries. Default is False (synchronous mode). Returns ------- @@ -303,9 +306,18 @@ def run_tap_query(self, adql): """ log.debug(f"Running TAP query on collection '{self.name}': {adql}") try: - result = self.tap_service.run_sync(adql) + if run_async: + result = self.tap_service.run_async(adql) + else: + result = self.tap_service.run_sync(adql) except DALQueryError as e: raise InvalidQueryError(f"TAP query failed for collection '{self.name}': {e}") + except DALServiceError as e: + if e.code == 504: + raise RuntimeError( + f"TAP query timed out for collection '{self.name}'. This may be due to a long-running query or " + "server issues. Consider setting run_async=True for more robust handling of long queries." + ) return result.to_table() def _fetch_catalogs(self): diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 69fd9d84ff..a14f2ab71b 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -148,7 +148,13 @@ def get_catalogs(self, collection=None): """ # If no collection specified, use the class attribute collection_obj = self._get_collection_obj(collection) if collection else self._collection - return collection_obj.catalogs + catalogs = collection_obj.catalogs + + # Do not expose tap_schema catalogs to users + mask = [not str(name).startswith('tap_schema') for name in catalogs['catalog_name']] + catalogs = catalogs[mask] + + return catalogs @class_or_instance def get_column_metadata(self, collection=None, catalog=None): @@ -230,6 +236,8 @@ def query_criteria( sort_by=None, sort_desc=False, filters=None, + run_async=False, + return_adql=False, version=None, pagesize=None, page=None, @@ -274,6 +282,13 @@ def query_criteria( filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. + run_async : bool, optional + If True, run the query in asynchronous mode. This mode is more robust and preferable + for long-running queries. If you encounter timeouts or connection issues with large queries, + set this to True. Default is False (synchronous mode). + return_adql : bool, optional + If True, return the ADQL query string instead of executing the query. Default is False. + When False, the ADQL query string is also returned in the metadata of the result table. version : str, optional Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional @@ -283,20 +298,24 @@ def query_criteria( **criteria Keyword arguments representing criteria filters to apply. - Criteria syntax - ---------------- - - Strings support wildcards using '*' (converted to SQL '%') and '%'. - - Lists are combined with OR for positive values; empty lists yield no matches. - - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using - the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. - - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values - for the same column are AND-combined, then ANDed with the OR of the positive values: - (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). - - Examples - -------- - - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) - - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + Criteria syntax: + + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + - Temporal columns can be specified as strings in a recognized date/time format (e.g., 'YYYY', 'YYYY-MM-DD', + 'YYYY-MM-DD hh:mm:ss', etc.), `~astropy.time.Time` objects, or `datetime` objects. The same comparison + operators and range syntax as numeric columns can be used to filter temporal columns + based on date/time values. + + Examples: + + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- @@ -411,14 +430,20 @@ def query_criteria( if offset: adql += f" OFFSET {offset}" + if return_adql: + return adql + # Execute the query - result_table = collection_obj.run_tap_query(adql) + result_table = collection_obj.run_tap_query(adql, run_async=run_async) + + # Add ADQL to table metadata for reference + result_table.meta["adql_query"] = adql if len(result_table) == 0: warnings.warn("The query returned no results.", NoResultsWarning) if count_only: - return result_table["count_all"][0] + return int(result_table["count_all"][0]) else: # TODO: Add schema browser URL to the result table metadata when available result_table.meta["collection"] = collection_obj.name @@ -462,6 +487,8 @@ def query_region( sort_by=None, sort_desc=False, filters=None, + run_async=False, + return_adql=False, version=None, pagesize=None, page=None, @@ -502,6 +529,13 @@ def query_region( filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. + run_async : bool, optional + If True, run the query in asynchronous mode. This mode is more robust and preferable + for long-running queries. If you encounter timeouts or connection issues with large queries, + set this to True. Default is False (synchronous mode). + return_adql : bool, optional + If True, return the ADQL query string instead of executing the query. Default is False. + When False, the ADQL query string is also returned in the metadata of the result table. version : str, optional Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional @@ -511,20 +545,24 @@ def query_region( **criteria Keyword arguments representing criteria filters to apply. - Criteria syntax - ---------------- - - Strings support wildcards using '*' (converted to SQL '%') and '%'. - - Lists are combined with OR for positive values; empty lists yield no matches. - - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using - the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. - - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values - for the same column are AND-combined, then ANDed with the OR of the positive values: - (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). - - Examples - -------- - - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) - - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + Criteria syntax: + + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + - Temporal columns can be specified as strings in a recognized date/time format (e.g., 'YYYY', 'YYYY-MM-DD', + 'YYYY-MM-DD hh:mm:ss', etc.), `~astropy.time.Time` objects, or `datetime` objects. The same comparison + operators and range syntax as numeric columns can be used to filter temporal columns + based on date/time values. + + Examples: + + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- @@ -554,6 +592,8 @@ def query_region( sort_by=sort_by, sort_desc=sort_desc, filters=filters, + run_async=run_async, + return_adql=return_adql, **criteria, ) @@ -595,6 +635,8 @@ def query_object( sort_by=None, sort_desc=False, filters=None, + run_async=False, + return_adql=False, version=None, pagesize=None, page=None, @@ -632,6 +674,13 @@ def query_object( filters : dict, optional Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column conflicts with a named parameter of this method. + run_async : bool, optional + If True, run the query in asynchronous mode. This mode is more robust and preferable + for long-running queries. If you encounter timeouts or connection issues with large queries, + set this to True. Default is False (synchronous mode). + return_adql : bool, optional + If True, return the ADQL query string instead of executing the query. Default is False. + When False, the ADQL query string is also returned in the metadata of the result table. version : str, optional Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional @@ -641,20 +690,24 @@ def query_object( **criteria Keyword arguments representing criteria filters to apply. - Criteria syntax - ---------------- - - Strings support wildcards using '*' (converted to SQL '%') and '%'. - - Lists are combined with OR for positive values; empty lists yield no matches. - - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using - the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. - - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values - for the same column are AND-combined, then ANDed with the OR of the positive values: - (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). - - Examples - -------- - - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) - - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) + Criteria syntax: + + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + - Temporal columns can be specified as strings in a recognized date/time format (e.g., 'YYYY', 'YYYY-MM-DD', + 'YYYY-MM-DD hh:mm:ss', etc.), `~astropy.time.Time` objects, or `datetime` objects. The same comparison + operators and range syntax as numeric columns can be used to filter temporal columns + based on date/time values. + + Examples: + + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- @@ -677,6 +730,8 @@ def query_object( sort_by=sort_by, sort_desc=sort_desc, filters=filters, + run_async=run_async, + return_adql=return_adql, **criteria, ) diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 6d3c11d22e..8e32f48c12 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -14,12 +14,12 @@ import pytest from astropy.coordinates import Angle, SkyCoord from astropy.io import fits -from astropy.io.votable import parse +from astropy.io.votable import from_table, parse from astropy.table import Table, unique from astropy.time import Time from astropy.utils.exceptions import AstropyDeprecationWarning from pyvo.dal import TAPResults -from pyvo.dal.exceptions import DALQueryError +from pyvo.dal.exceptions import DALQueryError, DALServiceError from pyvo.io.vosi import parse_capabilities from requests import HTTPError, Response @@ -135,6 +135,7 @@ def patch_post(request): @pytest.fixture def patch_tap(request, reset_catalogs_cache): + """Fixture to patch the TAPService used in Catalogs queries.""" mp = request.getfixturevalue("monkeypatch") mock_tap = vo_tap_mock() @@ -150,16 +151,11 @@ def patch_tap(request, reset_catalogs_cache): @pytest.fixture def reset_catalogs_cache(): + """Fixture to clear the collections cache before each test to ensure test isolation.""" Catalogs._collections_cache.clear() yield -def get_patch_tap_query(patch_tap): - args, _ = patch_tap.run_sync.call_args - query = args[0] - return query - - @pytest.fixture() def patch_boto3(monkeypatch, reset_cloud_state): """Fixture to patch boto3 client and resource for cloud access tests.""" @@ -310,11 +306,20 @@ def zcut_download_mockreturn(url, file_path): def vo_tap_mock(): + """Helper function to create a mock TAPService with predefined responses based on the input query.""" def run_sync_mock(query, **kwargs): + """Mock function to simulate TAPService run_sync and run_async methods.""" if 'invalid' in query: # Use this when wanting to simulate a DALQueryError # Where it occurs will depend on where you pass it (collection, catalog, parameter, etc.) raise DALQueryError('Simulated TAP query error for testing.') + elif 'timeout' in query: + # Use this when wanting to simulate a timeout error + raise DALServiceError('Simulated TAP service timeout for testing.', code=504) + elif 'COUNT' in query: + # Simulate a query that returns a count + votable = from_table(Table({'count_all': [42]})) # Example count value, can be adjusted as needed for tests + return TAPResults(votable) elif 'tap_schema.tables' in query: # Queries to get catalogs filename = data_path(DATA_FILES['tap_catalogs']) @@ -337,8 +342,9 @@ def run_sync_mock(query, **kwargs): # Mock TAPService mock_tap = MagicMock() mock_tap.run_sync.side_effect = run_sync_mock + mock_tap.run_async.side_effect = run_sync_mock - # Capabilities -> Not much to do here + # Capabilities parsing filename = data_path(DATA_FILES['tap_capabilities']) with open(filename, "rb") as f: caps = parse_capabilities(f) @@ -346,6 +352,17 @@ def run_sync_mock(query, **kwargs): return mock_tap + +def get_patch_tap_query(patch_tap, run_async=False): + """Helper function to extract the ADQL query string from the mock TAP service calls.""" + if run_async: + args, _ = patch_tap.run_async.call_args + else: + args, _ = patch_tap.run_sync.call_args + query = args[0] + return query + + ########################### # MissionSearchClass Test # ########################### @@ -1503,6 +1520,7 @@ def test_catalogs_attributes(patch_tap): def test_catalogs_get_catalogs(patch_tap): catalogs = Catalogs.get_catalogs("tic") assert isinstance(catalogs, Table) + assert not any(catalog.startswith("tap_schema") for catalog in catalogs["catalog_name"]) def test_catalogs_get_column_metadata(patch_tap): @@ -1527,6 +1545,9 @@ def test_catalogs_query_criteria(patch_tap): assert "CONTAINS" in query assert "CIRCLE" in query assert "TOP 2" in query + # ADQL should be saved in result meta + assert "adql_query" in result.meta + assert result.meta["adql_query"] == query # Region query result = Catalogs.query_criteria( @@ -1561,6 +1582,39 @@ def test_catalogs_query_criteria(patch_tap): assert "pmra > -10" in query assert "pmra < 10" in query + # Run with async + result = Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + run_async=True + ) + assert isinstance(result, Table) + assert len(result) > 0 + query = get_patch_tap_query(patch_tap, run_async=True) + assert "FROM dbo.catalogrecord" in query + + # Count only query + result = Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + count_only=True + ) + assert isinstance(result, int) + query = get_patch_tap_query(patch_tap) + assert "COUNT(*)" in query + + # Only return ADQL + result = Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + return_adql=True + ) + assert isinstance(result, str) + assert "FROM dbo.catalogrecord" in result + def test_catalogs_invalid_query_criteria(patch_tap): # Specifying both region and coordinates @@ -2227,6 +2281,15 @@ def test_catalogs_invalid_tap_query(patch_tap): allwise='invalid' ) + # Simulate a timeout + with pytest.raises(RuntimeError, match="TAP query timed out for collection 'tic'"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + allwise='timeout' + ) + def test_catalogs_invalid_spatial_query(patch_tap): # Force spatial query to fail diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index e9e6333f02..0a33f63dc2 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -1071,9 +1071,9 @@ def test_catalogs_collection(self): assert c.catalog == "dbo.publications" # Set the collection - c.collection = "caom" - assert c.collection == "caom" - assert c.catalog == "dbo.obspointing" + c.collection = "tic_v82" + assert c.collection == "tic_v82" + assert c.catalog == "tic_v82.source" def test_catalogs_get_collections(self): collections = Catalogs.get_collections() @@ -1085,6 +1085,7 @@ def test_catalogs_get_catalogs(self): assert isinstance(catalogs, Table) assert "catalog_name" in catalogs.colnames assert "description" in catalogs.colnames + assert not any(catalog.startswith("tap_schema") for catalog in catalogs["catalog_name"]) def test_catalogs_get_column_metadata(self): metadata = Catalogs.get_column_metadata("hsc", "dbo.SumMagAper2CatView") @@ -1131,6 +1132,9 @@ def test_catalogs_query_criteria(self): for i in range(len(result) - 1) if result["numimages"][i] == result["numimages"][i + 1] ) + # Assert that the ADQL query is included in the metadata and contains the correct filters + assert "adql_query" in result.meta + assert "SELECT" in result.meta["adql_query"] # Non-positional query with multiple filters and select_cols select_cols = [ @@ -1182,6 +1186,17 @@ def test_catalogs_query_criteria(self): assert isinstance(result_count, (int, np.integer)) assert result_count > 0 + # Return ADQL + result_adql = c.query_criteria(collection="goods", class_star=0.23, return_adql=True) + assert isinstance(result_adql, str) + assert "SELECT" in result_adql + + # Run async + result_async = c.query_criteria(collection="goods", class_star=0.23, run_async=True) + assert isinstance(result_async, Table) + assert len(result_async) > 0 + assert all(result_async["class_star"] == 0.23) + # Temporal filters and filter passed in through filters argument result = c.query_criteria(collection='caom', catalog='caommembers', @@ -1202,17 +1217,17 @@ def test_catalogs_query_criteria(self): def test_catalogs_query_criteria_error(self): # No results should warn user - # with pytest.warns(NoResultsWarning): - # Catalogs.query_criteria(collection="ps1_dr2", skycellid=-1) + with pytest.warns(NoResultsWarning): + Catalogs.query_criteria(collection="classy", z="<0") with pytest.warns(NoResultsWarning): Catalogs.query_criteria(collection="classy", target=[]) def test_catalogs_query_region(self): # Region search with polygon - select_cols = ["target_name", "obs_id", "s_ra", "s_dec", "s_region"] + select_cols = ["object_id", "raj2000", "dej2000"] result = Catalogs.query_region( - collection="caom", + collection="skymapperdr4", region="POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94", limit=5, select_cols=select_cols, @@ -1223,33 +1238,33 @@ def test_catalogs_query_region(self): # Assert that all results are within a radius of the specified polygon # We can't just check that the coordinates are within the polygon because # they may intersect the polygon without the center being within it - coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + coords = SkyCoord(result["raj2000"], result["dej2000"], unit="deg") polygon = SkyCoord(18.855, -6.945, unit="deg") separation = coords.separation(polygon) assert all(separation <= 0.1 * u.deg) # Region search with circle result = Catalogs.query_region( - collection="caom", region="CIRCLE ICRS 18.85 -6.95 0.01", limit=5, select_cols=select_cols + collection="skymapperdr4", region="CIRCLE ICRS 18.85 -6.95 0.01", limit=5, select_cols=select_cols ) assert isinstance(result, Table) assert len(result) <= 5 assert all(c in result.colnames for c in select_cols) # Assert that all results are within a radius of the specified circle - coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + coords = SkyCoord(result["raj2000"], result["dej2000"], unit="deg") center = SkyCoord(18.85, -6.95, unit="deg") separation = coords.separation(center) assert all(separation <= 0.1 * u.deg) def test_catalogs_query_object(self): # Object search - select_cols = ["target_name", "obs_id", "s_ra", "s_dec"] - result = Catalogs.query_object(collection="caom", object_name="M2", limit=5, select_cols=select_cols) + select_cols = ["object_id", "raj2000", "dej2000"] + result = Catalogs.query_object(collection="skymapperdr4", object_name="M2", limit=5, select_cols=select_cols) assert isinstance(result, Table) assert len(result) <= 5 assert all(c in result.colnames for c in select_cols) # Assert that all results are within a radius of the specified object - coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + coords = SkyCoord(result["raj2000"], result["dej2000"], unit="deg") m2_coords = SkyCoord(323.36258, -0.82325, unit="deg") separation = coords.separation(m2_coords) assert all(separation <= 0.1 * u.deg) @@ -1320,14 +1335,13 @@ def test_catalog_collection_discover_collections(self): assert CatalogCollection._discovered_collections is not None def test_catalog_collection_get_parent_collection(self): - parent = CatalogCollection.get_parent_collection("TIC") - assert parent == "tic" + parent = CatalogCollection.get_parent_collection("gaiadr3") + assert parent == "gaiadr3" - # parent = CatalogCollection.get_parent_collection("tic_v82") - # assert parent == "mast_catalogs" + parent = CatalogCollection.get_parent_collection("tic_v82") + assert parent == "mast_catalogs" - @pytest.mark.parametrize("collection", ["caom"]) - # @pytest.mark.parametrize("collection", ["caom", "tic_v82"]) + @pytest.mark.parametrize("collection", ["classy", "tic_v82"]) def test_catalog_collection_get_catalogs(self, collection): cc = CatalogCollection(collection) catalogs = cc._fetch_catalogs() @@ -1335,7 +1349,7 @@ def test_catalog_collection_get_catalogs(self, collection): assert len(catalogs) > 0 assert catalogs.colnames == ["catalog_name", "description"] - @pytest.mark.parametrize("collection", ["caom", "ullyses", "tic"]) + @pytest.mark.parametrize("collection", ["gaiadr3", "ullyses", "tic_v82"]) def test_catalog_collection_get_catalog_metadata(self, collection): cc = CatalogCollection(collection) default_catalog = cc.get_default_catalog() @@ -1358,14 +1372,14 @@ def test_catalog_collection_get_catalog_metadata(self, collection): assert default_metadata is default_metadata_cached def test_catalog_collection_invalid_get_catalog_metadata(self): - cc = CatalogCollection("TIC") + cc = CatalogCollection("tic_v82") invalid_catalog = "invalid_catalog" with pytest.raises( InvalidQueryError, match=f"Catalog '{invalid_catalog}' is not recognized for collection '{cc.name}'." ): cc.get_catalog_metadata(invalid_catalog) - @pytest.mark.parametrize("collection", ["tic", "classy", "ullyses"]) + @pytest.mark.parametrize("collection", ["ps1_dr2", "classy", "ullyses"]) def test_catalog_collection_get_default_catalog(self, collection): cc = CatalogCollection(collection) catalogs = cc._fetch_catalogs() @@ -1393,7 +1407,7 @@ def test_catalog_collection_run_tap_query(self): assert ((result["dec"] >= 12) & (result["dec"] <= 13)).all() def test_catalog_collection_verify_criteria(self): - cc = CatalogCollection("TIC") + cc = CatalogCollection("tic_v82") default_catalog = cc.get_default_catalog() result = cc._verify_criteria(default_catalog) diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index e53b16e8bd..97b31d688d 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -3,34 +3,45 @@ Catalog Queries *************** -The `~astroquery.mast.CatalogsClass` interface provides tools for discovering and querying the wide -range of astronomical catalogs hosted by MAST. These catalogs span multiple missions and surveys -and are organized into **collections**, each of which may contain one or more **catalogs** with -distinct schemas and capabilities. This interface is designed for **flexible, SQL-like querying** of -catalog data, including spatial searches and column-based filtering. - -At a high level, querying MAST catalogs follows three steps: - 1. Discover available collections and catalogs. - 2. Inspect catalog metadata to understand available fields and data types. - 3. Query the catalog using positional and/or criteria-based filters. +`~astroquery.mast.CatalogsClass` is a versatile tool for discovering and querying the wide range of astronomical catalogs hosted by the +[Mikulski Archive for Space Telescopes (MAST)](https://archive.stsci.edu/). `~astroquery.mast.CatalogsClass` is a Python wrapper +for our [MAST Table Access Protocol (TAP) Service](https://mast.stsci.edu/vo-tap/), which allows you to query for catalog +metadata and data. If you were querying the MAST TAP service directly, you would need to write your queries in +[Astronomical Data Query Language (ADQL)](https://www.ivoa.net/documents/latest/ADQL.html). With `~astroquery.mast.CatalogsClass`, +you can construct and execute these queries using a more intuitive Python interface, without needing to learn ADQL syntax. + +The catalogs available through MAST are diverse, covering a wide range of astronomical objects and phenomena. +They include data from various missions and surveys, such as the Hubble Space Telescope, Kepler, TESS, Gaia, and many more. +These catalogs are organized into **collections**, each of which may contain one or more catalogs with distinct schemas and capabilities. +The `~astroquery.mast.CatalogsClass` interface is designed for flexible querying of catalog data, including both spatial and non-spatial queries, +as well as the ability to filter results based on specific criteria. + +At a high level, querying MAST catalogs with `~astroquery.mast.CatalogsClass` involves the following steps: +1. **Discover** available collections and catalogs. +2. **Inspect** catalog metadata to understand available columns and data types, as well as the capabilities of each catalog. +3. **Query** the catalog using spatial and/or non-spatial criteria to retrieve relevant data. Collections and Catalogs ======================== -MAST catalogs are organized into **collections**, where each collection represents a related set of catalogs -with a shared scientific or mission context (for example, Hubble source catalogs, Gaia data releases, etc). -Within a collection, one or more **catalogs** may be available, each with its own schema and data. +MAST catalogs are organized into **collections**, where each collection represents a set of related catalogs +with a shared scientific or mission context (e.g., Hubble source catalogs, Gaia data releases, etc.). +Within a collection, one or more **catalogs** may be available, each with its own set of columns and capabilities. -`~astroquery.mast.CatalogsClass` maintains a current collection and catalog as attributes. If no collection or catalog +`~astroquery.mast.CatalogsClass` stores a ``collection`` and ``catalog`` as attributes. If no collection and/or catalog is specified in a query, these attributes will be used as defaults. The ``collection`` attribute is an object -representing the current collection, and the ``catalog`` attribute is a string representing the name of the current +representing the current collection, and the ``catalog`` attribute is a string representing the name of the current catalog within that collection. +The default value for the ``collection`` attribute is "hsc", referring to the [Hubble Source Catalog version 3](https://archive.stsci.edu/hst/hsc/). +The default value for ``catalog`` is "dbo.SumMagAper2CatView". This is a summary source catalog with data describing sources detected in Hubble images, +including their positions, magnitudes, and other properties. + .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> print("Default collection:", Catalogs.collection.name) + >>> print("Default collection:", Catalogs.collection) Default collection: hsc >>> print("Default catalog:", Catalogs.catalog) Default catalog: dbo.SumMagAper2CatView @@ -38,23 +49,51 @@ catalog within that collection. These attributes may be changed at any time to set new defaults. Both ``collection`` and ``catalog`` will be validated when set. When changing the collection, the catalog will be reset to the default for the new collection. +These attributes can be set with parameters when instantiating a `~astroquery.mast.CatalogsClass` object, or they can be changed +at any time after instantiation to set new defaults for subsequent queries. Both ``collection`` and ``catalog`` will be validated when set. +``collection`` must be a valid collection name, and ``catalog`` must be a valid catalog within the specified collection. When changing the +collection, the catalog will be reset to the default catalog for the new collection. + +Here, we'll change the value of ``collection`` to "ullyses", referring to the +[Hubble Ultraviolet Legacy Library of Young Stars as Essential Standards (ULLYSES)](https://ullyses.stsci.edu/) program. The default catalog +for this collection is "sciencemetadata", which contains metadata about the scientific exposures taken as part of the ULLYSES program, +including their coordinates, observation dates, instruments used, and other properties. + .. doctest-remote-data:: - >>> Catalogs.collection = "TIC" # set collection to TESS Input Catalog - >>> print("New collection:", Catalogs.collection.name) - New collection: tic + >>> Catalogs.collection = "ullyses" # set collection to ULLYSES + >>> print("New collection:", Catalogs.collection) + New collection: ullyses >>> print("New catalog:", Catalogs.catalog) - New catalog: dbo.CatalogRecord + New catalog: dbo.sciencemetadata -Discovering Available Collections ---------------------------------- +You can also create multiple instances of `~astroquery.mast.CatalogsClass` with different defaults, which can be useful for working with multiple catalogs +in the same script or notebook. -To list all available catalog collections hosted at MAST, use the -`~astroquery.mast.CatalogsClass.get_collections` method. +.. doctest-remote-data:: + + >>> hsc_catalog = Catalogs(collection="hsc") + >>> print("HSC collection:", hsc_catalog.collection) + HSC collection: hsc + >>> print("HSC catalog:", hsc_catalog.catalog) + HSC catalog: dbo.SumMagAper2CatView + >>> + >>> ullyses_catalog = Catalogs(collection="ullyses") + >>> print("ULLYSES collection:", ullyses_catalog.collection) + ULLYSES collection: ullyses + >>> print("ULLYSES catalog:", ullyses_catalog.catalog) + ULLYSES catalog: dbo.sciencemetadata + +Discovering Available Collections and Catalogs +=============================================== + +Discovering Available Collections +---------------------------------- +To list all of the catalog collections that are accessible via the MAST TAP Service, use the `~astroquery.mast.CatalogsClass.get_collections` method. This returns an `~astropy.table.Table` containing the names of all available collections. -Some historical collections are no longer supported for querying, and will not appear in this list. If a collection +Some historical collections are no longer supported for querying, and will not appear in this list. If a collection has been renamed or deprecated, Astroquery will issue a warning and suggest the appropriate replacement where possible. .. doctest-remote-data:: @@ -63,44 +102,39 @@ has been renamed or deprecated, Astroquery will issue a warning and suggest the >>> print(collections) # doctest: +IGNORE_OUTPUT collection_name --------------- + 3dhst + candels caom classy + deepspace gaiadr3 + goods hsc hscv2 missionmast + ps1_dr2 ps1dr1 ps1dr2 - ps1_dr2 registry skymapperdr4 tic + tic_v82 ullyses - goods - candels - 3dhst - deepspace - -Discovering Catalogs in a Collection -------------------------------------- -Once a collection is selected, you can list the catalogs available within it using the -`~astroquery.mast.CatalogsClass.get_catalogs` method. +Discovering Catalogs Within a Collection +----------------------------------------- -To query catalogs for a specific collection without changing the class state, you can pass the +Once a collection is selected, you can discover which catalogs are available within that collection using the +`~astroquery.mast.CatalogsClass.get_catalogs` method. This returns an `~astropy.table.Table` containing names and descriptions of the catalogs +within the currently selected collection. To query catalogs for a specific collection without changing the class state, you can pass the collection name as an argument to the method. .. doctest-remote-data:: - >>> catalogs = Catalogs.get_catalogs('hsc') + >>> catalogs = Catalogs.get_catalogs("hsc") >>> catalogs.pprint(max_width=-1) # doctest: +IGNORE_OUTPUT - catalog_name description + catalog_name description -------------------------- ------------------------------------------------------- - tap_schema.schemas description of schemas in this dataset - tap_schema.tables description of tables in this dataset - tap_schema.columns description of columns in this dataset - tap_schema.keys description of foreign keys in this dataset - tap_schema.key_columns description of foreign key columns in this dataset dbo.detailedcatalog Detailed list of source catalog parameters dbo.hcvdetailedview Detailed list of Hubble Catalog of Variables parameters dbo.hcvsummaryview Summary list of Hubble Catalog of Variables parameters @@ -111,280 +145,343 @@ collection name as an argument to the method. dbo.catalog_image_metadata Summary list of Image processing metadata Inspecting Catalog Metadata ----------------------------- +============================ + +Before querying a catalog, it's important to understand what data it contains and how that data is organized. Catalog metadata includes +information about the columns in the catalog (e.g., their names, data types, and descriptions) as well as the capabilities of the catalog +(e.g., what types of queries it supports). This information can help you construct effective queries and interpret the results correctly. -Before querying a catalog, it is often useful to inspect its metadata to understand the available columns, -data types, and supported query capabilities. The `~astroquery.mast.CatalogsClass.get_column_metadata` method returns -an `~astropy.table.Table` describing the catalog schema, including column names, data types, units, and descriptions. -This metadata can help you construct valid queries, select columns of interest, and understand which fields support -numeric comparisons, string matching, or spatial queries. +Inspecting Catalog Columns +--------------------------- -Again, you can specify a collection and catalog explicitly as inputs to the function, or use the current defaults. -If you only specify a collection, the default catalog for that collection will be used. +Use the `~astroquery.mast.CatalogsClass.get_column_metadata` method to inspect the columns of a catalog. This returns an `~astropy.table.Table` +with information about each column, including its name, data type, unit, and a description. This metadata is crucial for constructing valid queries, +selecting columns of interest, and understanding which columns support different criteria syntax. + +Again, you can specify the collection and catalog explicitly as inputs to the function, or you can rely on the default values stored in the class attributes. +If you only specify a collection, the default catalog for that collection will be used. If you only specify a catalog, the current collection will be used. .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_metadata = Catalogs.get_column_metadata('gaiadr3', 'dbo.gaia_source') + >>> catalog_metadata = Catalogs.get_column_metadata(collection="ullyses", catalog="dbo.sciencemetadata") >>> catalog_metadata[:5].pprint(max_width=-1) - column_name datatype unit ucd description - ------------ -------- ---- ----------------- ---------------------------- - solution_id long meta.version catalog version - designation char meta.id;meta.main designation - source_id long meta.id source id - random_index long meta.code random index - ref_epoch double yr time.epoch reference epoch julian years + column_name datatype unit ucd description + --------------------- -------- ---- -------------------- ----------------------------------- + target_id int meta.id;meta.main ullyses target id + target_name_ullyses char meta.id.assoc ullyses name of target + target_name_simbad char meta.id.assoc simbad name of target + target_name_hlsp char meta.id.assoc hlsp name of target + target_classification char src.class.stargalaxy target type: lmc; smc; t tau; low z +Inspecting Catalog Capabilities +------------------------------- -Querying Catalogs -================== +Each catalog has different capabilities, which are important to understand when constructing your queries. For example, only certain catalogs +support spatial queries based on a sky position or region. Use the `~astroquery.mast.CatalogsClass.supports_spatial_queries()` method to check +if a catalog supports spatial queries. -The Catalogs interface provides three closely related query methods. All three methods return results from a MAST -catalog as an `~astropy.table.Table` (or a scalar count, if requested), and all three support column-based filtering, -sorting, and result limiting. The primary difference between them is how positional constraints are specified. +.. doctest-remote-data:: -At a high level: - - `~astroquery.mast.CatalogsClass.query_criteria` is the most flexible method. It supports purely column-based queries, - purely positional queries, or a combination of both. + >>> supports_spatial_ullyses = Catalogs.supports_spatial_queries(collection="ullyses", catalog="dbo.sciencemetadata") + >>> print("Supports spatial queries:", supports_spatial_ullyses) + Supports spatial queries: False + >>> + >>> supports_spatial_hsc = Catalogs.supports_spatial_queries(collection="hsc", catalog="dbo.SumMagAper2CatView") + >>> print("Supports spatial queries:", supports_spatial_hsc) + Supports spatial queries: True - - `~astroquery.mast.CatalogsClass.query_region` is a convenience wrapper for positional queries using coordinates or an explicit region. +Querying Catalogs +================== - - `~astroquery.mast.CatalogsClass.query_object` is a convenience wrapper for cone searches centered on a resolved object name. +`~astroquery.mast.CatalogsClass` provides three main methods for querying catalogs: -All three methods ultimately construct and execute an ADQL query against the MAST TAP service. +- `~astroquery.mast.CatalogsClass.query_criteria` is the most flexible method. It supports purely column-based queries, purely spatial queries, or a combination of both. +- `~astroquery.mast.CatalogsClass.query_region` is a convenience method for spatial queries that use coordinates or a region on the sky. +- `~astroquery.mast.CatalogsClass.query_object` is a convenience method for spatial queries that use an object name resolved to coordinates. + +All three methods ultimately construct and execute an ADQL query against the MAST TAP service. All three support column-based filtering, sorting, and +limiting of results. The primary difference between them is whether and how spatial criteria are specified. Shared Query Parameters ------------------------ -The following parameters are supported by all three query methods: - - ``collection`` : The catalog collection to query. If not specified, the value of the instance's ``collection`` attribute is used. - - - ``catalog`` : The catalog within the collection to query. If not specified, the value of the instance's ``catalog`` attribute is used, or if ``collection`` is specified, the default catalog for that collection. - - - ``limit`` : Maximum number of rows to return (default: 5000). - - - ``offset`` : Number of rows to skip before returning results (default: 0). - - - ``count_only`` : If True, return only the number of matching records instead of the records themselves. - - - ``select_cols`` : A list of column names to include in the result. If omitted, all columns are returned. - - - ``sort_by``: One or more column names to sort by. - - - ``sort_desc`` : Whether to sort in descending order (either a single boolean applied to all ``sort_by`` columns or one per column). +The following parameters are shared across all three query methods: + + - ``collection`` : The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. + - ``collection``: The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. + - ``catalog``: The name of the catalog to query within the specified collection. If not specified, the ``catalog`` class attribute will be used as the default. + - ``limit``: An integer specifying the maximum number of results to return. The default is 5,000. + - ``offset``: An integer specifying the number of results to skip before starting to return results. This is useful for paginating through large result sets. Default is 0. + - ``count_only``: A boolean indicating whether to return only the count of matching results instead of the results themselves. Default is False. + - ``select_cols``: A list of column names to include in the results. If not specified, all columns will be returned. + - ``sort_by``: A string or list of strings specifying the column(s) to sort the results by. Default is None (no sorting). + - ``sort_desc``: A boolean or list of booleans specifying whether to sort in descending order for each column specified in ``sort_by``. + Default is False (ascending order). + - ``filters``: Another parameter used to specify criteria filters as a dictionary. Use this option when the name of a column conflicts + with a named parameter of this method. + - ``run_async``: If True, run the query in asynchronous mode. This mode is more robust and preferable for long-running queries. + - ``return_adql``: If True, return the ADQL query string instead of executing the query. This is useful for debugging or for users who want + to run the query directly against the TAP service. When False, the ADQL query string is also returned in the metadata of the result table. These parameters allow users to control the scope and format of their queries consistently across all three methods. -Writing Queries +Criteria Syntax ---------------- -The `~astroquery.mast.CatalogsClass.query_criteria` method supports both positional parameters and column-based filters. -Positional constraints are optional. +All query methods also allow you to filter results based on column values. Users may specify criteria using keyword arguments, where the keyword +is the column name and the value is the filter condition. Multiple criteria are combined using a logical **AND**. -Supported positional parameters include: - - ``coordinates`` : Sky coordinates around which to perform a cone search. - - ``object_name`` : Name of the object around which to perform a cone search. - - ``resolver`` : Resolver service to use for object name resolution. - - ``radius`` : Radius of the cone search around the specified coordinates or object name. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. - - ``region`` : Explicit region specification for the search. See :ref:`specifying-spatial-regions` below for more details. +Criteria syntax supports a variety of operations for filtering results: -If no positional parameters are supplied, the query is purely criteria-based. If positional parameters are supplied, -they are combined with any column-based criteria using logical **AND**. +- A single value (e.g. ``column=value``) will filter for rows where the column is equal to the value. +- A list of values (e.g. ``column=[value1, value2]``) will filter for rows where the column is equal to any of the values in the list (logical OR). +- A value prefixed with ``!`` (e.g. ``column="!value"``) will filter for rows where the column is not equal to the value. +- For string columns, a string with a wildcard character ``*`` (e.g. ``column="NGC*"``) will filter for rows where the column value matches the pattern, + where ``*`` can match any sequence of characters. +- For numeric columns, a string with a comparison operator (e.g. ``column=">value"``) will filter for rows where the column value satisfies the specified + comparison. Supported operators include ``<``, ``>``, ``<=``, and ``>=``. +- For numeric columns, a string with a range (e.g. ``column="value1..value2"``) will filter for rows where the column value falls within the specified range + (inclusive). +- For temporal columns, values can be specified as strings in a recognized date/time format (e.g. ``YYYY``, ``YYYY-MM-DD``, ``YYYY-MM-DD hh:mm:ss``, etc.), + ``astropy.time.Time`` objects, or ``datetime`` objects. The same comparison operators and range syntax as numeric columns can be used to filter temporal + columns based on date/time values. -.. doctest-remote-data:: +We'll use the Ullyses science metadata catalog to demonstrate a column-based query, since it doesn't support spatial queries. Let's filter for the following: - >>> from astroquery.mast import Catalogs - ... - >>> result = Catalogs.query_criteria(collection='hsc', - ... coordinates="322.49324 12.16683", - ... radius='2 arcsec', - ... sort_by=['numimages', 'starttime'], - ... sort_desc=[False, True], - ... limit=5, - ... select_cols=['matchid', 'matchra', 'matchdec', 'numimages', 'starttime']) - >>> result.pprint(max_width=-1) - matchid matchra matchdec numimages starttime - deg deg - --------- ------------------ ------------------ --------- -------------------------- - 100906349 322.4932839715549 12.166957658789572 1 2013-09-01 14:42:57.487000 - 37053748 322.49333237602906 12.167170257824768 1 2013-09-01 14:09:53.487000 - 61895629 322.493715383149 12.166629788750484 1 2011-10-22 08:10:21.217000 - 19779150 322.49277386636714 12.166728768957904 2 2013-09-01 14:09:53.487000 - 11562863 322.49294957070185 12.166668540816076 2 2006-05-02 01:13:43.920000 - -To filter results based on column values, users may specify criteria as keyword arguments, -where the keyword is the column name and the value is the desired filter. Multiple criteria are combined -using logical **AND**. +- Targets with a name that starts with "NGC". +- Targets that belong to spectral class "O" or "B". +- Targets that are NOT known binaries. +- Targets that are NOT classified as "Galaxy" or "Late O Dwarf". +- Targets with Gaia parallax less than -0.01 or greater than or equal to 0. +- Targets with effective temperature between 30,000 and 50,000 K. -Criteria syntax supports a variety of operations, including: +We will also select a subset of columns to return with the ``select_cols`` parameter. -- Exact matches by specifying a value for a column. - -- To filter by multiple values for a single column, use a list of values. This performs an **OR** operation between the values. +.. doctest-remote-data:: -- A filter value can be negated by prefixing it with ``!``, meaning that rows matching that value will be excluded from the results. - When a negated value is present in a list of filters, any positive values in that set are combined with **OR** logic, and any negated - values are combined with **AND** logic against the positives. + >>> result = Catalogs.query_criteria( + ... collection="ullyses", # Query the 'ullyses' collection + ... catalog="sciencemetadata", # Query the 'sciencemetadata' catalog + ... target_name_ullyses="NGC*", # Query for targets names starting with 'NGC' + ... sp_class=["O", "B"], # Query for targets with spectral class "O" or "B" + ... known_binary=False, # Query for targets that are not known binaries + ... target_classification=["!Galaxy", "!Late O Dwarf"], # Exclude targets classified as 'Galaxy' or 'Late O Dwarf' + ... gaia_parallax=["<-0.1", ">=0"], # Query for targets with Gaia parallax less than -0.01 or greater than or equal to 0 + ... star_teff="30000..50000", # Query for targets with effective temperature between 30,000 and 50,000 K + ... select_cols=["target_name_ullyses", "target_classification", "known_binary", "sp_class", "gaia_parallax", "star_teff"] + ... ) + >>> result[:5].pprint(max_width=-1) #doctest: +IGNORE_OUTPUT + target_name_ullyses target_classification known_binary sp_class gaia_parallax star_teff + mas K + ------------------- --------------------- ------------ -------- ------------- --------- + NGC346 ELS 043 Early B Dwarf False B -0.111579 33000.0 + NGC346 ELS 026 Early B Subgiant False O -0.047347 31000.0 + NGC346 ELS 028 Mid O Dwarf False O -0.069206 39600.0 + NGC346 ELS 007 Early O Dwarf False O -0.070696 42100.0 + NGC346 MPG 356 Mid O Dwarf False O -0.051241 38200.0 -- For columns with a numeric data type, filter using comparison values (``<``, ``>``, ``<=``, ``>=``). +Spatial Query Parameters +------------------------ - - ``<``: Return values less than or before the given number/date +If a catalog supports spatial queries, the following parameters can be used to specify the spatial region of interest: - - ``>``: Return values greater than or after the given number/date +- ``coordinates``: A string or `~astropy.coordinates.SkyCoord` object specifying the center of a cone search. This parameter is used in + conjunction with ``radius``. +- ``object_name``: A string specifying the name of an astronomical object to resolve to coordinates for a cone search. This parameter is used in + conjunction with ``radius``. +- ``resolver``: A string specifying the name of the resolver to use when resolving ``object_name`` to coordinates. This is only applicable when + ``object_name`` is provided. Default is None. +- ``radius``: The radius of a cone search around ``coordinates`` or ``object_name``. Can be defined as a string with units (e.g., "10 arcsec"), + a `~astropy.units.Quantity`, or a float in degrees. Default is 0.2 degrees. +- ``region``: Specifies the spatial region of interest for more complex spatial queries, such as polygon searches. + Please see the [Specifying Spatial Regions](#specifying-spatial-regions) section below for details on how to use this parameter. - - ``<=``: Return values less than or equal to the given number/date +If no spatial parameters are provided, the query is purely column-based and will not filter results based on position. If they are supplied, the +spatial parameters are combined with any column-based criteria using a logical **AND**, meaning that only results that satisfy both the spatial and +column-based criteria will be returned. - - ``>=``: Return values greater than or equal to the given number/date +We'll demonstrate a spatial query using the HSC summary source catalog, which supports spatial queries. Let's filter for the following: -- For columns with a numeric data type, select an inclusive range with the syntax ``'#..#'``. +- Sources within 2 arcseconds of the coordinates (322.49324, 12.16683). +- Sources with target names that are either "M-15" or start with "NGC". +- Sources with a start time between 2006 and 2013. -- Wildcards are special characters used in search patterns to represent one or more unknown characters, - allowing for flexible matching of strings. The wildcard characters are ``*`` and ``%`` and they replace any number - of characters preceding, following, or in between existing characters, depending on their placement. +The query will sort results first by the number of images in ascending order, and then by start time in descending order. We'll also limit +the number of results returned to 10 and select a subset of columns to return with the ``select_cols`` parameter. .. doctest-remote-data:: - >>> result = Catalogs.query_criteria(collection='ullyses', - ... catalog='sciencemetadata', - ... target_name_ullyses='NGC*', - ... target_classification='!Galaxy', - ... known_binary=False, - ... sp_class=['O', 'B'], - ... gaia_parallax=['<-0.1', '>=0'], - ... star_teff='30000..50000', - ... select_cols=['target_name_ullyses', 'target_classification', 'known_binary', 'sp_class', 'gaia_parallax', 'star_teff']) + >>> result = Catalogs.query_criteria( + ... collection="hsc", + ... coordinates="322.49324 12.16683", + ... radius="2 arcsec", # Query for sources within 2 arcseconds of the specified coordinates + ... targetname=["M-15", "NGC*"], # Query for targets with names 'M-15' or starting with 'NGC' + ... starttime="2006..2013", # Query for observations with starttime between 2006 and 2013 + ... sort_by=["numimages", "starttime"], # Sort results by number of images and then by starttime + ... sort_desc=[False, True], # Sort numimages in ascending order and starttime in descending order + ... limit=5, # Limit to 5 results + ... select_cols=["matchid", "matchra", "matchdec", "numimages", "starttime", "targetname"] + ... ) >>> result.pprint(max_width=-1) - target_name_ullyses target_classification known_binary sp_class gaia_parallax star_teff - mas K - ------------------- --------------------- ------------ -------- ------------- --------- - NGC346 ELS 043 Early B Dwarf False B -0.111579 33000.0 - NGC346 MPG 487 Late O Dwarf False O -0.389724 35800.0 - NGC 3109 EBU 20 Mid O Supergiant False O 0.876327 31150.0 - NGC 3109 EBU 34 Mid O Supergiant False O -0.126069 33050.0 - - -The `~astroquery.mast.CatalogsClass.query_region` and `~astroquery.mast.CatalogsClass.query_object` methods are -convenience wrappers around `~astroquery.mast.CatalogsClass.query_criteria`: - - - `~astroquery.mast.CatalogsClass.query_region` requires a positional constraint (``coordinates`` or ``region``). - - - `~astroquery.mast.CatalogsClass.query_object` requires an ``object_name`` and performs a cone search. - -Both methods also accept column-based criteria, which are applied in the same way as in `~astroquery.mast.CatalogsClass.query_criteria`. + matchid matchra matchdec numimages starttime targetname + deg deg + --------- ------------------ ------------------ --------- -------------------------- ---------- + 61895629 322.493715383149 12.166629788750484 1 2011-10-22 08:10:21.217000 M-15 + 11562863 322.49294957070185 12.166668540816076 2 2006-05-02 01:13:43.920000 NGC7078 + 16381110 322.4936372300021 12.166722963370844 3 2011-10-07 15:20:59.197000 M-15 + 105452327 322.4933282596331 12.16732046125442 3 2011-10-07 15:20:59.197000 M-15 + 49726591 322.4927927121447 12.166998250407733 3 2006-05-02 01:13:43.920000 NGC7078 + +The `~astroquery.mast.CatalogsClass.query_region` and `~astroquery.mast.CatalogsClass.query_object` methods are convenience methods for spatial queries. +`~astroquery.mast.CatalogsClass.query_region` allows you to specify a region on the sky using the ``coordinates``, ``radius``, and/or ``region`` parameters. +`~astroquery.mast.CatalogsClass.query_object` allows you to specify an ``object_name`` that will be resolved to coordinates and a ``radius`` for a cone search. +Both methods also support column-based criteria, sorting, and limiting of results, just like `~astroquery.mast.CatalogsClass.query_criteria`. + +For these queries, we will use the ``tic_v82`` collection, which refers to the +[TESS Input Catalog version 8.2](https://ui.adsabs.harvard.edu/abs/2022yCat.4039....0P/abstract). We'll use the `~astroquery.mast.CatalogsClass.query_region` +method to perform a simple cone search for sources within 1 arcminute of the coordinates (158.47924, -7.30962). .. doctest-remote-data:: - >>> result = Catalogs.query_region(collection='skymapperdr4', - ... coordinates="158.47924 -7.30962", - ... radius='1 arcmin', - ... select_cols=['object_id', 'raj2000', 'dej2000']) - >>> result.pprint(max_width=-1) - object_id raj2000 dej2000 - deg deg - ---------- ---------- --------- - 1116262239 158.475216 -7.29985 - 92307496 158.483015 -7.323183 - 92307428 158.46783 -7.319955 - 2151499732 158.466411 -7.319867 + >>> result = Catalogs.query_region( + ... collection="tic_v82", + ... catalog="source", + ... coordinates="158.47924 -7.30962", + ... radius="1 arcmin", + ... select_cols=["id", "ra", "dec"] + ... ) + >>> result.pprint(max_width=-1) #doctest: +IGNORE_OUTPUT + id ra dec + deg deg + --------- ---------------- ----------------- + 841736281 158.483019303286 -7.32320013067735 + 56661355 158.467833401313 -7.31994230664877 + 841736289 158.475246467012 -7.29984176473098 + +For this next query, we'll use the `~astroquery.mast.CatalogsClass.query_object` method to search for sources within 0.1 degrees of the object +["M11"](https://science.nasa.gov/mission/hubble/science/explore-the-night-sky/hubble-messier-catalog/messier-11/), which is an open star cluster +also known as the Wild Duck Cluster. We'll also filter for sources that are stars, sort results by the effective temperature of the source, and +limit the number of results to 10. .. doctest-remote-data:: - >>> result = Catalogs.query_object(collection='skymapperdr4', - ... object_name='M11', - ... radius=0.01, - ... catwise_id='2825m061*', - ... sort_by='mean_epoch', - ... select_cols=['object_id', 'raj2000', 'dej2000', 'catwise_id', 'mean_epoch']) - >>> result[:5].pprint(max_width=-1) - object_id raj2000 dej2000 catwise_id mean_epoch - deg deg d - ---------- ---------- --------- ------------------ ---------- - 277772662 282.75865 -6.273223 2825m061_b0-113841 56833.6242 - 277772668 282.762729 -6.274544 2825m061_b0-010099 56834.1188 - 2389914703 282.768791 -6.275556 2825m061_b0-045485 56834.6156 - 277772658 282.758508 -6.276218 2825m061_b0-049457 56834.6167 - 277772511 282.764593 -6.27897 2825m061_b0-046693 56895.2497 + >>> result = Catalogs.query_object( + ... collection="tic_v82", + ... object_name="M11", + ... radius=0.1, + ... objtype="STAR", + ... sort_by="teff", + ... select_cols=["id", "ra", "dec", "objtype", "teff"], + ... limit=10 + ... ) + >>> result.pprint(max_width=-1) + id ra dec objtype teff + deg deg K + --------- ---------------- ----------------- ------- ------ + 151449173 282.67863187603 -6.26184813416028 STAR 3025.0 + 151456872 282.798899858481 -6.3102827609116 STAR 3093.0 + 151455613 282.743580903863 -6.2188868480077 STAR 3260.0 + 31821653 282.851911590723 -6.26914233254122 STAR 3265.0 + 151456960 282.776343005483 -6.31830783055104 STAR 3281.0 + 151457177 282.777039432654 -6.34079199389906 STAR 3285.0 + 151457138 282.762562964854 -6.33622888690153 STAR 3290.0 + 151455292 282.769687283022 -6.18834508004063 STAR 3311.0 + 151455728 282.730778111903 -6.2287837491146 STAR 3325.0 + 151456312 282.737426625799 -6.26956499623787 STAR 3372.0 .. _specifying-spatial-regions: Specifying Spatial Regions -------------------------- -Catalogs that support spatial queries allow regions to be specified in several ways. +For catalogs that support spatial queries, there are several ways to specify the spatial region of interest. The simplest is a cone search, +defined by a center position and a radius. More complex regions, such as polygons, can also be specified using the ``region`` parameter. -Cone Searches -^^^^^^^^^^^^^ +Cone Search +^^^^^^^^^^^ Cone searches are the most common type of spatial query, defined by a center position and a radius. They may be specified using: - - ``coordinates`` and ``radius`` - - ``object_name`` and ``radius`` - - A `~regions.CircleSkyRegion` object as the ``region`` parameter - - A Space-Time Coordinate (STC) CIRCLE region string as the ``region`` parameter - -An STC-S CIRCLE region string has the following format: - -``CIRCLE [frame] `` - -For example: -``CIRCLE ICRS 210.8 54.35 0.05`` +- The ``coordinates`` and ``radius`` parameters together. +- The ``object_name`` and ``radius`` parameters together, where the object name is resolved to coordinates. +- The ``region`` parameter as: + - A `~regions.CircleSkyRegion` object from the `~regions` package. + - A Space-Time Coordinate (STC) string in the format ``CIRCLE [frame] ``, where ra, dec, and radius are in degrees. -This means: - - ICRS: Coordinate frame (optional; defaults to ICRS if omitted) - - 210.8: Right Ascension in degrees - - 54.35: Declination in degrees - - 0.05: Radius in degrees +Let's demonstrate this on the ``skymapperdr4`` collection, which contains catalogs from the +[SkyMapper Southern Survey: Data Release 4](https://skymapper.anu.edu.au/data-release/). The catalog we query will be "dr4.master", which is +a master catalog of sources detected in the SkyMapper DR4 survey, containing their positions, magnitudes, and other properties. Our cone +search will be centered on the coordinate (18.855, -6.945) with a radius of 0.01 degrees. .. doctest-remote-data:: - >>> result = Catalogs.query_region(collection='ps1_dr2', - ... region='CIRCLE ICRS 18.895 -6.944 0.01', - ... sort_by='objID', - ... select_cols=['objName', 'objID', 'raMean', 'decMean']) - >>> result[:5].pprint(max_width=-1) - objName objID raMean decMean - deg deg - ----------------------- ----------------- ----------------- ------------------ - PSX J011534.22-065706.1 99650188926188340 18.89261844 -6.95171552 - PSX J011534.90-065704.8 99650188954378784 18.89543744 -6.95134445 - PSX J011533.84-065651.9 99660188909803092 18.89100927680362 -6.947762708324004 - PSX J011533.97-065643.6 99660188915395866 18.8915527 -6.9454449 - PSX J011534.30-065636.0 99660188931668392 18.89294147 -6.94333417 - -Polygon Searches -^^^^^^^^^^^^^^^^^ + >>> from regions import CircleSkyRegion + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> + >>> circle_sky_region = CircleSkyRegion(center=SkyCoord(18.86, -6.95, unit='deg'), radius=0.01*u.deg) + >>> circle_stc_string = "CIRCLE ICRS 18.855 -6.945 0.01" + >>> + >>> result = Catalogs.query_region( + ... collection="skymapperdr4", + ... catalog="dr4.master", + ... region=circle_sky_region, # or use region=circle_stc_string + ... select_cols=["object_id", "raj2000", "dej2000"] + ... ) + >>> result.pprint(max_width=-1) + object_id raj2000 dej2000 + deg deg + ---------- --------- --------- + 2025217836 18.858273 -6.955493 + 2025218117 18.867198 -6.950921 + 2025218116 18.868135 -6.952049 + 2025217835 18.861348 -6.958706 + 1018207669 18.862164 -6.959528 + +Polygon Search +^^^^^^^^^^^^^^^ Polygon searches allow for more complex spatial queries by defining a polygonal region on the sky. They may be specified using any of the following as the ``region`` parameter: - - An iterable of coordinate pairs - - A `~regions.PolygonSkyRegion` object - - A Space-Time Coordinate (STC) POLYGON region string - -An STC-S POLYGON string has the form: - -``POLYGON [frame] ...`` - -For example: +- An iterable of (RA, Dec) tuples representing the vertices of the polygon. +- A `~regions.PolygonSkyRegion` object from the `~regions` package. +- A Space-Time Coordinate (STC) string in the format ``POLYGON [frame] ... ``, where (ra_i, dec_i) + are the vertices of the polygon in degrees. -``POLYGON ICRS 210.7 54.3 210.9 54.3 210.9 54.4 210.7 54.4`` +Keep in mind that at least three vertices are required to define a valid polygon, and the vertices should be ordered either clockwise or +counterclockwise. The polygon will be automatically closed by connecting the last vertex back to the first. -This defines a four-vertex polygon with vertices given as (RA, Dec) pairs in degrees. At least three vertices (six numbers) are required. +Let's search for sources in the Skymapper source catalog that fall within a four-sided polygon. The polygon has the same center as the +previous cone search, so the results may look similar! .. doctest-remote-data:: - >>> result = Catalogs.query_criteria(collection='caom', - ... region='POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94', - ... limit=5, - ... select_cols=['target_name', 'obs_id', 's_ra', 's_dec']) + >>> from regions import PolygonSkyRegion + >>> + >>> polygon_iter = [(18.85, -6.96), (18.87, -6.96), (18.87, -6.94), (18.85, -6.94)] + >>> polygon_sky_region = PolygonSkyRegion(vertices=SkyCoord( + ... [[18.85, -6.96], [18.87, -6.96], [18.87, -6.94], [18.85, -6.94]], + ... unit='deg' + ... )) + >>> polygon_stc_string = "POLYGON ICRS 18.85 -6.96 18.87 -6.96 18.87 -6.94 18.85 -6.94" + >>> + >>> result = Catalogs.query_region( + ... collection="skymapperdr4", + ... catalog="dr4.master", + ... region=polygon_sky_region, # or use region=polygon_sky_region or region=polygon_stc_string + ... select_cols=["object_id", "raj2000", "dej2000"] + ... ) >>> result.pprint(max_width=-1) - target_name obs_id s_ra s_dec - deg deg - ----------- ------------------------------------------------------------------------- ----------------- ------------------ - 408084461 hlsp_tglc_tess_ffi_gaiaid-2475555794352572672-s0003-cam1-ccd3_tess_v1_llc 18.85365901920825 -6.941643227687541 - 408084461 hlsp_t16_tess_ffi_s0003-cam1-ccd3-02475555794352572672_tess_v01 18.8536465402 -6.9416358523 - 408084461 hlsp_tasoc_tess_ffi_tic00408084461-s0003-cam1-ccd3-c1800_tess_v05 18.8532579321195 -6.94140657784404 - 408084461 hlsp_gsfc-eleanor-lite_tess_ffi_s0003-0000000408084461_tess_v1.0_lc 18.8532579321195 -6.941406577844042 - 408084461 hlsp_qlp_tess_ffi_s0097-0000000408084461_tess_v01_llc 18.8532579321 -6.94140657784 + object_id raj2000 dej2000 + deg deg + ---------- --------- --------- + 2025217836 18.858273 -6.955493 + 2025218117 18.867198 -6.950921 + 2025218116 18.868135 -6.952049 + 2025217835 18.861348 -6.958706 + 1018207669 18.862164 -6.959528 + 13693185 18.853648 -6.941675 + 2025218118 18.851606 -6.942183 + 1018207883 18.851708 -6.941662 Counting Results @@ -392,15 +489,31 @@ Counting Results All query methods support a ``count_only=True`` option, which returns only the number of matching records: +Each of the three query methods supports a ``count_only`` parameter. When set to ``True``, it returns only the number of matching results. +This can be useful for quickly assessing the size of a result set without having to retrieve all the data. + +Let's demonstrate this on the ``caom`` collection, which refers to the +[Common Archive Observation Model](https://www.ivoa.net/documents/CAOM/20240927/WD-CAOM-2.5-20240927.pdf), a data model used to +describe astronomical observations. We'll query the ``obspointing`` catalog, which contains metadata about scientific observations at MAST. +We'll perform a query to count the number of observations that are within 0.2 degrees of the exoplanet +[WASP-12b](https://science.nasa.gov/exoplanet-catalog/wasp-12-b/). You'll notice that this query takes several seconds to complete. +The ``obspointing`` catalog is huge, and if you were to run the same query without the ``count_only`` parameter, it would typically take +longer to return the full results, especially if there are many matching sources. + +For very long-running queries, you can also run the query in asynchronous mode by setting ``run_async=True``. This will prevent the query from timing out. + .. doctest-remote-data:: - >>> count = Catalogs.query_criteria(collection='skymapperdr4', - ... region=[(20, -5), (20, -6), (21, -6), (21, -5)], - ... count_only=True) + >>> count = Catalogs.query_criteria( + ... collection="caom", + ... catalog="obspointing", + ... object_name="WASP-12b", + ... radius=0.2, + ... count_only=True, + ... run_async=True + ... ) >>> print('Number of matching records:', count) - Number of matching records: 5479 - -This is useful for estimating result sizes before executing large queries. + Number of matching records: 6735 Deprecated Interfaces ====================== From f778d448c346fdadcff821db4c8d663e0989c5c0 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Fri, 15 May 2026 15:44:44 -0400 Subject: [PATCH 21/21] Update deprecation version, note about installing older version --- astroquery/mast/collections.py | 31 ++++++++-------- docs/mast/mast_catalog.rst | 64 ++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index a14f2ab71b..6ea08a25b3 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -200,25 +200,25 @@ def supports_spatial_queries(self, collection=None, catalog=None): @deprecated_renamed_argument( "version", None, - since="0.4.12", + since="0.4.13", message="The `version` argument is deprecated and " "will be removed in a future release. Please use `collection` and `catalog` instead.", ) @deprecated_renamed_argument( "pagesize", None, - since="0.4.12", + since="0.4.13", message="The `pagesize` argument is deprecated " "and will be removed in a future release. Please use `limit` instead.", ) @deprecated_renamed_argument( "page", None, - since="0.4.12", + since="0.4.13", message="The `page` argument is deprecated " "and will be removed in a future release. Please use `offset` instead.", ) - @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + @deprecated_renamed_argument("objectname", "object_name", since="0.4.13") def query_criteria( self, collection=None, @@ -454,21 +454,21 @@ def query_criteria( @deprecated_renamed_argument( "version", None, - since="0.4.12", + since="0.4.13", message="The `version` argument is deprecated and " "will be removed in a future release. Please use `collection` and `catalog` instead.", ) @deprecated_renamed_argument( "pagesize", None, - since="0.4.12", + since="0.4.13", message="The `pagesize` argument is deprecated " "and will be removed in a future release. Please use `limit` instead.", ) @deprecated_renamed_argument( "page", None, - since="0.4.12", + since="0.4.13", message="The `page` argument is deprecated " "and will be removed in a future release. Please use `offset` instead.", ) @@ -601,25 +601,25 @@ def query_region( @deprecated_renamed_argument( "version", None, - since="0.4.12", + since="0.4.13", message="The `version` argument is deprecated and " "will be removed in a future release. Please use `collection` and `catalog` instead.", ) @deprecated_renamed_argument( "pagesize", None, - since="0.4.12", + since="0.4.13", message="The `pagesize` argument is deprecated " "and will be removed in a future release. Please use `limit` instead.", ) @deprecated_renamed_argument( "page", None, - since="0.4.12", + since="0.4.13", message="The `page` argument is deprecated " "and will be removed in a future release. Please use `offset` instead.", ) - @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + @deprecated_renamed_argument("objectname", "object_name", since="0.4.13") def query_object( self, object_name, @@ -736,7 +736,7 @@ def query_object( ) @class_or_instance - @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + @deprecated(since="v0.4.13", message=("This function is deprecated and will be removed in a future release.")) def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None): """ Returns all the matches for a given Hubble Source Catalog MatchID. @@ -776,7 +776,7 @@ def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None) return self._current_connection.service_request_async(service, params, pagesize, page) @class_or_instance - @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + @deprecated(since="v0.4.13", message=("This function is deprecated and will be removed in a future release.")) def get_hsc_spectra_async(self, *, pagesize=None, page=None): """ Returns all Hubble Source Catalog spectra. @@ -797,7 +797,7 @@ def get_hsc_spectra_async(self, *, pagesize=None, page=None): self._current_connection = self._portal_api_connection return self._current_connection.service_request_async("Mast.HscSpectra.Db.All", {}, pagesize, page) - @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) + @deprecated(since="v0.4.13", message=("This function is deprecated and will be removed in a future release.")) def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_flag=False): """ Download one or more Hubble Source Catalog spectra. @@ -928,7 +928,8 @@ def _verify_collection(self, collection): if collection in self._no_longer_supported_collections: error_msg = ( f"Collection '{collection}' is no longer supported. To query from this catalog, " - f"please use a version of Astroquery older than 0.4.12." + f"please use a version of Astroquery older than 0.4.13. You can install version 0.4.12 using " + f"`pip install astroquery==0.4.12`" ) else: closest = difflib.get_close_matches(collection, self.available_collections, n=1) diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index 97b31d688d..d2c21c566b 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -4,10 +4,10 @@ Catalog Queries *************** `~astroquery.mast.CatalogsClass` is a versatile tool for discovering and querying the wide range of astronomical catalogs hosted by the -[Mikulski Archive for Space Telescopes (MAST)](https://archive.stsci.edu/). `~astroquery.mast.CatalogsClass` is a Python wrapper -for our [MAST Table Access Protocol (TAP) Service](https://mast.stsci.edu/vo-tap/), which allows you to query for catalog +`Mikulski Archive for Space Telescopes (MAST) `_. `~astroquery.mast.CatalogsClass` is a Python wrapper +for our `MAST Table Access Protocol (TAP) Service `_, which allows you to query for catalog metadata and data. If you were querying the MAST TAP service directly, you would need to write your queries in -[Astronomical Data Query Language (ADQL)](https://www.ivoa.net/documents/latest/ADQL.html). With `~astroquery.mast.CatalogsClass`, +`Astronomical Data Query Language (ADQL) `_. With `~astroquery.mast.CatalogsClass`, you can construct and execute these queries using a more intuitive Python interface, without needing to learn ADQL syntax. The catalogs available through MAST are diverse, covering a wide range of astronomical objects and phenomena. @@ -17,6 +17,7 @@ The `~astroquery.mast.CatalogsClass` interface is designed for flexible querying as well as the ability to filter results based on specific criteria. At a high level, querying MAST catalogs with `~astroquery.mast.CatalogsClass` involves the following steps: + 1. **Discover** available collections and catalogs. 2. **Inspect** catalog metadata to understand available columns and data types, as well as the capabilities of each catalog. 3. **Query** the catalog using spatial and/or non-spatial criteria to retrieve relevant data. @@ -33,7 +34,7 @@ is specified in a query, these attributes will be used as defaults. The ``collec representing the current collection, and the ``catalog`` attribute is a string representing the name of the current catalog within that collection. -The default value for the ``collection`` attribute is "hsc", referring to the [Hubble Source Catalog version 3](https://archive.stsci.edu/hst/hsc/). +The default value for the ``collection`` attribute is "hsc", referring to the `Hubble Source Catalog version 3 `_. The default value for ``catalog`` is "dbo.SumMagAper2CatView". This is a summary source catalog with data describing sources detected in Hubble images, including their positions, magnitudes, and other properties. @@ -55,7 +56,7 @@ at any time after instantiation to set new defaults for subsequent queries. Both collection, the catalog will be reset to the default catalog for the new collection. Here, we'll change the value of ``collection`` to "ullyses", referring to the -[Hubble Ultraviolet Legacy Library of Young Stars as Essential Standards (ULLYSES)](https://ullyses.stsci.edu/) program. The default catalog +`Hubble Ultraviolet Legacy Library of Young Stars as Essential Standards (ULLYSES) `_ program. The default catalog for this collection is "sciencemetadata", which contains metadata about the scientific exposures taken as part of the ULLYSES program, including their coordinates, observation dates, instruments used, and other properties. @@ -93,9 +94,6 @@ Discovering Available Collections To list all of the catalog collections that are accessible via the MAST TAP Service, use the `~astroquery.mast.CatalogsClass.get_collections` method. This returns an `~astropy.table.Table` containing the names of all available collections. -Some historical collections are no longer supported for querying, and will not appear in this list. If a collection -has been renamed or deprecated, Astroquery will issue a warning and suggest the appropriate replacement where possible. - .. doctest-remote-data:: >>> collections = Catalogs.get_collections() @@ -121,6 +119,12 @@ has been renamed or deprecated, Astroquery will issue a warning and suggest the tic_v82 ullyses +.. attention:: + Some historical collections are no longer supported for querying and will not appear in this list. If a collection + has been renamed or deprecated, Astroquery will issue a warning and suggest the appropriate replacement where possible. + To query a catalog that is no longer supported, you can install an older version of Astroquery that still supports it. + To install the latest version that supports legacy collections, use the following command: ``pip install astroquery==0.4.12`` + Discovering Catalogs Within a Collection ----------------------------------------- @@ -207,21 +211,21 @@ Shared Query Parameters The following parameters are shared across all three query methods: - - ``collection`` : The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. - - ``collection``: The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. - - ``catalog``: The name of the catalog to query within the specified collection. If not specified, the ``catalog`` class attribute will be used as the default. - - ``limit``: An integer specifying the maximum number of results to return. The default is 5,000. - - ``offset``: An integer specifying the number of results to skip before starting to return results. This is useful for paginating through large result sets. Default is 0. - - ``count_only``: A boolean indicating whether to return only the count of matching results instead of the results themselves. Default is False. - - ``select_cols``: A list of column names to include in the results. If not specified, all columns will be returned. - - ``sort_by``: A string or list of strings specifying the column(s) to sort the results by. Default is None (no sorting). - - ``sort_desc``: A boolean or list of booleans specifying whether to sort in descending order for each column specified in ``sort_by``. - Default is False (ascending order). - - ``filters``: Another parameter used to specify criteria filters as a dictionary. Use this option when the name of a column conflicts - with a named parameter of this method. - - ``run_async``: If True, run the query in asynchronous mode. This mode is more robust and preferable for long-running queries. - - ``return_adql``: If True, return the ADQL query string instead of executing the query. This is useful for debugging or for users who want - to run the query directly against the TAP service. When False, the ADQL query string is also returned in the metadata of the result table. +- ``collection`` : The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. +- ``collection``: The name of the catalog collection to query. If not specified, the ``collection`` class attribute will be used as the default. +- ``catalog``: The name of the catalog to query within the specified collection. If not specified, the ``catalog`` class attribute will be used as the default. +- ``limit``: An integer specifying the maximum number of results to return. The default is 5,000. +- ``offset``: An integer specifying the number of results to skip before starting to return results. This is useful for paginating through large result sets. Default is 0. +- ``count_only``: A boolean indicating whether to return only the count of matching results instead of the results themselves. Default is False. +- ``select_cols``: A list of column names to include in the results. If not specified, all columns will be returned. +- ``sort_by``: A string or list of strings specifying the column(s) to sort the results by. Default is None (no sorting). +- ``sort_desc``: A boolean or list of booleans specifying whether to sort in descending order for each column specified in ``sort_by``. + Default is False (ascending order). +- ``filters``: Another parameter used to specify criteria filters as a dictionary. Use this option when the name of a column conflicts + with a named parameter of this method. +- ``run_async``: If True, run the query in asynchronous mode. This mode is more robust and preferable for long-running queries. +- ``return_adql``: If True, return the ADQL query string instead of executing the query. This is useful for debugging or for users who want + to run the query directly against the TAP service. When False, the ADQL query string is also returned in the metadata of the result table. These parameters allow users to control the scope and format of their queries consistently across all three methods. @@ -294,7 +298,7 @@ If a catalog supports spatial queries, the following parameters can be used to s - ``radius``: The radius of a cone search around ``coordinates`` or ``object_name``. Can be defined as a string with units (e.g., "10 arcsec"), a `~astropy.units.Quantity`, or a float in degrees. Default is 0.2 degrees. - ``region``: Specifies the spatial region of interest for more complex spatial queries, such as polygon searches. - Please see the [Specifying Spatial Regions](#specifying-spatial-regions) section below for details on how to use this parameter. + Please see the `Specifying Spatial Regions`_ section below for details on how to use this parameter. If no spatial parameters are provided, the query is purely column-based and will not filter results based on position. If they are supplied, the spatial parameters are combined with any column-based criteria using a logical **AND**, meaning that only results that satisfy both the spatial and @@ -338,7 +342,7 @@ The `~astroquery.mast.CatalogsClass.query_region` and `~astroquery.mast.Catalogs Both methods also support column-based criteria, sorting, and limiting of results, just like `~astroquery.mast.CatalogsClass.query_criteria`. For these queries, we will use the ``tic_v82`` collection, which refers to the -[TESS Input Catalog version 8.2](https://ui.adsabs.harvard.edu/abs/2022yCat.4039....0P/abstract). We'll use the `~astroquery.mast.CatalogsClass.query_region` +`TESS Input Catalog version 8.2 `_. We'll use the `~astroquery.mast.CatalogsClass.query_region` method to perform a simple cone search for sources within 1 arcminute of the coordinates (158.47924, -7.30962). .. doctest-remote-data:: @@ -359,7 +363,7 @@ method to perform a simple cone search for sources within 1 arcminute of the coo 841736289 158.475246467012 -7.29984176473098 For this next query, we'll use the `~astroquery.mast.CatalogsClass.query_object` method to search for sources within 0.1 degrees of the object -["M11"](https://science.nasa.gov/mission/hubble/science/explore-the-night-sky/hubble-messier-catalog/messier-11/), which is an open star cluster +`M11 `_, which is an open star cluster also known as the Wild Duck Cluster. We'll also filter for sources that are stars, sort results by the effective temperature of the source, and limit the number of results to 10. @@ -409,7 +413,7 @@ Cone searches are the most common type of spatial query, defined by a center pos - A Space-Time Coordinate (STC) string in the format ``CIRCLE [frame] ``, where ra, dec, and radius are in degrees. Let's demonstrate this on the ``skymapperdr4`` collection, which contains catalogs from the -[SkyMapper Southern Survey: Data Release 4](https://skymapper.anu.edu.au/data-release/). The catalog we query will be "dr4.master", which is +`SkyMapper Southern Survey: Data Release 4 `_. The catalog we query will be "dr4.master", which is a master catalog of sources detected in the SkyMapper DR4 survey, containing their positions, magnitudes, and other properties. Our cone search will be centered on the coordinate (18.855, -6.945) with a radius of 0.01 degrees. @@ -487,16 +491,14 @@ previous cone search, so the results may look similar! Counting Results ----------------- -All query methods support a ``count_only=True`` option, which returns only the number of matching records: - Each of the three query methods supports a ``count_only`` parameter. When set to ``True``, it returns only the number of matching results. This can be useful for quickly assessing the size of a result set without having to retrieve all the data. Let's demonstrate this on the ``caom`` collection, which refers to the -[Common Archive Observation Model](https://www.ivoa.net/documents/CAOM/20240927/WD-CAOM-2.5-20240927.pdf), a data model used to +`Common Archive Observation Model `_, a data model used to describe astronomical observations. We'll query the ``obspointing`` catalog, which contains metadata about scientific observations at MAST. We'll perform a query to count the number of observations that are within 0.2 degrees of the exoplanet -[WASP-12b](https://science.nasa.gov/exoplanet-catalog/wasp-12-b/). You'll notice that this query takes several seconds to complete. +`WASP-12b `_. You'll notice that this query takes several seconds to complete. The ``obspointing`` catalog is huge, and if you were to run the same query without the ``count_only`` parameter, it would typically take longer to return the full results, especially if there are many matching sources.