diff --git a/CHANGES.rst b/CHANGES.rst index ac5c229ec3..3b34c14322 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -179,6 +179,8 @@ mast - Added ``pass_id`` as an alias for the ``pass`` column in query functions for the Roman mission to avoid conflicts with the reserved Python keyword. [#3588] - Update the cutout format request parameter in ``Zcut.download_cutouts`` to reflect a recent service change. [#3608] +- Added a ``mission`` parameter to methods in ``MastMissions`` to allow overriding the default mission for a query. [#3618] +- The ``MastMissions.get_available_missions`` method retrieves the list of available missions and caches it for future use. [#3618] jplspec diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index 94ae869a2e..443da70252 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -8,6 +8,7 @@ import difflib import json +import re import warnings from collections.abc import Iterable from json import JSONDecodeError @@ -22,19 +23,12 @@ from requests import HTTPError, RequestException from astroquery import log -from astroquery.exceptions import ( - InputWarning, - InvalidQueryError, - MaxResultsWarning, - NoResultsWarning, -) +from astroquery.exceptions import (InputWarning, InvalidQueryError, MaxResultsWarning, NoResultsWarning) from astroquery.mast import utils from astroquery.mast.core import MastQueryWithLogin from astroquery.utils import async_to_sync, commons from astroquery.utils.class_or_instance import class_or_instance -from . import conf - __all__ = ['MastMissionsClass', 'MastMissions'] @@ -60,13 +54,14 @@ class MastMissionsClass(MastQueryWithLogin): # Maximum number of input targets accepted in a single query _max_input_targets = 100 - def __init__(self, *, mission='hst', mast_token=None): + def __init__(self, *, mission=None, mast_token=None): super().__init__(mast_token=mast_token) self.dataset_kwds = { # column keywords corresponding to dataset ID 'hst': 'sci_data_set_name', 'jwst': 'fileSetName', 'roman': 'fileSetName', + 'roman_cgi': 'fileSetName', 'classy': 'Target', 'ullyses': 'observation_id', 'iue': 'iue_data_id' @@ -74,456 +69,232 @@ def __init__(self, *, mission='hst', mast_token=None): # Service attributes self.service = self._search # current API service - self.service_dict = {self._search: {'path': self._search}, - self._list_products: {'path': self._list_products}} + service_dict = {self._search: {'path': self._search}, + self._list_products: {'path': self._list_products}} + self._service_api_connection.set_service_params(service_dict, 'search') + + self._available_missions = None # Lazy loading of available missions from MAST Search API # Search attributes self._search_option_fields = ['limit', 'offset', 'sort_by', 'search_key', 'sort_desc', 'select_cols', 'skip_count', 'user_fields'] - self.mission = mission # current mission self.limit = 5000 # maximum number of results self.columns = dict() # columns configuration for each mission + # The default mission to query. This is used when a mission is not explicitly provided in query methods. + # Default initialization of this class should not trigger network requests + # Only set default mission if explicitly provided, otherwise defer to property setters + # which will handle defaults without network calls + if not mission: + self._mission = "hst" # default mission + else: + self.mission = mission # Use the setter for validation if mission is provided + + self._mission_to_search = self.mission # mission in focus for metadata search use in parse_result + @property def mission(self): + """The default mission to query.""" return self._mission @mission.setter def mission(self, value): - # Setter that updates the service parameters if the mission is changed + """Set the mission to query and verify that it is available in the MAST Search API.""" + self._verify_mission(value.lower()) self._mission = value.lower() # case-insensitive - self._service_api_connection.set_service_params(self.service_dict, f'search/{self.mission}') - def _extract_products(self, response): - """ - Extract products from the response of a `~requests.Response` object. + @property + def available_missions(self): + """Return a list of available missions in the MAST Search API.""" + if self._available_missions is None: + self._available_missions = self.get_available_missions() + return self._available_missions - Parameters - ---------- - response : `~requests.Response` - The response object containing the products data. + def get_available_missions(self): + """ + Return a list of available missions in the MAST Search API. Returns ------- - list - A list of products extracted from the response. + missions : list + List of available missions in MAST Search API. """ - combined = [] - for resp in response: - products = resp.json().get('products', []) - # Flatten if nested - if products and isinstance(products[0], list): - products = products[0] - combined.extend(products) - return combined + if self._available_missions is not None: + return self._available_missions - def _parse_result(self, response, *, verbose=False): # Used by the async_to_sync decorator functionality + missions = [] + html = utils._simple_request(f"{self._service_api_connection.MISSIONS_URL}docs").text + + for match in re.findall(r'[\w/\.-]+openapi[\w/\.-]*\.json', html): + if match.startswith("/"): + continue + + mission = re.search(r'(.+?)_api_openapi\.json$', match).group(1) + if mission: + missions.append(mission) + + return missions + + @class_or_instance + @deprecated_renamed_argument('objectname', 'object_names', since='0.4.12') + def query_criteria_async(self, *, coordinates=None, object_names=None, radius=3*u.arcmin, + limit=5000, offset=0, select_cols=None, resolver=None, mission=None, **criteria): """ - Parse the results of a `~requests.Response` objects and return an `~astropy.table.Table` of results. + Given a set of search criteria, returns a list of mission metadata. Parameters ---------- - response : `~requests.Response` - `~requests.Response` objects. - verbose : bool - (presently does nothing - there is no output with verbose set to - True or False) - Default False. Setting to True provides more extensive output. + coordinates : str, iterable of str, or `~astropy.coordinates` object + Coordinate target(s) around which to search. Can be specified as: + - A single coordinate string or `~astropy.coordinates.SkyCoord` object + - A comma-separated string of coordinates (e.g., "10.0 20.0, 15.0 25.0") + - An iterable of coordinate strings or coordinate objects + object_names : str or iterable of str, optional + Object name target(s) around which to search. Can be specified as: + - A single object name string + - A comma-separated string of object names (e.g., "M31, M51, NGC 1234") + - An iterable of object name strings + If both ``coordinates`` and ``object_names`` are provided, they are combined. + radius : str or `~astropy.units.Quantity` object + Default is 3 arcminutes. The radius around the coordinates to search within. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + The maximum supported query radius is 30 arcminutes. + limit : int + Default is 5000. The maximum number of dataset IDs in the results. + offset : int + Default is 0. The number of records you wish to skip before selecting records. + select_cols: iterable or str or None, optional + Default is None. Names of columns that will be included in the result table. + If None, a default set of columns will be returned. + Can either be an iterable of column names, a comma-separated string of column names, + or 'all'/'*' to return all available columns. + resolver : str, optional + Default is None. The resolver to use when resolving a named target into coordinates. Valid options are + "SIMBAD" and "NED". If not specified, the default resolver order will be used. Please see the + `STScI Archive Name Translation Application (SANTA) `__ + for more information. Default is None. + mission : str, optional + The mission for which to query. If not provided, the current value of the + ``mission`` attribute will be used. + **criteria + Criteria to apply. Valid criteria include coordinates, object_names, radius (as in + `~astroquery.mast.missions.MastMissionsClass.query_region` and + `~astroquery.mast.missions.MastMissionsClass.query_object` functions), + and all fields listed in the column documentation for the mission being queried. + List of all valid fields that can be used to match results on criteria can be retrieved by calling + `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. + To filter by multiple values for a single column, pass in a list of values or + a comma-separated string of values. For the Roman mission, you can also use the special "pass_id" + keyword as an alias for the "pass" column, which is a reserved keyword in Python. Returns ------- - response : `~astropy.table.Table` + response : list of `~requests.Response` + + Raises + ------ + InvalidQueryError + If the query radius is larger than the limit (30 arcminutes). """ - if self.service == self._search: - results = self._service_api_connection._parse_result(response, verbose, data_key='results') + self.limit = limit + self.service = self._search - # Add column descriptions to column metadata - column_list = self.get_column_list() - for col in results.columns: - if col in column_list['name']: - description = column_list[column_list['name'] == col]['description'].value[0] - results[col].meta = {'description': str(description)} + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) + self._mission_to_search = mission - # Add search parameters to table metadata - result_json = response.json() - results.meta['search_params'] = result_json.get('search_params', {}) + # Check that criteria arguments are valid + self._validate_criteria(mission=mission, **criteria) - # Warn if maximum results are returned - if len(results) >= self.limit: - warnings.warn("Maximum results returned, may not include all sources within radius.", - MaxResultsWarning) - return results + target_strings = None + if coordinates is not None or object_names is not None: + target_strings = self._parse_multiple_targets(coordinates=coordinates, + object_names=object_names, + resolver=resolver) - elif self.service == self._list_products: - products = self._extract_products(response) - return Table(products) + # if radius is just a number we assume degrees + radius = Angle(radius, u.arcmin) - def _validate_criteria(self, **criteria): - """ - Check that criteria keyword arguments are valid column names for the mission. - Raises InvalidQueryError if a criteria argument is invalid. + if radius > self._max_query_radius: + raise InvalidQueryError( + f"Query radius too large. Must be ≤{self._max_query_radius}, got {radius}." + ) - Parameters - ---------- - **criteria - Keyword arguments representing criteria filters to apply. + # build query + params = {"limit": self.limit, "offset": offset, "select_cols": self._parse_select_cols(select_cols, mission)} + if target_strings: + params["target"] = target_strings + params["radius"] = radius.arcsec + params["radius_units"] = 'arcseconds' - Raises - ------- - InvalidQueryError - If a keyword does not match any valid column names, an error is raised that suggests the closest - matching column name, if available. - """ - # Ensure that self.columns is populated - self.get_column_list() + self._build_params_from_criteria(params, **criteria) - # Check each criteria argument for validity - valid_cols = list(self.columns[self.mission]['name']) + self._search_option_fields - for kwd in criteria.keys(): - if kwd == "pass_id" and "pass_id" not in valid_cols and "pass" in valid_cols: - # Special case where the actual column name is "pass", but that's a reserved keyword in Python - # We allow "pass_id" as an alias - kwd = "pass" - col = next((name for name in valid_cols if name == kwd), None) - if not col: - closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) - error_msg = ( - f"Filter '{kwd}' does not exist. Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{kwd}' does not exist." - ) - raise InvalidQueryError(error_msg) + return self._service_api_connection.missions_request_async(self.service, params, mission) - def _build_params_from_criteria(self, params, **criteria): + @class_or_instance + def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, + select_cols=None, mission=None, **criteria): """ - Build the parameters for the API request based on the provided criteria. + Given a sky position (or positions) and radius, returns a list of matching dataset IDs. Parameters ---------- - params : dict - Dictionary to store the parameters for the API request. + coordinates : str, iterable of str, or `~astropy.coordinates` object + The target(s) around which to search. Can be specified as: + - A single coordinate string or `~astropy.coordinates.SkyCoord` object + - A comma-separated string of coordinates (e.g., "10.0 20.0, 15.0 25.0") + - An iterable of coordinate strings or `~astropy.coordinates` objects + radius : str or `~astropy.units.Quantity` object + Default is 3 arcminutes. The radius around the coordinates to search within. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + The maximum supported query radius is 30 arcminutes. + limit : int + Default is 5000. The maximum number of dataset IDs in the results. + offset : int + Default is 0. The number of records you wish to skip before selecting records. + select_cols: iterable or str or None, optional + Default is None. Names of columns that will be included in the result table. + If None, a default set of columns will be returned. + Can either be an iterable of column names, a comma-separated string of column names, + or 'all'/'*' to return all available columns. + mission : str, optional + The mission for which to query. If not provided, the current value of the + ``mission`` attribute will be used. **criteria - Keyword arguments representing criteria filters to apply. - """ - # Add each criterion to the params dictionary - params['conditions'] = [] - for prop, value in criteria.items(): - if prop not in self._search_option_fields: - if isinstance(value, list): - # Convert to comma-separated string if passed as a list - value = ','.join(str(item) for item in value) - params['conditions'].append({prop: value}) - else: - if prop == 'sort_by' and isinstance(value, str): - # Convert to list if passed as a string - value = [value] - if prop == 'sort_desc' and isinstance(value, bool): - # Convert to list if passed as a boolean - value = [value] - params[prop] = value - - def _parse_select_cols(self, select_cols): - """ - Parse the select_cols parameter to ensure it is in the correct format. - - Parameters - ---------- - select_cols : iterable or str or None - The select_cols parameter to parse. + Other mission-specific criteria arguments. + All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` + function. + For example, one can specify the output columns(select_cols) or use other filters(conditions). + To filter by multiple values for a single column, pass in a list of values or + a comma-separated string of values. For the Roman mission, you can also use the special "pass_id" + keyword as an alias for the "pass" column, which is a reserved keyword in Python. Returns ------- - list - A list of column names to select. + response : list of `~requests.Response` Raises ------ InvalidQueryError - If select_cols is not an iterable of strings, a comma-separated string, 'all', or '*'. - If any individual column name is not a string. + If the query radius is larger than the limit (30 arcminutes). """ - if select_cols is None: - if self.mission == 'ullyses': - select_cols = self._default_ullyses_cols - return select_cols - - # Handle special string cases first - all_columns = self.get_column_list()['name'].value.tolist() - if isinstance(select_cols, str): - if (select_cols.lower() == 'all' or select_cols == '*'): - return all_columns - # Comma-separated string - select_cols = select_cols.split(',') - - # Handle an iterable - elif isinstance(select_cols, Iterable): - # Convert to list so we can iterate multiple times safely - select_cols = list(select_cols) + return self.query_criteria_async(coordinates=coordinates, + radius=radius, + limit=limit, + offset=offset, + select_cols=select_cols, + mission=mission, + **criteria) - else: - raise InvalidQueryError( - "`select_cols` must be an iterable of column names, a comma-separated string, " - "'all', or '*'." - ) - - # Validate the column names - valid_select_cols = [] - for col in select_cols: - if not isinstance(col, str): - raise InvalidQueryError( - "`select_cols` must contain only strings (column names)." - ) - col = col.strip() - if col not in all_columns: - closest_match = difflib.get_close_matches(col, all_columns, n=1) - suggestion = f' Did you mean "{closest_match[0]}"?' if closest_match else '' - warnings.warn(f"Column '{col}' not found.{suggestion}", InputWarning) - else: - valid_select_cols.append(col) - - # Dataset ID column should always be returned - dataset_col = self.dataset_kwds.get(self.mission, None) - if dataset_col and dataset_col not in valid_select_cols: - valid_select_cols.append(dataset_col) - return valid_select_cols - - def _parse_multiple_targets(self, *, coordinates=None, object_names=None, resolver=None): - """ - Parse coordinate and object-name targets into a list of API target strings. - - Parameters - ---------- - coordinates : str, iterable of str, or `~astropy.coordinates` object, optional - Coordinate target(s). Can be a single coordinate string/object, a comma-separated - coordinate string, an iterable of coordinate strings/objects, or a vector - `~astropy.coordinates.SkyCoord`. - object_names : str or iterable of str, optional - Object-name target(s). Can be a single object name string, a comma-separated - object-name string, or an iterable of object-name strings. - resolver : str, optional - The resolver to use when resolving named targets into coordinates. - - Returns - ------- - list of str - A list of target strings in "ra dec" format for the API. - """ - def _as_list(values): - """Normalize the input values into a list of strings or coordinate objects.""" - if values is None: - items = [] - elif isinstance(values, str): - items = [item.strip() for item in values.split(',') if item.strip()] - elif isinstance(values, Iterable) and not isinstance(values, (SkyCoord, BaseCoordinateFrame)): - items = list(values) - else: - items = [values] - - return [item.strip() if isinstance(item, str) else item for item in items] - - def _is_legacy_ra_dec_pair(items): - """Detect ['ra', 'dec'] passed as one coordinate split on comma.""" - if len(items) != 2 or not all(isinstance(item, str) for item in items): - return False - - try: - float(items[0]) - float(items[1]) - return True - except ValueError: - return False - - coordinate_items = _as_list(coordinates) - object_name_items = _as_list(object_names) - - # Backward compatibility for historical single-coordinate input like: - # coordinates="10.5, -20.1" - if _is_legacy_ra_dec_pair(coordinate_items): - coordinate_items = [f"{coordinate_items[0]} {coordinate_items[1]}"] - - total_targets = len(coordinate_items) + len(object_name_items) - - if total_targets == 0: - raise InvalidQueryError('No targets were provided.') - - if total_targets > self._max_input_targets: - raise InvalidQueryError( - f'Too many input targets provided. Maximum supported is {self._max_input_targets}, ' - f'got {total_targets}.' - ) - - targets = [] - - # Parse coordinate targets - for coord in coordinate_items: - sc = commons.parse_coordinates(coord, return_frame='icrs') - # If input is a vector SkyCoord, iterate through each coordinate - if isinstance(sc, SkyCoord) and sc.isscalar is False: - for ra, dec in zip(sc.ra.deg, sc.dec.deg): - targets.append(f"{ra} {dec}") - else: - targets.append(f"{sc.ra.deg} {sc.dec.deg}") - - # Parse object name targets - if object_names: - resolved = utils.resolve_object(object_name_items, resolver=resolver) - for name in object_name_items: - sc = resolved if isinstance(resolved, SkyCoord) else resolved.get(name) - if sc: - targets.append(f"{sc.ra.deg} {sc.dec.deg}") - - return targets - - @class_or_instance - @deprecated_renamed_argument('objectname', 'object_names', since='0.4.12') - def query_criteria_async(self, *, coordinates=None, object_names=None, radius=3*u.arcmin, - limit=5000, offset=0, select_cols=None, resolver=None, **criteria): - """ - Given a set of search criteria, returns a list of mission metadata. - - Parameters - ---------- - coordinates : str, iterable of str, or `~astropy.coordinates` object - Coordinate target(s) around which to search. Can be specified as: - - A single coordinate string or `~astropy.coordinates.SkyCoord` object - - A comma-separated string of coordinates (e.g., "10.0 20.0, 15.0 25.0") - - An iterable of coordinate strings or coordinate objects - object_names : str or iterable of str, optional - Object name target(s) around which to search. Can be specified as: - - A single object name string - - A comma-separated string of object names (e.g., "M31, M51, NGC 1234") - - An iterable of object name strings - If both ``coordinates`` and ``object_names`` are provided, they are combined. - radius : str or `~astropy.units.Quantity` object - Default is 3 arcminutes. The radius around the coordinates to search within. - The string must be parsable by `~astropy.coordinates.Angle`. The - appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. - The maximum supported query radius is 30 arcminutes. - limit : int - Default is 5000. The maximum number of dataset IDs in the results. - offset : int - Default is 0. The number of records you wish to skip before selecting records. - select_cols: iterable or str or None, optional - Default is None. Names of columns that will be included in the result table. - If None, a default set of columns will be returned. - Can either be an iterable of column names, a comma-separated string of column names, - or 'all'/'*' to return all available columns. - resolver : str, optional - Default is None. The resolver to use when resolving a named target into coordinates. Valid options are - "SIMBAD" and "NED". If not specified, the default resolver order will be used. Please see the - `STScI Archive Name Translation Application (SANTA) `__ - for more information. Default is None. - **criteria - Criteria to apply. Valid criteria include coordinates, object_names, radius (as in - `~astroquery.mast.missions.MastMissionsClass.query_region` and - `~astroquery.mast.missions.MastMissionsClass.query_object` functions), - and all fields listed in the column documentation for the mission being queried. - List of all valid fields that can be used to match results on criteria can be retrieved by calling - `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. - To filter by multiple values for a single column, pass in a list of values or - a comma-separated string of values. For the Roman mission, you can also use the special "pass_id" - keyword as an alias for the "pass" column, which is a reserved keyword in Python. - - Returns - ------- - response : list of `~requests.Response` - - Raises - ------ - InvalidQueryError - If the query radius is larger than the limit (30 arcminutes). - """ - - self.limit = limit - self.service = self._search - - # Check that criteria arguments are valid - self._validate_criteria(**criteria) - - target_strings = None - if coordinates is not None or object_names is not None: - target_strings = self._parse_multiple_targets(coordinates=coordinates, - object_names=object_names, - resolver=resolver) - - # if radius is just a number we assume degrees - radius = Angle(radius, u.arcmin) - - if radius > self._max_query_radius: - raise InvalidQueryError( - f"Query radius too large. Must be ≤{self._max_query_radius}, got {radius}." - ) - - # build query - params = {"limit": self.limit, "offset": offset, 'select_cols': self._parse_select_cols(select_cols)} - if target_strings: - params["target"] = target_strings - params["radius"] = radius.arcsec - params["radius_units"] = 'arcseconds' - - self._build_params_from_criteria(params, **criteria) - - return self._service_api_connection.missions_request_async(self.service, params) - - @class_or_instance - def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, - select_cols=None, **criteria): - """ - Given a sky position (or positions) and radius, returns a list of matching dataset IDs. - - Parameters - ---------- - coordinates : str, iterable of str, or `~astropy.coordinates` object - The target(s) around which to search. Can be specified as: - - A single coordinate string or `~astropy.coordinates.SkyCoord` object - - A comma-separated string of coordinates (e.g., "10.0 20.0, 15.0 25.0") - - An iterable of coordinate strings or `~astropy.coordinates` objects - radius : str or `~astropy.units.Quantity` object - Default is 3 arcminutes. The radius around the coordinates to search within. - The string must be parsable by `~astropy.coordinates.Angle`. The - appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. - The maximum supported query radius is 30 arcminutes. - limit : int - Default is 5000. The maximum number of dataset IDs in the results. - offset : int - Default is 0. The number of records you wish to skip before selecting records. - select_cols: iterable or str or None, optional - Default is None. Names of columns that will be included in the result table. - If None, a default set of columns will be returned. - Can either be an iterable of column names, a comma-separated string of column names, - or 'all'/'*' to return all available columns. - **criteria - Other mission-specific criteria arguments. - All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` - function. - For example, one can specify the output columns(select_cols) or use other filters(conditions). - To filter by multiple values for a single column, pass in a list of values or - a comma-separated string of values. For the Roman mission, you can also use the special "pass_id" - keyword as an alias for the "pass" column, which is a reserved keyword in Python. - - Returns - ------- - response : list of `~requests.Response` - - Raises - ------ - InvalidQueryError - If the query radius is larger than the limit (30 arcminutes). - """ - return self.query_criteria_async(coordinates=coordinates, - radius=radius, - limit=limit, - offset=offset, - select_cols=select_cols, - **criteria) - - @class_or_instance - @deprecated_renamed_argument('objectname', 'object_names', since='0.4.12') - def query_object_async(self, object_names, *, radius=3*u.arcmin, limit=5000, offset=0, - select_cols=None, resolver=None, **criteria): - """ - Given an object name (or names), returns a list of matching rows. + @class_or_instance + @deprecated_renamed_argument('objectname', 'object_names', since='0.4.12') + def query_object_async(self, object_names, *, radius=3*u.arcmin, limit=5000, offset=0, + select_cols=None, resolver=None, mission=None, **criteria): + """ + Given an object name (or names), returns a list of matching rows. Parameters ---------- @@ -551,6 +322,9 @@ def query_object_async(self, object_names, *, radius=3*u.arcmin, limit=5000, off "SIMBAD" and "NED". If not specified, the default resolver order will be used. Please see the `STScI Archive Name Translation Application (SANTA) `__ for more information. Default is None. + mission : str, optional + The mission for which to query. If not provided, the current value of the + ``mission`` attribute will be used. **criteria Other mission-specific criteria arguments. All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` @@ -570,10 +344,11 @@ def query_object_async(self, object_names, *, radius=3*u.arcmin, limit=5000, off offset=offset, select_cols=select_cols, resolver=resolver, + mission=mission, **criteria) @class_or_instance - def get_product_list_async(self, datasets, *, batch_size=1000): + def get_product_list_async(self, datasets, *, batch_size=1000, mission=None): """ Given a dataset ID or list of dataset IDs, returns a list of associated data products. @@ -587,6 +362,9 @@ def get_product_list_async(self, datasets, *, batch_size=1000): batch_size : int, optional Default 1000. Number of dataset IDs to include in each batch request to the server. If you experience timeouts or connection errors, consider lowering this value. + mission : str, optional + The mission for which to query. If not provided, the current value of the + ``mission`` attribute will be used. Returns ------- @@ -595,10 +373,13 @@ def get_product_list_async(self, datasets, *, batch_size=1000): self.service = self._list_products + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) + if isinstance(datasets, Table) or isinstance(datasets, Row): - dataset_kwd = self.get_dataset_kwd() + dataset_kwd = self.get_dataset_kwd(mission=mission) if not dataset_kwd: - raise InvalidQueryError(f'Dataset keyword not found for mission "{self.mission}". Please input ' + raise InvalidQueryError(f'Dataset keyword not found for mission "{mission}". Please input ' 'dataset IDs as a string, list of strings, or `~astropy.table.Column`.') # Extract dataset IDs based on input type and mission @@ -627,7 +408,7 @@ def get_product_list_async(self, datasets, *, batch_size=1000): params={}, max_batch=batch_size, param_key="dataset_ids", - request_func=lambda p: self._service_api_connection.missions_request_async(self.service, p), + request_func=lambda p: self._service_api_connection.missions_request_async(self.service, p, mission), extract_func=lambda r: [r], # missions_request_async already returns one result desc=f"Fetching products for {len(datasets)} unique datasets" ) @@ -635,7 +416,7 @@ def get_product_list_async(self, datasets, *, batch_size=1000): # Return a list of responses return results - def get_unique_product_list(self, datasets, *, batch_size=1000): + def get_unique_product_list(self, datasets, *, batch_size=1000, mission=None): """ Given a dataset ID or list of dataset IDs, returns a list of associated data products with unique filenames. @@ -648,13 +429,16 @@ def get_unique_product_list(self, datasets, *, batch_size=1000): batch_size : int, optional Default 1000. Number of dataset IDs to include in each batch request to the server. If you experience timeouts or connection errors, consider lowering this value. + mission : str, optional + The mission for which to query. If not provided, the current value of the + ``mission`` attribute will be used. Returns ------- unique_products : `~astropy.table.Table` Table containing products with unique URIs. """ - products = self.get_product_list(datasets, batch_size=batch_size) + products = self.get_product_list(datasets, batch_size=batch_size, mission=mission) unique_products = utils.remove_duplicate_products(products, 'filename') if len(unique_products) < len(products): log.info("To return all products, use `MastMissions.get_product_list`") @@ -742,11 +526,12 @@ def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbo """ # Construct the full data URL based on mission - current_mission = mission.lower() if mission else self.mission + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) - if current_mission in ['hst', 'jwst', 'roman', 'roman_spectra', 'roman_cgi']: + if mission in ['hst', 'jwst', 'roman', 'roman_spectra', 'roman_cgi']: # HST, JWST, and RST have a dedicated endpoint for retrieving products - base_url = (f"{self._service_api_connection.MISSIONS_DOWNLOAD_URL}{current_mission}" + base_url = (f"{self._service_api_connection.MISSIONS_URL}{mission}" "/api/v0.1/retrieve_product") keyword = 'product_name' else: @@ -755,7 +540,7 @@ def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbo keyword = 'uri' # These files require a MAST URI and not just a filename if not uri.startswith('mast:'): - raise InvalidQueryError(f'For mission "{current_mission}", a full MAST URI is required ' + raise InvalidQueryError(f'For mission "{mission}", a full MAST URI is required ' f'for downloading. Got "{uri}".') data_url = base_url + f'?{keyword}=' + uri escaped_url = base_url + f'?{keyword}=' + quote(uri, safe='') @@ -800,72 +585,10 @@ def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbo return status, msg, url - def _download_files(self, products, base_dir, *, flat=False, cache=True, verbose=True): + def download_products(self, products, *, download_dir=None, flat=False, + cache=True, extension=None, mission=None, verbose=True, **filters): """ - Downloads files listed in an `~astropy.table.Table` of data products to a specified directory. - - Parameters - ---------- - products : `~astropy.table.Table` - Table containing products to be downloaded. - base_dir : str - Directory in which files will be downloaded. - flat : bool - Default is False. If True, all files are downloaded directly to `base_dir`, and no subdirectories - will be created. - cache : bool - Default is True. If file is found on disk, it will not be downloaded again. - verbose : bool, optional - Default is True. Whether to show download progress in the console. - - Returns - ------- - response : `~astropy.table.Table` - Table containing download results for each data product file. - """ - - manifest_entries = [] - base_dir = Path(base_dir) - - for data_product in products: - col_names = data_product.colnames - # Determine local path for each file - filename = data_product['filename'] - uri = data_product['uri'] if 'uri' in col_names else filename - dataset = None - if 'dataset' in col_names: - dataset = data_product['dataset'] - elif 'fileset' in col_names: - dataset = data_product['fileset'] - if not dataset and not flat: - raise InvalidQueryError('Data product is missing "dataset" or "fileset" field required for ' - 'constructing local download path. Specify `flat=True` to avoid this ' - 'requirement.') - - # If the products are a subscription JSON, they should include a mission field - mission = data_product['mission'].lower() if 'mission' in col_names else self.mission - - # Create the local file path - local_path = base_dir if flat else base_dir / 'mastDownload' / mission / dataset - local_path.mkdir(parents=True, exist_ok=True) - local_file_path = local_path / Path(filename).name - - # Download files and record status - status, msg, url = self.download_file(uri, - local_path=local_file_path, - cache=cache, - mission=mission, - verbose=verbose) - manifest_entries.append([local_file_path, status, msg, url]) - - # Return manifest as Astropy Table - manifest = Table(rows=manifest_entries, names=('Local Path', 'Status', 'Message', 'URL')) - return manifest - - def download_products(self, products, *, download_dir=None, flat=False, - cache=True, extension=None, verbose=True, **filters): - """ - Download specified data products. + Download specified data products. Parameters ---------- @@ -883,6 +606,10 @@ def download_products(self, products, *, download_dir=None, flat=False, Default is True. If file is found on disc, it will not be downloaded again. extension : string or list, optional Default is None. Filter by file extension. + mission : str, optional + The mission to which the products belong. If not provided, the current value of the + ``mission`` attribute will be used. + If verbose : bool, optional Default is True. Whether to show download progress in the console. **filters : @@ -900,6 +627,9 @@ def download_products(self, products, *, download_dir=None, flat=False, if not products: raise InvalidQueryError('No products specified for download.') + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) + # Ensure `products` is a Table, collecting products if necessary if (isinstance(products, str) and products.endswith('.json')) or isinstance(products, Path): # Products coming from local JSON filepath from subscription service @@ -941,24 +671,36 @@ def download_products(self, products, *, download_dir=None, flat=False, base_dir=download_dir, flat=flat, cache=cache, + mission=mission, verbose=verbose) return manifest @class_or_instance - def get_column_list(self): + def get_column_list(self, mission=None): """ For a mission, return a list of all searchable columns and their descriptions + Parameters + ---------- + mission : str, optional + The mission for which to retrieve the column list. If not provided, the current value + of the ``mission`` attribute will be used. + Returns ------- response : `~astropy.table.Table` that contains columns names, types, and descriptions """ - if not self.columns.get(self.mission): + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) + + if not self.columns.get(mission): try: # Send server request to get column list for current mission - params = {'mission': self.mission} - resp = utils._simple_request(f'{conf.server}/search/util/api/v0.1/column_list', params) + params = {'mission': mission} + resp = utils._simple_request( + f'{self._service_api_connection.MISSIONS_URL}util/api/v0.1/column_list', params + ) # Parse JSON and extract necessary info results = resp.json() @@ -969,36 +711,419 @@ def get_column_list(self): # Create Table with parsed data col_table = Table(rows=rows, names=('name', 'data_type', 'description')) - self.columns[self.mission] = col_table + self.columns[mission] = col_table except JSONDecodeError as ex: raise JSONDecodeError(f'Failed to decode JSON response while attempting to get column list' - f' for mission {self.mission}: {ex}') + f' for mission {mission}: {ex}') except RequestException as ex: raise ConnectionError(f'Failed to connect to the server while attempting to get column list' - f' for mission {self.mission}: {ex}') + f' for mission {mission}: {ex}') except KeyError as ex: raise KeyError(f'Expected key not found in response data while attempting to get column list' - f' for mission {self.mission}: {ex}') + f' for mission {mission}: {ex}') except Exception as ex: raise RuntimeError(f'An unexpected error occurred while attempting to get column list' - f' for mission {self.mission}: {ex}') + f' for mission {mission}: {ex}') - return self.columns[self.mission] + return self.columns[mission] - def get_dataset_kwd(self): + def get_dataset_kwd(self, *, mission=None): """ Return the Dataset ID keyword for the selected mission. If the keyword is unknown, returns None. + Parameters + ---------- + mission : str, optional + The mission for which to retrieve the dataset ID keyword. If not provided, the current value + of the ``mission`` attribute will be used. + Returns ------- keyword : str or None Dataset ID keyword or None if unknown. """ - if self.mission not in self.dataset_kwds: - log.warning('The mission "%s" does not have a known dataset ID keyword.', self.mission) + mission = mission.lower() if mission else self.mission + self._verify_mission(mission) + + if mission not in self.dataset_kwds: + log.warning('The mission "%s" does not have a known dataset ID keyword.', mission) return None - return self.dataset_kwds[self.mission] + return self.dataset_kwds[mission] + + def _verify_mission(self, mission): + """ + Verify that the specified mission is available in the MAST Search API. + + Parameters + ---------- + mission : str + The mission to verify. + + Raises + ------ + InvalidQueryError + If the specified mission is not available in the MAST Search API. + """ + if mission not in self.available_missions: + closest = difflib.get_close_matches(mission, self.available_missions, n=1) + suggestion = f" Did you mean '{closest[0]}'? " if closest else "" + raise InvalidQueryError( + f"Mission '{mission}' is not available in the MAST Search API.{suggestion} " + f"Available missions are: {', '.join(self.available_missions)}." + ) + + def _extract_products(self, response): + """ + Extract products from the response of a `~requests.Response` object. + + Parameters + ---------- + response : `~requests.Response` + The response object containing the products data. + + Returns + ------- + list + A list of products extracted from the response. + """ + combined = [] + for resp in response: + products = resp.json().get('products', []) + # Flatten if nested + if products and isinstance(products[0], list): + products = products[0] + combined.extend(products) + return combined + + def _parse_result(self, response, *, verbose=False): # Used by the async_to_sync decorator functionality + """ + Parse the results of a `~requests.Response` objects and return an `~astropy.table.Table` of results. + + Parameters + ---------- + response : `~requests.Response` + `~requests.Response` objects. + verbose : bool + (presently does nothing - there is no output with verbose set to + True or False) + Default False. Setting to True provides more extensive output. + + Returns + ------- + response : `~astropy.table.Table` + """ + + if self.service == self._search: + results = self._service_api_connection._parse_result(response, verbose, data_key='results') + + # Add column descriptions to column metadata + column_list = self.get_column_list(self._mission_to_search) + for col in results.columns: + if col in column_list['name']: + description = column_list[column_list['name'] == col]['description'].value[0] + results[col].meta = {'description': str(description)} + + # Add search parameters to table metadata + result_json = response.json() + results.meta['search_params'] = result_json.get('search_params', {}) + + # Warn if maximum results are returned + if len(results) >= self.limit: + warnings.warn("Maximum results returned, may not include all sources within radius.", + MaxResultsWarning) + return results + + elif self.service == self._list_products: + products = self._extract_products(response) + return Table(products) + + def _validate_criteria(self, *, mission=None, **criteria): + """ + Check that criteria keyword arguments are valid column names for the mission. + Raises InvalidQueryError if a criteria argument is invalid. + + Parameters + ---------- + mission : str, optional + The mission for which to validate criteria. If not provided, the current value of + the ``mission`` attribute will be used. + **criteria + Keyword arguments representing criteria filters to apply. + + Raises + ------- + InvalidQueryError + If a keyword does not match any valid column names, an error is raised that suggests the closest + matching column name, if available. + """ + # Ensure that self.columns is populated + self.get_column_list(mission) + + # Check each criteria argument for validity + valid_cols = list(self.columns[mission]['name']) + self._search_option_fields + for kwd in criteria.keys(): + if kwd == "pass_id" and "pass_id" not in valid_cols and "pass" in valid_cols: + # Special case where the actual column name is "pass", but that's a reserved keyword in Python + # We allow "pass_id" as an alias + kwd = "pass" + col = next((name for name in valid_cols if name == kwd), None) + if not col: + closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) + error_msg = ( + f"Filter '{kwd}' does not exist. Did you mean '{closest_match[0]}'?" + if closest_match + else f"Filter '{kwd}' does not exist." + ) + raise InvalidQueryError(error_msg) + + def _build_params_from_criteria(self, params, **criteria): + """ + Build the parameters for the API request based on the provided criteria. + + Parameters + ---------- + params : dict + Dictionary to store the parameters for the API request. + **criteria + Keyword arguments representing criteria filters to apply. + """ + # Add each criterion to the params dictionary + params['conditions'] = [] + for prop, value in criteria.items(): + if prop not in self._search_option_fields: + if isinstance(value, list): + # Convert to comma-separated string if passed as a list + value = ','.join(str(item) for item in value) + params['conditions'].append({prop: value}) + else: + if prop == 'sort_by' and isinstance(value, str): + # Convert to list if passed as a string + value = [value] + if prop == 'sort_desc' and isinstance(value, bool): + # Convert to list if passed as a boolean + value = [value] + params[prop] = value + + def _parse_select_cols(self, select_cols, mission): + """ + Parse the select_cols parameter to ensure it is in the correct format. + + Parameters + ---------- + select_cols : iterable or str or None + The select_cols parameter to parse. + mission : str + The mission for which to parse select_cols. If not provided, the current value + of the ``mission`` attribute will be used. + + Returns + ------- + list + A list of column names to select. + + Raises + ------ + InvalidQueryError + If select_cols is not an iterable of strings, a comma-separated string, 'all', or '*'. + If any individual column name is not a string. + """ + if select_cols is None: + if mission == 'ullyses': + return self._default_ullyses_cols + return [] + + # Handle special string cases first + all_columns = self.get_column_list(mission)['name'].value.tolist() + if isinstance(select_cols, str): + if (select_cols.lower() == 'all' or select_cols == '*'): + return all_columns + # Comma-separated string + select_cols = select_cols.split(',') + + # Handle an iterable + elif isinstance(select_cols, Iterable): + # Convert to list so we can iterate multiple times safely + select_cols = list(select_cols) + + else: + raise InvalidQueryError( + "`select_cols` must be an iterable of column names, a comma-separated string, " + "'all', or '*'." + ) + + # Validate the column names + valid_select_cols = [] + for col in select_cols: + if not isinstance(col, str): + raise InvalidQueryError( + "`select_cols` must contain only strings (column names)." + ) + col = col.strip() + if col not in all_columns: + closest_match = difflib.get_close_matches(col, all_columns, n=1) + suggestion = f' Did you mean "{closest_match[0]}"?' if closest_match else '' + warnings.warn(f"Column '{col}' not found.{suggestion}", InputWarning) + else: + valid_select_cols.append(col) + + # Dataset ID column should always be returned + dataset_col = self.dataset_kwds.get(mission, None) + if dataset_col and dataset_col not in valid_select_cols: + valid_select_cols.append(dataset_col) + return valid_select_cols + + def _parse_multiple_targets(self, *, coordinates=None, object_names=None, resolver=None): + """ + Parse coordinate and object-name targets into a list of API target strings. + + Parameters + ---------- + coordinates : str, iterable of str, or `~astropy.coordinates` object, optional + Coordinate target(s). Can be a single coordinate string/object, a comma-separated + coordinate string, an iterable of coordinate strings/objects, or a vector + `~astropy.coordinates.SkyCoord`. + object_names : str or iterable of str, optional + Object-name target(s). Can be a single object name string, a comma-separated + object-name string, or an iterable of object-name strings. + resolver : str, optional + The resolver to use when resolving named targets into coordinates. + + Returns + ------- + list of str + A list of target strings in "ra dec" format for the API. + """ + def _as_list(values): + """Normalize the input values into a list of strings or coordinate objects.""" + if values is None: + items = [] + elif isinstance(values, str): + items = [item.strip() for item in values.split(',') if item.strip()] + elif isinstance(values, Iterable) and not isinstance(values, (SkyCoord, BaseCoordinateFrame)): + items = list(values) + else: + items = [values] + + return [item.strip() if isinstance(item, str) else item for item in items] + + def _is_legacy_ra_dec_pair(items): + """Detect ['ra', 'dec'] passed as one coordinate split on comma.""" + if len(items) != 2 or not all(isinstance(item, str) for item in items): + return False + + try: + float(items[0]) + float(items[1]) + return True + except ValueError: + return False + + coordinate_items = _as_list(coordinates) + object_name_items = _as_list(object_names) + + # Backward compatibility for historical single-coordinate input like: + # coordinates="10.5, -20.1" + if _is_legacy_ra_dec_pair(coordinate_items): + coordinate_items = [f"{coordinate_items[0]} {coordinate_items[1]}"] + + total_targets = len(coordinate_items) + len(object_name_items) + + if total_targets == 0: + raise InvalidQueryError('No targets were provided.') + + if total_targets > self._max_input_targets: + raise InvalidQueryError( + f'Too many input targets provided. Maximum supported is {self._max_input_targets}, ' + f'got {total_targets}.' + ) + + targets = [] + + # Parse coordinate targets + for coord in coordinate_items: + sc = commons.parse_coordinates(coord, return_frame='icrs') + # If input is a vector SkyCoord, iterate through each coordinate + if isinstance(sc, SkyCoord) and sc.isscalar is False: + for ra, dec in zip(sc.ra.deg, sc.dec.deg): + targets.append(f"{ra} {dec}") + else: + targets.append(f"{sc.ra.deg} {sc.dec.deg}") + + # Parse object name targets + if object_names: + resolved = utils.resolve_object(object_name_items, resolver=resolver) + for name in object_name_items: + sc = resolved if isinstance(resolved, SkyCoord) else resolved.get(name) + if sc: + targets.append(f"{sc.ra.deg} {sc.dec.deg}") + + return targets + + def _download_files(self, products, base_dir, *, flat=False, cache=True, mission=None, verbose=True): + """ + Downloads files listed in an `~astropy.table.Table` of data products to a specified directory. + + Parameters + ---------- + products : `~astropy.table.Table` + Table containing products to be downloaded. + base_dir : str + Directory in which files will be downloaded. + flat : bool + Default is False. If True, all files are downloaded directly to `base_dir`, and no subdirectories + will be created. + cache : bool + Default is True. If file is found on disk, it will not be downloaded again. + mission : str, optional + The mission to which the products belong. If not provided, the current value of the ``mission`` + attribute will be used. + verbose : bool, optional + Default is True. Whether to show download progress in the console. + + Returns + ------- + response : `~astropy.table.Table` + Table containing download results for each data product file. + """ + + manifest_entries = [] + base_dir = Path(base_dir) + + for data_product in products: + col_names = data_product.colnames + # Determine local path for each file + filename = data_product['filename'] + uri = data_product['uri'] if 'uri' in col_names else filename + dataset = None + if 'dataset' in col_names: + dataset = data_product['dataset'] + elif 'fileset' in col_names: + dataset = data_product['fileset'] + if not dataset and not flat: + raise InvalidQueryError('Data product is missing "dataset" or "fileset" field required for ' + 'constructing local download path. Specify `flat=True` to avoid this ' + 'requirement.') + + # If the products are a subscription JSON, they should include a mission field + mission = data_product['mission'].lower() if 'mission' in col_names else mission + + # Create the local file path + local_path = base_dir if flat else base_dir / 'mastDownload' / mission / dataset + local_path.mkdir(parents=True, exist_ok=True) + local_file_path = local_path / Path(filename).name + + # Download files and record status + status, msg, url = self.download_file(uri, + local_path=local_file_path, + cache=cache, + mission=mission, + verbose=verbose) + manifest_entries.append([local_file_path, status, msg, url]) + + # Return manifest as Astropy Table + manifest = Table(rows=manifest_entries, names=('Local Path', 'Status', 'Message', 'URL')) + return manifest MastMissions = MastMissionsClass() diff --git a/astroquery/mast/services.py b/astroquery/mast/services.py index 697921ebf3..22139be03c 100644 --- a/astroquery/mast/services.py +++ b/astroquery/mast/services.py @@ -10,19 +10,16 @@ import warnings import numpy as np - -from astropy.table import Table, MaskedColumn +from astropy.table import MaskedColumn, Table from astropy.utils.decorators import deprecated_renamed_argument from .. import log +from ..exceptions import (BlankResponseWarning, InvalidQueryError, NoResultsWarning, TimeoutError) from ..query import BaseQuery from ..utils import async_to_sync from ..utils.class_or_instance import class_or_instance -from ..exceptions import BlankResponseWarning, InvalidQueryError, TimeoutError, NoResultsWarning - from . import conf - __all__ = ["ServiceAPI"] @@ -149,7 +146,7 @@ class ServiceAPI(BaseQuery): SERVICE_URL = conf.server REQUEST_URL = conf.server + "/api/v0.1/" - MISSIONS_DOWNLOAD_URL = conf.server + "/search/" + MISSIONS_URL = conf.server + "/search/" MAST_DOWNLOAD_URL = conf.server + "/api/v0.1/Download/file" SERVICES = {} @@ -366,21 +363,24 @@ def service_request_async(self, service, params, pagesize=None, page=None, use_j return response @class_or_instance - def missions_request_async(self, service, params): + def missions_request_async(self, service, params, mission): """ Builds and executes an asynchronous query to the MAST Search API. Parameters ---------- service : str - The MAST Search API service to query. Should be present in self.SERVICES. + The MAST Search API service to query. Should be present in self.SERVICES. params : dict - JSON object containing service parameters. + JSON object containing service parameters. + mission : str + The mission for which to query. + Returns ------- response : list of `~requests.Response` """ service_config = self.SERVICES.get(service.lower()) - request_url = self.REQUEST_URL + service_config.get('path') + request_url = self.MISSIONS_URL + f"{mission}/api/v0.1/" + f"{service_config.get('path')}" # Default headers headers = { diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 5e0a21f6d4..2a5de3fff1 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -62,3 +62,13 @@ To generate `~astroquery.mast.tests.data.resolver.json`, use the following: ... {'name': objects, 'outputFormat': 'json', 'resolveAll': 'true'}) >>> with open('resolver.json', 'w') as file: ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.available_missions.html`, use the following: + +.. doctest-remote-data:: + + >>> import requests + ... + >>> html = requests.get(f"https://mast.stsci.edu/search/docs").text + >>> with open("available_missions.html", "w") as file: + ... file.write(html) # doctest: +SKIP diff --git a/astroquery/mast/tests/data/available_missions.html b/astroquery/mast/tests/data/available_missions.html new file mode 100644 index 0000000000..ead3eff349 --- /dev/null +++ b/astroquery/mast/tests/data/available_missions.html @@ -0,0 +1,51 @@ + + + + + + MAST Search API Doc + + +
+
+ + + + + + + \ No newline at end of file diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index ad817949d3..a90ab356a5 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,16 +4,16 @@ import os import re import warnings +from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, patch -from pathlib import Path import astropy.units as u -import pytest import numpy as np -from astropy.table import Table, unique +import pytest from astropy.coordinates import SkyCoord from astropy.io import fits +from astropy.table import Table, unique from astropy.utils.exceptions import AstropyDeprecationWarning from requests import HTTPError, Response @@ -35,6 +35,7 @@ 'mission_search_results': 'mission_results.json', 'mission_columns': 'mission_columns.json', 'mission_products': 'mission_products.json', + 'available_missions': 'available_missions.html', 'columnsconfig': 'columnsconfig.json', 'ticcolumns': 'ticcolumns.json', 'ticcol_filtered': 'ticcolumns_filtered.json', @@ -179,6 +180,8 @@ def request_mockreturn(url, params={}): filename = data_path(DATA_FILES['panstarrs_columns']) elif 'path_lookup' in url: filename = data_path(DATA_FILES['get_cloud_paths']) + elif 'search/docs' in url: + filename = data_path(DATA_FILES['available_missions']) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -242,6 +245,30 @@ def zcut_download_mockreturn(url, file_path): ########################### +def test_missions_get_available_missions(): + # Access attribute + missions = MastMissions.available_missions + assert isinstance(missions, list) + assert len(missions) == 9 + + result = MastMissions.get_available_missions() + assert missions == result + + +def test_missions_verify_mission(): + MastMissions._verify_mission("hst") + + # Invalid mission with a possible match + with pytest.raises(InvalidQueryError, match="Did you mean 'hst'?"): + MastMissions._verify_mission("hsp") + + # Invalid mission with no match + with pytest.raises(InvalidQueryError) as err: + MastMissions._verify_mission("invalid") + assert "Did you mean" not in str(err.value) + assert "Mission 'invalid' is not available" in str(err.value) + + def test_missions_query_region_async(): responses = MastMissions.query_region_async(regionCoords, radius=0.002, sci_pi_last_name='GORDON') assert isinstance(responses, MockResponse) @@ -332,50 +359,50 @@ def test_missions_query_criteria(): def test_missions_parse_select_cols(): # Default columns - cols = MastMissions._parse_select_cols(None) # Default columns for HST - assert cols is None + cols = MastMissions._parse_select_cols(None, mission='hst') # Default columns for HST + assert cols == [] # All columns - all_cols = MastMissions._parse_select_cols('all') - assert all_cols == MastMissions.get_column_list()['name'].value.tolist() + all_cols = MastMissions._parse_select_cols('all', mission='hst') + assert all_cols == MastMissions.get_column_list(mission='hst')['name'].value.tolist() # Comma-separated string - string_cols = MastMissions._parse_select_cols('sci_pep_id, sci_instrume') + string_cols = MastMissions._parse_select_cols('sci_pep_id, sci_instrume', mission='hst') for col in ['sci_pep_id', 'sci_instrume', 'sci_data_set_name']: assert col in string_cols # List of columns - list_cols = MastMissions._parse_select_cols(['sci_pep_id', 'sci_instrume']) + list_cols = MastMissions._parse_select_cols(['sci_pep_id', 'sci_instrume'], mission='hst') for col in ['sci_pep_id', 'sci_instrume', 'sci_data_set_name']: assert col in list_cols # Tuple of columns - tuple_cols = MastMissions._parse_select_cols(('sci_pep_id', 'sci_instrume')) + tuple_cols = MastMissions._parse_select_cols(('sci_pep_id', 'sci_instrume'), mission='hst') for col in ['sci_pep_id', 'sci_instrume', 'sci_data_set_name']: assert col in tuple_cols # Generator of columns - gen_cols = MastMissions._parse_select_cols(col for col in ['sci_pep_id', 'sci_instrume']) + gen_cols = MastMissions._parse_select_cols((col for col in ['sci_pep_id', 'sci_instrume']), mission='hst') for col in ['sci_pep_id', 'sci_instrume', 'sci_data_set_name']: assert col in gen_cols # Error if invalid type with pytest.raises(InvalidQueryError, match="`select_cols` must be an iterable of column names"): - MastMissions._parse_select_cols(123) + MastMissions._parse_select_cols(123, mission='hst') # Error if an individual column is not a string with pytest.raises(InvalidQueryError, match="`select_cols` must contain only strings"): - MastMissions._parse_select_cols(['sci_pep_id', 123]) + MastMissions._parse_select_cols(['sci_pep_id', 123], mission='hst') # Warning for invalid column names with pytest.warns(InputWarning, match="Column 'invalid_column' not found."): - valid_cols = MastMissions._parse_select_cols(['sci_pep_id', 'invalid_column']) + valid_cols = MastMissions._parse_select_cols(['sci_pep_id', 'invalid_column'], mission='hst') assert 'sci_pep_id' in valid_cols assert 'invalid_column' not in valid_cols # Workaround for Ullyses mission default columns ullyses_mission = MastMissions(mission='ullyses') - ullyses_cols = ullyses_mission._parse_select_cols(None) + ullyses_cols = ullyses_mission._parse_select_cols(None, mission='ullyses') for col in MastMissions._default_ullyses_cols: assert col in ullyses_cols @@ -469,8 +496,8 @@ def test_missions_get_product_list_async(): assert 'Dataset list is empty' in str(err_empty.value) # No dataset keyword - with pytest.raises(InvalidQueryError, match='Dataset keyword not found for mission "invalid"'): - missions = MastMissions(mission='invalid') + with pytest.raises(InvalidQueryError, match='Dataset keyword not found for mission "roman_spectra".'): + missions = MastMissions(mission='roman_spectra') missions.get_product_list_async(Table({'a': [1, 2, 3]})) @@ -489,6 +516,10 @@ def test_missions_get_product_list(): result = MastMissions.get_product_list(datasets[:3]) assert isinstance(result, Table) + # Column input + result = MastMissions.get_product_list(datasets['sci_data_set_name']) + assert isinstance(result, Table) + # Table input result = MastMissions.get_product_list(datasets[0]) assert isinstance(result, Table) @@ -707,11 +738,11 @@ def test_missions_get_dataset_kwd(caplog): assert m.get_dataset_kwd() == 'Target' # Switch to an unknown - m.mission = 'Unknown' - assert m.mission == 'unknown' + m.mission = 'roman_spectra' + assert m.mission == 'roman_spectra' assert m.get_dataset_kwd() is None with caplog.at_level('WARNING', logger='astroquery'): - assert 'The mission "unknown" does not have a known dataset ID keyword' in caplog.text + assert 'The mission "roman_spectra" does not have a known dataset ID keyword' in caplog.text @pytest.mark.parametrize( diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index a3c1d0405c..543749f9a2 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -94,6 +94,13 @@ def test_resolve_object(self): # MissionSearchClass Test # ########################### + def test_missions_get_available_missions(self): + missions = MastMissions.get_available_missions() + assert isinstance(missions, list) + assert len(missions) > 5 + assert 'hst' in missions + assert 'jwst' in missions + def test_missions_get_column_list(self): columns = MastMissions().get_column_list() assert len(columns) > 1 @@ -168,7 +175,7 @@ def test_missions_query_criteria(self): assert result[cols].meta['description'] # Positional criteria search - result = MastMissions.query_criteria(object_names='NGC6121', + result = MastMissions.query_criteria(coordinates="245.89675 -26.52575", radius=0.1, sci_start_time='<2012', sci_actual_duration='0..200', @@ -179,11 +186,12 @@ def test_missions_query_criteria(self): assert (result['sci_start_time'] < '2012').all() assert ((result['sci_actual_duration'] >= 0) & (result['sci_actual_duration'] <= 200)).all() - # Search with multiple positional inputs - coord = SkyCoord(245.89675, -26.52575, unit='deg') - result = MastMissions.query_criteria(coordinates=[coord, "205.54842 28.37728"], - object_names=["M2", "M9"], - radius=0.1) + # Search with multiple positional inputs and a mission specified + coord = SkyCoord(9.242667, -73.3925521, unit='deg') + result = MastMissions.query_criteria(coordinates=[coord, "16.8094764 -72.2298700"], + object_names=["AA Tau", "AK Sco"], + radius=0.01, + mission="ullyses") assert len(set(result['search_pos'])) == 4 # Should have four different search positions # Raise error if invalid input is given @@ -408,7 +416,7 @@ def check_result(result, path): ('jwst', {'fileSetName': 'jw01189001001_02101_00001'}), ('classy', {'Target': 'J0021+0052'}), ('ullyses', {'host_galaxy_name': 'WLM', 'select_cols': ['observation_id']}), - ('roman', {'program': 3, 'pass_id': 1}), + ('roman', {'productLevel': 4, 'dataset_category': 'spectra'}), ('iue', {'iue_data_id': 'LWR08496'}), ]) def test_missions_workflow(self, tmp_path, mission, query_params): @@ -417,8 +425,7 @@ def test_missions_workflow(self, tmp_path, mission, query_params): # Roman requires extra setup to point towards the test server if mission == 'roman': - m._service_api_connection.SERVICE_URL = 'https://masttest.stsci.edu' - m._service_api_connection.REQUEST_URL = 'https://masttest.stsci.edu/search/roman/api/v0.1/' + m._service_api_connection.MISSIONS_URL = "https://masttest.stsci.edu/search/" # Criteria query datasets = m.query_criteria(**query_params) @@ -431,7 +438,7 @@ def test_missions_workflow(self, tmp_path, mission, query_params): assert len(prods) # Download products - result = m.download_products(prods[:3], + result = m.download_products(prods[:2], download_dir=tmp_path) for row in result: if row['Status'] == 'COMPLETE': @@ -555,7 +562,7 @@ def test_observations_query_criteria_async(self): # with position responses = Observations.query_criteria_async(filters=["NUV", "FUV"], - object_name="M10") + coordinates="254.28771 -4.10031") assert isinstance(responses, list) def test_observations_query_criteria(self): @@ -569,7 +576,7 @@ def test_observations_query_criteria(self): assert ((result['obs_collection'] == 'HST') | (result['obs_collection'] == 'HLA')).all() # with position - result = Observations.query_criteria(object_name="M10", + result = Observations.query_criteria(coordinates="254.28771 -4.10031", filters=["NUV", "FUV"], obs_collection="GALEX") assert isinstance(result, Table) @@ -577,7 +584,7 @@ def test_observations_query_criteria(self): assert (result['obs_collection'] == 'GALEX').all() assert sum(result['filters'] == 'NUV') == 4 - result = Observations.query_criteria(object_name="M10", + result = Observations.query_criteria(coordinates="254.28771 -4.10031", dataproduct_type="IMAGE", intentType="calibration") assert (result["intentType"] == "calibration").all() @@ -626,7 +633,7 @@ def test_observations_query_criteria_count(self): # product functions def test_observations_get_product_list_async(self): - test_obs = Observations.query_criteria(filters=["NUV", "FUV"], object_name="M10") + test_obs = Observations.query_criteria(filters=["NUV", "FUV"], coordinates="254.28771 -4.10031") responses = Observations.get_product_list_async(test_obs[0]["obsid"]) assert isinstance(responses, list) @@ -634,7 +641,7 @@ def test_observations_get_product_list_async(self): responses = Observations.get_product_list_async(test_obs[2:3]) assert isinstance(responses, list) - observations = Observations.query_criteria(object_name="M8", obs_collection=["K2", "IUE"]) + observations = Observations.query_criteria(coordinates="270.904 -24.387", obs_collection=["K2", "IUE"]) responses = Observations.get_product_list_async(observations[0]) assert isinstance(responses, list) @@ -646,7 +653,7 @@ def test_observations_get_product_list_async(self): assert isinstance(responses, list) def test_observations_get_product_list(self, capsys): - observations = Observations.query_criteria(object_name='M8', obs_collection=['K2', 'IUE']) + observations = Observations.query_criteria(coordinates="270.904 -24.387", obs_collection=['K2', 'IUE']) test_obs_id = str(observations[0]['obsid']) mult_obs_ids = str(observations[0]['obsid']) + ',' + str(observations[1]['obsid']) @@ -852,7 +859,7 @@ def check_result(result, path): assert os.path.exists(path) # get observations from GALEX instrument with query_criteria - observations = Observations.query_criteria(object_name='M10', + observations = Observations.query_criteria(coordinates="254.28771 -4.10031", radius=0.001, instrument_name='GALEX') @@ -1246,24 +1253,24 @@ def test_catalogs_query_criteria_async(self): # with position responses = Catalogs.query_criteria_async(catalog="Tic", - object_name="M10", + coordinates="254.28771 -4.10031", objType="EXTENDED") assert isinstance(responses, list) responses = Catalogs.query_criteria_async(catalog="CTL", - object_name="M10", + coordinates="254.28771 -4.10031", objType="EXTENDED") assert isinstance(responses, list) responses = Catalogs.query_criteria_async(catalog="DiskDetective", - object_name="M10", + coordinates="254.28771 -4.10031", radius=2, state="complete") assert isinstance(responses, list) responses = Catalogs.query_criteria_async(catalog="panstarrs", table="mean", - object_name="M10", + coordinates="254.28771 -4.10031", radius=.02, qualityFlag=48) assert isinstance(responses, Response) @@ -1303,23 +1310,24 @@ def check_result(result, exp_vals): # with position result = Catalogs.query_criteria(catalog="Tic", - object_name="M10", objType="EXTENDED") + coordinates="254.28771 -4.10031", + objType="EXTENDED") check_result(result, {'ID': '10000732589'}) - result = Catalogs.query_criteria(object_name='TIC 291067184', + result = Catalogs.query_criteria(coordinates="177.72837 6.26003", catalog="ctl", Tmag=[10.5, 11], POSflag="2mass") check_result(result, {'Tmag': 10.893}) result = Catalogs.query_criteria(catalog="DiskDetective", - object_name="M10", + coordinates="254.28771 -4.10031", radius=2, state="complete") check_result(result, {'designation': 'J165628.40-054630.8'}) result = Catalogs.query_criteria(catalog="panstarrs", - object_name="M10", + coordinates="254.28771 -4.10031", radius=.01, qualityFlag=32, zoneID=10306, @@ -1452,7 +1460,7 @@ def check_sector_table(sector_table): sector_table = Tesscut.get_sectors(coordinates=coord) check_sector_table(sector_table) - sector_table = Tesscut.get_sectors(object_name="M104") + sector_table = Tesscut.get_sectors(coordinates="189.99763 -11.62305") check_sector_table(sector_table) def test_tesscut_get_sectors_mt(self): @@ -1497,7 +1505,7 @@ def check_manifest(manifest, ext="fits"): path=str(tmpdir), inflate=False) check_manifest(manifest, ".zip") - manifest = Tesscut.download_cutouts(object_name="TIC 32449963", size=1, path=str(tmpdir)) + manifest = Tesscut.download_cutouts(coordinates="189.93837 -11.64712", size=1, path=str(tmpdir)) check_manifest(manifest, "fits") def test_tesscut_download_cutouts_mt(self, tmpdir): @@ -1538,7 +1546,7 @@ def check_cutout_hdu(cutout_hdus_list): sector=[28, 68]) check_cutout_hdu(cutout_hdus_list) - cutout_hdus_list = Tesscut.get_cutouts(object_name="TIC 32449963", + cutout_hdus_list = Tesscut.get_cutouts(coordinates="189.93837 -11.64712", size=1, sector=37) check_cutout_hdu(cutout_hdus_list) diff --git a/docs/mast/mast_missions.rst b/docs/mast/mast_missions.rst index f7e13f3d87..a81e587c96 100644 --- a/docs/mast/mast_missions.rst +++ b/docs/mast/mast_missions.rst @@ -3,9 +3,13 @@ Mission-Specific Queries ************************ -The `~astroquery.mast.MastMissionsClass` class allows for search queries based on mission-specific -metadata for a given data collection. This metadata includes header keywords, proposal information, and observational parameters. -The following missions/products are currently available for search: +The `~astroquery.mast.MastMissionsClass` is a versatile tool for accessing and working with datasets hosted by the +`Mikulski Archive for Space Telescopes (MAST) `_. ``MastMissions`` is a Python wrapper for the +`MAST Search API `_, which allows +users to search for mission-specific metadata and retrieve associated data products. This data is also findable through the `MAST +Search UI `_. + +The following missions/products are currently available for search as of June 2026: - `Hubble Space Telescope `_ (``'hst'``) @@ -19,42 +23,80 @@ The following missions/products are currently available for search: - `Hubble UV Legacy Library of Young Stars as Essential Standards `_ (``'ullyses'``) -An object of the ``MastMissions`` class is instantiated with a default mission of ``'hst'`` and -default service set to ``'search'``. The searchable metadata for Hubble encompasses all information that -was previously accessible through the original HST web search form. The metadata for Hubble and all other -available missions is also available through the `MAST Search UI `_. +The basic workflow for using the `~astroquery.mast.MastMissionsClass` is as follows: + +1. **Search** for mission datasets. +2. **Retrieve** a list of associated data products for the datasets. +3. **Access** the data products by reading them into memory or downloading them to your local machine. + +The workflow looks very similar to the `~astroquery.mast.ObservationsClass` workflow, but there are a few key differences to note, +and you should use the class that is best suited to your unique goals: + +- **API**: ``MastMissions`` uses the `MAST Search API `_ while ``Observations`` uses + the `MAST Portal API `_. +- **Collection**: ``MastMissions`` can only perform queries on a single collection, or “mission”, at a time. + ``Observations`` uses the `Common Archive Observation Model (CAOM) `_ and can + run queries across every available observational collection at the same time. +- **Filter Keywords**: ``MastMissions`` has an extensive selection of mission-specific keywords to use while writing queries. + ``Observations`` is limited to the `fields described by the CAOM `_ and has + no criteria with mission-specific meaning. + +In summary, ``MastMissions`` is well-suited for fast, mission-specific queries that might require a more extensive selection +of filter keywords. ``Observations`` is better for more broad, multi-mission searches. + +The Mission Attribute +======================= + +The `~astroquery.mast.MastMissionsClass` is designed to query a single mission at a time. The ``MastMissions`` stores a ``mission`` attribute. +If no mission is specified in a query, the ``mission`` attribute is used as the default. The ``mission`` attribute is a case-insensitive string +that corresponds to the mission name. The available missions can be retrieved with the `~astroquery.mast.MastMissionsClass.get_available_missions` method +or the `~astroquery.mast.MastMissionsClass.available_missions` property, which caches the list of missions after the first retrieval. .. doctest-remote-data:: >>> from astroquery.mast import MastMissions - >>> missions = MastMissions() - >>> missions.mission - 'hst' - >>> missions.service - 'search' + >>> MastMissions.get_available_missions() # doctest: +IGNORE_OUTPUT + ['hst', 'jwst', 'classy', 'ullyses', 'iue'] + +The default value for the ``mission`` attribute is "hst", referring to the Hubble Space Telescope. + +.. doctest-remote-data:: -Each ``MastMissions`` object can only make queries and download products from a single mission at a time. This mission can -be modified with the ``mission`` class attribute. This allows users to make queries to multiple missions with the same object. -To search for JWST metadata, the ``mission`` attribute is reassigned to ``'JWST'``. + >>> print("Default mission:", MastMissions.mission) + Default mission: hst + +This attribute can be modified at any time to set a new default, and the mission will be validated when set. This attribute can be set +when instantiating a ``MastMissions`` object or modified later. + +.. doctest-remote-data:: + + >>> MastMissions.mission = 'JWST' + >>> print("New mission:", MastMissions.mission) + New mission: jwst + +You can create multiple instances of ``MastMissions`` with a different default mission, which may be useful for comparing results +across missions or for running multiple queries with different missions in the same script without having to specify the mission each time. .. doctest-remote-data:: - >>> m = MastMissions() - >>> print(m.mission) - hst - >>> m.mission = 'JWST' - >>> print(m.mission) - jwst + + >>> classy_mission = MastMissions(mission="classy") + >>> print("Classy mission:", classy_mission.mission) + Classy mission: classy + >>> + >>> ullyses_mission = MastMissions(mission="ullyses") + >>> print("Ullyses mission:", ullyses_mission.mission) + Ullyses mission: ullyses Querying Missions ================== -The MastMissions interface provides three closely related query methods. All three methods return results as an `~astropy.table.Table` -and all three support column-based filtering, sorting, and result limiting. The primary difference between them is how positional +The MastMissions interface provides three closely related query methods. All three methods return results as an `~astropy.table.Table` +and all three support column-based filtering, sorting, and result limiting. The primary difference between them is how positional constraints are specified. At a high level: - - `~astroquery.mast.MastMissionsClass.query_criteria` is the most flexible method. It supports purely column-based queries, + - `~astroquery.mast.MastMissionsClass.query_criteria` is the most flexible method. It supports purely column-based queries, purely positional queries, or a combination of both. - `~astroquery.mast.MastMissionsClass.query_region` is a convenience wrapper for positional queries using coordinates. @@ -64,7 +106,7 @@ At a high level: Query Parameters ---------------- -The ``missions`` object can be used to search mission metadata by sky position, +The ``MastMissions`` object can be used to search mission metadata by sky position, object name, or other criteria. Keyword arguments may be used to specify output characteristics and filter on values such as instrument, exposure type, and principal investigator. The available column names for a mission can be retrieved @@ -72,13 +114,14 @@ using the `~astroquery.mast.MastMissionsClass.get_column_list` method. .. doctest-remote-data:: - >>> from astroquery.mast import MastMissions - >>> missions = MastMissions(mission='hst') + >>> missions = MastMissions(mission="hst") >>> columns = missions.get_column_list() Keyword arguments can also be used to refine results further. The following parameters are available: -- ``radius``: For positional searches only. Only return results within a certain distance from an object or set of coordinates. +- ``mission``: The mission to query. Give this argument to override the default mission specified by the ``mission`` attribute. + +- ``radius``: For positional searches only. Only return results within a certain distance from an object or set of coordinates. Default is 3 arcminutes. - ``limit``: The maximum number of results to return. Default is 5000. @@ -99,29 +142,32 @@ Writing Queries ---------------- The `~astroquery.mast.MastMissionsClass.query_criteria` method supports both positional parameters and column-based filters. -Positional constraints are optional. +Positional constraints are optional. Supported positional parameters include: - ``coordinates`` : Sky coordinates around which to perform a cone search. - ``object_name`` : Name(s) of the object(s) around which to perform a cone search. - ``resolver`` : Resolver service to use for object name resolution. - - ``radius`` : Radius of the cone searches around the specified coordinates or object names. Can be defined as an `~astropy.units.Quantity`, + - ``radius`` : Radius of the cone searches around the specified coordinates or object names. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. -Multiple coordinates or objects may be queried in a single request. The ``coordinates`` and ``object_names`` parameters +Multiple coordinates or objects may be queried in a single request. The ``coordinates`` and ``object_names`` parameters accept a single value, an iterable of values, or a comma-separated string. When multiple values are provided for either parameter, results matching *any* of the supplied positions are returned. .. doctest-remote-data:: >>> from astropy.coordinates import SkyCoord + >>> MastMissions.mission = "hst" >>> select_cols = ["sci_targname", "sci_pep_id", "sci_status"] - >>> results = missions.query_criteria(coordinates=[SkyCoord(245.89675, -26.52575, unit='deg'), "205.54842 28.37728"], - ... object_names=["M2", "M9"], - ... radius=0.1, - ... select_cols=select_cols, - ... sort_by='search_pos') + >>> results = MastMissions.query_criteria( + ... mission="hst", + ... coordinates=[SkyCoord(245.89675, -26.52575, unit='deg'), "205.54842 28.37728"], + ... object_names=["M2", "M9"], + ... radius=0.1, + ... select_cols=select_cols, + ... sort_by="search_pos") >>> results.pprint(max_width=-1) # doctest: +IGNORE_OUTPUT search_pos sci_data_set_name sci_targname sci_pep_id ang_sep sci_status ------------------- ----------------- ------------- ---------- -------------------- ---------- @@ -168,24 +214,25 @@ Criteria syntax supports several operations: - For numeric or date columns, select an inclusive range with the syntax ``'#..#'``. -- Wildcards are special characters used in search patterns to represent one or more unknown characters, +- Wildcards are special characters used in search patterns to represent one or more unknown characters, allowing for flexible matching of strings. The wildcard character is ``*`` and it replaces any number of characters preceding, following, or in between existing characters, depending on its placement. .. note:: - For the Roman mission, query methods also support the ``pass_id`` parameter as an alias for the ``pass`` column, + For the Roman mission, query methods also support the ``pass_id`` parameter as an alias for the ``pass`` column, which refers to a single iteration of a pass plan. This is to avoid conflicts with the reserved Python keyword. .. doctest-remote-data:: - >>> results = missions.query_criteria(sci_obs_type="IMAGE", - ... sci_instrume="!COS", - ... sci_spec_1234=["F150W", "F105W", "F110W"], - ... sci_dec=">0", - ... sci_actual_duration="1000..2000", - ... sci_targname="*GAL*", - ... select_cols=["sci_obs_type", "sci_spec_1234"]) + >>> results = missions.query_criteria( + ... sci_obs_type="IMAGE", + ... sci_instrume="!COS", + ... sci_spec_1234=["F150W", "F105W", "F110W"], + ... sci_dec=">0", + ... sci_actual_duration="1000..2000", + ... sci_targname="*GAL*", + ... select_cols=["sci_obs_type", "sci_spec_1234"]) >>> results[:5].pprint(max_width=-1) # doctest: +IGNORE_OUTPUT sci_data_set_name sci_targname sci_spec_1234 sci_obs_type @@ -196,7 +243,7 @@ Criteria syntax supports several operations: N4A702010 GAL-CLUS-0026+1653-ARCC F110W IMAGE N4A705010 GAL-CLUS-0026+1653-ARCC F110W IMAGE -The `~astroquery.mast.MastMissionsClass.query_region` and `~astroquery.mast.MastMissionsClass.query_object` methods are +The `~astroquery.mast.MastMissionsClass.query_region` and `~astroquery.mast.MastMissionsClass.query_object` methods are convenience wrappers around `~astroquery.mast.MastMissionsClass.query_criteria`: - `~astroquery.mast.MastMissionsClass.query_region` requires ``coordinates``. @@ -209,11 +256,12 @@ Both methods also accept column-based criteria, which are applied in the same wa >>> regionCoords = SkyCoord(210.80227, 54.34895, unit=('deg', 'deg')) >>> select_cols = ["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"] - >>> results = missions.query_region(regionCoords, - ... radius=3, - ... sci_pep_id=12556, - ... select_cols=select_cols, - ... sort_by='sci_targname') + >>> results = MastMissions.query_region( + ... regionCoords, + ... radius=3, + ... sci_pep_id=12556, + ... select_cols=select_cols, + ... sort_by="sci_targname") >>> results[:5].pprint(max_width=-1) # doctest: +IGNORE_OUTPUT
search_pos sci_data_set_name sci_targname sci_start_time sci_stop_time ang_sep sci_status @@ -226,10 +274,11 @@ Both methods also accept column-based criteria, which are applied in the same wa .. doctest-remote-data:: - >>> results = missions.query_object('M101', - ... radius=3, - ... select_cols=select_cols, - ... sort_by='sci_targname') + >>> results = MastMissions.query_object( + ... "M101", + ... radius=3, + ... select_cols=select_cols, + ... sort_by="sci_targname") >>> results[:5] # doctest: +IGNORE_OUTPUT
search_pos sci_data_set_name sci_targname sci_start_time sci_stop_time ang_sep sci_status @@ -248,21 +297,25 @@ Getting Product Lists ---------------------- Each observation returned from a MAST query can have one or more associated data products. Given -one or more datasets or dataset IDs, the `~astroquery.mast.MastMissionsClass.get_product_list` function +one or more datasets or dataset IDs, the `~astroquery.mast.MastMissionsClass.get_product_list` function will return a `~astropy.table.Table` containing the associated data products. -`~astroquery.mast.MastMissionsClass.get_product_list` also includes an optional ``batch_size`` parameter, -which controls how many datasets are sent to the MAST service per request. This can be useful for managing +To override the default mission, provide the ``mission`` parameter with the appropriate mission name. +If the mission is not specified, the default mission stored in the ``mission`` attribute will be used. + +`~astroquery.mast.MastMissionsClass.get_product_list` also includes an optional ``batch_size`` parameter, +which controls how many datasets are sent to the MAST service per request. This can be useful for managing memory usage or avoiding timeouts when requesting product lists for large numbers of datasets. If not provided, batch_size defaults to 1000. .. doctest-remote-data:: - >>> datasets = missions.query_criteria(sci_pep_id=12451, - ... sci_instrume='ACS', - ... sci_hlsp='>1') - >>> products = missions.get_product_list(datasets[:2], batch_size=1000) + >>> datasets = MastMissions.query_criteria( + ... sci_pep_id=12451, + ... sci_instrume="ACS", + ... sci_hlsp=">1") + >>> products = MastMissions.get_product_list(datasets[:2], batch_size=1000) >>> print(products[:5]) # doctest: +IGNORE_OUTPUT - product_key access dataset ... category size type + product_key access dataset ... category size type ---------------------------- ------ --------- ... ---------- --------- ------- JBTAA0010_jbtaa0010_asn.fits PUBLIC JBTAA0010 ... AUX 11520 science JBTAA0010_jbtaa0010_drz.fits PUBLIC JBTAA0010 ... CALIBRATED 214655040 science @@ -274,17 +327,17 @@ The keyword corresponding to the dataset ID varies between missions and can be r `~astroquery.mast.MastMissionsClass.get_dataset_kwd` method. .. doctest-remote-data:: - >>> dataset_id_kwd = missions.get_dataset_kwd() + >>> dataset_id_kwd = MastMissions.get_dataset_kwd() >>> print(dataset_id_kwd) sci_data_set_name - >>> products = missions.get_product_list(datasets[:2][dataset_id_kwd]) + >>> products = MastMissions.get_product_list(datasets[:2][dataset_id_kwd]) Some products may be associated with multiple datasets, and this table may contain duplicates. To return a list of products with unique filenames, use the `~astroquery.mast.MastMissionsClass.get_unique_product_list` function. .. doctest-remote-data:: - >>> unique_products = missions.get_unique_product_list(datasets[:2]) # doctest: +IGNORE_OUTPUT + >>> unique_products = MastMissions.get_unique_product_list(datasets[:2]) # doctest: +IGNORE_OUTPUT INFO: 16 of 206 products were duplicates. Only returning 190 unique product(s). [astroquery.mast.utils] INFO: To return all products, use `MastMissions.get_product_list` [astroquery.mast.missions] @@ -298,8 +351,8 @@ and any other of the product fields. The **AND** operation is applied between filters, and the **OR** operation is applied within each filter set, except in the case of negated values. A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results. -When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated -values are combined with **AND** logic against the positives. +When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated +values are combined with **AND** logic against the positives. For example: - ``file_suffix=['A', 'B', '!C']`` → (file_suffix != C) AND (file_suffix == A OR file_suffix == B) @@ -315,13 +368,14 @@ The filter below returns FITS products that are "science" type **and** less than **and** have a ``file_suffix`` of "ASN" (association files) **or** "JIF" (job information files). .. doctest-remote-data:: - >>> filtered = missions.filter_products(products, - ... extension='fits', - ... type='science', - ... size='<=20000', - ... file_suffix=['ASN', 'JIF']) + >>> filtered = MastMissions.filter_products( + ... products, + ... extension="fits", + ... type="science", + ... size="<=20000", + ... file_suffix=["ASN", "JIF"]) >>> print(filtered) # doctest: +IGNORE_OUTPUT - product_key access dataset ... category size type + product_key access dataset ... category size type ---------------------------- ------ --------- ... -------------- ----- ------- JBTAA0010_jbtaa0010_asn.fits PUBLIC JBTAA0010 ... AUX 11520 science JBTAA0020_jbtaa0020_asn.fits PUBLIC JBTAA0020 ... AUX 11520 science @@ -333,36 +387,40 @@ Downloding Data Downloading Data Products ------------------------- -The `~astroquery.mast.MastMissionsClass.download_products` function accepts a table of products like the one above -and will download the products to your local machine. Products may also be provided as dataset IDs with product filters, +The `~astroquery.mast.MastMissionsClass.download_products` function accepts a table of products like the one above +and will download the products to your local machine. Products may also be provided as dataset IDs with product filters, or as JSON product metadata sent by the MAST subscription service (either as a local JSON file or as in-memory data). By default, products will be downloaded into the current working directory, in a subdirectory called ``mastDownload``. -The full local filepaths will have the form ``mastDownload///file.`` You can change the download -directory using the ``download_dir`` parameter. If ``flat=True`` is specified, all files will be downloaded directly into the +The full local filepaths will have the form ``mastDownload///file.`` You can change the download +directory using the ``download_dir`` parameter. If ``flat=True`` is specified, all files will be downloaded directly into the ``download_dir`` without any subdirectories. +To override the default mission, provide the ``mission`` parameter with the appropriate mission name. +If the mission is not specified, the default mission stored in the ``mission`` attribute will be used. + .. doctest-remote-data:: - >>> manifest = missions.download_products(filtered) # doctest: +IGNORE_OUTPUT + >>> manifest = MastMissions.download_products(filtered) # doctest: +IGNORE_OUTPUT Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0010%2Fjbtaa0010_asn.fits to mastDownload/hst/JBTAA0010/jbtaa0010_asn.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0010%2Fjbtaa0010_jif.fits to mastDownload/hst/JBTAA0010/jbtaa0010_jif.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0020%2Fjbtaa0020_asn.fits to mastDownload/hst/JBTAA0020/jbtaa0020_asn.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0020%2Fjbtaa0020_jif.fits to mastDownload/hst/JBTAA0020/jbtaa0020_jif.fits ... [Done] >>> print(manifest) # doctest: +IGNORE_OUTPUT - Local Path Status Message URL + Local Path Status Message URL --------------------------------------------- -------- ------- ---- mastDownload/hst/JBTAA0010/jbtaa0010_asn.fits COMPLETE None None mastDownload/hst/JBTAA0010/jbtaa0010_jif.fits COMPLETE None None mastDownload/hst/JBTAA0020/jbtaa0020_asn.fits COMPLETE None None mastDownload/hst/JBTAA0020/jbtaa0020_jif.fits COMPLETE None None -The function also accepts dataset IDs and product filters as input for a more streamlined workflow. +The function also accepts dataset IDs and product filters as input for a more streamlined workflow. .. doctest-remote-data:: - >>> missions.download_products(['JBTAA0010', 'JBTAA0020'], - ... extension='fits', - ... type='science', - ... file_suffix=['ASN', 'JIF']) # doctest: +IGNORE_OUTPUT + >>> manifest = MastMissions.download_products( + ... ['JBTAA0010', 'JBTAA0020'], + ... extension="fits", + ... type="science", + ... file_suffix=["ASN", "JIF"]) # doctest: +IGNORE_OUTPUT Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0010%2Fjbtaa0010_asn.fits to mastDownload/hst/JBTAA0010/jbtaa0010_asn.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0010%2Fjbtaa0010_jif.fits to mastDownload/hst/JBTAA0010/jbtaa0010_jif.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0020%2Fjbtaa0020_asn.fits to mastDownload/hst/JBTAA0020/jbtaa0020_asn.fits ... [Done] @@ -374,11 +432,11 @@ Downloading a Single File To download a single data product file, use the `~astroquery.mast.MastMissionsClass.download_file` function with a MAST URI as input. Some missions (e.g., HST, JWST) accept direct filenames as input, but others require a fully-qualified ``mast:`` URI. -The default is to download the file to the current working directory, but you can specify the download directory or filepath with +The default is to download the file to the current working directory, but you can specify the download directory or filepath with the ``local_path`` keyword argument. .. doctest-remote-data:: - >>> result = missions.download_file('JBTAA0010/jbtaa0010_asn.fits') + >>> result = MastMissions.download_file("JBTAA0010/jbtaa0010_asn.fits") Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0010%2Fjbtaa0010_asn.fits to jbtaa0010_asn.fits ... [Done] >>> print(result) ('COMPLETE', None, None)