diff --git a/CHANGES.rst b/CHANGES.rst index ac5c229ec3..1b26623c50 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -178,6 +178,8 @@ mast and column descriptions in the column metadata. [#3588] - Added ``pass_id`` as an alias for the ``pass`` column in query functions for the Roman mission to avoid conflicts with the reserved Python keyword. [#3588] +- Added the ``MastMissions.read_product`` method to read data products directly into memory as an `~astropy.io.fits.HDUList` + or an `~asdf.AsdfFile` object. [#3593] - Update the cutout format request parameter in ``Zcut.download_cutouts`` to reflect a recent service change. [#3608] diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index 94ae869a2e..36abdbd708 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -7,6 +7,7 @@ """ import difflib +import importlib import json import warnings from collections.abc import Iterable @@ -16,13 +17,16 @@ import astropy.units as u import numpy as np +import requests from astropy.coordinates import Angle, BaseCoordinateFrame, SkyCoord +from astropy.io import fits from astropy.table import Column, Row, Table, vstack from astropy.utils.decorators import deprecated_renamed_argument from requests import HTTPError, RequestException from astroquery import log from astroquery.exceptions import ( + AuthenticationWarning, InputWarning, InvalidQueryError, MaxResultsWarning, @@ -35,6 +39,16 @@ from . import conf +try: + import fsspec +except ImportError: + fsspec = None + +try: + import asdf +except ImportError: + asdf = None + __all__ = ['MastMissionsClass', 'MastMissions'] @@ -713,35 +727,23 @@ def filter_products(self, products, *, extension=None, **filters): return products[filter_mask] - def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbose=True): + def _get_download_url(self, uri, mission=None): """ - Downloads a single file based on the data URI. + Construct the full download URL for a given data URI based on the mission. Parameters ---------- uri : str - The product filename or URI to be downloaded. - local_path : str - Directory or filename to which the file will be downloaded. Defaults to current working directory. - cache : bool - Default is True. If file is found on disk, it will not be downloaded again. + The product filename or URI for which to construct the download URL. mission : str, optional The mission to which the file belongs. If not provided, the current value of the ``mission`` attribute will be used. - verbose : bool, optional - Default is True. Whether to show download progress in the console. Returns ------- - status: str - Download status message. Either COMPLETE, SKIPPED, or ERROR. - msg : str - An error status message, if any. url : str The full URL download path. """ - - # Construct the full data URL based on mission current_mission = mission.lower() if mission else self.mission if current_mission in ['hst', 'jwst', 'roman', 'roman_spectra', 'roman_cgi']: @@ -754,11 +756,62 @@ def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbo base_url = self._service_api_connection.MAST_DOWNLOAD_URL keyword = 'uri' # These files require a MAST URI and not just a filename - if not uri.startswith('mast:'): + if not uri.startswith('mast'): raise InvalidQueryError(f'For mission "{current_mission}", a full MAST URI is required ' f'for downloading. Got "{uri}".') - data_url = base_url + f'?{keyword}=' + uri - escaped_url = base_url + f'?{keyword}=' + quote(uri, safe='') + + return base_url + f'?{keyword}=' + quote(uri, safe='') + + def _warn_no_auth(self, download_url): + """ + Warn the user that they are not authenticated to download the file, and provide guidance on how to authenticate. + + Parameters + ---------- + download_url : str + The URL that the user attempted to download from, which requires authentication. + """ + no_auth_msg = f'You are not authorized to access {download_url}.' + if self._authenticated: + no_auth_msg += ('\nYou do not have access to this data, or your authentication ' + 'token may be expired. You can generate a new token at ' + 'https://auth.mast.stsci.edu/token?suggested_name=Astroquery&' + 'suggested_scope=mast:exclusive_access') + else: + no_auth_msg += ('\nPlease authenticate yourself using the `~astroquery.mast.MastMissions.login` ' + 'function or initialize `~astroquery.mast.MastMissions` with an authentication ' + 'token.') + warnings.warn(no_auth_msg, AuthenticationWarning) + + def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbose=True): + """ + Downloads a single file based on the data URI. + + Parameters + ---------- + uri : str + The product filename or URI to be downloaded. + local_path : str + Directory or filename to which the file will be downloaded. Defaults to current working directory. + cache : bool + Default is True. If file is found on disk, it will not be downloaded again. + mission : str, optional + The mission to which the file belongs. If not provided, the current value of the ``mission`` attribute + will be used. + verbose : bool, optional + Default is True. Whether to show download progress in the console. + + Returns + ------- + status: str + Download status message. Either COMPLETE, SKIPPED, or ERROR. + msg : str + An error status message, if any. + url : str + The full URL download path. + """ + # Construct the full data URL based on mission + download_url = self._get_download_url(uri, mission=mission) # Determine local file path. Use current directory as default. filename = Path(uri).name @@ -773,30 +826,20 @@ def download_file(self, uri, *, local_path=None, cache=True, mission=None, verbo try: # Attempt file download - self._download_file(escaped_url, local_path, cache=cache, verbose=verbose) + self._download_file(download_url, local_path, cache=cache, verbose=verbose) # Check if file exists if not local_path.is_file() and status != 'SKIPPED': status = 'ERROR' msg = 'File was not downloaded' - url = data_url + url = download_url except HTTPError as err: if err.response.status_code == 401: - no_auth_msg = f'You are not authorized to download from {data_url}.' - if self._authenticated: - no_auth_msg += ('\nYou do not have access to download this data, or your authentication ' - 'token may be expired. You can generate a new token at ' - 'https://auth.mast.stsci.edu/token?suggested_name=Astroquery&' - 'suggested_scope=mast:exclusive_access') - else: - no_auth_msg += ('\nPlease authenticate yourself using the `~astroquery.mast.MastMissions.login` ' - 'function or initialize `~astroquery.mast.MastMissions` with an authentication ' - 'token.') - log.warning(no_auth_msg) + self._warn_no_auth(download_url) status = 'ERROR' msg = f'HTTPError: {err}' - url = data_url + url = download_url return status, msg, url @@ -823,7 +866,6 @@ def _download_files(self, products, base_dir, *, flat=False, cache=True, verbose response : `~astropy.table.Table` Table containing download results for each data product file. """ - manifest_entries = [] base_dir = Path(base_dir) @@ -945,6 +987,72 @@ def download_products(self, products, *, download_dir=None, flat=False, return manifest + def read_product(self, uri, *, mission=None, **kwargs): + """ + Reads a data product file directly from a URI into an appropriate in-memory object based on file type. + Supported file types are FITS and ASDF. + + FITS files are opened with `~astropy.io.fits.open` and are downloaded and cached on disk. ASDF files + are opened directly from the presigned S3 URL using `~fsspec.open` and `~asdf.open`, without being + downloaded to disk. + + Parameters + ---------- + uri : str + The product filename or URI to be read. + mission : str, optional + The mission to which the file belongs. If not provided, the current value of the ``mission`` attribute + will be used. + **kwargs + Additional keyword arguments to be passed to the file reading function (e.g., `~astropy.io.fits.open` + or `~asdf.open`). + """ + # Construct the full data URL based on mission + download_url = self._get_download_url(uri, mission=mission) + + try: + if uri.endswith((".fits", ".fits.gz")): + # Read in a + return fits.open(download_url, **kwargs) + elif uri.endswith(".asdf"): + if fsspec is None: + raise ImportError('The "fsspec" package is required to read products directly from a URI. ' + 'Please install it with `pip install fsspec` or install astroquery with ' + 'optional dependencies using `pip install astroquery[all]`.') + if asdf is None: + raise ImportError('The "asdf" package is required to read ASDF files. Please install it with ' + '`pip install asdf` or install astroquery with optional dependencies using ' + '`pip install astroquery[all]`.') + for package in ["gwcs", "lz4", "roman_datamodels"]: + if importlib.util.find_spec(package) is None: + warnings.warn(f'The "{package}" package is encouraged when reading ASDF files from the ' + 'Roman Space Telescope mission. Please install it with `pip install {package}` ' + 'or install astroquery with optional dependencies using ' + '`pip install astroquery[all]`.', ImportWarning) + + # Make an authenticated request to get the presigned S3 URL for the ASDF file + headers = {} + if self._authenticated: + headers["Authorization"] = f"token {self._auth_obj.session.cookies['mast_token']}" + resp = requests.get( + download_url, + headers=headers, + allow_redirects=False, + ) + resp.raise_for_status() + + # Use fsspec to open the file directly from the S3 URL, and then read it with asdf + fs = fsspec.filesystem("https") + f = fs.open(resp.headers["Location"], "rb") + return asdf.open(f, **kwargs) + else: + raise InvalidQueryError(f"Unsupported file type for reading: {uri}. " + "Supported types are .fits and .asdf.") + except HTTPError as err: + if err.response.status_code == 401: + self._warn_no_auth(download_url) + raise + @class_or_instance def get_column_list(self): """ diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index ad817949d3..64b78c4188 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,25 +4,25 @@ import os import re import warnings +from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, patch -from pathlib import Path import astropy.units as u -import pytest import numpy as np -from astropy.table import Table, unique +import pytest from astropy.coordinates import SkyCoord from astropy.io import fits +from astropy.table import Table, unique from astropy.utils.exceptions import AstropyDeprecationWarning from requests import HTTPError, Response +from astroquery.exceptions import (AuthenticationWarning, BlankResponseWarning, InputWarning, InvalidQueryError, + MaxResultsWarning, NoResultsWarning, RemoteServiceError, ResolverError) from astroquery.mast import (Catalogs, MastMissions, Observations, Tesscut, Zcut, Mast, utils, services, discovery_portal, auth, core, cloud) from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse -from astroquery.exceptions import (BlankResponseWarning, InvalidQueryError, InputWarning, MaxResultsWarning, - NoResultsWarning, RemoteServiceError, ResolverError) try: # Optional dependency import for cloud access functionality @@ -30,6 +30,12 @@ except ImportError: pass +try: + # Optional dependency import for ASDF file handling + import asdf +except ImportError: + pass + DATA_FILES = {'Mast.Caom.Cone': 'caom.json', 'Mast.Name.Lookup': 'resolver.json', 'mission_search_results': 'mission_results.json', @@ -75,23 +81,21 @@ def data_path(filename): @pytest.fixture(autouse=True) -def patch_post(request): - mp = request.getfixturevalue("monkeypatch") +def patch_post(mocker): + mocker.patch.object(utils, '_simple_request', request_mockreturn) + mocker.patch.object(discovery_portal.PortalAPI, '_request', post_mockreturn) + mocker.patch.object(services.ServiceAPI, '_request', service_mockreturn) + mocker.patch.object(auth.MastAuth, 'session_info', session_info_mockreturn) - mp.setattr(utils, '_simple_request', request_mockreturn) - mp.setattr(discovery_portal.PortalAPI, '_request', post_mockreturn) - mp.setattr(services.ServiceAPI, '_request', service_mockreturn) - mp.setattr(auth.MastAuth, 'session_info', session_info_mockreturn) + mocker.patch.object(Tesscut, '_download_file', tesscut_download_mockreturn) + mocker.patch.object(Zcut, '_download_file', zcut_download_mockreturn) + mocker.patch.object(core.MastQueryWithLogin, '_download_file', download_mockreturn) - mp.setattr(Tesscut, '_download_file', tesscut_download_mockreturn) - mp.setattr(Zcut, '_download_file', zcut_download_mockreturn) - mp.setattr(core.MastQueryWithLogin, '_download_file', download_mockreturn) - - return mp + return mocker @pytest.fixture() -def patch_boto3(monkeypatch, reset_cloud_state): +def patch_boto3(mocker, reset_cloud_state): """Fixture to patch boto3 client and resource for cloud access tests.""" pytest.importorskip('boto3') mock_client = MagicMock() @@ -100,8 +104,8 @@ def patch_boto3(monkeypatch, reset_cloud_state): mock_resource = MagicMock() mock_resource.Bucket.return_value.download_file.return_value = None - monkeypatch.setattr('boto3.client', lambda *args, **kwargs: mock_client) - monkeypatch.setattr('boto3.resource', lambda *args, **kwargs: mock_resource) + mocker.patch("boto3.client", return_value=mock_client) + mocker.patch("boto3.resource", return_value=mock_resource) return mock_client, mock_resource @@ -670,23 +674,24 @@ def test_missions_download_no_auth(caplog): # Exclusive access products should not be downloaded if user is not authenticated # User is not authenticated uri = 'unauthorized.fits' - result = MastMissions.download_file(uri) + with pytest.warns(AuthenticationWarning) as warn_auth: + result = MastMissions.download_file(uri) + msg = str(warn_auth[0].message) assert result[0] == 'ERROR' assert 'HTTPError' in result[1] - with caplog.at_level('WARNING', logger='astroquery'): - assert 'You are not authorized to download' in caplog.text - assert 'Please authenticate yourself' in caplog.text - caplog.clear() + assert 'You are not authorized' in msg + assert 'Please authenticate yourself' in msg # User is authenticated, but doesn't have proper permissions test_token = "56a9cf3df4c04052atest43feb87f282" MastMissions.login(token=test_token) - result = MastMissions.download_file(uri) + with pytest.warns(AuthenticationWarning) as warn_auth: + result = MastMissions.download_file(uri) + msg = str(warn_auth[0].message) assert result[0] == 'ERROR' assert 'HTTPError' in result[1] - with caplog.at_level('WARNING', logger='astroquery'): - assert 'You are not authorized to download' in caplog.text - assert 'You do not have access to download this data' in caplog.text + assert 'You are not authorized to access' in msg + assert 'You do not have access to this data' in msg def test_missions_get_dataset_kwd(caplog): @@ -729,6 +734,115 @@ def test_missions_radius_too_large(method, kwargs): getattr(m, method)(coordinates=coordinates, radius=radius, **kwargs) +@pytest.fixture +def mock_fits_open(mocker): + """Mock fits.open to return a valid HDUList without network access.""" + return mocker.patch("astropy.io.fits.open", return_value=fits.HDUList([fits.PrimaryHDU()])) + + +@pytest.fixture +def mock_asdf_open(mocker): + """Mock asdf.open and fsspec.filesystem for ASDF file reading without network access.""" + pytest.importorskip("asdf") + pytest.importorskip("fsspec") + + # Mock response from requests.get for the initial URL fetch + mock_response = MagicMock() + mock_response.status_code = 307 + mock_response.headers = { + "Location": "https://example-bucket.s3.amazonaws.com/test.asdf?signature=abc123" + } + mock_requests = MagicMock() + mock_requests.get.return_value = mock_response + mocker.patch("requests.get", mock_requests) + + # Mock fsspec.filesystem and the file handle + mock_fs = MagicMock() + mock_file = MagicMock() + mock_fs.open.return_value = mock_file + mocker.patch("fsspec.filesystem", return_value=mock_fs) + + # Create a mock AsdfFile + if asdf is not None: + mock_asdf_file = MagicMock(spec=asdf.AsdfFile) + else: + mock_asdf_file = MagicMock() + + return mocker.patch("asdf.open", return_value=mock_asdf_file) + + +def test_missions_read_product_fits(mocker, mock_fits_open): + """Test reading a FITS file product.""" + mocker.patch.object(MastMissions, "_get_download_url", return_value="https://test.url/file.fits") + uri = "file.fits" + obj = MastMissions.read_product(uri, mission='jwst') + assert isinstance(obj, fits.HDUList) + assert len(obj) == 1 # Should have at least the primary HDU + + # Call with additional kwargs to ensure they are passed to fits.open + obj = MastMissions.read_product(uri, mission='jwst', memmap=False) + mock_fits_open.assert_called_with("https://test.url/file.fits", memmap=False) + assert isinstance(obj, fits.HDUList) + + +def test_missions_read_product_asdf(mocker, mock_asdf_open): + """Test reading an ASDF file product.""" + # Mock importlib.util.find_spec to always return a spec (all packages present) + mocker.patch("importlib.util.find_spec", return_value=MagicMock()) + + uri = "file.asdf" + obj = MastMissions.read_product(uri, mission='roman') + assert isinstance(obj, asdf.AsdfFile) + + # Additional kwargs should be passed to asdf.open + obj = MastMissions.read_product(uri, mission='roman', copy_arrays=True) + # Verify asdf.open was called with kwargs + assert mock_asdf_open.call_args[1]['copy_arrays'] is True + assert isinstance(obj, asdf.AsdfFile) + + +def test_missions_read_product_asdf_missing_packages(mocker, mock_asdf_open): + """Test that warnings are issued for missing ASDF-related packages.""" + # Mock importlib.util.find_spec to return None for the packages + def mock_find_spec(package_name): + if package_name in ["gwcs", "lz4", "roman_datamodels"]: + return None # Simulate missing package + return MagicMock() # Return something for other packages + + mocker.patch("importlib.util.find_spec", side_effect=mock_find_spec) + + # Capture warnings + with pytest.warns(ImportWarning) as warning_list: + obj = MastMissions.read_product("test_file.asdf", mission='roman') + + # Check that warnings were issued for all three packages + assert len(warning_list) == 3 + warning_messages = [str(w.message) for w in warning_list] + + for package in ["gwcs", "lz4", "roman_datamodels"]: + assert any(package in msg and "encouraged" in msg for msg in warning_messages) + + # Verify the object is still returned correctly + assert isinstance(obj, asdf.AsdfFile) + + +def test_missions_read_product_unsupported_format(): + """Test that unsupported file formats raise an InvalidQueryError.""" + with pytest.raises(InvalidQueryError, match="Unsupported file type"): + MastMissions.read_product("unsupported_file.txt") + + +def test_missions_read_product_unauthorized(mocker): + """Test that unauthorized file access raises an error.""" + resp = Response() + resp.status_code = 401 + mocker.patch("astropy.io.fits.open", side_effect=HTTPError(response=resp)) + + with pytest.raises(HTTPError): + with pytest.warns(AuthenticationWarning, match="You are not authorized"): + MastMissions.read_product("file.fits", mission='jwst') + + ################### # MastClass tests # ################### @@ -1064,7 +1178,7 @@ def test_observations_filter_products(): @patch.object(Path, "is_file", return_value=True) -def test_observations_download_products(mock_is_file, patch_boto3, monkeypatch, tmpdir): +def test_observations_download_products(mock_is_file, patch_boto3, mocker, tmpdir): mock_resource = patch_boto3[1] obsid = '2003738726' data_uri = 'mast:HST/product/u9o40504m_c3m.fits' @@ -1132,7 +1246,7 @@ def test_observations_download_products(mock_is_file, patch_boto3, monkeypatch, assert result[0]['Status'] == 'SKIPPED' # Products not found in cloud - monkeypatch.setattr(Observations, 'get_cloud_uris', lambda *a, **k: {}) + mocker.patch.object(Observations, 'get_cloud_uris', lambda *a, **k: {}) with pytest.warns(NoResultsWarning, match='was not found in the cloud. Skipping download.'): result = Observations.download_products(obsid, dataURI=data_uri, @@ -1734,14 +1848,14 @@ def test_tesscut_download_cutouts_mt_no_sector(tmpdir): assert os.path.isfile(manifest[0]["Local Path"]) -def test_tesscut_get_cutouts_mt_no_sector_empty_results(monkeypatch): +def test_tesscut_get_cutouts_mt_no_sector_empty_results(mocker): """Test get_cutouts with moving target when no sectors are available. When get_sectors returns an empty table, the method should warn and return an empty list. """ # Mock get_sectors to return an empty Table empty_sector_table = Table(names=["sectorName", "sector", "camera", "ccd"], dtype=[str, int, int, int]) - monkeypatch.setattr(Tesscut, "get_sectors", lambda *args, **kwargs: empty_sector_table) + mocker.patch.object(Tesscut, "get_sectors", lambda *args, **kwargs: empty_sector_table) with pytest.warns(NoResultsWarning, match="Coordinates are not in any TESS sector"): cutout_hdus_list = Tesscut.get_cutouts(object_name="NonExistentObject", moving_target=True, size=5) @@ -1749,14 +1863,14 @@ def test_tesscut_get_cutouts_mt_no_sector_empty_results(monkeypatch): assert len(cutout_hdus_list) == 0 -def test_tesscut_download_cutouts_mt_no_sector_empty_results(tmpdir, monkeypatch): +def test_tesscut_download_cutouts_mt_no_sector_empty_results(tmpdir, mocker): """Test download_cutouts with moving target when no sectors are available. When get_sectors returns an empty table, the method should warn and return an empty Table. """ # Mock get_sectors to return an empty Table empty_sector_table = Table(names=["sectorName", "sector", "camera", "ccd"], dtype=[str, int, int, int]) - monkeypatch.setattr(Tesscut, "get_sectors", lambda *args, **kwargs: empty_sector_table) + mocker.patch.object(Tesscut, "get_sectors", lambda *args, **kwargs: empty_sector_table) with pytest.warns(NoResultsWarning, match="Coordinates are not in any TESS sector"): manifest = Tesscut.download_cutouts( diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index a3c1d0405c..53262b09b6 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -18,6 +18,11 @@ from ...exceptions import (InputWarning, InvalidQueryError, MaxResultsWarning, NoResultsWarning) from ..utils import ResolverError +try: + import asdf +except ImportError: + asdf = None + @pytest.fixture(scope="module") def msa_product_table(): @@ -404,11 +409,23 @@ def check_result(result, path): result = MastMissions.download_file(uri, local_path=local_path_file) check_result(result, local_path_file) + @pytest.mark.parametrize("uri, mission", [ + ("OFAE10C3Q/16j1600do_ccd.fits", "hst"), + ("jwst_niriss_trappars_0002.fits", "jwst"), + ("mast:HLSP/classy/j0021+0052/hlsp_classy_hst_cos_j0021+0052_multi_v1_coadded.fits", "classy"), + ]) + def test_missions_read_product_fits(self, uri, mission): + # Test that read_product can read a FITS file from a URI and return an HDUList + hdul = MastMissions.read_product(uri, mission=mission) + assert isinstance(hdul, fits.HDUList) + assert len(hdul) > 1 + hdul.close() + @pytest.mark.parametrize("mission, query_params", [ ('jwst', {'fileSetName': 'jw01189001001_02101_00001'}), ('classy', {'Target': 'J0021+0052'}), ('ullyses', {'host_galaxy_name': 'WLM', 'select_cols': ['observation_id']}), - ('roman', {'program': 3, 'pass_id': 1}), + ('roman', {'program': 163}), ('iue', {'iue_data_id': 'LWR08496'}), ]) def test_missions_workflow(self, tmp_path, mission, query_params): @@ -437,6 +454,18 @@ def test_missions_workflow(self, tmp_path, mission, query_params): if row['Status'] == 'COMPLETE': assert (row['Local Path']).is_file() + # Read a product + product = None + for prod in prods: + if prod['filename'].endswith(('.fits', '.asdf')): + product = prod + break + obj = m.read_product(product['uri'], mission=mission) + if mission == 'roman' and asdf is not None: + assert isinstance(obj, asdf.AsdfFile) + assert isinstance(obj, fits.HDUList) + obj.close() + ################### # MastClass tests # ################### diff --git a/docs/conf.py b/docs/conf.py index 037197122e..705da3ce2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,9 +13,10 @@ import datetime import sys -import tomllib from pathlib import Path +import tomllib + # Load all of the global Astropy configuration try: from sphinx_astropy.conf.v1 import * # noqa @@ -55,6 +56,8 @@ 'regions': ('https://astropy-regions.readthedocs.io/en/stable', None), 'mocpy': ('https://cds-astro.github.io/mocpy', None), 'pyvo': ('https://pyvo.readthedocs.io/en/stable', None), + 'asdf': ('https://asdf.readthedocs.io/en/stable', None), + 'fsspec': ('https://filesystem-spec.readthedocs.io/en/stable', None), }) # -- Project information ------------------------------------------------------ diff --git a/docs/index.rst b/docs/index.rst index 913d153322..4b80a26457 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,7 +40,7 @@ already installed, please make sure you use the ``--upgrade`` (or ``-U``) instal $ python -m pip install -U --pre astroquery To install all the mandatory and optional dependencies add the ``[all]`` -identifyer to the pip command above (or use ``[docs]`` or ``[test]`` for the +identifier to the pip command above (or use ``[docs]`` or ``[test]`` for the dependencies required to build the documentation or run the tests): .. code-block:: bash diff --git a/docs/mast/mast_missions.rst b/docs/mast/mast_missions.rst index f7e13f3d87..08b223f866 100644 --- a/docs/mast/mast_missions.rst +++ b/docs/mast/mast_missions.rst @@ -3,7 +3,7 @@ Mission-Specific Queries ************************ -The `~astroquery.mast.MastMissionsClass` class allows for search queries based on mission-specific +The `~astroquery.mast.MastMissionsClass` class allows for search queries based on mission-specific metadata for a given data collection. This metadata includes header keywords, proposal information, and observational parameters. The following missions/products are currently available for search: @@ -49,12 +49,12 @@ To search for JWST metadata, the ``mission`` attribute is reassigned to ``'JWST' Querying Missions ================== -The MastMissions interface provides three closely related query methods. All three methods return results as an `~astropy.table.Table` -and all three support column-based filtering, sorting, and result limiting. The primary difference between them is how positional +The MastMissions interface provides three closely related query methods. All three methods return results as an `~astropy.table.Table` +and all three support column-based filtering, sorting, and result limiting. The primary difference between them is how positional constraints are specified. At a high level: - - `~astroquery.mast.MastMissionsClass.query_criteria` is the most flexible method. It supports purely column-based queries, + - `~astroquery.mast.MastMissionsClass.query_criteria` is the most flexible method. It supports purely column-based queries, purely positional queries, or a combination of both. - `~astroquery.mast.MastMissionsClass.query_region` is a convenience wrapper for positional queries using coordinates. @@ -78,7 +78,7 @@ using the `~astroquery.mast.MastMissionsClass.get_column_list` method. Keyword arguments can also be used to refine results further. The following parameters are available: -- ``radius``: For positional searches only. Only return results within a certain distance from an object or set of coordinates. +- ``radius``: For positional searches only. Only return results within a certain distance from an object or set of coordinates. Default is 3 arcminutes. - ``limit``: The maximum number of results to return. Default is 5000. @@ -99,17 +99,17 @@ Writing Queries ---------------- The `~astroquery.mast.MastMissionsClass.query_criteria` method supports both positional parameters and column-based filters. -Positional constraints are optional. +Positional constraints are optional. Supported positional parameters include: - ``coordinates`` : Sky coordinates around which to perform a cone search. - ``object_name`` : Name(s) of the object(s) around which to perform a cone search. - ``resolver`` : Resolver service to use for object name resolution. - - ``radius`` : Radius of the cone searches around the specified coordinates or object names. Can be defined as an `~astropy.units.Quantity`, + - ``radius`` : Radius of the cone searches around the specified coordinates or object names. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. -Multiple coordinates or objects may be queried in a single request. The ``coordinates`` and ``object_names`` parameters +Multiple coordinates or objects may be queried in a single request. The ``coordinates`` and ``object_names`` parameters accept a single value, an iterable of values, or a comma-separated string. When multiple values are provided for either parameter, results matching *any* of the supplied positions are returned. @@ -117,7 +117,7 @@ results matching *any* of the supplied positions are returned. >>> from astropy.coordinates import SkyCoord >>> select_cols = ["sci_targname", "sci_pep_id", "sci_status"] - >>> results = missions.query_criteria(coordinates=[SkyCoord(245.89675, -26.52575, unit='deg'), "205.54842 28.37728"], + >>> results = missions.query_criteria(coordinates=[SkyCoord(245.89675, -26.52575, unit='deg'), "205.54842 28.37728"], ... object_names=["M2", "M9"], ... radius=0.1, ... select_cols=select_cols, @@ -168,13 +168,13 @@ Criteria syntax supports several operations: - For numeric or date columns, select an inclusive range with the syntax ``'#..#'``. -- Wildcards are special characters used in search patterns to represent one or more unknown characters, +- Wildcards are special characters used in search patterns to represent one or more unknown characters, allowing for flexible matching of strings. The wildcard character is ``*`` and it replaces any number of characters preceding, following, or in between existing characters, depending on its placement. .. note:: - For the Roman mission, query methods also support the ``pass_id`` parameter as an alias for the ``pass`` column, + For the Roman mission, query methods also support the ``pass_id`` parameter as an alias for the ``pass`` column, which refers to a single iteration of a pass plan. This is to avoid conflicts with the reserved Python keyword. .. doctest-remote-data:: @@ -196,7 +196,7 @@ Criteria syntax supports several operations: N4A702010 GAL-CLUS-0026+1653-ARCC F110W IMAGE N4A705010 GAL-CLUS-0026+1653-ARCC F110W IMAGE -The `~astroquery.mast.MastMissionsClass.query_region` and `~astroquery.mast.MastMissionsClass.query_object` methods are +The `~astroquery.mast.MastMissionsClass.query_region` and `~astroquery.mast.MastMissionsClass.query_object` methods are convenience wrappers around `~astroquery.mast.MastMissionsClass.query_criteria`: - `~astroquery.mast.MastMissionsClass.query_region` requires ``coordinates``. @@ -209,7 +209,7 @@ Both methods also accept column-based criteria, which are applied in the same wa >>> regionCoords = SkyCoord(210.80227, 54.34895, unit=('deg', 'deg')) >>> select_cols = ["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"] - >>> results = missions.query_region(regionCoords, + >>> results = missions.query_region(regionCoords, ... radius=3, ... sci_pep_id=12556, ... select_cols=select_cols, @@ -226,8 +226,8 @@ Both methods also accept column-based criteria, which are applied in the same wa .. doctest-remote-data:: - >>> results = missions.query_object('M101', - ... radius=3, + >>> results = missions.query_object('M101', + ... radius=3, ... select_cols=select_cols, ... sort_by='sci_targname') >>> results[:5] # doctest: +IGNORE_OUTPUT @@ -248,11 +248,11 @@ Getting Product Lists ---------------------- Each observation returned from a MAST query can have one or more associated data products. Given -one or more datasets or dataset IDs, the `~astroquery.mast.MastMissionsClass.get_product_list` function +one or more datasets or dataset IDs, the `~astroquery.mast.MastMissionsClass.get_product_list` function will return a `~astropy.table.Table` containing the associated data products. -`~astroquery.mast.MastMissionsClass.get_product_list` also includes an optional ``batch_size`` parameter, -which controls how many datasets are sent to the MAST service per request. This can be useful for managing +`~astroquery.mast.MastMissionsClass.get_product_list` also includes an optional ``batch_size`` parameter, +which controls how many datasets are sent to the MAST service per request. This can be useful for managing memory usage or avoiding timeouts when requesting product lists for large numbers of datasets. If not provided, batch_size defaults to 1000. @@ -262,7 +262,7 @@ If not provided, batch_size defaults to 1000. ... sci_hlsp='>1') >>> products = missions.get_product_list(datasets[:2], batch_size=1000) >>> print(products[:5]) # doctest: +IGNORE_OUTPUT - product_key access dataset ... category size type + product_key access dataset ... category size type ---------------------------- ------ --------- ... ---------- --------- ------- JBTAA0010_jbtaa0010_asn.fits PUBLIC JBTAA0010 ... AUX 11520 science JBTAA0010_jbtaa0010_drz.fits PUBLIC JBTAA0010 ... CALIBRATED 214655040 science @@ -298,8 +298,8 @@ and any other of the product fields. The **AND** operation is applied between filters, and the **OR** operation is applied within each filter set, except in the case of negated values. A filter value can be negated by prefiing it with ``!``, meaning that rows matching that value will be excluded from the results. -When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated -values are combined with **AND** logic against the positives. +When any negated value is present in a filter set, any positive values in that set are combined with **OR** logic, and the negated +values are combined with **AND** logic against the positives. For example: - ``file_suffix=['A', 'B', '!C']`` → (file_suffix != C) AND (file_suffix == A OR file_suffix == B) @@ -318,28 +318,56 @@ The filter below returns FITS products that are "science" type **and** less than >>> filtered = missions.filter_products(products, ... extension='fits', ... type='science', - ... size='<=20000', + ... size='<=20000', ... file_suffix=['ASN', 'JIF']) >>> print(filtered) # doctest: +IGNORE_OUTPUT - product_key access dataset ... category size type + product_key access dataset ... category size type ---------------------------- ------ --------- ... -------------- ----- ------- JBTAA0010_jbtaa0010_asn.fits PUBLIC JBTAA0010 ... AUX 11520 science JBTAA0020_jbtaa0020_asn.fits PUBLIC JBTAA0020 ... AUX 11520 science +Reading Data Products +====================== + +The `~astroquery.mast.MastMissionsClass.read_product` function allows you to read FITS or ASDF data products directly into memory as `~astropy.io.fits.HDUList` +or `~asdf.AsdfFile` objects, respectively. The function accepts a product URI or a direct filename (for certain missions) as input. + +FITS files are opened with `~astropy.io.fits.open` and are downloaded and cached locally. + +ASDF products from the Roman Space Telescope mission are opened directly with `~fsspec.open` and `~asdf.open`. The products are +read as `~asdf.AsdfFile` objects from presigned S3 URLs, without being downloaded locally. This requires the ``asdf`` and ``fsspec`` +packages to be installed. Other optional packages for reading products may be required depending on the product type and file format. +These packages are ``gwcs``, ``lz4``, and ``roman_datamodels``. To install astroquery with all optional dependencies, +use ``pip install astroquery[all]``. + +Remember that this method returns an open file object, so it is the user's responsibility to close the file when finished. This can be done +with a context manager or by calling the ``close()`` method on the returned object. + +.. doctest-remote-data:: + >>> obj = missions.read_product("jbtaa0010_asn.fits") # doctest: +IGNORE_OUTPUT + >>> print(type(obj)) + + >>> obj.info() # doctest: +IGNORE_OUTPUT + Filename: /Users/user/.astropy/cache/download/url/6555b8e890752df0de737940d3afb19d/contents + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 44 () + 1 ASN 1 BinTableHDU 25 3R x 3C [14A, 14A, L] + >>> obj.close() + Downloding Data =============== Downloading Data Products ------------------------- -The `~astroquery.mast.MastMissionsClass.download_products` function accepts a table of products like the one above -and will download the products to your local machine. Products may also be provided as dataset IDs with product filters, +The `~astroquery.mast.MastMissionsClass.download_products` function accepts a table of products like the one above +and will download the products to your local machine. Products may also be provided as dataset IDs with product filters, or as JSON product metadata sent by the MAST subscription service (either as a local JSON file or as in-memory data). By default, products will be downloaded into the current working directory, in a subdirectory called ``mastDownload``. -The full local filepaths will have the form ``mastDownload///file.`` You can change the download -directory using the ``download_dir`` parameter. If ``flat=True`` is specified, all files will be downloaded directly into the +The full local filepaths will have the form ``mastDownload///file.`` You can change the download +directory using the ``download_dir`` parameter. If ``flat=True`` is specified, all files will be downloaded directly into the ``download_dir`` without any subdirectories. .. doctest-remote-data:: @@ -349,14 +377,14 @@ directory using the ``download_dir`` parameter. If ``flat=True`` is specified, a Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0020%2Fjbtaa0020_asn.fits to mastDownload/hst/JBTAA0020/jbtaa0020_asn.fits ... [Done] Downloading URL https://mast.stsci.edu/search/hst/api/v0.1/retrieve_product?product_name=JBTAA0020%2Fjbtaa0020_jif.fits to mastDownload/hst/JBTAA0020/jbtaa0020_jif.fits ... [Done] >>> print(manifest) # doctest: +IGNORE_OUTPUT - Local Path Status Message URL + Local Path Status Message URL --------------------------------------------- -------- ------- ---- mastDownload/hst/JBTAA0010/jbtaa0010_asn.fits COMPLETE None None mastDownload/hst/JBTAA0010/jbtaa0010_jif.fits COMPLETE None None mastDownload/hst/JBTAA0020/jbtaa0020_asn.fits COMPLETE None None mastDownload/hst/JBTAA0020/jbtaa0020_jif.fits COMPLETE None None -The function also accepts dataset IDs and product filters as input for a more streamlined workflow. +The function also accepts dataset IDs and product filters as input for a more streamlined workflow. .. doctest-remote-data:: >>> missions.download_products(['JBTAA0010', 'JBTAA0020'], @@ -374,7 +402,7 @@ Downloading a Single File To download a single data product file, use the `~astroquery.mast.MastMissionsClass.download_file` function with a MAST URI as input. Some missions (e.g., HST, JWST) accept direct filenames as input, but others require a fully-qualified ``mast:`` URI. -The default is to download the file to the current working directory, but you can specify the download directory or filepath with +The default is to download the file to the current working directory, but you can specify the download directory or filepath with the ``local_path`` keyword argument. .. doctest-remote-data:: diff --git a/pyproject.toml b/pyproject.toml index bbe6c2c6cd..6b1d794993 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ all = [ "boto3", "botocore", "regions>=0.5", + "roman_datamodels", ] [build-system] diff --git a/tox.ini b/tox.ini index 1bd03f7e0d..d9648958f7 100644 --- a/tox.ini +++ b/tox.ini @@ -53,6 +53,8 @@ deps = oldestdeps-alldeps: mocpy==0.12 oldestdeps-alldeps: regions==0.5 oldestdeps-alldeps: astropy-healpix==0.7 + oldestdeps-alldeps: gwcs==0.18 + oldestdeps-alldeps: roman_datamodels==0.11.0 online: pytest-custom_exit_code