From 1aebba364eccf9bd360989196729861a49757b19 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Thu, 15 Jan 2026 15:38:16 -0500 Subject: [PATCH 01/11] Adding query_all() to heasarc. --- astroquery/heasarc/core.py | 235 +++++++++++++++++- astroquery/heasarc/tests/test_heasarc.py | 88 +++++++ .../heasarc/tests/test_heasarc_remote.py | 10 + docs/heasarc/heasarc.rst | 25 +- 4 files changed, 356 insertions(+), 2 deletions(-) diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index f92121ff70..e835bb4fd3 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -8,8 +8,9 @@ from astropy import coordinates from astropy import units as u from astropy.utils.decorators import deprecated, deprecated_renamed_argument - +from astropy.time import Time import pyvo +import re from astroquery import log from ..query import BaseQuery, BaseVOQuery @@ -612,6 +613,238 @@ def query_object(self, object_name, mission, *, return self.query_region(pos, catalog=mission, spatial='cone', get_query_payload=get_query_payload) + def _get_vector(ra=None, dec=None): + """ + If the input is a string name of a column like "a.ra", then this routine + constructs the unit vector column names that can be added to the SQL query + to represent the unit vector. If the input is a number, then it will actually + calculate the unit vector and return the values as strings to be added to the + SQL query. + + The former is used to fetch pre-computed unit vectors columns associated with + the table being queried. The latter is used to compute the input position unit + vector only once and put the numeric value in the query constraint. + """ + # Note that Astropy flips x and y compared to this, which is used internally + # despite the fact that our RA, DEC values are in ICRS. + try: + r, d = np.radians([float(ra), float(dec)]) + return ( + np.cos(d) * np.sin(r), + np.cos(d) * np.cos(r), + np.sin(d) + ) + except ValueError: + prefix = ra.split('.')[0] # e.g., 'a' from 'a.ra' + return (f"{prefix}.__x_ra_dec", f"{prefix}.__y_ra_dec", f"{prefix}.__z_ra_dec") + except Exception as e: + raise e + + def _fast_geometry_constraint(ra, dec, large=False, radius=None): + """ + Construct the spatial constraint to be added to the WHERE clause. It compares + the input position with the catalog's pre-computed unit vector columns + with the computation optimized for speed. The optimization was done by Tom McGlynn + for the Xamin GUI and the algorithm copied here. + + The master position tables are split into those where the default sensible search + radius is larger or smaller than 1 degree. + """ + vec0 = HeasarcClass._get_vector("a.ra", "a.dec") + vec1 = HeasarcClass._get_vector(ra, dec) + dot_product = " + ".join([f"{vec0[i]}*{vec1[i]}" for i in range(3)]) + if radius is not None: + if not isinstance(radius, (int, float)): + radius = radius.value + radius_condition = f"{dot_product} > (cos(radians(({radius}))))" + dec_condition = f"a.dec between {dec} - {radius} and {dec} + {radius}" + else: + # Assuming 'a.dsr' is the default search radius column in degrees. This value is + # defined by HEASARC curators for each table. + radius_condition = f"{dot_product} > (cos(radians((a.dsr*60/60))))" + dec_condition = f"a.dec between {dec} - a.dsr*60/60 and {dec} + a.dsr*60/60" + if large: + return f""" + ( ({radius_condition}) + and ({dec_condition}) ) + """ + else: + # Additional constraints on tables with search radii less than 1 deg, + # which speeds up the whole thing. + radius_condition_1deg = f"{dot_product} > {np.cos(np.radians(1.0))}" + dec_condition_1deg = f"a.dec between {float(dec) - 1} and {float(dec)+1}" + return f""" + ( ({radius_condition}) + and ({dec_condition}) + and ({radius_condition_1deg}) + and ({dec_condition_1deg}) + ) + """ + + def _time_constraint(start_time=None, end_time=None): + """" + Converts input string like "2025-01-02T01:00:00..2025-01-05T23:59:59" + into a decimal MJD constraint. + """ + start_mjd = Time(start_time, format='isot').mjd + end_mjd = Time(end_time, format='isot').mjd + return f"end_time > {start_mjd:.6f} AND start_time < {end_mjd:.6f}" + + def _query_matches(ra=None, dec=None, start_time=None, end_time=None, radius=None): + """ + Constructs the full SQL query including the spatial and time constraints. + Note that this queries multiple tables, as the HEASARC database has split + the master tables for efficiency. + """ + if ra is not None: + constraint_small = HeasarcClass._fast_geometry_constraint(ra, dec, large=False, radius=radius) + constraint_big = HeasarcClass._fast_geometry_constraint(ra, dec, large=True, radius=radius) + if start_time is not None: + constraint_time = HeasarcClass._time_constraint(start_time, end_time) + + tname1, tname2 = None, None + if ra is not None and start_time is None: + tname1 = 'pos_small' + tname2 = "pos_big" + elif ra is not None and start_time is not None: + tname1 = 'pos_time_small' + tname2 = 'pos_time_big' + constraint_small += f" AND {constraint_time}" + constraint_big += f" AND {constraint_time}" + elif ra is None and start_time is not None: + tname1 = 'time' + else: + raise ValueError("You must specify either a position or time range or both") + + if ra is not None: + full_query = f""" + select b.name as "table_name", count(*) as "count", b.description as + "description", b.regime as "regime", b.mission as "mission", b.type + as "obj_type" + from master_table.{tname1} as a,master_table.indexview as b + where ( ( a.table_name = b.name ) ) and + {constraint_small} + group by b.name , b.description , b.regime , b.mission , b.type + + union all + + select b.name as "table_name", count(*) as "count", b.description as + "description", b.regime as "regime", b.mission as "mission", b.type + as "obj_type" + from master_table.{tname2} as a,master_table.indexview as b + where ( ( a.table_name = b.name ) ) and + {constraint_big} + group by b.name , b.description , b.regime , b.mission , b.type + order by count desc + """ + else: + full_query = f""" + select b.name as "table_name", count(*) as "count", b.description as + "description", b.regime as "regime", b.mission as "mission", b.type + as "obj_type" + from master_table.{tname1} as a,master_table.indexview as b + where ( ( a.table_name = b.name ) ) and + {constraint_time} + group by b.name , b.description , b.regime , b.mission , b.type + order by count desc + """ + # remove all extraneous white space and line breaks + return re.sub(r'\s+', ' ', full_query.replace('\n', '')).strip() + return full_query + + def query_all(self, position=None, get_query_payload=False, start_time=None, + end_time=None, verbose=False, maxrec=None, radius=None): + """ + Query the HEASARC database to count matches at a given position for all available catalogs. + + Parameters + ---------- + position : str, `astropy.coordinates` object + The position around which to search. Must be a SkyCoord object or a string + that Astropy can convert. + start_time : str, `astropy.time` object + Beginning of time range of interest as a string in ISOT format + or Time object. + end_time : str, `astropy.time` object + End of time range of interest as a string in ISOT format + or Time object. + get_query_payload : bool, optional + If `True` then returns the generated ADQL query as str and does not send the query. + Defaults to `False`. + radius : str or `~astropy.units.Quantity` object + If this radius is None, the specified coordinate is compared to each mission + catalog entry using that catalog's default radius. This is based on the + approximate PSF. If you specify a radius in degrees, it uses that instead. + Be aware that for missions with large PSFs, when you search within a very small + radius, you may not find catalog entries that are within the PSF and + therefore might be of interest. + verbose : bool, optional + If True, prints additional information about the query. Default is False. + maxrec : int, optional + The maximum number of records to return. If None, all matching records are returned up to the server limit. + **kwargs : dict, optional + Additional keyword arguments: + + Returns + ------- + result : `~astropy.table.Table` + A table containing the results of the query, i.e. a list of catalogs + that have entries near the specified position, how many, and quick catalog + information. If no results are found, an empty table is returned and + a warning is issued. + + Raises + ------ + ValueError + If the position is not provided or is not a SkyCoord object. + + Notes + ----- + This method queries all HEASARC catalogs for sources near the specified position. + The results include the table name, number of matches, table description, regime, + mission, and object type for each catalog. + + The user can select the table name(s) of interest and then use the query_object(), query_region(), etc. + + The query uses the HEASARC TAP service to search position-only master tables efficiently. + + Examples + -------- + >>> from astropy.coordinates import SkyCoord + >>> from astropy import units as u + >>> position = SkyCoord(ra=10.68458, dec=41.26917, unit=(u.degree, u.degree), frame='icrs') + >>> result = Heasarc.query_all(position) + >>> print(result) + + """ + if position is not None: + coords_icrs = parse_coordinates(position).icrs + ra, dec = coords_icrs.ra.deg, coords_icrs.dec.deg + if position is None and start_time is not None: + ra = None + dec = None + if ((position is None and start_time is None)): + raise ValueError("A valid position and/or a time range must be provided.") + + full_query = HeasarcClass._query_matches(ra=ra, dec=dec, + start_time=start_time, + end_time=end_time, radius=radius) + + if get_query_payload: + return full_query + + response = self.query_tap(query=full_query, maxrec=maxrec) + + # save the response in case we want to use it later + self._last_result = response + + table = response.to_table() + if len(table) == 0: + warnings.warn( + NoResultsWarning("No matching rows were found in the query.") + ) + return table + def locate_data(self, query_result=None, catalog_name=None): """Get links to data products Use vo/datalinks to query the data products for some query_results. diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index 0091190f06..adffeb7089 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -7,6 +7,7 @@ from astropy.coordinates import SkyCoord from astropy.table import Table import astropy.units as u +from astropy.time import Time from astroquery.heasarc import Heasarc, HeasarcClass from astroquery.exceptions import InvalidQueryError @@ -719,3 +720,90 @@ def test_s3_mock_directory(s3_mock): assert os.path.exists(f"{tmpdir}/location/file1.txt") assert os.path.exists(f"{tmpdir}/location/sub/file2.txt") assert os.path.exists(f"{tmpdir}/location/sub/sub2/file3.txt") + + +def test__get_vec(): + # Test column name input + assert HeasarcClass._get_vec("a.ra", "a.dec") == \ + ("a.__x_ra_dec", "a.__y_ra_dec", "a.__z_ra_dec") + # Test numeric input + actual = HeasarcClass._get_vec("217.0", "-31.7") + desired = (-0.5120309075160554, -0.6794879643287802, -0.5254716510722678) + # Convert to float for comparison + assert all(abs(d - a) < 0.5 * (10 ** (-6)) for d, a in zip(desired, actual)) + + +def adql_str_comp(testing=str, reference=str): + "just makes sure whitespace changes don't matter" + import re + return re.sub(r'\s+', ' ', testing.replace('\n', '')).strip()\ + == re.sub(r'\s+', ' ', reference.replace('\n', '')).strip() + + +def test__constraint_matches(): + # Testing all together because it's easier to read this way. + constraint_small = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=False) + desired_small = """ + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) + and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) + and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) + and (a.dec between -32.7 and -30.7) + ) + """ + assert adql_str_comp(constraint_small, desired_small) + + constraint_large = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=True) + desired_large = """ + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) + and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) ) + """ + assert adql_str_comp(constraint_large, desired_large) + constraint_large_rad = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", + radius=0.5*u.deg, large=True) + assert "(a.dec between -31.7 - 0.5 and -31.7 + 0.5)" in constraint_large_rad + + constraint_time = HeasarcClass._time_constraint(start_time=Time("2017-01-01"), end_time=Time("2017-01-02")) + desired_time = "end_time > 57754.000000 AND start_time < 57755.000000" + assert adql_str_comp(constraint_time, desired_time) + + constraint_full = HeasarcClass._query_matches("217.0", "-31.7") + desired_full = f""" + select b.name as "table_name", count(*) as "count", b.description as + "description", b.regime as "regime", b.mission as "mission", b.type + as "obj_type" + from master_table.pos_small as a,master_table.indexview as b + where ( ( a.table_name = b.name ) ) and + {desired_small} + group by b.name , b.description , b.regime , b.mission , b.type + + union all + + select b.name as "table_name", count(*) as "count", b.description as + "description", b.regime as "regime", b.mission as "mission", b.type + as "obj_type" + from master_table.pos_big as a,master_table.indexview as b + where ( ( a.table_name = b.name ) ) and + {desired_large} + group by b.name , b.description , b.regime , b.mission , b.type + order by count desc + """ + assert adql_str_comp(constraint_full, desired_full) + + constraint_with_time = HeasarcClass._query_matches("217.0", "-31.7", + start_time="2017-01-01", + end_time="2020-01-02") + assert "end_time > 57754.000000 AND start_time < 58850.000000" in constraint_with_time + + +def test__query_all(): + # For some reason, the significant digits here don't give the same result as above. + full_with_strpos = Heasarc.query_all("217.0 -31.7", get_query_payload=True) + # in _query_matches and query_all, whitespaces get removed. + assert "( (a.__x_ra_dec*-0.5121892283646801 + a.__y_ra_dec*-0.6790813682341418 +" + "a.__z_ra_dec*-0.5258428374185955 > (cos(radians((a.dsr*60/60)))))" \ + in full_with_strpos + full_with_strtimes = Heasarc.query_all("217.0 -31.7", + start_time="2017-01-01", + end_time="2020-01-02", get_query_payload=True) + assert "end_time > 57754.0" in full_with_strtimes and \ + "start_time < 58850.0" in full_with_strtimes diff --git a/astroquery/heasarc/tests/test_heasarc_remote.py b/astroquery/heasarc/tests/test_heasarc_remote.py index 4d765c7583..53507de50e 100644 --- a/astroquery/heasarc/tests/test_heasarc_remote.py +++ b/astroquery/heasarc/tests/test_heasarc_remote.py @@ -360,3 +360,13 @@ def test_query_region_nohits(self): assert warnings[0].category == AstropyDeprecationWarning assert warnings[1].category == NoResultsWarning assert len(catalog) == 0 + + +@pytest.mark.remote_data +def test__query_all(): + result = Heasarc.query_all("217.0 -31.70", + start_time="2017-01-01", + end_time="2020-01-02") + assert len(result) == 7 + assert result[0]['table_name'] == 'intscw' + assert result[1]['count'] == 556 diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index 5aa7a28b36..950614e2dd 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -201,6 +201,28 @@ following for instance will find master catalogs that have keywords 'nicer' or ' nicermastr NICER Master Catalog swiftmastr Swift Master Catalog +Query All Available Catalogs +---------------------------- +If you need to know which catalogs are worth querying for your source, you can +use this function that takes advantage of a fast but limited HEASARC +`trick `__ + +.. doctest-remote-data:: + >>> from astroquery.heasarc import Heasarc + >>> from astropy.coordinates import SkyCoord + >>> from astropy import units as u + >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) + >>> matches = Heasarc.query_all(pos) + >>> matches[0:5].pprint() + table_name count description regime mission obj_type + ---------- ----- ---------------------------------------------------------------- ---------------- -------- ---------------- + hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 + intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral + intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral + icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube + comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro + + Then as above, you query the table(s) that look likely individually. Adding Column Constraints ---------------------------------------- @@ -221,8 +243,9 @@ Note that when column filters are given and no position is specified, the search defaults to an all-sky search. .. doctest-remote-data:: + + - >>> from astroquery.heasarc import Heasarc >>> tab = Heasarc.query_region( ... catalog='chanmaster', column_filters={'exposure': ('>', '190000')} ... ) From 693efbd64def0cabfd864f885251dfc46ed5bdd5 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Thu, 15 Jan 2026 15:47:51 -0500 Subject: [PATCH 02/11] flake8 issue I thought I had fixed --- astroquery/heasarc/tests/test_heasarc.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index adffeb7089..f1a915856e 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -736,17 +736,19 @@ def test__get_vec(): def adql_str_comp(testing=str, reference=str): "just makes sure whitespace changes don't matter" import re - return re.sub(r'\s+', ' ', testing.replace('\n', '')).strip()\ - == re.sub(r'\s+', ' ', reference.replace('\n', '')).strip() + return re.sub(r'\s+', ' ', testing.replace('\n', ' ')).strip()\ + == re.sub(r'\s+', ' ', reference.replace('\n', ' ')).strip() def test__constraint_matches(): # Testing all together because it's easier to read this way. constraint_small = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=False) desired_small = """ - ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) - and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) + and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) and (a.dec between -32.7 and -30.7) ) """ @@ -754,7 +756,8 @@ def test__constraint_matches(): constraint_large = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=True) desired_large = """ - ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) ) """ assert adql_str_comp(constraint_large, desired_large) @@ -762,7 +765,8 @@ def test__constraint_matches(): radius=0.5*u.deg, large=True) assert "(a.dec between -31.7 - 0.5 and -31.7 + 0.5)" in constraint_large_rad - constraint_time = HeasarcClass._time_constraint(start_time=Time("2017-01-01"), end_time=Time("2017-01-02")) + constraint_time = HeasarcClass._time_constraint(start_time=Time("2017-01-01"), + end_time=Time("2017-01-02")) desired_time = "end_time > 57754.000000 AND start_time < 57755.000000" assert adql_str_comp(constraint_time, desired_time) From 3a143ab967ec0e2fe84b26dee9eb2df59b24daab Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Fri, 16 Jan 2026 10:09:37 -0500 Subject: [PATCH 03/11] Changelog and fixed test (that somehow worked for me) --- CHANGES.rst | 2 +- astroquery/heasarc/tests/test_heasarc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4c4ee5ba47..909b2d7db3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ New Tools and Services ---------------------- - +Adding method heasarc.query_all(). See PR 3499. API changes diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index f1a915856e..bf9aae39a0 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -722,7 +722,7 @@ def test_s3_mock_directory(s3_mock): assert os.path.exists(f"{tmpdir}/location/sub/sub2/file3.txt") -def test__get_vec(): +def test__get_vector(): # Test column name input assert HeasarcClass._get_vec("a.ra", "a.dec") == \ ("a.__x_ra_dec", "a.__y_ra_dec", "a.__z_ra_dec") From 9355337be4a2b3dbee22d2f1ad9f4d46ef8ecc28 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Fri, 16 Jan 2026 10:13:38 -0500 Subject: [PATCH 04/11] dash --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 909b2d7db3..9b36743520 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ New Tools and Services ---------------------- -Adding method heasarc.query_all(). See PR 3499. +- Adding method heasarc.query_all(). See PR 3499. API changes From 6a89e1b52d11a2de4bcf2660f61c8b2517c63217 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Fri, 16 Jan 2026 12:01:02 -0500 Subject: [PATCH 05/11] things I thought I'd fixed --- astroquery/heasarc/tests/test_heasarc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index bf9aae39a0..3e9789c5fc 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -724,10 +724,10 @@ def test_s3_mock_directory(s3_mock): def test__get_vector(): # Test column name input - assert HeasarcClass._get_vec("a.ra", "a.dec") == \ + assert HeasarcClass._get_vector("a.ra", "a.dec") == \ ("a.__x_ra_dec", "a.__y_ra_dec", "a.__z_ra_dec") # Test numeric input - actual = HeasarcClass._get_vec("217.0", "-31.7") + actual = HeasarcClass._get_vector("217.0", "-31.7") desired = (-0.5120309075160554, -0.6794879643287802, -0.5254716510722678) # Convert to float for comparison assert all(abs(d - a) < 0.5 * (10 ** (-6)) for d, a in zip(desired, actual)) @@ -744,10 +744,10 @@ def test__constraint_matches(): # Testing all together because it's easier to read this way. constraint_small = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=False) desired_small = """ - ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) - and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) and (a.dec between -32.7 and -30.7) ) @@ -756,7 +756,7 @@ def test__constraint_matches(): constraint_large = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=True) desired_large = """ - ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) ) """ From 9343526277c8754bf5bb3023cb473a87c3c7dc8e Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Tue, 20 Jan 2026 10:09:00 -0500 Subject: [PATCH 06/11] Undid accidental edit. Moved CHANGES entry under heasarc. Removing whitespace. --- CHANGES.rst | 3 +-- docs/heasarc/heasarc.rst | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9b36743520..c679704400 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,8 +3,6 @@ New Tools and Services ---------------------- -- Adding method heasarc.query_all(). See PR 3499. - API changes ----------- @@ -29,6 +27,7 @@ heasarc - Add ``query_constraints`` to allow querying of different catalog columns. [#3403] - Add support for uploading tables when using TAP directly through ``query_tap``. [#3403] - Add automatic guessing for the data host in ``download_data``. [#3403] +- Adding method heasarc.query_all(). [#3499] gaia ^^^^ diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index 950614e2dd..18f15f050e 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -203,9 +203,9 @@ following for instance will find master catalogs that have keywords 'nicer' or ' Query All Available Catalogs ---------------------------- -If you need to know which catalogs are worth querying for your source, you can -use this function that takes advantage of a fast but limited HEASARC -`trick `__ +If you need to know which catalogs are worth querying for your source, you can +use this function that takes advantage of a fast but limited HEASARC +`trick `. .. doctest-remote-data:: >>> from astroquery.heasarc import Heasarc @@ -214,13 +214,13 @@ use this function that takes advantage of a fast but limited HEASARC >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) >>> matches = Heasarc.query_all(pos) >>> matches[0:5].pprint() - table_name count description regime mission obj_type + table_name count description regime mission obj_type ---------- ----- ---------------------------------------------------------------- ---------------- -------- ---------------- - hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 - intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral - intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral - icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube - comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro + hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 + intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral + intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral + icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube + comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro Then as above, you query the table(s) that look likely individually. @@ -243,9 +243,7 @@ Note that when column filters are given and no position is specified, the search defaults to an all-sky search. .. doctest-remote-data:: - - - + >>> from astroquery.heasarc import Heasarc >>> tab = Heasarc.query_region( ... catalog='chanmaster', column_filters={'exposure': ('>', '190000')} ... ) From c4ad9b14c3b042cc33e536b12ac77b6ba5a0d07f Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Tue, 20 Jan 2026 15:10:23 -0500 Subject: [PATCH 07/11] Added a max_offset_deg column to results, code simpler, updated tests and docs. --- astroquery/heasarc/core.py | 73 +++++++++++++----------- astroquery/heasarc/tests/test_heasarc.py | 8 ++- docs/heasarc/heasarc.rst | 15 +++-- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index e835bb4fd3..959da93a7a 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -696,9 +696,14 @@ def _query_matches(ra=None, dec=None, start_time=None, end_time=None, radius=Non Note that this queries multiple tables, as the HEASARC database has split the master tables for efficiency. """ + + offset_def = '' if ra is not None: constraint_small = HeasarcClass._fast_geometry_constraint(ra, dec, large=False, radius=radius) constraint_big = HeasarcClass._fast_geometry_constraint(ra, dec, large=True, radius=radius) + offset_def = f""", + MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',{ra},{dec}))) as max_offset_deg + """ if start_time is not None: constraint_time = HeasarcClass._time_constraint(start_time, end_time) @@ -716,41 +721,43 @@ def _query_matches(ra=None, dec=None, start_time=None, end_time=None, radius=Non else: raise ValueError("You must specify either a position or time range or both") - if ra is not None: - full_query = f""" - select b.name as "table_name", count(*) as "count", b.description as + # Note that these operations result in incorrect escaping of the quotes around + # 'ICRS' in the SQL string. These will be removed later. + select_block = f''' + select b.name as "table_name", count(*) as "count", b.description as "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type" + as "obj_type"{offset_def} + ''' + groupby_block = " b.name , b.description , b.regime , b.mission , b.type" + + if ra is not None: + full_query = f''' + {select_block} from master_table.{tname1} as a,master_table.indexview as b where ( ( a.table_name = b.name ) ) and {constraint_small} - group by b.name , b.description , b.regime , b.mission , b.type + group by {groupby_block} union all - select b.name as "table_name", count(*) as "count", b.description as - "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type" + {select_block} from master_table.{tname2} as a,master_table.indexview as b where ( ( a.table_name = b.name ) ) and {constraint_big} - group by b.name , b.description , b.regime , b.mission , b.type + group by {groupby_block} order by count desc - """ + ''' else: - full_query = f""" - select b.name as "table_name", count(*) as "count", b.description as - "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type" + full_query = f''' + {select_block} from master_table.{tname1} as a,master_table.indexview as b where ( ( a.table_name = b.name ) ) and {constraint_time} group by b.name , b.description , b.regime , b.mission , b.type order by count desc - """ + ''' # remove all extraneous white space and line breaks - return re.sub(r'\s+', ' ', full_query.replace('\n', '')).strip() - return full_query + return re.sub(r'\s+', ' ', full_query.replace('\n', '')).strip().replace("\'", "'") def query_all(self, position=None, get_query_payload=False, start_time=None, end_time=None, verbose=False, maxrec=None, radius=None): @@ -762,22 +769,23 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, position : str, `astropy.coordinates` object The position around which to search. Must be a SkyCoord object or a string that Astropy can convert. - start_time : str, `astropy.time` object + start_time : str, `astropy.time` object, optional Beginning of time range of interest as a string in ISOT format or Time object. - end_time : str, `astropy.time` object + end_time : str, `astropy.time` object, optional End of time range of interest as a string in ISOT format or Time object. - get_query_payload : bool, optional + get_query_payload : bool, optional, optional If `True` then returns the generated ADQL query as str and does not send the query. Defaults to `False`. - radius : str or `~astropy.units.Quantity` object + radius : str or `~astropy.units.Quantity` object, optional If this radius is None, the specified coordinate is compared to each mission catalog entry using that catalog's default radius. This is based on the - approximate PSF. If you specify a radius in degrees, it uses that instead. - Be aware that for missions with large PSFs, when you search within a very small - radius, you may not find catalog entries that are within the PSF and - therefore might be of interest. + approximate location uncertainty for each mission. If you specify a radius + in degrees, it uses that instead. Be aware that for missions with large + uncertainties, when you search within a very small radius, you may not find + some relevant catalog entries that therefore might be of interest. (E.g., a query + for Geminga without a radius specified will show the entry in the HEAO2. verbose : bool, optional If True, prints additional information about the query. Default is False. maxrec : int, optional @@ -790,8 +798,8 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, result : `~astropy.table.Table` A table containing the results of the query, i.e. a list of catalogs that have entries near the specified position, how many, and quick catalog - information. If no results are found, an empty table is returned and - a warning is issued. + information included the computed offset between the two positions in degrees. + If no results are found, an empty table is returned and a warning is issued. Raises ------ @@ -810,20 +818,21 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, Examples -------- + >>> from astroquery.heasarc import Heasarc >>> from astropy.coordinates import SkyCoord >>> from astropy import units as u - >>> position = SkyCoord(ra=10.68458, dec=41.26917, unit=(u.degree, u.degree), frame='icrs') - >>> result = Heasarc.query_all(position) - >>> print(result) + >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) + >>> matches = Heasarc.query_all(pos) + >>> matches[0:5].pprint() """ if position is not None: coords_icrs = parse_coordinates(position).icrs ra, dec = coords_icrs.ra.deg, coords_icrs.dec.deg - if position is None and start_time is not None: + elif position is None and start_time is not None: ra = None dec = None - if ((position is None and start_time is None)): + elif ((position is None and start_time is None)): raise ValueError("A valid position and/or a time range must be provided.") full_query = HeasarcClass._query_matches(ra=ra, dec=dec, diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index 3e9789c5fc..9c855fbd03 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -774,7 +774,8 @@ def test__constraint_matches(): desired_full = f""" select b.name as "table_name", count(*) as "count", b.description as "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type" + as "obj_type", + MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',217.0,-31.7))) as max_offset_deg from master_table.pos_small as a,master_table.indexview as b where ( ( a.table_name = b.name ) ) and {desired_small} @@ -784,7 +785,8 @@ def test__constraint_matches(): select b.name as "table_name", count(*) as "count", b.description as "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type" + as "obj_type", + MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',217.0,-31.7))) as max_offset_deg from master_table.pos_big as a,master_table.indexview as b where ( ( a.table_name = b.name ) ) and {desired_large} @@ -805,7 +807,7 @@ def test__query_all(): # in _query_matches and query_all, whitespaces get removed. assert "( (a.__x_ra_dec*-0.5121892283646801 + a.__y_ra_dec*-0.6790813682341418 +" "a.__z_ra_dec*-0.5258428374185955 > (cos(radians((a.dsr*60/60)))))" \ - in full_with_strpos + and "DISTANCE" in full_with_strpos full_with_strtimes = Heasarc.query_all("217.0 -31.7", start_time="2017-01-01", end_time="2020-01-02", get_query_payload=True) diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index 18f15f050e..176fb8af6c 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -214,14 +214,13 @@ use this function that takes advantage of a fast but limited HEASARC >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) >>> matches = Heasarc.query_all(pos) >>> matches[0:5].pprint() - table_name count description regime mission obj_type - ---------- ----- ---------------------------------------------------------------- ---------------- -------- ---------------- - hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 - intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral - intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral - icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube - comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro - +table_name count description regime mission obj_type max_offset_deg +---------- ----- ---------------------------------------------------------------- ---------------- -------- -------- ------------------ + hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 39.99700793332599 + intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral 9.999106184550586 + intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral 9.999106184550586 +icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube 0.997001295956274 + comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro 39.325681970373786 Then as above, you query the table(s) that look likely individually. Adding Column Constraints From e8a830832335cbbbcde21b199a74a84722c8e6df Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Wed, 21 Jan 2026 09:39:12 -0500 Subject: [PATCH 08/11] Clarified usage of radius argument, added tests. --- astroquery/heasarc/core.py | 20 ++++++++++++------- astroquery/heasarc/tests/test_heasarc.py | 4 ++++ .../heasarc/tests/test_heasarc_remote.py | 8 ++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index 959da93a7a..7addf6383d 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -780,12 +780,9 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, Defaults to `False`. radius : str or `~astropy.units.Quantity` object, optional If this radius is None, the specified coordinate is compared to each mission - catalog entry using that catalog's default radius. This is based on the - approximate location uncertainty for each mission. If you specify a radius - in degrees, it uses that instead. Be aware that for missions with large - uncertainties, when you search within a very small radius, you may not find - some relevant catalog entries that therefore might be of interest. (E.g., a query - for Geminga without a radius specified will show the entry in the HEAO2. + catalog entry using that catalog's default radius. (See get_default_radius().) + This is based on the approximate location uncertainty for each mission. If you + specify a radius in degrees, it uses that instead. verbose : bool, optional If True, prints additional information about the query. Default is False. maxrec : int, optional @@ -812,7 +809,16 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, The results include the table name, number of matches, table description, regime, mission, and object type for each catalog. - The user can select the table name(s) of interest and then use the query_object(), query_region(), etc. + By default, the search radius foreach table is adjusted according to the positional + accuracy in that table. This gives you results most likely to be relevant to your + search. But if you specify a radius, that will be used in all catalogs. + Be aware that for missions with large uncertainties, e.g., 40 degrees for hete2, + when you search within a very small radius, you may not find some relevant catalog + entries that might be of interest. And conversely, if you specify a larger + radius than the default, you will get more results further from the position specified. + + The user can then select the table name(s) of interest and use the query_object(), + query_region(), etc. The query uses the HEASARC TAP service to search position-only master tables efficiently. diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index 9c855fbd03..8255ba5361 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -800,6 +800,10 @@ def test__constraint_matches(): end_time="2020-01-02") assert "end_time > 57754.000000 AND start_time < 58850.000000" in constraint_with_time + constraint_with_radius = HeasarcClass._query_matches("217.0", "-31.7", radius=1.0) + assert "-31.7 - 1.0 and -31.7 + 1.0" in constraint_with_radius + assert "cos(radians((1.0)))" in constraint_with_radius + def test__query_all(): # For some reason, the significant digits here don't give the same result as above. diff --git a/astroquery/heasarc/tests/test_heasarc_remote.py b/astroquery/heasarc/tests/test_heasarc_remote.py index 53507de50e..b65a8ab6f6 100644 --- a/astroquery/heasarc/tests/test_heasarc_remote.py +++ b/astroquery/heasarc/tests/test_heasarc_remote.py @@ -370,3 +370,11 @@ def test__query_all(): assert len(result) == 7 assert result[0]['table_name'] == 'intscw' assert result[1]['count'] == 556 + + +result = Heasarc.query_all("217.0 -31.70", radius=0.1, + start_time="2017-01-01", + end_time="2020-01-02") +assert len(result) == 6 +assert result[0]['table_name'] == 'swiftbalog' +assert result[1]['count'] == 45 From 085d68e6f48d84a9bef0d19148a3755a47caed2c Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Wed, 21 Jan 2026 12:54:57 -0500 Subject: [PATCH 09/11] Clarifying docs. Fixing tests. --- astroquery/heasarc/tests/test_heasarc_remote.py | 15 +++++++-------- docs/heasarc/heasarc.rst | 17 +++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/astroquery/heasarc/tests/test_heasarc_remote.py b/astroquery/heasarc/tests/test_heasarc_remote.py index b65a8ab6f6..0942b22c54 100644 --- a/astroquery/heasarc/tests/test_heasarc_remote.py +++ b/astroquery/heasarc/tests/test_heasarc_remote.py @@ -366,15 +366,14 @@ def test_query_region_nohits(self): def test__query_all(): result = Heasarc.query_all("217.0 -31.70", start_time="2017-01-01", - end_time="2020-01-02") + end_time="2020-01-01") assert len(result) == 7 assert result[0]['table_name'] == 'intscw' assert result[1]['count'] == 556 - -result = Heasarc.query_all("217.0 -31.70", radius=0.1, - start_time="2017-01-01", - end_time="2020-01-02") -assert len(result) == 6 -assert result[0]['table_name'] == 'swiftbalog' -assert result[1]['count'] == 45 + result = Heasarc.query_all("217.0 -31.70", radius=1, + start_time="2017-01-01", + end_time="2020-01-01") + assert len(result) == 6 + assert result[0]['table_name'] == 'swiftbalog' + assert result[1]['count'] == 45 diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index 176fb8af6c..fc468b2fb5 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -204,7 +204,7 @@ following for instance will find master catalogs that have keywords 'nicer' or ' Query All Available Catalogs ---------------------------- If you need to know which catalogs are worth querying for your source, you can -use this function that takes advantage of a fast but limited HEASARC +use the query_all() function that takes advantage of a fast but limited HEASARC `trick `. .. doctest-remote-data:: @@ -214,13 +214,14 @@ use this function that takes advantage of a fast but limited HEASARC >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) >>> matches = Heasarc.query_all(pos) >>> matches[0:5].pprint() -table_name count description regime mission obj_type max_offset_deg ----------- ----- ---------------------------------------------------------------- ---------------- -------- -------- ------------------ - hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 39.99700793332599 - intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral 9.999106184550586 - intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral 9.999106184550586 -icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube 0.997001295956274 - comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro 39.325681970373786 + table_name count description regime mission obj_type max_offset_deg + ---------- ----- ---------------------------------------------------------------- ---------------- -------- -------- ------------------ + hete2tl 6971 HETE-2 Timeline Gamma-ray, X-ray hete-2 39.99700793332599 + intscw 4088 INTEGRAL Science Window Data Gamma-ray, X-ray integral 9.999106184550586 + intscwpub 2039 INTEGRAL Public Pointed Science Window Data Gamma-ray, X-ray integral 9.999106184550586 + icecubepsc 90 IceCube All-Sky Point-Source Neutrino Events Catalog (2008-2018) icecube 0.997001295956274 + comptel 52 CGRO/COMPTEL Low-Level Data and Maps Gamma-ray cgro 39.325681970373786 + Then as above, you query the table(s) that look likely individually. Adding Column Constraints From b8e499dfa26fa4dea7a91a2cb8b32518ae844f00 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Wed, 21 Jan 2026 16:14:20 -0500 Subject: [PATCH 10/11] Removed example from docstring because cannot get it to pass tox tests. --- astroquery/heasarc/core.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index 7addf6383d..3e9ebaeabd 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -822,15 +822,6 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, The query uses the HEASARC TAP service to search position-only master tables efficiently. - Examples - -------- - >>> from astroquery.heasarc import Heasarc - >>> from astropy.coordinates import SkyCoord - >>> from astropy import units as u - >>> pos = SkyCoord('217.0 -31.7', unit=u.deg) - >>> matches = Heasarc.query_all(pos) - >>> matches[0:5].pprint() - """ if position is not None: coords_icrs = parse_coordinates(position).icrs From 23c9d319656e365ca4a1d08c5904bd9940f242b6 Mon Sep 17 00:00:00 2001 From: TR Jaffe Date: Thu, 29 Jan 2026 14:57:03 -0500 Subject: [PATCH 11/11] mostly working fixed query with contortions to avoid backend issues --- astroquery/heasarc/core.py | 101 ++++++++++++++--------- astroquery/heasarc/tests/test_heasarc.py | 96 +++++++++++---------- 2 files changed, 115 insertions(+), 82 deletions(-) diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index 3e9ebaeabd..3da1ef16d2 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -10,7 +10,6 @@ from astropy.utils.decorators import deprecated, deprecated_renamed_argument from astropy.time import Time import pyvo -import re from astroquery import log from ..query import BaseQuery, BaseVOQuery @@ -661,8 +660,8 @@ def _fast_geometry_constraint(ra, dec, large=False, radius=None): else: # Assuming 'a.dsr' is the default search radius column in degrees. This value is # defined by HEASARC curators for each table. - radius_condition = f"{dot_product} > (cos(radians((a.dsr*60/60))))" - dec_condition = f"a.dec between {dec} - a.dsr*60/60 and {dec} + a.dsr*60/60" + radius_condition = f"{dot_product} > (cos(radians((a.dsr))))" + dec_condition = f"a.dec between {dec} - a.dsr and {dec} + a.dsr" if large: return f""" ( ({radius_condition}) @@ -696,24 +695,24 @@ def _query_matches(ra=None, dec=None, start_time=None, end_time=None, radius=Non Note that this queries multiple tables, as the HEASARC database has split the master tables for efficiency. """ - offset_def = '' if ra is not None: constraint_small = HeasarcClass._fast_geometry_constraint(ra, dec, large=False, radius=radius) constraint_big = HeasarcClass._fast_geometry_constraint(ra, dec, large=True, radius=radius) - offset_def = f""", - MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',{ra},{dec}))) as max_offset_deg - """ + # Note that at least in HEASARC implementation, using DISTANCE in a column + # definition is fine, but it's very slow in a WHERE clause. + offset_def = f''', MAX(DISTANCE(POINT('ICRS', a.ra, a.dec), + POINT('ICRS', {ra}, {dec}))) as max_offset_deg + ''' + if start_time is not None: constraint_time = HeasarcClass._time_constraint(start_time, end_time) tname1, tname2 = None, None if ra is not None and start_time is None: - tname1 = 'pos_small' - tname2 = "pos_big" + tname1, tname2 = 'pos_small', 'pos_big' elif ra is not None and start_time is not None: - tname1 = 'pos_time_small' - tname2 = 'pos_time_big' + tname1, tname2 = 'pos_time_small', 'pos_time_big' constraint_small += f" AND {constraint_time}" constraint_big += f" AND {constraint_time}" elif ra is None and start_time is not None: @@ -721,43 +720,65 @@ def _query_matches(ra=None, dec=None, start_time=None, end_time=None, radius=Non else: raise ValueError("You must specify either a position or time range or both") - # Note that these operations result in incorrect escaping of the quotes around - # 'ICRS' in the SQL string. These will be removed later. + # These columns are only in b, so can remove the b.column select_block = f''' - select b.name as "table_name", count(*) as "count", b.description as - "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type"{offset_def} + SELECT table_name, count(*) AS count, b.description, + b.regime, b.mission, b.type AS obj_type{offset_def} ''' - groupby_block = " b.name , b.description , b.regime , b.mission , b.type" + + groupby_block = "GROUP BY table_name, b.description, b.regime, b.mission, b.type" if ra is not None: full_query = f''' {select_block} - from master_table.{tname1} as a,master_table.indexview as b - where ( ( a.table_name = b.name ) ) and - {constraint_small} - group by {groupby_block} - - union all - + FROM master_table.{tname1} AS a, + master_table.indexview AS b + WHERE a.table_name = b.name AND {constraint_small} + {groupby_block} + UNION ALL {select_block} - from master_table.{tname2} as a,master_table.indexview as b - where ( ( a.table_name = b.name ) ) and - {constraint_big} - group by {groupby_block} - order by count desc + FROM master_table.{tname2} AS a, + master_table.indexview AS b + WHERE a.table_name = b.name AND {constraint_big} + {groupby_block} + ORDER BY count DESC ''' else: full_query = f''' - {select_block} - from master_table.{tname1} as a,master_table.indexview as b - where ( ( a.table_name = b.name ) ) and - {constraint_time} - group by b.name , b.description , b.regime , b.mission , b.type - order by count desc + {select_block} + FROM master_table.{tname1} AS a, master_table.indexview AS b + WHERE a.table_name = b.name AND {constraint_time} + {groupby_block} + ORDER BY count DESC ''' - # remove all extraneous white space and line breaks - return re.sub(r'\s+', ' ', full_query.replace('\n', '')).strip().replace("\'", "'") + + # rename for readability + full_query = f""" + select r.table_name as table_name, r.count as count, r.description as description, + r.regime as regime, r.mission as mission, r.obj_type as obj_type, + r.max_offset_deg as max_offset_deg from ({full_query}) as r""" + + # Join all parts of the query, ensuring simple spacing + return HeasarcClass._fix_sql_whitespace(full_query) + + def _fix_sql_whitespace(insql): + import re + return re.sub(r'\s+', ' ', insql).replace("\n", " ").strip() + + def _set_print_formats(table): + """ + Set the Astropy format (e.g., '.5f' or '.3e') so that + the columns look sensible. + """ + for colname in table.columns: + col = table[colname] + if col.dtype.kind not in 'f': + continue + if (abs(col.min()) < 1e-10 and abs(col.min()) > 0.0) or abs(col.max() > 1e10): + col.format = "%10e" + else: + col.format = "%10f" + return (table) def query_all(self, position=None, get_query_payload=False, start_time=None, end_time=None, verbose=False, maxrec=None, radius=None): @@ -849,7 +870,11 @@ def query_all(self, position=None, get_query_payload=False, start_time=None, warnings.warn( NoResultsWarning("No matching rows were found in the query.") ) - return table + return table + + # Because astropy Tables don't keep all the VOTable metadata, + # this prints in more sensible formats and avoids confusion. + return HeasarcClass._set_print_formats(table) def locate_data(self, query_result=None, catalog_name=None): """Get links to data products diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index 8255ba5361..9dee1f6981 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -734,33 +734,32 @@ def test__get_vector(): def adql_str_comp(testing=str, reference=str): - "just makes sure whitespace changes don't matter" - import re - return re.sub(r'\s+', ' ', testing.replace('\n', ' ')).strip()\ - == re.sub(r'\s+', ' ', reference.replace('\n', ' ')).strip() + return HeasarcClass._fix_sql_whitespace(testing) == HeasarcClass._fix_sql_whitespace(reference) def test__constraint_matches(): # Testing all together because it's easier to read this way. constraint_small = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=False) desired_small = """ - ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 - + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) - and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) - and (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 - + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) - and (a.dec between -32.7 and -30.7) - ) - """ - assert adql_str_comp(constraint_small, desired_small) + ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr))))) and + (a.dec between -31.7 - a.dsr and -31.7 + a.dsr) and + (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > 0.9998476951563913) + and (a.dec between -32.7 and -30.7) ) + """ + + assert HeasarcClass._fix_sql_whitespace(constraint_small) == \ + HeasarcClass._fix_sql_whitespace(desired_small) constraint_large = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", large=True) desired_large = """ ( (a.__x_ra_dec*-0.5120309075160554 + a.__y_ra_dec*-0.6794879643287802 - + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr*60/60))))) - and (a.dec between -31.7 - a.dsr*60/60 and -31.7 + a.dsr*60/60) ) + + a.__z_ra_dec*-0.5254716510722678 > (cos(radians((a.dsr))))) + and (a.dec between -31.7 - a.dsr and -31.7 + a.dsr) ) """ assert adql_str_comp(constraint_large, desired_large) + constraint_large_rad = HeasarcClass._fast_geometry_constraint("217.0", "-31.7", radius=0.5*u.deg, large=True) assert "(a.dec between -31.7 - 0.5 and -31.7 + 0.5)" in constraint_large_rad @@ -770,32 +769,38 @@ def test__constraint_matches(): desired_time = "end_time > 57754.000000 AND start_time < 57755.000000" assert adql_str_comp(constraint_time, desired_time) - constraint_full = HeasarcClass._query_matches("217.0", "-31.7") - desired_full = f""" - select b.name as "table_name", count(*) as "count", b.description as - "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type", - MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',217.0,-31.7))) as max_offset_deg - from master_table.pos_small as a,master_table.indexview as b - where ( ( a.table_name = b.name ) ) and - {desired_small} - group by b.name , b.description , b.regime , b.mission , b.type - - union all - - select b.name as "table_name", count(*) as "count", b.description as - "description", b.regime as "regime", b.mission as "mission", b.type - as "obj_type", - MAX(DISTANCE(POINT('ICRS', a.ra, a.dec),POINT('ICRS',217.0,-31.7))) as max_offset_deg - from master_table.pos_big as a,master_table.indexview as b - where ( ( a.table_name = b.name ) ) and - {desired_large} - group by b.name , b.description , b.regime , b.mission , b.type - order by count desc - """ - assert adql_str_comp(constraint_full, desired_full) - - constraint_with_time = HeasarcClass._query_matches("217.0", "-31.7", + constraint_full = HeasarcClass._query_matches("217.025", "-31.725") + desired_full = """ + select r.table_name as table_name, r.count as count, + r.description as description, r.regime as regime, + r.mission as mission, r.obj_type as obj_type, r.max_offset_deg + as max_offset_deg from ( SELECT table_name, count(*) + AS count, b.description, b.regime, b.mission, b.type + AS obj_type, MAX(DISTANCE(POINT('ICRS', a.ra, a.dec), + POINT('ICRS', 217.025, -31.725))) as max_offset_deg + FROM master_table.pos_small AS a, master_table.indexview + AS b WHERE a.table_name = b.name AND + ( (a.__x_ra_dec*-0.5121892283646801 + a.__y_ra_dec*-0.6790813682341418 + + a.__z_ra_dec*-0.5258428374185955 > (cos(radians((a.dsr))))) + and (a.dec between -31.725 - a.dsr and -31.725 + a.dsr) and + (a.__x_ra_dec*-0.5121892283646801 + a.__y_ra_dec*-0.6790813682341418 + + a.__z_ra_dec*-0.5258428374185955 > 0.9998476951563913) and + (a.dec between -32.725 and -30.725) ) GROUP BY table_name, + b.description, b.regime, b.mission, b.type UNION ALL + SELECT table_name, count(*) AS count, b.description, b.regime, + b.mission, b.type AS obj_type, MAX(DISTANCE(POINT('ICRS', + a.ra, a.dec), POINT('ICRS', 217.025, -31.725))) as max_offset_deg + FROM master_table.pos_big AS a, master_table.indexview AS b + WHERE a.table_name = b.name AND ( (a.__x_ra_dec*-0.5121892283646801 + + a.__y_ra_dec*-0.6790813682341418 + a.__z_ra_dec*-0.5258428374185955 > + (cos(radians((a.dsr))))) and (a.dec between -31.725 - a.dsr and -31.725 + a.dsr) + ) GROUP BY table_name, b.description, b.regime, b.mission, b.type + ORDER BY count DESC ) as r + """ + + assert HeasarcClass._fix_sql_whitespace(constraint_full) == HeasarcClass._fix_sql_whitespace(desired_full) + + constraint_with_time = HeasarcClass._query_matches("217.025", "-31.725", start_time="2017-01-01", end_time="2020-01-02") assert "end_time > 57754.000000 AND start_time < 58850.000000" in constraint_with_time @@ -809,9 +814,12 @@ def test__query_all(): # For some reason, the significant digits here don't give the same result as above. full_with_strpos = Heasarc.query_all("217.0 -31.7", get_query_payload=True) # in _query_matches and query_all, whitespaces get removed. - assert "( (a.__x_ra_dec*-0.5121892283646801 + a.__y_ra_dec*-0.6790813682341418 +" - "a.__z_ra_dec*-0.5258428374185955 > (cos(radians((a.dsr*60/60)))))" \ - and "DISTANCE" in full_with_strpos + assert HeasarcClass._fix_sql_whitespace("""( (a.__x_ra_dec*-0.5120309075160554 + + a.__y_ra_dec*-0.6794879643287802 + + a.__z_ra_dec*-0.5254716510722678 > + (cos(radians((a.dsr))))) + """) in HeasarcClass._fix_sql_whitespace(full_with_strpos) + full_with_strtimes = Heasarc.query_all("217.0 -31.7", start_time="2017-01-01", end_time="2020-01-02", get_query_payload=True)