Merge pull request astropy#3547 from keflavich/jplspec_2026

JPLSpec is even more down

Author: bsipocz

CHANGES.rst

@@ -185,6 +185,9 @@ linelists.jplspec
 ^^^^^^^^^^^^^^^^^
 
 - New location for jplspec.  astroquery.jplspec is now deprecated in favor of astroquery.linelists.jplspec [#3455]
+- Added ``use_getmolecule`` option to ``query_lines`` to bypass the JPL query
+  service and retrieve full molecule catalogs via ``get_molecule``, and added a
+  configurable ``ftp_cat_server`` with a Wayback Machine fallback. [#3547]
 
 mpc
 ^^^

astroquery/linelists/jplspec/__init__.py

@@ -15,6 +15,12 @@ class Conf(_config.ConfigNamespace):
         'https://spec.jpl.nasa.gov/cgi-bin/catform',
         'JPL Spectral Catalog URL.')
 
+    ftp_cat_server = _config.ConfigItem(
+        ['https://spec.jpl.nasa.gov/ftp/pub/catalog/',
+         'https://web.archive.org/web/20250630185813/https://spec.jpl.nasa.gov/ftp/pub/catalog/'],
+        'JPL FTP Catalog URL'
+    )
+
     timeout = _config.ConfigItem(
         60,
         'Time limit for connecting to JPL server.')

astroquery/linelists/jplspec/core.py

@@ -27,6 +27,7 @@ class JPLSpecClass(BaseQuery):
 
     # use the Configuration Items imported from __init__.py
     URL = conf.server
+    FTP_CAT_URL = conf.ftp_cat_server
     TIMEOUT = conf.timeout
 
     def __init__(self):
@@ -142,6 +143,7 @@ def query_lines(self, min_frequency, max_frequency, *,
                     parse_name_locally=False,
                     get_query_payload=False,
                     fallback_to_getmolecule=True,
+                    use_getmolecule=True,
                     cache=True):
         """
         Query the JPLSpec service for spectral lines.
@@ -153,7 +155,20 @@ def query_lines(self, min_frequency, max_frequency, *,
         governs whether `get_molecule` will be used when no results are returned
         by the query service.  This workaround is needed while JPLSpec's query
         tool is broken.
+
+        use_getmolecule is an option to skip the query service entirely and
+        retrieve full molecule catalogs via `get_molecule`. It is needed when
+        the JPL query server is unresponsive. Frequency and strength limits
+        are not applied in this mode.
         """
+        if use_getmolecule:
+            if get_query_payload:
+                return [('Mol', tuple(self._resolve_molecules(molecule, flags=flags,
+                                                              parse_name_locally=parse_name_locally)))]
+            mols = self._resolve_molecules(molecule, flags=flags,
+                                           parse_name_locally=parse_name_locally)
+            return self._build_table_from_get_molecule(mols)
+
         response = self.query_lines_async(min_frequency=min_frequency,
                                           max_frequency=max_frequency,
                                           min_strength=min_strength,
@@ -170,6 +185,45 @@ def query_lines(self, min_frequency, max_frequency, *,
 
     query_lines.__doc__ = process_asyncs.async_to_sync_docstr(query_lines_async.__doc__)
 
+    def _resolve_molecules(self, molecule, *, flags=0, parse_name_locally=False):
+        """Return a list of molecule identifiers to feed to get_molecule."""
+        if molecule is None or molecule == 'All':
+            raise InvalidQueryError("use_getmolecule requires an explicit molecule "
+                                    "or regex; 'All' is not supported.")
+        if parse_name_locally:
+            self.lookup_ids = build_lookup()
+            mols = list(self.lookup_ids.find(molecule, flags).values())
+            if len(mols) == 0:
+                raise InvalidQueryError('No matching species found.')
+            return mols
+        if isinstance(molecule, (list, tuple)):
+            return list(molecule)
+        return [molecule]
+
+    def _build_table_from_get_molecule(self, mols):
+        """
+        Fetch full catalog tables for each molecule and combine them.
+
+        ``mols`` should be passed through ``_resolve_molecules`` before being
+        sent to this function if it is user-specified, but if it comes directly
+        from a query, it should be trusted as-is.
+        """
+        self.lookup_ids = build_lookup()
+        tbs = [self.get_molecule(mol) for mol in mols]
+        if len(tbs) > 1:
+            for tb, mol in zip(tbs, mols):
+                tb['Name'] = self.lookup_ids.find(str(mol), flags=0)
+                for key in list(tb.meta.keys()):
+                    tb.meta[f'{mol}_{key}'] = tb.meta.pop(key)
+            tb = table.vstack(tbs)
+            tb.meta['molecule_list'] = list(mols)
+            return tb
+        else:
+            tb = tbs[0]
+            tb.meta['molecule_id'] = mols[0]
+            tb.meta['molecule_name'] = self.lookup_ids.find(str(mols[0]), flags=0)
+            return tb
+
     def _parse_result(self, response, *, verbose=False, fallback_to_getmolecule=False):
         """
         Parse a response into an `~astropy.table.Table`
@@ -203,26 +257,12 @@ def _parse_result(self, response, *, verbose=False, fallback_to_getmolecule=Fals
 
         if 'Zero lines were found' in response.text:
             if fallback_to_getmolecule:
-                self.lookup_ids = build_lookup()
                 payload = parse_qs(response.request.body)
-                tbs = [self.get_molecule(mol) for mol in payload['Mol']]
-                if len(tbs) > 1:
-                    mols = []
-                    for tb, mol in zip(tbs, payload['Mol']):
-                        tb['Name'] = self.lookup_ids.find(mol, flags=0)
-                        for key in list(tb.meta.keys()):
-                            tb.meta[f'{mol}_{key}'] = tb.meta.pop(key)
-                        mols.append(mol)
-                    tb = table.vstack(tbs)
-                    tb.meta['molecule_list'] = mols
-                else:
-                    tb = tbs[0]
-                    tb.meta['molecule_id'] = payload['Mol'][0]
-                    tb.meta['molecule_name'] = self.lookup_ids.find(payload['Mol'][0], flags=0)
-
-                return tb
-            else:
-                raise EmptyResponseError(f"Response was empty; message was '{response.text}'.")
+                return self._build_table_from_get_molecule(
+                    [payload['Mol']]
+                    if isinstance(payload['Mol'], str)
+                    else payload['Mol'])
+            raise EmptyResponseError(f"Response was empty; message was '{response.text}'.")
 
         # data starts at 0 since regex was applied
         # Warning for a result with more than 1000 lines:
@@ -320,7 +360,7 @@ def get_molecule(self, molecule_id, *, cache=True):
         molecule_str = parse_molid(molecule_id)
 
         # Construct the URL to the catalog file
-        url = f'https://spec.jpl.nasa.gov/ftp/pub/catalog/c{molecule_str}.cat'
+        url = f'{self.FTP_CAT_URL}/c{molecule_str}.cat'
 
         # Request the catalog file
         response = self._request(method='GET', url=url,

astroquery/linelists/jplspec/tests/test_jplspec.py

@@ -290,6 +290,7 @@ def test_query_lines_with_fallback():
                                 max_frequency=200 * u.GHz,
                                 min_strength=-500,
                                 molecule="28001 CO",
+                                use_getmolecule=False,
                                 fallback_to_getmolecule=False)
 
     # Test with fallback enabled - should call get_molecule
@@ -320,6 +321,7 @@ def test_query_lines_with_fallback():
             max_frequency=200 * u.GHz,
             min_strength=-500,
             molecule="28001 CO",
+            use_getmolecule=False,
             fallback_to_getmolecule=True)
 
         mock_get_molecule.assert_called_once_with('28001')

astroquery/linelists/jplspec/tests/test_jplspec_remote.py

@@ -3,16 +3,20 @@
 from astropy.table import Table
 
 from astroquery.linelists.jplspec import JPLSpec
-from astroquery.exceptions import EmptyResponseError
 
 
-@pytest.mark.xfail(reason="2025 server problems", raises=EmptyResponseError)
 @pytest.mark.remote_data
 def test_remote():
+    """
+    In 2025-2026, the JPLSpec server went down and this was marked 'xfail' for a while.
+
+    As of late April 2026, it's back online again.
+    """
     tbl = JPLSpec.query_lines(min_frequency=500 * u.GHz,
                               max_frequency=1000 * u.GHz,
                               min_strength=-500,
                               molecule="18003 H2O",
+                              use_getmolecule=False,
                               fallback_to_getmolecule=False)
     assert isinstance(tbl, Table)
     assert len(tbl) == 36
@@ -36,7 +40,7 @@ def test_remote_regex_fallback():
                               max_frequency=1000 * u.GHz,
                               min_strength=-500,
                               molecule=("28001", "28002", "28003"),
-                              fallback_to_getmolecule=True)
+                              use_getmolecule=True)
     assert isinstance(tbl, Table)
     tbl = tbl[((tbl['FREQ'].quantity > 500*u.GHz) & (tbl['FREQ'].quantity < 1*u.THz))]
     assert len(tbl) == 16
@@ -54,14 +58,13 @@ def test_remote_regex_fallback():
     assert tbl['FREQ'][15] == 946175.3151
 
 
-# Starting in 2025, the JPL CGI server that did search queries broke totally.  See #3363
-@pytest.mark.xfail(reason="2025 server problems", raises=EmptyResponseError)
 @pytest.mark.remote_data
 def test_remote_regex():
     tbl = JPLSpec.query_lines(min_frequency=500 * u.GHz,
                               max_frequency=1000 * u.GHz,
                               min_strength=-500,
                               molecule=("28001", "28002", "28003"),
+                              use_getmolecule=False,
                               fallback_to_getmolecule=False)
     assert isinstance(tbl, Table)
     assert len(tbl) == 16
@@ -130,7 +133,7 @@ def test_remote_fallback():
                               max_frequency=1000 * u.GHz,
                               min_strength=-500,
                               molecule="18003 H2O",
-                              fallback_to_getmolecule=True)
+                              use_getmolecule=True)
     assert isinstance(tbl, Table)
     tbl = tbl[((tbl['FREQ'].quantity > 500*u.GHz) & (tbl['FREQ'].quantity < 1*u.THz))]
     assert len(tbl) == 36

docs/linelists/jplspec/jplspec.rst

@@ -46,8 +46,8 @@ what each setting yields:
    ...                                molecule="28001 CO",
    ...                                get_query_payload=False)
    >>> response.pprint(max_lines=10)
-       FREQ      ERR    LGINT    DR    ELO     GUP  TAG  QNFMT QN' QN"  Lab 
-       MHz       MHz   nm2 MHz        1 / cm                                
+       FREQ      ERR    LGINT    DR    ELO     GUP  TAG  QNFMT QN' QN"  Lab
+       MHz       MHz   nm2 MHz        1 / cm
    ------------ ------ -------- --- ---------- --- ----- ----- --- --- -----
     115271.2018 0.0005  -5.0105   2        0.0   3 28001   101   1   0  True
        230538.0 0.0005  -4.1197   2      3.845   5 28001   101   2   1  True
@@ -69,7 +69,7 @@ The following example, with ``get_query_payload = True``, returns the payload:
    ...                                   molecule="28001 CO",
    ...                                   get_query_payload=True)
    >>> print(response)
-   [('MinNu', 100.0), ('MaxNu', 1000.0), ('MaxLines', 2000), ('UnitNu', 'GHz'), ('StrLim', -500), ('Mol', '28001 CO')]
+   [('Mol', ('28001 CO',))]
 
 The units of the columns of the query can be displayed by calling
 ``response.info``:
@@ -82,19 +82,19 @@ The units of the columns of the query can be displayed by calling
    ...                                molecule="28001 CO")
    >>> print(response.info)
    <Table length=91>
-    name  dtype    unit 
+    name  dtype    unit
    ----- ------- -------
     FREQ float64     MHz
      ERR float64     MHz
    LGINT float64 nm2 MHz
-      DR   int64        
+      DR   int64
      ELO float64  1 / cm
-     GUP   int64        
-     TAG   int64        
-   QNFMT   int64        
-     QN'   int64        
-     QN"   int64        
-     Lab    bool        
+     GUP   int64
+     TAG   int64
+   QNFMT   int64
+     QN'   int64
+     QN"   int64
+     Lab    bool
    <BLANKLINE>
 
 These come in handy for converting to other units easily, an example using a
@@ -103,8 +103,8 @@ simplified version of the data above is shown below:
 .. doctest-remote-data::
 
    >>> response['FREQ', 'ERR', 'ELO'].pprint(max_lines=10)
-       FREQ      ERR      ELO    
-       MHz       MHz     1 / cm  
+       FREQ      ERR      ELO
+       MHz       MHz     1 / cm
    ------------ ------ ----------
     115271.2018 0.0005        0.0
        230538.0 0.0005      3.845
@@ -226,8 +226,8 @@ to retrieve data from JPLSpec via astroquery.
    >>> print(f"Retrieved {len(table)} lines for CO")
    Retrieved 91 lines for CO
    >>> table[:5].pprint()
-       FREQ     ERR    LGINT   DR   ELO   GUP  TAG  QNFMT QN' QN" Lab 
-       MHz      MHz   nm2 MHz      1 / cm                             
+       FREQ     ERR    LGINT   DR   ELO   GUP  TAG  QNFMT QN' QN" Lab
+       MHz      MHz   nm2 MHz      1 / cm
    ----------- ------ ------- --- ------- --- ----- ----- --- --- ----
    115271.2018 0.0005 -5.0105   2     0.0   3 28001   101   1   0 True
       230538.0 0.0005 -4.1197   2   3.845   5 28001   101   2   1 True
@@ -272,8 +272,8 @@ to query these directly.
    ...                              molecule="H2O",
    ...                              parse_name_locally=True)
    >>> result.pprint(max_lines=10)
-       FREQ      ERR    LGINT    DR    ELO    GUP  TAG  QNFMT QN'1 QN"1 QN'2 QN"2 QN'3 QN"3 QN'4 QN"4  Lab 
-       MHz       MHz   nm2 MHz        1 / cm                                                               
+       FREQ      ERR    LGINT    DR    ELO    GUP  TAG  QNFMT QN'1 QN"1 QN'2 QN"2 QN'3 QN"3 QN'4 QN"4  Lab
+       MHz       MHz   nm2 MHz        1 / cm
    ------------ ------ -------- --- --------- --- ----- ----- ---- ---- ---- ---- ---- ---- ---- ---- -----
       8006.5805  2.851 -18.6204   3 6219.6192  45 18003  1404   22   21    4    7   18   15    0    0 False
      12478.2535 0.2051 -13.1006   3 3623.7652  31 18003  1404   15   16    7    4    9   12    0    0 False
@@ -375,7 +375,7 @@ If you are repeatedly getting failed queries, or bad/out-of-date results, try cl
     >>> from astroquery.linelists.jplspec import JPLSpec
     >>> JPLSpec.clear_cache()
 
-If this function is unavailable, upgrade your version of astroquery. 
+If this function is unavailable, upgrade your version of astroquery.
 The ``clear_cache`` function was introduced in version 0.4.7.dev8479.