Merge pull request astropy#3611 from esdc-esac-esa-int/ESAC_Euclid_EUCLIDSWRQ-292_multischema

EUCLID: new parameter schema in the methods get_product_list and get_scientific_product_list

Author: bsipocz

CHANGES.rst

@@ -64,6 +64,7 @@ esa.euclid
    product_type parameters dynamically. [#3601]
 - In the method, ``get_scientific_product_list``, the ``dsr_part3`` parameter now supports the additional value
    ``latest``. [#3601]
+- The methods ``get_product_list`` and ``get_scientific_product_list`` accept the new parameter ``schema``. [#3611]
 
 
 vizier

astroquery/esa/euclid/__init__.py

@@ -71,8 +71,6 @@ class Conf(_config.ConfigNamespace):
 
     PRODUCT_TYPES = ['observation', 'mosaic']
 
-    SCHEMAS = ['sedm']
-
     VALID_DATALINK_RETRIEVAL_TYPES = ['SPECTRA_BGS', 'SPECTRA_RGS']
 
     VALID_LINKING_PARAMETERS = {'SOURCE_ID', 'SOURCEPATCH_ID'}

astroquery/esa/euclid/core.py

@@ -944,7 +944,8 @@ def get_observation_products(self, *, id=None, schema="sedm", product_type=None,
 
         return files
 
-    def __get_tile_catalogue_list(self, *, tile_index, product_type, verbose=False):
+    def __get_tile_catalogue_list(self, *, tile_index, product_type, schema="sedm", dsr_part1=None,
+                                  dsr_part2=None, dsr_part3=None, verbose=False):
         """
         Get the list of products of a given EUCLID tile_index (mosaics)
 
@@ -989,6 +990,15 @@ def __get_tile_catalogue_list(self, *, tile_index, product_type, verbose=False):
                     dpdSleModelOutput: SLE Model Output
                 #. SIR
                     DpdSirCombinedSpectra: Combined Spectra Product
+        schema : str, optional
+            release name. Default value is 'sedm'.
+        dsr_part1: str, optional, default None
+            the data set release part 1: for OTF environment, the activity code; for REG and IDR, the target environment
+        dsr_part2: str, optional, default None
+            the data set release part 2: for OTF environment, the patch id (a positive integer); for REG and IDR,
+            the activity code
+        dsr_part3: int, optional, default None
+            the data set release part 3: for OTF, REG and IDR environment, the version (an integer greater than 1)
         verbose : bool, optional, default 'False'
             flag to display information about the process
 
@@ -1005,53 +1015,69 @@ def __get_tile_catalogue_list(self, *, tile_index, product_type, verbose=False):
         query = None
 
         if product_type in conf.MOSAIC_PRODUCTS:
+            dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'mosaic_product')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
+
             query = (
                 f"SELECT DISTINCT mosaic_product.file_name, mosaic_product.mosaic_product_oid, "
                 f"mosaic_product.tile_index, mosaic_product.instrument_name, mosaic_product.filter_name, "
                 f"mosaic_product.category, mosaic_product.second_type, mosaic_product.ra, mosaic_product.dec, "
-                f"mosaic_product.release_name, "
-                f"mosaic_product.technique FROM sedm.mosaic_product WHERE mosaic_product.tile_index = '{tile_index}' "
-                f"AND "
-                f"mosaic_product.product_type = '{product_type}';")
+                f"mosaic_product.release_name, mosaic_product.technique, mosaic_product.{self.dsr_1}, "
+                f"mosaic_product.{self.dsr_2}, mosaic_product.{self.dsr_3} FROM {schema}.mosaic_product "
+                f"WHERE mosaic_product.tile_index = '{tile_index}' "
+                f"AND mosaic_product.product_type = '{product_type}' {extra_condition};")
+
+        elif product_type in conf.BASIC_DOWNLOAD_DATA_PRODUCTS:
+            dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'basic_download_data')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
-        if product_type in conf.BASIC_DOWNLOAD_DATA_PRODUCTS:
             query = (
                 f"SELECT basic_download_data.basic_download_data_oid, basic_download_data.product_type, "
                 f"basic_download_data.product_id, CAST(basic_download_data.observation_id_list as text) AS "
                 f"observation_id_list, CAST(basic_download_data.tile_index_list as text) AS tile_index_list, "
                 f"CAST(basic_download_data.patch_id_list as text) AS patch_id_list, "
-                f"CAST(basic_download_data.filter_name as text) AS filter_name, basic_download_data.release_name FROM "
-                f"sedm.basic_download_data WHERE '{tile_index}'=ANY(tile_index_list) AND product_type = '"
-                f"{product_type}' "
+                f"CAST(basic_download_data.filter_name as text) AS filter_name, basic_download_data.release_name, "
+                f"basic_download_data.{self.dsr_1}, basic_download_data.{self.dsr_2}, "
+                f"basic_download_data.{self.dsr_3} FROM {schema}.basic_download_data "
+                f"WHERE '{tile_index}'=ANY(tile_index_list) AND product_type = '{product_type}' {extra_condition} "
                 f"ORDER BY observation_id_list ASC;")
 
-        if product_type in conf.COMBINED_SPECTRA_PRODUCTS:
+        elif product_type in conf.COMBINED_SPECTRA_PRODUCTS:
+            dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'combined_spectra')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
+
             query = (
                 f"SELECT combined_spectra.combined_spectra_oid, combined_spectra.lambda_range, "
                 f"combined_spectra.tile_index, combined_spectra.stc_s, combined_spectra.product_type, "
-                f"combined_spectra.product_id, combined_spectra.observation_id_list FROM sedm.combined_spectra "
+                f"combined_spectra.product_id, combined_spectra.observation_id_list, combined_spectra.{self.dsr_1}, "
+                f"combined_spectra.{self.dsr_2}, combined_spectra.{self.dsr_3} FROM {schema}.combined_spectra "
                 f"WHERE combined_spectra.tile_index = '{tile_index}' AND combined_spectra.product_type = '"
-                f"{product_type}';")
+                f"{product_type}' {extra_condition};")
+
+        elif product_type in conf.MER_SEGMENTATION_MAP_PRODUCTS:
+            dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'mer_segmentation_map')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
-        if product_type in conf.MER_SEGMENTATION_MAP_PRODUCTS:
             query = (
                 f"SELECT mer_segmentation_map.file_name, mer_segmentation_map.segmentation_map_oid, "
                 f"mer_segmentation_map.ra, mer_segmentation_map.dec, mer_segmentation_map.stc_s, "
                 f"mer_segmentation_map.tile_index, "
                 f"CAST(mer_segmentation_map.observation_id_list as TEXT) AS observation_id_list, "
-                f"mer_segmentation_map.product_type, mer_segmentation_map.product_id FROM sedm.mer_segmentation_map "
+                f"mer_segmentation_map.product_type, mer_segmentation_map.product_id, "
+                f"mer_segmentation_map.{self.dsr_1}, mer_segmentation_map.{self.dsr_2}, "
+                f"mer_segmentation_map.{self.dsr_3} FROM {schema}.mer_segmentation_map "
                 f"WHERE mer_segmentation_map.tile_index = '{tile_index}' AND "
-                f"mer_segmentation_map.product_type = '{product_type}';")
+                f"mer_segmentation_map.product_type = '{product_type}' {extra_condition};")
 
-        if query is None:
+        else:
             raise ValueError(f"Invalid product type {product_type}.")
 
         job = super().launch_job(query=query, output_format='votable_plain', verbose=verbose,
                                  format_with_results_compressed=('votable_gzip',))
         return job.get_results()
 
-    def get_product_list(self, *, observation_id=None, tile_index=None, product_type, dsr_part1=None, dsr_part2=None,
-                         dsr_part3=None, verbose=False):
+    def get_product_list(self, *, observation_id=None, tile_index=None, product_type, schema="sedm", dsr_part1=None,
+                         dsr_part2=None, dsr_part3=None, verbose=False):
         """
         Get the list of products of a given EUCLID id searching by observation_id or tile_index.
 
@@ -1140,6 +1166,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                     DpdSirCombinedSpectra: Combined Spectra Product \
                                            - We suggest to use ADQL to retrieve data (spectra) from this dataset.
                     dpdSirScienceFrame: Science Frame Product
+        schema : str, optional
+            release name. Default value is 'sedm'.
         dsr_part1: str, optional, default None
             the data set release part 1: for OTF environment, the activity code; for REG and IDR, the target environment
         dsr_part2: str, optional, default None
@@ -1164,11 +1192,12 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
             raise ValueError(self.__ERROR_MSG_REQUESTED_OBSERVATION_ID + "; " + self.__ERROR_MSG_REQUESTED_TILE_ID)
 
         if tile_index is not None:
-            return self.__get_tile_catalogue_list(tile_index=tile_index, product_type=product_type, verbose=verbose)
+            return self.__get_tile_catalogue_list(tile_index=tile_index, product_type=product_type, schema=schema,
+                                                  verbose=verbose)
 
         query = None
         if product_type in conf.OBSERVATION_STACK_PRODUCTS:
-            table = 'sedm.observation_stack'
+            table = f'{schema}.observation_stack'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'observation_stack')
             extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
@@ -1183,8 +1212,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                      f" observation_stack.observation_id = '{observation_id}' AND observation_stack.product_type = '"
                      f"{product_type}' {extra_condition};")
 
-        if product_type in conf.BASIC_DOWNLOAD_DATA_PRODUCTS:
-            table = 'sedm.basic_download_data'
+        elif product_type in conf.BASIC_DOWNLOAD_DATA_PRODUCTS:
+            table = f'{schema}.basic_download_data'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3,
                                                                'basic_download_data')
@@ -1202,8 +1231,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"{product_type}' {extra_condition}"
                 f"ORDER BY observation_id_list ASC;")
 
-        if product_type in conf.MER_SEGMENTATION_MAP_PRODUCTS:
-            table = 'sedm.mer_segmentation_map'
+        elif product_type in conf.MER_SEGMENTATION_MAP_PRODUCTS:
+            table = f'{schema}.mer_segmentation_map'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3,
                                                                'mer_segmentation_map')
@@ -1221,8 +1250,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"like '%,{observation_id}' OR CAST(observation_id_list as TEXT) like '%,{observation_id},%' ) AND "
                 f"mer_segmentation_map.product_type = '{product_type}' {extra_condition};")
 
-        if product_type in conf.RAW_FRAME_PRODUCTS:
-            table = 'sedm.raw_frame'
+        elif product_type in conf.RAW_FRAME_PRODUCTS:
+            table = f'{schema}.raw_frame'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'raw_frame')
             extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
@@ -1242,8 +1271,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"raw_frame.observation_id = '{observation_id}' "
                 f"AND raw_frame.instrument_name = '{instrument_name}' {extra_condition};")
 
-        if product_type in conf.CALIBRATED_FRAME_PRODUCTS:
-            table = 'sedm.calibrated_frame'
+        elif product_type in conf.CALIBRATED_FRAME_PRODUCTS:
+            table = f'{schema}.calibrated_frame'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'calibrated_frame')
             extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
@@ -1257,8 +1286,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"FROM {table} WHERE calibrated_frame.observation_id = '{observation_id}' AND "
                 f"calibrated_frame.product_type = '{product_type}' {extra_condition};")
 
-        if product_type in conf.FRAME_CATALOG_PRODUCTS:
-            table = 'sedm.frame_catalog'
+        elif product_type in conf.FRAME_CATALOG_PRODUCTS:
+            table = f'{schema}.frame_catalog'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'frame_catalog')
             extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
@@ -1272,8 +1301,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"WHERE frame_catalog.observation_id = '{observation_id}' AND frame_catalog.product_type = '"
                 f"{product_type}' {extra_condition};")
 
-        if product_type in conf.COMBINED_SPECTRA_PRODUCTS:
-            table = 'sedm.combined_spectra'
+        elif product_type in conf.COMBINED_SPECTRA_PRODUCTS:
+            table = f'{schema}.combined_spectra'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'combined_spectra')
             extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
@@ -1287,8 +1316,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"OR observation_id_list like '% {observation_id} %' ) AND "
                 f"combined_spectra.product_type = '{product_type}' {extra_condition};")
 
-        if product_type in conf.SIR_SCIENCE_FRAME_PRODUCTS:
-            table = 'sedm.sir_science_frame'
+        elif product_type in conf.SIR_SCIENCE_FRAME_PRODUCTS:
+            table = f'{schema}.sir_science_frame'
 
             dsr_condition = self.__get_data_set_release_by_env(dsr_part1, dsr_part2, dsr_part3, 'sir_science_frame')
             extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
@@ -1302,7 +1331,7 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"sir_science_frame.{self.dsr_3} FROM {table} WHERE sir_science_frame.observation_id = '"
                 f"{observation_id}' AND sir_science_frame.instrument_name = '{instrument_name}' {extra_condition};")
 
-        if query is None:
+        else:
             raise ValueError(f"Invalid product type {product_type}.")
 
         job = super().launch_job(query=query, output_format='votable_plain', verbose=verbose,
@@ -1720,8 +1749,8 @@ def get_valid_le3_configuration_values(self):
         return job.get_results()
 
     def get_scientific_product_list(self, *, observation_id=None, tile_index=None, category=None, group=None,
-                                    product_type=None, dataset_release='REGREPROC1_R2', dsr_part1=None, dsr_part2=None,
-                                    dsr_part3=None, verbose=False):
+                                    product_type=None, dataset_release='REGREPROC1_R2', schema="sedm", dsr_part1=None,
+                                    dsr_part2=None, dsr_part3=None, verbose=False):
         """ Gets the LE3 products (the high-level science data products).
 
         Please note that not all combinations of ``category``, ``group``, and ``product_type`` are valid. Use the
@@ -1741,6 +1770,8 @@ def get_scientific_product_list(self, *, observation_id=None, tile_index=None, c
             Product type.
         dataset_release : str, mandatory. Default REGREPROC1_R2
             Data release from which the data should be retrieved.
+        schema : str, optional, default 'sedm'
+            the data release
         dsr_part1: str, optional, default None
             the data set release part 1: for OTF environment, the activity code; for REG and IDR, the target environment
         dsr_part2: str, optional, default None
@@ -1817,7 +1848,7 @@ def get_scientific_product_list(self, *, observation_id=None, tile_index=None, c
             quoted_products = ", ".join(f"'{p}'" for p in valid_products)
             conditions.append(f"product_type IN ({quoted_products})")
 
-        table = "sedm.level_3"
+        table = f"{schema}.level_3"
         query = f" SELECT * FROM {table} WHERE {' AND '.join(conditions)} ORDER BY observation_id_list ASC "
 
         job = super().launch_job(query=query, output_format='csv', verbose=verbose,

astroquery/esa/euclid/tests/test_euclidtap.py

@@ -705,6 +705,39 @@ def test_get_product_list():
                                        verbose=True)
         assert results is not None, "Expected a valid table"
 
+    # use parameter schema
+    for product_type in conf.SIR_SCIENCE_FRAME_PRODUCTS:
+        results = tap.get_product_list(observation_id='13', product_type=product_type, schema='dr1', dsr_part2='PV-023',
+                                       dsr_part3=1, verbose=True)
+        assert results is not None, "Expected a valid table"
+
+
+def test_not_overlaping_values_in_product_groups():
+    # Test that no product appears in multiple groups
+    product_groups = {
+        "OBSERVATION_STACK_PRODUCTS": conf.OBSERVATION_STACK_PRODUCTS,
+        "BASIC_DOWNLOAD_DATA_PRODUCTS": conf.BASIC_DOWNLOAD_DATA_PRODUCTS,
+        "RAW_FRAME_PRODUCTS": conf.RAW_FRAME_PRODUCTS,
+        "MER_SEGMENTATION_MAP_PRODUCTS": conf.MER_SEGMENTATION_MAP_PRODUCTS,
+        "CALIBRATED_FRAME_PRODUCTS": conf.CALIBRATED_FRAME_PRODUCTS,
+        "FRAME_CATALOG_PRODUCTS": conf.FRAME_CATALOG_PRODUCTS,
+        "COMBINED_SPECTRA_PRODUCTS": conf.COMBINED_SPECTRA_PRODUCTS,
+        "SIR_SCIENCE_FRAME_PRODUCTS": conf.SIR_SCIENCE_FRAME_PRODUCTS,
+        "MOSAIC_PRODUCTS": conf.MOSAIC_PRODUCTS
+    }
+
+    seen = {}
+
+    for group_name, products in product_groups.items():
+        for product in products:
+            assert product not in seen, (
+                f"{product} is present in both "
+                f"{seen[product]} and {group_name}"
+            )
+            seen[product] = group_name
+
+    assert len(seen) == 31
+
 
 def test_get_product_list_by_tile_index():
     conn_handler = DummyConnHandler()
@@ -1622,6 +1655,12 @@ def test_get_scientific_data_product_list(mock_get_valid_le3_configuration_value
                                                  dsr_part2='PV-023', dsr_part3='latest', verbose=True)
     assert results is not None, "Expected a valid table"
 
+    # use parameter schema
+    results = euclid.get_scientific_product_list(product_type='DpdCovarTwoPCFWLClPosPos2D', schema='dr1',
+                                                 dsr_part1='CALBLOCK', dsr_part2='PV-023', dsr_part3='latest',
+                                                 verbose=True)
+    assert results is not None, "Expected a valid table"
+
 
 @patch.object(EuclidClass, 'get_valid_le3_configuration_values')
 def test_get_scientific_data_product_list_exceptions(mock_get_valid_le3_configuration_values):