Merge pull request #48 from observatorycontrolsystem/feature/thumbnails

Feature/thumbnails

Author: mgdaily

.github/workflows/build.yaml

@@ -27,15 +27,26 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e .[tests]
-        pip install coverage "coveralls<3.3"
+        pip install coverage
     - name: Run tests
-      run: coverage run -m pytest
+      run: | 
+        coverage run -m pytest
+        coverage xml
       env:
         OPENTSDB_PYTHON_METRICS_TEST_MODE: True
-    - name: Coveralls report
-      run: coveralls --service=github
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Generate and send coveralls report
+      uses: coverallsapp/github-action@v2
+      with:
+        parallel: true
+        format: cobertura
+  finish:
+    runs-on: ubuntu-latest
+    needs: run_tests
+    steps:
+      - name: Close parallel build
+        uses: coverallsapp/github-action@v2
+        with:
+          parallel-finished: true
 
   publish_to_pypi:
     # Only run this job if the run_tests job has succeeded, and if

CHANGELOG

@@ -1,3 +1,7 @@
+3.1.0
+2024-06-03
+Upgrade ocs_archive library, add support for uploading thumbnails to an OCS archive.
+
 3.0.6
 2024-01-31
 Upgrade ocs_archive library version to support selecting S3 addressing_style

ocs_ingester/archive.py

@@ -93,3 +93,29 @@ def post_frame(self, archive_record):
             **ingester_settings.EXTRA_METRICS_TAGS
         )
         return archive_record
+
+    @metrics.method_timer('ingester.post_thumbnail')
+    def post_thumbnail(self, archive_record):
+        response = requests.post(
+            '{0}thumbnails/'.format(self.api_root), json=archive_record, headers=self.headers
+        )
+        result = self.handle_response(response)
+        logger.info('Ingester posted thumbnail to archive', extra={
+            'tags': {
+                'filename': result.get('filename'),
+                'request_id': archive_record.get('request_id'),
+                'proposal_id': result.get('proposal_id'),
+                'id': result.get('id')
+            }
+        })
+        # Add some useful information from the result
+        archive_record['thumbnail_id'] = result.get('id')
+        # Record metric for the ingest lag (time between date of image vs date ingested)
+        ingest_lag = datetime.utcnow() - obs_end_time_from_dict(archive_record)
+        self.send_metric(
+            metric_name='ingester.ingest_lag',
+            value=ingest_lag.total_seconds(),
+            asynchronous=ingester_settings.SUBMIT_METRICS_ASYNCHRONOUSLY,
+            **ingester_settings.EXTRA_METRICS_TAGS
+        )
+        return archive_record

ocs_ingester/ingester.py

@@ -26,7 +26,6 @@
     >>>    ingested_record = ingester.upload_file_and_ingest_to_archive(fileobj)
 
 """
-
 from ocs_ingester.exceptions import BackoffRetryError, NonFatalDoNotRetryError, DoNotRetryError
 from ocs_ingester.archive import ArchiveService
 from ocs_ingester.utils.metrics import upload_and_collect_metrics, get_md5_and_collect_metrics
@@ -160,7 +159,7 @@ def upload_file_to_file_store(fileobj, path=None, file_metadata=None):
 
 def ingest_archive_record(version, record, api_root=ingester_settings.API_ROOT,
                           auth_token=ingester_settings.AUTH_TOKEN):
-    """Adds a record to the science archive database.
+    """Adds a frame record to the science archive database.
 
     Args:
         version (dict): Version information returned from the upload to S3
@@ -202,7 +201,8 @@ def ingest_archive_record(version, record, api_root=ingester_settings.API_ROOT,
 def upload_file_and_ingest_to_archive(fileobj, path=None, file_metadata=None,
                                       required_headers=archive_settings.REQUIRED_HEADERS,
                                       blacklist_headers=archive_settings.HEADER_BLACKLIST,
-                                      api_root=ingester_settings.API_ROOT, auth_token=ingester_settings.AUTH_TOKEN):
+                                      api_root=ingester_settings.API_ROOT, auth_token=ingester_settings.AUTH_TOKEN,
+                                      is_thumbnail=False, thumbnail_size=None):
     """Uploads a file to S3 and adds the associated record to the science archive database.
 
     This is a standalone function that runs all of the necessary steps to add data to the
@@ -218,6 +218,8 @@ def upload_file_and_ingest_to_archive(fileobj, path=None, file_metadata=None,
         auth_token (str): Science archive API authentication token
         required_headers (tuple): FITS headers that must be present
         blacklist_headers (tuple): FITS headers that should not be ingested
+        is_thumbnail (bool): Whether the file is a thumbnail
+        thumbnail_size (str): The size of the thumbnail
 
     Returns:
         dict: Information about the uploaded file and record. For example:
@@ -250,6 +252,11 @@ def upload_file_and_ingest_to_archive(fileobj, path=None, file_metadata=None,
     try:
         if file_metadata is None:
             file_metadata = {}
+        if is_thumbnail:
+            if thumbnail_size is None:
+                raise FileSpecificationException('thumbnail_size must be provided for thumbnail files')
+            file_metadata['size'] = thumbnail_size
+            required_headers = archive_settings.REQUIRED_THUMBNAIL_METADATA
         open_file = File(fileobj, path)
         datafile = FileFactory.get_datafile_class_for_extension(open_file.extension)(
             open_file, file_metadata, required_headers=required_headers, blacklist_headers=blacklist_headers
@@ -259,7 +266,7 @@ def upload_file_and_ingest_to_archive(fileobj, path=None, file_metadata=None,
         raise DoNotRetryError(str(fe))
 
     archive = ArchiveService(api_root=api_root, auth_token=auth_token)
-    ingester = Ingester(datafile, filestore, archive)
+    ingester = Ingester(datafile, filestore, archive, is_thumbnail=is_thumbnail)
     return ingester.ingest()
 
 
@@ -269,10 +276,11 @@ class Ingester(object):
     A single instance of this class is responsible for parsing a fits file,
     uploading the data to s3, and making a call to the archive api.
     """
-    def __init__(self, datafile, filestore, archive):
+    def __init__(self, datafile, filestore, archive, is_thumbnail=False):
         self.datafile = datafile
         self.filestore = filestore
         self.archive = archive
+        self.is_thumbnail = is_thumbnail
 
     def ingest(self):
         # Get the Md5 checksum of this file and check if it already exists in the archive
@@ -293,4 +301,5 @@ def ingest(self):
         record['area'] = self.datafile.get_wcs_corners()
         record['version_set'] = [version]
         record['basename'] = self.datafile.open_file.basename
-        return self.archive.post_frame(record)
+
+        return self.archive.post_thumbnail(record) if self.is_thumbnail else self.archive.post_frame(record)

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name='ocs-ingester',
-    version='3.0.6',
+    version='3.1.0',
     description='Ingest frames into the science archive of an observatory control system',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -22,7 +22,7 @@
         'astropy',
         'requests',
         'boto3',
-        'ocs_archive==0.3.0',
+        'ocs_archive==0.4.0',
         'python-dateutil',
         'lcogt-logging',
         'opentsdb-python-metrics>=0.2.0'

tests/__init__.py

@@ -57,6 +57,12 @@ def test_archive_post(self, post_mock, get_mock):
         self.assertTrue(post_mock.called)
         self.assertEqual(post_mock.call_args[0][0], 'http://fake/frames/')
 
+    def test_archive_post_thumbnail(self, post_mock, get_mock):
+        archive_service = ArchiveService(api_root='http://fake/', auth_token='')
+        archive_service.post_thumbnail({'observation_date': datetime.utcnow().isoformat()})
+        self.assertTrue(post_mock.called)
+        self.assertEqual(post_mock.call_args[0][0], 'http://fake/thumbnails/')
+
     def test_existing_md5(self, post_mock, get_mock):
         archive_service = ArchiveService(api_root='http://return1/', auth_token='')
         self.assertTrue(archive_service.version_exists(''))

tests/test_archive.py

@@ -15,7 +15,6 @@
 from ocs_ingester.ingester import (Ingester, upload_file_and_ingest_to_archive, ingest_archive_record,
                                    upload_file_to_file_store, validate_fits_and_create_archive_record)
 from ocs_ingester.exceptions import DoNotRetryError, NonFatalDoNotRetryError
-from ocs_ingester.settings import settings
 
 opentsdb_python_metrics.metric_wrappers.test_mode = True
 
@@ -53,6 +52,11 @@
     'cptnrs03-fa13-20150219-0001-e92-summary.pdf'
 )
 
+JPG_FILE = os.path.join(
+    OTHER_PATH,
+    'tfn0m419-sq32-20240426-0097-e91-small.jpg'
+)
+
 
 def mock_hashlib_md5(*args, **kwargs):
     class MockHash(object):
@@ -70,10 +74,10 @@ def hexdigest(self):
 filestore_mock = MagicMock()
 filestore_mock.store_file = MagicMock(return_value={'md5': 'fakemd5'})
 
-def mocked_ingester(datafile_real, fake_filestore, fake_archive):
+def mocked_ingester(datafile_real, fake_filestore, fake_archive, is_thumbnail):
     class MockIngester(Ingester):
         def __init__(self):
-            super().__init__(datafile_real, filestore_mock, archive_mock)
+            super().__init__(datafile_real, filestore_mock, archive_mock, is_thumbnail)
 
     return MockIngester()
 
@@ -108,6 +112,7 @@ def setUp(self):
         self.open_files = [File(open(os.path.join(FITS_PATH, f), 'rb')) for f in os.listdir(FITS_PATH)]
         self.data_files = [FileFactory.get_datafile_class_for_extension(open_file.extension)(open_file) for open_file in self.open_files]
         self.mock_metadata = {'PROPID': 'INGEST-TEST-2021',
+                              'DAY-OBS': '20150219',
                               'DATE-OBS': '2015-02-19T13:56:05.261',
                               'INSTRUME': 'nres03',
                               'SITEID': 'cpt',
@@ -282,3 +287,38 @@ def test_ingest_pdf_with_meta(self, ingester_mock):
             upload_file_and_ingest_to_archive(fileobj, file_metadata=self.mock_metadata)
             self.assertTrue(filestore_mock.store_file.called)
             self.assertTrue(archive_mock.post_frame.called)
+
+    @patch('ocs_ingester.ingester.Ingester', side_effect=mocked_ingester)
+    def test_ingest_jpg_no_meta(self, ingester_mock):
+        with open(JPG_FILE, 'rb') as fileobj:
+            with self.assertRaises(DoNotRetryError):
+                upload_file_and_ingest_to_archive(fileobj)
+            self.assertFalse(filestore_mock.store_file.called)
+            self.assertFalse(archive_mock.post_frame.called)
+
+    @patch('ocs_ingester.ingester.Ingester', side_effect=mocked_ingester)
+    def test_ingest_jpg_missing_keyword(self, ingester_mock):
+        bad_metadata = copy(self.mock_metadata)
+        del bad_metadata['BLKUID']
+        with open(JPG_FILE, 'rb') as fileobj:
+            with self.assertRaises(DoNotRetryError):
+                upload_file_and_ingest_to_archive(fileobj, file_metadata=bad_metadata)
+            self.assertFalse(filestore_mock.store_file.called)
+            self.assertFalse(archive_mock.post_frame.called)
+
+    @patch('ocs_ingester.ingester.Ingester', side_effect=mocked_ingester)
+    def test_ingest_jpg_with_incomplete_meta(self, ingester_mock):
+        with open(JPG_FILE, 'rb') as fileobj:
+            with self.assertRaises(DoNotRetryError):
+                upload_file_and_ingest_to_archive(fileobj, file_metadata=self.mock_metadata, is_thumbnail=True, thumbnail_size='small')
+            self.assertFalse(filestore_mock.store_file.called)
+            self.assertFalse(archive_mock.post_thumbnail.called)
+
+    @patch('ocs_ingester.ingester.Ingester', side_effect=mocked_ingester)
+    def test_ingest_jpg_with_complete_meta(self, ingester_mock):
+        test_metadata = copy(self.mock_metadata)
+        test_metadata['frame_basename'] = 'tfn0m419-sq32-20240426-0097-e91'
+        with open(JPG_FILE, 'rb') as fileobj:
+            upload_file_and_ingest_to_archive(fileobj, file_metadata=test_metadata, is_thumbnail=True, thumbnail_size='small')
+            self.assertTrue(filestore_mock.store_file.called)
+            self.assertTrue(archive_mock.post_thumbnail.called)