diff --git a/tom_base/settings.py b/tom_base/settings.py index 35bc64fe8..88d57f033 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -285,9 +285,12 @@ TOM_CADENCE_STRATEGIES = [ 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', + 'tom_observations.cadences.retry_failed_observations.RetryUntilDeadlineStrategy', 'tom_observations.cadences.resume_cadence_after_failure.ResumeCadenceAfterFailureStrategy' ] +OBS_WINDOW_MINIMUM = 24 + # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" # See https://tomtoolkit.github.io/docs/target_fields for documentation on this feature # For example: diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index cfc663d8a..9b8536c44 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -1,6 +1,8 @@ -from datetime import datetime, timedelta +from datetime import timedelta from dateutil.parser import parse +from django.conf import settings import logging +from django.utils import timezone from tom_observations.cadence import BaseCadenceForm, CadenceStrategy from tom_observations.models import ObservationRecord @@ -40,25 +42,53 @@ def update_observation_payload(self, observation_payload): def run(self): # gets the most recent observation because the next observation is just going to modify these parameters last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + if not last_obs: + return # Make a call to the facility to get the current status of the observation facility = get_service_class(last_obs.facility)() + start_keyword, end_keyword = facility.get_start_end_keywords() facility.update_observation_status(last_obs.observation_id) # Updates the DB record last_obs.refresh_from_db() # Gets the record updates - # Boilerplate to get necessary properties for future calls - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = last_obs.parameters - # Cadence logic # If the observation hasn't finished, do nothing if not last_obs.terminal: return - elif last_obs.failed: # If the observation failed + + if last_obs.status == 'CANCELED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + logger.info(f'Observation {last_obs} was canceled, stopping dynamic cadence') + return + + # Boilerplate to get necessary properties for future calls + observation_payload = last_obs.parameters.copy() + scheduled_end = last_obs.scheduled_end + if not scheduled_end: + logger.info(f'No observation end scheduled yet, falling back to end: {observation_payload[end_keyword]}') + scheduled_end = parse(observation_payload[end_keyword]) + + if isinstance(scheduled_end, str): + scheduled_end = parse(scheduled_end) + + if timezone.is_naive(scheduled_end): + scheduled_end = timezone.make_aware(scheduled_end) + + observation_payload['scheduled_end'] = scheduled_end.isoformat() + logger.info(f'Scheduled observation end: {scheduled_end}') + + if last_obs.failed: # If the observation failed # Submit next observation to be taken as soon as possible with the same window length - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) - observation_payload[start_keyword] = datetime.now().isoformat() - observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() + cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + if cadence_frequency is None: + raise Exception(f'The {self.name} strategy requires a cadence_frequency cadence_parameter.') + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(cadence_frequency, window_min) + now = timezone.now() + observation_payload[start_keyword] = now.isoformat() + observation_payload[end_keyword] = (now + timedelta(hours=window_length)).isoformat() + else: # If the observation succeeded # Advance window normally according to cadence parameters observation_payload = self.advance_window( @@ -70,10 +100,14 @@ def run(self): # Submission of the new observation to the facility obs_type = last_obs.parameters.get('observation_type') form = facility.get_form(obs_type)(data=observation_payload) + logger.info(f'obs payload: {observation_payload}') if form.is_valid(): observation_ids = facility.submit_observation(form.observation_payload()) else: - logger.error(msg=f'Unable to submit next cadenced observation: {form.errors}') + logger.error( + msg=f'Unable to submit next cadenced observation: {form.errors} ' + f'for ObservationRecord.id: {last_obs.id}' + ) raise Exception(f'Unable to submit next cadenced observation: {form.errors}') # Creation of corresponding ObservationRecord objects for the observations @@ -88,27 +122,36 @@ def run(self): ) # Add ObservationRecords to the DynamicCadence self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() new_observations.append(record) + self.dynamic_cadence.observation_group.save() # Update the status of the ObservationRecords in the DB for obsr in new_observations: - facility = get_service_class(obsr.facility)() facility.update_observation_status(obsr.observation_id) + obsr.refresh_from_db() # commit the updated observation status return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') - if not cadence_frequency: + if cadence_frequency is None: raise Exception(f'The {self.name} strategy requires a cadence_frequency cadence_parameter.') advance_window_hours = cadence_frequency - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(cadence_frequency, window_min) + + scheduled_end = observation_payload['scheduled_end'] + + if isinstance(scheduled_end, str): + scheduled_end = parse(scheduled_end) + + if timezone.is_naive(scheduled_end): + scheduled_end = timezone.make_aware(scheduled_end) - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) - if new_start < datetime.now(): # Ensure that the new window isn't in the past - new_start = datetime.now() - new_end = new_start + window_length + new_start = scheduled_end + timedelta(hours=advance_window_hours) + if new_start < timezone.now(): # Ensure that the new window isn't in the past + new_start = timezone.now() + new_end = new_start + timedelta(hours=window_length) observation_payload[start_keyword] = new_start.isoformat() observation_payload[end_keyword] = new_end.isoformat() diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index fc15217bb..93b26ba43 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -1,66 +1,209 @@ from datetime import timedelta from dateutil.parser import parse +import logging +from django.conf import settings +from django.utils import timezone from tom_observations.cadence import BaseCadenceForm, CadenceStrategy from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class +logger = logging.getLogger(__name__) + class RetryFailedObservationsForm(BaseCadenceForm): pass -class RetryFailedObservationsStrategy(CadenceStrategy): +class BaseRetryFailedObservationsStrategy(CadenceStrategy): """ - The RetryFailedObservationsStrategy immediately re-submits all observations within an observation group a certain - number of hours later, as specified by ``advance_window_hours``. + The BaseRetryFailedObservationsStrategy immediately re-submits all observations within an observation + group a certain number of hours later, as specified by ``advance_window_hours``. This strategy requires the DynamicCadence to have a parameter ``cadence_frequency``. """ name = 'Retry Failed Observations' - description = """This strategy immediately re-submits a cadenced observation without amending any other part of the - cadence.""" + description = """This strategy immediately re-submits a cadenced observation without amending any + other part of the cadence.""" form = RetryFailedObservationsForm + def retry_observation(self, first_obs, last_obs, facility): + ''' + Default retry observations, for BaseRetry strategy (retry until successful), the default is to always + retry the observations until obtained + ''' + return True + + def notify_success(self, obs): + ''' + Function to add a call to slack or email on successful single time observations + ''' + return + def run(self): - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] + records = self.dynamic_cadence.observation_group.observation_records.all().order_by('created') + first_obs = records.first() + last_obs = records.last() + + if not first_obs or not last_obs: + return + + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + + if not last_obs.terminal: + return + + if last_obs.status == 'COMPLETED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + logger.info(f'Observation {last_obs} completed; turned off dynamic cadence') + return self.notify_success(last_obs) + + if last_obs.status == 'CANCELED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + logger.info(f'Observation {last_obs} was canceled, stopping dynamic cadence') + return + + if not self.retry_observation(first_obs, last_obs, facility): + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + logger.info(f'Stopping retry cadence for observation group {self.dynamic_cadence.observation_group.id}') + return + + return self.submit_retry_observation(first_obs, last_obs, facility) + + def submit_retry_observation(self, first_obs, last_obs, facility): + observation_payload = last_obs.parameters.copy() + + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword, + first_obs=first_obs, facility=facility + ) + + if observation_payload is None: + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + logger.info( + f'No retry window remaining for observation group ' + f'{self.dynamic_cadence.observation_group.id}; deactivated silently' + ) + return + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(data=observation_payload) + + if not form.is_valid(): + logger.error( + msg=f'Unable to submit next observation: {form.errors} ' + f'for ObservationRecord.id: {last_obs.id}' + ) + raise Exception(f'Unable to submit next observation: {form.errors}') + + observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=observation_payload, + observation_id=observation_id, ) - obs_type = obs.parameters.get('observation_type', None) - form = facility.get_form(obs_type)(data=observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=obs.target, - facility=facility.name, - parameters=observation_payload, - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) + self.dynamic_cadence.observation_group.observation_records.add(record) + new_observations.append(record) + + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) + obsr.refresh_from_db() return new_observations - def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): + def advance_window(self, observation_payload, + start_keyword='start', end_keyword='end', first_obs=None, facility=None): cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') - if not cadence_frequency: - raise Exception(f'The {self.name} strategy requires a cadence_frequency cadence_parameter.') - advance_window_hours = cadence_frequency - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) - new_end = parse(observation_payload[end_keyword]) + timedelta(hours=advance_window_hours) + if cadence_frequency is None: + raise Exception( + f'The {self.name} strategy requires a cadence_frequency cadence_parameter.' + ) + + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(cadence_frequency, window_min) + + new_start = timezone.now() + new_end = new_start + timedelta(hours=window_length) + observation_payload[start_keyword] = new_start.isoformat() observation_payload[end_keyword] = new_end.isoformat() + return observation_payload + +class RetryFailedObservationsStrategy(BaseRetryFailedObservationsStrategy): + """ + Retry indefinitely until the observation succeeds. + """ + pass + + +class RetryUntilDeadlineStrategy(BaseRetryFailedObservationsStrategy): + """ + Retry in short windows until either the observation succeeds, or the + original cadence_frequency interval has elapsed. + """ + + def retry_observation(self, first_obs, last_obs, facility): + deadline = self.get_deadline(first_obs, facility) + return timezone.now() < deadline + + def advance_window(self, observation_payload, + start_keyword='start', end_keyword='end', first_obs=None, facility=None): + cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + if cadence_frequency is None: + raise Exception( + f'The {self.name} strategy requires a cadence_frequency cadence_parameter.' + ) + + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(cadence_frequency, window_min) + + deadline = self.get_deadline(first_obs, facility) + new_start = timezone.now() + + if new_start >= deadline: + return None + + new_end = min(new_start + timedelta(hours=window_length), deadline) + + if new_end <= new_start: + return None + + observation_payload[start_keyword] = new_start.isoformat() + observation_payload[end_keyword] = new_end.isoformat() return observation_payload + + def get_deadline(self, first_obs, facility): + cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + if cadence_frequency is None: + raise Exception( + f'The {self.name} strategy requires a cadence_frequency cadence_parameter.' + ) + + start_keyword, _ = facility.get_start_end_keywords() + start_value = first_obs.parameters.get(start_keyword) + + if not start_value: + raise Exception( + f'Could not determine original start time for ' + f'ObservationRecord.id={first_obs.id}' + ) + + original_start = parse(start_value) if isinstance(start_value, str) else start_value + if timezone.is_naive(original_start): + original_start = timezone.make_aware(original_start) + + return original_start + timedelta(hours=cadence_frequency) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 9e96b4972..0daf6abfc 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -789,7 +789,7 @@ class LCOPhotometricSequenceForm(LCOOldStyleObservationForm): The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the configuration of multiple filters, as well as a more intuitive proactive cadence form. """ - valid_instruments = ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG'] + valid_instruments = ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG', '0M4-SCICAM-QHY600'] valid_filters = ['U', 'B', 'V', 'R', 'I', 'up', 'gp', 'rp', 'ip', 'zs', 'w', 'unknown'] cadence_frequency = forms.IntegerField(required=True, help_text='in hours') @@ -808,7 +808,11 @@ def __init__(self, *args, **kwargs): # Massage cadence form to be SNEx-styled self.fields['cadence_strategy'] = forms.ChoiceField( - choices=[('', 'Once in the next'), ('ResumeCadenceAfterFailureStrategy', 'Repeating every')], + choices=[ + ('ResumeCadenceAfterFailureStrategy', 'Repeating every'), + ('RetryUntilDeadlineStrategy', 'Once in the next'), + ('RetryFailedObservationsStrategy', 'Retry until successful'), + ], required=False, ) for field_name in ['exposure_time', 'exposure_count', 'filter']: @@ -816,8 +820,7 @@ def __init__(self, *args, **kwargs): if self.fields.get('groups'): self.fields['groups'].label = 'Data granted to' for field_name in ['start', 'end']: - self.fields[field_name].widget = forms.HiddenInput() - self.fields[field_name].required = False + self.fields[field_name] = forms.CharField(required=False, widget=forms.HiddenInput()) self.helper.layout = Layout( Row( @@ -882,8 +885,13 @@ def clean(self): - Adds an end time that corresponds with the cadence frequency """ cleaned_data = super().clean() + logger.info(f'cleaned data: {cleaned_data}') start = cleaned_data.get('start') - cleaned_data['end'] = datetime.strftime(parse(start) + timedelta(hours=cleaned_data['cadence_frequency']), + cadence_frequency = cleaned_data['cadence_frequency'] + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(window_min, cadence_frequency) + + cleaned_data['end'] = datetime.strftime(parse(start) + timedelta(hours=window_length), '%Y-%m-%dT%H:%M:%S') return cleaned_data @@ -966,7 +974,11 @@ def __init__(self, *args, **kwargs): self.fields['name'].widget.attrs['placeholder'] = 'Name' self.fields['min_lunar_distance'].widget.attrs['placeholder'] = 'Degrees' self.fields['cadence_strategy'] = forms.ChoiceField( - choices=[('', 'Once in the next'), ('ResumeCadenceAfterFailureStrategy', 'Repeating every')], + choices=[ + ('ResumeCadenceAfterFailureStrategy', 'Repeating every'), + ('RetryUntilDeadlineStrategy', 'Once in the next'), + ('RetryFailedObservationsStrategy', 'Retry until successful'), + ], required=False, label='' ) @@ -978,8 +990,7 @@ def __init__(self, *args, **kwargs): if self.fields.get('groups'): self.fields['groups'].label = 'Data granted to' for field_name in ['start', 'end']: - self.fields[field_name].widget = forms.HiddenInput() - self.fields[field_name].required = False + self.fields[field_name] = forms.CharField(required=False, widget=forms.HiddenInput()) self.helper.layout = Layout( Div( @@ -1058,8 +1069,13 @@ def clean(self): """ cleaned_data = super().clean() cleaned_data['instrument_type'] = '2M0-FLOYDS-SCICAM' # SNEx only submits spectra to FLOYDS + start = cleaned_data.get('start') - cleaned_data['end'] = datetime.strftime(parse(start) + timedelta(hours=cleaned_data['cadence_frequency']), + cadence_frequency = cleaned_data['cadence_frequency'] + window_min = getattr(settings, 'OBS_WINDOW_MINIMUM', 24) + window_length = min(cadence_frequency, window_min) + + cleaned_data['end'] = datetime.strftime(parse(start) + timedelta(hours=window_length), '%Y-%m-%dT%H:%M:%S') return cleaned_data diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 73adbbec8..2ce50529e 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -36,6 +36,10 @@ def handle(self, *args, **kwargs): continue if not new_observations: logger.log(msg=f'No changes from dynamic cadence {cg}', level=logging.INFO) + elif new_observations == 'COMPLETED': + logger.log(msg=f'''Single observation obtained for {cg}, + no new observation submitted.''', + level=logging.INFO) else: logger.log(msg=f'''Cadence update completed for dynamic cadence {cg}, {len(new_observations)} new observations created.''', diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 0b8dc115a..6384f15fc 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -1,4 +1,5 @@ from django.test import TestCase +from django.utils import timezone from unittest.mock import patch from datetime import datetime, timedelta from dateutil.parser import parse @@ -6,7 +7,9 @@ from .factories import ObservingRecordFactory, SiderealTargetFactory from tom_observations.models import ObservationGroup, DynamicCadence from tom_observations.cadences.resume_cadence_after_failure import ResumeCadenceAfterFailureStrategy -from tom_observations.cadences.retry_failed_observations import RetryFailedObservationsStrategy +from tom_observations.cadences.retry_failed_observations import ( + RetryFailedObservationsStrategy, RetryUntilDeadlineStrategy +) mock_instruments = { @@ -56,29 +59,60 @@ def setUp(self): self.group.observation_records.add(*observing_records) self.group.save() self.dynamic_cadence = DynamicCadence.objects.create( - cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, - observation_group=self.group) + cadence_strategy='RetryFailedObservationsStrategy', cadence_parameters={'cadence_frequency': 72}, + active=True, observation_group=self.group) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'WINDOW_EXPIRED', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence_failed_obs(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} num_records = self.group.observation_records.count() - observing_record = self.group.observation_records.first() - observing_record.status = 'CANCELED' - observing_record.save() strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) new_records = strategy.run() self.group.refresh_from_db() - # Make sure the candence run created a new observation. + self.dynamic_cadence.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) - # assert that the newly added observation record has a window of exactly 3 days - # later than the canceled observation. - self.assertEqual( - parse(observing_record.parameters['start']), - parse(new_records[0].parameters['start']) - timedelta(days=3) + self.assertAlmostEqual( + parse(new_records[0].parameters['start']), + timezone.now(), + delta=timedelta(seconds=5) ) + self.assertTrue(self.dynamic_cadence.active) - @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', - 'scheduled_start': None, 'scheduled_end': None}) + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'COMPLETED', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence_successful_obs(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertFalse(self.dynamic_cadence.active) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'CANCELED', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_canceled_deactivates(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertFalse(self.dynamic_cadence.active) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'WINDOW_EXPIRED', 'scheduled_start': None, 'scheduled_end': None}) def test_resume_when_failed_cadence_failed_obs(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, mock_proposal_choices, mock_get_insts): mock_validate_obs.return_value = {} @@ -87,12 +121,14 @@ def test_resume_when_failed_cadence_failed_obs(self, mock_get_obs_status, mock_v strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) new_records = strategy.run() self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) self.assertAlmostEqual( - parse(new_records[0].parameters["start"]), - datetime.now(), + parse(new_records[0].parameters['start']), + timezone.now(), delta=timedelta(seconds=5), ) + self.assertTrue(self.dynamic_cadence.active) @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'COMPLETED', 'scheduled_start': None, 'scheduled_end': None}) @@ -107,8 +143,9 @@ def test_resume_when_failed_cadence_successful_obs(self, mock_get_obs_status, mo self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) self.assertAlmostEqual( - parse(obsr.parameters['start']).replace(second=0, microsecond=0), - parse(new_records[0].parameters['start']).replace(second=0, microsecond=0) - timedelta(days=3) + parse(new_records[0].parameters['start']), + timezone.make_aware(parse(obsr.parameters['end'])) + timedelta(hours=72), + delta=timedelta(seconds=5) ) @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'COMPLETED', @@ -127,10 +164,26 @@ def test_resume_when_failed_cadence_invalid_date(self, mock_get_obs_status, mock self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) self.assertAlmostEqual( - datetime.now().replace(second=0, microsecond=0), - parse(new_records[0].parameters['start']).replace(second=0, microsecond=0) + timezone.now(), + parse(new_records[0].parameters['start']), + delta=timedelta(seconds=5) ) + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'CANCELED', 'scheduled_start': None, 'scheduled_end': None}) + def test_resume_when_canceled_deactivates(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertFalse(self.dynamic_cadence.active) + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'COMPLETED', 'scheduled_start': None, 'scheduled_end': None}) def test_resume_when_failed_cadence_obs_invalid(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, @@ -140,3 +193,73 @@ def test_resume_when_failed_cadence_obs_invalid(self, mock_get_obs_status, mock_ strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) with self.assertRaises(Exception): strategy.run() + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'PENDING', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_not_terminal_noop(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertTrue(self.dynamic_cadence.active) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'PENDING', 'scheduled_start': None, 'scheduled_end': None}) + def test_resume_when_not_terminal_noop(self, mock_get_obs_status, mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertTrue(self.dynamic_cadence.active) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'WINDOW_EXPIRED', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_until_deadline_before_deadline_retries(self, mock_get_obs_status, + mock_validate_obs, mock_submit_obs, + mock_proposal_choices, mock_get_insts): + mock_validate_obs.return_value = {} + num_records = self.group.observation_records.count() + + strategy = RetryUntilDeadlineStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertEqual(num_records + 1, self.group.observation_records.count()) + self.assertAlmostEqual( + parse(new_records[0].parameters['start']), + timezone.now(), + delta=timedelta(seconds=5) + ) + self.assertTrue(self.dynamic_cadence.active) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', + return_value={'state': 'WINDOW_EXPIRED', 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_until_deadline_after_deadline_deactivates(self, mock_get_obs_status, mock_validate_obs, + mock_submit_obs, mock_proposal_choices, + mock_get_insts): + mock_validate_obs.return_value = {} + first_obs = self.group.observation_records.order_by('created').first() + first_obs.parameters = {**first_obs.parameters, + 'start': (datetime.now() - timedelta(hours=100)).strftime('%Y-%m-%dT%H:%M:%S')} + first_obs.save() + num_records = self.group.observation_records.count() + + strategy = RetryUntilDeadlineStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + self.dynamic_cadence.refresh_from_db() + self.assertIsNone(new_records) + self.assertEqual(num_records, self.group.observation_records.count()) + self.assertFalse(self.dynamic_cadence.active)