Merge pull request #1426 from TOMToolkit/1396-update-reduced-datums-for-existing-target-via-data-service

1396 update reduced datums for existing target via data service

Author: jchate6

docs/_static/dataservices_doc/advanced_form.png

@@ -181,7 +181,7 @@ Editing this file will allow you to overwrite any of the css used at the base le
 styling established by TOMToolkit themes such as "Dark Mode".
 
 The TOMToolkit has several built in variables that allow you to establish a theme for your TOM. You can see these in
-`root` dictionay in yoyr ``custom.css``. Altering these will change the appearance of large portions of the website.
+`root` dictionary in your ``custom.css``. Altering these will change the appearance of large portions of the website.
 
 As an example, let's change the background color from white (#ffffff) to an off-white (#efead6).
 Add the following in your ``custom.css`` after the comments:

docs/_static/dataservices_doc/query_form_collapsed.png

@@ -141,6 +141,11 @@ td, th {
     float: left;
 }
 
+.bottom {
+  position: absolute;
+  bottom: 20%;
+}
+
 /* Create copy button class */
 .copy-button {
     opacity: 0.75; /* Make it slightly transparent */

docs/brokers/create_dataservice.rst

@@ -74,6 +74,10 @@ def __init__(self, *args, **kwargs):
             ),
         )
 
+    def get_simple_form_partial(self):
+        """Returns a path to a simplified bare-minimum partial form that can be used to access the DataService."""
+        return 'tom_dataservices/tns/partials/tns_simple_form.html'
+
 
 class TNSDataService(DataService):
     """
@@ -98,14 +102,6 @@ class TNSDataService(DataService):
     info_url = 'https://tom-toolkit.readthedocs.io/en/latest/api/tom_alerts/brokers.html#module-tom_alerts.brokers.tns'
     query_results_table = 'tom_dataservices/tns/partials/tns_query_results_table.html'
 
-    def get_simple_form_partial(self):
-        """Returns a path to a simplified bare-minimum partial form that can be used to access the DataService."""
-        return 'tom_dataservices/tns/partials/tns_simple_form.html'
-
-    # def get_advanced_form_partial(self):
-    #     """Returns a path to a full or advanced partial form that can be used to access the DataService."""
-    #     return 'tom_dataservices/tns/partials/tns_advanced_form.html'
-
     @classmethod
     def urls(cls, **kwargs) -> dict:
         """Dictionary of URLS for the TNS API."""

docs/customization/customize_templates.rst

@@ -1,12 +1,19 @@
 from abc import ABC, abstractmethod
 import logging
-from typing import List, Tuple
+from typing import List
+from urllib.parse import urlencode
 
 from django.conf import settings
 from django.apps import apps
 from django.utils.module_loading import import_string
+from django.db import IntegrityError
+from django.utils.safestring import mark_safe
+from django.contrib import messages
+from django.urls import reverse
 
-from tom_targets.models import TargetName
+from guardian.shortcuts import assign_perm
+
+from tom_targets.models import TargetName, Target
 
 logger = logging.getLogger(__name__)
 
@@ -119,6 +126,22 @@ def build_query_parameters(self, parameters, **kwargs):
         """Builds the query parameters from the form data"""
         raise NotImplementedError(f'build_query_parameters method has not been implemented for {self.name}')
 
+    # Include this method if you wish for the TOM to be able to query data for an individual Target.
+    # def build_query_parameters_from_target(self, target, **kwargs):
+    #     """
+    #     This is a method that builds query parameters based on an existing target object that will be recognized by
+    #     `query_service()`.
+    #     This can be done by either by re-creating the form fields set by the Data Service Form and then calling
+    #     `self.build_query_parameters()` with the results, or we can reproduce a limited set of parameters uniquely for
+    #     a target query.
+
+    #     :param target: A target object to be queried
+    #     :return: query_parameters (usually a dict) that can be understood by `query_service()`
+    #     """
+    #     raise NotImplementedError('build_query_parameters_from_target method has not been implemented' +
+    #                               f'for {self.name}.'
+    #                               )
+
     def build_headers(self, *args, **kwargs):
         """Builds the headers for the query"""
         return {}
@@ -193,31 +216,33 @@ def get_success_message(self, **kwargs):
         """Returns a success message to display in the UI after making the query."""
         return "Query completed successfully."
 
-    def get_simple_form_partial(self):
-        """Returns a path to a simplified bare-minimum partial form that can be used to access the DataService."""
-        return None
-
-    def get_advanced_form_partial(self):
-        """Returns a path to a full or advanced partial form that can be used to access the DataService."""
-        return None
-
     def query_forced_photometry(self, query_parameters, **kwargs):
         """Set up and run a specialized query for a DataService’s forced photometry service."""
-        return self.query_service(query_parameters, **kwargs)
+        raise NotImplementedError(f'query_forced_photometry method has not been implemented for {self.name}')
 
     def query_photometry(self, query_parameters, **kwargs):
         """Set up and run a specialized query for a DataService’s photometry service."""
-        return self.query_service(query_parameters, **kwargs)
+        raise NotImplementedError(f'query_photometry method has not been implemented for {self.name}')
 
     def query_spectroscopy(self, query_parameters, **kwargs):
         """Set up and run a specialized query for a DataService’s spectroscopy service."""
-        return self.query_service(query_parameters, **kwargs)
+        raise NotImplementedError(f'query_spectroscopy method has not been implemented for {self.name}')
 
-    def query_reduced_data(self, query_parameters, **kwargs):
+    def query_reduced_data(self, target, **kwargs):
         """Set up and run a specialized query to retrieve Reduced Datums from a Data Service"""
-        phot_results = self.query_photometry(query_parameters, **kwargs)
-        spec_results = self.query_spectroscopy(query_parameters, **kwargs)
-        forced_phot_results = self.query_forced_photometry(query_parameters, **kwargs)
+        query_parameters = self.build_query_parameters_from_target(target)
+        try:
+            phot_results = self.query_photometry(query_parameters, **kwargs)
+        except NotImplementedError:
+            phot_results = []
+        try:
+            spec_results = self.query_spectroscopy(query_parameters, **kwargs)
+        except NotImplementedError:
+            spec_results = []
+        try:
+            forced_phot_results = self.query_forced_photometry(query_parameters, **kwargs)
+        except NotImplementedError:
+            forced_phot_results = []
         return {'photometry': phot_results,
                 'spectroscopy': spec_results,
                 'forced_photometry': forced_phot_results}
@@ -242,7 +267,7 @@ def query_targets(self, query_parameters, **kwargs) -> List[dict]:
 
         :param query_parameters: This is the output from build_query_parameters()
         :return: A list of dictionaries describing the resulting targets. Include 'reduced_datums' and/or 'aliases' as
-        keys in this dictionary to add associated data and alternate names without perfoming additional queries.
+          keys in this dictionary to add associated data and alternate names without performing additional queries.
         :rtype: List[dict]
         """
         return [{}]
@@ -271,19 +296,28 @@ def to_reduced_datums(self, target, data_results=None, **kwargs):
         of query_reduced_data() to create_reduced_datums_from_query()
         :param target: Target object to associate with the ReducedDatum
         :param data_results: Query results from the DataService storing observation data. This should be a dictionary
-            with each key being a data_type (i.e. Photometry, Spectroscopy, etc.)
+        with each key being a data_type (i.e. Photometry, Spectroscopy, etc.)
         """
         if not data_results:
             raise MissingDataException('No Reduced Data dictionary found.')
+        reduced_datum_list = []
         for key in data_results.keys():
-            self.create_reduced_datums_from_query(target, data_results[key], key, **kwargs)
-        return
+            reduced_datum_list += self.create_reduced_datums_from_query(target, data_results[key], key, **kwargs)
+        return reduced_datum_list
+
+    def create_reduced_datums_from_query(self, target, data: List, data_type=None, **kwargs) -> List:
+        """
+        Create and save new reduced_datums of the appropriate data_type from the query results
+        Be sure to use `ReducedDatum.objects.get_or_create()` when creating new objects.
 
-    def create_reduced_datums_from_query(self, target, data=None, data_type=None, **kwargs):
-        """Create and save new reduced_datums of the appropriate data_type from the query results"""
+        :param target: Target Object to be associated with the reduced data
+        :param data: List of data dictionaries of the appropriate `data_type`
+        :param data_type: An appropriate data type as listed in tom_dataproducts.models.DATA_TYPE_CHOICES
+        :return: List of Reduced datums (either retrieved or created)
+        """
         raise NotImplementedError
 
-    def to_target(self, target_result=None, **kwargs) -> Tuple[dict, dict, dict]:
+    def to_target(self, target_result=None, **kwargs):
         """
         Upper level function to create a new target from the query results
         This method is not intended to be extended. This method passes a single instance of the output
@@ -292,15 +326,38 @@ def to_target(self, target_result=None, **kwargs) -> Tuple[dict, dict, dict]:
         Intended usage: Call to_target on each element of the target_data list of dictionaries from query_target.
         (see views.py::CreateTargetFromQueryView)
         :param target_results: Dictionary containing target information.
-        :returns: Target object, dictionary of target_extras, and list of aliases
+        :returns: Target object
         """
         if not target_result:
             raise MissingDataException('No query results. Did you call query_service()?')
         else:
             target = self.create_target_from_query(target_result, **kwargs)
             extras = self.create_target_extras_from_query(target_result, **kwargs)
             aliases = self.create_aliases_from_query(target_result.get('aliases', []), **kwargs)
-            return target, extras, aliases
+
+            request = kwargs.get('request')
+            try:
+                target.save(extras=extras, names=aliases)
+                # Give the user access to the target they created
+                if request:
+                    target.give_user_access(request.user)
+                    for group in request.user.groups.all():
+                        assign_perm('tom_targets.view_target', group, target)
+                        assign_perm('tom_targets.change_target', group, target)
+                        assign_perm('tom_targets.delete_target', group, target)
+            except IntegrityError:
+                # errors.append(target.name)
+                target = Target.objects.get(name=target.name)
+                messages.warning(request,
+                                 mark_safe(
+                                    f"""The target,
+                                    <a href="{reverse('targets:detail', kwargs={'pk': target.id})}">
+                                    {target.name}</a> already exists, any new data has been ingested.
+                                    You can <a href="{reverse('targets:create') + '?' +
+                                                      urlencode(target.as_dict())}">create</a> a new target anyway.
+                                    """)
+                                 )
+            return target
 
     def create_target_from_query(self, target_result, **kwargs):
         """Create a new target from a single instance of the target results.

tom_common/static/tom_common/css/main.css

@@ -1,8 +1,13 @@
-from crispy_forms.layout import Layout
+from typing import List
+
 from django import forms
+from django.urls import reverse
 from crispy_forms.helper import FormHelper
+from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit
 
 from tom_dataservices.models import DataServiceQuery
+from tom_dataservices.dataservices import get_data_service_classes
+from tom_targets.models import Target
 
 
 class BaseQueryForm(forms.Form):
@@ -27,11 +32,23 @@ def __init__(self, *args, **kwargs):
         self.helper.form_tag = False
         self.helper.layout = self.get_layout()
 
+    def simple_fields(self) -> List[str]:
+        """Return List of fields to be included in the simple form."""
+        return []
+
     def get_layout(self):
-        exclude = ["query_save", "query_name"]
+        exclude = ["query_save", "query_name"] + self.simple_fields()
         field_keys = [f for f in self.fields.keys() if f not in exclude]
         return Layout(*field_keys)
 
+    def get_simple_form_partial(self):
+        """Returns a path to a simplified bare-minimum partial form that can be used to access the DataService."""
+        return None
+
+    def get_advanced_form_partial(self):
+        """Returns a path to a full or advanced partial form that can be used to access the DataService."""
+        return None
+
     def save(self, query_id=None):
         """
         Saves the form data as a DataServiceQuery object
@@ -48,3 +65,36 @@ def save(self, query_id=None):
         query.parameters = self.cleaned_data
         query.save()
         return query
+
+
+class UpdateDataFromDataServiceForm(forms.Form):
+    target = forms.ModelChoiceField(
+        Target.objects.all(),
+        widget=forms.HiddenInput(),
+        required=False
+    )
+    data_service = forms.ChoiceField(required=True, choices=[])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        data_service_list = get_data_service_classes()
+        data_service_choices = []
+        for name, data_service in data_service_list.items():
+            if hasattr(data_service, 'build_query_parameters_from_target'):
+                data_service_choices.append((name, name))
+        self.fields['data_service'].choices = data_service_choices
+        self.helper = FormHelper()
+        self.helper.form_action = reverse('tom_dataservices:update-data')
+        self.helper.layout = Layout(
+            'target',
+            Row(
+                Column(
+                    'data_service'
+                    ),
+                Column(
+                    ButtonHolder(
+                        Submit('Update', 'Update Reduced Data'), css_class="bottom"
+                        )
+                    ),
+                )
+        )

tom_dataservices/data_services/tns.py

@@ -0,0 +1,6 @@
+{% load bootstrap4 %}
+
+
+{% for field in simple_fields %}
+    {% bootstrap_field field %}
+{% endfor %}

tom_dataservices/dataservices.py

@@ -0,0 +1,9 @@
+{% load bootstrap4 crispy_forms_tags static %}
+
+<h4>Get Reduced Data from Data Service</h4>
+    {% csrf_token %}
+    {% if update_from_dataservice_form.fields.data_service.choices %}
+        {% crispy update_from_dataservice_form %}
+    {% else %}
+        You don't have any Data Services installed capable of retrieving data for this target.
+    {% endif %}