Merge branch 'Derek-Furst/retracted-status' of https://github.com/hubmapconsortium/entity-api into dev-integrate

Author: yuanzhou

entity-api-spec.yaml

@@ -645,7 +645,8 @@ components:
             - Submitted
             - Incomplete
             - Approval
-          description: 'One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval'
+            - Retracted
+          description: 'One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval|Retracted'
         title:
           type: string
           description: 'The dataset title.'
@@ -792,9 +793,6 @@ components:
           type: string
           format: uuid
           description: 'The thumbnail image file previously uploaded to delete. Provide as a string of the file_uuid like: "c35002f9c3d49f8b77e1e2cd4a01803d"'
-        sub_status:
-          type: string
-          description: 'A sub-status provided to further define the status. The only current allowable value is "Retracted"'
         retraction_reason:
           type: string
           description: 'Information recorded about why a the dataset was retracted.'
@@ -1101,7 +1099,8 @@ components:
             - Submitted
             - Incomplete
             - Approval
-          description: 'One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval'
+            - Retracted
+          description: 'One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval|Retracted'
         title:
           type: string
           description: 'The Publication title.'

src/app.py

@@ -542,7 +542,7 @@ def get_sorted_revisions(neo4j_driver, uuid):
 
 def get_sorted_multi_revisions(neo4j_driver, uuid, fetch_all=True, property_key=False):
     results = []
-    match_case = '' if fetch_all is True else 'AND prev.status = "Published" AND next.status = "Published" '
+    match_case = '' if fetch_all is True else 'AND prev.status IN ["Published", "Retracted"] AND next.status IN ["Published", "Retracted"] '
     collect_prop = f".{property_key}" if property_key else ''
 
     query = (
@@ -1095,7 +1095,7 @@ def get_all_dataset_samples(neo4j_driver, dataset_uuid):
 def get_sankey_info(neo4j_driver, public_only):
     public_only_query = " "
     if public_only:
-        public_only_query = f"AND toLower(ds.status) = 'published' "
+        public_only_query = f"AND toLower(ds.status) IN ['published', 'retracted'] "
     query = (f"MATCH (donor:Donor)-[:ACTIVITY_INPUT]->(organ_activity:Activity)-[:ACTIVITY_OUTPUT]-> "
             f"(organ:Sample {{sample_category:'organ'}})-[*]->(a:Activity)-[:ACTIVITY_OUTPUT]->(ds:Dataset) "
             f"WHERE toLower(a.creation_action) = 'create dataset activity' "
@@ -1138,7 +1138,7 @@ def get_sankey_info(neo4j_driver, public_only):
 def get_unpublished(neo4j_driver):
     query = (
         "MATCH (ds:Dataset)<-[*]-(d:Donor) "
-        "WHERE ds.status <> 'Published' and ds.status <> 'Hold' "
+        "WHERE NOT ds.status IN ['Published', 'Hold', 'Retracted'] "
         # specimen_type -> sample_category 12/15/2022
         "OPTIONAL MATCH (ds)<-[*]-(s:Sample {sample_category:'organ'}) "
         "RETURN distinct ds.data_types as data_types, ds.group_name as organization, ds.uuid as uuid, "

src/app_neo4j_queries.py

@@ -408,7 +408,7 @@ ENTITIES:
           - validate_dataset_not_component
         generated: true
         indexed: true
-        description: "One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval"
+        description: "One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete|Approval|Retracted"
         before_create_trigger: set_dataset_status_new
         after_create_trigger: set_status_history
         after_update_trigger: update_status
@@ -640,18 +640,7 @@ ENTITIES:
       retraction_reason:
         type: string
         indexed: true
-        before_property_update_validators:
-          - validate_if_retraction_permitted
-          - validate_sub_status_provided
         description: 'Information recorded about why a the dataset was retracted.'
-      sub_status:
-        type: string
-        indexed: true
-        before_property_update_validators:
-          - validate_if_retraction_permitted
-          - validate_retraction_reason_provided
-          - validate_retracted_dataset_sub_status_value
-        description: 'A sub-status provided to further define the status. The only current allowable value is "Retracted"'
       provider_info:
         type: string
         indexed: true

src/schema/provenance_schema.yaml

@@ -1470,12 +1470,12 @@ def normalize_entity_type(entity_type):
 Parameters
 ----------
 status : str
-    One of the status types: New|Processing|QA|Published|Error|Hold|Invalid|Approval
+    One of the status types: New|Processing|QA|Published|Error|Hold|Invalid|Approval|Retracted
     
 Returns
 -------
 string
-    One of the normalized status types: New|Processing|QA|Published|Error|Hold|Invalid|Approval
+    One of the normalized status types: New|Processing|QA|Published|Error|Hold|Invalid|Approval|Retracted
 """
 def normalize_status(status):
     if status.lower() == "qa":

src/schema/schema_manager.py

@@ -1692,7 +1692,7 @@ def get_component_dataset_uuids(neo4j_driver, uuid):
 def count_attached_published_datasets(neo4j_driver, entity_type, uuid):
     query = (f"MATCH (e:{entity_type})-[:ACTIVITY_INPUT|ACTIVITY_OUTPUT*]->(d:Dataset) "
              # Use the string function toLower() to avoid case-sensetivity issue
-             f"WHERE e.uuid='{uuid}' AND toLower(d.status) = 'published' "
+             f"WHERE e.uuid='{uuid}' AND toLower(d.status) IN ['published', 'retracted'] "
              # COLLECT() returns a list
              # apoc.coll.toSet() reruns a set containing unique nodes
              f"RETURN COUNT(d) AS {record_field_name}")

src/schema/schema_neo4j_queries.py

@@ -1766,7 +1766,7 @@ def update_status(property_key, normalized_type, request_args, user_token, exist
     set_status_history(property_key, normalized_type, request_args, user_token, existing_data_dict, new_data_dict)
 
     # Only apply to non-published parent datasets
-    if status.lower() != 'published':
+    if status.lower() not in  ['published', 'retracted']:
         # Only sync the child component datasets status for Multi-Assay Split
         component_dataset_uuids = schema_neo4j_queries.get_component_dataset_uuids(schema_manager.get_neo4j_driver_instance(), uuid)
 

src/schema/schema_triggers.py

@@ -298,8 +298,9 @@ def halt_DOI_if_unpublished_dataset(property_key, normalized_entity_type, reques
         # simply get the existing, distinct 'data_access_level' setting for all the Datasets in the Collection
         distinct_dataset_statuses = schema_neo4j_queries.get_collection_datasets_statuses(neo4j_driver_instance
                                                                                           ,existing_data_dict['uuid'])
-    if len( distinct_dataset_statuses) != 1 or \
-            distinct_dataset_statuses[0].lower() != SchemaConstants.DATASET_STATUS_PUBLISHED:
+    PUBLIC_STATUSES = {SchemaConstants.DATASET_STATUS_PUBLISHED, 'retracted'}
+
+    if not all(status.lower() in PUBLIC_STATUSES for status in distinct_dataset_statuses):
         raise ValueError(f"Unable to modify existing {existing_data_dict['entity_type']}"
                          f" {existing_data_dict['uuid']} for DOI since it contains unpublished Datasets.")
 
@@ -416,6 +417,7 @@ def validate_dataset_status_value(property_key, normalized_entity_type, request,
     accepted_status_values = [
         'new', 'processing', 'published', 'qa', 'error', 'hold', 'invalid', 'submitted', 'incomplete', 'approval'
     ]
+    # Retracted intentionally omitted. Status will be set to retracted only on a manual basis
     new_status = new_data_dict[property_key].lower()
 
     if new_status not in accepted_status_values:
@@ -426,7 +428,7 @@ def validate_dataset_status_value(property_key, normalized_entity_type, request,
 
     # If status == 'Published' already in Neo4j, then fail for any changes at all
     # Because once published, the dataset should be read-only
-    if existing_data_dict['status'].lower() == SchemaConstants.DATASET_STATUS_PUBLISHED:
+    if existing_data_dict['status'].lower() in [SchemaConstants.DATASET_STATUS_PUBLISHED, 'retracted']:
         raise ValueError(f"The status of this {normalized_entity_type} is already 'Published', status change is not allowed")
 
     # HTTP header names are case-insensitive
@@ -480,6 +482,7 @@ def validate_status_changed(property_key, normalized_entity_type, request, exist
     The json data in request body, already after the regular validations
 """
 def validate_if_retraction_permitted(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
+    # This validator is currently unused. Keeping it in case we decide we want an api endpoint for retraction at some point. 
     if 'status' not in existing_data_dict:
         raise KeyError("Missing 'status' key in 'existing_data_dict' during calling 'validate_if_retraction_permitted()' validator method.")
 
@@ -506,27 +509,6 @@ def validate_if_retraction_permitted(property_key, normalized_entity_type, reque
         raise ValueError("Permission denied, retraction is not allowed")
 
 
-"""
-Validate the sub_status field is also provided when Dataset.retraction_reason is provided on update via PUT
-
-Parameters
-----------
-property_key : str
-    The target property key
-normalized_type : str
-    Submission
-request: Flask request object
-    The instance of Flask request passed in from application request
-existing_data_dict : dict
-    A dictionary that contains all existing entity properties
-new_data_dict : dict
-    The json data in request body, already after the regular validations
-"""
-def validate_sub_status_provided(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
-    if 'sub_status' not in new_data_dict:
-        raise ValueError("Missing sub_status field when retraction_reason is provided")
-
-
 """
 Validate the reaction_reason field is also provided when Dataset.sub_status is provided on update via PUT
 
@@ -548,31 +530,6 @@ def validate_retraction_reason_provided(property_key, normalized_entity_type, re
         raise ValueError("Missing retraction_reason field when sub_status is provided")
 
 
-"""
-Validate the provided value of Dataset.sub_status on update via PUT
-
-Parameters
-----------
-property_key : str
-    The target property key
-normalized_type : str
-    Submission
-request: Flask request object
-    The instance of Flask request passed in from application request
-existing_data_dict : dict
-    A dictionary that contains all existing entity properties
-new_data_dict : dict
-    The json data in request body, already after the regular validations
-"""
-def validate_retracted_dataset_sub_status_value(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
-    # Use lowercase for comparison
-    accepted_sub_status_values = ['retracted']
-    sub_status = new_data_dict[property_key].lower()
-
-    if sub_status not in accepted_sub_status_values:
-        raise ValueError("Invalid sub_status value of the Dataset to be retracted")
-
-
 """
 Validate the provided value of Upload.status on update via PUT
 
@@ -1019,7 +976,7 @@ def _validate_application_header(applications_allowed, request_headers):
 def _is_entity_locked_against_update(existing_entity_dict):
     entity_type = existing_entity_dict['entity_type']
     if entity_type in ['Publication','Dataset']:
-        if 'status' in existing_entity_dict and existing_entity_dict['status'] == 'Published':
+        if 'status' in existing_entity_dict and existing_entity_dict['status'] in ['Published', 'Retracted']:
             raise schema_errors.LockedEntityUpdateException(f"Permission denied to change a published/public {entity_type}.")
     elif entity_type in ['Donor','Sample']:
         if 'data_access_level' in existing_entity_dict and existing_entity_dict['data_access_level'] == 'public':

src/schema/schema_validators.py

@@ -601,7 +601,8 @@ x-ref-components:
                 - Hold
                 - Invalid
                 - Approval
-              description: "One of: New|Processing|QA|Published|Error|Hold|Invalid|Approval"
+                - Retracted
+              description: "One of: New|Processing|QA|Published|Error|Hold|Invalid|Approval|Retracted"
         title:
           type: string
           description: "The dataset title."

src/schema_templating/example-yaml-templates/api-template-test/entity-Template.yaml

@@ -92,7 +92,8 @@ Dataset:
             - Hold
             - Invalid
             - Approval
-          description: "One of: New|Processing|QA|Published|Error|Hold|Invalid|Approval"
+            - Retracted
+          description: "One of: New|Processing|QA|Published|Error|Hold|Invalid|Approval|Retracted"
     title:
       type: string
       description: "The dataset title."