added new status retracted which behaves the same as published. Misc other required changes

Author: DerekFurstPitt

src/app.py

@@ -492,7 +492,7 @@ def get_ancestor_organs(id):
     public_entity = True
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             public_entity = False
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
@@ -981,7 +981,7 @@ def get_entity_provenance(id):
 
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
     else:
@@ -1262,7 +1262,7 @@ def create_entity(entity_type):
 
             # Only published datasets can have revisions made of them. Verify that that status of the Dataset specified
             # by previous_revision_uuid is published. Else, bad request error.
-            if previous_version_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+            if previous_version_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
                 bad_request_error(f"The previous_revision_uuid specified for this dataset must be 'Published' in order to create a new revision from it")
 
     # If the preceding "additional validations" did not raise an error,
@@ -1655,7 +1655,7 @@ def get_ancestors(id):
 
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             public_entity = False
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
@@ -1862,7 +1862,7 @@ def get_parents(id):
 
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             public_entity = False
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
@@ -2070,7 +2070,7 @@ def get_siblings(id):
 
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             public_entity = False
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
@@ -2207,7 +2207,7 @@ def get_tuplets(id):
 
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         # Only published/public datasets don't require token
-        if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+        if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
             public_entity = False
             # Token is required and the user must belong to HuBMAP-READ group
             token = get_user_token(request, non_public_access_required = True)
@@ -2459,7 +2459,7 @@ def get_collections(id):
     if not schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         bad_request_error(f"Unsupported entity type of id {id}: {normalized_entity_type}")
 
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         public_entity = False
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required = True)
@@ -2567,7 +2567,7 @@ def get_uploads(id):
     if not schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset'):
         bad_request_error(f"Unsupported entity type of id {id}: {normalized_entity_type}")
 
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required = True)
 
@@ -2883,7 +2883,7 @@ def get_globus_url(id):
         globus_server_uuid = app.config['GLOBUS_PROTECTED_ENDPOINT_UUID']
         access_dir = access_level_prefix_dir(app.config['PROTECTED_DATA_SUBDIR'])
         dir_path = dir_path + access_dir + group_name + "/"
-    elif (entity_data_access_level == ACCESS_LEVEL_PROTECTED) and (entity_dict['status'] == 'Published'):
+    elif (entity_data_access_level == ACCESS_LEVEL_PROTECTED) and (entity_dict['status'] in ['Published', 'Retracted']):
         globus_server_uuid = app.config['GLOBUS_PUBLIC_ENDPOINT_UUID']
         access_dir = access_level_prefix_dir(app.config['PUBLIC_DATA_SUBDIR'])
         dir_path = dir_path +  access_dir + "/"
@@ -3016,7 +3016,7 @@ def get_dataset_revision_number(id):
         bad_request_error("The entity of given id is not a Dataset or Publication")
 
     # Only published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required = True)
 
@@ -3170,7 +3170,7 @@ def get_revisions_list(id):
         bad_request_error("The entity is not a Dataset. Found entity type:" + normalized_entity_type)
 
     # Only published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required=True)
 
@@ -3200,7 +3200,7 @@ def get_revisions_list(id):
             normalized_revisions_list.pop(0)
 
             # Also hide the 'next_revision_uuid' of the second last revision from response
-            if 'next_revision_uuid' in normalized_revisions_list[0]:
+            if normalized_revisions_list and 'next_revision_uuid' in normalized_revisions_list[0]:
                 normalized_revisions_list[0].pop('next_revision_uuid')
 
     # Now all we need to do is to compose the result list
@@ -3255,7 +3255,7 @@ def get_associated_organs_from_dataset(id):
     excluded_fields = schema_manager.get_fields_to_exclude('Sample')
     public_entity = True
     # published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         public_entity = False
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required=True)
@@ -3316,7 +3316,7 @@ def get_associated_samples_from_dataset(id):
         bad_request_error("The entity of given id is not a Dataset or Publication")
     public_entity = True
     # published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         # Token is required and the user must belong to HuBMAP-READ group
         public_entity = False
         token = get_user_token(request, non_public_access_required=True)
@@ -3376,7 +3376,7 @@ def get_associated_donors_from_dataset(id):
         bad_request_error("The entity of given id is not a Dataset or Publication")
     public_entity = True
     # published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         public_entity = False
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required=True)
@@ -3451,7 +3451,7 @@ def get_prov_info_for_dataset(id):
         bad_request_error("The entity of given id is not a Dataset")
 
     # published/public datasets don't require token
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         # Token is required and the user must belong to HuBMAP-READ group
         token = get_user_token(request, non_public_access_required=True)
 
@@ -3959,14 +3959,14 @@ def paired_dataset(id):
     if normalized_entity_type != 'Dataset':
         bad_request_error("The target entity of the specified id is not a Dataset")
 
-    if entity_dict['status'].lower() != DATASET_STATUS_PUBLISHED:
+    if entity_dict['status'].lower() not in [DATASET_STATUS_PUBLISHED, 'retracted']:
         if not user_in_hubmap_read_group(request):
             forbidden_error("Access not granted")
 
     paired_dataset = app_neo4j_queries.get_paired_dataset(neo4j_driver_instance, uuid, data_type, search_depth)
     out_list = []
     for result in paired_dataset:
-        if user_in_hubmap_read_group(request) or result['status'].lower() == 'published':
+        if user_in_hubmap_read_group(request) or result['status'].lower() in ['published', 'retracted']:
             out_list.append(result['uuid'])
     if len(out_list) < 1:
         not_found_error(f"Search for paired datasets of type {data_type} for dataset with id {uuid} returned no results")
@@ -4512,7 +4512,7 @@ def _get_entity_visibility(normalized_entity_type, entity_dict):
     # it can be used along with the user's authorization to determine access.
     entity_visibility=DataVisibilityEnum.NONPUBLIC
     if schema_manager.entity_type_instanceof(normalized_entity_type, 'Dataset') and \
-       entity_dict['status'].lower() == DATASET_STATUS_PUBLISHED:
+       entity_dict['status'].lower() in [DATASET_STATUS_PUBLISHED, 'retracted']:
         entity_visibility=DataVisibilityEnum.PUBLIC
     elif schema_manager.entity_type_instanceof(normalized_entity_type, 'Collection') and \
         'registered_doi' in entity_dict and \
@@ -4524,12 +4524,10 @@ def _get_entity_visibility(normalized_entity_type, entity_dict):
             # Get the data_access_level for each Dataset in the Collection from Neo4j
             collection_dataset_statuses = schema_neo4j_queries.get_collection_datasets_statuses(neo4j_driver_instance
                                                                                                 ,entity_dict['uuid'])
+            PUBLIC_STATUSES = {SchemaConstants.DATASET_STATUS_PUBLISHED, "retracted"}
 
-            # If the list of distinct statuses for Datasets in the Collection only has one entry, and
-            # it is 'published', the Collection is public
-            if len(collection_dataset_statuses) == 1 and \
-                collection_dataset_statuses[0].lower() == SchemaConstants.DATASET_STATUS_PUBLISHED:
-                entity_visibility=DataVisibilityEnum.PUBLIC
+            if all(status.lower() in PUBLIC_STATUSES for status in collection_dataset_statuses):
+                entity_visibility = DataVisibilityEnum.PUBLIC
     elif normalized_entity_type == 'Upload':
         # Upload entities require authorization to access, so keep the
         # entity_visibility as non-public, as initialized outside block.

src/app_neo4j_queries.py

@@ -349,7 +349,7 @@ def get_sorted_revisions(neo4j_driver, uuid):
 
 def get_sorted_multi_revisions(neo4j_driver, uuid, fetch_all=True, property_key=False):
     results = []
-    match_case = '' if fetch_all is True else 'AND prev.status = "Published" AND next.status = "Published" '
+    match_case = '' if fetch_all is True else 'AND prev.status IN ["Published", "Retracted"] AND next.status IN ["Published", "Retracted"] '
     collect_prop = f".{property_key}" if property_key else ''
 
     query = (
@@ -902,7 +902,7 @@ def get_all_dataset_samples(neo4j_driver, dataset_uuid):
 def get_sankey_info(neo4j_driver, public_only):
     public_only_query = " "
     if public_only:
-        public_only_query = f"AND toLower(ds.status) = 'published' "
+        public_only_query = f"AND toLower(ds.status) IN ['published', 'retracted'] "
     query = (f"MATCH (donor:Donor)-[:ACTIVITY_INPUT]->(organ_activity:Activity)-[:ACTIVITY_OUTPUT]-> "
             f"(organ:Sample {{sample_category:'organ'}})-[*]->(a:Activity)-[:ACTIVITY_OUTPUT]->(ds:Dataset) "
             f"WHERE toLower(a.creation_action) = 'create dataset activity' "
@@ -945,7 +945,7 @@ def get_sankey_info(neo4j_driver, public_only):
 def get_unpublished(neo4j_driver):
     query = (
         "MATCH (ds:Dataset)<-[*]-(d:Donor) "
-        "WHERE ds.status <> 'Published' and ds.status <> 'Hold' "
+        "WHERE NOT ds.status IN ['Published', 'Hold', 'Retracted'] "
         # specimen_type -> sample_category 12/15/2022
         "OPTIONAL MATCH (ds)<-[*]-(s:Sample {sample_category:'organ'}) "
         "RETURN distinct ds.data_types as data_types, ds.group_name as organization, ds.uuid as uuid, "

src/schema/provenance_schema.yaml

@@ -640,9 +640,6 @@ ENTITIES:
       retraction_reason:
         type: string
         indexed: true
-        before_property_update_validators:
-          - validate_if_retraction_permitted
-          - validate_sub_status_provided
         description: 'Information recorded about why a the dataset was retracted.'
       provider_info:
         type: string

src/schema/schema_neo4j_queries.py

@@ -1692,7 +1692,7 @@ def get_component_dataset_uuids(neo4j_driver, uuid):
 def count_attached_published_datasets(neo4j_driver, entity_type, uuid):
     query = (f"MATCH (e:{entity_type})-[:ACTIVITY_INPUT|ACTIVITY_OUTPUT*]->(d:Dataset) "
              # Use the string function toLower() to avoid case-sensetivity issue
-             f"WHERE e.uuid='{uuid}' AND toLower(d.status) = 'published' "
+             f"WHERE e.uuid='{uuid}' AND toLower(d.status) IN ['published', 'retracted'] "
              # COLLECT() returns a list
              # apoc.coll.toSet() reruns a set containing unique nodes
              f"RETURN COUNT(d) AS {record_field_name}")

src/schema/schema_triggers.py

@@ -1766,7 +1766,7 @@ def update_status(property_key, normalized_type, request_args, user_token, exist
     set_status_history(property_key, normalized_type, request_args, user_token, existing_data_dict, new_data_dict)
 
     # Only apply to non-published parent datasets
-    if status.lower() != 'published':
+    if status.lower() not in  ['published', 'retracted']:
         # Only sync the child component datasets status for Multi-Assay Split
         component_dataset_uuids = schema_neo4j_queries.get_component_dataset_uuids(schema_manager.get_neo4j_driver_instance(), uuid)
 

src/schema/schema_validators.py

@@ -298,8 +298,9 @@ def halt_DOI_if_unpublished_dataset(property_key, normalized_entity_type, reques
         # simply get the existing, distinct 'data_access_level' setting for all the Datasets in the Collection
         distinct_dataset_statuses = schema_neo4j_queries.get_collection_datasets_statuses(neo4j_driver_instance
                                                                                           ,existing_data_dict['uuid'])
-    if len( distinct_dataset_statuses) != 1 or \
-            distinct_dataset_statuses[0].lower() != SchemaConstants.DATASET_STATUS_PUBLISHED:
+    PUBLIC_STATUSES = {SchemaConstants.DATASET_STATUS_PUBLISHED, 'retracted'}
+
+    if not all(status.lower() in PUBLIC_STATUSES for status in distinct_dataset_statuses):
         raise ValueError(f"Unable to modify existing {existing_data_dict['entity_type']}"
                          f" {existing_data_dict['uuid']} for DOI since it contains unpublished Datasets.")
 
@@ -414,8 +415,9 @@ def validate_application_header_before_property_update(property_key, normalized_
 def validate_dataset_status_value(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
     # Use lowercase for comparison
     accepted_status_values = [
-        'new', 'processing', 'published', 'qa', 'error', 'hold', 'invalid', 'submitted', 'incomplete', 'approval', 'retracted'
+        'new', 'processing', 'published', 'qa', 'error', 'hold', 'invalid', 'submitted', 'incomplete', 'approval'
     ]
+    # Retracted intentionally omitted. Status will be set to retracted only on a manual basis
     new_status = new_data_dict[property_key].lower()
 
     if new_status not in accepted_status_values:
@@ -426,7 +428,7 @@ def validate_dataset_status_value(property_key, normalized_entity_type, request,
 
     # If status == 'Published' already in Neo4j, then fail for any changes at all
     # Because once published, the dataset should be read-only
-    if existing_data_dict['status'].lower() == SchemaConstants.DATASET_STATUS_PUBLISHED:
+    if existing_data_dict['status'].lower() in [SchemaConstants.DATASET_STATUS_PUBLISHED, 'retracted']:
         raise ValueError(f"The status of this {normalized_entity_type} is already 'Published', status change is not allowed")
 
     # HTTP header names are case-insensitive
@@ -480,6 +482,7 @@ def validate_status_changed(property_key, normalized_entity_type, request, exist
     The json data in request body, already after the regular validations
 """
 def validate_if_retraction_permitted(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
+    # This validator is currently unused. Keeping it in case we decide we want an api endpoint for retraction at some point. 
     if 'status' not in existing_data_dict:
         raise KeyError("Missing 'status' key in 'existing_data_dict' during calling 'validate_if_retraction_permitted()' validator method.")
 
@@ -506,27 +509,6 @@ def validate_if_retraction_permitted(property_key, normalized_entity_type, reque
         raise ValueError("Permission denied, retraction is not allowed")
 
 
-"""
-Validate the sub_status field is also provided when Dataset.retraction_reason is provided on update via PUT
-
-Parameters
-----------
-property_key : str
-    The target property key
-normalized_type : str
-    Submission
-request: Flask request object
-    The instance of Flask request passed in from application request
-existing_data_dict : dict
-    A dictionary that contains all existing entity properties
-new_data_dict : dict
-    The json data in request body, already after the regular validations
-"""
-def validate_sub_status_provided(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
-    if 'sub_status' not in new_data_dict:
-        raise ValueError("Missing sub_status field when retraction_reason is provided")
-
-
 """
 Validate the reaction_reason field is also provided when Dataset.sub_status is provided on update via PUT
 
@@ -1019,7 +1001,7 @@ def _validate_application_header(applications_allowed, request_headers):
 def _is_entity_locked_against_update(existing_entity_dict):
     entity_type = existing_entity_dict['entity_type']
     if entity_type in ['Publication','Dataset']:
-        if 'status' in existing_entity_dict and existing_entity_dict['status'] == 'Published':
+        if 'status' in existing_entity_dict and existing_entity_dict['status'] in ['Published', 'Retracted']:
             raise schema_errors.LockedEntityUpdateException(f"Permission denied to change a published/public {entity_type}.")
     elif entity_type in ['Donor','Sample']:
         if 'data_access_level' in existing_entity_dict and existing_entity_dict['data_access_level'] == 'public':