Add biobank merge code (#1304)

* Add biobank merge code
* Skip biobank merge non-fatally when S3 download or combine fails
* Add try_upload/download S3 helpers while preserving fatal wrappers

Co-authored-by: Manda Wilson <1458628+mandawilson@users.noreply.github.com>

Author: callachennault

import-scripts/fetch-dmp-data-for-import.sh

@@ -13,6 +13,36 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
     # localize global variables / jar names and functions
     source $PORTAL_HOME/scripts/dmp-import-vars-functions.sh
 
+    # Merge biobank patient clinical data when present (already synced via downloadFromS3AllStudies).
+    # Clinical merge failure is non-fatal; an existing biobank timeline file is kept for CDM timeline merge.
+    function merge_biobank_clinical_data() {
+        local study_data_home="$1"
+
+        local input_clinical_file="$study_data_home/data_clinical_patient.txt"
+        local biobank_clinical_file="$study_data_home/data_clinical_patient_biobank.txt"
+        local biobank_timeline_file="$study_data_home/data_timeline_biobank_specimen.txt"
+        local merged_clinical_file="$study_data_home/data_clinical_patient_merged.txt"
+
+        if [ -f "$biobank_clinical_file" ]; then
+            $PYTHON3_BINARY $PORTAL_HOME/scripts/combine_files_py3.py -i "$input_clinical_file" "$biobank_clinical_file" -o "$merged_clinical_file" -c "PATIENT_ID" -m left
+            if [ $? -gt 0 ]; then
+                echo "`date`: Warning: failed to merge biobank patient clinical data for ${study_data_home}; skipping biobank clinical merge for this import."
+                rm -f "$merged_clinical_file"
+            else
+                mv "$merged_clinical_file" "$input_clinical_file"
+                rm -f "$biobank_clinical_file"
+            fi
+        else
+            echo "`date`: Warning: biobank patient clinical file not found in ${study_data_home}; skipping biobank clinical merge for this import."
+        fi
+
+        if [ -f "$biobank_timeline_file" ]; then
+            echo "`date`: Biobank timeline data present in ${study_data_home}."
+        fi
+
+        return 0
+    }
+
     ## STATUS FLAGS
 
     # Flags indicating whether a study can be updated
@@ -733,6 +763,9 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
             else
                 echo "UNLINKED_ARCHER subset successful! Creating cancer type case lists..."
                 echo $(date)
+
+                merge_biobank_clinical_data "$MSK_ARCHER_DATA_HOME"
+
                 # add metadata headers and overrides before importing
                 $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskarcher -f $MSK_ARCHER_DATA_HOME/data_clinical* -i $PORTAL_HOME/scripts/cdm_metadata.json
                 if [ $? -gt 0 ] ; then
@@ -806,6 +839,9 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
     else
         echo "MSKSOLIDHEME merge successful! Creating cancer type case lists..."
         echo $(date)
+
+        merge_biobank_clinical_data "$MSK_SOLID_HEME_DATA_HOME"
+
         # add metadata headers and overrides before importing
         $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskimpact -f $MSK_SOLID_HEME_DATA_HOME/data_clinical_sample.txt -i $PORTAL_HOME/scripts/cdm_metadata.json
         $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskimpact -f $MSK_SOLID_HEME_DATA_HOME/data_clinical_patient.txt -i $PORTAL_HOME/scripts/cdm_metadata.json

import-scripts/s3_functions.sh

@@ -5,18 +5,25 @@ if ! [ -n "$PORTAL_HOME" ] ; then
     exit 1
 fi
 
-# Uploads a file or directory to an S3 bucket (from local to S3)
-# If a directory is provided, files removed locally will be deleted from S3
-# PATH_TO_UPLOAD must end in the same path as PATH_IN_S3, or the script exits with an error
-function upload_to_s3() {
+# Uploads a file or directory to an S3 bucket (from local to S3).
+# If a directory is provided, files removed locally will be deleted from S3.
+# Returns 0 on success, 1 on failure (does not exit the shell).
+#
+# Args:
+#   $1 PATH_TO_UPLOAD  Local file or directory to upload (must exist).
+#   $2 PATH_IN_S3      S3 key prefix within the bucket (no s3:// prefix). Must match
+#                      the trailing path of PATH_TO_UPLOAD, unless empty to sync the
+#                      bucket root (e.g. upload_to_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks").
+#   $3 BUCKET_NAME     S3 bucket name (e.g. mskimpact-databricks).
+function try_upload_to_s3() {
     PATH_TO_UPLOAD="$1"
     PATH_IN_S3="$2"
     BUCKET_NAME="$3"
 
     # Check if path exists
     if [ ! -e "$PATH_TO_UPLOAD" ]; then
         echo "`date`: Path '$PATH_TO_UPLOAD' does not exist, exiting..."
-        exit 1
+        return 1
     fi
 
     # Normalize paths
@@ -28,7 +35,7 @@ function upload_to_s3() {
     if [ -n "$PATH_IN_S3_CLEAN" ]; then
         if [[ "$PATH_TO_UPLOAD_ABS" != *"/$PATH_IN_S3_CLEAN" && "$PATH_TO_UPLOAD_ABS" != "$PATH_IN_S3_CLEAN" ]]; then
             echo "`date`: ERROR – PATH_IN_S3 ('$PATH_IN_S3') must exactly match the trailing path of '$PATH_TO_UPLOAD_ABS'. Exiting..."
-            exit 1
+            return 1
         fi
     fi
 
@@ -56,38 +63,61 @@ function upload_to_s3() {
             --profile saml
     else
         echo "`date`: '$PATH_TO_UPLOAD' is neither a file nor a directory, exiting..."
-        exit 1
+        return 1
     fi
 
     if [ $? -ne 0 ]; then
         echo "`date`: Failed to upload '$PATH_TO_UPLOAD' to S3, exiting..."
+        return 1
+    fi
+}
+
+# Uploads a file or directory to an S3 bucket (from local to S3).
+# If a directory is provided, files removed locally will be deleted from S3.
+# Exits 1 on failure.
+#
+# Args:
+#   $1 PATH_TO_UPLOAD  Local file or directory to upload (must exist).
+#   $2 PATH_IN_S3      S3 key prefix within the bucket (no s3:// prefix). Must match
+#                      the trailing path of PATH_TO_UPLOAD, unless empty to sync the
+#                      bucket root (e.g. upload_to_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks").
+#   $3 BUCKET_NAME     S3 bucket name (e.g. mskimpact-databricks).
+function upload_to_s3() {
+    if ! try_upload_to_s3 "$1" "$2" "$3"; then
         exit 1
     fi
 }
 
-# Syncs a file or directory from S3 to a local path
-# Files not in S3 will be removed from the local destination (for directories)
-# PATH_TO_OVERWRITE must end in PATH_IN_S3 (if provided), and cannot be "/"
-function download_from_s3() {
+# Syncs a file or directory from S3 to a local path.
+# Files not in S3 will be removed from the local destination (for directories).
+# Returns 0 on success, 1 on failure (does not exit the shell).
+#
+# Args:
+#   $1 PATH_TO_OVERWRITE  Local file or directory to sync into (must already exist; cannot be "/").
+#   $2 PATH_IN_S3         S3 key prefix within the bucket (no s3:// prefix). Must match
+#                         the trailing path of PATH_TO_OVERWRITE, unless empty to sync the
+#                         bucket root (e.g. download_from_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks").
+#   $3 BUCKET_NAME        S3 bucket name (e.g. mskimpact-databricks).
+function try_download_from_s3() {
     PATH_TO_OVERWRITE="$1"
     PATH_IN_S3="$2"
     BUCKET_NAME="$3"
 
     # Normalize local path
     if [ -z "$PATH_TO_OVERWRITE" ]; then
         echo "`date`: PATH_TO_OVERWRITE is empty, exiting..."
-        exit 1
+        return 1
     fi
 
     PATH_TO_OVERWRITE_ABS=$(realpath "$PATH_TO_OVERWRITE" 2>/dev/null)
     if [ $? -ne 0 ]; then
         echo "`date`: '$PATH_TO_OVERWRITE' does not exist, exiting..."
-        exit 1
+        return 1
     fi
 
     if [ "$PATH_TO_OVERWRITE_ABS" == "/" ]; then
         echo "`date`: Refusing to sync to root directory '/', exiting..."
-        exit 1
+        return 1
     fi
 
     # Clean PATH_IN_S3
@@ -97,7 +127,7 @@ function download_from_s3() {
     if [ -n "$PATH_IN_S3_CLEAN" ]; then
         if [[ "$PATH_TO_OVERWRITE_ABS" != *"/$PATH_IN_S3_CLEAN" && "$PATH_TO_OVERWRITE_ABS" != "$PATH_IN_S3_CLEAN" ]]; then
             echo "`date`: ERROR – PATH_IN_S3 ('$PATH_IN_S3') must exactly match the trailing path of '$PATH_TO_OVERWRITE_ABS'. Exiting..."
-            exit 1
+            return 1
         fi
     fi
 
@@ -129,11 +159,27 @@ function download_from_s3() {
 
     else
         echo "`date`: '$PATH_TO_OVERWRITE_ABS' is neither a file nor a directory, exiting..."
-        exit 1
+        return 1
     fi
 
     if [ $? -ne 0 ]; then
         echo "`date`: Failed to download from '$S3_SOURCE', exiting..."
+        return 1
+    fi
+}
+
+# Syncs a file or directory from S3 to a local path.
+# Files not in S3 will be removed from the local destination (for directories).
+# Exits 1 on failure.
+#
+# Args:
+#   $1 PATH_TO_OVERWRITE  Local file or directory to sync into (must already exist; cannot be "/").
+#   $2 PATH_IN_S3         S3 key prefix within the bucket (no s3:// prefix). Must match
+#                         the trailing path of PATH_TO_OVERWRITE, unless empty to sync the
+#                         bucket root (e.g. download_from_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks").
+#   $3 BUCKET_NAME        S3 bucket name (e.g. mskimpact-databricks).
+function download_from_s3() {
+    if ! try_download_from_s3 "$1" "$2" "$3"; then
         exit 1
     fi
 }