diff --git a/import-scripts/fetch-dmp-data-for-import.sh b/import-scripts/fetch-dmp-data-for-import.sh index 8a2cfdb17..ecaf157df 100755 --- a/import-scripts/fetch-dmp-data-for-import.sh +++ b/import-scripts/fetch-dmp-data-for-import.sh @@ -13,6 +13,36 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" # localize global variables / jar names and functions source $PORTAL_HOME/scripts/dmp-import-vars-functions.sh + # Merge biobank patient clinical data when present (already synced via downloadFromS3AllStudies). + # Clinical merge failure is non-fatal; an existing biobank timeline file is kept for CDM timeline merge. + function merge_biobank_clinical_data() { + local study_data_home="$1" + + local input_clinical_file="$study_data_home/data_clinical_patient.txt" + local biobank_clinical_file="$study_data_home/data_clinical_patient_biobank.txt" + local biobank_timeline_file="$study_data_home/data_timeline_biobank_specimen.txt" + local merged_clinical_file="$study_data_home/data_clinical_patient_merged.txt" + + if [ -f "$biobank_clinical_file" ]; then + $PYTHON3_BINARY $PORTAL_HOME/scripts/combine_files_py3.py -i "$input_clinical_file" "$biobank_clinical_file" -o "$merged_clinical_file" -c "PATIENT_ID" -m left + if [ $? -gt 0 ]; then + echo "`date`: Warning: failed to merge biobank patient clinical data for ${study_data_home}; skipping biobank clinical merge for this import." + rm -f "$merged_clinical_file" + else + mv "$merged_clinical_file" "$input_clinical_file" + rm -f "$biobank_clinical_file" + fi + else + echo "`date`: Warning: biobank patient clinical file not found in ${study_data_home}; skipping biobank clinical merge for this import." + fi + + if [ -f "$biobank_timeline_file" ]; then + echo "`date`: Biobank timeline data present in ${study_data_home}." + fi + + return 0 + } + ## STATUS FLAGS # Flags indicating whether a study can be updated @@ -733,6 +763,9 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" else echo "UNLINKED_ARCHER subset successful! Creating cancer type case lists..." echo $(date) + + merge_biobank_clinical_data "$MSK_ARCHER_DATA_HOME" + # add metadata headers and overrides before importing $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskarcher -f $MSK_ARCHER_DATA_HOME/data_clinical* -i $PORTAL_HOME/scripts/cdm_metadata.json if [ $? -gt 0 ] ; then @@ -806,6 +839,9 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock" else echo "MSKSOLIDHEME merge successful! Creating cancer type case lists..." echo $(date) + + merge_biobank_clinical_data "$MSK_SOLID_HEME_DATA_HOME" + # add metadata headers and overrides before importing $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskimpact -f $MSK_SOLID_HEME_DATA_HOME/data_clinical_sample.txt -i $PORTAL_HOME/scripts/cdm_metadata.json $PYTHON_BINARY $PORTAL_HOME/scripts/add_clinical_attribute_metadata_headers.py -s mskimpact -f $MSK_SOLID_HEME_DATA_HOME/data_clinical_patient.txt -i $PORTAL_HOME/scripts/cdm_metadata.json diff --git a/import-scripts/s3_functions.sh b/import-scripts/s3_functions.sh index 3738306a5..46a9f2ab4 100644 --- a/import-scripts/s3_functions.sh +++ b/import-scripts/s3_functions.sh @@ -5,10 +5,17 @@ if ! [ -n "$PORTAL_HOME" ] ; then exit 1 fi -# Uploads a file or directory to an S3 bucket (from local to S3) -# If a directory is provided, files removed locally will be deleted from S3 -# PATH_TO_UPLOAD must end in the same path as PATH_IN_S3, or the script exits with an error -function upload_to_s3() { +# Uploads a file or directory to an S3 bucket (from local to S3). +# If a directory is provided, files removed locally will be deleted from S3. +# Returns 0 on success, 1 on failure (does not exit the shell). +# +# Args: +# $1 PATH_TO_UPLOAD Local file or directory to upload (must exist). +# $2 PATH_IN_S3 S3 key prefix within the bucket (no s3:// prefix). Must match +# the trailing path of PATH_TO_UPLOAD, unless empty to sync the +# bucket root (e.g. upload_to_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks"). +# $3 BUCKET_NAME S3 bucket name (e.g. mskimpact-databricks). +function try_upload_to_s3() { PATH_TO_UPLOAD="$1" PATH_IN_S3="$2" BUCKET_NAME="$3" @@ -16,7 +23,7 @@ function upload_to_s3() { # Check if path exists if [ ! -e "$PATH_TO_UPLOAD" ]; then echo "`date`: Path '$PATH_TO_UPLOAD' does not exist, exiting..." - exit 1 + return 1 fi # Normalize paths @@ -28,7 +35,7 @@ function upload_to_s3() { if [ -n "$PATH_IN_S3_CLEAN" ]; then if [[ "$PATH_TO_UPLOAD_ABS" != *"/$PATH_IN_S3_CLEAN" && "$PATH_TO_UPLOAD_ABS" != "$PATH_IN_S3_CLEAN" ]]; then echo "`date`: ERROR – PATH_IN_S3 ('$PATH_IN_S3') must exactly match the trailing path of '$PATH_TO_UPLOAD_ABS'. Exiting..." - exit 1 + return 1 fi fi @@ -56,19 +63,42 @@ function upload_to_s3() { --profile saml else echo "`date`: '$PATH_TO_UPLOAD' is neither a file nor a directory, exiting..." - exit 1 + return 1 fi if [ $? -ne 0 ]; then echo "`date`: Failed to upload '$PATH_TO_UPLOAD' to S3, exiting..." + return 1 + fi +} + +# Uploads a file or directory to an S3 bucket (from local to S3). +# If a directory is provided, files removed locally will be deleted from S3. +# Exits 1 on failure. +# +# Args: +# $1 PATH_TO_UPLOAD Local file or directory to upload (must exist). +# $2 PATH_IN_S3 S3 key prefix within the bucket (no s3:// prefix). Must match +# the trailing path of PATH_TO_UPLOAD, unless empty to sync the +# bucket root (e.g. upload_to_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks"). +# $3 BUCKET_NAME S3 bucket name (e.g. mskimpact-databricks). +function upload_to_s3() { + if ! try_upload_to_s3 "$1" "$2" "$3"; then exit 1 fi } -# Syncs a file or directory from S3 to a local path -# Files not in S3 will be removed from the local destination (for directories) -# PATH_TO_OVERWRITE must end in PATH_IN_S3 (if provided), and cannot be "/" -function download_from_s3() { +# Syncs a file or directory from S3 to a local path. +# Files not in S3 will be removed from the local destination (for directories). +# Returns 0 on success, 1 on failure (does not exit the shell). +# +# Args: +# $1 PATH_TO_OVERWRITE Local file or directory to sync into (must already exist; cannot be "/"). +# $2 PATH_IN_S3 S3 key prefix within the bucket (no s3:// prefix). Must match +# the trailing path of PATH_TO_OVERWRITE, unless empty to sync the +# bucket root (e.g. download_from_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks"). +# $3 BUCKET_NAME S3 bucket name (e.g. mskimpact-databricks). +function try_download_from_s3() { PATH_TO_OVERWRITE="$1" PATH_IN_S3="$2" BUCKET_NAME="$3" @@ -76,18 +106,18 @@ function download_from_s3() { # Normalize local path if [ -z "$PATH_TO_OVERWRITE" ]; then echo "`date`: PATH_TO_OVERWRITE is empty, exiting..." - exit 1 + return 1 fi PATH_TO_OVERWRITE_ABS=$(realpath "$PATH_TO_OVERWRITE" 2>/dev/null) if [ $? -ne 0 ]; then echo "`date`: '$PATH_TO_OVERWRITE' does not exist, exiting..." - exit 1 + return 1 fi if [ "$PATH_TO_OVERWRITE_ABS" == "/" ]; then echo "`date`: Refusing to sync to root directory '/', exiting..." - exit 1 + return 1 fi # Clean PATH_IN_S3 @@ -97,7 +127,7 @@ function download_from_s3() { if [ -n "$PATH_IN_S3_CLEAN" ]; then if [[ "$PATH_TO_OVERWRITE_ABS" != *"/$PATH_IN_S3_CLEAN" && "$PATH_TO_OVERWRITE_ABS" != "$PATH_IN_S3_CLEAN" ]]; then echo "`date`: ERROR – PATH_IN_S3 ('$PATH_IN_S3') must exactly match the trailing path of '$PATH_TO_OVERWRITE_ABS'. Exiting..." - exit 1 + return 1 fi fi @@ -129,11 +159,27 @@ function download_from_s3() { else echo "`date`: '$PATH_TO_OVERWRITE_ABS' is neither a file nor a directory, exiting..." - exit 1 + return 1 fi if [ $? -ne 0 ]; then echo "`date`: Failed to download from '$S3_SOURCE', exiting..." + return 1 + fi +} + +# Syncs a file or directory from S3 to a local path. +# Files not in S3 will be removed from the local destination (for directories). +# Exits 1 on failure. +# +# Args: +# $1 PATH_TO_OVERWRITE Local file or directory to sync into (must already exist; cannot be "/"). +# $2 PATH_IN_S3 S3 key prefix within the bucket (no s3:// prefix). Must match +# the trailing path of PATH_TO_OVERWRITE, unless empty to sync the +# bucket root (e.g. download_from_s3 "$DMP_DATA_HOME" "" "mskimpact-databricks"). +# $3 BUCKET_NAME S3 bucket name (e.g. mskimpact-databricks). +function download_from_s3() { + if ! try_download_from_s3 "$1" "$2" "$3"; then exit 1 fi }