@@ -20,6 +20,7 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
2020 IMPORT_STATUS_HEME=0
2121 IMPORT_STATUS_ARCHER=0
2222 IMPORT_STATUS_ACCESS=0
23+ IMPORT_STATUS_ACCESSHEME=0
2324
2425 # Flags for ARCHER structural variants merge failure
2526 ARCHER_MERGE_IMPACT_FAIL=0
@@ -30,12 +31,14 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
3031 EXPORT_SUPP_DATE_HEME_FAIL=0
3132 EXPORT_SUPP_DATE_ARCHER_FAIL=0
3233 EXPORT_SUPP_DATE_ACCESS_FAIL=0
34+ EXPORT_SUPP_DATE_ACCESSHEME_FAIL=0
3335
3436 # Assume fetchers have failed until they complete successfully
3537 FETCH_CVR_IMPACT_FAIL=1
3638 FETCH_CVR_HEME_FAIL=1
3739 FETCH_CVR_ARCHER_FAIL=1
3840 FETCH_CVR_ACCESS_FAIL=1
41+ FETCH_CVR_ACCESSHEME_FAIL=1
3942
4043 UNLINKED_ARCHER_SUBSET_FAIL=0
4144 MIXEDPACT_MERGE_FAIL=0
@@ -149,6 +152,13 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
149152 sendPreImportFailureMessageMskPipelineLogsSlack " ACCESS Redcap export of mskaccess_data_clinical_supp_date"
150153 fi
151154
155+ echo " exporting accessheme data_clinical_supp_date.txt from redcap"
156+ export_project_from_redcap $MSK_ACCESSHEME_DATA_HOME mskaccess_heme_data_clinical_supp_date
157+ if [ $? -gt 0 ] ; then
158+ EXPORT_SUPP_DATE_ACCESSHEME_FAIL=1
159+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESS Redcap export of mskaccess_heme_data_clinical_supp_date"
160+ fi
161+
152162 # IF WE CANCEL ANY IMPORT, LET REDCAP GET AHEAD OF CURRENCY, BUT DON'T LET THE REPOSITORY HEAD ADVANCE [REVERT]
153163 printTimeStampedDataProcessingStepMessage " export of cvr clinical files from redcap"
154164 echo " exporting impact data_clinical.txt from redcap"
@@ -179,6 +189,12 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
179189 sendPreImportFailureMessageMskPipelineLogsSlack " ACCESS Redcap export of mskaccess_data_clinical_cvr"
180190 fi
181191
192+ echo " exporting accessheme data_clinical.txt from redcap"
193+ export_project_from_redcap $MSK_ACCESSHEME_DATA_HOME mskaccess_heme_data_clinical
194+ if [ $? -gt 0 ] ; then
195+ IMPORT_STATUS_ACCESSHEME=1
196+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESS Redcap export of mskaccess_heme_data_clinical_cvr"
197+ fi
182198 # -----------------------------------------------------------------------------------------------------------
183199 # MSKIMPACT DATA FETCHES
184200 # TODO: move other pre-import/data-fetch steps here (i.e exporting raw files from redcap)
@@ -391,6 +407,40 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
391407 fi
392408 fi
393409
410+ # -----------------------------------------------------------------------------------------------------------
411+ # ACCESSHEME DATA FETCHES
412+ printTimeStampedDataProcessingStepMessage " ACCESSHEME data processing"
413+
414+ if [ $IMPORT_STATUS_ACCESSHEME -eq 0 ] ; then
415+ # fetch new/updated access samples using CVR Web service (must come after git fetching).
416+ drop_dead_instant_step=$( date --date=" +3hours" -Iseconds) # nearly 3 hours from now
417+ drop_dead_instant_string=$( find_earlier_instant " $drop_dead_instant_step " " $DROP_DEAD_INSTANT_END_TO_END " )
418+ printTimeStampedDataProcessingStepMessage " CVR fetch for accessheme"
419+ # access has -b option to block warnings for samples with zero variants (all samples will have zero variants)
420+ $JAVA_BINARY $JAVA_CVR_FETCHER_ARGS -d $MSK_ACCESSHEME_DATA_HOME -p $MSK_ACCESSHEME_PRIVATE_DATA_HOME -n data_clinical_mskaccessheme_data_clinical.txt -i mskaccess_heme -s -b -r 50 $CVR_TEST_MODE_ARGS -z $drop_dead_instant_string
421+ if [ $? -gt 0 ] ; then
422+ echo " CVR ACCESSHEME fetch failed!"
423+ echo " This will not affect importing of mskimpact"
424+ cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard
425+ cd $DMP_PRIVATE_DATA_HOME ; $GIT_BINARY reset HEAD --hard
426+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESSHEME CVR Fetch"
427+ IMPORT_STATUS_ACCESSHEME=1
428+ else
429+ # check for PHI
430+ $PYTHON_BINARY $PORTAL_HOME /scripts/phi-scanner.py -a $PIPELINES_CONFIG_HOME /properties/fetch-cvr/phi-scanner-attributes.txt -j $MSK_ACCESSHEME_PRIVATE_DATA_HOME /cvr_data.json
431+ if [ $? -gt 0 ] ; then
432+ echo " PHI attributes found in $MSK_ACCESS_PRIVATE_DATA_HOME /cvr_data.json! ACCESSHEME will not be imported!"
433+ cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard
434+ cd $DMP_PRIVATE_DATA_HOME ; $GIT_BINARY reset HEAD --hard
435+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESSHEME PHI attributes scan failed on $MSK_ACCESSHEME_PRIVATE_DATA_HOME /cvr_data.json"
436+ IMPORT_STATUS_ACCESSHEME=1
437+ else
438+ FETCH_CVR_ACCESSHEME_FAIL=0
439+ cd $MSK_ACCESSHEME_DATA_HOME ; $GIT_BINARY add ./* ; $GIT_BINARY commit -m " Latest ACCESSHEME dataset"
440+ cd $MSK_ACCESSHEME_PRIVATE_DATA_HOME ; $GIT_BINARY add ./* ; $GIT_BINARY commit -m " Latest ACCESSHEME dataset"
441+ fi
442+ fi
443+ fi
394444 # -----------------------------------------------------------------------------------------------------------
395445 # GENERATE CANCER TYPE CASE LISTS AND SUPP DATE ADDED FILES
396446 # NOTE: Even though cancer type case lists are not needed for MSKIMPACT, HEMEPACT for the portal
@@ -442,6 +492,18 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
442492 cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard
443493 fi
444494
495+ # generate case lists by cancer type and add "DATE ADDED" info to clinical data for ACCESSHEME
496+ if [ $IMPORT_STATUS_ACCESSHEME -eq 0 ] && [ $FETCH_CVR_ACCESSHEME_FAIL -eq 0 ] ; then
497+ # TODO: double check args for this
498+ addCancerTypeCaseLists $MSK_ACCESSHEME_DATA_HOME " mskaccess_heme" " data_clinical_mskaccess_heme_data_clinical.txt"
499+ cd $MSK_ACCESSHEME_DATA_HOME ; $GIT_BINARY add case_lists ; $GIT_BINARY commit -m " Latest ACCESSHEME Dataset: Case Lists"
500+ if [ $EXPORT_SUPP_DATE_ACCESSHEME_FAIL -eq 0 ] ; then
501+ addDateAddedData $MSK_ACCESSHEME_DATA_HOME " data_clinical_mskaccess_heme_data_clinical.txt" " data_clinical_mskaccess_heme_data_clinical_supp_date.txt"
502+ cd $MSK_ACCESSHEME_DATA_HOME ; $GIT_BINARY add data_clinical_mskaccess_heme_data_clinical_supp_date.txt ; $GIT_BINARY commit -m " Latest ACCESSHEME Dataset: SUPP DATE ADDED"
503+ fi
504+ cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard
505+ fi
506+
445507 # -----------------------------------------------------------------------------------------------------------
446508 # ADDITIONAL PROCESSING
447509
@@ -558,6 +620,23 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
558620 fi
559621 fi
560622
623+ # # ACCESSHEME imports
624+
625+ # imports access cvr data into redcap
626+ if [ $FETCH_CVR_ACCESSHEME_FAIL -eq 0 ] ; then
627+ import_accessheme_cvr_to_redcap
628+ if [ $? -gt 0 ] ; then
629+ IMPORT_STATUS_ACCESSHEME=1
630+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESSHEME CVR Redcap Import"
631+ fi
632+ if [ $EXPORT_SUPP_DATE_ACCESSHEME_FAIL -eq 0 ] ; then
633+ import_accessheme_supp_date_to_redcap
634+ if [ $? -gt 0 ] ; then
635+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESSHEME Supp Date Redcap Import. Project is now empty, data restoration required"
636+ fi
637+ fi
638+ fi
639+
561640 echo " Import into redcap finished"
562641
563642 # -------------------------------------------------------------
@@ -575,8 +654,11 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
575654 echo " removing raw clinical & timeline files for mskaccess"
576655 remove_raw_clinical_timeline_data_files $MSK_ACCESS_DATA_HOME
577656
657+ echo " removing raw clinical & timeline files for mskaccess"
658+ remove_raw_clinical_timeline_data_files $MSK_ACCESSHEME_DATA_HOME
659+
578660 # commit raw file cleanup - study staging directories should only contain files for portal import
579- $GIT_BINARY commit -m " Raw clinical and timeline file cleanup: MSKIMPACT, HEMEPACT, ARCHER, ACCESS"
661+ $GIT_BINARY commit -m " Raw clinical and timeline file cleanup: MSKIMPACT, HEMEPACT, ARCHER, ACCESS, ACCESSHEME "
580662
581663 # -------------------------------------------------------------
582664 # REDCAP EXPORTS - CBIO STAGING FORMATS
@@ -656,6 +738,19 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
656738 fi
657739 fi
658740
741+ printTimeStampedDataProcessingStepMessage " export of redcap data for accessheme"
742+ if [ $IMPORT_STATUS_ACCESSHEME -eq 0 ] ; then
743+ export_stable_id_from_redcap mskaccess $MSK_ACCESSHEME_DATA_HOME
744+ if [ $? -gt 0 ] ; then
745+ IMPORT_STATUS_ACCESSHEME=1
746+ cd $DMP_DATA_HOME ; $GIT_BINARY reset HEAD --hard
747+ sendPreImportFailureMessageMskPipelineLogsSlack " ACCESSHEME Redcap Export"
748+ else
749+ touch $MSK_ACCESSHEME_CONSUME_TRIGGER
750+ cd $MSK_ACCESSHEME_DATA_HOME ; $GIT_BINARY add * ; $GIT_BINARY commit -m " Latest ACCESSHEME Dataset: Clinical and Timeline"
751+ fi
752+ fi
753+
659754 # -------------------------------------------------------------
660755 # UNLINKED ARCHER DATA PROCESSING
661756 # NOTE: This processing should only occur if (1) PROCESS_UNLINKED_ARCHER_STUDY=1 and
@@ -729,9 +824,9 @@ MY_FLOCK_FILEPATH="/data/portal-cron/cron-lock/fetch-dmp-data-for-import.lock"
729824 echo $( date)
730825 fi
731826
732- printTimeStampedDataProcessingStepMessage " merge of MSK-IMPACT, HEMEPACT, ACCESS data for MSKSOLIDHEME"
827+ printTimeStampedDataProcessingStepMessage " merge of MSK-IMPACT, HEMEPACT, ACCESS, ACCESSHEME data for MSKSOLIDHEME"
733828 # MSKSOLIDHEME merge and check exit code
734- $PYTHON_BINARY $PORTAL_HOME /scripts/merge.py -d $MSK_SOLID_HEME_DATA_HOME -i mskimpact -m " true" -e $MAPPED_ARCHER_SAMPLES_FILE $MSK_IMPACT_DATA_HOME $MSK_HEMEPACT_DATA_HOME $MSK_ACCESS_DATA_HOME
829+ $PYTHON_BINARY $PORTAL_HOME /scripts/merge.py -d $MSK_SOLID_HEME_DATA_HOME -i mskimpact -m " true" -e $MAPPED_ARCHER_SAMPLES_FILE $MSK_IMPACT_DATA_HOME $MSK_HEMEPACT_DATA_HOME $MSK_ACCESS_DATA_HOME $MSK_ACCESSHEME_DATA_HOME
735830 if [ $? -gt 0 ] ; then
736831 echo " MSKSOLIDHEME merge failed! Study will not be updated in the portal."
737832 echo $( date)
0 commit comments