Merge branch 'lcms_metab_lipid_cicd' into 'lcms_metab'

Add CI/CD for lipidomics and makefile command for downloading lipidomics test data

See merge request mass-spectrometry/metams!21

Author: kheal

.github/workflows/gcms_workflow_test.yml .github/workflows/workflow_tests.yml.github/workflows/gcms_workflow_test.yml renamed to .github/workflows/workflow_tests.yml

@@ -1,4 +1,4 @@
-name: Test GC/MS workflow
+name: Test workflows
 
 on:
   push:
@@ -22,20 +22,36 @@ jobs:
         curl -fsSL https://get.docker.com -o get-docker.sh
         sh get-docker.sh
         sudo usermod -aG docker $USER
-  
-    - name: Build Docker Image
-      run: |
-        docker build -t local-metams:latest .
 
     - name: Install MiniWDL
       run: |
         python -m pip install --upgrade pip
         pip install miniwdl
-
+    
+    - name: Download test data for lipidomics
+      run: |
+        make get-lipid-test-data
+    
+    - name: Run Lipidomics WDL with lipidomics test data
+      run: |
+        miniwdl run wdl/metaMS_lcmslipidomics.wdl -i wdl/metams_input_lipidomics.json --verbose --no-cache --copy-input-files
+    
     - name: Run GCMS WDL with pushed Docker Image
       run: |
         miniwdl run wdl/metaMS_gcms.wdl -i wdl/metams_input_gcms.json --verbose --no-cache --copy-input-files
+  
+    - name: Build Docker Image
+      run: |
+        docker build -t local-metams:latest .
     
     - name: Run GCMS WDL with local Docker Image based on Dockerfile
       run: |
-        miniwdl run wdl/metaMS_gcms.wdl -i wdl/metams_input_gcms_local_docker.json --verbose --no-cache --copy-input-files
+        miniwdl run wdl/metaMS_gcms.wdl -i wdl/metams_input_gcms_local_docker.json --verbose --no-cache --copy-input-files
+    
+    - name: Run Lipidomics WDL with local Docker Image based on Dockerfile
+      run: |
+        miniwdl run wdl/metaMS_lcmslipidomics.wdl -i wdl/metams_input_lipidomics_local_docker.json --verbose --no-cache --copy-input-files
+    
+    - name: Remove lipidomics test data
+      run: |
+        rm -rf test_data

.gitignore

@@ -44,4 +44,7 @@ share/python-wheels/
 *.token
 
 # msp databases
-*.msp
+*.msp
+
+# test_data
+test_data/

MAINTAINERS.md

@@ -3,10 +3,11 @@
 ## Table of Contents
 1. [Introduction](#introduction)
 2. [Setting Up the Development Environment](#setting-up-the-development-environment)
-3. [Releasing a new version](#releasing-a-new-version)
+3. [Testing a new or modified workflow](#testing-a-new-or-modified-workflow)
+4. [Releasing a new version](#releasing-a-new-version)
     1. [Bump Version Numbers](#bump-version-numbers)
     2. [Push Updated Docker Image](#push-updated-docker-image)
-4. [Generate Documentation](#generate-documentation)
+5. [Generate Documentation](#generate-documentation)
 
 ## Introduction
 This guide provides instructions for developers working on the project. It covers setting up the development environment, pushing Docker images, generating documentation, and bumping version numbers.
@@ -31,6 +32,14 @@ This guide provides instructions for developers working on the project. It cover
     pip install -r requirements-dev.txt
     ```
 
+## Testing a new or modified workflow
+
+To test a workflow before an associated docker image is pushed, you can run the workflow locally using the docker image that is built locally according to the dockerfile in the workflow directory.  To do this, your wdl must have the option to change the docker image in which the workflow is run, and this image must be specified in the input json file.  For example, the following command will run the lipidomcis workflow using the docker image that is built locally.  See the input file [here](wdl/metams_input_lipidomics_local_docker.json) for an example of how to specify the docker image in the input file.
+
+    ```sh
+    miniwdl run wdl/metaMS_lcmslipidomics.wdl -i wdl/metams_input_lipidomics_local_docker.json --verbose --no-cache --copy-input-files
+    ```
+
 ## Releasing a new version
 
 The following steps should be followed in order
@@ -54,7 +63,7 @@ To bump the repo *and all workflows*, run one of the following commands.  This w
     make patch
     ```
 ### Push Updated Docker Image
-Use the github action to push a docker image to the appropriate dockerhub repository.
+Use the github action to push a docker image to the appropriate dockerhub repository. This will use the version number that was set in the previous step and will build and push a docker image according to the dockerfile in the root of the repo.  The docker image will be tagged with the version number and the latest tag.
 
 ## Generating Documentation
 

Makefile

@@ -119,9 +119,21 @@ wdl-run-gcms-local:
 	@make docker-build-local
 	@miniwdl run wdl/metaMS_gcms.wdl -i wdl/metams_input_gcms_local_docker.json --verbose --no-cache --copy-input-files
 
+get-lipid-test-data:
+	@echo "Downloading test data for lipidomics"
+	@mkdir -p test_data
+	@curl -L -o test_data/test_lipid_data.zip https://nmdcdemo.emsl.pnnl.gov/lipidomics/test_data/metams_lipid_test_data/test_lipid_data.zip
+	@unzip test_data/test_lipid_data.zip -d test_data/
+	@rm test_data/test_lipid_data.zip
+	@echo "Test data downloaded and unzipped"
+
 wdl-run-lipid :
 	miniwdl run wdl/metaMS_lcmslipidomics.wdl -i wdl/metams_input_lipidomics.json --verbose --no-cache --copy-input-files
 
+wdl-run-lipid-local:
+	@make docker-build-local
+	miniwdl run wdl/metaMS_lcmslipidomics.wdl -i wdl/metams_input_lipidomics_local_docker.json --verbose --no-cache --copy-input-files
+
 convert_lipid_rst_to_md:
     # convert the lipid documentation from rst to md and render it into html
 	pandoc -f rst -t markdown -o docs/lcms_lipidomics/README_LCMS_LIPID.md docs/lcms_lipidomics/index.rst

metaMS/lcms_lipidomics_workflow.py

@@ -291,10 +291,6 @@ def export_results(myLCMSobj, out_path, molecular_metadata=None, final=False):
     exporter.to_hdf(overwrite=True)
     if final:
         exporter.report_to_csv(molecular_metadata=molecular_metadata)
-    else:
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            exporter.report_to_csv()
 
 def run_lipid_sp_ms1(file_in, out_path, params_toml, scan_translator):
     """Run signal processing and associated mass feature generation for a lipidomics LCMS file
@@ -336,7 +332,11 @@ def run_lipid_sp_ms1(file_in, out_path, params_toml, scan_translator):
             ]
         )
     )
-    mz_dict = {myLCMSobj.polarity: precursor_mz_list}
+    mz_dict = {myLCMSobj.polarity: precursor_mz_list.copy()}
+    myLCMSobj.spectra_parser = None 
+    del myLCMSobj
+    gc.collect()
+
     return mz_dict
 
 def prep_metadata(mz_dicts, out_dir, db_location):
@@ -599,8 +599,6 @@ def run_lcms_lipidomics_workflow(
     scan_translator = lipid_workflow_params.scan_translator_path
 
     click.echo("Starting lipidomics workflow for " + str(len(files_list)) + " file(s), using " +  str(cores) + " core(s)")
-    gc.collect()
-
     # Run signal processing, get associated ms1, add associated ms2, do ms1 molecular search, and export intermediate results
     if cores == 1 or len(files_list) == 1:
         mz_dicts = []
@@ -624,13 +622,11 @@ def run_lcms_lipidomics_workflow(
                 for file_in, file_out in zip(files_list, out_paths_list)
             ]
             mz_dicts = pool.starmap(run_lipid_sp_ms1, args)
-    gc.collect() 
-        
+
     # Prepare metadata for searching
     click.echo("Preparing metadata for ms2 spectral search")
     metadata = prep_metadata(mz_dicts, out_dir, lipid_workflow_params.db_location)
     del mz_dicts
-    gc.collect()
 
     # Run ms2 spectral search and export final results
     click.echo("Starting ms2 spectral search and exporting final results")
@@ -644,4 +640,4 @@ def run_lcms_lipidomics_workflow(
             args = [(file_out, metadata, scan_translator) for file_out in out_paths_list]
             pool.starmap(run_lipid_ms2, args)
 
-    gc.collect()
+    click.echo("Lipidomics workflow complete")

wdl/metaMS_lcmslipidomics.wdl

@@ -1,7 +1,14 @@
 version 1.0
 
 workflow lcmsLipidomics {
-    call runMetaMSLCMSLipidomics
+    input {
+        String? docker_image  # Optional input for Docker image
+    }
+
+    call runMetaMSLCMSLipidomics {
+        input:
+            docker_image = docker_image
+    }
 
     output {
         String out = runMetaMSLCMSLipidomics.out
@@ -17,6 +24,7 @@ task runMetaMSLCMSLipidomics {
         File db_location
         File scan_translator_path
         Int cores
+        String? docker_image
     }
 
     command {
@@ -27,6 +35,24 @@ task runMetaMSLCMSLipidomics {
             -d ${db_location} \
             -s ${scan_translator_path} \
             -j ${cores}
+        EXIT_CODE=$?
+        num_inputs=$(echo ${sep=' ' file_paths} | wc -w)
+        num_subdirs=$(find "${output_directory}" -mindepth 1 -maxdepth 1 -type d | wc -l)
+        all_ok=1
+        for subdir in "${output_directory}"/*/; do
+            if [ -d "$subdir" ]; then
+                if ! ls "$subdir"/*.csv 1> /dev/null 2>&1; then
+                    all_ok=0
+                    break
+                fi
+            fi
+        done
+
+        if [ "$all_ok" -eq 1 ] && [ "$num_subdirs" -eq "$num_inputs" ]; then
+            exit 0
+        else
+            exit $EXIT_CODE
+        fi
     }
 
     output {
@@ -35,6 +61,6 @@ task runMetaMSLCMSLipidomics {
     }
 
     runtime {
-        docker: "microbiomedata/metams:3.1.0"
+        docker: "~{if defined(docker_image) then docker_image else 'microbiomedata/metams:3.1.0'}"
     }
 }

wdl/metams_input_lipidomics.json

@@ -1,11 +1,11 @@
 {
     "lcmsLipidomics.runMetaMSLCMSLipidomics.file_paths": [
-        "/Users/heal742/Library/CloudStorage/OneDrive-PNNL/Documents/_DMS_data/_NMDC/_blanchard_lipidomics/raw/Blanch_Nat_Lip_C_4_AB_M_08_NEG_25Jan18_Brandi-WCSH5801.raw",
-        "/Users/heal742/Library/CloudStorage/OneDrive-PNNL/Documents/_DMS_data/_NMDC/_blanchard_lipidomics/raw/Blanch_Nat_Lip_C_4_AB_M_08_POS_23Jan18_Brandi-WCSH5801.raw"
+        "./test_data/test_lipid_data/Blanch_Nat_Lip_C_4_AB_M_08_NEG_25Jan18_Brandi-WCSH5801.raw",
+        "./test_data/test_lipid_data/Blanch_Nat_Lip_C_4_AB_M_08_POS_23Jan18_Brandi-WCSH5801.raw"
     ],
     "lcmsLipidomics.runMetaMSLCMSLipidomics.output_directory": "output",
-    "lcmsLipidomics.runMetaMSLCMSLipidomics.corems_toml_path": "/Users/heal742/LOCAL/05_NMDC/02_MetaMS/data_processing/configurations/emsl_lipidomics_corems_params.toml",
-    "lcmsLipidomics.runMetaMSLCMSLipidomics.db_location": "/Users/heal742/LOCAL/05_NMDC/00_Lipid_Databse/lipid_db/lipid_ref.sqlite",
-    "lcmsLipidomics.runMetaMSLCMSLipidomics.scan_translator_path": "./configuration/lipid_configs/emsl_lipidomics_scan_translator.toml",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.corems_toml_path": "./test_data/test_lipid_data/emsl_lipidomics_corems_params_test.toml",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.db_location": "./test_data/test_lipid_data/lipid_ref.sqlite",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.scan_translator_path": "./test_data/test_lipid_data/emsl_lipidomics_scan_translator.toml",
     "lcmsLipidomics.runMetaMSLCMSLipidomics.cores": 1
 }

wdl/metams_input_lipidomics_local_docker.json

@@ -0,0 +1,12 @@
+{
+    "lcmsLipidomics.docker_image": "local-metams:latest",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.file_paths": [
+        "./test_data/test_lipid_data/Blanch_Nat_Lip_C_4_AB_M_08_NEG_25Jan18_Brandi-WCSH5801.raw",
+        "./test_data/test_lipid_data/Blanch_Nat_Lip_C_4_AB_M_08_POS_23Jan18_Brandi-WCSH5801.raw"
+    ],
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.output_directory": "output",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.corems_toml_path": "./test_data/test_lipid_data/emsl_lipidomics_corems_params_test.toml",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.db_location": "./test_data/test_lipid_data/lipid_ref.sqlite",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.scan_translator_path": "./test_data/test_lipid_data/emsl_lipidomics_scan_translator.toml",
+    "lcmsLipidomics.runMetaMSLCMSLipidomics.cores": 1
+}