Drug2cell dev

.gitignore

@@ -91,6 +91,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+*.ipynb
 
 # IPython
 profile_default/
@@ -130,6 +131,7 @@ ipython_config.py
 .pdm.toml
 .pdm-python
 .pdm-build/
+config_dev.toml
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
@@ -149,6 +151,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+*_venv/
 
 # Spyder project settings
 .spyderproject
@@ -189,3 +192,14 @@ cython_debug/
 
 # Other
 .vscode/
+
+# DS_Store files
+.DS_Store
+*.DS_Store
+*/.DS_Store
+
+# HPC outputs
+*.sh.e*
+*.sh.o*
+*.e*
+*.o*

HPC_jobs/HLCA_full_col_inf.txt

@@ -0,0 +1,92 @@
+Disease distribution:
+disease
+normal                                   1305099
+COVID-19                                  341761
+pulmonary fibrosis                        268932
+interstitial lung disease                  68456
+chronic obstructive pulmonary disease      67943
+lung adenocarcinoma                        62807
+pneumonia                                  31923
+chronic rhinitis                           29137
+lung large cell carcinoma                  21167
+squamous cell lung carcinoma               20631
+cystic fibrosis                            17590
+lymphangioleiomyomatosis                   12374
+pleomorphic carcinoma                      10765
+hypersensitivity pneumonitis               10379
+non-specific interstitial pneumonia         8597
+pulmonary sarcoidosis                       4886
+Name: count, dtype: int64
+Tissue type distribution:
+tissue_type
+tissue    2282447
+Name: count, dtype: int64
+Cell type distribution:
+cell_type
+unknown                                                 623313
+alveolar macrophage                                     276291
+pulmonary alveolar type 2 cell                          156713
+respiratory basal cell                                  119037
+CD8-positive, alpha-beta T cell                         113641
+multiciliated columnar cell of tracheobronchial tree    113074
+elicited macrophage                                      88248
+CD4-positive, alpha-beta T cell                          82698
+nasal mucosa goblet cell                                 72359
+classical monocyte                                       64980
+capillary endothelial cell                               64583
+club cell                                                50366
+natural killer cell                                      48439
+alveolar type 1 fibroblast cell                          38328
+B cell                                                   37863
+plasma cell                                              33111
+pulmonary alveolar type 1 cell                           32306
+vein endothelial cell                                    31951
+CD1c-positive myeloid dendritic cell                     31359
+non-classical monocyte                                   26257
+alveolar adventitial fibroblast                          23259
+mast cell                                                21323
+tracheobronchial smooth muscle cell                      16676
+pulmonary artery endothelial cell                        14724
+endothelial cell of lymphatic vessel                     13067
+lung pericyte                                            11680
+epithelial cell of lower respiratory tract               11124
+bronchus fibroblast of lung                               8750
+lung macrophage                                           8317
+multiciliated epithelial cell                             7421
+respiratory tract hillock cell                            5570
+T cell                                                    3908
+plasmacytoid dendritic cell                               3686
+tracheobronchial serous cell                              2775
+serous secreting cell                                     2504
+dendritic cell                                            2360
+mesothelial cell                                          2261
+bronchial goblet cell                                     2257
+myofibroblast cell                                        1926
+pulmonary alveolar epithelial cell                        1815
+ionocyte                                                  1803
+acinar cell                                               1799
+smooth muscle cell                                        1788
+conventional dendritic cell                               1446
+mucus secreting cell                                      1425
+tracheobronchial goblet cell                              1190
+fibroblast                                                1093
+brush cell of tracheobronchial tree                        641
+stromal cell                                               553
+pulmonary neuroendocrine cell                              328
+hematopoietic stem cell                                     61
+Name: count, dtype: int64
+Subject type
+subject_type
+organ_donor      333516
+alive_healthy    190446
+alive_disease     60982
+Name: count, dtype: int64
+
+====================================
+CPU Time used: 00:04:29
+CPU Percent: 78%
+Memory usage: 89613436kb
+Approx Power usage: 0.0008
+Walltime usage: 00:08:03
+
+====================================

HPC_jobs/HLCA_full_col_names.txt

@@ -0,0 +1,34 @@
+Index(['suspension_type', 'donor_id', 'is_primary_data',
+       'assay_ontology_term_id', 'cell_type_ontology_term_id',
+       'development_stage_ontology_term_id', 'disease_ontology_term_id',
+       'self_reported_ethnicity_ontology_term_id', 'tissue_ontology_term_id',
+       'sex_ontology_term_id', '3'_or_5'', 'BMI', 'age_or_mean_of_age_range',
+       'age_range', 'anatomical_region_ccf_score',
+       'ann_coarse_for_GWAS_and_modeling', 'ann_finest_level', 'ann_level_1',
+       'ann_level_2', 'ann_level_3', 'ann_level_4', 'ann_level_5',
+       'cause_of_death', 'core_or_extension', 'dataset', 'fresh_or_frozen',
+       'log10_total_counts', 'lung_condition', 'mixed_ancestry',
+       'original_ann_level_1', 'original_ann_level_2', 'original_ann_level_3',
+       'original_ann_level_4', 'original_ann_level_5',
+       'original_ann_nonharmonized', 'reannotation_type', 'sample',
+       'scanvi_label', 'sequencing_platform', 'smoking_status', 'study',
+       'subject_type', 'tissue_coarse_unharmonized',
+       'tissue_detailed_unharmonized', 'tissue_dissociation_protocol',
+       'tissue_level_2', 'tissue_level_3', 'tissue_sampling_method',
+       'total_counts', 'transf_ann_level_1_label', 'transf_ann_level_1_uncert',
+       'transf_ann_level_2_label', 'transf_ann_level_2_uncert',
+       'transf_ann_level_3_label', 'transf_ann_level_3_uncert',
+       'transf_ann_level_4_label', 'transf_ann_level_4_uncert',
+       'transf_ann_level_5_label', 'transf_ann_level_5_uncert', 'tissue_type',
+       'cell_type', 'assay', 'disease', 'sex', 'tissue',
+       'self_reported_ethnicity', 'development_stage', 'observation_joinid'],
+      dtype='object')
+
+====================================
+CPU Time used: 00:04:22
+CPU Percent: 86%
+Memory usage: 73536196kb
+Approx Power usage: 0.0008
+Walltime usage: 00:08:49
+
+====================================

HPC_jobs/HLCA_full_tissue_type.txt

@@ -0,0 +1,141 @@
+tissue_coarse_unharmonized distribution:
+tissue_coarse_unharmonized
+parenchyma              455629
+airway                  218015
+nose                    160397
+Trachea                  20411
+Intermediate Bronchi     18658
+Inferior turbinate       18126
+Distal Bronchi           17292
+segmental_bronchi         9010
+Name: count, dtype: int64
+----------
+tissue_detailed_unharmonized distribution:
+tissue_detailed_unharmonized
+Lower left lobe                                       68321
+right bronchus (intermediate)                         55806
+right inferior turbinate                              45557
+side unknown, lower lobe                              41220
+trachea                                               33593
+right upper lobe                                      29677
+Upper left lobe                                       25485
+Trachea                                               24166
+parenchyma                                            22734
+lobar_bronchi                                         22160
+left lung, superior lobe, apical-posterior segment    18612
+right middle lobe, alveolar region                    18541
+4th generation airway                                 18038
+Parenchyma                                            17222
+2nd-3rd generation airway                             16980
+left lower lobe, alveolar region                      16898
+Lower inferior turbinate                              14596
+anterior carina                                       11549
+right middle lobe, bronchioles                        10251
+upper left lobe                                        9822
+left main bronchus                                     9417
+lower right lobe                                       9137
+segmental_bronchi                                      9010
+posterior carina                                       8862
+right lobe                                             7821
+left lower lobe, bronchi                               7769
+lower left lobe                                        7718
+left upper lobe, alveolar region                       7523
+left lobe                                              7122
+middle right lobe                                      5976
+upper lobe                                             5923
+upper right lobe                                       5619
+parenchyma left lower lobe                             5348
+left lower lobe                                        4516
+lower lobe                                             4249
+Lower right lobe                                       4146
+right lower lobe                                       3966
+## lobe                                                3183
+parenchyma right upper lobe                            2269
+Bronchi                                                 445
+parenchyma right middle lobe                            399
+Name: count, dtype: int64
+----------
+tissue_dissociation_protocol distribution:
+tissue_dissociation_protocol
+Collagenase D + DNAse             224250
+Dispase + collagenase             121894
+Cold protease 1h                  120044
+Collagenase + Elastase + DNAse     60982
+Cold protease overnight            33593
+Collagenase A + DNAse              24181
+Name: count, dtype: int64
+----------
+tissue_level_2 distribution:
+tissue_level_2
+parenchyma lower lobe           118634
+inferior turbinate               78279
+lobular bronchi                  74464
+parenchyma upper lobe            61735
+trachea                          54004
+distal lobular airways           27543
+parenchyma right middle lobe     18541
+left or right main bronchus       9417
+segmental bronchi                 7769
+Name: count, dtype: int64
+----------
+tissue_level_3 distribution:
+tissue_level_3
+parenchyma left lower lobe          65053
+inferior turbinate right nostril    53378
+parenchyma right upper lobe         29677
+parenchyma left upper lobe          26135
+distal airways middle right lobe    16227
+parenchyma right lower lobe          8112
+inferior turbinate left nostril      7122
+lobular bronchi upper left lobe      7049
+lobular bronchi lower right lobe     5056
+distal airways lower right lobe      4081
+lobular bronchi lower left lobe      3411
+lobular bronchi upper right lobe     3142
+distal airways upper left lobe       2773
+distal airways upper right lobe      2477
+distal airways lower left lobe       1985
+Name: count, dtype: int64
+----------
+tissue_sampling_method distribution:
+tissue_sampling_method
+donor_lung                  702873
+lung_explant                428431
+surgical_resection          292307
+autopsy                     205374
+balf                        202476
+brush                       164056
+biopsy                      100467
+scraping                     82297
+balf_cryopreserved           51689
+donor_lung_cryopreserved     46500
+autopsy_cryopreserved         4366
+biopsy_cryopreserved          1611
+Name: count, dtype: int64
+----------
+tissue distribution:
+tissue
+lung                  1506615
+lung parenchyma        356202
+respiratory airway     304611
+nose                   115019
+Name: count, dtype: int64
+----------
+ann_level_1 distribution:
+ann_level_1
+Immune         1222972
+Epithelial      705307
+Stroma          181028
+Endothelial     156585
+Unknown          16555
+Name: count, dtype: int64
+----------
+
+====================================
+CPU Time used: 00:04:34
+CPU Percent: 70%
+Memory usage: 88996004kb
+Approx Power usage: 0.0008
+Walltime usage: 00:09:22
+
+====================================

HPC_jobs/hlca_full.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#PBS -l walltime=5:0:0
+#PBS -lselect=1:ncpus=16:mem=512gb
+
+# Load production tools
+module load tools/prod
+
+# Load python and bundle
+module load Biopython/1.84-foss-2024a
+
+# Change to directory
+cd /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline
+
+# Activate virtual environment
+source xenium_5k_venv/bin/activate
+
+# Run
+#python /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline/src/manual_src/hlca_full_filt_process.py
+python /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline/src/manual_src/umap_hlca.py

HPC_jobs/hpc_run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#PBS -l walltime=5:0:0
+#PBS -lselect=1:ncpus=16:mem=512gb
+
+# Load production tools
+module load tools/prod
+
+# Load python and bundle
+module load Biopython/1.84-foss-2024a
+
+# Change to directory
+cd /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline
+
+# Activate virtual environment
+source xenium_5k_venv/bin/activate
+
+# Run
+python -m recode_st config_files/config_HPC_dev.toml

HPC_jobs/hpc_run_analysis.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#PBS -l walltime=1:0:0
+#PBS -lselect=1:ncpus=8:mem=256gb
+
+# Load production tools
+module load tools/prod
+
+# Load python and bundle
+module load Biopython/1.84-foss-2024a
+
+# Change to directory
+cd /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline
+
+# Activate virtual environment
+source xenium_5k_venv/bin/activate
+
+# Run
+python -m recode_st config_files/config_HPC_drug2cell.toml

HPC_jobs/hpc_run_format.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#PBS -l walltime=5:0:0
+#PBS -lselect=1:ncpus=16:mem=512gb
+
+# Load production tools
+module load tools/prod
+
+# Load python and bundle
+module load Biopython/1.84-foss-2024a
+
+# Change to directory
+cd /rds/general/user/sep22/home/Projects/Xenium_5k_analysis_pipeline
+
+# Activate virtual environment
+source xenium_5k_venv/bin/activate
+
+# Run
+python -m recode_st config_files/config_HPC_format.toml