CNAM-153 reorganizes config files based on env

Author: danielpes

src/main/resources/config/filtering-cmap.conf

@@ -0,0 +1,25 @@
+env_name = "cmap"
+
+paths = {
+  input = {
+    dcir = "/shared/flat_data/joins/DCIR"
+    pmsi_mco = "/shared/flat_data/joins/MCO"
+    pmsi_had = "/shared/flat_data/joins/HAD"
+    pmsi_ssr = "/shared/flat_data/joins/SSR"
+    ir_ben = "/shared/flat_data/IR_BEN_R"
+    ir_imb = "/shared/flat_data/IR_IMB_R"
+    ir_pha = "/shared/value_tables/IR_PHA_R"
+    dosages = "/shared/value_tables/DOSE_PER_MOLECULE.CSV"
+  }
+  output = {
+    root = "/shared/filtered_data"
+    patients = "/shared/filtered_data/patients"
+    flat_events = "/shared/filtered_data/flat_events"
+  }
+}
+
+dates = {
+  age_reference = [2010, 12, 31, 23, 59, 59]
+  study_start = [2010, 1, 1]
+  study_end = [2014, 12, 31, 23, 59, 59]
+}

src/main/resources/config/filtering-cnam.conf

@@ -0,0 +1,25 @@
+env_name = "cnam"
+
+paths = {
+  input = {
+    dcir = "/shared/flat_data/joins/DCIR"
+    pmsi_mco = "/shared/flat_data/joins/MCO"
+    pmsi_had = "/shared/flat_data/joins/HAD"
+    pmsi_ssr = "/shared/flat_data/joins/SSR"
+    ir_ben = "/shared/flat_data/IR_BEN_R"
+    ir_imb = "/shared/flat_data/IR_IMB_R"
+    ir_pha = "/shared/value_tables/IR_PHA_R"
+    dosages = "/shared/value_tables/DOSE_PER_MOLECULE.CSV"
+  }
+  output = {
+    root = "/shared/filtered_data"
+    patients = "/shared/filtered_data/patients"
+    flat_events = "/shared/filtered_data/flat_events"
+  }
+}
+
+dates = {
+  age_reference = [2006, 12, 31, 23, 59, 59]
+  study_start = [2006, 1, 1]
+  study_end = [2009, 12, 31, 23, 59, 59]
+}

src/main/resources/config/filtering-default.conf

@@ -0,0 +1,97 @@
+default = {
+  env_name = "test"
+  drug_categories = ["A10"]
+  cancer_definition = "broad"
+  disease_code = "C67"
+  mco_death_code = 9
+
+  limits = {
+    min_year = 1900
+    max_year = 2020
+    min_month = 1
+    max_month = 12
+    min_gender = 1
+    max_gender = 2
+    min_age = 18
+    max_age = 120
+  }
+
+  cox_hypothesis = {
+    trackloss = {
+      threshold = 4
+      delay = 2
+    }
+    follow-up = {
+      delay = 6
+    }
+    observation = {
+      start = "first_molecule" # or "study_start"
+      end = "study_end"
+    }
+    exposures = {
+      min_purchases = 2
+      start_delay = 3
+      purchases_window = 6
+      only_first = false
+      delayed_entries = {
+        filter = true
+        delay = 12
+      }
+      diagnosed_patients = {
+        filter = true
+        delay = 0
+      }
+    }
+  }
+
+  ltsccs_hypothesis = {
+    observation = {
+      start = "first_molecule" # "study_start"
+      end = "study_end"
+    }
+    exposures = {
+      min_purchases = 2
+      start_delay = 3
+      purchases_window = 6
+      only_first = false
+      delayed_entries = {
+        filter = true
+        delay = 12
+      }
+      diagnosed_patients = {
+        filter = true
+        delay = 0
+      }
+    }
+  }
+
+  mlpp_hypothesis = {
+    observation = {
+      start = "first_molecule" # "study_start"
+      end = "study_end"
+    }
+    exposures = {
+      min_purchases = 2
+      start_delay = 3
+      purchases_window = 6
+      only_first = false
+      delayed_entries = {
+        filter = true
+        delay = 12
+      }
+      diagnosed_patients = {
+        filter = true
+        delay = 0
+      }
+    }
+  }
+}
+
+cnam = ${default}
+cnam = include "filtering-cnam.conf"
+
+cmap = ${default}
+cmap = include "filtering-cmap.conf"
+
+test = ${default}
+test = include "filtering-test.conf"

src/main/resources/config/filtering-test.conf

@@ -0,0 +1,25 @@
+env_name = "test"
+
+paths = {
+  input = {
+    dcir = "src/test/resources/test-input/DCIR.parquet"
+    pmsi_mco = "src/test/resources/test-input/MCO.parquet"
+    pmsi_had = "src/test/resources/test-input/HAD.parquet"
+    pmsi_ssr = "src/test/resources/test-input/SSR.parquet"
+    ir_ben = "src/test/resources/test-input/IR_BEN_R.parquet"
+    ir_imb = "src/test/resources/test-input/IR_IMB_R.parquet"
+    ir_pha = "src/test/resources/test-input/IR_PHA_R.parquet"
+    dosages = "src/test/resources/test-input/DOSE_PER_MOLECULE.CSV"
+  }
+  output = {
+    root = "target/test/output"
+    patients = "target/test/output/patients"
+    flat_events = "target/test/output/flat_events"
+  }
+}
+
+dates = {
+  age_reference = [2006, 12, 31, 23, 59, 59]
+  study_start = [2006, 1, 1]
+  study_end = [2009, 12, 31, 23, 59, 59]
+}

src/main/resources/filtering-default.conf

@@ -43,7 +43,7 @@ object FilteringConfig {
     val configPath: String = sqlContext.getConf("conf", "")
     val environment: String = sqlContext.getConf("env", "test")
 
-    val defaultConfig = ConfigFactory.parseResources("filtering-default.conf").resolve().getConfig(environment)
+    val defaultConfig = ConfigFactory.parseResources("config/filtering-default.conf").resolve().getConfig(environment)
     val newConfig = ConfigFactory.parseFile(new java.io.File(configPath))
 
     newConfig.withFallback(defaultConfig).resolve()

src/main/scala/fr/polytechnique/cmap/cnam/filtering/FilteringConfig.scala

@@ -19,14 +19,14 @@ class Sources(
 
 object Sources {
   def apply(
-    dcir: Option[DataFrame] = None,
-    pmsiMco: Option[DataFrame] = None,
-    pmsiHad: Option[DataFrame] = None,
-    pmsiSsr: Option[DataFrame] = None,
-    irBen: Option[DataFrame] = None,
-    irImb: Option[DataFrame] = None,
-    irPha: Option[DataFrame] = None,
-    dosages: Option[DataFrame] = None) = {
+      dcir: Option[DataFrame] = None,
+      pmsiMco: Option[DataFrame] = None,
+      pmsiHad: Option[DataFrame] = None,
+      pmsiSsr: Option[DataFrame] = None,
+      irBen: Option[DataFrame] = None,
+      irImb: Option[DataFrame] = None,
+      irPha: Option[DataFrame] = None,
+      dosages: Option[DataFrame] = None) = {
     new Sources(dcir, pmsiMco, pmsiHad, pmsiSsr, irBen, irImb, irPha, dosages)
   }
 }

src/main/scala/fr/polytechnique/cmap/cnam/filtering/Sources.scala

@@ -95,7 +95,7 @@ object CoxMain extends Main {
         case Array(arg1, args2) => (args(0), args(1), true)
         case _ => ("test", "broad", true)
       }
-    val config: Config = ConfigFactory.parseResources("filtering-default.conf").getConfig(environment)
+    val config: Config = ConfigFactory.parseResources("config/filtering-default.conf").getConfig(environment)
     coxFeaturing(sqlContext, config, cancerDefinition, filterDelayedPatients)
     stopContext()
   }

src/main/scala/fr/polytechnique/cmap/cnam/filtering/cox/CoxMain.scala

@@ -57,7 +57,7 @@ object LTSCCSMain extends Main {
   override def main(args: Array[String]): Unit = {
     startContext()
     val environment = if (args.nonEmpty) args(0) else "test"
-    val config: Config = ConfigFactory.parseResources("filtering-default.conf").getConfig(environment)
+    val config: Config = ConfigFactory.parseResources("config/filtering-default.conf").getConfig(environment)
     runLTSCCS(sqlContext, config)
     stopContext()
   }