Merge pull request MIT-LCP#2063 from Chessing234/fix/sqlite-import-limit-and-strip

Fix sqlite --limit on chunks + strip() CSV table names

Author: alistairewj

mimic-iii/buildmimic/oracle/add_oracle_rowdelimiter.py

@@ -63,7 +63,11 @@ def main(argv):
         print('Cannot find input file {}'.format(fn_in))
         sys.exit(2)
 
-    fn_out=fn_in.strip('.csv')+'_output.csv'
+    fn_out = (
+        fn_in[: -len(".csv")] + "_output.csv"
+        if fn_in.lower().endswith(".csv")
+        else fn_in + "_output.csv"
+    )
     print('\n'+'~'*40)
     print('Input filename = {}'.format(fn_in))
     print('Delimiter = {}'.format(delimiter))

mimic-iii/buildmimic/sqlite/import.py

@@ -10,22 +10,32 @@
 CHUNKSIZE = 10 ** 6
 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME)
 
+
+def _table_name_from_csv(filename: str) -> str:
+    """Derive SQL table name from a CSV path (literal suffix, not str.strip)."""
+    name = filename
+    for suffix in (".csv.gz", ".csv"):
+        if name.lower().endswith(suffix):
+            name = name[: -len(suffix)]
+            break
+    return name.lower()
+
+
 if os.path.exists(DATABASE_NAME):
     msg = "File {} already exists.".format(DATABASE_NAME)
     print(msg)
     sys.exit()
 
 for f in glob("*.csv.gz"):
     print("Starting processing {}".format(f))
+    table = _table_name_from_csv(f)
     if os.path.getsize(f) < THRESHOLD_SIZE:
         df = pd.read_csv(f, index_col="ROW_ID")
-        df.to_sql(f.strip(".csv.gz").lower(), CONNECTION_STRING)
+        df.to_sql(table, CONNECTION_STRING)
     else:
         # If the file is too large, let's do the work in chunks
         for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
-            chunk.to_sql(
-                f.strip(".csv.gz").lower(), CONNECTION_STRING, if_exists="append"
-            )
+            chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
     print("Finished processing {}".format(f))
 
 print("Should be all done!")

mimic-iv/buildmimic/sqlite/import.py

@@ -165,7 +165,7 @@ def main():
             else:
                 # If the file is too large, let's do the work in chunks
                 for chunk in pd.read_csv(f, chunksize=CHUNKSIZE, low_memory=False, dtype=mimic_dtypes):
-                    chunk = process_dataframe(chunk)
+                    chunk = process_dataframe(chunk, subjects=subjects)
                     chunk.to_sql(tablename, connection, if_exists="append", index=False)
                     row_counts[tablename] += len(chunk)
             print("done!")