diff --git a/mimic-iii/buildmimic/oracle/add_oracle_rowdelimiter.py b/mimic-iii/buildmimic/oracle/add_oracle_rowdelimiter.py index 39f85294e..adb5fc8a2 100644 --- a/mimic-iii/buildmimic/oracle/add_oracle_rowdelimiter.py +++ b/mimic-iii/buildmimic/oracle/add_oracle_rowdelimiter.py @@ -63,7 +63,11 @@ def main(argv): print('Cannot find input file {}'.format(fn_in)) sys.exit(2) - fn_out=fn_in.strip('.csv')+'_output.csv' + fn_out = ( + fn_in[: -len(".csv")] + "_output.csv" + if fn_in.lower().endswith(".csv") + else fn_in + "_output.csv" + ) print('\n'+'~'*40) print('Input filename = {}'.format(fn_in)) print('Delimiter = {}'.format(delimiter)) diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 3e5cfc377..801651c7a 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,17 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) + +def _table_name_from_csv(filename: str) -> str: + """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" + name = filename + for suffix in (".csv.gz", ".csv"): + if name.lower().endswith(suffix): + name = name[: -len(suffix)] + break + return name.lower() + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) @@ -17,15 +28,14 @@ for f in glob("*.csv.gz"): print("Starting processing {}".format(f)) + table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: df = pd.read_csv(f, index_col="ROW_ID") - df.to_sql(f.strip(".csv.gz").lower(), CONNECTION_STRING) + df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): - chunk.to_sql( - f.strip(".csv.gz").lower(), CONNECTION_STRING, if_exists="append" - ) + chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) print("Should be all done!") diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 1cb8eb0d2..68d436479 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -165,7 +165,7 @@ def main(): else: # If the file is too large, let's do the work in chunks for chunk in pd.read_csv(f, chunksize=CHUNKSIZE, low_memory=False, dtype=mimic_dtypes): - chunk = process_dataframe(chunk) + chunk = process_dataframe(chunk, subjects=subjects) chunk.to_sql(tablename, connection, if_exists="append", index=False) row_counts[tablename] += len(chunk) print("done!")