Merge pull request #170 from ncsa/feature/integrate_parallel_into_sim

Feature/integrate parallel into sim

Author: joshfactorial

config_template/simple_template.yml

@@ -26,12 +26,8 @@ rng_seed: .
 min_mutations: .
 overwrite_output: .
 
-outdir: .
-final_prefix: .
-by: .
+mode: .
 size: .
 jobs: .
-neat_cmd: .
-samtools: .
 cleanup_splits: .
 reuse_splits: .

config_template/template_neat_config.yml

@@ -140,43 +140,38 @@ min_mutations: .
 # type: bool | required = no | default = false
 overwrite_output: .
 
-# Top-level output directory for splits, per-chunk outputs, and stitched results.
-# Relative paths are interpreted against the CURRENT WORKING DIRECTORY.
-# If omitted (or set to .), it defaults to: <cwd>/<config_stem>_parallel
-# type = string | required: no
-outdir: .
-
-# Location (prefix, no extension) for stitched outputs.
-# If relative, it is resolved under outdir (i.e., <outdir>/<final_prefix>*).
-# Default is "stitched/final".
-# type = string | required: no | default = stitched/final
-final_prefix: .
-
 # How to split the input reference for parallelization
+# If parallel_mode = false, this option has no effect.
 # type = string | required: no | default = contig | values: contig, size
-by: .
+mode: .
 
 # Target chunk size if by = size (overlap = read_len * 2).
-# Default is 500000 when by = size.
+# Default is 500000 when by = size. Not used for by = contig.
 # type = int | required: no | default = 500000 (when by=size)
 size: .
 
-# Maximum number of concurrent NEAT jobs
-# type = int | required: no | default = (CPU count)
-jobs: .
-
-# Command used to launch the simulator (CLI mode)
-# type = string | required: no | default = "neat read-simulator"
-neat_cmd: .
-
-# Path to samtools (binary name if on PATH)
-# type = string | required: no | default = samtools
-samtools: .
+# Maximum number of concurrent NEAT jobs (threads or hyperthreads) to run.
+# Note that assigning a value higher than your available number of threads
+# could result in performance issues in the computer running the job.
+# If parallel_mode = false, this option has no effect.
+# type = int | required: no | default = all available.
+threads: .
 
 # Delete the 'splits' directory after stitching completes
+# Note if parallel_mode = false, this option has no effect.
 # type = bool | required: no | default = false
 cleanup_splits: .
 
-# Reuse existing files in 'splits' and skip the split step
-# type = bool | required: no | default = false
+# Reuse existing files in 'splits' and skip the split step. This
+# option should be a path to the directory containing the splits.
+# Note if parallel_mode = None, this option has no effect.
+# type = str | required: no | default = None
 reuse_splits: .
+
+# For recombining bams, you will need to have samtools installed
+# if no installation path is specified here, Python will check
+# for an installation of samtools in PATH. If Python can't
+# find it, it will throw an error and exit.
+# Note if parallel_mode = None, or produce_bam = false, this option has no effect,
+#   and will not be checked
+samtools: .

neat/cli/cli.py

@@ -138,7 +138,10 @@ def main(parser: argparse.ArgumentParser, arguments: list[str]) -> int:
         else:
             end = time.time()
             log.info(
-                f"command finished successfully; execution took {(end - start)/60:.2f} m"
+                f"command finished successfully; execution took {(end - start)*1000:.2f} ms\nd"
+            )
+            log.info(
+                f"command finished successfully; execution took {(end - start) / 60:.2f} m"
             )
         return 0
 

neat/cli/commands/options.py

@@ -4,14 +4,25 @@
 
 __all__ = ["output_group"]
 
+import os
+
 from .base import Group
 
 output_group = Group("output", is_mutually_exclusive=True, required=True)
 output_group.add_argument(
     "-o",
-    "--output",
-    dest="output",
+    "--output_dir",
+    dest="output_dir",
     type=str,
-    help="Path (including filename prefix) to the output file",
-    default=None
+    help="Path to the output directory. Will create if not present.",
+    default=os.getcwd()
 )
+
+output_group.add_argument(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    type=str,
+    help="Prefix to use to name files",
+    default="neat_sim"
+)

neat/cli/commands/parallel.py

@@ -44,4 +44,4 @@ def execute(self, arguments: argparse.Namespace):
 
         :param arguments: The namespace with arguments and their values.
         """
-        read_simulator_runner(arguments.config, arguments.output)
+        read_simulator_runner(arguments.config, arguments.output_dir, arguments.prefix)

neat/cli/commands/read_simulator.py

@@ -182,4 +182,4 @@ def validate_output_path(path: str | Path, is_file: bool = True, overwrite: bool
                 _LOG.error(f"cannot write to '{path}', access denied")
                 sys.exit(11)
         else:
-            path.parent.mkdir(parents=True, exist_ok=True)
+            path.mkdir(parents=True, exist_ok=True)