WIP: parametric sweeps and redo generate and submit

Author: kks32

README.md

@@ -79,14 +79,14 @@ client = DSClient()
 client.systems.establish_credentials("frontera")
 
 # Submit a job
-job_request = client.jobs.generate_request(
+job_request = client.jobs.generate(
     app_id="matlab-r2023a",
     input_dir_uri="/MyData/analysis/input/",
     script_filename="run_analysis.m",
     max_minutes=30,
     allocation="your_allocation"
 )
-job = client.jobs.submit_request(job_request)
+job = client.jobs.submit(job_request)
 final_status = job.monitor()
 ```
 

_toc.yml

@@ -20,13 +20,15 @@ parts:
       - file: docs/examples/mpm
       - file: docs/examples/opensees
       - file: docs/examples/openfoam
+      - file: docs/examples/pylauncher
       - file: docs/examples/tms_credentials
       - file: docs/examples/database
   - caption: API Reference
     chapters:
       - file: docs/api/index
       - file: docs/api/client
       - file: docs/api/jobs
+      - file: docs/api/launcher
       - file: docs/api/files
       - file: docs/api/apps
       - file: docs/api/systems

dapi/__init__.py

@@ -35,12 +35,12 @@
     >>> files = client.files.list("/MyData/uploads/")
     
     >>> # Job submission and monitoring
-    >>> job_request = client.jobs.generate_request(
+    >>> job_request = client.jobs.generate(
     ...     app_id="matlab-r2023a",
     ...     input_dir_uri="/MyData/analysis/input/",
     ...     script_filename="run_analysis.m"
     ... )
-    >>> job = client.jobs.submit_request(job_request)
+    >>> job = client.jobs.submit(job_request)
     >>> final_status = job.monitor()
     
     >>> # Database access

dapi/client.py

@@ -5,6 +5,7 @@
 from . import files as files_module
 from . import jobs as jobs_module
 from . import systems as systems_module
+from . import launcher as launcher_module
 from .db.accessor import DatabaseAccessor
 
 # Import only the necessary classes/functions from jobs
@@ -366,6 +367,103 @@ def revoke_credentials(
         )
 
 
+class ParametricSweepMethods:
+    """Interface for PyLauncher parameter sweeps.
+
+    - ``generate`` — preview (``preview=True``) or write sweep files.
+    - ``submit`` — submit the sweep job to TACC.
+    """
+
+    def __init__(self, tapis_client):
+        self._tapis = tapis_client
+
+    def generate(
+        self,
+        base_command: str,
+        sweep: Dict[str, Any],
+        directory: str = None,
+        *,
+        placeholder_style: str = "token",
+        debug: str = None,
+        preview: bool = False,
+    ):
+        """Generate PyLauncher sweep files or preview the parameter grid.
+
+        With ``preview=True``, returns a DataFrame of all parameter
+        combinations — no files are written.
+
+        Otherwise, expands *base_command* into one command per combination
+        and writes ``runsList.txt`` and ``call_pylauncher.py`` into
+        *directory*. Returns the list of generated commands.
+
+        Args:
+            base_command: Command template with placeholders matching sweep keys.
+            sweep: Mapping of placeholder name to sequence of values.
+            directory: Directory to write files into (created if needed).
+                Required when *preview* is ``False``.
+            placeholder_style: ``"token"`` (default) for bare ``ALPHA``,
+                or ``"braces"`` for ``{ALPHA}``.
+            debug: Optional debug string (e.g. ``"host+job"``).
+            preview: If ``True``, return a DataFrame (dry run).
+
+        Returns:
+            ``List[str]`` of commands, or ``pandas.DataFrame`` when
+            *preview* is ``True``.
+        """
+        return launcher_module.generate_sweep(
+            base_command, sweep, directory,
+            placeholder_style=placeholder_style, debug=debug, preview=preview,
+        )
+
+    def submit(
+        self,
+        directory: str,
+        app_id: str,
+        allocation: str,
+        *,
+        node_count: Optional[int] = None,
+        cores_per_node: Optional[int] = None,
+        max_minutes: Optional[int] = None,
+        queue: Optional[str] = None,
+        **kwargs,
+    ):
+        """Submit a PyLauncher sweep job.
+
+        Translates *directory* to a Tapis URI, builds a job request with
+        ``call_pylauncher.py`` as the script, and submits it.
+
+        Args:
+            directory: Path to the input directory containing
+                ``runsList.txt`` and ``call_pylauncher.py``
+                (e.g. ``"/MyData/sweep/"``).
+            app_id: Tapis application ID (e.g. ``"openseespy-s3"``).
+            allocation: TACC allocation to charge.
+            node_count: Number of compute nodes.
+            cores_per_node: Cores per node.
+            max_minutes: Maximum runtime in minutes.
+            queue: Execution queue name.
+            **kwargs: Additional arguments passed to
+                ``ds.jobs.generate()``.
+
+        Returns:
+            SubmittedJob: A job object for monitoring via ``.monitor()``.
+        """
+        input_uri = files_module.get_ds_path_uri(self._tapis, directory)
+        job_request = jobs_module.generate_job_request(
+            tapis_client=self._tapis,
+            app_id=app_id,
+            input_dir_uri=input_uri,
+            script_filename="call_pylauncher.py",
+            node_count=node_count,
+            cores_per_node=cores_per_node,
+            max_minutes=max_minutes,
+            queue=queue,
+            allocation=allocation,
+            **kwargs,
+        )
+        return jobs_module.submit_job_request(self._tapis, job_request)
+
+
 class JobMethods:
     """Interface for Tapis job submission, monitoring, and management.
 
@@ -374,6 +472,10 @@ class JobMethods:
 
     Args:
         tapis_client (Tapis): Authenticated Tapis client instance.
+
+    Attributes:
+        parametric_sweep (ParametricSweepMethods): Interface for PyLauncher
+            parameter sweep generation.
     """
 
     def __init__(self, tapis_client: Tapis):
@@ -383,9 +485,10 @@ def __init__(self, tapis_client: Tapis):
             tapis_client (Tapis): Authenticated Tapis client instance.
         """
         self._tapis = tapis_client
+        self.parametric_sweep = ParametricSweepMethods(tapis_client)
 
     # Method to generate the request dictionary
-    def generate_request(
+    def generate(
         self,
         app_id: str,
         input_dir_uri: str,
@@ -456,7 +559,7 @@ def generate_request(
             JobSubmissionError: If job request generation fails.
 
         Example:
-            >>> job_request = ds.jobs.generate_request(
+            >>> job_request = ds.jobs.generate(
             ...     app_id="matlab-r2023a",
             ...     input_dir_uri="tapis://designsafe.storage.default/username/input/",
             ...     script_filename="run_analysis.m",
@@ -491,27 +594,26 @@ def generate_request(
         )
 
     # Method to submit the generated request dictionary
-    def submit_request(self, job_request: Dict[str, Any]) -> SubmittedJob:
-        """Submit a pre-generated job request dictionary to Tapis.
+    def submit(self, job_request: Dict[str, Any]) -> SubmittedJob:
+        """Submit a job request dictionary to Tapis.
 
-        This method takes a complete job request dictionary (typically generated
-        by generate_request) and submits it to Tapis for execution.
+        Takes a job request dictionary (typically from ``generate()``) and
+        submits it to Tapis for execution.
 
         Args:
-            job_request (Dict[str, Any]): Complete job request dictionary containing
-                all necessary job parameters and configuration.
+            job_request (Dict[str, Any]): Complete job request dictionary.
 
         Returns:
             SubmittedJob: A SubmittedJob object for monitoring and managing the job.
 
         Raises:
             ValueError: If job_request is not a dictionary.
-            JobSubmissionError: If the Tapis submission fails or encounters an error.
+            JobSubmissionError: If the Tapis submission fails.
 
         Example:
-            >>> job_request = ds.jobs.generate_request(...)
-            >>> submitted_job = ds.jobs.submit_request(job_request)
-            >>> print(f"Job submitted with UUID: {submitted_job.uuid}")
+            >>> job_request = ds.jobs.generate(...)
+            >>> job = ds.jobs.submit(job_request)
+            >>> print(f"Job submitted with UUID: {job.uuid}")
         """
         return jobs_module.submit_job_request(self._tapis, job_request)
 

dapi/exceptions.py

@@ -155,7 +155,7 @@ class JobSubmissionError(DapiException):
 
     Example:
         >>> try:
-        ...     job = client.jobs.submit_request(invalid_job_request)
+        ...     job = client.jobs.submit(invalid_job_request)
         ... except JobSubmissionError as e:
         ...     print(f"Job submission failed: {e}")
         ...     if e.response:

dapi/launcher.py

@@ -0,0 +1,139 @@
+"""PyLauncher parameter sweep utilities for DesignSafe.
+
+This module provides functions for generating parameter sweeps and writing
+PyLauncher input files. These are pure local operations — PyLauncher itself
+runs on TACC compute nodes, not locally.
+
+Functions:
+    generate_sweep: Generate sweep commands and optionally write PyLauncher input files.
+"""
+
+from __future__ import annotations
+
+from itertools import product
+from pathlib import Path
+from typing import Any, List, Mapping, Sequence, Union
+
+import pandas as pd
+
+
+def _validate_sweep(sweep: Mapping[str, Sequence[Any]]) -> None:
+    """Validate sweep values are non-empty, non-string sequences."""
+    for k, vals in sweep.items():
+        if not isinstance(vals, Sequence) or isinstance(vals, (str, bytes)):
+            raise TypeError(f"sweep[{k!r}] must be a non-string sequence of values.")
+        if len(vals) == 0:
+            raise ValueError(f"sweep[{k!r}] is empty; provide at least one value.")
+
+
+def _expand_commands(
+    base_command: str,
+    sweep: Mapping[str, Sequence[Any]],
+    placeholder_style: str,
+) -> List[str]:
+    """Expand a command template into all parameter combinations."""
+    if not sweep:
+        return [base_command]
+
+    _validate_sweep(sweep)
+
+    if placeholder_style not in ("token", "braces"):
+        raise ValueError("placeholder_style must be 'token' or 'braces'.")
+
+    keys = list(sweep.keys())
+    commands: List[str] = []
+    for combo in product(*[sweep[k] for k in keys]):
+        cmd = base_command
+        for k, v in zip(keys, combo):
+            if placeholder_style == "token":
+                cmd = cmd.replace(k, str(v))
+            else:
+                cmd = cmd.replace("{" + k + "}", str(v))
+        commands.append(cmd)
+
+    return commands
+
+
+def generate_sweep(
+    base_command: str,
+    sweep: Mapping[str, Sequence[Any]],
+    directory: Union[str, Path, None] = None,
+    *,
+    placeholder_style: str = "token",
+    debug: str | None = None,
+    preview: bool = False,
+) -> Union[List[str], pd.DataFrame]:
+    """Generate sweep commands and write PyLauncher input files.
+
+    When *preview* is ``True``, returns a DataFrame of all parameter
+    combinations without writing any files — useful for inspecting the
+    sweep in a notebook before committing.
+
+    When *preview* is ``False`` (default), expands *base_command* into one
+    command per parameter combination and writes ``runsList.txt`` and
+    ``call_pylauncher.py`` into *directory*.
+
+    Args:
+        base_command: Command template containing placeholders that match
+            keys in *sweep*. Environment variables like ``$WORK`` or
+            ``$SLURM_JOB_ID`` are left untouched.
+        sweep: Mapping of placeholder name to a sequence of values.
+            Example: ``{"ALPHA": [0.3, 0.5], "BETA": [1, 2]}``.
+        directory: Directory to write files into. Created if it doesn't
+            exist. Required when *preview* is ``False``.
+        placeholder_style: How placeholders appear in *base_command*:
+
+            - ``"token"`` (default): bare tokens, e.g. ``ALPHA``
+            - ``"braces"``: brace-wrapped, e.g. ``{ALPHA}``
+
+        debug: Optional debug string passed to ``ClassicLauncher``
+            (e.g. ``"host+job"``). Ignored when *preview* is ``True``.
+        preview: If ``True``, return a DataFrame of parameter combinations
+            without writing files.
+
+    Returns:
+        ``List[str]`` of generated commands when *preview* is ``False``,
+        or a ``pandas.DataFrame`` of parameter combinations when ``True``.
+
+    Raises:
+        TypeError: If a sweep value is not a non-string sequence.
+        ValueError: If a sweep value is empty, *placeholder_style* is
+            invalid, or *directory* is missing when *preview* is ``False``.
+    """
+    if sweep:
+        _validate_sweep(sweep)
+
+    if preview:
+        if not sweep:
+            return pd.DataFrame()
+        keys = list(sweep.keys())
+        rows = [dict(zip(keys, combo)) for combo in product(*[sweep[k] for k in keys])]
+        return pd.DataFrame(rows)
+
+    if directory is None:
+        raise ValueError("directory is required when preview=False.")
+
+    commands = _expand_commands(base_command, sweep, placeholder_style)
+
+    dirpath = Path(directory)
+    dirpath.mkdir(parents=True, exist_ok=True)
+
+    # Write runsList.txt
+    (dirpath / "runsList.txt").write_text(
+        "\n".join(commands) + "\n", encoding="utf-8"
+    )
+
+    # Write call_pylauncher.py
+    if debug is not None:
+        script = (
+            "import pylauncher\n"
+            f'pylauncher.ClassicLauncher("runsList.txt", debug="{debug}")\n'
+        )
+    else:
+        script = (
+            "import pylauncher\n"
+            'pylauncher.ClassicLauncher("runsList.txt")\n'
+        )
+    (dirpath / "call_pylauncher.py").write_text(script, encoding="utf-8")
+
+    return commands