Auto-TMS on DSClient init, configurable jobs list output

- DSClient() now auto-establishes TMS credentials on TACC systems
  (frontera, stampede3, ls6). Skips gracefully on errors/no allocation.
- ds.jobs.list() supports output="df" (default), "list", or "raw".
- Simplify auth docs: DB credentials use built-in defaults, no .env needed.
- Remove manual TMS step from quickstart (now automatic).
- 59 tests pass (8 new for setup_tms, 6 new for output formats).

Author: kks32

dapi/client.py

@@ -92,6 +92,9 @@ def __init__(self, tapis_client: Optional[Tapis] = None, **auth_kwargs):
         self.systems = SystemMethods(self.tapis)
         self.db = DatabaseAccessor()
 
+        # Auto-setup TMS credentials on TACC execution systems
+        systems_module.setup_tms_credentials(self.tapis)
+
 
 # --- AppMethods and FileMethods remain the same ---
 class AppMethods:
@@ -582,34 +585,41 @@ def list(
         app_id: Optional[str] = None,
         status: Optional[str] = None,
         limit: int = 100,
+        output: str = "df",
         verbose: bool = False,
     ):
-        """List jobs as a pandas DataFrame with optional filtering.
+        """List jobs with optional filtering.
 
-        Fetches jobs from Tapis ordered by creation date (newest first)
-        and returns them as a DataFrame. Filters are applied client-side.
+        Fetches jobs from Tapis ordered by creation date (newest first).
+        Filters are applied client-side.
 
         Args:
             app_id (str, optional): Filter by application ID.
             status (str, optional): Filter by job status (e.g., "FINISHED").
                 Case-insensitive.
             limit (int, optional): Maximum jobs to fetch. Defaults to 100.
+            output (str, optional): Output format. "df" for pandas DataFrame
+                (default), "list" for list of dicts, "raw" for TapisResult
+                objects.
             verbose (bool, optional): Print job count. Defaults to False.
 
         Returns:
-            pd.DataFrame: Job metadata with formatted datetime columns.
+            Depends on output: DataFrame, list of dicts, or list of
+            TapisResult objects.
 
         Raises:
             JobMonitorError: If the Tapis API call fails.
 
         Example:
             >>> df = ds.jobs.list(app_id="matlab-r2023a", status="FINISHED")
-            >>> print(df[["name", "uuid", "status", "created_dt"]])
+            >>> jobs = ds.jobs.list(output="list")
+            >>> raw = ds.jobs.list(limit=10, output="raw")
         """
         return jobs_module.list_jobs(
             self._tapis,
             app_id=app_id,
             status=status,
             limit=limit,
+            output=output,
             verbose=verbose,
         )

dapi/jobs.py

@@ -1348,9 +1348,10 @@ def list_jobs(
     app_id: Optional[str] = None,
     status: Optional[str] = None,
     limit: int = 100,
+    output: str = "df",
     verbose: bool = False,
-) -> pd.DataFrame:
-    """Fetch Tapis jobs and return them as a pandas DataFrame.
+):
+    """Fetch Tapis jobs with optional filtering.
 
     Retrieves jobs from Tapis ordered by creation date (newest first)
     and optionally filters by app ID and/or status. Filters are applied
@@ -1362,22 +1363,29 @@ def list_jobs(
         status: Filter by job status (e.g., "FINISHED", "FAILED").
             Case-insensitive.
         limit: Maximum number of jobs to fetch from Tapis. Defaults to 100.
+        output: Output format. "df" returns a pandas DataFrame (default),
+            "list" returns a list of dicts, "raw" returns the raw
+            TapisResult objects.
         verbose: If True, prints the number of jobs found.
 
     Returns:
-        DataFrame with job metadata and formatted datetime columns.
-        Priority columns appear first: name, uuid, status, appId, appVersion,
-        created_dt, ended_dt. Additional datetime columns include _dt
-        (timezone-aware) and _date (date only) variants for created, ended,
-        remoteStarted, and lastUpdated.
+        Depends on ``output``:
+        - "df": pandas DataFrame with formatted datetime columns.
+        - "list": list of dicts with job metadata.
+        - "raw": list of TapisResult objects as returned by the API.
 
     Raises:
         JobMonitorError: If the Tapis API call fails.
+        ValueError: If output format is not recognized.
 
     Example:
         >>> df = list_jobs(t, app_id="matlab-r2023a", status="FINISHED")
-        >>> print(df[["name", "uuid", "status", "created_dt"]])
+        >>> jobs = list_jobs(t, output="list")
+        >>> raw = list_jobs(t, limit=10, output="raw")
     """
+    if output not in ("df", "list", "raw"):
+        raise ValueError(f"output must be 'df', 'list', or 'raw', got '{output}'")
+
     try:
         jobs_list = tapis_client.jobs.getJobList(
             limit=limit,
@@ -1391,17 +1399,48 @@ def list_jobs(
     if not jobs_list:
         if verbose:
             print("Found 0 jobs.")
+        if output == "raw":
+            return []
+        if output == "list":
+            return []
         return pd.DataFrame()
 
+    # For raw output, apply filters manually on TapisResult objects
+    if output == "raw":
+        results = jobs_list
+        if app_id:
+            results = [j for j in results if getattr(j, "appId", None) == app_id]
+        if status:
+            results = [
+                j for j in results
+                if getattr(j, "status", "").upper() == status.upper()
+            ]
+        if verbose:
+            print(f"Found {len(results)} jobs.")
+        return results
+
     # Convert TapisResult objects to dicts
     jobs_dicts = [job.__dict__ for job in jobs_list]
-    df = pd.DataFrame(jobs_dicts)
 
     # Apply client-side filters
-    if app_id and "appId" in df.columns:
-        df = df[df["appId"] == app_id]
-    if status and "status" in df.columns:
-        df = df[df["status"] == status.upper()]
+    if app_id:
+        jobs_dicts = [j for j in jobs_dicts if j.get("appId") == app_id]
+    if status:
+        jobs_dicts = [
+            j for j in jobs_dicts if j.get("status", "").upper() == status.upper()
+        ]
+
+    if verbose:
+        print(f"Found {len(jobs_dicts)} jobs.")
+
+    if output == "list":
+        return jobs_dicts
+
+    # Build DataFrame
+    df = pd.DataFrame(jobs_dicts)
+
+    if df.empty:
+        return df
 
     # Add formatted datetime columns
     time_cols = ["created", "ended", "remoteStarted", "lastUpdated"]
@@ -1421,7 +1460,4 @@ def list_jobs(
 
     df = df.reset_index(drop=True)
 
-    if verbose:
-        print(f"Found {len(df)} jobs.")
-
     return df

dapi/systems.py

@@ -1,7 +1,7 @@
 # dapi/systems.py
 from tapipy.tapis import Tapis
 from tapipy.errors import BaseTapyException, UnauthorizedError, NotFoundError
-from typing import List, Any, Optional
+from typing import Dict, List, Any, Optional
 from .exceptions import SystemInfoError, CredentialError
 
 
@@ -292,3 +292,77 @@ def revoke_credentials(
             f"Unexpected error revoking credentials for user '{effective_username}' "
             f"on system '{system_id}': {e}"
         ) from e
+
+
+# Default TACC execution systems that use TMS_KEYS
+TACC_SYSTEMS = ["frontera", "stampede3", "ls6"]
+
+
+def setup_tms_credentials(
+    t: Tapis,
+    systems: Optional[List[str]] = None,
+) -> Dict[str, str]:
+    """Check and establish TMS credentials on execution systems.
+
+    For each system, checks if credentials exist and creates them if missing.
+    Failures are handled gracefully — a system that can't be reached or where
+    the user lacks an allocation is skipped with a warning.
+
+    Args:
+        t: Authenticated Tapis client instance.
+        systems: List of system IDs to set up. Defaults to TACC_SYSTEMS
+            (frontera, stampede3, ls6).
+
+    Returns:
+        Dict mapping system_id to status: "ready", "created", or "skipped".
+    """
+    if systems is None:
+        systems = TACC_SYSTEMS
+
+    username = getattr(t, "username", None)
+    if not username:
+        print("Warning: Could not determine username. Skipping TMS setup.")
+        return {s: "skipped" for s in systems}
+
+    results = {}
+
+    for system_id in systems:
+        try:
+            # Check if system uses TMS_KEYS
+            system_details = t.systems.getSystem(systemId=system_id)
+            authn_method = getattr(system_details, "defaultAuthnMethod", None)
+
+            if authn_method != "TMS_KEYS":
+                results[system_id] = "skipped"
+                continue
+
+            # Check existing credentials
+            if check_credentials(t, system_id, username):
+                results[system_id] = "ready"
+                continue
+
+            # Try to create credentials
+            t.systems.createUserCredential(
+                systemId=system_id,
+                userName=username,
+                createTmsKeys=True,
+            )
+            results[system_id] = "created"
+
+        except Exception:
+            results[system_id] = "skipped"
+
+    # Print summary
+    ready = [s for s, v in results.items() if v in ("ready", "created")]
+    created = [s for s, v in results.items() if v == "created"]
+    skipped = [s for s, v in results.items() if v == "skipped"]
+
+    if ready:
+        msg = f"TMS credentials ready: {', '.join(ready)}"
+        if created:
+            msg += f" (newly created: {', '.join(created)})"
+        print(msg)
+    if skipped:
+        print(f"TMS credentials skipped: {', '.join(skipped)}")
+
+    return results

docs/authentication.md

@@ -327,65 +327,37 @@ print(os.access('.env', os.R_OK))
 
 ### Database Connection Issues
 
-For database-specific authentication issues:
+Database connections use built-in public read-only credentials by default -- no `.env` setup is required for database access. If you need to override the defaults (e.g., for a private database instance), you can set environment variables:
 
-```python
-# Check database environment variables
-import os
-print("NGL_DB_USER:", os.getenv('NGL_DB_USER'))
-print("VP_DB_USER:", os.getenv('VP_DB_USER'))
-print("EQ_DB_USER:", os.getenv('EQ_DB_USER'))
-```
-
-Required database environment variables:
 ```bash
-# NGL Database
-export NGL_DB_USER="dspublic"
-export NGL_DB_PASSWORD="your_password"
-export NGL_DB_HOST="db_host"
-export NGL_DB_PORT="3306"
-
-# VP Database 
-export VP_DB_USER="dspublic"
-export VP_DB_PASSWORD="your_password"
-export VP_DB_HOST="db_host"
-export VP_DB_PORT="3306"
-
-# Earthquake Recovery Database
-export EQ_DB_USER="dspublic"
-export EQ_DB_PASSWORD="your_password"
-export EQ_DB_HOST="db_host"
-export EQ_DB_PORT="3306"
+# Optional: override database credentials via .env or environment
+NGL_DB_USER=your_user
+NGL_DB_PASSWORD=your_password
+NGL_DB_HOST=your_host
+NGL_DB_PORT=3306
 ```
 
-## Example: Complete Setup
+The same pattern applies for VP (`VP_DB_*`) and Earthquake Recovery (`EQ_DB_*`) databases.
 
-Here's a complete example of setting up authentication:
+## Example: Complete Setup
 
 ```python
-# 1. Create .env file
+# 1. Create .env file (only Tapis credentials required)
 with open('.env', 'w') as f:
  f.write('DESIGNSAFE_USERNAME=your_username\n')
  f.write('DESIGNSAFE_PASSWORD=your_password\n')
- f.write('NGL_DB_USER=dspublic\n')
- f.write('NGL_DB_PASSWORD=your_db_password\n')
- f.write('NGL_DB_HOST=db_host\n')
- f.write('NGL_DB_PORT=3306\n')
 
-# 2. Initialize client
+# 2. Initialize client (auto-sets up TMS credentials)
 from dapi import DSClient
 ds = DSClient()
 
-# 3. Test authentication
-print("Testing TAPIS API access...")
+# 3. Test
 apps = ds.apps.find("matlab", verbose=False)
 print(f"Found {len(apps)} MATLAB apps")
 
-print("Testing database access...")
+# Database works out of the box -- no extra credentials needed
 df = ds.db.ngl.read_sql("SELECT COUNT(*) FROM SITE")
 print(f"NGL database has {df.iloc[0, 0]} sites")
-
-print("All authentication successful!")
 ```
 
 ## Troubleshooting

docs/jobs.md

@@ -22,7 +22,7 @@ from dapi import DSClient
 
 ds = DSClient()
 
-# List all recent jobs (default: last 100)
+# List all recent jobs (default: last 100, returns DataFrame)
 df = ds.jobs.list()
 print(df[["name", "uuid", "status", "appId", "created_dt"]])
 
@@ -41,7 +41,22 @@ print(f"Finished jobs: {len(finished)}")
 print(finished.groupby("appId").size())
 ```
 
-The returned DataFrame includes formatted datetime columns (`created_dt`, `ended_dt`, `created_date`, etc.) for easy time-based analysis.
+### Output Formats
+
+By default `list()` returns a pandas DataFrame. Use the `output` parameter for other formats:
+
+```python
+# DataFrame (default) -- includes formatted datetime columns
+df = ds.jobs.list()
+
+# List of dicts -- lightweight, no pandas dependency
+jobs = ds.jobs.list(output="list")
+for job in jobs:
+    print(f"{job['name']}: {job['status']}")
+
+# Raw TapisResult objects -- for advanced Tapis API usage
+raw = ds.jobs.list(output="raw")
+```
 
 ## Application Discovery
 

docs/quickstart.md

@@ -61,20 +61,13 @@ ds = DSClient()
 # Output: Authentication successful.
 ```
 
-### Step 1b: Establish TMS Credentials (One-Time)
+`DSClient()` automatically sets up TMS credentials on TACC execution systems (Frontera, Stampede3, LS6). You'll see a summary like:
 
-Before submitting jobs, ensure you have TMS credentials on the execution system:
-
-```python
-# One-time setup per system -- safe to call repeatedly
-ds.systems.establish_credentials("frontera")
-# Output: TMS credentials established for user 'myuser' on system 'frontera'.
-
-# Or if already established:
-# Output: Credentials already exist for user 'myuser' on system 'frontera'. No action taken.
+```
+TMS credentials ready: frontera, stampede3, ls6
 ```
 
-See the [Authentication Guide](authentication.md#tms-credentials-execution-system-access) for details.
+Systems where you don't have an allocation are silently skipped. See the [Authentication Guide](authentication.md#tms-credentials-execution-system-access) for manual control.
 
 ### Step 2: Explore Available Applications