Add ds.systems.list(), improve publications search with filters

Systems:
- ds.systems.list(): DataFrame of HPC and storage systems with credential status
- Filter by category: "hpc", "storage", "all"
- Filters out internal, duplicate, and project-specific systems
- Add examples/systems.ipynb

Publications search:
- Add specific filters: pi, keyword, publication_type
- Filters combine with AND logic, all case-insensitive
- ds.publications.search(pi="Rathje", publication_type="simulation")

Update docs and examples for both features.

Author: kks32

dapi/client.py

@@ -361,23 +361,47 @@ def list(self, limit: int = 100, offset: int = 0, output: str = "df"):
             self._tapis, limit=limit, offset=offset, output=output
         )
 
-    def search(self, query: str, limit: int = 100, output: str = "df"):
-        """Search published datasets by keyword, title, or PI name.
+    def search(
+        self,
+        query: Optional[str] = None,
+        *,
+        pi: Optional[str] = None,
+        keyword: Optional[str] = None,
+        publication_type: Optional[str] = None,
+        limit: int = 100,
+        output: str = "df",
+    ):
+        """Search published datasets with optional filters.
+
+        All filters are AND-combined and case-insensitive.
 
         Args:
-            query (str): Search term (case-insensitive).
-            limit (int, optional): Max publications to fetch before filtering. Defaults to 100.
+            query (str, optional): General search across title, description, keywords, PI.
+            pi (str, optional): Filter by PI name.
+            keyword (str, optional): Filter by keyword.
+            publication_type (str, optional): Filter by type: "simulation",
+                "experimental", "field_recon", "other", "hybrid_simulation".
+            limit (int, optional): Max publications to fetch. Defaults to 100.
             output (str, optional): "df" for DataFrame (default), "list" for dicts.
 
         Returns:
             DataFrame or List[Dict]: Matching publications.
 
         Example:
             >>> ds.publications.search("liquefaction")
-            >>> ds.publications.search("lateral spreading", limit=500)
+            >>> ds.publications.search(pi="Rathje")
+            >>> ds.publications.search(
+            ...     keyword="storm surge", publication_type="simulation"
+            ... )
         """
         return publications_module.search_publications(
-            self._tapis, query, limit=limit, output=output
+            self._tapis,
+            query,
+            pi=pi,
+            keyword=keyword,
+            publication_type=publication_type,
+            limit=limit,
+            output=output,
         )
 
     def get(self, project_id: str) -> Dict:
@@ -435,6 +459,30 @@ def __init__(self, tapis_client: Tapis):
         """
         self._tapis = tapis_client
 
+    def list(self, category: Optional[str] = None, output: str = "df"):
+        """List Tapis systems you have access to.
+
+        Filters out internal and project-specific systems by default.
+
+        Args:
+            category (str, optional): "hpc" for execution systems,
+                "storage" for storage systems, "all" for everything,
+                None for HPC + storage (default).
+            output (str, optional): "df" for DataFrame (default), "list" for dicts.
+
+        Returns:
+            DataFrame or List[Dict]: Systems with id, host, category, authn, credentials.
+
+        Example:
+            >>> ds.systems.list()  # HPC + storage
+            >>> ds.systems.list("hpc")  # HPC only with credential status
+            >>> ds.systems.list("storage")  # Storage only
+            >>> ds.systems.list("all")  # Everything including internal
+        """
+        return systems_module.list_systems(
+            self._tapis, category=category, output=output
+        )
+
     def queues(self, system_id: str, verbose: bool = True) -> List[Any]:
         """List logical queues available on a Tapis execution system.
 

dapi/publications.py

@@ -88,31 +88,44 @@ def list_publications(
 
 def search_publications(
     t: Tapis,
-    query: str,
+    query: Optional[str] = None,
+    *,
+    pi: Optional[str] = None,
+    keyword: Optional[str] = None,
+    publication_type: Optional[str] = None,
     limit: int = 100,
     output: str = "df",
 ) -> Union[pd.DataFrame, List[Dict]]:
-    """Search published datasets by keyword, title, or PI name.
+    """Search published datasets with optional filters.
 
-    Searches across title, description, keywords, and PI name fields.
-    Filtering is done client-side since the API doesn't support server-side search.
+    All filters are case-insensitive and combined with AND logic.
+    At least one filter (query, pi, keyword, or publication_type) must be provided.
 
     Args:
         t (Tapis): Authenticated Tapis client instance.
-        query (str): Search term (case-insensitive).
-        limit (int, optional): Maximum publications to fetch before filtering. Defaults to 100.
+        query (str, optional): General search across title, description, keywords, and PI.
+        pi (str, optional): Filter by PI name (partial match).
+        keyword (str, optional): Filter by keyword (partial match against keywords list).
+        publication_type (str, optional): Filter by type: "simulation", "experimental",
+            "field_recon", "other", "hybrid_simulation".
+        limit (int, optional): Max publications to fetch before filtering. Defaults to 100.
         output (str, optional): "df" for DataFrame (default), "list" for list of dicts.
 
     Returns:
         Union[pd.DataFrame, List[Dict]]: Matching publications.
 
     Raises:
         FileOperationError: If the API request fails.
-        ValueError: If output format is invalid.
+        ValueError: If output format is invalid or no filters provided.
     """
     if output not in ("df", "list"):
         raise ValueError(f"output must be 'df' or 'list', got '{output}'")
 
+    if not any([query, pi, keyword, publication_type]):
+        raise ValueError(
+            "At least one filter must be provided: query, pi, keyword, or publication_type."
+        )
+
     headers = _get_auth_headers(t)
     try:
         resp = requests.get(
@@ -125,31 +138,48 @@ def search_publications(
     except requests.RequestException as e:
         raise FileOperationError(f"Failed to search publications: {e}") from e
 
-    q = query.lower()
     data = resp.json()
     matches = []
     for p in data.get("result", []):
-        searchable = " ".join(
-            [
-                str(p.get("title", "")),
-                str(p.get("description", "")),
-                " ".join(p.get("keywords", [])),
-                _pi_display(p.get("pi")),
-                str(p.get("projectId", "")),
-            ]
-        ).lower()
-        if q in searchable:
-            pi = p.get("pi")
-            matches.append(
-                {
-                    "projectId": p.get("projectId"),
-                    "title": p.get("title"),
-                    "pi": _pi_display(pi),
-                    "type": p.get("type"),
-                    "keywords": p.get("keywords", []),
-                    "created": p.get("created"),
-                }
-            )
+        # Apply filters with AND logic
+        if query:
+            searchable = " ".join(
+                [
+                    str(p.get("title", "")),
+                    str(p.get("description", "")),
+                    " ".join(p.get("keywords", [])),
+                    _pi_display(p.get("pi")),
+                    str(p.get("projectId", "")),
+                ]
+            ).lower()
+            if query.lower() not in searchable:
+                continue
+
+        if pi:
+            pi_name = _pi_display(p.get("pi")).lower()
+            if pi.lower() not in pi_name:
+                continue
+
+        if keyword:
+            kw_lower = keyword.lower()
+            kw_list = [k.lower() for k in p.get("keywords", [])]
+            if not any(kw_lower in k for k in kw_list):
+                continue
+
+        if publication_type:
+            if (p.get("type") or "").lower() != publication_type.lower():
+                continue
+
+        matches.append(
+            {
+                "projectId": p.get("projectId"),
+                "title": p.get("title"),
+                "pi": _pi_display(p.get("pi")),
+                "type": p.get("type"),
+                "keywords": p.get("keywords", []),
+                "created": p.get("created"),
+            }
+        )
 
     if output == "list":
         return matches

dapi/systems.py

@@ -1,10 +1,127 @@
 # dapi/systems.py
+import pandas as pd
 from tapipy.tapis import Tapis
 from tapipy.errors import BaseTapyException, UnauthorizedError, NotFoundError
-from typing import Dict, List, Any, Optional
+from typing import Dict, List, Any, Optional, Union
 from .exceptions import SystemInfoError, CredentialError
 
 
+# Known DesignSafe system categories
+_KNOWN_HPC = {"stampede3", "frontera", "ls6", "vista"}
+_KNOWN_STORAGE = {
+    "designsafe.storage.default",
+    "designsafe.storage.community",
+    "designsafe.storage.published",
+    "nees.public",
+}
+_INTERNAL_PREFIXES = ("project-", "apcd.", "wma-", "ds-stko", "cloud.data", "c4-")
+_DUPLICATE_SUFFIXES = (".tms", ".designsafe", "-simcenter")
+_STORAGE_PREFIXES = ("designsafe.storage.",)
+
+
+def list_systems(
+    t: Tapis,
+    category: Optional[str] = None,
+    output: str = "df",
+) -> Union[pd.DataFrame, List[Dict]]:
+    """List Tapis systems the user has access to.
+
+    Filters out internal, duplicate, and project-specific systems by default,
+    showing only the systems useful for job submission and data access.
+
+    Args:
+        t (Tapis): Authenticated Tapis client instance.
+        category (str, optional): Filter by category:
+            "hpc" for execution systems (stampede3, frontera, ls6, vista),
+            "storage" for storage systems (MyData, CommunityData, etc.),
+            "all" for all systems without filtering.
+            If None, shows HPC + storage (excludes internal/project systems).
+        output (str, optional): "df" for DataFrame (default), "list" for dicts.
+
+    Returns:
+        Union[pd.DataFrame, List[Dict]]: Systems with id, host, type, category, credentials.
+
+    Raises:
+        SystemInfoError: If the API request fails.
+        ValueError: If output or category is invalid.
+    """
+    if output not in ("df", "list"):
+        raise ValueError(f"output must be 'df' or 'list', got '{output}'")
+    if category is not None and category not in ("hpc", "storage", "all"):
+        raise ValueError(
+            f"category must be 'hpc', 'storage', 'all', or None, got '{category}'"
+        )
+
+    try:
+        all_systems = t.systems.getSystems(listType="ALL", limit=200)
+    except BaseTapyException as e:
+        raise SystemInfoError(f"Failed to list systems: {e}") from e
+
+    username = getattr(t, "username", None)
+    rows = []
+
+    for s in all_systems:
+        sid = s.id
+        host = getattr(s, "host", "")
+        can_exec = getattr(s, "canExec", False)
+        authn = getattr(s, "defaultAuthnMethod", "")
+
+        # Classify
+        if sid in _KNOWN_HPC:
+            cat = "hpc"
+        elif sid in _KNOWN_STORAGE:
+            cat = "storage"
+        elif (
+            any(sid.startswith(pfx) for pfx in _STORAGE_PREFIXES)
+            and sid not in _KNOWN_STORAGE
+        ):
+            cat = "internal"
+        elif sid.startswith("project-"):
+            cat = "project"
+        elif any(sid.endswith(sfx) for sfx in _DUPLICATE_SUFFIXES):
+            cat = "internal"
+        elif any(sid.startswith(pfx) for pfx in _INTERNAL_PREFIXES):
+            cat = "internal"
+        elif sid == "maverick2":
+            cat = "internal"
+        elif can_exec:
+            cat = "hpc"
+        else:
+            cat = "other"
+
+        # Filter
+        if category == "hpc" and cat != "hpc":
+            continue
+        if category == "storage" and cat != "storage":
+            continue
+        if category is None and cat not in ("hpc", "storage"):
+            continue
+        # category == "all" shows everything
+
+        # Check TMS credentials for HPC systems
+        has_creds = None
+        if cat == "hpc" and authn == "TMS_KEYS" and username:
+            try:
+                has_creds = check_credentials(t, sid, username)
+            except Exception:
+                has_creds = None
+
+        rows.append(
+            {
+                "id": sid,
+                "host": host,
+                "category": cat,
+                "authn": authn,
+                "credentials": has_creds,
+            }
+        )
+
+    if output == "list":
+        return rows
+
+    return pd.DataFrame(rows)
+
+
 def list_system_queues(t: Tapis, system_id: str, verbose: bool = True) -> List[Any]:
     """
     Retrieves the list of batch logical queues available on a specific Tapis execution system.

docs/examples.md

@@ -23,6 +23,13 @@ Search and browse published datasets on DesignSafe.
 
 ---
 
+### Systems
+List HPC and storage systems, check credentials, and view queues.
+
+[![Try on DesignSafe](https://raw.githubusercontent.com/DesignSafe-CI/dapi/main/DesignSafe-Badge.svg)](https://jupyter.designsafe-ci.org/hub/user-redirect/lab/tree/CommunityData/dapi/systems.ipynb)
+
+---
+
 ### Application Management
 Discover and manage applications on DesignSafe.
 

docs/publications.md

@@ -23,20 +23,26 @@ DataFrame columns: `projectId`, `title`, `pi`, `type`, `keywords`, `created`.
 
 ## Search publications
 
-Search across titles, descriptions, keywords, and PI names. Case-insensitive.
+Search with specific filters or general text. All filters are case-insensitive and combined with AND logic.
 
 ```python
-# By topic
+# General text search (across title, description, keywords, PI)
 ds.publications.search("liquefaction")
 
-# By PI
-ds.publications.search("Rathje")
+# Filter by PI name
+ds.publications.search(pi="Rathje")
 
-# By project type keyword
-ds.publications.search("storm surge")
+# Filter by keyword
+ds.publications.search(keyword="storm surge")
+
+# Filter by publication type: simulation, experimental, field_recon, other, hybrid_simulation
+ds.publications.search(publication_type="simulation")
+
+# Combine filters (AND logic)
+ds.publications.search(keyword="storm surge", publication_type="simulation")
 
 # Increase search pool (API returns 100 by default)
-ds.publications.search("earthquake", limit=500)
+ds.publications.search(keyword="earthquake", limit=500)
 ```
 
 ## Get publication details