Skip to content

Commit 401094f

Browse files
kks32kks32
committed
Get queues on system
1 parent 18ce6e9 commit 401094f

21 files changed

Lines changed: 14459 additions & 157 deletions

‎dapi/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
AuthenticationError,
1010
FileOperationError,
1111
AppDiscoveryError,
12+
SystemInfoError,
1213
JobSubmissionError,
1314
JobMonitorError,
1415
)
@@ -43,6 +44,7 @@
4344
"AuthenticationError",
4445
"FileOperationError",
4546
"AppDiscoveryError",
47+
"SystemInfoError"
4648
"JobSubmissionError",
4749
"JobMonitorError",
4850
]

‎dapi/client.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from . import apps as apps_module
55
from . import files as files_module
66
from . import jobs as jobs_module
7+
from . import systems as systems_module
78
from .db.accessor import DatabaseAccessor
89

910
# Import only the necessary classes/functions from jobs
@@ -32,6 +33,7 @@ def __init__(self, tapis_client: Optional[Tapis] = None, **auth_kwargs):
3233
self.apps = AppMethods(self.tapis)
3334
self.files = FileMethods(self.tapis)
3435
self.jobs = JobMethods(self.tapis)
36+
self.systems = SystemMethods(self.tapis)
3537
self.db = DatabaseAccessor()
3638

3739

@@ -63,8 +65,14 @@ def download(self, *args, **kwargs):
6365
def list(self, *args, **kwargs) -> List[Tapis]:
6466
return files_module.list_files(self._tapis, *args, **kwargs)
6567

68+
class SystemMethods:
69+
def __init__(self, tapis_client: Tapis):
70+
self._tapis = tapis_client
6671

67-
# --- JobMethods Updated ---
72+
def list_queues(self, system_id: str, verbose: bool = True) -> List[Any]:
73+
"""Lists logical queues for a given Tapis system."""
74+
return systems_module.list_system_queues(self._tapis, system_id, verbose=verbose)
75+
6876
class JobMethods:
6977
def __init__(self, tapis_client: Tapis):
7078
self._tapis = tapis_client

‎dapi/exceptions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ class AppDiscoveryError(DapiException):
2424

2525
pass
2626

27+
class SystemInfoError(DapiException):
28+
"""Error retrieving information about Tapis systems or queues."""
29+
pass
30+
2731

2832
class JobSubmissionError(DapiException):
2933
"""Error during job definition validation or submission."""

‎dapi/files.py‎

Lines changed: 127 additions & 143 deletions
Large diffs are not rendered by default.

‎dapi/systems.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# dapi/systems.py
2+
from tapipy.tapis import Tapis
3+
from tapipy.errors import BaseTapyException
4+
from typing import List, Any, Optional
5+
from .exceptions import SystemInfoError
6+
7+
def list_system_queues(t: Tapis, system_id: str, verbose: bool = True) -> List[Any]:
8+
"""
9+
Retrieves the list of batch logical queues available on a specific Tapis execution system.
10+
11+
Args:
12+
t: Authenticated Tapis client instance.
13+
system_id: The ID of the execution system (e.g., 'frontera', 'stampede2').
14+
verbose: If True, prints the found queues.
15+
16+
Returns:
17+
A list of queue objects (typically TapisResult instances or similar dict-like structures)
18+
defined for the system. Returns an empty list if the system exists but has no queues defined.
19+
20+
Raises:
21+
SystemInfoError: If the system is not found or an API error occurs.
22+
"""
23+
if not system_id:
24+
raise ValueError("system_id cannot be empty.")
25+
26+
try:
27+
if verbose:
28+
print(f"\nFetching queue information for system '{system_id}'...")
29+
30+
# Get system details - Fetch the full object to ensure queues are included
31+
# Removed 'select' parameter for simplicity and robustness against API variations
32+
system_details = t.systems.getSystem(systemId=system_id)
33+
34+
# Use 'batchLogicalQueues' based on the direct API call result
35+
queues = getattr(system_details, 'batchLogicalQueues', [])
36+
37+
if not queues:
38+
# Check if the system itself was found but just has no queues
39+
try:
40+
# Minimal check to confirm system existence if queues list was empty
41+
# This might be slightly redundant if getSystem above succeeded, but safe.
42+
t.systems.getSystem(systemId=system_id, select="id")
43+
if verbose:
44+
# Updated message
45+
print(f"System '{system_id}' found, but it has no batch logical queues defined.")
46+
return [] # Return empty list as system exists but has no queues
47+
except BaseTapyException as e_check:
48+
# If this minimal check fails with 404, the system wasn't found initially
49+
if hasattr(e_check, "response") and e_check.response and e_check.response.status_code == 404:
50+
raise SystemInfoError(f"Execution system '{system_id}' not found.") from e_check
51+
else: # Other error during the existence check
52+
raise SystemInfoError(f"Error confirming existence of system '{system_id}': {e_check}") from e_check
53+
54+
55+
if verbose:
56+
# Updated message
57+
print(f"Found {len(queues)} batch logical queues for system '{system_id}':")
58+
for q in queues:
59+
name = getattr(q, 'name', 'N/A')
60+
hpc_queue = getattr(q, 'hpcQueueName', 'N/A') # Actual scheduler queue name
61+
max_jobs = getattr(q, 'maxJobs', 'N/A')
62+
max_user_jobs = getattr(q, 'maxUserJobs', 'N/A')
63+
max_mins = getattr(q, 'maxMinutes', 'N/A')
64+
max_nodes = getattr(q, 'maxNodeCount', 'N/A')
65+
# Add more attributes if desired (e.g., maxMemoryMB, maxCoresPerNode)
66+
print(f" - Name: {name} (HPC Queue: {hpc_queue}, Max Jobs: {max_jobs}, Max User Jobs: {max_user_jobs}, Max Mins: {max_mins}, Max Nodes: {max_nodes})")
67+
68+
print()
69+
70+
# The items in the list are TapisResult objects themselves
71+
return queues
72+
73+
except BaseTapyException as e:
74+
if hasattr(e, "response") and e.response and e.response.status_code == 404:
75+
raise SystemInfoError(f"Execution system '{system_id}' not found.") from e
76+
else:
77+
raise SystemInfoError(f"Failed to retrieve queues for system '{system_id}': {e}") from e
78+
except Exception as e:
79+
raise SystemInfoError(f"An unexpected error occurred while fetching queues for system '{system_id}': {e}") from e

‎examples/mpm.ipynb‎

Lines changed: 136 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
},
1919
{
2020
"cell_type": "code",
21-
"execution_count": 13,
21+
"execution_count": 9,
2222
"id": "839fa332-70a6-4818-a190-18c9ca109c28",
2323
"metadata": {},
2424
"outputs": [
@@ -79,7 +79,7 @@
7979
"Building wheels for collected packages: dapi\n",
8080
" Building editable for dapi (pyproject.toml) ... \u001b[?25ldone\n",
8181
"\u001b[?25h Created wheel for dapi: filename=dapi-1.0.0-py3-none-any.whl size=3825 sha256=f9fd4761dd2e940e7cd0f96795280478e27aacfbc14275ffb57c0ee44f9b129c\n",
82-
" Stored in directory: /private/var/folders/w8/xz590jyd7r36zmxcspgzj3z40000gn/T/pip-ephem-wheel-cache-qfmkjb13/wheels/98/df/91/ed70fe2dca11c3c6e5b6e8e6eef18c373a119d095037f892a3\n",
82+
" Stored in directory: /private/var/folders/w8/xz590jyd7r36zmxcspgzj3z40000gn/T/pip-ephem-wheel-cache-tph3acf9/wheels/98/df/91/ed70fe2dca11c3c6e5b6e8e6eef18c373a119d095037f892a3\n",
8383
"Successfully built dapi\n",
8484
"Installing collected packages: dapi\n",
8585
"Successfully installed dapi-1.0.0\n",
@@ -96,19 +96,10 @@
9696
},
9797
{
9898
"cell_type": "code",
99-
"execution_count": 1,
99+
"execution_count": 5,
100100
"id": "35fca324-ee48-41c8-84a1-78ad7b03aae8",
101101
"metadata": {},
102-
"outputs": [
103-
{
104-
"name": "stderr",
105-
"output_type": "stream",
106-
"text": [
107-
"/Users/krishna/Library/Caches/pypoetry/virtualenvs/dapi-ptztLUqK-py3.13/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
108-
" from .autonotebook import tqdm as notebook_tqdm\n"
109-
]
110-
}
111-
],
102+
"outputs": [],
112103
"source": [
113104
"import os\n",
114105
"\n",
@@ -120,6 +111,7 @@
120111
" AppDiscoveryError,\n",
121112
" FileOperationError,\n",
122113
" JobSubmissionError,\n",
114+
" SystemInfoError,\n",
123115
" JobMonitorError,\n",
124116
" # Optionally import status constants if you want to check against them explicitly\n",
125117
" STATUS_TIMEOUT,\n",
@@ -785,6 +777,137 @@
785777
"\n",
786778
"print(f\"App Description: {app_details}\")"
787779
]
780+
},
781+
{
782+
"cell_type": "code",
783+
"execution_count": 6,
784+
"id": "9aaef98a",
785+
"metadata": {},
786+
"outputs": [
787+
{
788+
"name": "stdout",
789+
"output_type": "stream",
790+
"text": [
791+
"\n",
792+
"--- System Queue Information ---\n",
793+
"\n",
794+
"Fetching queue information for system 'frontera'...\n",
795+
"Found 10 batch logical queues for system 'frontera':\n",
796+
" - Name: flex (HPC Queue: flex, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 128)\n",
797+
" - Name: development (HPC Queue: development, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 120, Max Nodes: 40)\n",
798+
" - Name: normal (HPC Queue: normal, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 512)\n",
799+
" - Name: large (HPC Queue: large, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 2048)\n",
800+
" - Name: debug (HPC Queue: debug, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 8368)\n",
801+
" - Name: rtx (HPC Queue: rtx, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 22)\n",
802+
" - Name: rtx-dev (HPC Queue: rtx-dev, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 120, Max Nodes: 2)\n",
803+
" - Name: nvdimm (HPC Queue: nvdimm, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 120, Max Nodes: 4)\n",
804+
" - Name: small (HPC Queue: small, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 2880, Max Nodes: 2)\n",
805+
" - Name: grace (HPC Queue: grace, Max Jobs: -1, Max User Jobs: N/A, Max Mins: 7200, Max Nodes: 30)\n",
806+
"\n",
807+
"Does 'development' queue exist on Frontera? True\n",
808+
"\n",
809+
"Fetching queue information for system 'non-existent-system'...\n",
810+
"Error getting system info: Failed to retrieve queues for system 'non-existent-system': message: SYSAPI_NOT_FOUND Record not found. jwtTenant: designsafe jwtUser: kks32 OboTenant: designsafe OboUser: kks32 System: non-existent-system\n",
811+
"-----------------------------\n"
812+
]
813+
}
814+
],
815+
"source": [
816+
"# --- Example: List Queues for Frontera ---\n",
817+
"try:\n",
818+
" print(\"\\n--- System Queue Information ---\")\n",
819+
" frontera_queues = ds.systems.list_queues(\"frontera\")\n",
820+
" # You can now inspect the 'frontera_queues' list\n",
821+
" # Example: Find if 'development' queue exists\n",
822+
" dev_queue_exists = any(q.name == 'development' for q in frontera_queues)\n",
823+
" print(f\"Does 'development' queue exist on Frontera? {dev_queue_exists}\")\n",
824+
"\n",
825+
" # Example: List queues for a non-existent system\n",
826+
" ds.systems.list_queues(\"non-existent-system\") # This would raise SystemInfoError\n",
827+
"\n",
828+
"except SystemInfoError as e:\n",
829+
" print(f\"Error getting system info: {e}\")\n",
830+
"except Exception as e:\n",
831+
" print(f\"An unexpected error occurred: {e}\")\n",
832+
"print(\"-----------------------------\")"
833+
]
834+
},
835+
{
836+
"cell_type": "markdown",
837+
"id": "c3fb073a",
838+
"metadata": {},
839+
"source": [
840+
"## Verify TAPIS paths"
841+
]
842+
},
843+
{
844+
"cell_type": "code",
845+
"execution_count": 6,
846+
"id": "e074a3c3",
847+
"metadata": {},
848+
"outputs": [
849+
{
850+
"name": "stdout",
851+
"output_type": "stream",
852+
"text": [
853+
"\n",
854+
"Translating and verifying path: /MyData/mpm-benchmarks/2d/uniaxial_stress/\n",
855+
"Translated '/MyData/mpm-benchmarks/2d/uniaxial_stress/' to 'tapis://designsafe.storage.default/kks32/mpm-benchmarks/2d/uniaxial_stress/' using t.username\n",
856+
"Verifying existence of translated path: tapis://designsafe.storage.default/kks32/mpm-benchmarks/2d/uniaxial_stress/\n",
857+
"Checking system 'designsafe.storage.default' for path 'kks32/mpm-benchmarks/2d/uniaxial_stress/'...\n",
858+
"Verification successful: Path exists.\n",
859+
"Input Directory Tapis URI (verified): tapis://designsafe.storage.default/kks32/mpm-benchmarks/2d/uniaxial_stress/\n",
860+
"\n",
861+
"Translating and verifying non-existent path: /MyData/this/path/does/not/exist/\n",
862+
"Translated '/MyData/this/path/does/not/exist/' to 'tapis://designsafe.storage.default/kks32/this/path/does/not/exist/' using t.username\n",
863+
"Verifying existence of translated path: tapis://designsafe.storage.default/kks32/this/path/does/not/exist/\n",
864+
"Checking system 'designsafe.storage.default' for path 'kks32/this/path/does/not/exist/'...\n",
865+
"Error during path translation/verification: Verification error for path 'kks32/this/path/does/not/exist/' on system 'designsafe.storage.default': message: FILES_CLIENT_SSH_NOT_FOUND Path not found. OboTenant: designsafe OboUser: kks32 System: designsafe.storage.default EffectiveUser: kks32 Host: cloud.data.tacc.utexas.edu RootDir: /data/designsafe/mydata Path: kks32/this/path/does/not/exist\n"
866+
]
867+
},
868+
{
869+
"ename": "SystemExit",
870+
"evalue": "Stopping notebook due to path verification error.",
871+
"output_type": "error",
872+
"traceback": [
873+
"An exception has occurred, use %tb to see the full traceback.\n",
874+
"\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m Stopping notebook due to path verification error.\n"
875+
]
876+
},
877+
{
878+
"name": "stderr",
879+
"output_type": "stream",
880+
"text": [
881+
"/Users/krishna/Library/Caches/pypoetry/virtualenvs/dapi-ptztLUqK-py3.13/lib/python3.13/site-packages/IPython/core/interactiveshell.py:3585: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
882+
" warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
883+
]
884+
}
885+
],
886+
"source": [
887+
"# --- Translate Path with Verification ---\n",
888+
"ds_path: str = \"/MyData/mpm-benchmarks/2d/uniaxial_stress/\"\n",
889+
"ds_path_nonexistent: str = \"/MyData/this/path/does/not/exist/\"\n",
890+
"\n",
891+
"try:\n",
892+
" # Translate and verify the existing path\n",
893+
" print(f\"\\nTranslating and verifying path: {ds_path}\")\n",
894+
" input_uri = ds.files.translate_path_to_uri(ds_path, verify_exists=True)\n",
895+
" print(f\"Input Directory Tapis URI (verified): {input_uri}\")\n",
896+
"\n",
897+
" # Example: Try translating a non-existent path with verification (will raise error)\n",
898+
" print(f\"\\nTranslating and verifying non-existent path: {ds_path_nonexistent}\")\n",
899+
" input_uri_bad = ds.files.translate_path_to_uri(ds_path_nonexistent, verify_exists=True)\n",
900+
" print(f\"This line should not be reached.\")\n",
901+
"\n",
902+
"except FileOperationError as e:\n",
903+
" print(f\"Error during path translation/verification: {e}\")\n",
904+
" # Decide how to handle the error (e.g., stop notebook, use default, etc.)\n",
905+
" # For this example, we'll stop if verification fails.\n",
906+
" raise SystemExit(\"Stopping notebook due to path verification error.\")\n",
907+
"except Exception as e:\n",
908+
" print(f\"An unexpected error occurred during path translation: {e}\")\n",
909+
" raise SystemExit(\"Stopping notebook due to unexpected path translation error.\")\n"
910+
]
788911
}
789912
],
790913
"metadata": {

0 commit comments

Comments
 (0)