Skip to content

Commit dc083a8

Browse files
committed
Add ds.projects module for DesignSafe project management
- New projects.py: list_projects, get_project, list_project_files, resolve_project_uuid via DesignSafe portal API (/api/projects/v2/) - New ProjectMethods class on DSClient: ds.projects.list(), .get(), .files() - Move _resolve_project_uuid from files.py to projects.py - Add docs/projects.md and update nav in myst.yml and docs/index.md - Update docs/files.md with NHERI-Published and NEES path formats - Add tests/projects/test_projects.py (9 tests) - Add examples/projects.ipynb with working outputs
1 parent aaaad3f commit dc083a8

11 files changed

Lines changed: 866 additions & 45 deletions

File tree

dapi/client.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from . import jobs as jobs_module
77
from . import systems as systems_module
88
from . import launcher as launcher_module
9+
from . import projects as projects_module
910
from .db.accessor import DatabaseAccessor
1011

1112
# Import only the necessary classes/functions from jobs
@@ -31,6 +32,7 @@ class DSClient:
3132
apps (AppMethods): Interface for application discovery and details.
3233
files (FileMethods): Interface for file operations (upload, download, list).
3334
jobs (JobMethods): Interface for job submission and monitoring.
35+
projects (ProjectMethods): Interface for DesignSafe project management.
3436
systems (SystemMethods): Interface for system information and queues.
3537
db (DatabaseAccessor): Interface for database connections and queries.
3638
@@ -90,6 +92,7 @@ def __init__(self, tapis_client: Optional[Tapis] = None, **auth_kwargs):
9092
self.apps = AppMethods(self.tapis)
9193
self.files = FileMethods(self.tapis)
9294
self.jobs = JobMethods(self.tapis)
95+
self.projects = ProjectMethods(self.tapis)
9396
self.systems = SystemMethods(self.tapis)
9497
self.db = DatabaseAccessor()
9598

@@ -250,6 +253,74 @@ def list(self, *args, **kwargs) -> List[Tapis]:
250253
return files_module.list_files(self._tapis, *args, **kwargs)
251254

252255

256+
class ProjectMethods:
257+
"""Interface for DesignSafe project management.
258+
259+
Provides methods for listing projects, getting project metadata,
260+
and listing files within projects.
261+
262+
Args:
263+
tapis_client (Tapis): Authenticated Tapis client instance.
264+
"""
265+
266+
def __init__(self, tapis_client: Tapis):
267+
self._tapis = tapis_client
268+
269+
def list(self, limit: int = 100, offset: int = 0) -> List[Dict]:
270+
"""List DesignSafe projects you have access to.
271+
272+
Args:
273+
limit (int, optional): Maximum projects to return. Defaults to 100.
274+
offset (int, optional): Number of projects to skip. Defaults to 0.
275+
276+
Returns:
277+
List[Dict]: List of project dicts with uuid, projectId, title, pi, etc.
278+
279+
Example:
280+
>>> projects = ds.projects.list()
281+
>>> for p in projects[:3]:
282+
... print(f"{p['projectId']} - {p['title']}")
283+
"""
284+
return projects_module.list_projects(self._tapis, limit=limit, offset=offset)
285+
286+
def get(self, project_id: str) -> Dict:
287+
"""Get detailed metadata for a project.
288+
289+
Args:
290+
project_id (str): Project ID (e.g., "PRJ-1305").
291+
292+
Returns:
293+
Dict: Project metadata including title, description, PI, team,
294+
DOIs, award numbers, keywords, systemId.
295+
296+
Example:
297+
>>> info = ds.projects.get("PRJ-6270")
298+
>>> print(info["title"])
299+
>>> print(info["systemId"])
300+
"""
301+
return projects_module.get_project(self._tapis, project_id)
302+
303+
def files(self, project_id: str, path: str = "/", limit: int = 100) -> List:
304+
"""List files in a project.
305+
306+
Args:
307+
project_id (str): Project ID (e.g., "PRJ-1305").
308+
path (str, optional): Path within the project. Defaults to "/".
309+
limit (int, optional): Max items to return. Defaults to 100.
310+
311+
Returns:
312+
List: List of Tapis file objects.
313+
314+
Example:
315+
>>> files = ds.projects.files("PRJ-1305", "/Training/")
316+
>>> for f in files[:5]:
317+
... print(f"{f.name} ({f.type})")
318+
"""
319+
return projects_module.list_project_files(
320+
self._tapis, project_id, path=path, limit=limit
321+
)
322+
323+
253324
class SystemMethods:
254325
"""Interface for Tapis system information and queue management.
255326

dapi/files.py

Lines changed: 5 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
import os
33
import urllib.parse
44

5-
import requests
65
from tapipy.tapis import Tapis
76
from tapipy.errors import BaseTapyException
87
from .exceptions import FileOperationError, AuthenticationError
@@ -157,48 +156,6 @@ def tapis_uri_to_local_path(tapis_uri: str) -> str:
157156
return tapis_uri
158157

159158

160-
def _resolve_project_uuid(t: Tapis, project_id: str) -> str:
161-
"""Resolve a DesignSafe project ID (e.g., PRJ-1305) to its Tapis system UUID.
162-
163-
Queries the DesignSafe projects API to find the project UUID, then
164-
returns the corresponding Tapis system ID.
165-
166-
Args:
167-
t (Tapis): Authenticated Tapis client instance (used for the access token).
168-
project_id (str): The DesignSafe project ID (e.g., "PRJ-1305").
169-
170-
Returns:
171-
str: The Tapis system ID (e.g., "project-7997906542076432871-242ac11c-0001-012").
172-
173-
Raises:
174-
FileOperationError: If the project cannot be found or the API request fails.
175-
"""
176-
token = t.access_token.access_token
177-
headers = {"X-Tapis-Token": token, "Authorization": f"Bearer {token}"}
178-
try:
179-
resp = requests.get(
180-
"https://designsafe-ci.org/api/projects/v2/",
181-
headers=headers,
182-
params={"limit": 100},
183-
timeout=30,
184-
)
185-
resp.raise_for_status()
186-
projects = resp.json().get("result", [])
187-
for p in projects:
188-
val = p.get("value", {})
189-
if val.get("projectId", "") == project_id:
190-
uuid = p["uuid"]
191-
return f"project-{uuid}"
192-
except requests.RequestException as e:
193-
raise FileOperationError(
194-
f"Failed to query DesignSafe projects API for '{project_id}': {e}"
195-
) from e
196-
197-
raise FileOperationError(
198-
f"Project '{project_id}' not found. Ensure you have access to this project."
199-
)
200-
201-
202159
def get_ds_path_uri(t: Tapis, path: str, verify_exists: bool = False) -> str:
203160
"""Translate DesignSafe-style paths to Tapis URIs.
204161
@@ -352,7 +309,11 @@ def get_ds_path_uri(t: Tapis, path: str, verify_exists: bool = False) -> str:
352309
)
353310
else:
354311
# Resolve PRJ number via DesignSafe projects API
355-
found_system_id = _resolve_project_uuid(t, project_id_part)
312+
from . import projects as projects_module
313+
314+
found_system_id = projects_module.resolve_project_uuid(
315+
t, project_id_part
316+
)
356317

357318
input_uri = f"tapis://{found_system_id}/{path_within_project}"
358319
print(f"Translated '{path}' to '{input_uri}'")

dapi/projects.py

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
# dapi/projects.py
2+
import requests
3+
from tapipy.tapis import Tapis
4+
from tapipy.errors import BaseTapyException
5+
from .exceptions import FileOperationError
6+
from typing import Dict, List, Optional
7+
8+
9+
_DS_PROJECTS_API = "https://designsafe-ci.org/api/projects/v2/"
10+
11+
12+
def _get_auth_headers(t: Tapis) -> Dict[str, str]:
13+
"""Build authentication headers from a Tapis client."""
14+
token = t.access_token.access_token
15+
return {"X-Tapis-Token": token, "Authorization": f"Bearer {token}"}
16+
17+
18+
def list_projects(t: Tapis, limit: int = 100, offset: int = 0) -> List[Dict]:
19+
"""List DesignSafe projects the authenticated user has access to.
20+
21+
Args:
22+
t (Tapis): Authenticated Tapis client instance.
23+
limit (int, optional): Maximum number of projects to return. Defaults to 100.
24+
offset (int, optional): Number of projects to skip. Defaults to 0.
25+
26+
Returns:
27+
List[Dict]: List of project dictionaries with keys:
28+
- uuid (str): Project UUID
29+
- projectId (str): Project ID (e.g., "PRJ-1305")
30+
- title (str): Project title
31+
- pi (dict): Principal investigator info (username, fname, lname)
32+
- created (str): Creation timestamp
33+
- lastUpdated (str): Last update timestamp
34+
35+
Raises:
36+
FileOperationError: If the API request fails.
37+
"""
38+
headers = _get_auth_headers(t)
39+
try:
40+
resp = requests.get(
41+
_DS_PROJECTS_API,
42+
headers=headers,
43+
params={"limit": limit, "offset": offset},
44+
timeout=30,
45+
)
46+
resp.raise_for_status()
47+
except requests.RequestException as e:
48+
raise FileOperationError(f"Failed to list projects: {e}") from e
49+
50+
data = resp.json()
51+
projects = []
52+
for p in data.get("result", []):
53+
val = p.get("value", {})
54+
users = val.get("users", [])
55+
pi = next((u for u in users if u.get("role") == "pi"), None)
56+
projects.append(
57+
{
58+
"uuid": p.get("uuid"),
59+
"projectId": val.get("projectId"),
60+
"title": val.get("title"),
61+
"pi": pi,
62+
"created": p.get("created"),
63+
"lastUpdated": p.get("lastUpdated"),
64+
}
65+
)
66+
return projects
67+
68+
69+
def get_project(t: Tapis, project_id: str) -> Dict:
70+
"""Get detailed metadata for a DesignSafe project.
71+
72+
Args:
73+
t (Tapis): Authenticated Tapis client instance.
74+
project_id (str): Project ID (e.g., "PRJ-1305") or project UUID.
75+
76+
Returns:
77+
Dict: Project metadata with keys:
78+
- uuid (str): Project UUID
79+
- projectId (str): Project ID
80+
- title (str): Project title
81+
- description (str): Project description
82+
- pi (dict): Principal investigator info
83+
- coPis (list): Co-PIs
84+
- teamMembers (list): Team members
85+
- awardNumbers (list): Award/grant numbers
86+
- keywords (list): Keywords
87+
- dois (list): Associated DOIs
88+
- projectType (str): Project type
89+
- created (str): Creation timestamp
90+
- lastUpdated (str): Last update timestamp
91+
- systemId (str): Tapis system ID for file access
92+
93+
Raises:
94+
FileOperationError: If the project is not found or the API request fails.
95+
"""
96+
headers = _get_auth_headers(t)
97+
try:
98+
resp = requests.get(
99+
f"{_DS_PROJECTS_API}{project_id}/",
100+
headers=headers,
101+
timeout=30,
102+
)
103+
resp.raise_for_status()
104+
except requests.RequestException as e:
105+
raise FileOperationError(f"Failed to get project '{project_id}': {e}") from e
106+
107+
data = resp.json()
108+
bp = data.get("baseProject", {})
109+
val = bp.get("value", {})
110+
users = val.get("users", [])
111+
pi = next((u for u in users if u.get("role") == "pi"), None)
112+
uuid = bp.get("uuid", "")
113+
114+
return {
115+
"uuid": uuid,
116+
"projectId": val.get("projectId"),
117+
"title": val.get("title"),
118+
"description": val.get("description"),
119+
"pi": pi,
120+
"coPis": val.get("coPis", []),
121+
"teamMembers": val.get("teamMembers", []),
122+
"awardNumbers": val.get("awardNumbers", []),
123+
"keywords": val.get("keywords", []),
124+
"dois": val.get("dois", []),
125+
"projectType": val.get("projectType"),
126+
"created": bp.get("created"),
127+
"lastUpdated": bp.get("lastUpdated"),
128+
"systemId": f"project-{uuid}" if uuid else None,
129+
}
130+
131+
132+
def list_project_files(
133+
t: Tapis, project_id: str, path: str = "/", limit: int = 100
134+
) -> List:
135+
"""List files in a DesignSafe project.
136+
137+
Resolves the project ID to a Tapis system and lists files at the given path.
138+
139+
Args:
140+
t (Tapis): Authenticated Tapis client instance.
141+
project_id (str): Project ID (e.g., "PRJ-1305").
142+
path (str, optional): Path within the project. Defaults to "/".
143+
limit (int, optional): Maximum number of items to return. Defaults to 100.
144+
145+
Returns:
146+
List: List of Tapis file objects with name, type, size, etc.
147+
148+
Raises:
149+
FileOperationError: If the project is not found or file listing fails.
150+
"""
151+
project = get_project(t, project_id)
152+
system_id = project["systemId"]
153+
if not system_id:
154+
raise FileOperationError(
155+
f"Could not determine Tapis system ID for project '{project_id}'."
156+
)
157+
158+
if not path:
159+
path = "/"
160+
161+
try:
162+
results = t.files.listFiles(systemId=system_id, path=path, limit=limit)
163+
return results
164+
except BaseTapyException as e:
165+
raise FileOperationError(
166+
f"Failed to list files in project '{project_id}' at path '{path}': {e}"
167+
) from e
168+
169+
170+
def resolve_project_uuid(t: Tapis, project_id: str) -> str:
171+
"""Resolve a DesignSafe project ID (e.g., PRJ-1305) to its Tapis system ID.
172+
173+
Args:
174+
t (Tapis): Authenticated Tapis client instance.
175+
project_id (str): The DesignSafe project ID (e.g., "PRJ-1305").
176+
177+
Returns:
178+
str: The Tapis system ID (e.g., "project-7997906542076432871-242ac11c-0001-012").
179+
180+
Raises:
181+
FileOperationError: If the project cannot be found.
182+
"""
183+
headers = _get_auth_headers(t)
184+
try:
185+
resp = requests.get(
186+
_DS_PROJECTS_API,
187+
headers=headers,
188+
params={"limit": 100},
189+
timeout=30,
190+
)
191+
resp.raise_for_status()
192+
projects = resp.json().get("result", [])
193+
for p in projects:
194+
val = p.get("value", {})
195+
if val.get("projectId", "") == project_id:
196+
uuid = p["uuid"]
197+
return f"project-{uuid}"
198+
except requests.RequestException as e:
199+
raise FileOperationError(
200+
f"Failed to query DesignSafe projects API for '{project_id}': {e}"
201+
) from e
202+
203+
raise FileOperationError(
204+
f"Project '{project_id}' not found. Ensure you have access to this project."
205+
)

docs/files.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,10 @@ ds.files.to_path("tapis://designsafe.storage.community/datasets/eq.csv")
4848
| `~/MyData/...` | `designsafe.storage.default/<username>/...` |
4949
| `jupyter/MyData/...` | `designsafe.storage.default/<username>/...` |
5050
| `/CommunityData/...` | `designsafe.storage.community/...` |
51+
| `/NHERI-Published/...` | `designsafe.storage.published/...` |
52+
| `/NEES/...` | `nees.public/...` |
5153
| `/projects/PRJ-XXXX/...` | `project-<uuid>/...` (auto-discovered) |
54+
| `/MyProjects/PRJ-XXXX/...` | `project-<uuid>/...` (auto-discovered) |
5255
| `tapis://...` | passed through unchanged |
5356

5457
## List files

docs/index.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ For background on DesignSafe compute environments, storage, and workflow design,
3838
- [Jobs](jobs.md) -- submit, monitor, and manage computational jobs
3939
- [Apps](apps.md) -- find applications and their IDs
4040
- [Files](files.md) -- path translation, upload, download
41+
- [Projects](projects.md) -- list, inspect, and access project files
4142
- [Systems](systems.md) -- queues and TMS credentials
4243
- [Database Access](database.md) -- query DesignSafe research databases
4344

0 commit comments

Comments
 (0)