Skip to content

Commit e4ce399

Browse files
kks32kks32
committed
Fix list_files failing on root paths with empty path component
Tapis requires a non-empty path argument. When listing root of a storage system (e.g., tapis://designsafe.storage.community/), the parsed path is empty. Default to "/" in that case.
1 parent 9af7efa commit e4ce399

2 files changed

Lines changed: 153 additions & 20 deletions

File tree

‎dapi/files.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,8 @@ def list_files(
542542
"""
543543
try:
544544
system_id, path = _parse_tapis_uri(remote_uri)
545+
if not path:
546+
path = "/"
545547
print(f"Listing files in system '{system_id}' at path '{path}'...")
546548
# URL-encode the path for API call
547549
encoded_path = _safe_quote(path)

‎examples/files.ipynb‎

Lines changed: 151 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,27 @@
2222
},
2323
{
2424
"cell_type": "code",
25-
"execution_count": null,
25+
"execution_count": 1,
2626
"metadata": {},
27-
"outputs": [],
27+
"outputs": [
28+
{
29+
"name": "stderr",
30+
"output_type": "stream",
31+
"text": [
32+
"/Users/krishna/dev/DesignSafe/Dapi-Tapis/dapi/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
33+
" from .autonotebook import tqdm as notebook_tqdm\n"
34+
]
35+
},
36+
{
37+
"name": "stdout",
38+
"output_type": "stream",
39+
"text": [
40+
"Authentication successful.\n",
41+
"DatabaseAccessor initialized. Connections will be created on first access.\n",
42+
"TMS credentials ready: frontera, stampede3, ls6\n"
43+
]
44+
}
45+
],
2846
"source": [
2947
"from dapi import DSClient\n",
3048
"\n",
@@ -42,9 +60,18 @@
4260
},
4361
{
4462
"cell_type": "code",
45-
"execution_count": null,
63+
"execution_count": 2,
4664
"metadata": {},
47-
"outputs": [],
65+
"outputs": [
66+
{
67+
"name": "stdout",
68+
"output_type": "stream",
69+
"text": [
70+
"Translated '/MyData/folder/' to 'tapis://designsafe.storage.default/kks32/folder/' using t.username\n",
71+
"MyData: tapis://designsafe.storage.default/kks32/folder/\n"
72+
]
73+
}
74+
],
4875
"source": [
4976
"# MyData\n",
5077
"uri = ds.files.to_uri(\"/MyData/folder/\")\n",
@@ -53,9 +80,18 @@
5380
},
5481
{
5582
"cell_type": "code",
56-
"execution_count": null,
83+
"execution_count": 3,
5784
"metadata": {},
58-
"outputs": [],
85+
"outputs": [
86+
{
87+
"name": "stdout",
88+
"output_type": "stream",
89+
"text": [
90+
"Translated '/CommunityData/app_examples/' to 'tapis://designsafe.storage.community/app_examples/'\n",
91+
"CommunityData: tapis://designsafe.storage.community/app_examples/\n"
92+
]
93+
}
94+
],
5995
"source": [
6096
"# CommunityData\n",
6197
"uri = ds.files.to_uri(\"/CommunityData/app_examples/\")\n",
@@ -64,9 +100,18 @@
64100
},
65101
{
66102
"cell_type": "code",
67-
"execution_count": null,
103+
"execution_count": 4,
68104
"metadata": {},
69-
"outputs": [],
105+
"outputs": [
106+
{
107+
"name": "stdout",
108+
"output_type": "stream",
109+
"text": [
110+
"Translated '/NHERI-Published/PRJ-1271/' to 'tapis://designsafe.storage.published/PRJ-1271/'\n",
111+
"NHERI-Published: tapis://designsafe.storage.published/PRJ-1271/\n"
112+
]
113+
}
114+
],
70115
"source": [
71116
"# NHERI-Published\n",
72117
"uri = ds.files.to_uri(\"/NHERI-Published/PRJ-1271/\")\n",
@@ -75,9 +120,18 @@
75120
},
76121
{
77122
"cell_type": "code",
78-
"execution_count": null,
123+
"execution_count": 5,
79124
"metadata": {},
80-
"outputs": [],
125+
"outputs": [
126+
{
127+
"name": "stdout",
128+
"output_type": "stream",
129+
"text": [
130+
"Translated '/NEES/' to 'tapis://nees.public/'\n",
131+
"NEES: tapis://nees.public/\n"
132+
]
133+
}
134+
],
81135
"source": [
82136
"# NEES\n",
83137
"uri = ds.files.to_uri(\"/NEES/\")\n",
@@ -86,9 +140,18 @@
86140
},
87141
{
88142
"cell_type": "code",
89-
"execution_count": null,
143+
"execution_count": 6,
90144
"metadata": {},
91-
"outputs": [],
145+
"outputs": [
146+
{
147+
"name": "stdout",
148+
"output_type": "stream",
149+
"text": [
150+
"Translated '/MyProjects/PRJ-1305/' to 'tapis://project-7997906542076432871-242ac11c-0001-012/'\n",
151+
"MyProjects: tapis://project-7997906542076432871-242ac11c-0001-012/\n"
152+
]
153+
}
154+
],
92155
"source": [
93156
"# Projects - dapi resolves the PRJ number to the Tapis system UUID\n",
94157
"uri = ds.files.to_uri(\"/MyProjects/PRJ-1305/\")\n",
@@ -104,9 +167,38 @@
104167
},
105168
{
106169
"cell_type": "code",
107-
"execution_count": null,
170+
"execution_count": 7,
108171
"metadata": {},
109-
"outputs": [],
172+
"outputs": [
173+
{
174+
"name": "stdout",
175+
"output_type": "stream",
176+
"text": [
177+
"Translated '/CommunityData/' to 'tapis://designsafe.storage.community/'\n",
178+
"Listing files in system 'designsafe.storage.community' at path ''...\n"
179+
]
180+
},
181+
{
182+
"ename": "FileOperationError",
183+
"evalue": "Tapis file listing failed for 'tapis://designsafe.storage.community/': message: path is a required argument and cannot be None.",
184+
"output_type": "error",
185+
"traceback": [
186+
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
187+
"\u001b[31mInvalidInputError\u001b[39m Traceback (most recent call last)",
188+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/dapi/files.py:548\u001b[39m, in \u001b[36mlist_files\u001b[39m\u001b[34m(t, remote_uri, limit, offset)\u001b[39m\n\u001b[32m 547\u001b[39m encoded_path = _safe_quote(path)\n\u001b[32m--> \u001b[39m\u001b[32m548\u001b[39m results = \u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlistFiles\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 549\u001b[39m \u001b[43m \u001b[49m\u001b[43msystemId\u001b[49m\u001b[43m=\u001b[49m\u001b[43msystem_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencoded_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlimit\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[43m=\u001b[49m\u001b[43moffset\u001b[49m\n\u001b[32m 550\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 551\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFound \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(results)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m items.\u001b[39m\u001b[33m\"\u001b[39m)\n",
189+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/.venv/lib/python3.13/site-packages/tapipy/util.py:152\u001b[39m, in \u001b[36mretriable.<locals>.wrapper\u001b[39m\u001b[34m(self, _config, *args, **kwargs)\u001b[39m\n\u001b[32m 150\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m152\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exception\n\u001b[32m 154\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exception\n",
190+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/.venv/lib/python3.13/site-packages/tapipy/util.py:138\u001b[39m, in \u001b[36mretriable.<locals>.wrapper\u001b[39m\u001b[34m(self, _config, *args, **kwargs)\u001b[39m\n\u001b[32m 137\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m138\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mop__call__\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 139\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
191+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/.venv/lib/python3.13/site-packages/tapipy/tapis.py:1024\u001b[39m, in \u001b[36mOperation.__call__\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m 1023\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m param.get(\u001b[33m\"\u001b[39m\u001b[33mrequired\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m p_val:\n\u001b[32m-> \u001b[39m\u001b[32m1024\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m errors.InvalidInputError(msg=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparam[\u001b[33m'\u001b[39m\u001b[33mname\u001b[39m\u001b[33m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is a required argument and cannot be None.\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 1025\u001b[39m \u001b[38;5;66;03m# replace the parameter in the path template with the parameter value\u001b[39;00m\n",
192+
"\u001b[31mInvalidInputError\u001b[39m: message: path is a required argument and cannot be None.",
193+
"\nThe above exception was the direct cause of the following exception:\n",
194+
"\u001b[31mFileOperationError\u001b[39m Traceback (most recent call last)",
195+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# List files in CommunityData\u001b[39;00m\n\u001b[32m 2\u001b[39m uri = ds.files.to_uri(\u001b[33m\"\u001b[39m\u001b[33m/CommunityData/\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m files = \u001b[43mds\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlist\u001b[49m\u001b[43m(\u001b[49m\u001b[43muri\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m files[:\u001b[32m10\u001b[39m]:\n\u001b[32m 5\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mf.name\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m50s\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mf.type\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n",
196+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/dapi/client.py:250\u001b[39m, in \u001b[36mFileMethods.list\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 235\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mlist\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs) -> List[Tapis]:\n\u001b[32m 236\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"List files and directories in a Tapis storage system path.\u001b[39;00m\n\u001b[32m 237\u001b[39m \n\u001b[32m 238\u001b[39m \u001b[33;03m This is a convenience wrapper around files_module.list_files().\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 248\u001b[39m \u001b[33;03m FileOperationError: If the listing operation fails.\u001b[39;00m\n\u001b[32m 249\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m250\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfiles_module\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlist_files\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_tapis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
197+
"\u001b[36mFile \u001b[39m\u001b[32m~/dev/DesignSafe/Dapi-Tapis/dapi/dapi/files.py:557\u001b[39m, in \u001b[36mlist_files\u001b[39m\u001b[34m(t, remote_uri, limit, offset)\u001b[39m\n\u001b[32m 555\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m FileOperationError(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mRemote path not found at \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mremote_uri\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 556\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m557\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m FileOperationError(\n\u001b[32m 558\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTapis file listing failed for \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mremote_uri\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 559\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 560\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mException\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 561\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m FileOperationError(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to list files at \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mremote_uri\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n",
198+
"\u001b[31mFileOperationError\u001b[39m: Tapis file listing failed for 'tapis://designsafe.storage.community/': message: path is a required argument and cannot be None."
199+
]
200+
}
201+
],
110202
"source": [
111203
"# List files in CommunityData\n",
112204
"uri = ds.files.to_uri(\"/CommunityData/\")\n",
@@ -117,9 +209,29 @@
117209
},
118210
{
119211
"cell_type": "code",
120-
"execution_count": null,
212+
"execution_count": 8,
121213
"metadata": {},
122-
"outputs": [],
214+
"outputs": [
215+
{
216+
"name": "stdout",
217+
"output_type": "stream",
218+
"text": [
219+
"Translated '/MyProjects/PRJ-1305/Training/' to 'tapis://project-7997906542076432871-242ac11c-0001-012/Training/'\n",
220+
"Listing files in system 'project-7997906542076432871-242ac11c-0001-012' at path 'Training/'...\n",
221+
"Found 26 items.\n",
222+
"2023-NHERI-Academy dir\n",
223+
"2024-NHERI-AI-Academy dir\n",
224+
"2025-SPARC dir\n",
225+
"Computational-Workflows-on-DesignSafe dir\n",
226+
"dapi dir\n",
227+
"Real-Time_Traffic_Incident_Reports_20250818.csv file\n",
228+
"REU_Workshop_2018 dir\n",
229+
"SampleData dir\n",
230+
"ScientificProgramming-Python2024 dir\n",
231+
"template dir\n"
232+
]
233+
}
234+
],
123235
"source": [
124236
"# List files in a project\n",
125237
"uri = ds.files.to_uri(\"/MyProjects/PRJ-1305/Training/\")\n",
@@ -137,9 +249,20 @@
137249
},
138250
{
139251
"cell_type": "code",
140-
"execution_count": null,
252+
"execution_count": 9,
141253
"metadata": {},
142-
"outputs": [],
254+
"outputs": [
255+
{
256+
"name": "stdout",
257+
"output_type": "stream",
258+
"text": [
259+
"/home/jupyter/MyData/results/\n",
260+
"/home/jupyter/CommunityData/app_examples/\n",
261+
"/home/jupyter/NHERI-Published/PRJ-1271/\n",
262+
"/home/jupyter/NEES/NEES-2011-1050.groups/\n"
263+
]
264+
}
265+
],
143266
"source": [
144267
"# Convert Tapis URIs back to JupyterHub paths\n",
145268
"print(ds.files.to_path(\"tapis://designsafe.storage.default/kks32/results/\"))\n",
@@ -186,13 +309,21 @@
186309
],
187310
"metadata": {
188311
"kernelspec": {
189-
"display_name": "Python 3",
312+
"display_name": ".venv",
190313
"language": "python",
191314
"name": "python3"
192315
},
193316
"language_info": {
317+
"codemirror_mode": {
318+
"name": "ipython",
319+
"version": 3
320+
},
321+
"file_extension": ".py",
322+
"mimetype": "text/x-python",
194323
"name": "python",
195-
"version": "3.12.0"
324+
"nbconvert_exporter": "python",
325+
"pygments_lexer": "ipython3",
326+
"version": "3.13.5"
196327
}
197328
},
198329
"nbformat": 4,

0 commit comments

Comments
 (0)