Skip to content

Commit d9e80ff

Browse files
ap--ap--
authored
Fix iterdir behaviour when raising NotADirectory for all UPath subclasses (#506)
* upath.core: raise NotADirectory in iterdir for all UPath classes * upath.implementations: remove obsolete overrides * tests: a filesystem resource can be file and dir at the same time * tests: fix data iterdir test * tests: mock iterdir test succeeds now
1 parent 76a9430 commit d9e80ff

15 files changed

Lines changed: 9 additions & 121 deletions

File tree

‎upath/core.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1183,7 +1183,11 @@ def iterdir(self) -> Iterator[Self]:
11831183
base = self
11841184
if self.parts[-1:] == ("",):
11851185
base = self.parent
1186-
for name in base.fs.listdir(base.path):
1186+
fs = base.fs
1187+
base_path = base.path
1188+
if not fs.isdir(base_path):
1189+
raise NotADirectoryError(str(self))
1190+
for name in fs.listdir(base_path):
11871191
# fsspec returns dictionaries
11881192
if isinstance(name, dict):
11891193
name = name.get("name")
@@ -1192,7 +1196,7 @@ def iterdir(self) -> Iterator[Self]:
11921196
continue
11931197
# only want the path name with iterdir
11941198
_, _, name = name.removesuffix(sep).rpartition(self.parser.sep)
1195-
yield base.with_segments(base.path, name)
1199+
yield base.with_segments(base_path, name)
11961200

11971201
def __open_reader__(self) -> BinaryIO:
11981202
return self.fs.open(self.path, mode="rb")

‎upath/implementations/cached.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,13 @@
88
from upath.types import JoinablePathLike
99

1010
if TYPE_CHECKING:
11-
from collections.abc import Iterator
1211
from collections.abc import Mapping
1312
from typing import Any
1413
from typing import Literal
1514

1615
if sys.version_info >= (3, 11):
17-
from typing import Self
1816
from typing import Unpack
1917
else:
20-
from typing_extensions import Self
2118
from typing_extensions import Unpack
2219

2320
from fsspec import AbstractFileSystem
@@ -62,8 +59,3 @@ def storage_options(self) -> Mapping[str, Any]:
6259
so = self._storage_options.copy()
6360
so.pop("fo", None)
6461
return MappingProxyType(so)
65-
66-
def iterdir(self) -> Iterator[Self]:
67-
if self.is_file():
68-
raise NotADirectoryError(str(self))
69-
yield from super().iterdir()

‎upath/implementations/cloud.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import sys
4-
from collections.abc import Iterator
54
from typing import TYPE_CHECKING
65
from typing import Any
76

@@ -14,10 +13,8 @@
1413
from typing import Literal
1514

1615
if sys.version_info >= (3, 11):
17-
from typing import Self
1816
from typing import Unpack
1917
else:
20-
from typing_extensions import Self
2118
from typing_extensions import Unpack
2219

2320
from upath._chain import FSSpecChainParser
@@ -88,11 +85,6 @@ def mkdir(
8885
raise FileExistsError(self.path)
8986
super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok)
9087

91-
def iterdir(self) -> Iterator[Self]:
92-
if self.is_file():
93-
raise NotADirectoryError(str(self))
94-
yield from super().iterdir()
95-
9688

9789
class GCSPath(CloudPath):
9890
__slots__ = ()

‎upath/implementations/ftp.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import sys
4-
from collections.abc import Iterator
54
from ftplib import error_perm as FTPPermanentError # nosec B402
65
from typing import TYPE_CHECKING
76

@@ -53,12 +52,6 @@ def mkdir(
5352
return
5453
raise FileExistsError(str(self)) from e
5554

56-
def iterdir(self) -> Iterator[Self]:
57-
if not self.is_dir():
58-
raise NotADirectoryError(str(self))
59-
else:
60-
return super().iterdir()
61-
6255
def rename(
6356
self,
6457
target: WritablePathLike,

‎upath/implementations/github.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from __future__ import annotations
66

77
import sys
8-
from collections.abc import Iterator
98
from collections.abc import Sequence
109
from typing import TYPE_CHECKING
1110

@@ -16,10 +15,8 @@
1615
from typing import Literal
1716

1817
if sys.version_info >= (3, 11):
19-
from typing import Self
2018
from typing import Unpack
2119
else:
22-
from typing_extensions import Self
2320
from typing_extensions import Unpack
2421

2522
from upath._chain import FSSpecChainParser
@@ -52,11 +49,6 @@ def path(self) -> str:
5249
return ""
5350
return pth
5451

55-
def iterdir(self) -> Iterator[Self]:
56-
if self.is_file():
57-
raise NotADirectoryError(str(self))
58-
yield from super().iterdir()
59-
6052
@property
6153
def parts(self) -> Sequence[str]:
6254
parts = super().parts

‎upath/implementations/hdfs.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import sys
4-
from collections.abc import Iterator
54
from typing import TYPE_CHECKING
65

76
from upath.core import UPath
@@ -11,10 +10,8 @@
1110
from typing import Literal
1211

1312
if sys.version_info >= (3, 11):
14-
from typing import Self
1513
from typing import Unpack
1614
else:
17-
from typing_extensions import Self
1815
from typing_extensions import Unpack
1916

2017
from upath._chain import FSSpecChainParser
@@ -42,8 +39,3 @@ def mkdir(
4239
if not exist_ok and self.exists():
4340
raise FileExistsError(str(self))
4441
super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok)
45-
46-
def iterdir(self) -> Iterator[Self]:
47-
if self.is_file():
48-
raise NotADirectoryError(str(self))
49-
yield from super().iterdir()

‎upath/implementations/http.py‎

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -65,40 +65,6 @@ def path(self) -> str:
6565
sr = urlsplit(super().path)
6666
return sr._replace(path=sr.path or "/").geturl()
6767

68-
def is_file(self, *, follow_symlinks: bool = True) -> bool:
69-
if not follow_symlinks:
70-
warnings.warn(
71-
f"{type(self).__name__}.is_file(follow_symlinks=False):"
72-
" is currently ignored.",
73-
UserWarning,
74-
stacklevel=2,
75-
)
76-
try:
77-
next(super().iterdir())
78-
except (StopIteration, NotADirectoryError):
79-
return True
80-
except FileNotFoundError:
81-
return False
82-
else:
83-
return False
84-
85-
def is_dir(self, *, follow_symlinks: bool = True) -> bool:
86-
if not follow_symlinks:
87-
warnings.warn(
88-
f"{type(self).__name__}.is_dir(follow_symlinks=False):"
89-
" is currently ignored.",
90-
UserWarning,
91-
stacklevel=2,
92-
)
93-
try:
94-
next(super().iterdir())
95-
except (StopIteration, NotADirectoryError):
96-
return False
97-
except FileNotFoundError:
98-
return False
99-
else:
100-
return True
101-
10268
def stat(self, follow_symlinks: bool = True) -> StatResultType:
10369
if not follow_symlinks:
10470
warnings.warn(

‎upath/implementations/memory.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import sys
4-
from collections.abc import Iterator
54
from typing import TYPE_CHECKING
65

76
from upath.core import UPath
@@ -11,10 +10,8 @@
1110
from typing import Literal
1211

1312
if sys.version_info >= (3, 11):
14-
from typing import Self
1513
from typing import Unpack
1614
else:
17-
from typing_extensions import Self
1815
from typing_extensions import Unpack
1916

2017
from upath._chain import FSSpecChainParser
@@ -36,11 +33,6 @@ def __init__(
3633
**storage_options: Unpack[MemoryStorageOptions],
3734
) -> None: ...
3835

39-
def iterdir(self) -> Iterator[Self]:
40-
if not self.is_dir():
41-
raise NotADirectoryError(str(self))
42-
yield from super().iterdir()
43-
4436
@property
4537
def path(self) -> str:
4638
path = super().path

‎upath/implementations/sftp.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import sys
4-
from collections.abc import Iterator
54
from typing import TYPE_CHECKING
65

76
from upath.core import UPath
@@ -11,10 +10,8 @@
1110
from typing import Literal
1211

1312
if sys.version_info >= (3, 11):
14-
from typing import Self
1513
from typing import Unpack
1614
else:
17-
from typing_extensions import Self
1815
from typing_extensions import Unpack
1916

2017
from upath._chain import FSSpecChainParser
@@ -48,9 +45,3 @@ def __str__(self) -> str:
4845
if path_str.startswith(("ssh:///", "sftp:///")):
4946
return path_str.removesuffix("/")
5047
return path_str
51-
52-
def iterdir(self) -> Iterator[Self]:
53-
if not self.is_dir():
54-
raise NotADirectoryError(str(self))
55-
else:
56-
return super().iterdir()

‎upath/implementations/smb.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import sys
44
import warnings
5-
from collections.abc import Iterator
65
from typing import TYPE_CHECKING
76
from typing import Any
87

@@ -73,12 +72,6 @@ def mkdir(
7372
if not self.is_dir():
7473
raise FileExistsError(str(self))
7574

76-
def iterdir(self) -> Iterator[Self]:
77-
if not self.is_dir():
78-
raise NotADirectoryError(str(self))
79-
else:
80-
return super().iterdir()
81-
8275
def rename(
8376
self,
8477
target: WritablePathLike,

0 commit comments

Comments
 (0)