From 23fd379e18f3b05b29036fbaef9221e2a2a3f355 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 20 Dec 2025 18:29:33 +0100 Subject: [PATCH 1/5] upath.core: raise NotADirectory in iterdir for all UPath classes --- upath/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/upath/core.py b/upath/core.py index fdc66d95..a8a72669 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1183,7 +1183,11 @@ def iterdir(self) -> Iterator[Self]: base = self if self.parts[-1:] == ("",): base = self.parent - for name in base.fs.listdir(base.path): + fs = base.fs + base_path = base.path + if not fs.isdir(base_path): + raise NotADirectoryError(str(self)) + for name in fs.listdir(base_path): # fsspec returns dictionaries if isinstance(name, dict): name = name.get("name") @@ -1192,7 +1196,7 @@ def iterdir(self) -> Iterator[Self]: continue # only want the path name with iterdir _, _, name = name.removesuffix(sep).rpartition(self.parser.sep) - yield base.with_segments(base.path, name) + yield base.with_segments(base_path, name) def __open_reader__(self) -> BinaryIO: return self.fs.open(self.path, mode="rb") From fb705a1592cf7e909d9f8a52890b35cd7f8ef748 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 20 Dec 2025 18:36:23 +0100 Subject: [PATCH 2/5] upath.implementations: remove obsolete overrides --- upath/implementations/cached.py | 8 -------- upath/implementations/cloud.py | 8 -------- upath/implementations/ftp.py | 7 ------- upath/implementations/github.py | 8 -------- upath/implementations/hdfs.py | 8 -------- upath/implementations/http.py | 34 --------------------------------- upath/implementations/memory.py | 8 -------- upath/implementations/sftp.py | 9 --------- upath/implementations/smb.py | 7 ------- upath/implementations/tar.py | 2 -- upath/implementations/zip.py | 8 -------- 11 files changed, 107 deletions(-) diff --git a/upath/implementations/cached.py b/upath/implementations/cached.py index 0da528a6..b3e23c98 100644 --- a/upath/implementations/cached.py +++ b/upath/implementations/cached.py @@ -8,16 +8,13 @@ from upath.types import JoinablePathLike if TYPE_CHECKING: - from collections.abc import Iterator from collections.abc import Mapping from typing import Any from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from fsspec import AbstractFileSystem @@ -62,8 +59,3 @@ def storage_options(self) -> Mapping[str, Any]: so = self._storage_options.copy() so.pop("fo", None) return MappingProxyType(so) - - def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index f662472e..3f037796 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from typing import TYPE_CHECKING from typing import Any @@ -14,10 +13,8 @@ from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -88,11 +85,6 @@ def mkdir( raise FileExistsError(self.path) super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok) - def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() - class GCSPath(CloudPath): __slots__ = () diff --git a/upath/implementations/ftp.py b/upath/implementations/ftp.py index 69698441..6f383245 100644 --- a/upath/implementations/ftp.py +++ b/upath/implementations/ftp.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from ftplib import error_perm as FTPPermanentError # nosec B402 from typing import TYPE_CHECKING @@ -53,12 +52,6 @@ def mkdir( return raise FileExistsError(str(self)) from e - def iterdir(self) -> Iterator[Self]: - if not self.is_dir(): - raise NotADirectoryError(str(self)) - else: - return super().iterdir() - def rename( self, target: WritablePathLike, diff --git a/upath/implementations/github.py b/upath/implementations/github.py index f8e47706..b95dd11e 100644 --- a/upath/implementations/github.py +++ b/upath/implementations/github.py @@ -5,7 +5,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from collections.abc import Sequence from typing import TYPE_CHECKING @@ -16,10 +15,8 @@ from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -52,11 +49,6 @@ def path(self) -> str: return "" return pth - def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() - @property def parts(self) -> Sequence[str]: parts = super().parts diff --git a/upath/implementations/hdfs.py b/upath/implementations/hdfs.py index 72dae2a3..368dff2e 100644 --- a/upath/implementations/hdfs.py +++ b/upath/implementations/hdfs.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from typing import TYPE_CHECKING from upath.core import UPath @@ -11,10 +10,8 @@ from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -42,8 +39,3 @@ def mkdir( if not exist_ok and self.exists(): raise FileExistsError(str(self)) super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok) - - def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() diff --git a/upath/implementations/http.py b/upath/implementations/http.py index 34b8ec45..166819f5 100644 --- a/upath/implementations/http.py +++ b/upath/implementations/http.py @@ -65,40 +65,6 @@ def path(self) -> str: sr = urlsplit(super().path) return sr._replace(path=sr.path or "/").geturl() - def is_file(self, *, follow_symlinks: bool = True) -> bool: - if not follow_symlinks: - warnings.warn( - f"{type(self).__name__}.is_file(follow_symlinks=False):" - " is currently ignored.", - UserWarning, - stacklevel=2, - ) - try: - next(super().iterdir()) - except (StopIteration, NotADirectoryError): - return True - except FileNotFoundError: - return False - else: - return False - - def is_dir(self, *, follow_symlinks: bool = True) -> bool: - if not follow_symlinks: - warnings.warn( - f"{type(self).__name__}.is_dir(follow_symlinks=False):" - " is currently ignored.", - UserWarning, - stacklevel=2, - ) - try: - next(super().iterdir()) - except (StopIteration, NotADirectoryError): - return False - except FileNotFoundError: - return False - else: - return True - def stat(self, follow_symlinks: bool = True) -> StatResultType: if not follow_symlinks: warnings.warn( diff --git a/upath/implementations/memory.py b/upath/implementations/memory.py index a912645b..ff9482f9 100644 --- a/upath/implementations/memory.py +++ b/upath/implementations/memory.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from typing import TYPE_CHECKING from upath.core import UPath @@ -11,10 +10,8 @@ from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -36,11 +33,6 @@ def __init__( **storage_options: Unpack[MemoryStorageOptions], ) -> None: ... - def iterdir(self) -> Iterator[Self]: - if not self.is_dir(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() - @property def path(self) -> str: path = super().path diff --git a/upath/implementations/sftp.py b/upath/implementations/sftp.py index ff924a52..e4204757 100644 --- a/upath/implementations/sftp.py +++ b/upath/implementations/sftp.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from collections.abc import Iterator from typing import TYPE_CHECKING from upath.core import UPath @@ -11,10 +10,8 @@ from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -48,9 +45,3 @@ def __str__(self) -> str: if path_str.startswith(("ssh:///", "sftp:///")): return path_str.removesuffix("/") return path_str - - def iterdir(self) -> Iterator[Self]: - if not self.is_dir(): - raise NotADirectoryError(str(self)) - else: - return super().iterdir() diff --git a/upath/implementations/smb.py b/upath/implementations/smb.py index 53ea99a3..c967e370 100644 --- a/upath/implementations/smb.py +++ b/upath/implementations/smb.py @@ -2,7 +2,6 @@ import sys import warnings -from collections.abc import Iterator from typing import TYPE_CHECKING from typing import Any @@ -73,12 +72,6 @@ def mkdir( if not self.is_dir(): raise FileExistsError(str(self)) - def iterdir(self) -> Iterator[Self]: - if not self.is_dir(): - raise NotADirectoryError(str(self)) - else: - return super().iterdir() - def rename( self, target: WritablePathLike, diff --git a/upath/implementations/tar.py b/upath/implementations/tar.py index e383329e..9853b4e3 100644 --- a/upath/implementations/tar.py +++ b/upath/implementations/tar.py @@ -62,8 +62,6 @@ def stat( return UPathStatResult.from_info(info) def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) it = iter(super().iterdir()) p0 = next(it) if p0.name != "": diff --git a/upath/implementations/zip.py b/upath/implementations/zip.py index 73500657..237cae15 100644 --- a/upath/implementations/zip.py +++ b/upath/implementations/zip.py @@ -8,14 +8,11 @@ from upath.types import JoinablePathLike if TYPE_CHECKING: - from collections.abc import Iterator from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -38,11 +35,6 @@ def __init__( **storage_options: Unpack[ZipStorageOptions], ) -> None: ... - def iterdir(self) -> Iterator[Self]: - if self.is_file(): - raise NotADirectoryError(str(self)) - yield from super().iterdir() - if sys.version_info >= (3, 11): def mkdir( From 6ca19983ee367a2adabbbef32da037ef73b920f7 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 20 Dec 2025 18:37:57 +0100 Subject: [PATCH 3/5] tests: a filesystem resource can be file and dir at the same time --- upath/tests/cases.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/upath/tests/cases.py b/upath/tests/cases.py index f93e1dd0..16255fe0 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -115,10 +115,8 @@ def test_is_dir(self): assert not (self.path / "not-existing-dir").is_dir() def test_is_file(self): - path = self.path / "file1.txt" - assert path.is_file() - assert not self.path.is_file() - + path_exists = self.path / "file1.txt" + assert path_exists.is_file() assert not (self.path / "not-existing-file.txt").is_file() def test_is_absolute(self): From 5d1569eebae5bef74ba7345376109ba48353d9e2 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 20 Dec 2025 18:38:26 +0100 Subject: [PATCH 4/5] tests: fix data iterdir test --- upath/tests/implementations/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upath/tests/implementations/test_data.py b/upath/tests/implementations/test_data.py index 248fb5c5..ac632a42 100644 --- a/upath/tests/implementations/test_data.py +++ b/upath/tests/implementations/test_data.py @@ -61,7 +61,7 @@ def test_is_file(self): assert self.path.is_file() def test_iterdir(self): - with pytest.raises(NotImplementedError): + with pytest.raises(NotADirectoryError): list(self.path.iterdir()) @pytest.mark.skip(reason="DataPath does not have directories") From 5a7abd5d1b216ca405a1a3207538e5e594203ac1 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 20 Dec 2025 18:39:19 +0100 Subject: [PATCH 5/5] tests: mock iterdir test succeeds now --- upath/tests/test_core.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 7864cf3f..ef8138de 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -17,7 +17,6 @@ from .cases import BaseTests from .utils import only_on_windows from .utils import skip_on_windows -from .utils import xfail_if_version @skip_on_windows @@ -73,12 +72,6 @@ def test_home(self): ): type(self.path).home() - @xfail_if_version("fsspec", reason="", ge="2024.2.0") - def test_iterdir_no_dir(self): - # the mock filesystem is basically just LocalFileSystem, - # so this test would need to have an iterdir fix. - super().test_iterdir_no_dir() - @pytest.mark.skipif( sys.platform.startswith("win"), reason="mock fs is not well defined on windows",