Fix iterdir behaviour when raising NotADirectory for all UPath subclasses

upath/core.py

@@ -1183,7 +1183,11 @@ def iterdir(self) -> Iterator[Self]:
         base = self
         if self.parts[-1:] == ("",):
             base = self.parent
-        for name in base.fs.listdir(base.path):
+        fs = base.fs
+        base_path = base.path
+        if not fs.isdir(base_path):
+            raise NotADirectoryError(str(self))
+        for name in fs.listdir(base_path):
             # fsspec returns dictionaries
             if isinstance(name, dict):
                 name = name.get("name")
@@ -1192,7 +1196,7 @@ def iterdir(self) -> Iterator[Self]:
                 continue
             # only want the path name with iterdir
             _, _, name = name.removesuffix(sep).rpartition(self.parser.sep)
-            yield base.with_segments(base.path, name)
+            yield base.with_segments(base_path, name)
 
     def __open_reader__(self) -> BinaryIO:
         return self.fs.open(self.path, mode="rb")

upath/implementations/cached.py

@@ -8,16 +8,13 @@
 from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
     from collections.abc import Mapping
     from typing import Any
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from fsspec import AbstractFileSystem
@@ -62,8 +59,3 @@ def storage_options(self) -> Mapping[str, Any]:
         so = self._storage_options.copy()
         so.pop("fo", None)
         return MappingProxyType(so)
-
-    def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()

upath/implementations/cloud.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from typing import TYPE_CHECKING
 from typing import Any
 
@@ -14,10 +13,8 @@
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -88,11 +85,6 @@ def mkdir(
             raise FileExistsError(self.path)
         super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok)
 
-    def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()
-
 
 class GCSPath(CloudPath):
     __slots__ = ()

upath/implementations/ftp.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from ftplib import error_perm as FTPPermanentError  # nosec B402
 from typing import TYPE_CHECKING
 
@@ -53,12 +52,6 @@ def mkdir(
                 return
             raise FileExistsError(str(self)) from e
 
-    def iterdir(self) -> Iterator[Self]:
-        if not self.is_dir():
-            raise NotADirectoryError(str(self))
-        else:
-            return super().iterdir()
-
     def rename(
         self,
         target: WritablePathLike,

upath/implementations/github.py

@@ -5,7 +5,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
@@ -16,10 +15,8 @@
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -52,11 +49,6 @@ def path(self) -> str:
             return ""
         return pth
 
-    def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()
-
     @property
     def parts(self) -> Sequence[str]:
         parts = super().parts

upath/implementations/hdfs.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from typing import TYPE_CHECKING
 
 from upath.core import UPath
@@ -11,10 +10,8 @@
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -42,8 +39,3 @@ def mkdir(
         if not exist_ok and self.exists():
             raise FileExistsError(str(self))
         super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok)
-
-    def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()

upath/implementations/http.py

@@ -65,40 +65,6 @@ def path(self) -> str:
         sr = urlsplit(super().path)
         return sr._replace(path=sr.path or "/").geturl()
 
-    def is_file(self, *, follow_symlinks: bool = True) -> bool:
-        if not follow_symlinks:
-            warnings.warn(
-                f"{type(self).__name__}.is_file(follow_symlinks=False):"
-                " is currently ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
-        try:
-            next(super().iterdir())
-        except (StopIteration, NotADirectoryError):
-            return True
-        except FileNotFoundError:
-            return False
-        else:
-            return False
-
-    def is_dir(self, *, follow_symlinks: bool = True) -> bool:
-        if not follow_symlinks:
-            warnings.warn(
-                f"{type(self).__name__}.is_dir(follow_symlinks=False):"
-                " is currently ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
-        try:
-            next(super().iterdir())
-        except (StopIteration, NotADirectoryError):
-            return False
-        except FileNotFoundError:
-            return False
-        else:
-            return True
-
     def stat(self, follow_symlinks: bool = True) -> StatResultType:
         if not follow_symlinks:
             warnings.warn(

upath/implementations/memory.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from typing import TYPE_CHECKING
 
 from upath.core import UPath
@@ -11,10 +10,8 @@
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -36,11 +33,6 @@ def __init__(
             **storage_options: Unpack[MemoryStorageOptions],
         ) -> None: ...
 
-    def iterdir(self) -> Iterator[Self]:
-        if not self.is_dir():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()
-
     @property
     def path(self) -> str:
         path = super().path

upath/implementations/sftp.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import sys
-from collections.abc import Iterator
 from typing import TYPE_CHECKING
 
 from upath.core import UPath
@@ -11,10 +10,8 @@
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -48,9 +45,3 @@ def __str__(self) -> str:
         if path_str.startswith(("ssh:///", "sftp:///")):
             return path_str.removesuffix("/")
         return path_str
-
-    def iterdir(self) -> Iterator[Self]:
-        if not self.is_dir():
-            raise NotADirectoryError(str(self))
-        else:
-            return super().iterdir()

upath/implementations/smb.py

@@ -2,7 +2,6 @@
 
 import sys
 import warnings
-from collections.abc import Iterator
 from typing import TYPE_CHECKING
 from typing import Any
 
@@ -73,12 +72,6 @@ def mkdir(
             if not self.is_dir():
                 raise FileExistsError(str(self))
 
-    def iterdir(self) -> Iterator[Self]:
-        if not self.is_dir():
-            raise NotADirectoryError(str(self))
-        else:
-            return super().iterdir()
-
     def rename(
         self,
         target: WritablePathLike,

upath/implementations/tar.py

@@ -62,8 +62,6 @@ def stat(
         return UPathStatResult.from_info(info)
 
     def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
         it = iter(super().iterdir())
         p0 = next(it)
         if p0.name != "":

upath/implementations/zip.py

@@ -8,14 +8,11 @@
 from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
     from typing import Literal
 
     if sys.version_info >= (3, 11):
-        from typing import Self
         from typing import Unpack
     else:
-        from typing_extensions import Self
         from typing_extensions import Unpack
 
     from upath._chain import FSSpecChainParser
@@ -38,11 +35,6 @@ def __init__(
             **storage_options: Unpack[ZipStorageOptions],
         ) -> None: ...
 
-    def iterdir(self) -> Iterator[Self]:
-        if self.is_file():
-            raise NotADirectoryError(str(self))
-        yield from super().iterdir()
-
     if sys.version_info >= (3, 11):
 
         def mkdir(

upath/tests/cases.py

@@ -115,10 +115,8 @@ def test_is_dir(self):
         assert not (self.path / "not-existing-dir").is_dir()
 
     def test_is_file(self):
-        path = self.path / "file1.txt"
-        assert path.is_file()
-        assert not self.path.is_file()
-
+        path_exists = self.path / "file1.txt"
+        assert path_exists.is_file()
         assert not (self.path / "not-existing-file.txt").is_file()
 
     def test_is_absolute(self):

upath/tests/implementations/test_data.py

@@ -61,7 +61,7 @@ def test_is_file(self):
         assert self.path.is_file()
 
     def test_iterdir(self):
-        with pytest.raises(NotImplementedError):
+        with pytest.raises(NotADirectoryError):
             list(self.path.iterdir())
 
     @pytest.mark.skip(reason="DataPath does not have directories")

upath/tests/test_core.py

@@ -17,7 +17,6 @@
 from .cases import BaseTests
 from .utils import only_on_windows
 from .utils import skip_on_windows
-from .utils import xfail_if_version
 
 
 @skip_on_windows
@@ -73,12 +72,6 @@ def test_home(self):
         ):
             type(self.path).home()
 
-    @xfail_if_version("fsspec", reason="", ge="2024.2.0")
-    def test_iterdir_no_dir(self):
-        # the mock filesystem is basically just LocalFileSystem,
-        # so this test would need to have an iterdir fix.
-        super().test_iterdir_no_dir()
-
     @pytest.mark.skipif(
         sys.platform.startswith("win"),
         reason="mock fs is not well defined on windows",