Add support for Python 3.12

.github/workflows/tests.yml

@@ -42,7 +42,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3

cloudpathlib/cloudpath.py

@@ -8,10 +8,9 @@
     PosixPath,
     PurePosixPath,
     WindowsPath,
-    _make_selector,
-    _posix_flavour,
     _PathParents,
 )
+
 import shutil
 import sys
 from typing import (
@@ -44,6 +43,17 @@
 else:
     from typing_extensions import Self
 
+if sys.version_info >= (3, 12):
+    from pathlib import posixpath as _posix_flavour  # type: ignore[attr-defined]
+    from pathlib import _make_selector  # type: ignore[attr-defined]
+else:
+    from pathlib import _posix_flavour  # type: ignore[attr-defined]
+    from pathlib import _make_selector as _make_selector_pathlib  # type: ignore[attr-defined]
+
+    def _make_selector(pattern_parts, _flavour, case_sensitive=True):
+        return _make_selector_pathlib(tuple(pattern_parts), _flavour)
+
+
 from cloudpathlib.enums import FileCacheMode
 
 from . import anypath
@@ -406,7 +416,7 @@
                 ".glob is only supported within a bucket or container; you can use `.iterdir` to list buckets; for example, CloudPath('s3://').iterdir()"
             )
 
-    def _glob(self, selector, recursive: bool) -> Generator[Self, None, None]:
+    def _build_subtree(self, recursive):
         # build a tree structure for all files out of default dicts
         Tree: Callable = lambda: defaultdict(Tree)
 
@@ -433,7 +443,10 @@
             nodes = (p for p in parts)
             _build_tree(file_tree, next(nodes, None), nodes, is_dir)
 
-        file_tree = dict(file_tree)  # freeze as normal dict before passing in
+        return dict(file_tree)  # freeze as normal dict before passing in
+
+    def _glob(self, selector, recursive: bool) -> Generator[Self, None, None]:
+        file_tree = self._build_subtree(recursive)
 
         root = _CloudPathSelectable(
             self.name,
@@ -445,11 +458,15 @@
             # select_from returns self.name/... so strip before joining
             yield (self / str(p)[len(self.name) + 1 :])
 
-    def glob(self, pattern: str) -> Generator[Self, None, None]:
+    def glob(
+        self, pattern: str, case_sensitive: Optional[bool] = None
+    ) -> Generator[Self, None, None]:
         self._glob_checks(pattern)
 
         pattern_parts = PurePosixPath(pattern).parts
-        selector = _make_selector(tuple(pattern_parts), _posix_flavour)
+        selector = _make_selector(
+            tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
+        )
 
         yield from self._glob(
             selector,
@@ -458,11 +475,15 @@
             in pattern,  # recursive listing needed if explicit ** or any sub folder in pattern
         )
 
-    def rglob(self, pattern: str) -> Generator[Self, None, None]:
+    def rglob(
+        self, pattern: str, case_sensitive: Optional[bool] = None
+    ) -> Generator[Self, None, None]:
         self._glob_checks(pattern)
 
         pattern_parts = PurePosixPath(pattern).parts
-        selector = _make_selector(("**",) + tuple(pattern_parts), _posix_flavour)
+        selector = _make_selector(
+            ("**",) + tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
+        )
 
         yield from self._glob(selector, True)
 
@@ -471,6 +492,41 @@
             if f != self:  # iterdir does not include itself in pathlib
                 yield f
 
+    @staticmethod
+    def _walk_results_from_tree(root, tree, top_down=True):
+        """Utility to yield tuples in the form expected by `.walk` from the file
+        tree constructed by `_build_substree`.
+        """
+        dirs = []
+        files = []
+        for item, branch in tree.items():
+            files.append(item) if branch is None else dirs.append(item)
+
+        if top_down:
+            yield root, dirs, files
+
+        for dir in dirs:
+            yield from CloudPath._walk_results_from_tree(root / dir, tree[dir], top_down=top_down)
+
+        if not top_down:
+            yield root, dirs, files
+
+    def walk(
+        self,
+        top_down: bool = True,
+        on_error: Optional[Callable] = None,
+        follow_symlinks: bool = False,
+    ) -> Generator[Tuple[Self, List[str], List[str]], None, None]:
+        try:
+            file_tree = self._build_subtree(recursive=True)  # walking is always recursive
+            yield from self._walk_results_from_tree(self, file_tree, top_down=top_down)
+
+        except Exception as e:
+            if on_error is not None:
+                on_error(e)
+            else:
+                raise
+
     def open(
         self,
         mode: str = "r",
@@ -647,6 +703,9 @@
         with self.open(mode="r", encoding=encoding, errors=errors) as f:
             return f.read()
 
+    def is_junction(self):
+        return False  # only windows paths can be junctions, not cloudpaths
+
     # ====================== DISPATCHED TO POSIXPATH FOR PURE PATHS ======================
     # Methods that are dispatched to exactly how pathlib.PurePosixPath would calculate it on
     # self._path for pure paths (does not matter if file exists);
@@ -692,8 +751,8 @@
 
         return self._dispatch_to_path("__truediv__", other)
 
-    def joinpath(self, *args: Union[str, os.PathLike]) -> Self:
-        return self._dispatch_to_path("joinpath", *args)
+    def joinpath(self, *pathsegments: Union[str, os.PathLike]) -> Self:
+        return self._dispatch_to_path("joinpath", *pathsegments)
 
     def absolute(self) -> Self:
         return self
@@ -704,7 +763,7 @@
     def resolve(self, strict: bool = False) -> Self:
         return self
 
-    def relative_to(self, other: Self) -> PurePosixPath:
+    def relative_to(self, other: Self, walk_up: bool = False) -> PurePosixPath:
         # We don't dispatch regularly since this never returns a cloud path (since it is relative, and cloud paths are
         # absolute)
         if not isinstance(other, CloudPath):
@@ -713,7 +772,13 @@
             raise ValueError(
                 f"{self} is a {self.cloud_prefix} path, but {other} is a {other.cloud_prefix} path"
             )
-        return self._path.relative_to(other._path)
+
+        kwargs = dict(walk_up=walk_up)
+
+        if sys.version_info < (3, 12):
+            kwargs.pop("walk_up")
+
+        return self._path.relative_to(other._path, **kwargs)  # type: ignore[call-arg]
 
     def is_relative_to(self, other: Self) -> bool:
         try:
@@ -726,12 +791,17 @@
     def name(self) -> str:
         return self._dispatch_to_path("name")
 
-    def match(self, path_pattern: str) -> bool:
+    def match(self, path_pattern: str, case_sensitive: Optional[bool] = None) -> bool:
         # strip scheme from start of pattern before testing
         if path_pattern.startswith(self.anchor + self.drive + "/"):
             path_pattern = path_pattern[len(self.anchor + self.drive + "/") :]
 
-        return self._dispatch_to_path("match", path_pattern)
+        kwargs = dict(case_sensitive=case_sensitive)
+
+        if sys.version_info < (3, 12):
+            kwargs.pop("case_sensitive")
+
+        return self._dispatch_to_path("match", path_pattern, **kwargs)
 
     @property
     def parent(self) -> Self:
@@ -771,6 +841,9 @@
     def with_name(self, name: str) -> Self:
         return self._dispatch_to_path("with_name", name)
 
+    def with_segments(self, *pathsegments) -> Self:
+        return self._new_cloudpath("/".join(pathsegments))
+
     def with_suffix(self, suffix: str) -> Self:
         return self._dispatch_to_path("with_suffix", suffix)
 
@@ -1244,3 +1317,16 @@
         )
 
     _scandir = scandir  # Py 3.11 compatibility
+
+    def walk(self):
+        # split into dirs and files
+        dirs_files = defaultdict(list)
+        with self.scandir(self) as items:
+            for child in items:
+                dirs_files[child.is_dir()].append(child)
+
+            # top-down, so yield self before recursive call
+            yield self, [f.name for f in dirs_files[True]], [f.name for f in dirs_files[False]]
+
+            for child_dir in dirs_files[True]:
+                yield from child_dir.walk()

pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 requires-python = ">=3.7"
 dependencies = [
@@ -49,7 +50,7 @@ all = ["cloudpathlib[azure]", "cloudpathlib[gs]", "cloudpathlib[s3]"]
 
 [tool.black]
 line-length = 99
-target-version = ['py37', 'py38', 'py39', 'py310', 'py311']
+target-version = ['py37', 'py38', 'py39', 'py310', 'py311', 'py312']
 include = '\.pyi?$|\.ipynb$'
 extend-exclude = '''
 /(

requirements-dev.txt

@@ -20,7 +20,8 @@ pillow
 psutil
 pydantic
 pytest
-pytest-cases
+# pytest-cases
+git+https://github.com/jayqi/python-pytest-cases@packaging-version
 pytest-cov
 pytest-xdist
 python-dotenv

tests/performance/perf_file_listing.py

@@ -10,3 +10,12 @@ def glob(folder, recursive):
         return {"n_items": len(list(folder.rglob("*.item")))}
     else:
         return {"n_items": len(list(folder.glob("*.item")))}
+
+
+def walk(folder):
+    n_items = 0
+
+    for _, _, files in folder.walk():
+        n_items += len(files)
+
+    return {"n_items": n_items}

tests/performance/runner.py

@@ -14,7 +14,7 @@
 from cloudpathlib import CloudPath
 
 
-from perf_file_listing import folder_list, glob
+from perf_file_listing import folder_list, glob, walk
 
 
 # make loguru and tqdm play nicely together
@@ -137,6 +137,15 @@ def main(root, iterations, burn_in):
                 PerfRunConfig(name="Glob deep non-recursive", args=[deep, False], kwargs={}),
             ],
         ),
+        (
+            "Walk scenarios",
+            walk,
+            [
+                PerfRunConfig(name="Walk shallow", args=[shallow], kwargs={}),
+                PerfRunConfig(name="Walk normal", args=[normal], kwargs={}),
+                PerfRunConfig(name="Walk deep", args=[deep], kwargs={}),
+            ],
+        ),
     ]
 
     logger.info(

tests/test_cloudpath_instantiation.py

@@ -1,4 +1,7 @@
+import inspect
 import os
+from pathlib import PurePath
+import re
 
 import pytest
 
@@ -83,3 +86,59 @@ def test_dependencies_not_loaded(rig, monkeypatch):
 def test_is_pathlike(rig):
     p = rig.create_cloud_path("dir_0")
     assert isinstance(p, os.PathLike)
+
+
+def test_public_interface_is_superset(rig):
+    """Test that a CloudPath has all of the Path methods and properties. For methods
+    we also ensure that the only difference in the signature is that a CloudPath has
+    optional additional kwargs (which are likely added in subsequent Python versions).
+    """
+    lp = PurePath(".")
+    cp = rig.create_cloud_path("dir_0/file0_0.txt")
+
+    # Use regex to find the methods not implemented that are listed in the CloudPath code
+    not_implemented_section = re.search(
+        r"# =+ NOT IMPLEMENTED =+\n(.+?)\n\n", inspect.getsource(CloudPath), re.DOTALL
+    )
+
+    if not_implemented_section:
+        methods_not_implemented_str = not_implemented_section.group(1)
+        methods_not_implemented = re.findall(r"# (\w+)", methods_not_implemented_str)
+
+    for name, lp_member in inspect.getmembers(lp):
+        if name.startswith("_") or name in methods_not_implemented:
+            continue
+
+        # checks all public methods and properties
+        cp_member = getattr(cp, name, None)
+        assert cp_member is not None, f"CloudPath missing {name}"
+
+        # for methods, checks the function signature
+        if callable(lp_member):
+            cp_signature = inspect.signature(cp_member)
+            lp_signature = inspect.signature(lp_member)
+
+            # all parameters for Path method should be part of CloudPath signature
+            for parameter in lp_signature.parameters:
+                # some parameters like _deprecated in Path.is_relative_to are not really part of the signature
+                if parameter.startswith("_") or (
+                    name == "joinpath" and parameter in ["args", "pathsegments"]
+                ):  # handle arg name change in 3.12
+                    continue
+
+                assert (
+                    parameter in cp_signature.parameters
+                ), f"CloudPath.{name} missing parameter {parameter}"
+
+            # extra parameters for CloudPath method should be optional with defaults
+            for parameter, param_details in cp_signature.parameters.items():
+                if name == "joinpath" and parameter in [
+                    "args",
+                    "pathsegments",
+                ]:  # handle arg name change in 3.12
+                    continue
+
+                if parameter not in lp_signature.parameters:
+                    assert (
+                        param_details.default is not inspect.Parameter.empty
+                    ), f"CloudPath.{name} added parameter {parameter} without a default"