From 210e5083c107a7dceb2c1964c32a4b6fc81cdf7d Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Mon, 20 Feb 2023 16:53:47 -0500
Subject: [PATCH 1/8] refactor: deprecate imp for importlib

---
 invoke/config.py        |  3 +-
 invoke/loader.py        | 88 ++++++++++++++++++++---------------------
 invoke/program.py       |  1 +
 sites/www/changelog.rst |  2 +
 tests/loader.py         | 28 +++++++------
 tests/program.py        |  1 +
 6 files changed, 64 insertions(+), 59 deletions(-)

diff --git a/invoke/config.py b/invoke/config.py
index c38afc67..116f774a 100644
--- a/invoke/config.py
+++ b/invoke/config.py
@@ -908,8 +908,7 @@ def _load_yaml(self, path: PathLike) -> Any:
         with open(path) as fd:
             return yaml.safe_load(fd)
 
-    def _load_yml(self, path: PathLike) -> Any:
-        return self._load_yaml(path)
+    _load_yml = _load_yaml
 
     def _load_json(self, path: PathLike) -> Any:
         with open(path) as fd:
diff --git a/invoke/loader.py b/invoke/loader.py
index 23bffdf0..5550a5c0 100644
--- a/invoke/loader.py
+++ b/invoke/loader.py
@@ -1,8 +1,9 @@
 import os
 import sys
-import imp
+from importlib.machinery import ModuleSpec
+from importlib.util import module_from_spec, spec_from_file_location
 from types import ModuleType
-from typing import Any, IO, Optional, Tuple
+from typing import Any, Optional, Tuple
 
 from . import Config
 from .exceptions import CollectionNotFound
@@ -29,7 +30,7 @@ def __init__(self, config: Optional["Config"] = None) -> None:
             config = Config()
         self.config = config
 
-    def find(self, name: str) -> Tuple[IO, str, Tuple[str, str, int]]:
+    def find(self, name: str) -> Optional[ModuleSpec]:
         """
         Implementation-specific finder method seeking collection ``name``.
 
@@ -65,28 +66,19 @@ def load(self, name: Optional[str] = None) -> Tuple[ModuleType, str]:
         """
         if name is None:
             name = self.config.tasks.collection_name
-        # Find the named tasks module, depending on implementation.
-        # Will raise an exception if not found.
-        fd, path, desc = self.find(name)
-        try:
-            # Ensure containing directory is on sys.path in case the module
-            # being imported is trying to load local-to-it names.
-            parent = os.path.dirname(path)
-            if parent not in sys.path:
-                sys.path.insert(0, parent)
-            # FIXME: deprecated capability that needs replacement
-            # Actual import
-            module = imp.load_module(name, fd, path, desc)  # type: ignore
-            # Return module + path.
-            # TODO: is there a reason we're not simply having clients refer to
-            # os.path.dirname(module.__file__)?
-            return module, parent
-        finally:
-            # Ensure we clean up the opened file object returned by find(), if
-            # there was one (eg found packages, vs modules, don't open any
-            # file.)
-            if fd:
-                fd.close()
+        spec = self.find(name)
+        if spec and spec.loader and spec.origin:
+            path = spec.origin
+            if os.path.isfile(spec.origin):
+                path = os.path.dirname(spec.origin)
+            if path not in sys.path:
+                sys.path.insert(0, path)
+            module = module_from_spec(spec)
+            spec.loader.exec_module(module)
+            return module, os.path.dirname(spec.origin)
+        msg = "ImportError loading {!r}, raising ImportError"
+        debug(msg.format(name))
+        raise ImportError
 
 
 class FilesystemLoader(Loader):
@@ -103,6 +95,8 @@ class FilesystemLoader(Loader):
     # as auto-dashes, and has to grow one of those for every bit Collection
     # ever needs to know
     def __init__(self, start: Optional[str] = None, **kwargs: Any) -> None:
+        # finder = kwargs.pop("finder_class", CollectionFinder)
+        # sys.meta_path.append(finder)
         super().__init__(**kwargs)
         if start is None:
             start = self.config.tasks.search_root
@@ -113,25 +107,31 @@ def start(self) -> str:
         # Lazily determine default CWD if configured value is falsey
         return self._start or os.getcwd()
 
-    def find(self, name: str) -> Tuple[IO, str, Tuple[str, str, int]]:
-        # Accumulate all parent directories
-        start = self.start
-        debug("FilesystemLoader find starting at {!r}".format(start))
-        parents = [os.path.abspath(start)]
-        parents.append(os.path.dirname(parents[-1]))
-        while parents[-1] != parents[-2]:
-            parents.append(os.path.dirname(parents[-1]))
-        # Make sure we haven't got duplicates on the end
-        if parents[-1] == parents[-2]:
-            parents = parents[:-1]
-        # Use find_module with our list of parents. ImportError from
-        # find_module means "couldn't find" not "found and couldn't import" so
-        # we turn it into a more obvious exception class.
+    def find(self, name: str) -> Optional[ModuleSpec]:
+        debug("FilesystemLoader find starting at {!r}".format(self.start))
+        spec = None
+        module = "{}.py".format(name)
+        paths = self.start.split(os.sep)
         try:
-            tup = imp.find_module(name, parents)
-            debug("Found module: {!r}".format(tup[1]))
-            return tup
-        except ImportError:
+            for x in reversed(range(len(paths) + 1)):
+                path = os.sep.join(paths[0:x])
+                if module in os.listdir(path):
+                    spec = spec_from_file_location(
+                        name, os.path.join(path, module)
+                    )
+                    break
+                elif name in os.listdir(path) and os.path.exists(
+                    os.path.join(path, "__init__.py")
+                ):
+                    spec = spec_from_file_location(
+                        name, os.path.join(path, "__init__.py")
+                    )
+                    break
+            if spec:
+                debug("Found module: {!r}".format(spec))
+                return spec
+        except (FileNotFoundError, ModuleNotFoundError):
             msg = "ImportError loading {!r}, raising CollectionNotFound"
             debug(msg.format(name))
-            raise CollectionNotFound(name=name, start=start)
+            raise CollectionNotFound(name=name, start=self.start)
+        return None
diff --git a/invoke/program.py b/invoke/program.py
index c7e5cd00..474aab32 100644
--- a/invoke/program.py
+++ b/invoke/program.py
@@ -577,6 +577,7 @@ def execute(self) -> None:
             # TODO: worth trying to wrap both of these and raising ImportError
             # for cases where module exists but class name does not? More
             # "normal" but also its own possible source of bugs/confusion...
+            print(module_path, class_name)
             module = import_module(module_path)
             klass = getattr(module, class_name)
         executor = klass(self.collection, self.config, self.core)
diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst
index 0088923c..deed0ece 100644
--- a/sites/www/changelog.rst
+++ b/sites/www/changelog.rst
@@ -2,6 +2,8 @@
 Changelog
 =========
 
+- :support:`675` Implement `importlib` and deprecate `imp` module. Patches
+  provided by Jesse P. Johnson
 - :bug:`376` Resolve equality comparison bug for non-collections. Patch via
   Jesse P. Johnson
 - :support:`906` Implement type hints and type checking tests with mypy to
diff --git a/tests/loader.py b/tests/loader.py
index 9d6065c7..f5a080dd 100644
--- a/tests/loader.py
+++ b/tests/loader.py
@@ -1,7 +1,7 @@
-import imp
 import os
 import sys
-import types
+from importlib.util import spec_from_file_location
+from types import ModuleType
 
 from pytest import raises
 
@@ -21,8 +21,13 @@ class _BasicLoader(Loader):
     """
 
     def find(self, name):
-        self.fd, self.path, self.desc = t = imp.find_module(name, [support])
-        return t
+        path = os.path.join(support, name)
+        if os.path.exists(f"{path}.py"):
+            path = f"{path}.py"
+        elif os.path.exists(path):
+            path = os.path.join(path, "__init__.py")
+        spec = spec_from_file_location(name, path)
+        return spec
 
 
 class Loader_:
@@ -33,7 +38,7 @@ def exhibits_default_config_object(self):
 
     def returns_module_and_location(self):
         mod, path = _BasicLoader().load("namespacing")
-        assert isinstance(mod, types.ModuleType)
+        assert isinstance(mod, ModuleType)
         assert path == support
 
     def may_configure_config_via_constructor(self):
@@ -52,11 +57,6 @@ def doesnt_dupliate_parent_dir_addition(self):
         # other tests will pollute it (!).
         assert sys.path.count(support) == 1
 
-    def closes_opened_file_object(self):
-        loader = _BasicLoader()
-        loader.load("foo")
-        assert loader.fd.closed
-
     def can_load_package(self):
         loader = _BasicLoader()
         # make sure it doesn't explode
@@ -89,7 +89,7 @@ def exposes_start_point_as_attribute(self):
         assert FSLoader().start == os.getcwd()
 
     def start_point_is_configurable_via_kwarg(self):
-        start = "/tmp/"
+        start = "/tmp"
         assert FSLoader(start=start).start == start
 
     def start_point_is_configurable_via_config(self):
@@ -102,13 +102,15 @@ def raises_CollectionNotFound_if_not_found(self):
 
     def raises_ImportError_if_found_collection_cannot_be_imported(self):
         # Instead of masking with a CollectionNotFound
-        with raises(ImportError):
+        with raises(ModuleNotFoundError):
             self.loader.load("oops")
 
+    # TODO: Need CollectionImportError here
+
     def searches_towards_root_of_filesystem(self):
         # Loaded while root is in same dir as .py
         directly = self.loader.load("foo")
         # Loaded while root is multiple dirs deeper than the .py
         deep = os.path.join(support, "ignoreme", "ignoremetoo")
         indirectly = FSLoader(start=deep).load("foo")
-        assert directly == indirectly
+        assert directly[1] == indirectly[1]
diff --git a/tests/program.py b/tests/program.py
index 2a6d4f01..045c13c3 100644
--- a/tests/program.py
+++ b/tests/program.py
@@ -844,6 +844,7 @@ def nontrivial_trees_are_sorted_by_namespace_and_depth(self):
 
 """
             stdout, _ = run("-c tree --list")
+            print('--------', stdout)
             assert expected == stdout
 
         class namespace_limiting:

From ac9b4baa9587d1903faa6bfe8d621f7e65794fc0 Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Tue, 21 Feb 2023 22:29:50 -0500
Subject: [PATCH 2/8] refactor: deprecate imp for importlib

---
 invoke/loader.py  | 12 ++++++++----
 invoke/program.py |  1 -
 tests/program.py  |  3 +--
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/invoke/loader.py b/invoke/loader.py
index 5550a5c0..19417813 100644
--- a/invoke/loader.py
+++ b/invoke/loader.py
@@ -120,11 +120,15 @@ def find(self, name: str) -> Optional[ModuleSpec]:
                         name, os.path.join(path, module)
                     )
                     break
-                elif name in os.listdir(path) and os.path.exists(
-                    os.path.join(path, "__init__.py")
-                ):
+                elif name in os.listdir(path):
+                    #  and os.path.exists(
+                    #     os.path.join(path, name, "__init__.py")
+                    # ):
+                    basepath = os.path.join(path, name)
                     spec = spec_from_file_location(
-                        name, os.path.join(path, "__init__.py")
+                        name,
+                        os.path.join(basepath, "__init__.py"),
+                        submodule_search_locations=[basepath],
                     )
                     break
             if spec:
diff --git a/invoke/program.py b/invoke/program.py
index 474aab32..c7e5cd00 100644
--- a/invoke/program.py
+++ b/invoke/program.py
@@ -577,7 +577,6 @@ def execute(self) -> None:
             # TODO: worth trying to wrap both of these and raising ImportError
             # for cases where module exists but class name does not? More
             # "normal" but also its own possible source of bugs/confusion...
-            print(module_path, class_name)
             module = import_module(module_path)
             klass = getattr(module, class_name)
         executor = klass(self.collection, self.config, self.core)
diff --git a/tests/program.py b/tests/program.py
index 045c13c3..b609228e 100644
--- a/tests/program.py
+++ b/tests/program.py
@@ -844,12 +844,11 @@ def nontrivial_trees_are_sorted_by_namespace_and_depth(self):
 
 """
             stdout, _ = run("-c tree --list")
-            print('--------', stdout)
             assert expected == stdout
 
         class namespace_limiting:
             def argument_limits_display_to_given_namespace(self):
-                stdout, _ = run("-c tree --list build")
+                stdout, a = run("-c tree --list build")
                 expected = """Available 'build' tasks:
 
   .all (.everything)      Build all necessary artifacts.

From 9c68797556da6cac1da2e1c2157a8c9922108d19 Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 20:22:17 -0500
Subject: [PATCH 3/8] test: revert change

---
 tests/program.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/program.py b/tests/program.py
index b609228e..2a6d4f01 100644
--- a/tests/program.py
+++ b/tests/program.py
@@ -848,7 +848,7 @@ def nontrivial_trees_are_sorted_by_namespace_and_depth(self):
 
         class namespace_limiting:
             def argument_limits_display_to_given_namespace(self):
-                stdout, a = run("-c tree --list build")
+                stdout, _ = run("-c tree --list build")
                 expected = """Available 'build' tasks:
 
   .all (.everything)      Build all necessary artifacts.

From 282cc4396050254fe9dfadce82bc35bb295d15da Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 22:15:42 -0500
Subject: [PATCH 4/8] refactor: replace deprecated load_module

---
 invoke/config.py | 8 +++++++-
 invoke/loader.py | 9 ++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/invoke/config.py b/invoke/config.py
index 116f774a..54f780be 100644
--- a/invoke/config.py
+++ b/invoke/config.py
@@ -2,8 +2,10 @@
 import json
 import os
 import types
+from importlib.util import spec_from_loader
 from os import PathLike
 from os.path import join, splitext, expanduser
+from types import ModuleType
 from typing import Any, Dict, Iterator, Optional, Tuple, Type, Union
 
 from .env import Environment
@@ -24,7 +26,11 @@
 def load_source(name: str, path: str) -> Dict[str, Any]:
     if not os.path.exists(path):
         return {}
-    return vars(SourceFileLoader("mod", path).load_module())
+    loader = SourceFileLoader("mod", path)
+    mod = ModuleType("mod")
+    mod.__spec__ = spec_from_loader("mod", loader)
+    loader.exec_module(mod)
+    return vars(mod)
 
 
 class DataProxy:
diff --git a/invoke/loader.py b/invoke/loader.py
index 19417813..9172a41d 100644
--- a/invoke/loader.py
+++ b/invoke/loader.py
@@ -34,7 +34,7 @@ def find(self, name: str) -> Optional[ModuleSpec]:
         """
         Implementation-specific finder method seeking collection ``name``.
 
-        Must return a 4-tuple valid for use by `imp.load_module`, which is
+        Must return a ModuleSpec valid for use by `importlib`, which is
         typically a name string followed by the contents of the 3-tuple
         returned by `imp.find_module` (``file``, ``pathname``,
         ``description``.)
@@ -120,10 +120,9 @@ def find(self, name: str) -> Optional[ModuleSpec]:
                         name, os.path.join(path, module)
                     )
                     break
-                elif name in os.listdir(path):
-                    #  and os.path.exists(
-                    #     os.path.join(path, name, "__init__.py")
-                    # ):
+                elif name in os.listdir(path) and os.path.exists(
+                    os.path.join(path, name, "__init__.py")
+                ):
                     basepath = os.path.join(path, name)
                     spec = spec_from_file_location(
                         name,

From ece848e2dce3e5736d7e16e9dbd8dc709c50d0a5 Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 22:29:09 -0500
Subject: [PATCH 5/8] refactor: replace deprecated load_module

---
 invoke/loader.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/invoke/loader.py b/invoke/loader.py
index 9172a41d..4c18846e 100644
--- a/invoke/loader.py
+++ b/invoke/loader.py
@@ -36,8 +36,8 @@ def find(self, name: str) -> Optional[ModuleSpec]:
 
         Must return a ModuleSpec valid for use by `importlib`, which is
         typically a name string followed by the contents of the 3-tuple
-        returned by `imp.find_module` (``file``, ``pathname``,
-        ``description``.)
+        returned by `importlib.module_from_spec` (``name``, ``loader``,
+        ``origin``.)
 
         For a sample implementation, see `.FilesystemLoader`.
 
@@ -113,6 +113,7 @@ def find(self, name: str) -> Optional[ModuleSpec]:
         module = "{}.py".format(name)
         paths = self.start.split(os.sep)
         try:
+            # walk the path upwards to check for dynamic import
             for x in reversed(range(len(paths) + 1)):
                 path = os.sep.join(paths[0:x])
                 if module in os.listdir(path):

From 471752dd65a0d939c2aab234f1770a2055163ab6 Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 22:42:33 -0500
Subject: [PATCH 6/8] refactor: replace deprecated load_module

---
 invoke/loader.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/invoke/loader.py b/invoke/loader.py
index 4c18846e..628c4958 100644
--- a/invoke/loader.py
+++ b/invoke/loader.py
@@ -69,10 +69,13 @@ def load(self, name: Optional[str] = None) -> Tuple[ModuleType, str]:
         spec = self.find(name)
         if spec and spec.loader and spec.origin:
             path = spec.origin
+            # Ensure containing directory is on sys.path in case the module
+            # being imported is trying to load local-to-it names.
             if os.path.isfile(spec.origin):
                 path = os.path.dirname(spec.origin)
             if path not in sys.path:
                 sys.path.insert(0, path)
+            # Actual import
             module = module_from_spec(spec)
             spec.loader.exec_module(module)
             return module, os.path.dirname(spec.origin)

From 0b6e21f722ca277cfd4d2d0ba785e2176c78e917 Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 22:50:12 -0500
Subject: [PATCH 7/8] refactor: replace deprecated load_module

---
 tests/loader.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/loader.py b/tests/loader.py
index f5a080dd..58f713d7 100644
--- a/tests/loader.py
+++ b/tests/loader.py
@@ -113,4 +113,5 @@ def searches_towards_root_of_filesystem(self):
         # Loaded while root is multiple dirs deeper than the .py
         deep = os.path.join(support, "ignoreme", "ignoremetoo")
         indirectly = FSLoader(start=deep).load("foo")
+        assert directly[0].__spec__ == indirectly[0].__spec__
         assert directly[1] == indirectly[1]

From dea3ba244c8e14abaf369dfd558fa4ec715ee7ba Mon Sep 17 00:00:00 2001
From: "Jesse P. Johnson" <jpj6652@gmail.com>
Date: Wed, 22 Feb 2023 22:51:33 -0500
Subject: [PATCH 8/8] refactor: replace deprecated load_module

---
 tests/loader.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/loader.py b/tests/loader.py
index 58f713d7..35036322 100644
--- a/tests/loader.py
+++ b/tests/loader.py
@@ -113,5 +113,6 @@ def searches_towards_root_of_filesystem(self):
         # Loaded while root is multiple dirs deeper than the .py
         deep = os.path.join(support, "ignoreme", "ignoremetoo")
         indirectly = FSLoader(start=deep).load("foo")
+        assert directly[0].__file__ == indirectly[0].__file__
         assert directly[0].__spec__ == indirectly[0].__spec__
         assert directly[1] == indirectly[1]