bpo-32225: Implementation of PEP 562 (#4731)

Implement PEP 562: module __getattr__ and __dir__.
The implementation simply updates module_getattro and
module_dir.

Author: ilevkivskyi

Doc/reference/datamodel.rst

@@ -1512,6 +1512,51 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances.
    returned. :func:`dir` converts the returned sequence to a list and sorts it.
 
 
+Customizing module attribute access
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. index::
+   single: __getattr__ (module attribute)
+   single: __dir__ (module attribute)
+   single: __class__ (module attribute)
+
+Special names ``__getattr__`` and ``__dir__`` can be also used to customize
+access to module attributes. The ``__getattr__`` function at the module level
+should accept one argument which is the name of an attribute and return the
+computed value or raise an :exc:`AttributeError`. If an attribute is
+not found on a module object through the normal lookup, i.e.
+:meth:`object.__getattribute__`, then ``__getattr__`` is searched in
+the module ``__dict__`` before raising an :exc:`AttributeError`. If found,
+it is called with the attribute name and the result is returned.
+
+The ``__dir__`` function should accept no arguments, and return a list of
+strings that represents the names accessible on module. If present, this
+function overrides the standard :func:`dir` search on a module.
+
+For a more fine grained customization of the module behavior (setting
+attributes, properties, etc.), one can set the ``__class__`` attribute of
+a module object to a subclass of :class:`types.ModuleType`. For example::
+
+   import sys
+   from types import ModuleType
+
+   class VerboseModule(ModuleType):
+       def __repr__(self):
+           return f'Verbose {self.__name__}'
+
+       def __setattr__(self, attr, value):
+           print(f'Setting {attr}...')
+           setattr(self, attr, value)
+
+   sys.modules[__name__].__class__ = VerboseModule
+
+.. note::
+   Defining module ``__getattr__`` and setting module ``__class__`` only
+   affect lookups made using the attribute access syntax -- directly accessing
+   the module globals (whether by code within the module, or via a reference
+   to the module's globals dictionary) is unaffected.
+
+
 .. _descriptors:
 
 Implementing Descriptors

Doc/whatsnew/3.7.rst

@@ -159,6 +159,24 @@ effort will be made to add such support.
        PEP written by Erik M. Bray; implementation by Masayuki Yamamoto.
 
 
+PEP 562: Customization of access to module attributes
+-----------------------------------------------------
+
+It is sometimes convenient to customize or otherwise have control over access
+to module attributes. A typical example is managing deprecation warnings.
+Typical workarounds are assigning ``__class__`` of a module object to
+a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules``
+item with a custom wrapper instance. This procedure is now simplified by
+recognizing ``__getattr__`` defined directly in a module that would act like
+a normal ``__getattr__`` method, except that it will be defined on module
+*instances*.
+
+.. seealso::
+
+    :pep:`562` -- Module ``__getattr__`` and ``__dir__``
+       PEP written and implemented by Ivan Levkivskyi
+
+
 PEP 564: Add new time functions with nanosecond resolution
 ----------------------------------------------------------
 

Lib/test/bad_getattr.py

@@ -0,0 +1,4 @@
+x = 1
+
+__getattr__ = "Surprise!"
+__dir__ = "Surprise again!"

Lib/test/bad_getattr2.py

@@ -0,0 +1,7 @@
+def __getattr__():
+    "Bad one"
+
+x = 1
+
+def __dir__(bad_sig):
+    return []

Lib/test/bad_getattr3.py

@@ -0,0 +1,5 @@
+def __getattr__(name):
+    if name != 'delgetattr':
+        raise AttributeError
+    del globals()['__getattr__']
+    raise AttributeError

Lib/test/good_getattr.py

@@ -0,0 +1,11 @@
+x = 1
+
+def __dir__():
+    return ['a', 'b', 'c']
+
+def __getattr__(name):
+    if name == "yolo":
+        raise AttributeError("Deprecated, use whatever instead")
+    return f"There is {name}"
+
+y = 2

Lib/test/test_module.py

@@ -125,6 +125,57 @@ def test_weakref(self):
         gc_collect()
         self.assertIs(wr(), None)
 
+    def test_module_getattr(self):
+        import test.good_getattr as gga
+        from test.good_getattr import test
+        self.assertEqual(test, "There is test")
+        self.assertEqual(gga.x, 1)
+        self.assertEqual(gga.y, 2)
+        with self.assertRaisesRegex(AttributeError,
+                                    "Deprecated, use whatever instead"):
+            gga.yolo
+        self.assertEqual(gga.whatever, "There is whatever")
+        del sys.modules['test.good_getattr']
+
+    def test_module_getattr_errors(self):
+        import test.bad_getattr as bga
+        from test import bad_getattr2
+        self.assertEqual(bga.x, 1)
+        self.assertEqual(bad_getattr2.x, 1)
+        with self.assertRaises(TypeError):
+            bga.nope
+        with self.assertRaises(TypeError):
+            bad_getattr2.nope
+        del sys.modules['test.bad_getattr']
+        if 'test.bad_getattr2' in sys.modules:
+            del sys.modules['test.bad_getattr2']
+
+    def test_module_dir(self):
+        import test.good_getattr as gga
+        self.assertEqual(dir(gga), ['a', 'b', 'c'])
+        del sys.modules['test.good_getattr']
+
+    def test_module_dir_errors(self):
+        import test.bad_getattr as bga
+        from test import bad_getattr2
+        with self.assertRaises(TypeError):
+            dir(bga)
+        with self.assertRaises(TypeError):
+            dir(bad_getattr2)
+        del sys.modules['test.bad_getattr']
+        if 'test.bad_getattr2' in sys.modules:
+            del sys.modules['test.bad_getattr2']
+
+    def test_module_getattr_tricky(self):
+        from test import bad_getattr3
+        # these lookups should not crash
+        with self.assertRaises(AttributeError):
+            bad_getattr3.one
+        with self.assertRaises(AttributeError):
+            bad_getattr3.delgetattr
+        if 'test.bad_getattr3' in sys.modules:
+            del sys.modules['test.bad_getattr3']
+
     def test_module_repr_minimal(self):
         # reprs when modules have no __file__, __name__, or __loader__
         m = ModuleType('foo')

Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst

@@ -0,0 +1,2 @@
+PEP 562: Add support for module ``__getattr__`` and ``__dir__``. Implemented by Ivan
+Levkivskyi.

Objects/moduleobject.c

@@ -679,12 +679,19 @@ module_repr(PyModuleObject *m)
 static PyObject*
 module_getattro(PyModuleObject *m, PyObject *name)
 {
-    PyObject *attr, *mod_name;
+    PyObject *attr, *mod_name, *getattr;
     attr = PyObject_GenericGetAttr((PyObject *)m, name);
-    if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError))
+    if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) {
         return attr;
+    }
     PyErr_Clear();
     if (m->md_dict) {
+        _Py_IDENTIFIER(__getattr__);
+        getattr = _PyDict_GetItemId(m->md_dict, &PyId___getattr__);
+        if (getattr) {
+            PyObject* stack[1] = {name};
+            return _PyObject_FastCall(getattr, stack, 1);
+        }
         _Py_IDENTIFIER(__name__);
         mod_name = _PyDict_GetItemId(m->md_dict, &PyId___name__);
         if (mod_name && PyUnicode_Check(mod_name)) {
@@ -730,8 +737,15 @@ module_dir(PyObject *self, PyObject *args)
     PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__);
 
     if (dict != NULL) {
-        if (PyDict_Check(dict))
-            result = PyDict_Keys(dict);
+        if (PyDict_Check(dict)) {
+            PyObject *dirfunc = PyDict_GetItemString(dict, "__dir__");
+            if (dirfunc) {
+                result = _PyObject_CallNoArg(dirfunc);
+            }
+            else {
+                result = PyDict_Keys(dict);
+            }
+        }
         else {
             const char *name = PyModule_GetName(self);
             if (name)