typing.Self: support Self in variable annotations.

docs/support.md

@@ -17,7 +17,7 @@ of pytype.
       * [Third-Party Libraries](#third-party-libraries)
 
 <!-- Created by https://github.com/ekalinin/github-markdown-toc -->
-<!-- Added by: rechen, at: Tue Oct 10 01:57:19 PM PDT 2023 -->
+<!-- Added by: rechen, at: Wed Oct 25 03:33:16 PM PDT 2023 -->
 
 <!--te-->
 
@@ -82,7 +82,7 @@ Feature
 [PEP 646 -- Variadic Generics][646]                                                     | 3.11    | ❌        |
 [PEP 647 -- User-Defined Type Guards][647]                                              | 3.10    | ✅        |
 [PEP 655 -- Marking individual TypedDict items as required or potentially-missing][655] | 3.11    | ❌        |
-[PEP 673 -- Self Type][673]                                                             | 3.11    | 🟡        | [#1283][self]
+[PEP 673 -- Self Type][673]                                                             | 3.11    | ✅        |
 [PEP 675 -- Arbitrary Literal String Type][675]                                         | 3.11    | ❌        |
 [PEP 681 -- Data Class Transforms][681]                                                 | 3.11    | 🟡        |
 [PEP 695 -- Type Parameter Syntax][695]                                                 | 3.12    | ❌        |

pytype/abstract/_classes.py

@@ -561,6 +561,8 @@ def convert_as_instance_attribute(self, name, instance):
           subst[itm.full_name] = self.ctx.convert.constant_to_value(
               itm.type_param, {}).instantiate(
                   self.ctx.root_node, container=instance)
+        subst[f"{self.full_name}.Self"] = instance.to_variable(
+            self.ctx.root_node)
         # Set all other type parameters to Any. See
         # test_recursive_types:PyiTest.test_callable for a case in which it is
         # not an error to have an unsubstituted type parameter here.

pytype/annotation_utils.py

@@ -325,12 +325,18 @@ def init_annotation(self, node, name, annot, container=None, extra_key=None):
       d.from_annotation = name
     return node, value
 
+  def _in_class_frame(self):
+    frame = self.ctx.vm.frame
+    if not frame.func:
+      return False
+    return (isinstance(frame.func.data, abstract.BoundFunction) or
+            frame.func.data.is_attribute_of_class)
+
   def extract_and_init_annotation(self, node, name, var):
     """Extracts an annotation from var and instantiates it."""
     frame = self.ctx.vm.frame
     substs = frame.substs
-    if frame.func and (isinstance(frame.func.data, abstract.BoundFunction) or
-                       frame.func.data.is_attribute_of_class):
+    if self._in_class_frame():
       self_var = frame.first_arg
       if self_var:
         # self_var is an instance of (a subclass of) the class on which
@@ -370,7 +376,14 @@ def _sub_and_instantiate(self, node, name, typ, substs):
                                                  instantiate_unbound=False)
     else:
       substituted_type = typ
-    _, value = self.init_annotation(node, name, substituted_type)
+    if typ.formal and self._in_class_frame():
+      class_substs = abstract_utils.combine_substs(
+          substs, [{"typing.Self": self.ctx.vm.frame.first_arg}])
+      type_for_value = self.sub_one_annotation(node, typ, class_substs,
+                                               instantiate_unbound=False)
+    else:
+      type_for_value = substituted_type
+    _, value = self.init_annotation(node, name, type_for_value)
     return substituted_type, value
 
   def apply_annotation(self, node, op, name, value):
@@ -431,6 +444,10 @@ def extract_annotation(
     if typ.formal and allowed_type_params is not None:
       allowed_type_params = (allowed_type_params |
                              self.get_callable_type_parameter_names(typ))
+      if (self.ctx.vm.frame.func and
+          (isinstance(self.ctx.vm.frame.func.data, abstract.BoundFunction) or
+           self.ctx.vm.frame.func.data.is_class_builder)):
+        allowed_type_params.add("typing.Self")
       illegal_params = []
       for x in self.get_type_parameters(typ):
         if not allowed_type_params.intersection([x.name, x.full_name]):
@@ -441,6 +458,9 @@ def extract_annotation(
     return typ
 
   def _log_illegal_params(self, illegal_params, stack, typ, name):
+    out_of_scope_params = utils.unique_list(illegal_params)
+    details = "TypeVar(s) %s not in scope" % ", ".join(
+        repr(p) for p in out_of_scope_params)
     if self.ctx.vm.frame.func:
       method = self.ctx.vm.frame.func.data
       if isinstance(method, abstract.BoundFunction):
@@ -449,20 +469,6 @@ def _log_illegal_params(self, illegal_params, stack, typ, name):
       else:
         desc = "class" if method.is_class_builder else "method"
         frame_name = method.name
-    else:
-      desc, frame_name = None, None
-    out_of_scope_params = []
-    for param in utils.unique_list(illegal_params):
-      if param == "Self" and desc == "class":
-        self.ctx.errorlog.not_supported_yet(
-            stack, "Using typing.Self in a variable annotation")
-      else:
-        out_of_scope_params.append(param)
-    if not out_of_scope_params:
-      return
-    details = "TypeVar(s) %s not in scope" % ", ".join(
-        repr(p) for p in out_of_scope_params)
-    if desc:
       details += f" for {desc} {frame_name!r}"
     if "AnyStr" in out_of_scope_params:
       str_type = "Union[str, bytes]"

pytype/attribute.py

@@ -450,6 +450,13 @@ def _lookup_variable_annotation(self, node, base, name, valself):
           node, typ, [subst], instantiate_unbound=False)
     else:
       return typ, None
+    if typ.formal and valself:
+      if isinstance(valself.data, abstract.Class):
+        self_var = valself.data.instantiate(node)
+      else:
+        self_var = valself.AssignToNewVariable(node)
+      typ = self.ctx.annotation_utils.sub_one_annotation(
+          node, typ, [{"typing.Self": self_var}], instantiate_unbound=False)
     _, attr = self.ctx.annotation_utils.init_annotation(node, name, typ)
     return typ, attr
 
@@ -540,9 +547,6 @@ def _get_member(self, node, obj, name, valself):
     if isinstance(obj, mixin.LazyMembers):
       if not valself:
         subst = None
-      elif isinstance(valself.data, abstract.ParameterizedClass):
-        subst = {f"{valself.data.full_name}.{k}": v.instantiate(node)
-                 for k, v in valself.data.formal_type_parameters.items()}
       elif isinstance(valself.data, abstract.Instance):
         # We need to rebind the parameter values at the root because that's the
         # node at which load_lazy_attribute() converts pyvals.
@@ -559,6 +563,13 @@ def _get_member(self, node, obj, name, valself):
             #   class Child(Base): ...  # equivalent to `class Child(Base[Any])`
             # When this happens, parameter values are implicitly set to Any.
             subst[k] = self.ctx.new_unsolvable(self.ctx.root_node)
+        subst[f"{obj.full_name}.Self"] = valself.AssignToNewVariable()
+      elif isinstance(valself.data, abstract.Class):
+        subst = {f"{obj.full_name}.Self":
+                 valself.data.instantiate(self.ctx.root_node)}
+        if isinstance(valself.data, abstract.ParameterizedClass):
+          for k, v in valself.data.formal_type_parameters.items():
+            subst[f"{valself.data.full_name}.{k}"] = v.instantiate(node)
       else:
         subst = None
       member = obj.load_lazy_attribute(name, subst)

pytype/overlays/fiddle_overlay.py

@@ -247,6 +247,7 @@ def make_instance(
   cache_key = (ctx.root_node, underlying, subclass_name)
   if cache_key in _INSTANCE_CACHE:
     return node, _INSTANCE_CACHE[cache_key]
+  _INSTANCE_CACHE[cache_key] = ctx.convert.unsolvable  # recursion handling
 
   instance_class = {"Config": Config, "Partial": Partial}[subclass_name]
   # Create the specialized class Config[underlying] or Partial[underlying]

pytype/pytd/visitors.py

@@ -1755,7 +1755,8 @@ def _IsBoundTypeParam(self, node):
 
   def VisitTypeParameter(self, node):
     """Add scopes to type parameters, track unbound params."""
-    if self.constant_name and not self._IsBoundTypeParam(node):
+    if (self.constant_name and node.name != "Self" and
+        not self._IsBoundTypeParam(node)):
       raise ContainerError("Unbound type parameter {} in {}".format(
           node.name, self._GetFullName(self.constant_name)))
     scope = self._GetScope(node.name)

pytype/tests/test_typing_self.py

@@ -184,6 +184,78 @@ def f(self) -> int:  # signature-mismatch
           return 0
     """)
 
+  def test_class_attribute(self):
+    self.Check("""
+      from typing_extensions import Self
+      class Foo:
+        x: Self
+      class Bar(Foo):
+        pass
+      assert_type(Foo.x, Foo)
+      assert_type(Foo().x, Foo)
+      assert_type(Bar.x, Bar)
+      assert_type(Bar().x, Bar)
+    """)
+
+  def test_instance_attribute(self):
+    self.Check("""
+      from typing_extensions import Self
+      class Foo:
+        def __init__(self, x: Self):
+          self.x = x
+          self.y: Self = __any_object__
+      class Bar(Foo):
+        pass
+      assert_type(Foo(__any_object__).x, Foo)
+      assert_type(Foo(__any_object__).y, Foo)
+      assert_type(Bar(__any_object__).x, Bar)
+      assert_type(Bar(__any_object__).y, Bar)
+    """)
+
+  def test_cast(self):
+    self.Check("""
+      from typing import cast
+      from typing_extensions import Self
+      class Foo:
+        def f(self):
+          return cast(Self, __any_object__)
+      class Bar(Foo):
+        pass
+      assert_type(Foo().f(), Foo)
+      assert_type(Bar().f(), Bar)
+    """)
+
+  def test_generic_attribute(self):
+    self.Check("""
+      from typing import Generic, TypeVar
+      from typing_extensions import Self
+      T = TypeVar('T')
+      class C(Generic[T]):
+        x: Self
+      class D(C[T]):
+        pass
+      assert_type(C[int].x, C[int])
+      assert_type(C[int]().x, C[int])
+      assert_type(D[str].x, D[str])
+      assert_type(D[str]().x, D[str])
+    """)
+
+  def test_attribute_mismatch(self):
+    self.CheckWithErrors("""
+      from typing import Protocol
+      from typing_extensions import Self
+      class C(Protocol):
+        x: Self
+      class Ok:
+        x: 'Ok'
+      class Bad:
+        x: int
+      def f(c: C):
+        pass
+      f(Ok())
+      f(Bad())  # wrong-arg-types
+    """)
+
 
 class SelfPyiTest(test_base.BaseTest):
   """Tests for typing.Self usage in type stubs."""
@@ -333,6 +405,59 @@ def f(self) -> int:  # signature-mismatch
             return 0
       """)
 
+  def test_attribute(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import Self
+      class A:
+        x: Self
+    """)]):
+      self.Check("""
+        import foo
+        class B(foo.A):
+          pass
+        assert_type(foo.A.x, foo.A)
+        assert_type(foo.A().x, foo.A)
+        assert_type(B.x, B)
+        assert_type(B().x, B)
+      """)
+
+  def test_generic_attribute(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import Generic, Self, TypeVar
+      T = TypeVar('T')
+      class A(Generic[T]):
+        x: Self
+    """)]):
+      self.Check("""
+        import foo
+        from typing import TypeVar
+        T = TypeVar('T')
+        class B(foo.A[T]):
+          pass
+        assert_type(foo.A[str].x, foo.A[str])
+        assert_type(foo.A[int]().x, foo.A[int])
+        assert_type(B[int].x, B[int])
+        assert_type(B[str]().x, B[str])
+      """)
+
+  def test_attribute_mismatch(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import Protocol, Self
+      class C(Protocol):
+        x: Self
+    """)]):
+      self.CheckWithErrors("""
+        import foo
+        class Ok:
+          x: 'Ok'
+        class Bad:
+          x: str
+        def f(c: foo.C):
+          pass
+        f(Ok())
+        f(Bad())  # wrong-arg-types
+      """)
+
 
 class SelfReingestTest(test_base.BaseTest):
   """Tests for outputting typing.Self to a stub and reading the stub back in."""
@@ -354,6 +479,22 @@ def f(self) -> Self: ...""")
     actual = pytd_utils.Print(ty)
     self.assertMultiLineEqual(expected, actual)
 
+  def test_attribute_output(self):
+    ty = self.Infer("""
+      from typing_extensions import Self
+      class A:
+        x: Self
+        def __init__(self):
+          self.y: Self = __any_object__
+    """)
+    self.assertTypesMatchPytd(ty, """
+      from typing import Self
+      class A:
+        x: Self
+        y: Self
+        def __init__(self) -> None: ...
+    """)
+
   def test_instance_method_return(self):
     with self.DepTree([("foo.py", """
       from typing_extensions import Self
@@ -538,14 +679,6 @@ def f(x) -> Self:  # invalid-annotation[e]
     self.assertErrorSequences(
         errors, {"e": ["'typing.Self' outside of a class"]})
 
-  def test_variable_annotation(self):
-    self.CheckWithErrors("""
-      from typing_extensions import Self
-      class C:
-        x: Self  # not-supported-yet
-        y = ...  # type: Self  # not-supported-yet
-    """)
-
   def test_variable_annotation_not_in_class(self):
     errors = self.CheckWithErrors("""
       from typing_extensions import Self