Google sync

CHANGELOG

@@ -1,3 +1,14 @@
+Version 2023.10.24:
+
+Updates:
+* Add support for typing.Self in method type annotations.
+
+Bug fixes:
+* Support pattern matching against a tuple of match variables.
+* Fix crash caused by use of a recursive type in a callable.
+* When setting a type from assertIsInstance narrow the original type if
+  possible.
+
 Version 2023.10.17:
 
 Updates:

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2023.10.17'
+__version__ = '2023.10.24'

pytype/abstract/_base.py

@@ -442,7 +442,7 @@ def _compute_template(val: Any):
             val, "Cannot inherit from Generic[...] multiple times")
       for item in base.template:
         param = base.formal_type_parameters.get(item.name)
-        template.append(param.with_module(val.full_name))
+        template.append(param.with_scope(val.full_name))
 
   if template:
     # All type parameters in the base classes should appear in
@@ -453,7 +453,7 @@ def _compute_template(val: Any):
           for item in base.template:
             param = base.formal_type_parameters.get(item.name)
             if _isinstance(base, "TypeParameter"):
-              t = param.with_module(val.full_name)
+              t = param.with_scope(val.full_name)
               if t not in template:
                 raise abstract_utils.GenericTypeError(
                     val, "Generic should contain all the type variables")
@@ -466,7 +466,7 @@ def _compute_template(val: Any):
         for item in base.template:
           param = base.formal_type_parameters.get(item.name)
           if _isinstance(param, "TypeParameter"):
-            seq.append(param.with_module(val.full_name))
+            seq.append(param.with_scope(val.full_name))
         seqs.append(seq)
     try:
       template.extend(mro.MergeSequences(seqs))

pytype/abstract/_classes.py

@@ -137,8 +137,6 @@ def get_first_opcode(self):
     return self._first_opcode
 
   def update_method_type_params(self):
-    if not self.template:
-      return
     # For function type parameters check
     methods = []
     # members of self._undecorated_methods that will be ignored for updating
@@ -168,7 +166,7 @@ def type_param_check(self):
       # in current generic class
       inner_cls_types = self.collect_inner_cls_types()
       for cls, item in inner_cls_types:
-        nitem = item.with_module(self.full_name)
+        nitem = item.with_scope(self.full_name)
         if nitem in self.template:
           raise abstract_utils.GenericTypeError(
               self, ("Generic class [%s] and its nested generic class [%s] "
@@ -189,7 +187,7 @@ def collect_inner_cls_types(self, max_depth=5):
         mbr = abstract_utils.get_atomic_value(
             mbr, default=self.ctx.convert.unsolvable)
         if isinstance(mbr, InterpreterClass) and mbr.template:
-          templates.update([(mbr, item.with_module(None))
+          templates.update([(mbr, item.with_scope(None))
                             for item in mbr.template])
           templates.update(mbr.collect_inner_cls_types(max_depth - 1))
     return templates

pytype/abstract/_function_base.py

@@ -219,23 +219,23 @@ def __init__(self, callself, underlying):
       self.alias_map = None
 
   def _get_self_annot(self, callself):
-    if not self._has_self_type_param():
+    self_type = self._get_self_type_param()
+    if not self_type:
       return abstract_utils.get_generic_type(callself)
-    self_type = self.ctx.convert.lookup_value("typing", "Self")
     if "classmethod" in self.underlying.decorators:
       return _classes.ParameterizedClass(
           self.ctx.convert.type_type, {abstract_utils.T: self_type}, self.ctx)
     else:
       return self_type
 
-  def _has_self_type_param(self):
+  def _get_self_type_param(self):
     if not isinstance(self.underlying, SignedFunction):
-      return False
+      return None
     for annot in self.underlying.signature.annotations.values():
-      if any(param.full_name == "typing.Self"
-             for param in self.ctx.annotation_utils.get_type_parameters(annot)):
-        return True
-    return False
+      for param in self.ctx.annotation_utils.get_type_parameters(annot):
+        if param.full_name == "typing.Self":
+          return param
+    return None
 
   def argcount(self, node):
     return self.underlying.argcount(node) - 1  # account for self
@@ -666,7 +666,7 @@ def generator():
   def update_signature_scope(self, cls):
     self.signature.excluded_types.update(
         [t.name for t in cls.template])
-    self.signature.add_scope(cls.full_name)
+    self.signature.add_scope(cls)
 
 
 class SimpleFunction(SignedFunction):

pytype/abstract/_interpreter_function.py

@@ -267,7 +267,7 @@ def _inner_cls_check(self, last_frame):
     all_type_parameters = []
     for annot in self.signature.annotations.values():
       params = self.ctx.annotation_utils.get_type_parameters(annot)
-      all_type_parameters.extend(itm.with_module(None) for itm in params)
+      all_type_parameters.extend(itm.with_scope(None) for itm in params)
 
     if all_type_parameters:
       for key, value in last_frame.f_locals.pyval.items():
@@ -277,7 +277,7 @@ def _inner_cls_check(self, last_frame):
             key == value.name):
           # `value` is a nested class definition.
           inner_cls_types = value.collect_inner_cls_types()
-          inner_cls_types.update([(value, item.with_module(None))
+          inner_cls_types.update([(value, item.with_scope(None))
                                   for item in value.template])
           # Report errors in a deterministic order.
           for cls, item in sorted(inner_cls_types, key=lambda typ: typ[1].name):

pytype/abstract/_typing.py

@@ -272,7 +272,7 @@ def _build_value(self, node, inner, ellipses):
       # usually, the parameterized class inherits the base class's template.
       # Protocol[T, ...] is a shorthand for Protocol, Generic[T, ...].
       template_params = [
-          param.with_module(base_cls.full_name)
+          param.with_scope(base_cls.full_name)
           for param in typing.cast(Tuple[TypeParameter, ...], processed_inner)]
     else:
       template_params = None
@@ -348,7 +348,11 @@ def __init__(self, param, instance, ctx):
     super().__init__(param.name, ctx)
     self.cls = self.param = param
     self.instance = instance
-    self.module = param.module
+    self.scope = param.scope
+
+  @property
+  def full_name(self):
+    return f"{self.scope}.{self.name}" if self.scope else self.name
 
   def call(self, node, func, args, alias_map=None):
     var = self.instance.get_instance_type_parameter(self.name)
@@ -391,26 +395,35 @@ def __init__(self,
                bound=None,
                covariant=False,
                contravariant=False,
-               module=None):
+               scope=None):
     super().__init__(name, ctx)
     # TODO(b/217789659): PEP-612 does not mention constraints, but ParamSpecs
     # ignore all the extra parameters anyway..
     self.constraints = constraints
     self.bound = bound
     self.covariant = covariant
     self.contravariant = contravariant
-    self.module = module
+    self.scope = scope
+
+  @_base.BaseValue.module.setter
+  def module(self, module):
+    super(_TypeVariable, _TypeVariable).module.fset(self, module)
+    self.scope = module
+
+  @property
+  def full_name(self):
+    return f"{self.scope}.{self.name}" if self.scope else self.name
 
   def is_generic(self):
     return not self.constraints and not self.bound
 
   def copy(self):
     return self.__class__(self.name, self.ctx, self.constraints, self.bound,
-                          self.covariant, self.contravariant, self.module)
+                          self.covariant, self.contravariant, self.scope)
 
-  def with_module(self, module):
+  def with_scope(self, scope):
     res = self.copy()
-    res.module = module
+    res.scope = scope
     return res
 
   def __eq__(self, other):
@@ -420,7 +433,7 @@ def __eq__(self, other):
               self.bound == other.bound and
               self.covariant == other.covariant and
               self.contravariant == other.contravariant and
-              self.module == other.module)
+              self.scope == other.scope)
     return NotImplemented
 
   def __ne__(self, other):
@@ -433,7 +446,7 @@ def __hash__(self):
   def __repr__(self):
     return ("{!s}({!r}, constraints={!r}, bound={!r}, module={!r})"
             .format(self.__class__.__name__, self.name, self.constraints,
-                    self.bound, self.module))
+                    self.bound, self.scope))
 
   def instantiate(self, node, container=None):
     var = self.ctx.program.NewVariable()

pytype/abstract/function.py

@@ -137,12 +137,12 @@ def has_param_annotations(self):
   def posonly_params(self):
     return self.param_names[:self.posonly_count]
 
-  def add_scope(self, module):
+  def add_scope(self, cls):
     """Add scope for type parameters in annotations."""
     annotations = {}
     for key, val in self.annotations.items():
       annotations[key] = val.ctx.annotation_utils.add_scope(
-          val, self.excluded_types, module)
+          val, self.excluded_types, cls)
     self.annotations = annotations
 
   def _postprocess_annotation(self, name, annotation):
@@ -164,8 +164,8 @@ def set_annotation(self, name, annotation):
   def del_annotation(self, name):
     del self.annotations[name]  # Raises KeyError if annotation does not exist.
 
-  def check_type_parameter_count(self, stack, opcode):
-    """Check the count of type parameters in function."""
+  def check_type_parameters(self, stack, opcode, is_attribute_of_class):
+    """Check type parameters in function."""
     if not self.annotations:
       return
     c = collections.Counter()
@@ -191,7 +191,12 @@ def check_type_parameter_count(self, stack, opcode):
       c.update(params)
     bare_alias_errors = set()
     for param, count in c.items():
-      if param.full_name == "typing.Self" or param.name in self.excluded_types:
+      if param.name in self.excluded_types:
+        continue
+      if param.full_name == "typing.Self":
+        if not is_attribute_of_class and not self.name.endswith(".__new__"):
+          ctx.errorlog.invalid_annotation(
+              stack, param, "Cannot use 'typing.Self' outside of a class")
         continue
       if count == 1 and not (param.constraints or param.bound or
                              param.covariant or param.contravariant):

pytype/annotation_utils.py

@@ -179,7 +179,7 @@ def get_late_annotations(self, annot):
       for _, typ in annot.get_inner_types():
         yield from self.get_late_annotations(typ)
 
-  def add_scope(self, annot, types, module, seen=None):
+  def add_scope(self, annot, types, cls, seen=None):
     """Add scope for type parameters.
 
     In original type class, all type parameters that should be added a scope
@@ -188,24 +188,29 @@ def add_scope(self, annot, types, module, seen=None):
     Args:
       annot: The type class.
       types: A type name list that should be added a scope.
-      module: Module name.
+      cls: The class that type parameters should be scoped to.
       seen: Already seen types.
 
     Returns:
       The type with fresh type parameters that have been added the scope.
     """
     if seen is None:
       seen = {annot}
-    elif annot in seen:
+    elif annot in seen or not annot.formal:
       return annot
     else:
       seen.add(annot)
     if isinstance(annot, abstract.TypeParameter):
       if annot.name in types:
-        return annot.with_module(module)
-      return annot
+        return annot.with_scope(cls.full_name)
+      elif annot.full_name == "typing.Self":
+        bound_annot = annot.copy()
+        bound_annot.bound = cls
+        return bound_annot
+      else:
+        return annot
     elif isinstance(annot, mixin.NestedAnnotation):
-      inner_types = [(key, self.add_scope(typ, types, module, seen))
+      inner_types = [(key, self.add_scope(typ, types, cls, seen))
                      for key, typ in annot.get_inner_types()]
       return annot.replace(inner_types)
     return annot
@@ -221,7 +226,7 @@ def get_type_parameters(self, annot, seen=None):
       seen: A seen set.
     """
     seen = seen or set()
-    if annot in seen:
+    if annot in seen or not annot.formal:
       return []
     if isinstance(annot, mixin.NestedAnnotation):
       # We track parameterized classes to avoid recursion errors when a class
@@ -247,7 +252,7 @@ def get_callable_type_parameter_names(self, val: abstract.BaseValue):
     stack = [val]
     while stack:
       annot = stack.pop()
-      if annot in seen:
+      if annot in seen or not annot.formal:
         continue
       seen.add(annot)
       if annot.full_name == "typing.Callable":
@@ -339,7 +344,7 @@ def extract_and_init_annotation(self, node, name, var):
           for cls in v_cls.mro:
             if cls.name == defining_cls_name:
               # Normalize type parameter names by dropping the scope.
-              type_params.extend(p.with_module(None) for p in cls.template)
+              type_params.extend(p.with_scope(None) for p in cls.template)
               defining_classes.append(cls)
               break
         self_substs = tuple(
@@ -431,24 +436,39 @@ def extract_annotation(
         if not allowed_type_params.intersection([x.name, x.full_name]):
           illegal_params.append(x.name)
       if illegal_params:
-        details = "TypeVar(s) %s not in scope" % ", ".join(
-            repr(p) for p in utils.unique_list(illegal_params))
-        if self.ctx.vm.frame.func:
-          method = self.ctx.vm.frame.func.data
-          if isinstance(method, abstract.BoundFunction):
-            desc = "class"
-            frame_name = method.name.rsplit(".", 1)[0]
-          else:
-            desc = "class" if method.is_class_builder else "method"
-            frame_name = method.name
-          details += f" for {desc} {frame_name!r}"
-        if "AnyStr" in illegal_params:
-          str_type = "Union[str, bytes]"
-          details += (f"\nNote: For all string types, use {str_type}.")
-        self.ctx.errorlog.invalid_annotation(stack, typ, details, name)
+        self._log_illegal_params(illegal_params, stack, typ, name)
         return self.ctx.convert.unsolvable
     return typ
 
+  def _log_illegal_params(self, illegal_params, stack, typ, name):
+    if self.ctx.vm.frame.func:
+      method = self.ctx.vm.frame.func.data
+      if isinstance(method, abstract.BoundFunction):
+        desc = "class"
+        frame_name = method.name.rsplit(".", 1)[0]
+      else:
+        desc = "class" if method.is_class_builder else "method"
+        frame_name = method.name
+    else:
+      desc, frame_name = None, None
+    out_of_scope_params = []
+    for param in utils.unique_list(illegal_params):
+      if param == "Self" and desc == "class":
+        self.ctx.errorlog.not_supported_yet(
+            stack, "Using typing.Self in a variable annotation")
+      else:
+        out_of_scope_params.append(param)
+    if not out_of_scope_params:
+      return
+    details = "TypeVar(s) %s not in scope" % ", ".join(
+        repr(p) for p in out_of_scope_params)
+    if desc:
+      details += f" for {desc} {frame_name!r}"
+    if "AnyStr" in out_of_scope_params:
+      str_type = "Union[str, bytes]"
+      details += (f"\nNote: For all string types, use {str_type}.")
+    self.ctx.errorlog.invalid_annotation(stack, typ, details, name)
+
   def eval_multi_arg_annotation(self, node, func, annot, stack):
     """Evaluate annotation for multiple arguments (from a type comment)."""
     args, errorlog = self._eval_expr_as_tuple(node, annot, stack)

pytype/convert.py

@@ -834,7 +834,7 @@ def _constant_to_value(self, pyval, subst, get_node):
           self.ctx,
           constraints=constraints,
           bound=bound,
-          module=pyval.scope)
+          scope=pyval.scope)
     elif isinstance(pyval, (pytd.ParamSpecArgs, pytd.ParamSpecKwargs)):
       # TODO(b/217789659): Support these.
       return self.unsolvable

pytype/output.py

@@ -271,7 +271,7 @@ def value_instance_to_pytd_type(self, node, v, instance, seen, view):
       return pytd.AnythingType()
 
   def _type_variable_to_pytd_type(self, node, v, seen, view):
-    if (v.module in self._scopes or
+    if (v.scope in self._scopes or
         isinstance(v.instance, abstract_utils.DummyContainer)):
       if isinstance(v, abstract.TYPE_VARIABLE_INSTANCES):
         return self._type_variable_to_def(node, v.param, v.param.name)