Merge pull request #1559 from google/google_sync

Google sync

docs/support.md

@@ -17,7 +17,7 @@ of pytype.
       * [Third-Party Libraries](#third-party-libraries)
 
 <!-- Created by https://github.com/ekalinin/github-markdown-toc -->
-<!-- Added by: rechen, at: Mon Dec 18 11:42:58 AM PST 2023 -->
+<!-- Added by: rechen, at: Wed Jan  3 02:26:12 PM PST 2024 -->
 
 <!--te-->
 
@@ -79,10 +79,11 @@ Feature
 [PEP 613 -- Explicit Type Aliases][613]                                                 | 3.10    | ✅        |
 [PEP 647 -- User-Defined Type Guards][647]                                              | 3.10    | ✅        |
 [PEP 646 -- Variadic Generics][646]                                                     | 3.11    | ❌        | [#1525][variadic-generics]
-[PEP 655 -- Marking individual TypedDict items as required or potentially-missing][655] | 3.11    | ❌        | [#1551][typed-dict-requirements]
+[PEP 655 -- Marking individual TypedDict items as required or potentially-missing][655] | 3.11    | ✅        |
 [PEP 673 -- Self Type][673]                                                             | 3.11    | ✅        |
 [PEP 675 -- Arbitrary Literal String Type][675]                                         | 3.11    | ❌        | [#1552][literal-string]
 [PEP 681 -- Data Class Transforms][681]                                                 | 3.11    | 🟡        | [#1553][dataclass-transform]
+[PEP 692 -- Using TypedDict for more precise **kwargs typing][692]                      | 3.12    | ❌        | [#1558][typeddict-unpack]
 [PEP 695 -- Type Parameter Syntax][695]                                                 | 3.12    | ❌        |
 [PEP 698 -- Override Decorator for Static Typing][698]                                  | 3.12    | ✅        |
 Custom Recursive Types                                                                  | 3.6     | ✅        |
@@ -186,6 +187,7 @@ Tensorflow | 🟡        | Minimal, Google-internal
 [673]: https://www.python.org/dev/peps/pep-0673
 [675]: https://peps.python.org/pep-0675/
 [681]: https://peps.python.org/pep-0681/
+[692]: https://peps.python.org/pep-0692/
 [695]: https://peps.python.org/pep-0695/
 [698]: https://peps.python.org/pep-0698/
 [annotated]: https://github.com/google/pytype/issues/791
@@ -205,5 +207,5 @@ Tensorflow | 🟡        | Minimal, Google-internal
 [pytype-typing-faq]: https://google.github.io/pytype/typing_faq.html
 [self]: https://github.com/google/pytype/issues/1283
 [type-guards]: https://github.com/google/pytype/issues/916
-[typed-dict-requirements]: https://github.com/google/pytype/issues/1551
+[typeddict-unpack]: https://github.com/google/pytype/issues/1558
 [variadic-generics]: https://github.com/google/pytype/issues/1525

pytype/abstract/_typing.py

@@ -736,6 +736,9 @@ def __setattr__(self, name, value):
       return super().__setattr__(name, value)
     return self._type.__setattr__(name, value)
 
+  def __contains__(self, name):
+    return self.resolved and name in self._type
+
   def resolve(self, node, f_globals, f_locals):
     """Resolve the late annotation."""
     if self.resolved:

pytype/abstract/abstract_utils.py

@@ -727,16 +727,20 @@ def get_type_parameter_substitutions(
   return subst
 
 
+def is_type_variable(val: _BaseValueType):
+  """Check if a value is a type variable (TypeVar or ParamSpec)."""
+  return _isinstance(val, ("TypeParameter", "ParamSpec"))
+
+
 def build_generic_template(
     type_params: Sequence[_BaseValueType], base_type: _BaseValueType
 ) -> Tuple[Sequence[str], Sequence[_TypeParamType]]:
   """Build a typing.Generic template from a sequence of type parameters."""
-  if not all(_isinstance(item, "TypeParameter") for item in type_params):
+  if not all(is_type_variable(item) for item in type_params):
     base_type.ctx.errorlog.invalid_annotation(
         base_type.ctx.vm.frames, base_type,
         "Parameters to Generic[...] must all be type variables")
-    type_params = [item for item in type_params
-                   if _isinstance(item, "TypeParameter")]
+    type_params = [item for item in type_params if is_type_variable(item)]
 
   template = [item.name for item in type_params]
 

pytype/constant_folding.py

@@ -336,10 +336,15 @@ def build_tuple(tup):
               other_elts = tuple(_Constant(('prim', e), v, None, other.op)
                                  for (_, e), v in zip(other_et, other.value))
             elif other_tag == 'prim':
-              assert other_et == str
-              other_et = {other.typ}
-              other_elts = tuple(_Constant(('prim', str), v, None, other.op)
-                                 for v in other.value)
+              if other_et == str:
+                other_et = {other.typ}
+                other_elts = tuple(_Constant(('prim', str), v, None, other.op)
+                                   for v in other.value)
+              else:
+                # We have some malformed code, e.g. [*42]
+                name = other_et.__name__
+                msg = f'Value after * must be an iterable, not {name}'
+                raise ConstantError(msg, op)
             else:
               other_elts = other.elements
             typ = (tag, et | set(other_et))

pytype/constant_folding_test.py

@@ -84,6 +84,11 @@ def test_str_to_list(self):
         (1, ("list", str), ["a", "b", "c"], [str, str, str])
     ])
 
+  @test_utils.skipBeforePy((3, 9), "Test for new LIST_EXTEND opcode in 3.9")
+  def test_bad_extend(self):
+    with self.assertRaises(constant_folding.ConstantError):
+      self._process("a = [1, 2, *3]")
+
   def test_map(self):
     actual = self._process("a = {'x': 1, 'y': '2'}")
     self.assertCountEqual(actual, [

pytype/overlays/dataclass_overlay.py

@@ -22,6 +22,7 @@ def __init__(self, ctx):
     member_map = {
         "dataclass": Dataclass.make,
         "field": FieldFunction.make,
+        "replace": Replace.make,
     }
     ast = ctx.loader.import_name("dataclasses")
     super().__init__(ctx, "dataclasses", member_map, ast)
@@ -220,3 +221,75 @@ def match_initvar(var):
 def match_classvar(var):
   """Unpack the type parameter from ClassVar[T]."""
   return abstract_utils.match_type_container(var, "typing.ClassVar")
+
+
+class Replace(abstract.PyTDFunction):
+  """Implements dataclasses.replace."""
+
+  @classmethod
+  def make(cls, ctx, module="dataclasses"):
+    return super().make("replace", ctx, module)
+
+  def _match_args_sequentially(self, node, args, alias_map, match_all_views):
+    ret = super()._match_args_sequentially(
+        node, args, alias_map, match_all_views
+    )
+    if not args.posargs:
+      # This is a weird case where pytype thinks the call can succeed, but
+      # there's no concrete `__obj` in the posargs.
+      # This can happen when `dataclasses.replace` is called with **kwargs:
+      #   @dataclasses.dataclass
+      #   class A:
+      #     replace = dataclasses.replace
+      #     def do(self, **kwargs):
+      #       return self.replace(**kwargs)
+      # (Yes, this is a simplified example of real code.)
+      # Since **kwargs is opaque magic, we can't do more type checking.
+      return ret
+    # _match_args_sequentially has succeeded, so we know we have 1 posarg (the
+    # object) and some number of named args (the new fields).
+    (obj,) = args.posargs
+    if len(obj.data) != 1:
+      return ret
+    obj = abstract_utils.get_atomic_value(obj)
+    # There are some cases where the user knows that obj will be a dataclass
+    # instance, but we don't. These instances are commonly false positives, so
+    # we should ignore them.
+    # (Consider a generic function where an `obj: T` is passed to replace().)
+    if (
+        obj.cls == self.ctx.convert.unsolvable
+        or not abstract_utils.is_dataclass(obj.cls)
+    ):
+      return ret
+    invalid_names = tuple(
+        name for name in args.namedargs.keys() if name not in obj.cls
+    )
+    if invalid_names:
+      # pylint: disable=line-too-long
+      # If we use the signature of replace() in the error message, it will be
+      # very confusing to users:
+      #   Invalid keyword arguments (y, z) to function dataclasses.replace [wrong-keyword-args]
+      #            Expected: (__obj, **changes)
+      #     Actually passed: (__obj, y, z)
+      # Instead, we construct a fake signature that shows the expected fields:
+      #   Invalid keyword arguments (y, z) to function dataclasses.replace [wrong-keyword-args]
+      #            Expected: (__obj: Test, *, x)
+      #     Actually passed: (__obj: Test, y, z)
+      # We also cheat a little bit by making sure the type of the object is
+      # included in the signature, pointing users towards more info.
+      # pylint: enable=line-too-long
+      fields = obj.cls.metadata["__dataclass_fields__"]
+      s = self.signatures[0].signature
+      sig = function.Signature(
+          name=s.name,
+          param_names=(f"__obj: {obj.cls.full_name}",),
+          posonly_count=s.posonly_count,
+          varargs_name=s.varargs_name,
+          kwonly_params=tuple(f.name for f in fields),
+          defaults=s.defaults,
+          kwargs_name=None,
+          annotations={},  # not used when printing errors.
+          postprocess_annotations=False,
+      )
+      raise function.WrongKeywordArgs(sig, args, self.ctx, invalid_names)
+    return ret

pytype/pyi/parser.py

@@ -279,7 +279,11 @@ class _MetadataVisitor(visitor.BaseVisitor):
   def visit_Call(self, node):
     posargs = tuple(evaluator.literal_eval(x) for x in node.args)
     kwargs = {x.arg: evaluator.literal_eval(x.value) for x in node.keywords}
-    return (node.func.id, posargs, kwargs)
+    if isinstance(node.func, astlib.Attribute):
+      func_name = _attribute_to_name(node.func)
+    else:
+      func_name = node.func
+    return (func_name.id, posargs, kwargs)
 
   def visit_Dict(self, node):
     return evaluator.literal_eval(node)

pytype/pyi/parser_test.py

@@ -2885,6 +2885,19 @@ class Foo:
           x: Annotated[int, Signal]
     """)
 
+  def test_attribute_access_and_call(self):
+    self.check("""
+      from typing import Annotated, Any
+      a: Any
+      def f() -> Annotated[list[int], a.b.C(3)]: ...
+    """, """
+      from typing import Annotated, Any, List
+
+      a: Any
+
+      def f() -> Annotated[List[int], {'tag': 'call', 'fn': 'a.b.C', 'posargs': (3,), 'kwargs': {}}]: ...
+    """)
+
 
 class ErrorTest(test_base.UnitTest):
   """Test parser errors."""

pytype/tests/test_dataclasses.py

@@ -784,6 +784,24 @@ class X:
         def __init__(self, b: int, a: str = ...) -> None: ...
     """)
 
+  def test_replace_wrong_keyword_args(self):
+    self.CheckWithErrors("""
+      import dataclasses
+      @dataclasses.dataclass
+      class Test:
+        x: int
+      x = Test(1)
+      dataclasses.replace(x, y=1, z=2)  # wrong-keyword-args
+    """)
+
+  def test_replace_not_a_dataclass(self):
+    self.CheckWithErrors("""
+      import dataclasses
+      class Test:
+        pass
+      dataclasses.replace(Test(), y=1, z=2)  # wrong-arg-types
+    """)
+
 
 class TestPyiDataclass(test_base.BaseTest):
   """Tests for @dataclasses in pyi files."""
@@ -1283,6 +1301,52 @@ class B(foo.A):
         b = B(1, '1')
       """)
 
+  def test_replace_wrong_keyword_args(self):
+    with self.DepTree([("foo.pyi", """
+        import dataclasses
+        @dataclasses.dataclass
+        class Test:
+            x: int
+            def __init__(self, x: int) -> None: ...
+    """)]):
+      self.CheckWithErrors("""
+        import dataclasses
+        import foo
+        x = foo.Test(1)
+        dataclasses.replace(x, y=1, z=2)  # wrong-keyword-args
+      """)
+
+  def test_replace_late_annotation(self):
+    # Regression test: LateAnnotations (like `z: Z`) should behave
+    # like their underlying types once resolved. The dataclass overlay
+    # relies on this behavior.
+    self.Check("""
+      from __future__ import annotations
+      import dataclasses
+      @dataclasses.dataclass
+      class A:
+        z: Z
+        def do(self):
+          return dataclasses.replace(self.z, name="A")
+      @dataclasses.dataclass
+      class Z:
+        name: str
+    """)
+
+  def test_replace_as_method_with_kwargs(self):
+    # This is a weird case where replace is added as a method, then called
+    # with kwargs. This makes pytype unable to see that `self` is the object
+    # being modified, and also caused a crash when the dataclass overlay tries
+    # to unpack the object being modified from the args.
+    self.Check("""
+      import dataclasses
+      @dataclasses.dataclass
+      class WithKwargs:
+        replace = dataclasses.replace
+        def do(self, **kwargs):
+            return self.replace(**kwargs)
+    """)
+
 
 if __name__ == "__main__":
   test_base.main()

pytype/tests/test_paramspec.py

@@ -187,6 +187,18 @@ def g(x: int, y: str) -> bool:
       assert_type(b, bool)
     """)
 
+  def test_use_as_protocol_parameter(self):
+    self.Check("""
+      from typing import ParamSpec, Protocol, TypeVar
+
+      P = ParamSpec('P')
+      T = TypeVar('T')
+
+      class CallLogger(Protocol[P, T]):
+        def args(self, *args: P.args, **kwargs: P.kwargs) -> None:
+          pass
+    """)
+
 
 _DECORATOR_PYI = """
   from typing import TypeVar, ParamSpec, Callable, List