Google sync

CHANGELOG

@@ -1,3 +1,24 @@
+Version 2024.02.09:
+
+Updates:
+* Remove 'deep' and 'store_all_calls' options.
+* Remove duplicate pytype inputs and outputs.
+
+Bug fixes:
+* Fix module resolution bug in load_pytd.
+* Pattern matching:
+  * Fix a corner case in pattern matching where the first case is None.
+  * Fix a corner case when comparing to Any in a case statement.
+  * Fix a false redundant-match when matching instances of a nonexhaustive type.
+  * Do not attempt to track matching if we don't recognise a CMP as an instance.
+  * Do not attempt to track matches if the match variable contains an Any.
+  * Rework the check for an out-of-order opcode in a match block.
+* Fix a crash when calling get() on a TypedDict instance.
+* Don't crash when inferring a type for an uncalled attrs.define.
+* Handle aliased imports in type stubs better.
+* Teach pytype that zip is actually a class.
+* Catch bad external types in type annotations.
+
 Version 2024.01.24:
 
 Updates:

pylintrc

@@ -15,6 +15,7 @@ disable=
   arguments-differ,
   arguments-out-of-order,
   assigning-non-slot,
+  assignment-from-no-return,
   attribute-defined-outside-init,
   bad-mcs-classmethod-argument,
   bad-option-value,

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2024.01.24'
+__version__ = '2024.02.09'

pytype/blocks/process_blocks.py

@@ -128,20 +128,3 @@ def adjust_returns(code, block_returns):
           new_line = next(lines, None)
           if new_line:
             op.line = new_line
-
-
-def check_out_of_order(code):
-  """Check if a line of code is executed out of order."""
-  # This sometimes happens due to compiler optimisations, and needs to be
-  # recorded so that we don't trigger code that is only meant to execute when
-  # the main flow of control reaches a certain line.
-  last_line = []
-  for block in code.order:
-    for op in block:
-      if not last_line or last_line[-1].line == op.line:
-        last_line.append(op)
-      else:
-        if op.line < last_line[-1].line:
-          for x in last_line:
-            x.metadata.is_out_of_order = True
-        last_line = [op]

pytype/pattern_matching.py

@@ -252,19 +252,20 @@ class _Matches:
   """Tracks branches of match statements."""
 
   def __init__(self, ast_matches):
-    self.start_to_end = {}
+    self.start_to_end = {}  # match_line : match_end_line
     self.end_to_starts = collections.defaultdict(list)
-    self.match_cases = {}
-    self.defaults = set()
-    self.as_names = {}
-    self.matches = []
+    self.match_cases = {}  # opcode_line : match_line
+    self.defaults = set()  # lines with defaults
+    self.as_names = {}  # case_end_line : case_as_name
+    self.unseen_cases = {}  # match_line : num_unseen_cases
 
     for m in ast_matches.matches:
       self._add_match(m.start, m.end, m.cases)
 
   def _add_match(self, start, end, cases):
     self.start_to_end[start] = end
     self.end_to_starts[end].append(start)
+    self.unseen_cases[start] = len(cases)
     for c in cases:
       for i in range(c.start, c.end + 1):
         self.match_cases[i] = start
@@ -273,6 +274,10 @@ def _add_match(self, start, end, cases):
       if c.as_name:
         self.as_names[c.end] = c.as_name
 
+  def register_case(self, match_line, case_line):
+    assert self.match_cases[case_line] == match_line
+    self.unseen_cases[match_line] -= 1
+
   def __repr__(self):
     return f"""
       Matches: {sorted(self.start_to_end.items())}
@@ -301,10 +306,9 @@ def __init__(self, ast_matches, ctx):
     self.ctx = ctx
 
   def _get_option_tracker(
-      self, match_var: cfg.Variable, case_line: int
+      self, match_var: cfg.Variable, match_line: int
   ) -> _OptionTracker:
     """Get the option tracker for a match line."""
-    match_line = self.matches.match_cases[case_line]
     if (match_line not in self._option_tracker or
         match_var.id not in self._option_tracker[match_line]):
       self._option_tracker[match_line][match_var.id] = (
@@ -323,8 +327,16 @@ def _make_instance_for_match(self, node, types):
       ret.append(self.ctx.vm.init_class(node, cls))
     return self.ctx.join_variables(node, ret)
 
+  def _register_case_branch(self, op: opcodes.Opcode) -> Optional[int]:
+    match_line = self.matches.match_cases.get(op.line)
+    if match_line is None:
+      return None
+    self.matches.register_case(match_line, op.line)
+    return match_line
+
   def instantiate_case_var(self, op, match_var, node):
-    tracker = self._get_option_tracker(match_var, op.line)
+    match_line = self.matches.match_cases[op.line]
+    tracker = self._get_option_tracker(match_var, match_line)
     if tracker.cases[op.line]:
       # We have matched on one or more classes in this case.
       types = [x.typ for x in tracker.cases[op.line]]
@@ -360,14 +372,16 @@ def register_match_type(self, op: opcodes.Opcode):
     self._match_types[match_line].add(_MatchTypes.make(op))
 
   def add_none_branch(self, op: opcodes.Opcode, match_var: cfg.Variable):
-    if op.line in self.matches.match_cases:
-      tracker = self._get_option_tracker(match_var, op.line)
-      tracker.cover_from_none(op.line)
-      if not tracker.is_complete:
-        return None
-      else:
-        # This is the last remaining case, and will always succeed.
-        return True
+    match_line = self._register_case_branch(op)
+    if not match_line:
+      return None
+    tracker = self._get_option_tracker(match_var, match_line)
+    tracker.cover_from_none(op.line)
+    if not tracker.is_complete:
+      return None
+    else:
+      # This is the last remaining case, and will always succeed.
+      return True
 
   def add_cmp_branch(
       self,
@@ -377,12 +391,13 @@ def add_cmp_branch(
       case_var: cfg.Variable
   ) -> _MatchSuccessType:
     """Add a compare-based match case branch to the tracker."""
-    if cmp_type not in (slots.CMP_EQ, slots.CMP_IS):
+    match_line = self._register_case_branch(op)
+    if not match_line:
       return None
 
-    match_line = self.matches.match_cases.get(op.line)
-    if not match_line:
+    if cmp_type not in (slots.CMP_EQ, slots.CMP_IS):
       return None
+
     match_type = self._match_types[match_line]
 
     try:
@@ -403,7 +418,7 @@ def add_cmp_branch(
     # (enum or union of literals) that we are tracking.
     if not tracker:
       if _is_literal_match(match_var) or _is_enum_match(match_var, case_val):
-        tracker = self._get_option_tracker(match_var, op.line)
+        tracker = self._get_option_tracker(match_var, match_line)
 
     # If none of the above apply we cannot do any sort of tracking.
     if not tracker:
@@ -425,32 +440,31 @@ def add_cmp_branch(
   def add_class_branch(self, op: opcodes.Opcode, match_var: cfg.Variable,
                        case_var: cfg.Variable) -> _MatchSuccessType:
     """Add a class-based match case branch to the tracker."""
-    tracker = self._get_option_tracker(match_var, op.line)
+    match_line = self._register_case_branch(op)
+    if not match_line:
+      return None
+    tracker = self._get_option_tracker(match_var, match_line)
     tracker.cover(op.line, case_var)
     return tracker.is_complete or None
 
   def add_default_branch(self, op: opcodes.Opcode) -> _MatchSuccessType:
     """Add a default match case branch to the tracker."""
-    match_line = self.matches.match_cases.get(op.line)
-    if match_line is None:
-      return None
-    if match_line in self._option_tracker:
-      for opt in self._option_tracker[match_line].values():
-        # We no longer check for exhaustive or redundant matches once we hit a
-        # default case.
-        opt.invalidate()
-      return True
-    else:
+    match_line = self._register_case_branch(op)
+    if not match_line or match_line not in self._option_tracker:
       return None
 
+    for opt in self._option_tracker[match_line].values():
+      # We no longer check for exhaustive or redundant matches once we hit a
+      # default case.
+      opt.invalidate()
+    return True
+
   def check_ending(
       self,
       op: opcodes.Opcode,
       implicit_return: bool = False
   ) -> List[IncompleteMatch]:
     """Check if we have ended a match statement with leftover cases."""
-    if op.metadata.is_out_of_order:
-      return []
     line = op.line
     if implicit_return:
       done = set()
@@ -464,6 +478,10 @@ def check_ending(
     ret = []
     for i in done:
       for start in self.matches.end_to_starts[i]:
+        if self.matches.unseen_cases[start] > 0:
+          # We have executed some opcode out of order and thus gone past the end
+          # of the match block before seeing all case branches.
+          continue
         trackers = self._option_tracker[start]
         for tracker in trackers.values():
           if tracker.is_valid:

pytype/pytd/visitors.py

@@ -548,7 +548,10 @@ def VisitNamedType(self, t):
     if (isinstance(item, pytd.Constant) and
         item.name == "typing_extensions.TypedDict"):
       return self.to_type(pytd.NamedType("typing.TypedDict"))
-    return self.to_type(item)
+    try:
+      return self.to_type(item)
+    except NotImplementedError as e:
+      raise SymbolLookupError(f"{item} is not a type") from e
 
   def VisitClassType(self, t):
     new_type = self.VisitNamedType(t)
@@ -820,6 +823,12 @@ def VisitNamedType(self, node):
           resolved_node = self.to_type(self._LookupItemRecursive(node.name))
         except KeyError:
           resolved_node = node  # lookup failures are handled later
+        except NotImplementedError as e:
+          # to_type() can raise NotImplementedError, but _LookupItemRecursive
+          # shouldn't return a pytd node that can't be turned into a type in
+          # this specific case. As such, it's impossible to test this case.
+          # But it's irresponsible to just crash on it, so here we are.
+          raise SymbolLookupError(f"{node.name} is not a type") from e
         else:
           if isinstance(resolved_node, pytd.ClassType):
             resolved_node.name = node.name

pytype/rewrite/CMakeLists.txt

@@ -15,6 +15,8 @@ py_library(
     abstract
   SRCS
     abstract.py
+  DEPS
+    pytype.blocks.blocks
 )
 
 py_test(
@@ -60,6 +62,7 @@ py_library(
     frame.py
   DEPS
     .abstract
+    .stack
     pytype.blocks.blocks
     pytype.rewrite.flow.flow
 )
@@ -78,12 +81,34 @@ py_test(
     pytype.rewrite.tests.test_utils
 )
 
+py_library(
+  NAME
+    stack
+  SRCS
+    stack.py
+  DEPS
+    .abstract
+    pytype.rewrite.flow.flow
+)
+
+py_test(
+  NAME
+    stack_test
+  SRCS
+    stack_test.py
+  DEPS
+    .abstract
+    .stack
+    pytype.rewrite.flow.flow
+)
+
 py_library(
   NAME
     vm
   SRCS
     vm.py
   DEPS
+    .abstract
     .frame
     pytype.blocks.blocks
     pytype.rewrite.flow.flow
@@ -95,6 +120,7 @@ py_test(
   SRCS
     vm_test.py
   DEPS
+    .abstract
     .vm
     pytype.pyc.pyc
     pytype.rewrite.tests.test_utils

pytype/rewrite/abstract.py

@@ -1,5 +1,7 @@
 """Abstract representations of Python values."""
 
+from pytype.blocks import blocks
+
 
 class BaseValue:
   pass
@@ -15,3 +17,20 @@ def __repr__(self):
 
   def __eq__(self, other):
     return type(self) == type(other) and self.constant == other.constant  # pylint: disable=unidiomatic-typecheck
+
+
+class Function(BaseValue):
+
+  def __init__(self, name: str, code: blocks.OrderedCode):
+    self.name = name
+    self.code = code
+
+  def __repr__(self):
+    return f'Function({self.name})'
+
+
+class _Null(BaseValue):
+  pass
+
+
+NULL = _Null()

pytype/rewrite/analyze.py

@@ -44,8 +44,9 @@ def check_types(
     init_maximum_depth: int = _INIT_MAXIMUM_DEPTH,
     maximum_depth: int = _MAXIMUM_DEPTH,
 ) -> Analysis:
-  """Check types for the given source code."""
-  _analyze(src, options, loader, init_maximum_depth, maximum_depth)
+  """Checks types for the given source code."""
+  vm = _make_vm(src, options, loader, init_maximum_depth, maximum_depth)
+  vm.analyze_all_defs()
   return Analysis(Context(), None, None)
 
 
@@ -56,28 +57,27 @@ def infer_types(
     init_maximum_depth: int = _INIT_MAXIMUM_DEPTH,
     maximum_depth: int = _MAXIMUM_DEPTH,
 ) -> Analysis:
-  """Infer types for the given source code."""
-  _analyze(src, options, loader, init_maximum_depth, maximum_depth)
+  """Infers types for the given source code."""
+  vm = _make_vm(src, options, loader, init_maximum_depth, maximum_depth)
+  vm.infer_stub()
   ast = pytd.TypeDeclUnit('inferred + unknowns', (), (), (), (), ())
   deps = pytd.TypeDeclUnit('<all>', (), (), (), (), ())
   return Analysis(Context(), ast, deps)
 
 
-def _analyze(
+def _make_vm(
     src: str,
     options: config.Options,
     loader: load_pytd.Loader,
     init_maximum_depth: int,
     maximum_depth: int,
-) -> None:
-  """Analyze the given source code."""
+) -> vm_lib.VirtualMachine:
+  """Creates abstract virtual machine for given source code."""
   del loader, init_maximum_depth, maximum_depth
   code = _get_bytecode(src, options)
   # TODO(b/241479600): Populate globals from builtins.
-  globals_ = {}
-  vm = vm_lib.VirtualMachine(code, globals_)
-  vm.run()
-  # TODO(b/241479600): Analyze classes and functions.
+  initial_globals = {}
+  return vm_lib.VirtualMachine(code, initial_globals)
 
 
 def _get_bytecode(src: str, options: config.Options) -> blocks.OrderedCode: