Merge pull request #7339 from erik-krogh/pyPerf

Python: Cache more predicates to improve performance.

Author: erik-krogh

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -7,8 +7,9 @@
  */
 
 // Importing python under the `py` namespace to avoid importing `CallNode` from `Flow.qll` and thereby having a naming conflict with `API::CallNode`.
-private import python as py
+private import python as PY
 import semmle.python.dataflow.new.DataFlow
+private import semmle.python.internal.CachedStages
 
 /**
  * Provides classes and predicates for working with APIs used in a database.
@@ -427,13 +428,13 @@ module API {
       /** An abstract representative for imports of the module called `name`. */
       MkModuleImport(string name) {
         // Ignore the following module name for Python 2, as we alias `__builtin__` to `builtins` elsewhere
-        (name != "__builtin__" or py::major_version() = 3) and
+        (name != "__builtin__" or PY::major_version() = 3) and
         (
           imports(_, name)
           or
           // When we `import foo.bar.baz` we want to create API graph nodes also for the prefixes
           // `foo` and `foo.bar`:
-          name = any(py::ImportExpr e | not e.isRelative()).getAnImportedModuleName()
+          name = any(PY::ImportExpr e | not e.isRelative()).getAnImportedModuleName()
         )
         or
         // The `builtins` module should always be implicitly available
@@ -469,7 +470,7 @@ module API {
      * Ignores relative imports, such as `from ..foo.bar import baz`.
      */
     private predicate imports(DataFlow::Node imp, string name) {
-      exists(py::ImportExprNode iexpr |
+      exists(PY::ImportExprNode iexpr |
         imp.asCfgNode() = iexpr and
         not iexpr.getNode().isRelative() and
         name = iexpr.getNode().getImportedModuleName()
@@ -492,7 +493,7 @@ module API {
      *
      * `moduleImport("foo").getMember("bar")`
      */
-    private TApiNode potential_import_star_base(py::Scope s) {
+    private TApiNode potential_import_star_base(PY::Scope s) {
       exists(DataFlow::Node n |
         n.asCfgNode() = ImportStar::potentialImportStarBase(s) and
         use(result, n)
@@ -515,17 +516,17 @@ module API {
         )
         or
         // TODO: I had expected `DataFlow::AttrWrite` to contain the attribute writes from a dict, that's how JS works.
-        exists(py::Dict dict, py::KeyValuePair item |
+        exists(PY::Dict dict, PY::KeyValuePair item |
           dict = pred.asExpr() and
           dict.getItem(_) = item and
-          lbl = Label::member(item.getKey().(py::StrConst).getS()) and
+          lbl = Label::member(item.getKey().(PY::StrConst).getS()) and
           rhs.asExpr() = item.getValue()
         )
         or
-        exists(py::CallableExpr fn | fn = pred.asExpr() |
+        exists(PY::CallableExpr fn | fn = pred.asExpr() |
           not fn.getInnerScope().isAsync() and
           lbl = Label::return() and
-          exists(py::Return ret |
+          exists(PY::Return ret |
             rhs.asExpr() = ret.getValue() and
             ret.getScope() = fn.getInnerScope()
           )
@@ -568,7 +569,7 @@ module API {
         // Subclassing a node
         lbl = Label::subclass() and
         exists(DataFlow::Node superclass | pred.flowsTo(superclass) |
-          ref.asExpr().(py::ClassExpr).getABase() = superclass.asExpr()
+          ref.asExpr().(PY::ClassExpr).getABase() = superclass.asExpr()
         )
         or
         // awaiting
@@ -579,7 +580,7 @@ module API {
         )
       )
       or
-      exists(DataFlow::Node def, py::CallableExpr fn |
+      exists(DataFlow::Node def, PY::CallableExpr fn |
         rhs(base, def) and fn = trackDefNode(def).asExpr()
       |
         exists(int i |
@@ -598,7 +599,7 @@ module API {
       lbl = Label::member(any(string name | ref = Builtins::likelyBuiltin(name)))
       or
       // Unknown variables that may belong to a module imported with `import *`
-      exists(py::Scope s |
+      exists(PY::Scope s |
         base = potential_import_star_base(s) and
         lbl =
           Label::member(any(string name |
@@ -618,7 +619,7 @@ module API {
       )
       or
       // Ensure the Python 2 `__builtin__` module gets the name of the Python 3 `builtins` module.
-      py::major_version() = 2 and
+      PY::major_version() = 2 and
       nd = MkModuleImport("builtins") and
       imports(ref, "__builtin__")
       or
@@ -683,6 +684,7 @@ module API {
      */
     cached
     DataFlow::LocalSourceNode trackUseNode(DataFlow::LocalSourceNode src) {
+      Stages::TypeTracking::ref() and
       result = trackUseNode(src, DataFlow::TypeTracker::end()) and
       not result instanceof DataFlow::ModuleVariableNode
     }
@@ -759,18 +761,18 @@ module API {
           exists(Builtins::likelyBuiltin(member)) or
           ImportStar::namePossiblyDefinedInImportStar(_, member, _) or
           Impl::prefix_member(_, member, _) or
-          member = any(py::Dict d).getAnItem().(py::KeyValuePair).getKey().(py::StrConst).getS()
+          member = any(PY::Dict d).getAnItem().(PY::KeyValuePair).getKey().(PY::StrConst).getS()
         } or
         MkLabelUnknownMember() or
         MkLabelParameter(int i) {
           exists(any(DataFlow::CallCfgNode c).getArg(i))
           or
-          exists(any(py::Function f).getArg(i))
+          exists(any(PY::Function f).getArg(i))
         } or
         MkLabelKeywordParameter(string name) {
           exists(any(DataFlow::CallCfgNode c).getArgByName(name))
           or
-          exists(any(py::Function f).getArgByName(name))
+          exists(any(PY::Function f).getArgByName(name))
         } or
         MkLabelReturn() or
         MkLabelSubclass() or

python/ql/lib/semmle/python/AstExtended.qll

@@ -1,4 +1,5 @@
 import python
+private import semmle.python.internal.CachedStages
 
 /** A syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
 abstract class AstNode extends AstNode_ {
@@ -17,9 +18,14 @@ abstract class AstNode extends AstNode_ {
    * NOTE: For some statements and other purely syntactic elements,
    * there may not be a `ControlFlowNode`
    */
-  ControlFlowNode getAFlowNode() { py_flow_bb_node(result, this, _, _) }
+  cached
+  ControlFlowNode getAFlowNode() {
+    Stages::AST::ref() and
+    py_flow_bb_node(result, this, _, _)
+  }
 
   /** Gets the location for this AST node */
+  cached
   Location getLocation() { none() }
 
   /**
@@ -35,6 +41,7 @@ abstract class AstNode extends AstNode_ {
    * Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
    * Scope.getAStmt().
    */
+  cached
   abstract AstNode getAChildNode();
 
   /**
@@ -44,12 +51,16 @@ abstract class AstNode extends AstNode_ {
    * Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
    * Scope.getAStmt() applied to the parent.
    */
-  AstNode getParentNode() { result.getAChildNode() = this }
+  cached
+  AstNode getParentNode() {
+    Stages::AST::ref() and
+    result.getAChildNode() = this
+  }
 
   /** Whether this contains `inner` syntactically */
   predicate contains(AstNode inner) { this.getAChildNode+() = inner }
 
-  pragma[noinline]
+  pragma[nomagic]
   private predicate containsInScope(AstNode inner, Scope scope) {
     this.contains(inner) and
     not inner instanceof Scope and
@@ -106,9 +117,16 @@ class Comprehension extends Comprehension_, AstNode {
 
   override string toString() { result = "Comprehension" }
 
-  override Location getLocation() { result = Comprehension_.super.getLocation() }
+  override Location getLocation() {
+    Stages::AST::ref() and
+    result = Comprehension_.super.getLocation()
+  }
 
-  override AstNode getAChildNode() { result = this.getASubExpression() }
+  pragma[nomagic]
+  override AstNode getAChildNode() {
+    Stages::AST::ref() and
+    result = this.getASubExpression()
+  }
 
   Expr getASubExpression() {
     result = this.getIter() or

python/ql/lib/semmle/python/Exprs.qll

@@ -1,14 +1,19 @@
 import python
 private import semmle.python.pointsto.PointsTo
 private import semmle.python.objects.ObjectInternal
+private import semmle.python.internal.CachedStages
 
 /** An expression */
 class Expr extends Expr_, AstNode {
   /** Gets the scope of this expression */
   override Scope getScope() { py_scopes(this, result) }
 
   /** Gets a textual representation of this element. */
-  override string toString() { result = "Expression" }
+  cached
+  override string toString() {
+    Stages::AST::ref() and
+    result = "Expression"
+  }
 
   /** Gets the module in which this expression occurs */
   Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }

python/ql/lib/semmle/python/Flow.qll

@@ -1,5 +1,6 @@
 import python
 private import semmle.python.pointsto.PointsTo
+private import semmle.python.internal.CachedStages
 
 /*
  * Note about matching parent and child nodes and CFG splitting:
@@ -122,7 +123,9 @@ class ControlFlowNode extends @py_flow_node {
   AstNode getNode() { py_flow_bb_node(this, result, _, _) }
 
   /** Gets a textual representation of this element. */
+  cached
   string toString() {
+    Stages::DataFlow::ref() and
     exists(Scope s | s.getEntryNode() = this | result = "Entry node for " + s.toString())
     or
     exists(Scope s | s.getANormalExit() = this | result = "Exit node for " + s.toString())
@@ -191,7 +194,9 @@ class ControlFlowNode extends @py_flow_node {
   BasicBlock getBasicBlock() { result.contains(this) }
 
   /** Gets the scope containing this flow node */
+  cached
   Scope getScope() {
+    Stages::AST::ref() and
     if this.getNode() instanceof Scope
     then
       /* Entry or exit node */
@@ -614,7 +619,9 @@ class UnaryExprNode extends ControlFlowNode {
  * and nodes implicitly assigned in class and function definitions and imports.
  */
 class DefinitionNode extends ControlFlowNode {
+  cached
   DefinitionNode() {
+    Stages::AST::ref() and
     exists(Assign a | a.getATarget().getAFlowNode() = this)
     or
     exists(AnnAssign a | a.getTarget().getAFlowNode() = this and exists(a.getValue()))
@@ -673,6 +680,7 @@ abstract class SequenceNode extends ControlFlowNode {
   ControlFlowNode getAnElement() { result = this.getElement(_) }
 
   /** Gets the control flow node for the nth element of this sequence */
+  cached
   abstract ControlFlowNode getElement(int n);
 }
 
@@ -681,6 +689,7 @@ class TupleNode extends SequenceNode {
   TupleNode() { toAst(this) instanceof Tuple }
 
   override ControlFlowNode getElement(int n) {
+    Stages::AST::ref() and
     exists(Tuple t | this.getNode() = t and result.getNode() = t.getElt(n)) and
     (
       result.getBasicBlock().dominates(this.getBasicBlock())
@@ -998,11 +1007,13 @@ class BasicBlock extends @py_flow_node {
   string toString() { result = "BasicBlock" }
 
   /** Whether this basic block strictly dominates the other */
-  pragma[nomagic]
-  predicate strictlyDominates(BasicBlock other) { other.getImmediateDominator+() = this }
+  cached
+  predicate strictlyDominates(BasicBlock other) {
+    Stages::AST::ref() and
+    other.getImmediateDominator+() = this
+  }
 
   /** Whether this basic block dominates the other */
-  pragma[nomagic]
   predicate dominates(BasicBlock other) {
     this = other
     or
@@ -1011,6 +1022,7 @@ class BasicBlock extends @py_flow_node {
 
   cached
   BasicBlock getImmediateDominator() {
+    Stages::AST::ref() and
     this.firstNode().getImmediateDominator().getBasicBlock() = result
   }
 
@@ -1048,7 +1060,11 @@ class BasicBlock extends @py_flow_node {
   }
 
   /** Gets a successor to this basic block */
-  BasicBlock getASuccessor() { result = this.getLastNode().getASuccessor().getBasicBlock() }
+  cached
+  BasicBlock getASuccessor() {
+    Stages::AST::ref() and
+    result = this.getLastNode().getASuccessor().getBasicBlock()
+  }
 
   /** Gets a predecessor to this basic block */
   BasicBlock getAPredecessor() { result.getASuccessor() = this }
@@ -1118,7 +1134,11 @@ class BasicBlock extends @py_flow_node {
   }
 
   /** Holds if this basic block strictly reaches the other. Is the start of other reachable from the end of this. */
-  predicate strictlyReaches(BasicBlock other) { this.getASuccessor+() = other }
+  cached
+  predicate strictlyReaches(BasicBlock other) {
+    Stages::AST::ref() and
+    this.getASuccessor+() = other
+  }
 
   /** Holds if this basic block reaches the other. Is the start of other reachable from the end of this. */
   predicate reaches(BasicBlock other) { this = other or this.strictlyReaches(other) }

python/ql/lib/semmle/python/Function.qll

@@ -167,18 +167,18 @@ class Function extends Function_, Scope, AstNode {
 
 /** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
 class FunctionDef extends Assign {
+  FunctionExpr f;
+
   /* syntax: def name(...): ... */
   FunctionDef() {
     /* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
-    exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())
+    this.getValue() = f or this.getValue() = f.getADecoratorCall()
   }
 
   override string toString() { result = "FunctionDef" }
 
   /** Gets the function for this statement */
-  Function getDefinedFunction() {
-    exists(FunctionExpr func | this.containsInScope(func) and result = func.getInnerScope())
-  }
+  Function getDefinedFunction() { result = f.getInnerScope() }
 
   override Stmt getLastStatement() { result = this.getDefinedFunction().getLastStatement() }
 }

python/ql/lib/semmle/python/Import.qll

@@ -1,5 +1,6 @@
 import python
 private import semmle.python.types.Builtins
+private import semmle.python.internal.CachedStages
 
 /**
  * An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
@@ -203,7 +204,9 @@ class Import extends Import_ {
 /** An import * statement */
 class ImportStar extends ImportStar_ {
   /* syntax: from modname import * */
+  cached
   ImportExpr getModuleExpr() {
+    Stages::AST::ref() and
     result = this.getModule()
     or
     result = this.getModule().(ImportMember).getModule()

python/ql/lib/semmle/python/Module.qll

@@ -1,6 +1,7 @@
 import python
 private import semmle.python.objects.ObjectAPI
 private import semmle.python.objects.Modules
+private import semmle.python.internal.CachedStages
 
 /**
  * A module. This is the top level element in an AST, corresponding to a source file.
@@ -221,7 +222,9 @@ private predicate transitively_imported_from_entry_point(File file) {
   )
 }
 
+cached
 string moduleNameFromFile(Container file) {
+  Stages::AST::ref() and
   exists(string basename |
     basename = moduleNameFromBase(file) and
     legalShortName(basename)