-
Notifications
You must be signed in to change notification settings - Fork 296
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[x2cpg] Performance improvements in Overlay Passes (#4227)
* Changes to parallelize linkToSingle method processing Changes to parallelize linkToSingle method processing * Converted StaticCallLinker to parallel pass * Some testing changes * Improvments in StaticCallLinker and LinkToMultiple methods 1. Made changes in `StaticCallLinker to divide the list of chunks further into smaller batches so that the in memory data generated in one batch becomes smaller and in turn when it gets absorbed the given memory will be garbage collected to free up the memory. 2. Introduced parallism inside `linkToMultiple` method handling/ * Few debug logs along with failing unit tests 1. Added few debug logs while linking `CALL` nodes to `METHOD` nodes if the number of METHOD nodes found is greater than 5. 2. There were wrong `FieldAccess` nodes being created, which is resulting into large memory being consumed in `StaticCallLinker` pass. Added respective failing unit tests in ignored state. These unit tests needs to be fixed. * Updated unit tests with working one and the failing one * Review comment fixes * Review comment fixes * Fix for fieldAccess operator call node In case of chained field access like `a.b.c` for `.c`, it was creating a `CALL` node with methodFullName set to `<operator>.fieldAccess` and name set to "c". Hence there were many `METHOD` sutbs being created for each of such `CALL` nodes with fullName set to `<operator>.fieldAccess`. Which is resulting into many edges being created mapping each fieldAccess `CALL` nodes to each of these `METHOD` stubs inside `StaticCallLinker` Pass. This change fixes the issue. * Changed TypeUsagePass and StaticCallLinker 1. Converted TypeUsagePass into two passes with ForkJoinParallelPass 2. Converted StaticCallLinker pass to extend from ForkJoinParallelPass * removed unwanted code * changes as per review comment * Review comment fixes * review comment fixes * review comment fixes
- Loading branch information
1 parent
310f209
commit a02559d
Showing
9 changed files
with
131 additions
and
122 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package io.joern.x2cpg.passes.base | ||
|
|
||
| import io.joern.x2cpg.utils.LinkingUtil | ||
| import io.shiftleft.codepropertygraph.Cpg | ||
| import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} | ||
| import io.shiftleft.passes.ForkJoinParallelCpgPass | ||
| import overflowdb.Node | ||
| import overflowdb.traversal.* | ||
|
|
||
| class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { | ||
| val srcLabels = List( | ||
| NodeTypes.METHOD_PARAMETER_IN, | ||
| NodeTypes.METHOD_PARAMETER_OUT, | ||
| NodeTypes.METHOD_RETURN, | ||
| NodeTypes.MEMBER, | ||
| NodeTypes.LITERAL, | ||
| NodeTypes.CALL, | ||
| NodeTypes.LOCAL, | ||
| NodeTypes.IDENTIFIER, | ||
| NodeTypes.BLOCK, | ||
| NodeTypes.METHOD_REF, | ||
| NodeTypes.TYPE_REF, | ||
| NodeTypes.UNKNOWN | ||
| ) | ||
|
|
||
| def generateParts(): Array[List[Node]] = { | ||
| val nodes = cpg.graph.nodes(srcLabels: _*).toList | ||
| nodes.grouped(getBatchSize(nodes.size)).toArray | ||
| } | ||
| def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { | ||
| linkToSingle( | ||
| cpg = cpg, | ||
| srcNodes = part, | ||
| srcLabels = srcLabels, | ||
| dstNodeLabel = NodeTypes.TYPE, | ||
| edgeType = EdgeTypes.EVAL_TYPE, | ||
| dstNodeMap = typeFullNameToNode(cpg, _), | ||
| dstFullNameKey = PropertyNames.TYPE_FULL_NAME, | ||
| dstGraph = builder, | ||
| None | ||
| ) | ||
| } | ||
| } |
30 changes: 30 additions & 0 deletions
30
joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package io.joern.x2cpg.passes.base | ||
|
|
||
| import io.joern.x2cpg.utils.LinkingUtil | ||
| import io.shiftleft.codepropertygraph.Cpg | ||
| import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} | ||
| import io.shiftleft.passes.ForkJoinParallelCpgPass | ||
| import overflowdb.Node | ||
| import overflowdb.traversal.* | ||
|
|
||
| class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { | ||
| val srcLabels = List(NodeTypes.TYPE) | ||
|
|
||
| def generateParts(): Array[List[Node]] = { | ||
| val nodes = cpg.graph.nodes(srcLabels: _*).toList | ||
| nodes.grouped(getBatchSize(nodes.size)).toArray | ||
| } | ||
| def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { | ||
| linkToSingle( | ||
| cpg = cpg, | ||
| srcNodes = part, | ||
| srcLabels = srcLabels, | ||
| dstNodeLabel = NodeTypes.TYPE_DECL, | ||
| edgeType = EdgeTypes.REF, | ||
| dstNodeMap = typeDeclFullNameToNode(cpg, _), | ||
| dstFullNameKey = PropertyNames.TYPE_DECL_FULL_NAME, | ||
| dstGraph = builder, | ||
| None | ||
| ) | ||
| } | ||
| } |
48 changes: 0 additions & 48 deletions
48
joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeUsagePass.scala
This file was deleted.
Oops, something went wrong.
8 changes: 6 additions & 2 deletions
8
...-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
72 changes: 24 additions & 48 deletions
72
...cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/StaticCallLinker.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,64 +1,40 @@ | ||
| package io.joern.x2cpg.passes.callgraph | ||
|
|
||
| import io.joern.x2cpg.utils.LinkingUtil | ||
| import io.shiftleft.codepropertygraph.Cpg | ||
| import io.shiftleft.codepropertygraph.generated.nodes._ | ||
| import io.shiftleft.codepropertygraph.generated.nodes.* | ||
| import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes} | ||
| import io.shiftleft.passes.CpgPass | ||
| import io.shiftleft.semanticcpg.language._ | ||
| import io.shiftleft.passes.ForkJoinParallelCpgPass | ||
| import io.shiftleft.semanticcpg.language.* | ||
| import org.slf4j.{Logger, LoggerFactory} | ||
|
|
||
| import scala.collection.mutable | ||
| class StaticCallLinker(cpg: Cpg) extends ForkJoinParallelCpgPass[Seq[Call]](cpg) with LinkingUtil { | ||
|
|
||
| class StaticCallLinker(cpg: Cpg) extends CpgPass(cpg) { | ||
|
|
||
| import StaticCallLinker._ | ||
| private val methodFullNameToNode = mutable.Map.empty[String, List[Method]] | ||
|
|
||
| override def run(dstGraph: DiffGraphBuilder): Unit = { | ||
| private val logger: Logger = LoggerFactory.getLogger(classOf[StaticCallLinker]) | ||
|
|
||
| cpg.method.foreach { method => | ||
| methodFullNameToNode.updateWith(method.fullName) { | ||
| case Some(l) => Some(method :: l) | ||
| case None => Some(List(method)) | ||
| } | ||
| } | ||
| override def generateParts(): Array[Seq[Call]] = { | ||
| val calls = cpg.call.l | ||
| calls.grouped(getBatchSize(calls.size)).toArray | ||
| } | ||
|
|
||
| cpg.call.foreach { call => | ||
| override def runOnPart(builder: DiffGraphBuilder, calls: Seq[Call]): Unit = { | ||
| calls.foreach { call => | ||
| try { | ||
| linkCall(call, dstGraph) | ||
| call.dispatchType match { | ||
| case DispatchTypes.STATIC_DISPATCH | DispatchTypes.INLINED => | ||
| val resolvedMethods = cpg.method.fullNameExact(call.methodFullName).l | ||
| resolvedMethods.foreach(dst => builder.addEdge(call, dst, EdgeTypes.CALL)) | ||
| val size = resolvedMethods.size | ||
| // Add the debug logs with number of METHOD nodes found for given method full name | ||
| if size > 1 then logger.debug(s"Total $size METHOD nodes found for -> ${call.methodFullName}") | ||
| case DispatchTypes.DYNAMIC_DISPATCH => | ||
| // Do nothing | ||
| case _ => logger.warn(s"Unknown dispatch type on dynamic CALL ${call.code}") | ||
| } | ||
| } catch { | ||
| case exception: Exception => | ||
| throw new RuntimeException(exception) | ||
| logger.error(s"Exception in StaticCallLinker: ", exception) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private def linkCall(call: Call, dstGraph: DiffGraphBuilder): Unit = { | ||
| call.dispatchType match { | ||
| case DispatchTypes.STATIC_DISPATCH | DispatchTypes.INLINED => | ||
| linkStaticCall(call, dstGraph) | ||
| case DispatchTypes.DYNAMIC_DISPATCH => | ||
| // Do nothing | ||
| case _ => logger.warn(s"Unknown dispatch type on dynamic CALL ${call.code}") | ||
| } | ||
| } | ||
|
|
||
| private def linkStaticCall(call: Call, dstGraph: DiffGraphBuilder): Unit = { | ||
| val resolvedMethodOption = methodFullNameToNode.get(call.methodFullName) | ||
| if (resolvedMethodOption.isDefined) { | ||
| resolvedMethodOption.get.foreach { dst => | ||
| dstGraph.addEdge(call, dst, EdgeTypes.CALL) | ||
| } | ||
| } else { | ||
| logger.info( | ||
| s"Unable to link static CALL with METHOD_FULL_NAME ${call.methodFullName}, NAME ${call.name}, " + | ||
| s"SIGNATURE ${call.signature}, CODE ${call.code}" | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| object StaticCallLinker { | ||
| private val logger: Logger = LoggerFactory.getLogger(classOf[StaticCallLinker]) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters