Merge remote-tracking branch 'origin/master' into dave/c#/call-nodes

dataflowengineoss/build.sbt

@@ -1,6 +1,6 @@
 name := "dataflowengineoss"
 
-dependsOn(Projects.semanticcpg, Projects.x2cpg)
+dependsOn(Projects.semanticcpg, Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
   "org.antlr"               % "antlr4-runtime"             % Versions.antlr,

dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/testfixtures/SemanticTestCpg.scala

@@ -0,0 +1,52 @@
+package io.joern.dataflowengineoss.testfixtures
+
+import io.joern.dataflowengineoss.DefaultSemantics
+import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
+import io.joern.dataflowengineoss.queryengine.EngineContext
+import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics}
+import io.joern.x2cpg.testfixtures.TestCpg
+import io.shiftleft.semanticcpg.layers.LayerCreatorContext
+
+/** Extends the capabilities of the test CPG to handle the configuration of data-flow enhancements.
+  */
+trait SemanticTestCpg { this: TestCpg =>
+
+  protected var _withOssDataflow                = false
+  protected var _extraFlows                     = List.empty[FlowSemantic]
+  protected implicit var context: EngineContext = EngineContext()
+
+  /** Allows one to enable data-flow analysis capabilities to the TestCpg.
+    */
+  def withOssDataflow(value: Boolean = true): this.type = {
+    _withOssDataflow = value
+    this
+  }
+
+  /** Allows one to add additional semantics to the engine context during PDG creation.
+    */
+  def withExtraFlows(value: List[FlowSemantic] = List.empty): this.type = {
+    _extraFlows = value
+    this
+  }
+
+  /** Some frontends require OSS data-flow to execute after post-processing, so we choose to expose this method without
+    * defining where it's executed.
+    */
+  def applyOssDataFlow(): Unit = {
+    if (_withOssDataflow) {
+      val context = new LayerCreatorContext(this)
+      val options = new OssDataFlowOptions(extraFlows = _extraFlows)
+      new OssDataFlow(options).run(context)
+      this.context = EngineContext(Semantics.fromList(DefaultSemantics().elements ++ _extraFlows))
+    }
+  }
+
+}
+
+/** Allows the tests to make use of the data-flow engine and any additional semantics.
+  */
+trait SemanticCpgTestFixture(extraFlows: List[FlowSemantic] = List.empty) {
+
+  implicit val context: EngineContext = EngineContext(Semantics.fromList(DefaultSemantics().elements ++ extraFlows))
+
+}

joern-cli/frontends/c2cpg/build.sbt

@@ -1,6 +1,6 @@
 name := "c2cpg"
 
-dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % Test, Projects.x2cpg % "compile->compile;test->test")
+dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
   "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",

joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/CCodeToCpgSuite.scala

@@ -1,11 +1,12 @@
 package io.joern.c2cpg.testfixtures
 
 import better.files.File
-import io.joern.c2cpg.{C2Cpg, Config}
 import io.joern.c2cpg.parser.FileDefaults
+import io.joern.c2cpg.{C2Cpg, Config}
+import io.joern.dataflowengineoss.semanticsloader.FlowSemantic
+import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg}
 import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend}
 import io.shiftleft.codepropertygraph.Cpg
-import org.scalatest.Inside
 
 trait C2CpgFrontend extends LanguageFrontend {
   def execute(sourceCodePath: java.io.File): Cpg = {
@@ -23,8 +24,18 @@ trait C2CpgFrontend extends LanguageFrontend {
   }
 }
 
-class DefaultTestCpgWithC(val fileSuffix: String) extends DefaultTestCpg with C2CpgFrontend
+class DefaultTestCpgWithC(val fileSuffix: String) extends DefaultTestCpg with C2CpgFrontend with SemanticTestCpg {
+  override protected def applyPasses(): Unit = {
+    super.applyPasses()
+    applyOssDataFlow()
+  }
+}
 
-class CCodeToCpgSuite(fileSuffix: String = FileDefaults.C_EXT)
-    extends Code2CpgFixture(() => new DefaultTestCpgWithC(fileSuffix))
-    with Inside
+class CCodeToCpgSuite(
+  fileSuffix: String = FileDefaults.C_EXT,
+  withOssDataflow: Boolean = false,
+  extraFlows: List[FlowSemantic] = List.empty
+) extends Code2CpgFixture(() =>
+      new DefaultTestCpgWithC(fileSuffix).withOssDataflow(withOssDataflow).withExtraFlows(extraFlows)
+    )
+    with SemanticCpgTestFixture(extraFlows)

joern-cli/frontends/csharpsrc2cpg/build.sbt

@@ -7,7 +7,7 @@ import scala.util.Try
 
 name := "csharpsrc2cpg"
 
-dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test")
+dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 lazy val appProperties = settingKey[Config]("App Properties")
 appProperties := {

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala

@@ -18,6 +18,9 @@ class DotNetAstGenRunner(config: Config) extends AstGenRunnerBase(config) {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
+  // The x86 variant seems to run well enough on MacOS M-family chips, whereas the ARM build crashes
+  override val MacArm: String = MacX86
+
   override def fileFilter(file: String, out: File): Boolean = {
     file.stripSuffix(".json").replace(out.pathAsString, config.inputPath) match {
       case filePath if isIgnoredByUserConfig(filePath) => false

joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/testfixtures/CSharpCode2CpgFixture.scala

@@ -2,8 +2,8 @@ package io.joern.csharpsrc2cpg.testfixtures
 
 import io.joern.csharpsrc2cpg.{CSharpSrc2Cpg, Config}
 import io.joern.dataflowengineoss.language.Path
-import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
-import io.joern.dataflowengineoss.queryengine.EngineContext
+import io.joern.dataflowengineoss.semanticsloader.FlowSemantic
+import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg}
 import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend}
 import io.joern.x2cpg.{ValidationMode, X2Cpg}
 import io.shiftleft.codepropertygraph.generated.Cpg
@@ -13,12 +13,20 @@ import org.scalatest.Inside
 
 import java.io.File
 
-class CSharpCode2CpgFixture(withPostProcessing: Boolean = false, withDataFlow: Boolean = false)
-    extends Code2CpgFixture(() => new DefaultTestCpgWithCSharp(withPostProcessing, withDataFlow))
+class CSharpCode2CpgFixture(
+  withPostProcessing: Boolean = false,
+  withDataFlow: Boolean = false,
+  extraFlows: List[FlowSemantic] = List.empty
+) extends Code2CpgFixture(() =>
+      new DefaultTestCpgWithCSharp()
+        .withOssDataflow(withDataFlow)
+        .withExtraFlows(extraFlows)
+        .withPostProcessingPasses(withPostProcessing)
+    )
+    with SemanticCpgTestFixture(extraFlows)
     with Inside {
 
-  implicit val resolver: ICallResolver           = NoResolve
-  implicit lazy val engineContext: EngineContext = EngineContext()
+  implicit val resolver: ICallResolver = NoResolve
 
   protected def flowToResultPairs(path: Path): List[(String, Integer)] =
     path.resultPairs().collect { case (firstElement: String, secondElement: Option[Integer]) =>
@@ -47,22 +55,11 @@ class CSharpCode2CpgFixture(withPostProcessing: Boolean = false, withDataFlow: B
        |""".stripMargin
 }
 
-class DefaultTestCpgWithCSharp(withPostProcessing: Boolean, withDataFlow: Boolean)
-    extends DefaultTestCpg
-    with CSharpFrontend {
+class DefaultTestCpgWithCSharp extends DefaultTestCpg with CSharpFrontend with SemanticTestCpg {
 
-  override def applyPasses(): Unit = {
-    X2Cpg.applyDefaultOverlays(this)
-
-    if (withPostProcessing) {
-      CSharpSrc2Cpg.postProcessingPasses(this, config).foreach(_.createAndApply())
-    }
-
-    if (withDataFlow) {
-      val context = new LayerCreatorContext(this)
-      val options = new OssDataFlowOptions()
-      new OssDataFlow(options).run(context)
-    }
+  override def applyPostProcessingPasses(): Unit = {
+    CSharpSrc2Cpg.postProcessingPasses(this, config).foreach(_.createAndApply())
+    super.applyPostProcessingPasses()
   }
 
 }

joern-cli/frontends/gosrc2cpg/build.sbt

@@ -5,7 +5,7 @@ import com.typesafe.config.{Config, ConfigFactory}
 
 name := "gosrc2cpg"
 
-dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test")
+dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
   "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,

joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala

@@ -1,15 +1,13 @@
 package io.joern.go2cpg.testfixtures
 
 import better.files.File
-import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
-import io.joern.dataflowengineoss.queryengine.EngineContext
-import io.joern.gosrc2cpg.datastructures.GoGlobal
+import io.joern.dataflowengineoss.semanticsloader.FlowSemantic
+import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg}
 import io.joern.gosrc2cpg.{Config, GoSrc2Cpg}
 import io.joern.x2cpg.X2Cpg
 import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve}
-import io.shiftleft.semanticcpg.layers.LayerCreatorContext
 import org.scalatest.Inside
 trait Go2CpgFrontend extends LanguageFrontend {
   def execute(sourceCodePath: java.io.File): Cpg = {
@@ -25,29 +23,22 @@ trait Go2CpgFrontend extends LanguageFrontend {
   }
 }
 
-class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with Go2CpgFrontend {
-
-  private var _withOssDataflow = false
-
-  def withOssDataflow(value: Boolean = true): this.type = {
-    _withOssDataflow = value
-    this
-  }
-
-  override def applyPasses(): Unit = {
-    X2Cpg.applyDefaultOverlays(this)
-    if (_withOssDataflow) {
-      val context = new LayerCreatorContext(this)
-      val options = new OssDataFlowOptions()
-      new OssDataFlow(options).run(context)
-    }
+class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with Go2CpgFrontend with SemanticTestCpg {
+  override protected def applyPasses(): Unit = {
+    super.applyPasses()
+    applyOssDataFlow()
   }
 }
 
-class GoCodeToCpgSuite(fileSuffix: String = ".go", withOssDataflow: Boolean = false)
-    extends Code2CpgFixture(() => new DefaultTestCpgWithGo(fileSuffix).withOssDataflow(withOssDataflow))
+class GoCodeToCpgSuite(
+  fileSuffix: String = ".go",
+  withOssDataflow: Boolean = false,
+  extraFlows: List[FlowSemantic] = List.empty
+) extends Code2CpgFixture(() =>
+      new DefaultTestCpgWithGo(fileSuffix).withOssDataflow(withOssDataflow).withExtraFlows(extraFlows)
+    )
+    with SemanticCpgTestFixture(extraFlows)
     with Inside {
-  implicit val resolver: ICallResolver           = NoResolve
-  implicit lazy val engineContext: EngineContext = EngineContext()
+  implicit val resolver: ICallResolver = NoResolve
 
 }

joern-cli/frontends/javasrc2cpg/build.sbt

@@ -1,6 +1,6 @@
 name := "javasrc2cpg"
 
-dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test")
+dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
   "io.shiftleft"           %% "codepropertygraph"             % Versions.cpg,

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala

@@ -219,6 +219,12 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator =>
       typeInfoCalc
         .fullName(parameter.getType)
         .orElse(scope.lookupType(parameter.getTypeAsString))
+        // In a scenario where we have an import of an external type e.g. `import foo.bar.Baz` and
+        // this parameter's type is e.g. `Baz<String>`, the lookup will fail. However, if we lookup
+        // for `Baz` instead (i.e. without type arguments), then the lookup will succeed.
+        .orElse(
+          Try(parameter.getType.asClassOrInterfaceType).toOption.flatMap(t => scope.lookupType(t.getNameAsString))
+        )
         .map(_ ++ maybeArraySuffix)
         .getOrElse(s"${Defines.UnresolvedNamespace}.${parameter.getTypeAsString}")
     val evalStrat =

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodParameterTests.scala

@@ -135,4 +135,46 @@ class MethodParameterTests2 extends JavaSrcCode2CpgFixture {
       }
     }
   }
+
+  "imported method parameter's external, non-generic type" should {
+    val cpg = code("""
+        |import foo.bar.Baz;
+        |class Main {
+        | void run(Baz p1) {}
+        |}
+        |""".stripMargin)
+
+    "have correct type for parameter p1" in {
+      val List(param) = cpg.method.name("run").parameter.name("p1").l
+      param.typeFullName shouldBe "foo.bar.Baz"
+    }
+  }
+
+  "imported method parameter's internal, generic type" should {
+    val cpg = code("""
+        |import java.util.*;
+        |class Main {
+        | void run(List<String> p1) {}
+        |}
+        |""".stripMargin)
+
+    "have correct type for parameter p1" in {
+      val List(param) = cpg.method.name("run").parameter.name("p1").l
+      param.typeFullName shouldBe "java.util.List"
+    }
+  }
+
+  "imported method parameter's external, generic type" should {
+    val cpg = code("""
+        |import foo.bar.Baz;
+        |class Main {
+        |  void run(Baz<String> p1) {}
+        |}
+        |""".stripMargin)
+
+    "have correct type for parameter p1" in {
+      val List(param) = cpg.method.name("run").parameter.name("p1").l
+      param.typeFullName shouldBe "foo.bar.Baz"
+    }
+  }
 }

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala

@@ -1,15 +1,14 @@
 package io.joern.javasrc2cpg.testfixtures
 
-import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
 import io.joern.dataflowengineoss.queryengine.EngineContext
+import io.joern.dataflowengineoss.semanticsloader.FlowSemantic
+import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg}
 import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg}
 import io.joern.x2cpg.X2Cpg
-import io.joern.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, TestCpg}
+import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg}
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.PropertyNames
-import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Literal, StoredNode}
+import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Literal}
 import io.shiftleft.semanticcpg.language.*
-import io.shiftleft.semanticcpg.layers.LayerCreatorContext
 
 import java.io.File
 
@@ -23,31 +22,29 @@ trait JavaSrcFrontend extends LanguageFrontend {
   }
 }
 
-class JavaSrcTestCpg(enableTypeRecovery: Boolean = false) extends TestCpg with JavaSrcFrontend {
-  private var _withOssDataflow = false
+class JavaSrcTestCpg(enableTypeRecovery: Boolean = false)
+    extends DefaultTestCpg
+    with JavaSrcFrontend
+    with SemanticTestCpg {
 
-  def withOssDataflow(value: Boolean = true): this.type = {
-    _withOssDataflow = value
-    this
-  }
-
-  override def applyPasses(): Unit = {
-    X2Cpg.applyDefaultOverlays(this)
+  override protected def applyPasses(): Unit = {
+    super.applyPasses()
     if (enableTypeRecovery) JavaSrc2Cpg.typeRecoveryPasses(this).foreach(_.createAndApply())
-    if (_withOssDataflow) {
-      val context = new LayerCreatorContext(this)
-      val options = new OssDataFlowOptions()
-      new OssDataFlow(options).run(context)
-    }
+    applyOssDataFlow()
   }
 
 }
 
-class JavaSrcCode2CpgFixture(withOssDataflow: Boolean = false, enableTypeRecovery: Boolean = false)
-    extends Code2CpgFixture(() => new JavaSrcTestCpg(enableTypeRecovery).withOssDataflow(withOssDataflow)) {
+class JavaSrcCode2CpgFixture(
+  withOssDataflow: Boolean = false,
+  extraFlows: List[FlowSemantic] = List.empty,
+  enableTypeRecovery: Boolean = false
+) extends Code2CpgFixture(() =>
+      new JavaSrcTestCpg(enableTypeRecovery).withOssDataflow(withOssDataflow).withExtraFlows(extraFlows)
+    )
+    with SemanticCpgTestFixture(extraFlows) {
 
-  implicit val resolver: ICallResolver           = NoResolve
-  implicit lazy val engineContext: EngineContext = EngineContext()
+  implicit val resolver: ICallResolver = NoResolve
 
   def getConstSourceSink(
     cpg: Cpg,