[c#] Basic Static Calls & Improved Scoping, Type Map (#4040)

* Have type map able to generate type stubs from JSON input
* Replaced initial simple stack with custom scope object for better scoping
* Generating basic static call nodes with arguments and receivers
* Updated DotNetAstGen

TODO: Load imports and known types into scope for basic type resolution

joern-cli/frontends/csharpsrc2cpg/build.sbt

@@ -37,9 +37,8 @@ lazy val AstgenWinArm   = "dotnetastgen-win-arm.exe"
 lazy val AstgenLinux    = "dotnetastgen-linux"
 lazy val AstgenLinuxArm = "dotnetastgen-linux-arm"
 lazy val AstgenMac      = "dotnetastgen-macos"
-lazy val AstgenMacArm   = "dotnetastgen-macos-arm"
 
-lazy val AllPlatforms = Seq(AstgenWin, AstgenWinArm, AstgenLinux, AstgenLinuxArm, AstgenMac, AstgenMacArm)
+lazy val AllPlatforms = Seq(AstgenWin, AstgenWinArm, AstgenLinux, AstgenLinuxArm, AstgenMac)
 
 lazy val astGenDlUrl = settingKey[String]("astgen download url")
 astGenDlUrl := s"https://github.com/joernio/DotNetAstGen/releases/download/v${astGenVersion.value}/"
@@ -72,11 +71,7 @@ astGenBinaryNames := {
           case Environment.ArchitectureType.ARM => Seq(AstgenLinuxArm)
         }
         Seq(AstgenLinux)
-      case Environment.OperatingSystemType.Mac =>
-        Environment.architecture match {
-          case Environment.ArchitectureType.X86 => Seq(AstgenMac)
-          case Environment.ArchitectureType.ARM => Seq(AstgenMacArm)
-        }
+      case Environment.OperatingSystemType.Mac => Seq(AstgenMac)
       case Environment.OperatingSystemType.Unknown =>
         AllPlatforms
     }
@@ -91,13 +86,9 @@ astGenDlTask := {
   astGenBinaryNames.value.foreach { fileName =>
     val dest = astGenDir / fileName
     if (!dest.exists) {
-      val url            = s"${astGenDlUrl.value}${fileName.stripSuffix(".exe")}.zip"
-      val downloadedFile = files.File(SimpleCache.downloadMaybe(url).toPath)
-      files.File.temporaryDirectory("joern-").apply { unzipTarget =>
-        downloadedFile.unzipTo(unzipTarget)
-        unzipTarget.list.filter(_.name == fileName).foreach(exec => IO.copyFile(exec.toJava, dest))
-      }
-      downloadedFile.delete(swallowIOExceptions = true)
+      val url            = s"${astGenDlUrl.value}$fileName"
+      val downloadedFile = SimpleCache.downloadMaybe(url)
+      IO.copyFile(downloadedFile, dest)
     }
   }
 

joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf

@@ -1,3 +1,3 @@
 csharpsrc2cpg {
-    dotnetastgen_version: "0.12.0"
+    dotnetastgen_version: "0.13.0"
 }

joern-cli/frontends/csharpsrc2cpg/src/main/resources/builtin_types.json

@@ -0,0 +1,30 @@
+{
+  "System": [
+    {
+      "name": "Console",
+      "methods": [
+        {
+          "name": "Write"
+        },
+        {
+          "name": "WriteLine"
+        }
+      ],
+      "fields": []
+    }
+  ],
+  "System.Buffers": [
+    {
+      "name": "ArrayBufferWriter",
+      "methods": [
+        {
+          "name": "Advance"
+        },
+        {
+          "name": "Clear"
+        }
+      ],
+      "fields": []
+    }
+  ]
+}

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/TypeMap.scala

@@ -1,5 +1,7 @@
 package io.joern.csharpsrc2cpg
 
+import com.typesafe.config.impl.*
+import com.typesafe.config.{Config, ConfigFactory}
 import io.joern.csharpsrc2cpg.astcreation.AstCreatorHelper
 import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.{
   ClassDeclaration,
@@ -10,32 +12,67 @@ import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.{
 import io.joern.csharpsrc2cpg.parser.{DotNetJsonAst, DotNetJsonParser, DotNetNodeInfo, ParserKeys}
 import io.joern.x2cpg.astgen.AstGenRunner.AstGenRunnerResult
 import io.joern.x2cpg.datastructures.Stack.Stack
+import io.joern.x2cpg.utils.ConcurrentTaskUtil
 import io.shiftleft.codepropertygraph.generated.nodes.NewNode
+import io.shiftleft.semanticcpg.language.*
+import org.slf4j.LoggerFactory
+import upickle.default.*
 
+import java.io.InputStream
 import java.nio.file.Paths
 import scala.collection.mutable
+import scala.jdk.CollectionConverters.*
+import scala.util.{Failure, Success, Try}
+
+type NamespaceToTypeMap = Map[String, Set[CSharpType]]
+
+/** A mapping of type stubs of known types within the scope of the analysis.
+  *
+  * @param astGenResult
+  *   the parsed application code.
+  * @param initialMappings
+  *   any additional mappings to add to the scope.
+  * @see
+  *   [[io.joern.csharpsrc2cpg.TypeMap.jsonToInitialMapping]] for generating initial mappings.
+  */
+class TypeMap(astGenResult: AstGenRunnerResult, initialMappings: List[NamespaceToTypeMap] = List.empty) {
+
+  private val logger = LoggerFactory.getLogger(getClass)
+
+  private def builtinTypes: NamespaceToTypeMap =
+    jsonToInitialMapping(getClass.getResourceAsStream("/builtin_types.json")) match
+      case Failure(exception) => logger.warn("Unable to parse JSON type entry from builtin types", exception); Map.empty
+      case Success(mapping)   => mapping
+
+  /** Converts a JSON type mapping to a NamespaceToTypeMap entry.
+    * @param jsonInputStream
+    *   a JSON file as an input stream.
+    * @return
+    *   the resulting type map in a Try
+    */
+  def jsonToInitialMapping(jsonInputStream: InputStream): Try[NamespaceToTypeMap] =
+    Try(read[NamespaceToTypeMap](ujson.Readable.fromByteArray(jsonInputStream.readAllBytes())))
 
-class TypeMap(astGenResult: AstGenRunnerResult) {
-
-  private val namespaceToType: Map[String, Set[CSharpType]] = astGenResult.parsedFiles
-    .map { file =>
+  private val namespaceToType: NamespaceToTypeMap = {
+    def typeMapTasks = astGenResult.parsedFiles.map { file =>
       val parserResult    = DotNetJsonParser.readFile(Paths.get(file))
       val compilationUnit = AstCreatorHelper.createDotNetNodeInfo(parserResult.json(ParserKeys.AstRoot))
       () => parseCompilationUnit(compilationUnit)
-    }
-    .map(task => task()) // TODO: To be parallelized with https://github.com/joernio/joern/pull/4009
-    .foldLeft(Map.empty[String, Set[CSharpType]])((a, b) => {
+    }.iterator
+    val typeMaps = ConcurrentTaskUtil.runUsingSpliterator(typeMapTasks).flatMap(_.toOption)
+    (builtinTypes +: typeMaps ++: initialMappings).foldLeft(Map.empty[String, Set[CSharpType]])((a, b) => {
       val accumulator = mutable.HashMap.from(a)
       val allKeys     = accumulator.keySet ++ b.keySet
 
       allKeys.foreach(k =>
         accumulator.updateWith(k) {
-          case Some(existing) => b.get(k).map(x => x ++ existing)
-          case None           => b.get(k)
+          case Some(existing) => Option(a.getOrElse(k, Set.empty) ++ b.getOrElse(k, Set.empty) ++ existing)
+          case None           => Option(a.getOrElse(k, Set.empty) ++ b.getOrElse(k, Set.empty))
         }
       )
       accumulator.toMap
     })
+  }
 
   /** For the given namespace, returns the declared classes.
     */
@@ -86,7 +123,7 @@ class TypeMap(astGenResult: AstGenRunnerResult) {
           case _                 => List.empty
       }
       .toList
-    CSharpType(className, members)
+    CSharpType(className, members.collectAll[CSharpMethod].l, members.collectAll[CSharpField].l)
   }
 
   private def parseMethodDeclaration(methodDecl: DotNetNodeInfo): List[CSharpMethod] = {
@@ -106,12 +143,8 @@ class TypeMap(astGenResult: AstGenRunnerResult) {
 
 }
 
-sealed trait CSharpMember {
-  def name: String
-}
-
-case class CSharpField(name: String) extends CSharpMember
+case class CSharpField(name: String) derives ReadWriter
 
-case class CSharpMethod(name: String) extends CSharpMember
+case class CSharpMethod(name: String) derives ReadWriter
 
-case class CSharpType(name: String, members: List[CSharpMember]) extends CSharpMember
+case class CSharpType(name: String, methods: List[CSharpMethod], fields: List[CSharpField]) derives ReadWriter

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala

@@ -1,18 +1,19 @@
 package io.joern.csharpsrc2cpg.astcreation
 
 import io.joern.csharpsrc2cpg.TypeMap
+import io.joern.csharpsrc2cpg.datastructures.CSharpScope
 import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.*
 import io.joern.csharpsrc2cpg.parser.{DotNetNodeInfo, ParserKeys}
 import io.joern.x2cpg.astgen.{AstGenNodeBuilder, ParserResult}
 import io.joern.x2cpg.datastructures.Scope
-import io.joern.x2cpg.datastructures.Stack.Stack
+import io.joern.x2cpg.datastructures.Stack.{Stack, StackWrapper}
 import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode}
 import io.shiftleft.codepropertygraph.generated.NodeTypes
 import io.shiftleft.codepropertygraph.generated.nodes.{NewFile, NewNode}
 import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 import ujson.Value
-import io.joern.x2cpg.datastructures.Stack.StackWrapper
+
 import java.math.BigInteger
 import java.security.MessageDigest
 
@@ -28,8 +29,7 @@ class AstCreator(val relativeFileName: String, val parserResult: ParserResult, v
 
   protected val logger: Logger = LoggerFactory.getLogger(getClass)
 
-  protected val methodAstParentStack                             = new Stack[NewNode]()
-  protected val scope: Scope[String, (NewNode, String), NewNode] = new Scope()
+  protected val scope: CSharpScope = new CSharpScope()
 
   override def createAst(): DiffGraphBuilder = {
     val hash = String.format(
@@ -71,7 +71,7 @@ class AstCreator(val relativeFileName: String, val parserResult: ParserResult, v
       case IdentifierName            => Seq(astForIdentifier(nodeInfo))
       case LocalDeclarationStatement => astForLocalDeclarationStatement(nodeInfo)
       case GlobalStatement           => astForGlobalStatement(nodeInfo)
-      case _: LiteralExpr            => Seq(astForLiteralExpression(nodeInfo))
+      case _: LiteralExpr            => astForLiteralExpression(nodeInfo)
       case _                         => notHandledYet(nodeInfo)
     }
   }

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala

@@ -4,8 +4,8 @@ import io.joern.csharpsrc2cpg.astcreation
 import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.*
 import io.joern.csharpsrc2cpg.parser.{DotNetJsonAst, DotNetNodeInfo, ParserKeys}
 import io.joern.x2cpg.{Ast, Defines, ValidationMode}
-import io.shiftleft.codepropertygraph.generated.{DispatchTypes, PropertyNames}
 import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewMethod, NewNamespaceBlock, NewTypeDecl}
+import io.shiftleft.codepropertygraph.generated.{DispatchTypes, PropertyNames}
 import ujson.Value
 
 import scala.util.{Failure, Success, Try}
@@ -37,17 +37,15 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
   }
 
   protected def astFullName(node: DotNetNodeInfo): String = {
-    methodAstParentStack.headOption match
-      case Some(head: NewNamespaceBlock) => s"${head.fullName}.${nameFromNode(node)}"
-      case Some(head: NewMethod)         => s"${head.fullName}.${nameFromNode(node)}"
-      case Some(head: NewTypeDecl)       => s"${head.fullName}.${nameFromNode(node)}"
-      case _                             => nameFromNode(node)
+    scope.surroundingScopeFullName match
+      case Some(fullName) => s"$fullName.${nameFromNode(node)}"
+      case _              => nameFromNode(node)
   }
 
   protected def getTypeFullNameFromAstNode(ast: Seq[Ast]): String = {
     ast.headOption
       .flatMap(_.root)
-      .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString)
+      .map(_.properties.getOrElse(PropertyNames.TYPE_FULL_NAME, "ANY").toString)
       .getOrElse("ANY")
   }
 
@@ -64,7 +62,10 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
         BuiltinTypes.Float
       case NumericLiteralExpression if node.code.matches("^\\d+\\.?\\d*[m|M]?$") => // e.g. 2m or 2.1M
         BuiltinTypes.Decimal
-      case StringLiteralExpression if node.code.matches("^\"\\w+\"$") => BuiltinTypes.String
+      case StringLiteralExpression => BuiltinTypes.DotNetTypeMap(BuiltinTypes.String)
+      case IdentifierName          =>
+        // TODO: Look at scope object for possible types
+        "ANY"
       case _ =>
         Try(createDotNetNodeInfo(node.json(ParserKeys.Type))) match
           case Success(typeNode) =>

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala

@@ -1,5 +1,6 @@
 package io.joern.csharpsrc2cpg.astcreation
 
+import io.joern.csharpsrc2cpg.datastructures.{BlockScope, MethodScope, NamespaceScope, TypeScope}
 import io.joern.csharpsrc2cpg.parser.{DotNetNodeInfo, ParserKeys}
 import io.joern.x2cpg.datastructures.Stack.StackWrapper
 import io.joern.x2cpg.utils.NodeBuilders.newModifierNode
@@ -22,10 +23,8 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
       .columnNumber(columnEnd(namespace))
       .filename(relativeFileName)
       .fullName(fullName)
-    methodAstParentStack.push(namespaceBlock)
-    scope.pushNewScope(namespaceBlock)
+    scope.pushNewScope(NamespaceScope(fullName))
     val memberAsts = namespace.json(ParserKeys.Members).arr.flatMap(astForNode).toSeq
-    methodAstParentStack.pop()
     scope.popScope()
     Seq(Ast(namespaceBlock).withChildren(memberAsts))
   }
@@ -34,11 +33,9 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
     val name     = nameFromNode(classDecl)
     val fullName = astFullName(classDecl)
     val typeDecl = typeDeclNode(classDecl, name, fullName, relativeFileName, code(classDecl))
-    methodAstParentStack.push(typeDecl)
-    scope.pushNewScope(typeDecl)
+    scope.pushNewScope(TypeScope(fullName))
     val modifiers = astForModifiers(classDecl)
     val members   = astForMembers(classDecl.json(ParserKeys.Members).arr.map(createDotNetNodeInfo).toSeq)
-    methodAstParentStack.pop()
     scope.popScope()
     val typeDeclAst = Ast(typeDecl)
       .withChildren(modifiers)
@@ -74,7 +71,7 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
     val identifierAst = astForIdentifier(varDecl, typeFullName)
     val _localNode    = localNode(varDecl, name, name, typeFullName)
     val localNodeAst  = Ast(_localNode)
-    scope.addToScope(name, (_localNode, typeFullName))
+    scope.addToScope(name, _localNode)
     val assignmentNode = callNode(
       varDecl,
       code(varDecl),
@@ -112,9 +109,9 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
       methodSignature(methodReturn, params.flatMap(_.nodes.collectFirst { case x: NewMethodParameterIn => x }))
     val fullName    = s"${astFullName(methodDecl)}:$signature"
     val methodNode_ = methodNode(methodDecl, name, code(methodDecl), fullName, Option(signature), relativeFileName)
-    methodAstParentStack.push(methodNode_)
-    scope.pushNewScope(methodNode_)
-    val body      = astForMethodBody(createDotNetNodeInfo(methodDecl.json(ParserKeys.Body)))
+    scope.pushNewScope(MethodScope(fullName))
+    val body = astForMethodBody(createDotNetNodeInfo(methodDecl.json(ParserKeys.Body)))
+    scope.popScope()
     val modifiers = astForModifiers(methodDecl).flatMap(_.nodes).collect { case x: NewModifier => x }
     val thisNode =
       if (!modifiers.exists(_.modifierType == ModifierTypes.STATIC)) astForThisNode(methodDecl)
@@ -137,20 +134,19 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
   }
 
   private def astForThisNode(methodDecl: DotNetNodeInfo): Ast = {
-    val name = "this"
-    val typeFullName =
-      methodAstParentStack.headOption.map(_.properties.getOrElse(PropertyNames.FULL_NAME, "ANY").toString)
-    val param =
-      parameterInNode(methodDecl, name, name, 0, false, EvaluationStrategies.BY_SHARING.name, typeFullName)
+    val name         = "this"
+    val typeFullName = scope.surroundingTypeDeclFullName.getOrElse("ANY")
+    val param = parameterInNode(methodDecl, name, name, 0, false, EvaluationStrategies.BY_SHARING.name, typeFullName)
     Ast(param)
   }
 
   private def astForMethodBody(body: DotNetNodeInfo): Ast = {
-    val block      = blockNode(body)
+    val block = blockNode(body)
+    scope.pushNewScope(BlockScope)
     val statements = body.json(ParserKeys.Statements).arr.flatMap(astForNode).toList
-    methodAstParentStack.pop()
+    val _blockAst  = blockAst(block, statements)
     scope.popScope()
-    blockAst(block, statements)
+    _blockAst
   }
 
   private def nodeToMethodReturn(methodReturn: DotNetNodeInfo): NewMethodReturn = {
@@ -179,18 +175,10 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
     )
     val implicitAccessModifier = accessModifiers match
       // Internal is default for top-level definitions
-      case Nil
-          if methodAstParentStack.isEmpty || !methodAstParentStack
-            .take(2)
-            .map(_.label())
-            .distinct
-            .contains(NodeTypes.METHOD) =>
-        Ast(newModifierNode(ModifierTypes.INTERNAL))
+      case Nil if scope.isTopLevel => Ast(newModifierNode(ModifierTypes.INTERNAL))
       // Private is default for nested definitions
-      case Nil
-          if methodAstParentStack.headOption.exists(x => x.isInstanceOf[NewMethod] || x.isInstanceOf[NewTypeDecl]) =>
-        Ast(newModifierNode(ModifierTypes.PRIVATE))
-      case _ => Ast()
+      case Nil => Ast(newModifierNode(ModifierTypes.PRIVATE))
+      case _   => Ast()
 
     implicitAccessModifier :: explicitModifiers
   }