[gosrc2cpg] - Handling for package level global variable access (#3703)

1. Handled package level global variable access with FieldAccess `CALL`
node with respective handling to set the TypeFullName properly.
2. Made changes to handle the situation where we couldn't identify the
TypeFullName in that situation, we are setting the TypeFullName of the
receiver with respective labels postfixed to it like `<FieldAccess>` or
`<ReturnType>`.
3. Updated respective unit tests along with a few more unit tests.

TODO:
Need to handle Global Variable Declaration.

Fixes #3695

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala

@@ -49,7 +49,7 @@ class AstCreator(val relPathFileName: String, val parserResult: ParserResult, go
   private def astForTranslationUnit(rootNode: ParserNodeInfo): Ast = {
     val namespaceBlock = NewNamespaceBlock()
       .name(fullyQualifiedPackage)
-      .fullName(s"$relPathFileName:${fullyQualifiedPackage}")
+      .fullName(s"$relPathFileName:$fullyQualifiedPackage")
       .filename(relPathFileName)
     methodAstParentStack.push(namespaceBlock)
     val rootAst = Ast(namespaceBlock).withChild(

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala

@@ -1,5 +1,6 @@
 package io.joern.gosrc2cpg.astcreation
 
+import io.joern.gosrc2cpg.datastructures.GoGlobal
 import io.joern.gosrc2cpg.parser.ParserAst.*
 import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo}
 import io.joern.x2cpg
@@ -40,11 +41,10 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode)
     Seq(Ast(newImportNode(s"import $importedAsReplacement$importedEntity", importedEntity, importedAs, basicLit)))
   }
 
-  private def astForValueSpec(valueSpec: ParserNodeInfo): Seq[Ast] = {
+  protected def astForValueSpec(valueSpec: ParserNodeInfo, recordVar: Boolean = false): Seq[Ast] = {
     val typeFullName = Try(valueSpec.json(ParserKeys.Type)) match
       case Success(typeJson) =>
-        val typeInfoNode            = createParserNodeInfo(typeJson)
-        val (typeFullName, _, _, _) = processTypeInfo(typeInfoNode)
+        val (typeFullName, _, _, _) = processTypeInfo(createParserNodeInfo(typeJson))
         Some(typeFullName)
       case _ => None
 
@@ -54,7 +54,7 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode)
           (valueSpec.json(ParserKeys.Names).arr.toList zip valueSpec.json(ParserKeys.Values).arr.toList)
             .map { case (lhs, rhs) => (createParserNodeInfo(lhs), createParserNodeInfo(rhs)) }
             .map { case (lhsParserNode, rhsParserNode) =>
-              astForAssignmentCallNode(lhsParserNode, rhsParserNode, typeFullName, valueSpec.code)
+              astForAssignmentCallNode(lhsParserNode, rhsParserNode, typeFullName, valueSpec.code, recordVar)
             }
             .unzip
         localAsts ++: assCallAsts
@@ -64,7 +64,7 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode)
           .arr
           .flatMap { parserNode =>
             val localParserNode = createParserNodeInfo(parserNode)
-            Seq(astForLocalNode(localParserNode, typeFullName)) ++: astForNode(localParserNode)
+            Seq(astForLocalNode(localParserNode, typeFullName, recordVar)) ++: astForNode(localParserNode)
           }
           .toSeq
 
@@ -74,11 +74,12 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode)
     lhsParserNode: ParserNodeInfo,
     rhsParserNode: ParserNodeInfo,
     typeFullName: Option[String],
-    code: String
+    code: String,
+    recordVar: Boolean = false
   ): (Ast, Ast) = {
     val rhsAst          = astForBooleanLiteral(rhsParserNode)
     val rhsTypeFullName = typeFullName.getOrElse(getTypeFullNameFromAstNode(rhsAst))
-    val localAst        = astForLocalNode(lhsParserNode, Some(rhsTypeFullName))
+    val localAst        = astForLocalNode(lhsParserNode, Some(rhsTypeFullName), recordVar)
     val lhsAst          = astForNode(lhsParserNode)
     val arguments       = lhsAst ++: rhsAst
     val cNode = callNode(
@@ -93,11 +94,19 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode)
     (callAst(cNode, arguments), localAst)
   }
 
-  protected def astForLocalNode(localParserNode: ParserNodeInfo, typeFullName: Option[String]): Ast = {
+  protected def astForLocalNode(
+    localParserNode: ParserNodeInfo,
+    typeFullName: Option[String],
+    recordVar: Boolean = false
+  ): Ast = {
     val name = localParserNode.json(ParserKeys.Name).str
     if name != "_" then {
-      val node = localNode(localParserNode, name, localParserNode.code, typeFullName.getOrElse(Defines.anyTypeName))
-      scope.addToScope(name, (node, typeFullName.getOrElse(Defines.anyTypeName)))
+      val typeFullNameStr = typeFullName.getOrElse(Defines.anyTypeName)
+      val node            = localNode(localParserNode, name, localParserNode.code, typeFullNameStr)
+
+      if recordVar then
+        GoGlobal.recordStructTypeMemberType(s"$fullyQualifiedPackage${Defines.dot}$name", typeFullNameStr)
+      else scope.addToScope(name, (node, typeFullNameStr))
       Ast(node)
     } else {
       Ast()

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala

@@ -138,7 +138,10 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio
     val callMethodFullName = s"$receiverTypeFullName.$methodName"
     val (returnTypeFullNameCache, signatureCache) =
       GoGlobal.methodFullNameReturnTypeMap
-        .getOrDefault(callMethodFullName, (Defines.anyTypeName, s"$callMethodFullName()"))
+        .getOrDefault(
+          callMethodFullName,
+          (s"$receiverTypeFullName.$methodName.${Defines.ReturnType}.${XDefines.Unknown}", s"$callMethodFullName()")
+        )
     (methodName, signatureCache, callMethodFullName, returnTypeFullNameCache, receiverAst)
   }
 }

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala

@@ -42,14 +42,43 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi
       case _ => Seq.empty
   }
 
-  protected def astForFieldAccess(info: ParserNodeInfo): Seq[Ast] = {
-    val identifierAsts       = astForNode(info.json(ParserKeys.X))
+  private def processReceiver(info: ParserNodeInfo): (Seq[Ast], String) = {
+    val xnode           = createParserNodeInfo(info.json(ParserKeys.X))
+    val fieldIdentifier = info.json(ParserKeys.Sel)(ParserKeys.Name).str
+    xnode.node match
+      case Ident =>
+        Try(xnode.json(ParserKeys.Obj)) match
+          case Success(_) =>
+            // The presence of "Obj" field indicates its variable identifier and not an alias
+            receiverAstAndFullName(xnode, fieldIdentifier)
+          case _ =>
+            // Otherwise its an alias to imported namespace using which global variable is getting accessed
+            val alias            = xnode.json(ParserKeys.Name).str
+            val receiverFullName = resolveAliasToFullName(alias, fieldIdentifier)
+            (
+              astForNode(xnode),
+              GoGlobal.structTypeMemberTypeMapping.getOrDefault(
+                receiverFullName,
+                s"$receiverFullName${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}"
+              )
+            )
+      case _ =>
+        // This will take care of chained calls
+        receiverAstAndFullName(xnode, fieldIdentifier)
+  }
+
+  private def receiverAstAndFullName(xnode: ParserNodeInfo, fieldIdentifier: String): (Seq[Ast], String) = {
+    val identifierAsts       = astForNode(xnode)
     val receiverTypeFullName = getTypeFullNameFromAstNode(identifierAsts)
-    val fieldIdentifier      = info.json(ParserKeys.Sel)(ParserKeys.Name).str
     val fieldTypeFullName = GoGlobal.structTypeMemberTypeMapping.getOrDefault(
-      receiverTypeFullName + Defines.dot + fieldIdentifier,
-      XDefines.Unknown
+      s"$receiverTypeFullName${Defines.dot}$fieldIdentifier",
+      s"$receiverTypeFullName${Defines.dot}$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}"
     )
+    (identifierAsts, fieldTypeFullName)
+  }
+  protected def astForFieldAccess(info: ParserNodeInfo): Seq[Ast] = {
+    val (identifierAsts, fieldTypeFullName) = processReceiver(info)
+    val fieldIdentifier                     = info.json(ParserKeys.Sel)(ParserKeys.Name).str
     val fieldIdentifierNode = NewFieldIdentifier()
       .canonicalName(fieldIdentifier)
       .lineNumber(line(info))

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala

@@ -1,6 +1,7 @@
 package io.joern.gosrc2cpg.astcreation
 
 import io.joern.gosrc2cpg.datastructures.GoGlobal
+import io.joern.gosrc2cpg.parser.ParserAst.{GenDecl, ValueSpec}
 import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo}
 import io.joern.x2cpg.Ast
 import ujson.{Arr, Obj, Value}
@@ -11,16 +12,40 @@ trait CacheBuilder { this: AstCreator =>
 
   def buildCache(): Unit = {
     try {
-      findAndProcess(parserResult.json)
       // Declared package name and namespace ending folder token is not matching then cache the alias to namespace mapping
       if (!fullyQualifiedPackage.endsWith(declaredPackageName)) {
         GoGlobal.recordAliasToNamespaceMapping(declaredPackageName, fullyQualifiedPackage)
       }
+      findAndProcess(parserResult.json)
+      processPackageLevelGolbalVaraiblesAndConstants(parserResult.json)
     } catch
       case ex: Exception =>
         logger.warn(s"Error: While processing - ${parserResult.fullPath}", ex)
   }
 
+  private def processPackageLevelGolbalVaraiblesAndConstants(json: Value): Unit = {
+    json(ParserKeys.Decls).arrOpt
+      .getOrElse(List())
+      .map(createParserNodeInfo)
+      .foreach(decl => {
+        decl.node match
+          case GenDecl =>
+            decl
+              .json(ParserKeys.Specs)
+              .arrOpt
+              .getOrElse(List())
+              .map(createParserNodeInfo)
+              .foreach(spec => {
+                spec.node match
+                  case ValueSpec => astForValueSpec(spec, true)
+                  case _         =>
+                  // Only process ValueSpec
+              })
+          case _ =>
+          // Only process GenDecl
+      })
+  }
+
   private def findAndProcess(json: Value): Unit = {
     json match {
       case obj: Obj =>

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/Defines.scala

@@ -10,6 +10,8 @@ object Defines {
   val chan                           = "chan"
   val This: String                   = "this"
   val Bool                           = "bool"
+  val FieldAccess                    = "<FieldAccess>"
+  val ReturnType                     = "<ReturnType>"
 
   val primitiveTypeMap: Map[String, String] =
     // This list is prepared with reference to primitives defined at https://pkg.go.dev/builtin#pkg-types

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala

@@ -22,9 +22,29 @@ object GoGlobal extends Global {
     *
     * In above sample as the package name `fpkg` is different from `lib` this one will be cached in the map
     */
-  val aliasToNameSpaceMapping: ConcurrentHashMap[String, String]               = new ConcurrentHashMap()
+  val aliasToNameSpaceMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap()
+
+  // Mapping method fullname to its return type and signature
   val methodFullNameReturnTypeMap: ConcurrentHashMap[String, (String, String)] = new ConcurrentHashMap()
-  val structTypeMemberTypeMapping: ConcurrentHashMap[String, String]           = new ConcurrentHashMap()
+
+  /** Mapping fully qualified name of the member variable of a struct type to it's type It will also maintain the type
+    * mapping for package level global variables. e.g.
+    *
+    * module namespace = joern.io/sample
+    *
+    * package sample
+    *
+    * type Person struct{ Age int}
+    *
+    * var ( HostURL = "http://api.sample.com" )
+    *
+    * It will map
+    *
+    * `joern.io/sample.Person.Age` - `int`
+    *
+    * `joern.io/sample.HostURL` - `string`
+    */
+  val structTypeMemberTypeMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap()
 
   def recordAliasToNamespaceMapping(alias: String, namespace: String): Unit = {
     aliasToNameSpaceMapping.putIfAbsent(alias, namespace)

joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala

@@ -2,6 +2,7 @@ package io.joern.go2cpg.passes.ast
 
 import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite
 import io.joern.gosrc2cpg.Config
+import io.shiftleft.codepropertygraph.generated.Operators
 import io.shiftleft.semanticcpg.language.*
 
 class DownloadDependencyTest extends GoCodeToCpgSuite {
@@ -18,7 +19,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite {
       "go.mod"
     ).moreCode("""
         |package main
-        |
+        |import "github.com/google/uuid"
         |func main()  {
         |  var uud = uuid.NewString()
         |}
@@ -30,4 +31,37 @@ class DownloadDependencyTest extends GoCodeToCpgSuite {
       x.typeFullName shouldBe "string"
     }
   }
+
+  "unresolved dependency tests" should {
+    val cpg = code(
+      """
+        |module joern.io/sample
+        |go 1.18
+        |require (
+        |	joern.io/sampletwo v1.3.1
+        |)
+        |""".stripMargin,
+      "go.mod"
+    ).moreCode("""
+          |package main
+          |import "joern.io/sampletwo"
+          |func main()  {
+          |  var a = sampletwo.Person{Name:"Pandurang"}
+          |  var b = a.Name
+          |  var c = a.FullName()
+          |  var d = a.Process().FullName()
+          |  var e = a.Process().SomeField
+          |}
+          |""".stripMargin)
+
+    "Be correct for CALL Node typeFullNames" in {
+      val List(a, b, c, d, e, f, g) = cpg.call.nameNot(Operators.assignment).l
+      a.typeFullName shouldBe "joern.io/sampletwo.Person"
+      b.typeFullName shouldBe "joern.io/sampletwo.Person.Name.<FieldAccess>.<unknown>"
+      c.typeFullName shouldBe "joern.io/sampletwo.Person.FullName.<ReturnType>.<unknown>"
+      d.typeFullName shouldBe "joern.io/sampletwo.Person.Process.<ReturnType>.<unknown>.FullName.<ReturnType>.<unknown>"
+      e.typeFullName shouldBe "joern.io/sampletwo.Person.Process.<ReturnType>.<unknown>"
+      f.typeFullName shouldBe "joern.io/sampletwo.Person.Process.<ReturnType>.<unknown>.SomeField.<FieldAccess>.<unknown>"
+    }
+  }
 }