From 5106dd3a3bbc8c85a2401c9e33a0fb845dfcd748 Mon Sep 17 00:00:00 2001 From: Pandurang Patil <5101898+pandurangpatil@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:03:29 +0530 Subject: [PATCH] [Gosrc2cpg] download dependencies and caching improvements (#4352) * Some code refactor and optimisations 1. Removed some unwanted brackets 2. Parallelised downloading of the dependencies and processing them. * Partial changes to add failing unit tests. * Unit tests to cover expected situations to lower the memory footprint * minor changes * Initial download dependency optimisation 1. Record which dependencies are getting used as well as which subpackages are getting used. 2. Only download those dependencies which are directly getting imported or used. * handling for used dependencies 1. Handling for the downloading of dependencies only if those are getting used in the main code. 2. While doing optimisation, came across a bug where if more than one packages with the same name created in the code. Then it was creating package level `TypeDecl` and `NamspaceBlock` only once. Introduced few unit test which covers these use cases as well as made the respective handling for the same. * Fixing one more unit test from first * changes to not cache unwanted imports 1. Made changes to not cache unwanted imports from dependency source code. 2. Made changes to cache only used imports in source code with all the non aliased imports to global cache and aliased ones in the context of file `AstCreator`. 3. Caching only those packages whose package name is different from enclosing folder name inside global cache. * few test corrections as per updated changes * minor updates * Optimisation to cache lamdbda type info Optimisation to cache lamdbda type info * optimisations to store method meta data along with strcut type metata 1. Changed the storage structure to minimize the amount of data being stored for method meta data cache and struct type members type information. 2. Made respective changes to fix all the breaking unit tests. * not caching namespaces having starting letter in small case * Fix for issue related to package TypeDecl 1. While making the optimisations, while processing imports if the main source code package is being imported and processed. In some cases TypeDecl for package level global variables wasn't getting created. 2. Identified the issue and made a fix for the same. * Ignoring few unit tests which needs to be updated with improvements. * review comment fixes --- .../scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala | 49 +-- .../astcreation/AstCreatorHelper.scala | 6 +- .../AstForGenDeclarationCreator.scala | 21 +- .../astcreation/AstForLambdaCreator.scala | 26 +- .../AstForMethodCallExpressionCreator.scala | 62 ++-- .../astcreation/AstForPrimitivesCreator.scala | 4 +- .../astcreation/AstForTypeDeclCreator.scala | 34 ++- .../gosrc2cpg/astcreation/CacheBuilder.scala | 109 +++++-- .../gosrc2cpg/datastructures/GoGlobal.scala | 175 ++++++++--- .../io/joern/gosrc2cpg/model/GoMod.scala | 25 +- .../passes/DownloadDependenciesPass.scala | 49 +-- .../MethodAndTypeCacheBuilderPass.scala | 4 +- .../passes/ast/DownloadDependencyTest.scala | 286 +++++++++++++++++- .../ast/GlobalVariableAndConstantTests.scala | 84 +++++ .../go2cpg/passes/ast/TypeFullNameTests.scala | 4 +- .../testfixtures/GoCodeToCpgSuite.scala | 38 ++- 16 files changed, 760 insertions(+), 216 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala index cc5c0a26044d..ad8b6d9da1ae 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala @@ -22,31 +22,42 @@ import io.shiftleft.codepropertygraph.generated.Languages import java.nio.file.Paths import scala.util.Try -class GoSrc2Cpg extends X2CpgFrontend[Config] { +class GoSrc2Cpg(goGlobalOption: Option[GoGlobal] = Option(GoGlobal())) extends X2CpgFrontend[Config] { private val report: Report = new Report() + private var goMod: Option[GoModHelper] = None def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => File.usingTemporaryDirectory("gosrc2cpgOut") { tmpDir => - val goGlobal = GoGlobal() - new MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() - val astGenResult = new AstGenRunner(config).execute(tmpDir).asInstanceOf[GoAstGenRunnerResult] - val goMod = new GoModHelper( - Some(config), - astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) - ) - if (config.fetchDependencies) { - goGlobal.processingDependencies = true - new DownloadDependenciesPass(goMod, goGlobal, config).process() - goGlobal.processingDependencies = false - } - val astCreators = - new MethodAndTypeCacheBuilderPass(Some(cpg), astGenResult.parsedFiles, config, goMod, goGlobal).process() - new AstCreationPass(cpg, astCreators, report).createAndApply() - if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then - new PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() - report.print() + goGlobalOption + .orElse(Option(GoGlobal())) + .foreach(goGlobal => { + new MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() + val astGenResult = new AstGenRunner(config).execute(tmpDir).asInstanceOf[GoAstGenRunnerResult] + goMod = Some( + new GoModHelper( + Some(config), + astGenResult.parsedModFile + .flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) + ) + ) + goGlobal.mainModule = goMod.flatMap(modHelper => modHelper.getModMetaData().map(mod => mod.module.name)) + val astCreators = + new MethodAndTypeCacheBuilderPass(Some(cpg), astGenResult.parsedFiles, config, goMod.get, goGlobal) + .process() + if (config.fetchDependencies) { + goGlobal.processingDependencies = true + new DownloadDependenciesPass(goMod.get, goGlobal, config).process() + goGlobal.processingDependencies = false + } + new AstCreationPass(cpg, astCreators, report).createAndApply() + if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then + new PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() + report.print() + }) } } } + + def getGoModHelper: GoModHelper = goMod.get } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala index 882285ba0366..84fcb66b50ec 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala @@ -131,8 +131,8 @@ trait AstCreatorHelper { this: AstCreator => .toMap } - protected def resolveAliasToFullName(alias: String, typeOrMethodName: String): String = { - s"${aliasToNameSpaceMapping.getOrElse(alias, goGlobal.aliasToNameSpaceMapping.getOrDefault(alias, s"${XDefines.Unknown}.<$alias>"))}.$typeOrMethodName" + protected def resolveAliasToFullName(alias: String): String = { + s"${aliasToNameSpaceMapping.getOrElse(alias, goGlobal.aliasToNameSpaceMapping.getOrDefault(alias, s"${XDefines.Unknown}.<$alias>"))}" } protected def generateTypeFullName( typeName: Option[String] = None, @@ -156,7 +156,7 @@ trait AstCreatorHelper { this: AstCreator => Defines.primitiveTypeMap.getOrElse(typname, s"$fullyQualifiedPackage.$typname") } case Some(alias) => - resolveAliasToFullName(alias, typname) + s"${resolveAliasToFullName(alias)}.$typname" } private def internalTypeFullName( diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala index e5b4ce8d31cb..50bc5d30144d 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala @@ -65,11 +65,12 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) val localParserNode = createParserNodeInfo(parserNode) if globalStatements then { val variableName = localParserNode.json(ParserKeys.Name).str - if (checkForDependencyFlags(variableName)) { + if (goGlobal.checkForDependencyFlags(variableName)) { // While processing the dependencies code ignoring package level global variables starting with lower case letter // as these variables are only accessible within package. So those will not be referred from main source code. - goGlobal.recordStructTypeMemberType( - s"$fullyQualifiedPackage${Defines.dot}$variableName", + goGlobal.recordStructTypeMemberTypeInfo( + fullyQualifiedPackage, + variableName, typeFullName.getOrElse(Defines.anyTypeName) ) astForGlobalVarAndConstants(typeFullName.getOrElse(Defines.anyTypeName), localParserNode) @@ -94,8 +95,8 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) val rhsTypeFullName = typeFullName.getOrElse(getTypeFullNameFromAstNode(rhsAst)) if (globalStatements) { val variableName = lhsParserNode.json(ParserKeys.Name).str - if (checkForDependencyFlags(variableName)) { - goGlobal.recordStructTypeMemberType(s"$fullyQualifiedPackage${Defines.dot}$variableName", rhsTypeFullName) + if (goGlobal.checkForDependencyFlags(variableName)) { + goGlobal.recordStructTypeMemberTypeInfo(fullyQualifiedPackage, variableName, rhsTypeFullName) astForGlobalVarAndConstants(rhsTypeFullName, lhsParserNode, Some(rhsAst)) } (Ast(), Ast()) @@ -160,14 +161,4 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) Ast() } } - - /** While processing the dependencies code ignoring package level global variables, constants, types, and functions - * starting with lower case letter as those are only accessible within package. So those will not be referred from - * main source code. - * @param name - * @return - */ - protected def checkForDependencyFlags(name: String): Boolean = { - !goGlobal.processingDependencies || goGlobal.processingDependencies && name.headOption.exists(_.isUpper) - } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala index e640444d1f52..41d80224c68c 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.{LambdaTypeInfo, MethodCacheMetaData} import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.datastructures.Stack.StackWrapper import io.joern.x2cpg.utils.NodeBuilders.newModifierNode @@ -8,6 +9,8 @@ import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodRetur import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} import ujson.Value +import scala.jdk.CollectionConverters.* + trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => def astForFuncLiteral(funcLiteral: ParserNodeInfo): Seq[Ast] = { @@ -17,9 +20,8 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: .collectFirst({ case m: NewMethod if !m.fullName.endsWith(parserResult.filename) => m.fullName }) .getOrElse(fullyQualifiedPackage) val fullName = s"$baseFullName.$lambdaName" - val (signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) = generateLambdaSignature( - createParserNodeInfo(funcLiteral.json(ParserKeys.Type)) - ) + val LambdaFunctionMetaData(signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) = + generateLambdaSignature(createParserNodeInfo(funcLiteral.json(ParserKeys.Type))) val methodNode_ = methodNode(funcLiteral, lambdaName, funcLiteral.code, fullName, Some(signature), relPathFileName) methodAstParentStack.push(methodNode_) scope.pushNewScope(methodNode_) @@ -40,7 +42,7 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: typeDeclNode_.astParentType(NodeTypes.TYPE_DECL).astParentFullName(fullyQualifiedPackage) else typeDeclNode_.astParentType(NodeTypes.METHOD).astParentFullName(baseFullName) val structTypes = Option(goGlobal.lambdaSignatureToLambdaTypeMap.get(signature)) match { - case Some(types) => types.map(_._1) + case Some(types) => types.asScala.map(_.lambdaStructTypeFullName) case None => Seq.empty } typeDeclNode_.inheritsFromTypeFullName(structTypes) @@ -50,13 +52,11 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: methodNode_.astParentFullName(fullName) Ast.storeInDiffGraph(astForMethod, diffGraph) } - goGlobal.recordFullNameToReturnType(fullName, returnTypeStr, signature) + goGlobal.recordMethodMetadata(baseFullName, lambdaName, MethodCacheMetaData(returnTypeStr, signature)) Seq(Ast(methodRefNode(funcLiteral, funcLiteral.code, fullName, fullName))) } - protected def generateLambdaSignature( - funcType: ParserNodeInfo - ): (String, String, NewMethodReturn, Value, Map[String, List[String]]) = { + protected def generateLambdaSignature(funcType: ParserNodeInfo): LambdaFunctionMetaData = { val genericTypeMethodMap: Map[String, List[String]] = Map() // TODO: While handling the tuple return type we need to handle it here as well. val (returnTypeStr, returnTypeInfo) = @@ -68,6 +68,14 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: val paramSignature = parameterSignature(params, genericTypeMethodMap) val signature = s"${XDefines.ClosurePrefix}($paramSignature)$returnTypeStr" - (signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) + LambdaFunctionMetaData(signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) } } + +case class LambdaFunctionMetaData( + signature: String, + returnTypeStr: String, + methodReturn: NewMethodReturn, + params: Value, + genericTypeMethodMap: Map[String, List[String]] +) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala index 8b7fe7bdaeb1..1882d37cc897 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.MethodCacheMetaData import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} @@ -94,26 +95,22 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio // Then we are assuming that the given function is defined inside same package as that of current file's package. // This assumption will be invalid when another package is imported with alias "." val methodFullName = s"$fullyQualifiedPackage.$methodName" - val (returnTypeFullNameCache, signatureCache) = - goGlobal.methodFullNameReturnTypeMap - .getOrDefault(methodFullName, (Defines.anyTypeName, s"$methodFullName()")) + val methodInfo = goGlobal + .getMethodMetadata(fullyQualifiedPackage, methodName) + .getOrElse(MethodCacheMetaData(Defines.anyTypeName, s"$methodFullName()")) val (signature, fullName, returnTypeFullName) = - Defines.builtinFunctions.getOrElse(methodName, (signatureCache, methodFullName, returnTypeFullNameCache)) + Defines.builtinFunctions.getOrElse(methodName, (methodInfo.signature, methodFullName, methodInfo.returnType)) val probableLambdaTypeFullName = scope.lookupVariable(methodName) match case Some((_, lambdaTypeFullName)) => Some(lambdaTypeFullName) case _ => - Option(goGlobal.structTypeMemberTypeMapping.get(methodFullName)) match - case Some(globalLambdaTypeFullName) => Some(globalLambdaTypeFullName) - case _ => None + goGlobal.getStructTypeMemberType(fullyQualifiedPackage, methodName) val (postLambdaFullname, postLambdaSignature, postLambdaReturnTypeFullName) = probableLambdaTypeFullName match case Some(lambdaTypeFullName) => - Option( - goGlobal.methodFullNameReturnTypeMap - .get(lambdaTypeFullName) - ) match - case Some((lambdaReturnTypeFullNameCache, lambdaSignatureCache)) => - (lambdaTypeFullName, lambdaSignatureCache, lambdaReturnTypeFullNameCache) - case _ => (fullName, signature, returnTypeFullName) + val (nameSpaceName, lambdaName) = goGlobal.splitNamespaceFromMember(lambdaTypeFullName) + goGlobal.getMethodMetadata(nameSpaceName, lambdaName) match { + case Some(metaData) => (lambdaTypeFullName, metaData.signature, metaData.returnType) + case _ => (fullName, signature, returnTypeFullName) + } case _ => (fullName, signature, returnTypeFullName) (methodName, postLambdaSignature, postLambdaFullname, postLambdaReturnTypeFullName, Seq.empty) @@ -126,18 +123,21 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio processReceiverAst(methodName, xnode) case _ => // Otherwise its an alias to imported namespace on which method call is made - val alias = xnode.json(ParserKeys.Name).str - val callMethodFullName = - resolveAliasToFullName(alias, methodName) + val alias = xnode.json(ParserKeys.Name).str + val fullNamespace = resolveAliasToFullName(alias) + val callMethodFullName = s"$fullNamespace.$methodName" val lambdaFullName = - goGlobal.structTypeMemberTypeMapping.getOrDefault(callMethodFullName, callMethodFullName) - val (returnTypeFullNameCache, signatureCache) = Option( - goGlobal.methodFullNameReturnTypeMap - .get(lambdaFullName) - ) match - case Some((returnTypeFullName, signature)) => (returnTypeFullName, signature) - case _ => (s"$callMethodFullName.${Defines.ReturnType}.${XDefines.Unknown}", s"$callMethodFullName()") - + goGlobal.getStructTypeMemberType(fullNamespace, methodName).getOrElse(callMethodFullName) + val (nameSpace, memberName) = goGlobal.splitNamespaceFromMember(lambdaFullName) + val MethodCacheMetaData(returnTypeFullNameCache, signatureCache) = + goGlobal + .getMethodMetadata(nameSpace, memberName) + .getOrElse( + MethodCacheMetaData( + s"$callMethodFullName.${Defines.ReturnType}.${XDefines.Unknown}", + s"$callMethodFullName()" + ) + ) (methodName, signatureCache, lambdaFullName, returnTypeFullNameCache, Seq.empty) case _ => // This will take care of chained method calls. It will call `astForCallExpression` in recursive way, @@ -157,12 +157,14 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio .getOrElse(Defines.anyTypeName) .stripPrefix("*") val callMethodFullName = s"$receiverTypeFullName.$methodName" - val (returnTypeFullNameCache, signatureCache) = - goGlobal.methodFullNameReturnTypeMap - .getOrDefault( - callMethodFullName, - (s"$receiverTypeFullName.$methodName.${Defines.ReturnType}.${XDefines.Unknown}", s"$callMethodFullName()") + val MethodCacheMetaData(returnTypeFullNameCache, signatureCache) = goGlobal + .getMethodMetadata(receiverTypeFullName, methodName) + .getOrElse( + MethodCacheMetaData( + s"$receiverTypeFullName.$methodName.${Defines.ReturnType}.${XDefines.Unknown}", + s"$callMethodFullName()" ) + ) (methodName, signatureCache, callMethodFullName, returnTypeFullNameCache, receiverAst) } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala index d7c2bc356d4d..58f1e3097fdc 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala @@ -1,6 +1,5 @@ package io.joern.gosrc2cpg.astcreation -import io.joern.gosrc2cpg.datastructures.GoGlobal import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.utils.NodeBuilders.newOperatorCallNode @@ -72,11 +71,12 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t Ast(node).withRefEdge(node, variable) case _ => // If its not local node then check if its global member variable of package TypeDecl - Option(goGlobal.structTypeMemberTypeMapping.get(s"$fullyQualifiedPackage${Defines.dot}$identifierName")) match + goGlobal.getStructTypeMemberType(fullyQualifiedPackage, identifierName) match { case Some(fieldTypeFullName) => astForPackageGlobalFieldAccess(fieldTypeFullName, identifierName, ident) case _ => // TODO: something is wrong here. Refer to SwitchTests -> "be correct for switch case 4" Ast(identifierNode(ident, identifierName, ident.json(ParserKeys.Name).str, Defines.anyTypeName)) + } } } else { Ast() diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala index 71c88029ecd2..f727c976ab75 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala @@ -1,5 +1,5 @@ package io.joern.gosrc2cpg.astcreation -import io.joern.gosrc2cpg.datastructures.GoGlobal +import io.joern.gosrc2cpg.datastructures.LambdaTypeInfo import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg @@ -13,7 +13,7 @@ import scala.util.{Success, Try} trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def astForTypeSpec(typeSpecNode: ParserNodeInfo): Seq[Ast] = { - val (name, fullName, memberAsts) = processTypeSepc(typeSpecNode.json) + val (name, fullName, memberAsts) = processTypeSepc(createParserNodeInfo(typeSpecNode.json)) val typeDeclNode_ = typeDeclNode(typeSpecNode, name, fullName, relPathFileName, typeSpecNode.code) val modifier = addModifier(typeDeclNode_, name) @@ -21,8 +21,8 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi } protected def processFuncType(typeNode: ParserNodeInfo, typeDeclFullName: String): Seq[Ast] = { - val (signature, returnTypeFullName, _, _, _) = generateLambdaSignature(typeNode) - goGlobal.recordLambdaSigntureToLambdaType(signature, typeDeclFullName, returnTypeFullName) + val LambdaFunctionMetaData(signature, returnTypeFullName, _, _, _) = generateLambdaSignature(typeNode) + goGlobal.recordLambdaSigntureToLambdaType(signature, LambdaTypeInfo(typeDeclFullName, returnTypeFullName)) Seq.empty } @@ -39,7 +39,9 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi .map(fieldInfo => { val fieldNodeInfo = createParserNodeInfo(fieldInfo) val fieldName = fieldNodeInfo.json(ParserKeys.Name).str - goGlobal.recordStructTypeMemberType(typeDeclFullName + Defines.dot + fieldName, typeFullName) + if (goGlobal.checkForDependencyFlags(fieldName)) { + goGlobal.recordStructTypeMemberTypeInfo(typeDeclFullName, fieldName, typeFullName) + } Ast(memberNode(typeInfo, fieldName, fieldNodeInfo.code, typeFullName)) }) }) @@ -58,14 +60,15 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi receiverAstAndFullName(xnode, fieldIdentifier) case _ => // Otherwise its an alias to imported namespace using which global variable is getting accessed - val alias = xnode.json(ParserKeys.Name).str - val receiverFullName = resolveAliasToFullName(alias, fieldIdentifier) + val alias = xnode.json(ParserKeys.Name).str + val nameSpace = resolveAliasToFullName(alias) ( astForNode(xnode), - goGlobal.structTypeMemberTypeMapping.getOrDefault( - receiverFullName, - s"$receiverFullName${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" - ) + goGlobal + .getStructTypeMemberType(nameSpace, fieldIdentifier) + .getOrElse( + s"$nameSpace.$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" + ) ) case _ => // This will take care of chained calls @@ -75,10 +78,11 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi private def receiverAstAndFullName(xnode: ParserNodeInfo, fieldIdentifier: String): (Seq[Ast], String) = { val identifierAsts = astForNode(xnode) val receiverTypeFullName = getTypeFullNameFromAstNode(identifierAsts) - val fieldTypeFullName = goGlobal.structTypeMemberTypeMapping.getOrDefault( - s"$receiverTypeFullName${Defines.dot}$fieldIdentifier", - s"$receiverTypeFullName${Defines.dot}$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" - ) + val fieldTypeFullName = goGlobal + .getStructTypeMemberType(receiverTypeFullName, fieldIdentifier) + .getOrElse( + s"$receiverTypeFullName${Defines.dot}$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" + ) (identifierAsts, fieldTypeFullName) } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala index 88369e45444a..b1cf71451e02 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.MethodCacheMetaData import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern @@ -17,19 +18,21 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre def buildCache(cpgOpt: Option[Cpg]): DiffGraphBuilder = { val diffGraph = new DiffGraphBuilder try { - - cpgOpt.map { _ => - // We don't want to process this part when third party dependencies are being processed. - val result = goGlobal.recordAliasToNamespaceMapping(declaredPackageName, fullyQualifiedPackage) - if (result == null) { - // if result is null that means item got added first time otherwise it has been already added to global map - val rootNode = createParserNodeInfo(parserResult.json) - val ast = astForPackage(rootNode) - Ast.storeInDiffGraph(ast, diffGraph) + if (checkIfGivenDependencyPackageCanBeProcessed()) { + cpgOpt.map { _ => + // We don't want to process this part when third party dependencies are being processed. + if (goGlobal.recordForThisNamespace(fullyQualifiedPackage)) { + // java.util.Set.Add method will return true when set already doesn't contain the same value. + val rootNode = createParserNodeInfo(parserResult.json) + val ast = astForPackage(rootNode) + Ast.storeInDiffGraph(ast, diffGraph) + } } + identifyAndRecordPackagesWithDifferentName() + findAndProcess(parserResult.json) + // NOTE: For dependencies we are just caching the global variables Types. + processPackageLevelGolbalVaraiblesAndConstants(parserResult.json) } - findAndProcess(parserResult.json) - processPackageLevelGolbalVaraiblesAndConstants(parserResult.json) } catch { case ex: Exception => logger.warn(s"Error: While processing - ${parserResult.fullPath}", ex) @@ -37,6 +40,16 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre diffGraph } + private def checkIfGivenDependencyPackageCanBeProcessed(): Boolean = + !goGlobal.processingDependencies || goGlobal.processingDependencies && goGlobal.aliasToNameSpaceMapping + .containsValue(fullyQualifiedPackage) + + private def identifyAndRecordPackagesWithDifferentName(): Unit = { + // record the package to full namespace mapping only when declared package name is not matching with containing folder name + if (declaredPackageName != fullyQualifiedPackage.split("/").last) + goGlobal.recordAliasToNamespaceMapping(declaredPackageName, fullyQualifiedPackage) + } + private def astForPackage(rootNode: ParserNodeInfo): Ast = { val pathTokens = relPathFileName.split(fileSeparateorPattern) val packageFolderPath = if (pathTokens.nonEmpty && pathTokens.size > 1) { @@ -84,17 +97,17 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.ImportSpec" && !json.obj.contains( ParserKeys.NodeReferenceId - ) + ) && !goGlobal.processingDependencies ) { - processImports(obj) + // NOTE: Dependency code is not being processed here. + processImports(obj, true) } else if ( json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.TypeSpec" && !json.obj.contains( ParserKeys.NodeReferenceId ) ) { - createParserNodeInfo(obj) - processTypeSepc(obj) + processTypeSepc(createParserNodeInfo(obj)) } else if ( json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.FuncDecl" && !json.obj.contains( @@ -107,10 +120,20 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.ValueSpec" && !json.obj.contains( ParserKeys.NodeReferenceId - ) + ) && !goGlobal.processingDependencies ) { + // NOTE: Dependency code is not being processed here. createParserNodeInfo(obj) + } else if ( + json.obj + .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.FuncLit" && !json.obj.contains( + ParserKeys.NodeReferenceId + ) && !goGlobal.processingDependencies + ) { + // NOTE: Dependency code is not being processed here. + processFuncLiteral(obj) } + obj.value.values.foreach(subJson => findAndProcess(subJson)) case arr: Arr => arr.value.foreach(subJson => findAndProcess(subJson)) @@ -118,14 +141,21 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre } } - protected def processTypeSepc(typeSepc: Value): (String, String, Seq[Ast]) = { - val name = typeSepc(ParserKeys.Name)(ParserKeys.Name).str - if (checkForDependencyFlags(name)) { + private def processFuncLiteral(funcLit: Value): Unit = { + val LambdaFunctionMetaData(signature, _, _, _, _) = generateLambdaSignature( + createParserNodeInfo(funcLit(ParserKeys.Type)) + ) + goGlobal.recordForThisLamdbdaSignature(signature) + } + + protected def processTypeSepc(typeSepc: ParserNodeInfo): (String, String, Seq[Ast]) = { + val name = typeSepc.json(ParserKeys.Name)(ParserKeys.Name).str + if (goGlobal.checkForDependencyFlags(name)) { // Ignoring recording the Type details when we are processing dependencies code with Type name starting with lower case letter // As the Types starting with lower case letters will only be accessible within that package. Which means // these Types are not going to get referred from main source code. val fullName = fullyQualifiedPackage + Defines.dot + name - val typeNode = createParserNodeInfo(typeSepc(ParserKeys.Type)) + val typeNode = createParserNodeInfo(typeSepc.json(ParserKeys.Type)) val ast = typeNode.node match { // As of don't see any use case where InterfaceType needs to be handled. case InterfaceType => Seq.empty @@ -140,37 +170,50 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre ("", "", Seq.empty) } - protected def processImports(importDecl: Value): (String, String) = { + protected def processImports(importDecl: Value, recordFindings: Boolean = false): (String, String) = { val importedEntity = importDecl(ParserKeys.Path).obj(ParserKeys.Value).str.replaceAll("\"", "") - val importedAs = + if (recordFindings) { + goMod.recordUsedDependencies(importedEntity) + } + val importedAsOption = Try(importDecl(ParserKeys.Name).obj(ParserKeys.Name).str).toOption - .getOrElse(importedEntity.split("/").last) - - aliasToNameSpaceMapping.put(importedAs, importedEntity) - (importedEntity, importedAs) + importedAsOption match { + case Some(importedAs) => + // As these alias could be different for each file. Hence we maintain the cache at file level. + if (recordFindings) + aliasToNameSpaceMapping.put(importedAs, importedEntity) + (importedEntity, importedAs) + case _ => + val derivedImportedAs = importedEntity.split("/").last + if (recordFindings) + goGlobal.recordAliasToNamespaceMapping(derivedImportedAs, importedEntity) + (importedEntity, derivedImportedAs) + } } protected def processFuncDecl(funcDeclVal: Value): MethodMetadata = { val name = funcDeclVal(ParserKeys.Name).obj(ParserKeys.Name).str - if (checkForDependencyFlags(name)) { + if (goGlobal.checkForDependencyFlags(name)) { // Ignoring recording the method details when we are processing dependencies code with functions name starting with lower case letter // As the functions starting with lower case letters will only be accessible within that package. Which means // these methods / functions are not going to get referred from main source code. val receiverInfo = getReceiverInfo(Try(funcDeclVal(ParserKeys.Recv))) - val methodFullname = receiverInfo match + val (methodFullname, recordNamespace) = receiverInfo match case Some(_, typeFullName, _, _) => - s"$typeFullName.$name" + (s"$typeFullName.$name", typeFullName) case _ => - s"$fullyQualifiedPackage.$name" + (s"$fullyQualifiedPackage.$name", fullyQualifiedPackage) // TODO: handle multiple return type or tuple (int, int) val genericTypeMethodMap = processTypeParams(funcDeclVal(ParserKeys.Type)) val (returnTypeStr, _) = getReturnType(funcDeclVal(ParserKeys.Type), genericTypeMethodMap).headOption - .getOrElse(("", null)) + .getOrElse((Defines.voidTypeName, null)) val params = funcDeclVal(ParserKeys.Type)(ParserKeys.Params)(ParserKeys.List) val signature = - s"$methodFullname(${parameterSignature(params, genericTypeMethodMap)})$returnTypeStr" - goGlobal.recordFullNameToReturnType(methodFullname, returnTypeStr, signature) + s"$methodFullname(${parameterSignature(params, genericTypeMethodMap)})${ + if returnTypeStr == Defines.voidTypeName then "" else returnTypeStr + }" + goGlobal.recordMethodMetadata(recordNamespace, name, MethodCacheMetaData(returnTypeStr, signature)) MethodMetadata(name, methodFullname, signature, params, receiverInfo, genericTypeMethodMap) } else MethodMetadata() diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala index c3b0501afb98..9a42a1bbcb65 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala @@ -1,12 +1,13 @@ package io.joern.gosrc2cpg.datastructures import io.joern.x2cpg.Ast +import org.slf4j.LoggerFactory -import java.util.concurrent.ConcurrentHashMap - +import java.util.concurrent.{ConcurrentHashMap, ConcurrentSkipListSet} class GoGlobal { - - var processingDependencies = false + private val logger = LoggerFactory.getLogger(getClass) + var mainModule: Option[String] = None + var processingDependencies = false /** This map will only contain the mapping for those packages whose package name is different from the enclosing * folder name @@ -23,13 +24,124 @@ class GoGlobal { */ val aliasToNameSpaceMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap() - val lambdaSignatureToLambdaTypeMap: ConcurrentHashMap[String, Set[(String, String)]] = new ConcurrentHashMap() + /** This map will record the Type FullName of Struct Type defined for Lambda Expression along with return type + * fullname against the lambda signature. + * + * This will help map the Lambda TypeFullName with the respective Struct Type as supper Type + */ + val lambdaSignatureToLambdaTypeMap: ConcurrentHashMap[String, java.util.Set[LambdaTypeInfo]] = new ConcurrentHashMap() val pkgLevelVarAndConstantAstMap: ConcurrentHashMap[String, Set[(Ast, String)]] = new ConcurrentHashMap() - // Mapping method fullname to its return type and signature - val methodFullNameReturnTypeMap: ConcurrentHashMap[String, (String, String)] = new ConcurrentHashMap() + val nameSpaceMetaDataMap: ConcurrentHashMap[String, NameSpaceMetaData] = new ConcurrentHashMap() + + def recordAliasToNamespaceMapping(alias: String, namespace: String): Unit = synchronized { + val existingVal = aliasToNameSpaceMapping.putIfAbsent(alias, namespace) + // NOTE: !namespace.startsWith(mainModule.get) this check will not add the mapping for main source code imports. + // This will make sure to add the entry in CacheBuilder, which in turn creates the required Package level TypeDecl AST structure as well. + if (existingVal == null && (mainModule == None || (mainModule != None && !namespace.startsWith(mainModule.get)))) { + recordForThisNamespace(namespace) + } else if (existingVal != namespace) { + // TODO: This might need better way of recording the information. + logger.warn(s"more than one namespaces are found for given alias `$alias` -> `$existingVal` and `$namespace`") + } + } + + def recordForThisNamespace(namespace: String): Boolean = { + val existing = nameSpaceMetaDataMap.putIfAbsent(namespace, NameSpaceMetaData()) + existing == null + } + + def getMethodMetadata(namespace: String, methodName: String): Option[MethodCacheMetaData] = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + Option(existingNamespace.methodMetaMap.get(methodName)) + case _ => + None + } + } + + def recordMethodMetadata(namespace: String, methodName: String, methodMetaData: MethodCacheMetaData): Unit = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + existingNamespace.methodMetaMap.put(methodName, methodMetaData) + case _ => + // handling for types and lambda functions defined inside methods. Wrapping method becomes the part of their namespace. + val (wrappingNamespace, membertoken) = splitNamespaceFromMember(namespace) + // now check if this namespace is present in the map. If yes then make the new entry for this sub namespace + if (nameSpaceMetaDataMap.containsKey(wrappingNamespace) && checkForDependencyFlags(membertoken)) { + recordForThisNamespace(namespace) + recordMethodMetadata(namespace, methodName, methodMetaData) + } + } + } + + def getStructTypeMemberType(namespace: String, memberName: String): Option[String] = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + Option(existingNamespace.structTypeMembers.get(memberName)) + case _ => + None + } + } + def recordStructTypeMemberTypeInfo(namespace: String, memberName: String, memberType: String): Unit = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + existingNamespace.structTypeMembers.put(memberName, memberType) + case _ => + val (wrappingNamespace, membertoken) = splitNamespaceFromMember(namespace) + if (nameSpaceMetaDataMap.containsKey(wrappingNamespace) && checkForDependencyFlags(membertoken)) { + recordForThisNamespace(namespace) + recordStructTypeMemberTypeInfo(namespace, memberName, memberType) + } + } + } + + def recordPkgLevelVarAndConstantAst(pkg: String, ast: Ast, filePath: String): Unit = synchronized { + Option(pkgLevelVarAndConstantAstMap.get(pkg)) match { + case Some(existingList) => + val t = (ast, filePath) + pkgLevelVarAndConstantAstMap.put(pkg, existingList + t) + case None => pkgLevelVarAndConstantAstMap.put(pkg, Set((ast, filePath))) + } + } + def recordForThisLamdbdaSignature(signature: String): Unit = { + lambdaSignatureToLambdaTypeMap.putIfAbsent(signature, new ConcurrentSkipListSet()) + } + + def recordLambdaSigntureToLambdaType(signature: String, lambdaTypeInfo: LambdaTypeInfo): Unit = { + Option(lambdaSignatureToLambdaTypeMap.get(signature)) match { + case Some(existingList) => + existingList.add(lambdaTypeInfo) + case _ => + } + } + + def splitNamespaceFromMember(fullName: String): (String, String) = { + if (fullName.contains('.')) { + val lastDotIndex = fullName.lastIndexOf('.') + val nameSpaceName = fullName.substring(0, lastDotIndex) + val memberName = fullName.substring(lastDotIndex + 1) + (nameSpaceName, memberName) + } else { + (fullName, "") + } + } + + /** While processing the dependencies code ignoring package level global variables, constants, types, and functions + * starting with lower case letter as those are only accessible within package. So those will not be referred from + * main source code. + * + * @param name + * @return + */ + def checkForDependencyFlags(name: String): Boolean = { + !processingDependencies || processingDependencies && name.headOption.exists(_.isUpper) + } +} + +case class NameSpaceMetaData( /** Mapping fully qualified name of the member variable of a struct type to it's type It will also maintain the type * mapping for package level global variables. e.g. * @@ -47,44 +159,21 @@ class GoGlobal { * * `joern.io/sample.HostURL` - `string` */ - val structTypeMemberTypeMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap() - - def recordAliasToNamespaceMapping(alias: String, namespace: String): String = { - aliasToNameSpaceMapping.putIfAbsent(alias, namespace) - } + structTypeMembers: ConcurrentHashMap[String, String] = new ConcurrentHashMap(), + // Mapping method fullname to its return type and signature, lambda expression return type also getting recorded under this map + methodMetaMap: ConcurrentHashMap[String, MethodCacheMetaData] = ConcurrentHashMap() +) - def recordStructTypeMemberType(memberFullName: String, memberType: String): Unit = { - structTypeMemberTypeMapping.putIfAbsent(memberFullName, memberType) - } +case class MethodCacheMetaData(returnType: String, signature: String) - def recordFullNameToReturnType(methodFullName: String, returnType: String, signature: String): Unit = { - methodFullNameReturnTypeMap.putIfAbsent(methodFullName, (returnType, signature)) - } - - def recordPkgLevelVarAndConstantAst(pkg: String, ast: Ast, filePath: String): Unit = { - synchronized { - Option(pkgLevelVarAndConstantAstMap.get(pkg)) match { - case Some(existingList) => - val t = (ast, filePath) - pkgLevelVarAndConstantAstMap.put(pkg, existingList + t) - case None => pkgLevelVarAndConstantAstMap.put(pkg, Set((ast, filePath))) - } +case class LambdaTypeInfo(lambdaStructTypeFullName: String, returnTypeFullname: String) + extends Comparable[LambdaTypeInfo] { + override def compareTo(that: LambdaTypeInfo): Int = { + val lambdaStructTypeFullNameComparison = this.lambdaStructTypeFullName.compareTo(that.lambdaStructTypeFullName) + if (lambdaStructTypeFullNameComparison != 0) { + lambdaStructTypeFullNameComparison + } else { + this.returnTypeFullname.compareTo(that.returnTypeFullname) } } - - def recordLambdaSigntureToLambdaType( - signature: String, - lambdaStructTypeFullName: String, - returnTypeFullname: String - ): Unit = { - synchronized { - Option(lambdaSignatureToLambdaTypeMap.get(signature)) match { - case Some(existingList) => - val t = (lambdaStructTypeFullName, returnTypeFullname) - lambdaSignatureToLambdaTypeMap.put(signature, existingList + t) - case None => lambdaSignatureToLambdaTypeMap.put(signature, Set((lambdaStructTypeFullName, returnTypeFullname))) - } - } - } - } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala index 83b2ec4cfda8..414db4f34c4f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala @@ -5,6 +5,10 @@ import io.circe.{Decoder, HCursor} import io.joern.gosrc2cpg.Config import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern +import java.util.Set +import java.util.concurrent.ConcurrentSkipListSet +import scala.util.control.Breaks.* + class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { def getModMetaData(): Option[GoMod] = meta @@ -41,6 +45,22 @@ class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { val tokens = meta.get.module.name +: pathTokens.dropRight(1).filterNot(x => x == null || x.trim.isEmpty) tokens.mkString("/") } + + def recordUsedDependencies(importStmt: String): Unit = { + breakable { + meta.map(mod => + // TODO: && also add a check for builtin package imports to skip those + if (!importStmt.startsWith(mod.module.name)) { + for (dependency <- mod.dependencies) { + if (importStmt.startsWith(dependency.module)) { + dependency.beingUsed = true + dependency.usedPackages.add(importStmt) + } + } + } + ) + } + } } case class GoMod(fileFullPath: String, module: GoModModule, dependencies: List[GoModDependency]) @@ -55,10 +75,12 @@ case class GoModDependency( module: String, version: String, indirect: Boolean, + var beingUsed: Boolean, lineNo: Option[Int] = None, colNo: Option[Int] = None, endLineNo: Option[Int] = None, - endColNo: Option[Int] = None + endColNo: Option[Int] = None, + usedPackages: Set[String] = new ConcurrentSkipListSet[String]() ) object CirceEnDe { @@ -94,6 +116,7 @@ object CirceEnDe { module = module.getOrElse(""), version = version.getOrElse(""), indirect = indirect.getOrElse(false), + beingUsed = false, lineNo = lineNo.toOption, colNo = colNo.toOption, endLineNo = endLineNo.toOption, diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala index 4b91bfe4cd0f..7629e04eb6cb 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala @@ -12,6 +12,9 @@ import org.slf4j.LoggerFactory import java.io.File as JFile import java.nio.file.Paths +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} import scala.util.{Failure, Success, Try} class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, config: Config) { @@ -25,23 +28,30 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, con private def setupDummyProjectAndDownload(prjDir: String): Unit = { parentGoMod .getModMetaData() - .map(mod => { - ExternalCommand.run("go mod init joern.io/temp", prjDir) match + .foreach(mod => { + ExternalCommand.run("go mod init joern.io/temp", prjDir) match { case Success(_) => - mod.dependencies - .filter(dep => config.includeIndirectDependencies || !dep.indirect) - .foreach(dependency => { - val dependencyStr = s"${dependency.module}@${dependency.version}" - val cmd = s"go get $dependencyStr" - ExternalCommand.run(cmd, prjDir) match - case Success(_) => - print(". ") - processDependency(dependencyStr) - case Failure(f) => - logger.error(s"\t- command '${cmd}' failed", f) + val futures = mod.dependencies + .filter(dep => dep.beingUsed) + .map(dependency => { + Future { + val dependencyStr = s"${dependency.module}@${dependency.version}" + val cmd = s"go get $dependencyStr" + val results = synchronized(ExternalCommand.run(cmd, prjDir)) + results match { + case Success(_) => + print(". ") + processDependency(dependencyStr) + case Failure(f) => + logger.error(s"\t- command '$cmd' failed", f) + } + } }) + val allResults: Future[List[Unit]] = Future.sequence(futures) + Await.result(allResults, Duration.Inf) case Failure(f) => logger.error("\t- command 'go mod init joern.io/temp' failed", f) + } }) } @@ -49,13 +59,18 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, con val gopath = Try(sys.env("GOPATH")).getOrElse(Seq(os.home, "go").mkString(JFile.separator)) val dependencyLocation = (Seq(gopath, "pkg", "mod") ++ dependencyStr.split("/")).mkString(JFile.separator) File.usingTemporaryDirectory("godep") { astLocation => - val config = Config().withInputPath(dependencyLocation) - val astGenResult = new AstGenRunner(config).execute(astLocation).asInstanceOf[GoAstGenRunnerResult] + val depConfig = Config() + .withInputPath(dependencyLocation) + .withIgnoredFilesRegex(config.ignoredFilesRegex.toString()) + .withIgnoredFiles(config.ignoredFiles.toList) + // TODO: Need to implement mechanism to filter and process only used namespaces(folders) of the dependency. + // In order to achieve this filtering, we need to add support for inclusive rule with goastgen utility first. + val astGenResult = new AstGenRunner(depConfig).execute(astLocation).asInstanceOf[GoAstGenRunnerResult] val goMod = new GoModHelper( - Some(config), + Some(depConfig), astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) ) - new MethodAndTypeCacheBuilderPass(None, astGenResult.parsedFiles, config, goMod, goGlobal).process() + new MethodAndTypeCacheBuilderPass(None, astGenResult.parsedFiles, depConfig, goMod, goGlobal).process() } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala index 2e025d9ecfdd..b2bdb95e10bf 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala @@ -23,7 +23,7 @@ class MethodAndTypeCacheBuilderPass( ) { def process(): Seq[AstCreator] = { val futures = astFiles - .map(file => { + .map(file => Future { val parserResult = GoAstJsonParser.readFile(Paths.get(file)) val relPathFileName = SourceFiles.toRelativePath(parserResult.fullPath, config.inputPath) @@ -31,7 +31,7 @@ class MethodAndTypeCacheBuilderPass( val diffGraph = astCreator.buildCache(cpgOpt) (astCreator, diffGraph) } - }) + ) val allResults: Future[List[(AstCreator, DiffGraphBuilder)]] = Future.sequence(futures) val results = Await.result(allResults, Duration.Inf) val (astCreators, diffGraphs) = results.unzip diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala index a96e97f7e396..a889fcd9a641 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala @@ -2,14 +2,17 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.joern.gosrc2cpg.Config +import io.joern.gosrc2cpg.astcreation.Defines +import io.joern.gosrc2cpg.datastructures.{GoGlobal, LambdaTypeInfo, MethodCacheMetaData, NameSpaceMetaData} import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* +import scala.jdk.CollectionConverters.* + class DownloadDependencyTest extends GoCodeToCpgSuite { - // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 - // ignoring the below uni tests, which tries to download the dependencies. + val IGNORE_TEST_FILE_REGEX = ".*_test(s)?.*" "Simple use case of third-party dependency download" should { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -34,9 +37,10 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { } } - // TODO: These tests were working, something has broken. Will fix it in next PR. + // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 + // ignoring the below uni tests, which tries to download the dependencies. "Download dependency example with different package and namespace name" ignore { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -63,7 +67,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "Check if we are able to identify the type of constants accessible out side dependencies code" in { val List(t) = cpg.local("test").l - t.typeFullName shouldBe "string" + t.typeFullName shouldBe "github.com/aerospike/aerospike-client-go/v6.privilegeCode" } } @@ -124,10 +128,10 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { } } - // Note: methodFullName of call node is not resolving as per DownloadDependency so ignoring - // the below unit tests, which tries to download the dependencies and resolve it. + // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 + // ignoring the below uni tests, which tries to download the dependencies. "dependency resolution having type struct" ignore { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -149,7 +153,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { |func (c *Client) setValue() { | key := "key" | value := "value" - | err := c.rdb.Set(key, value).Err() + | err := c.rdb.Close() |} |""".stripMargin) .withConfig(config) @@ -158,13 +162,265 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { val List(typeDeclNode) = cpg.typeDecl.nameExact("Client").l typeDeclNode.fullName shouldBe "main.Client" typeDeclNode.member.size shouldBe 1 - typeDeclNode.member.head.typeFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.." + typeDeclNode.member.head.typeFullName shouldBe "github.com/redis/go-redis/v9.UniversalClient" + } + + "Test call node" ignore { + // TODO: Need to handle interface Type for caching the meta data to make this test work. + val List(callNode) = cpg.call.name("Close").l + callNode.typeFullName shouldBe "error" + callNode.methodFullName shouldBe "github.com/redis/go-redis/v9.UnversalClient.Close" + } + } + + "If the dependency is not getting used then it " should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |func main() { + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "not be downloaded " in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 1 + val List(dep) = dependencies + dep.module shouldBe "github.com/rs/zerolog" + dep.beingUsed shouldBe false } - "Test call node" in { - val List(callNode) = cpg.call.name("Set").l - callNode.typeFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.Set.." - callNode.methodFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.Set" + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping in the case current sample + goGlobal.aliasToNameSpaceMapping.size() shouldBe 0 + + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 0 + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + "not create any entry in method full name to return type map" in { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + "not create any entry in struct member to type map" in { + // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 + } + } + + "The dependency is getting imported somewhere but not getting used then it" should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + | github.com/google/uuid v1.3.1 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |import "github.com/rs/zerolog" + |func main() { + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "download the dependency" in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 2 + val List(depone, deptwo) = dependencies + depone.module shouldBe "github.com/rs/zerolog" + depone.beingUsed shouldBe true + + deptwo.module shouldBe "github.com/google/uuid" + deptwo.beingUsed shouldBe false + } + + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping + goGlobal.aliasToNameSpaceMapping.size() shouldBe 1 + goGlobal.aliasToNameSpaceMapping.values().toArray shouldBe Array("github.com/rs/zerolog") + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 0 + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in method full name to return type map" ignore { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in struct member to type map" ignore { + // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 + } + } + + "The dependency is getting imported and used in the code then it" should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + | github.com/google/uuid v1.3.1 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |import ( + | "github.com/rs/zerolog" + | "github.com/rs/zerolog/log" + |) + |func main() { + | var eventHandler = func(e *zerolog.Event, level zerolog.Level, message string){ + | } + | zerolog.SetGlobalLevel(zerolog.InfoLevel) + | log.Error().Msg("Error message") + | log.Warn().Msg("Warning message") + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "Be correct for CALL Node typeFullNames" in { + val List(a, b, c, d, e) = + cpg.call.where(_.and(_.nameNot(Operators.fieldAccess), _.nameNot(Operators.assignment))).l + a.typeFullName shouldBe "void" + b.typeFullName shouldBe "void" + c.typeFullName shouldBe "*github.com/rs/zerolog.Event" + d.typeFullName shouldBe "void" + e.typeFullName shouldBe "*github.com/rs/zerolog.Event" + } + + "download the dependency" in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 2 + val List(depone, deptwo) = dependencies + depone.module shouldBe "github.com/rs/zerolog" + depone.beingUsed shouldBe true + + deptwo.module shouldBe "github.com/google/uuid" + deptwo.beingUsed shouldBe false + } + + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping unless the folder name and package name is different. + goGlobal.aliasToNameSpaceMapping.size() shouldBe 2 + goGlobal.aliasToNameSpaceMapping.values().toArray shouldBe Array( + "github.com/rs/zerolog", + "github.com/rs/zerolog/log" + ) + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 1 + goGlobal.lambdaSignatureToLambdaTypeMap + .values() + .toArray() + .map(_.asInstanceOf[java.util.Set[LambdaTypeInfo]].asScala) + .flatMap(_.toList) shouldBe Array(LambdaTypeInfo("github.com/rs/zerolog.HookFunc", "void")) + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in method full name to return type map" ignore { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + // TODO: While doing the implementation we need update this test + // Lambda expression return types are also getting recorded under this map + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in struct member to type map" ignore { + // TODO: This test might require to update when we implement + // 1. Struct Type is directly being used + // 2. Struct Type is being passed as parameter or returned as value of method that is being used. + // 3. A method of Struct Type being used. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 } } } + +// TODO: Add unit tests with imports having builtin packages. diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala index 252e3117ff4d..ed85c1092310 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala @@ -4,6 +4,7 @@ import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File @@ -292,4 +293,87 @@ class GlobalVariableAndConstantTests extends GoCodeToCpgSuite { c.code shouldBe "personName" } } + + "Multiple packages with the same name but different paths" should { + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + |""".stripMargin, + "go.mod" + ).moreCode( + """ + |package lib1 + | + |const ( + | SchemeHTTP = "http" + |) + | + |""".stripMargin, + Seq("lib1", "typelib.go").mkString(File.separator) + ).moreCode( + """ + |package lib1 + | + |const ( + | SchemeHTTP = "something" + |) + | + |""".stripMargin, + Seq("another", "lib1", "dummy.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/sample/lib1" + |import anlib1 "joern.io/sample/another/lib1" + |func main() { + | var a = lib1.SchemeHTTP.value() + | var b = anlib1.SchemeHTTP.value() + |} + |""".stripMargin, + "main.go" + ) + + "Check package Type Decl" in { + val List(x) = cpg.typeDecl("main").l + x.fullName shouldBe "main" + } + + "Traversal from package type decl to global variable member nodes" in { + val List(f) = cpg.typeDecl("joern.io/sample/another/lib1").l + val List(a) = f.member.l + a.name shouldBe "SchemeHTTP" + a.typeFullName shouldBe "string" + val List(s) = cpg.typeDecl("joern.io/sample/lib1").l + val List(b) = s.member.l + b.name shouldBe "SchemeHTTP" + b.typeFullName shouldBe "string" + } + + "Create two package level TypeDecls for each package" in { + cpg.typeDecl.fullName.l shouldBe List("joern.io/sample/another/lib1", "joern.io/sample/lib1", "main") + } + + "Create two namespace blocks for each package" in { + cpg.namespaceBlock.filenameNot(FileTraversal.UNKNOWN).namespace.name.l shouldBe List( + "joern.io/sample/another/lib1", + "joern.io/sample/lib1", + "main" + ) + } + + "Be correct for Field Access CALL Node for Global variable access" in { + val List(a, b, c, d) = cpg.call(Operators.fieldAccess).l + a.typeFullName shouldBe "string" + b.typeFullName shouldBe "string" + c.method.fullName shouldBe "joern.io/sample/another/lib1" + d.method.fullName shouldBe "joern.io/sample/lib1" + } + + "Check methodfullname of constant imported from other package " in { + val List(callNode, callNode2) = cpg.call("value").l + callNode.methodFullName shouldBe "string.value" + callNode2.methodFullName shouldBe "string.value" + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala index 4df545631ea6..5d803f4310e5 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala @@ -1,6 +1,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite +import io.joern.gosrc2cpg.datastructures.GoGlobal import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* @@ -625,6 +626,7 @@ class TypeFullNameTests extends GoCodeToCpgSuite { } "Method call return value assigned to variable type check" should { + val goGlobal = GoGlobal() val cpg = code( """ |module joern.io/sample @@ -673,7 +675,7 @@ class TypeFullNameTests extends GoCodeToCpgSuite { |} |""".stripMargin, "main.go" - ) + ).withGoGlobal(goGlobal) "Call node typeFullName check with primitive return type" in { val List(bar) = cpg.call("bar").l diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala index 112206c8868d..eec45b3f0002 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala @@ -3,31 +3,47 @@ package io.joern.go2cpg.testfixtures import better.files.File import io.joern.dataflowengineoss.semanticsloader.FlowSemantic import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} +import io.joern.gosrc2cpg.datastructures.GoGlobal +import io.joern.gosrc2cpg.model.GoModHelper import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} -import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} +import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} import org.scalatest.Inside -trait Go2CpgFrontend extends LanguageFrontend { +class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with SemanticTestCpg { + + private var goGlobal: Option[GoGlobal] = None + private var goSrc2Cpg: Option[GoSrc2Cpg] = None + override protected def applyPasses(): Unit = { + super.applyPasses() + applyOssDataFlow() + } + + def withGoGlobal(goGlobal: GoGlobal): this.type = { + setGoGlobal(goGlobal) + this + } + + private def setGoGlobal(goGlobal: GoGlobal): Unit = { + if (this.goGlobal.isDefined) { + throw new RuntimeException("Frontend GoGlobal may only be set once per test") + } + this.goGlobal = Some(goGlobal) + } + def execute(sourceCodePath: java.io.File): Cpg = { val cpgOutFile = File.newTemporaryFile("go2cpg.bin") cpgOutFile.deleteOnExit() - val go2cpg = new GoSrc2Cpg() + goSrc2Cpg = Some(new GoSrc2Cpg(this.goGlobal)) val config = getConfig() .collectFirst { case x: Config => x } .getOrElse(Config()) .withInputPath(sourceCodePath.getAbsolutePath) .withOutputPath(cpgOutFile.pathAsString) - go2cpg.createCpg(config).get + goSrc2Cpg.get.createCpg(config).get } -} -class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with Go2CpgFrontend with SemanticTestCpg { - override protected def applyPasses(): Unit = { - super.applyPasses() - applyOssDataFlow() - } + def getModHelper(): GoModHelper = goSrc2Cpg.get.getGoModHelper } class GoCodeToCpgSuite(