diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala index cc5c0a26044d..ad8b6d9da1ae 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala @@ -22,31 +22,42 @@ import io.shiftleft.codepropertygraph.generated.Languages import java.nio.file.Paths import scala.util.Try -class GoSrc2Cpg extends X2CpgFrontend[Config] { +class GoSrc2Cpg(goGlobalOption: Option[GoGlobal] = Option(GoGlobal())) extends X2CpgFrontend[Config] { private val report: Report = new Report() + private var goMod: Option[GoModHelper] = None def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => File.usingTemporaryDirectory("gosrc2cpgOut") { tmpDir => - val goGlobal = GoGlobal() - new MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() - val astGenResult = new AstGenRunner(config).execute(tmpDir).asInstanceOf[GoAstGenRunnerResult] - val goMod = new GoModHelper( - Some(config), - astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) - ) - if (config.fetchDependencies) { - goGlobal.processingDependencies = true - new DownloadDependenciesPass(goMod, goGlobal, config).process() - goGlobal.processingDependencies = false - } - val astCreators = - new MethodAndTypeCacheBuilderPass(Some(cpg), astGenResult.parsedFiles, config, goMod, goGlobal).process() - new AstCreationPass(cpg, astCreators, report).createAndApply() - if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then - new PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() - report.print() + goGlobalOption + .orElse(Option(GoGlobal())) + .foreach(goGlobal => { + new MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() + val astGenResult = new AstGenRunner(config).execute(tmpDir).asInstanceOf[GoAstGenRunnerResult] + goMod = Some( + new GoModHelper( + Some(config), + astGenResult.parsedModFile + .flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) + ) + ) + goGlobal.mainModule = goMod.flatMap(modHelper => modHelper.getModMetaData().map(mod => mod.module.name)) + val astCreators = + new MethodAndTypeCacheBuilderPass(Some(cpg), astGenResult.parsedFiles, config, goMod.get, goGlobal) + .process() + if (config.fetchDependencies) { + goGlobal.processingDependencies = true + new DownloadDependenciesPass(goMod.get, goGlobal, config).process() + goGlobal.processingDependencies = false + } + new AstCreationPass(cpg, astCreators, report).createAndApply() + if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then + new PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() + report.print() + }) } } } + + def getGoModHelper: GoModHelper = goMod.get } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala index 882285ba0366..84fcb66b50ec 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala @@ -131,8 +131,8 @@ trait AstCreatorHelper { this: AstCreator => .toMap } - protected def resolveAliasToFullName(alias: String, typeOrMethodName: String): String = { - s"${aliasToNameSpaceMapping.getOrElse(alias, goGlobal.aliasToNameSpaceMapping.getOrDefault(alias, s"${XDefines.Unknown}.<$alias>"))}.$typeOrMethodName" + protected def resolveAliasToFullName(alias: String): String = { + s"${aliasToNameSpaceMapping.getOrElse(alias, goGlobal.aliasToNameSpaceMapping.getOrDefault(alias, s"${XDefines.Unknown}.<$alias>"))}" } protected def generateTypeFullName( typeName: Option[String] = None, @@ -156,7 +156,7 @@ trait AstCreatorHelper { this: AstCreator => Defines.primitiveTypeMap.getOrElse(typname, s"$fullyQualifiedPackage.$typname") } case Some(alias) => - resolveAliasToFullName(alias, typname) + s"${resolveAliasToFullName(alias)}.$typname" } private def internalTypeFullName( diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala index e5b4ce8d31cb..50bc5d30144d 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForGenDeclarationCreator.scala @@ -65,11 +65,12 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) val localParserNode = createParserNodeInfo(parserNode) if globalStatements then { val variableName = localParserNode.json(ParserKeys.Name).str - if (checkForDependencyFlags(variableName)) { + if (goGlobal.checkForDependencyFlags(variableName)) { // While processing the dependencies code ignoring package level global variables starting with lower case letter // as these variables are only accessible within package. So those will not be referred from main source code. - goGlobal.recordStructTypeMemberType( - s"$fullyQualifiedPackage${Defines.dot}$variableName", + goGlobal.recordStructTypeMemberTypeInfo( + fullyQualifiedPackage, + variableName, typeFullName.getOrElse(Defines.anyTypeName) ) astForGlobalVarAndConstants(typeFullName.getOrElse(Defines.anyTypeName), localParserNode) @@ -94,8 +95,8 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) val rhsTypeFullName = typeFullName.getOrElse(getTypeFullNameFromAstNode(rhsAst)) if (globalStatements) { val variableName = lhsParserNode.json(ParserKeys.Name).str - if (checkForDependencyFlags(variableName)) { - goGlobal.recordStructTypeMemberType(s"$fullyQualifiedPackage${Defines.dot}$variableName", rhsTypeFullName) + if (goGlobal.checkForDependencyFlags(variableName)) { + goGlobal.recordStructTypeMemberTypeInfo(fullyQualifiedPackage, variableName, rhsTypeFullName) astForGlobalVarAndConstants(rhsTypeFullName, lhsParserNode, Some(rhsAst)) } (Ast(), Ast()) @@ -160,14 +161,4 @@ trait AstForGenDeclarationCreator(implicit withSchemaValidation: ValidationMode) Ast() } } - - /** While processing the dependencies code ignoring package level global variables, constants, types, and functions - * starting with lower case letter as those are only accessible within package. So those will not be referred from - * main source code. - * @param name - * @return - */ - protected def checkForDependencyFlags(name: String): Boolean = { - !goGlobal.processingDependencies || goGlobal.processingDependencies && name.headOption.exists(_.isUpper) - } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala index e640444d1f52..41d80224c68c 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.{LambdaTypeInfo, MethodCacheMetaData} import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.datastructures.Stack.StackWrapper import io.joern.x2cpg.utils.NodeBuilders.newModifierNode @@ -8,6 +9,8 @@ import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodRetur import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} import ujson.Value +import scala.jdk.CollectionConverters.* + trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => def astForFuncLiteral(funcLiteral: ParserNodeInfo): Seq[Ast] = { @@ -17,9 +20,8 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: .collectFirst({ case m: NewMethod if !m.fullName.endsWith(parserResult.filename) => m.fullName }) .getOrElse(fullyQualifiedPackage) val fullName = s"$baseFullName.$lambdaName" - val (signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) = generateLambdaSignature( - createParserNodeInfo(funcLiteral.json(ParserKeys.Type)) - ) + val LambdaFunctionMetaData(signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) = + generateLambdaSignature(createParserNodeInfo(funcLiteral.json(ParserKeys.Type))) val methodNode_ = methodNode(funcLiteral, lambdaName, funcLiteral.code, fullName, Some(signature), relPathFileName) methodAstParentStack.push(methodNode_) scope.pushNewScope(methodNode_) @@ -40,7 +42,7 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: typeDeclNode_.astParentType(NodeTypes.TYPE_DECL).astParentFullName(fullyQualifiedPackage) else typeDeclNode_.astParentType(NodeTypes.METHOD).astParentFullName(baseFullName) val structTypes = Option(goGlobal.lambdaSignatureToLambdaTypeMap.get(signature)) match { - case Some(types) => types.map(_._1) + case Some(types) => types.asScala.map(_.lambdaStructTypeFullName) case None => Seq.empty } typeDeclNode_.inheritsFromTypeFullName(structTypes) @@ -50,13 +52,11 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: methodNode_.astParentFullName(fullName) Ast.storeInDiffGraph(astForMethod, diffGraph) } - goGlobal.recordFullNameToReturnType(fullName, returnTypeStr, signature) + goGlobal.recordMethodMetadata(baseFullName, lambdaName, MethodCacheMetaData(returnTypeStr, signature)) Seq(Ast(methodRefNode(funcLiteral, funcLiteral.code, fullName, fullName))) } - protected def generateLambdaSignature( - funcType: ParserNodeInfo - ): (String, String, NewMethodReturn, Value, Map[String, List[String]]) = { + protected def generateLambdaSignature(funcType: ParserNodeInfo): LambdaFunctionMetaData = { val genericTypeMethodMap: Map[String, List[String]] = Map() // TODO: While handling the tuple return type we need to handle it here as well. val (returnTypeStr, returnTypeInfo) = @@ -68,6 +68,14 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: val paramSignature = parameterSignature(params, genericTypeMethodMap) val signature = s"${XDefines.ClosurePrefix}($paramSignature)$returnTypeStr" - (signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) + LambdaFunctionMetaData(signature, returnTypeStr, methodReturn, params, genericTypeMethodMap) } } + +case class LambdaFunctionMetaData( + signature: String, + returnTypeStr: String, + methodReturn: NewMethodReturn, + params: Value, + genericTypeMethodMap: Map[String, List[String]] +) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala index 8b7fe7bdaeb1..1882d37cc897 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.MethodCacheMetaData import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} @@ -94,26 +95,22 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio // Then we are assuming that the given function is defined inside same package as that of current file's package. // This assumption will be invalid when another package is imported with alias "." val methodFullName = s"$fullyQualifiedPackage.$methodName" - val (returnTypeFullNameCache, signatureCache) = - goGlobal.methodFullNameReturnTypeMap - .getOrDefault(methodFullName, (Defines.anyTypeName, s"$methodFullName()")) + val methodInfo = goGlobal + .getMethodMetadata(fullyQualifiedPackage, methodName) + .getOrElse(MethodCacheMetaData(Defines.anyTypeName, s"$methodFullName()")) val (signature, fullName, returnTypeFullName) = - Defines.builtinFunctions.getOrElse(methodName, (signatureCache, methodFullName, returnTypeFullNameCache)) + Defines.builtinFunctions.getOrElse(methodName, (methodInfo.signature, methodFullName, methodInfo.returnType)) val probableLambdaTypeFullName = scope.lookupVariable(methodName) match case Some((_, lambdaTypeFullName)) => Some(lambdaTypeFullName) case _ => - Option(goGlobal.structTypeMemberTypeMapping.get(methodFullName)) match - case Some(globalLambdaTypeFullName) => Some(globalLambdaTypeFullName) - case _ => None + goGlobal.getStructTypeMemberType(fullyQualifiedPackage, methodName) val (postLambdaFullname, postLambdaSignature, postLambdaReturnTypeFullName) = probableLambdaTypeFullName match case Some(lambdaTypeFullName) => - Option( - goGlobal.methodFullNameReturnTypeMap - .get(lambdaTypeFullName) - ) match - case Some((lambdaReturnTypeFullNameCache, lambdaSignatureCache)) => - (lambdaTypeFullName, lambdaSignatureCache, lambdaReturnTypeFullNameCache) - case _ => (fullName, signature, returnTypeFullName) + val (nameSpaceName, lambdaName) = goGlobal.splitNamespaceFromMember(lambdaTypeFullName) + goGlobal.getMethodMetadata(nameSpaceName, lambdaName) match { + case Some(metaData) => (lambdaTypeFullName, metaData.signature, metaData.returnType) + case _ => (fullName, signature, returnTypeFullName) + } case _ => (fullName, signature, returnTypeFullName) (methodName, postLambdaSignature, postLambdaFullname, postLambdaReturnTypeFullName, Seq.empty) @@ -126,18 +123,21 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio processReceiverAst(methodName, xnode) case _ => // Otherwise its an alias to imported namespace on which method call is made - val alias = xnode.json(ParserKeys.Name).str - val callMethodFullName = - resolveAliasToFullName(alias, methodName) + val alias = xnode.json(ParserKeys.Name).str + val fullNamespace = resolveAliasToFullName(alias) + val callMethodFullName = s"$fullNamespace.$methodName" val lambdaFullName = - goGlobal.structTypeMemberTypeMapping.getOrDefault(callMethodFullName, callMethodFullName) - val (returnTypeFullNameCache, signatureCache) = Option( - goGlobal.methodFullNameReturnTypeMap - .get(lambdaFullName) - ) match - case Some((returnTypeFullName, signature)) => (returnTypeFullName, signature) - case _ => (s"$callMethodFullName.${Defines.ReturnType}.${XDefines.Unknown}", s"$callMethodFullName()") - + goGlobal.getStructTypeMemberType(fullNamespace, methodName).getOrElse(callMethodFullName) + val (nameSpace, memberName) = goGlobal.splitNamespaceFromMember(lambdaFullName) + val MethodCacheMetaData(returnTypeFullNameCache, signatureCache) = + goGlobal + .getMethodMetadata(nameSpace, memberName) + .getOrElse( + MethodCacheMetaData( + s"$callMethodFullName.${Defines.ReturnType}.${XDefines.Unknown}", + s"$callMethodFullName()" + ) + ) (methodName, signatureCache, lambdaFullName, returnTypeFullNameCache, Seq.empty) case _ => // This will take care of chained method calls. It will call `astForCallExpression` in recursive way, @@ -157,12 +157,14 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio .getOrElse(Defines.anyTypeName) .stripPrefix("*") val callMethodFullName = s"$receiverTypeFullName.$methodName" - val (returnTypeFullNameCache, signatureCache) = - goGlobal.methodFullNameReturnTypeMap - .getOrDefault( - callMethodFullName, - (s"$receiverTypeFullName.$methodName.${Defines.ReturnType}.${XDefines.Unknown}", s"$callMethodFullName()") + val MethodCacheMetaData(returnTypeFullNameCache, signatureCache) = goGlobal + .getMethodMetadata(receiverTypeFullName, methodName) + .getOrElse( + MethodCacheMetaData( + s"$receiverTypeFullName.$methodName.${Defines.ReturnType}.${XDefines.Unknown}", + s"$callMethodFullName()" ) + ) (methodName, signatureCache, callMethodFullName, returnTypeFullNameCache, receiverAst) } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala index d7c2bc356d4d..58f1e3097fdc 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala @@ -1,6 +1,5 @@ package io.joern.gosrc2cpg.astcreation -import io.joern.gosrc2cpg.datastructures.GoGlobal import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.utils.NodeBuilders.newOperatorCallNode @@ -72,11 +71,12 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t Ast(node).withRefEdge(node, variable) case _ => // If its not local node then check if its global member variable of package TypeDecl - Option(goGlobal.structTypeMemberTypeMapping.get(s"$fullyQualifiedPackage${Defines.dot}$identifierName")) match + goGlobal.getStructTypeMemberType(fullyQualifiedPackage, identifierName) match { case Some(fieldTypeFullName) => astForPackageGlobalFieldAccess(fieldTypeFullName, identifierName, ident) case _ => // TODO: something is wrong here. Refer to SwitchTests -> "be correct for switch case 4" Ast(identifierNode(ident, identifierName, ident.json(ParserKeys.Name).str, Defines.anyTypeName)) + } } } else { Ast() diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala index 71c88029ecd2..f727c976ab75 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForTypeDeclCreator.scala @@ -1,5 +1,5 @@ package io.joern.gosrc2cpg.astcreation -import io.joern.gosrc2cpg.datastructures.GoGlobal +import io.joern.gosrc2cpg.datastructures.LambdaTypeInfo import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg @@ -13,7 +13,7 @@ import scala.util.{Success, Try} trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def astForTypeSpec(typeSpecNode: ParserNodeInfo): Seq[Ast] = { - val (name, fullName, memberAsts) = processTypeSepc(typeSpecNode.json) + val (name, fullName, memberAsts) = processTypeSepc(createParserNodeInfo(typeSpecNode.json)) val typeDeclNode_ = typeDeclNode(typeSpecNode, name, fullName, relPathFileName, typeSpecNode.code) val modifier = addModifier(typeDeclNode_, name) @@ -21,8 +21,8 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi } protected def processFuncType(typeNode: ParserNodeInfo, typeDeclFullName: String): Seq[Ast] = { - val (signature, returnTypeFullName, _, _, _) = generateLambdaSignature(typeNode) - goGlobal.recordLambdaSigntureToLambdaType(signature, typeDeclFullName, returnTypeFullName) + val LambdaFunctionMetaData(signature, returnTypeFullName, _, _, _) = generateLambdaSignature(typeNode) + goGlobal.recordLambdaSigntureToLambdaType(signature, LambdaTypeInfo(typeDeclFullName, returnTypeFullName)) Seq.empty } @@ -39,7 +39,9 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi .map(fieldInfo => { val fieldNodeInfo = createParserNodeInfo(fieldInfo) val fieldName = fieldNodeInfo.json(ParserKeys.Name).str - goGlobal.recordStructTypeMemberType(typeDeclFullName + Defines.dot + fieldName, typeFullName) + if (goGlobal.checkForDependencyFlags(fieldName)) { + goGlobal.recordStructTypeMemberTypeInfo(typeDeclFullName, fieldName, typeFullName) + } Ast(memberNode(typeInfo, fieldName, fieldNodeInfo.code, typeFullName)) }) }) @@ -58,14 +60,15 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi receiverAstAndFullName(xnode, fieldIdentifier) case _ => // Otherwise its an alias to imported namespace using which global variable is getting accessed - val alias = xnode.json(ParserKeys.Name).str - val receiverFullName = resolveAliasToFullName(alias, fieldIdentifier) + val alias = xnode.json(ParserKeys.Name).str + val nameSpace = resolveAliasToFullName(alias) ( astForNode(xnode), - goGlobal.structTypeMemberTypeMapping.getOrDefault( - receiverFullName, - s"$receiverFullName${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" - ) + goGlobal + .getStructTypeMemberType(nameSpace, fieldIdentifier) + .getOrElse( + s"$nameSpace.$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" + ) ) case _ => // This will take care of chained calls @@ -75,10 +78,11 @@ trait AstForTypeDeclCreator(implicit withSchemaValidation: ValidationMode) { thi private def receiverAstAndFullName(xnode: ParserNodeInfo, fieldIdentifier: String): (Seq[Ast], String) = { val identifierAsts = astForNode(xnode) val receiverTypeFullName = getTypeFullNameFromAstNode(identifierAsts) - val fieldTypeFullName = goGlobal.structTypeMemberTypeMapping.getOrDefault( - s"$receiverTypeFullName${Defines.dot}$fieldIdentifier", - s"$receiverTypeFullName${Defines.dot}$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" - ) + val fieldTypeFullName = goGlobal + .getStructTypeMemberType(receiverTypeFullName, fieldIdentifier) + .getOrElse( + s"$receiverTypeFullName${Defines.dot}$fieldIdentifier${Defines.dot}${Defines.FieldAccess}${Defines.dot}${XDefines.Unknown}" + ) (identifierAsts, fieldTypeFullName) } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala index 88369e45444a..b1cf71451e02 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/CacheBuilder.scala @@ -1,5 +1,6 @@ package io.joern.gosrc2cpg.astcreation +import io.joern.gosrc2cpg.datastructures.MethodCacheMetaData import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern @@ -17,19 +18,21 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre def buildCache(cpgOpt: Option[Cpg]): DiffGraphBuilder = { val diffGraph = new DiffGraphBuilder try { - - cpgOpt.map { _ => - // We don't want to process this part when third party dependencies are being processed. - val result = goGlobal.recordAliasToNamespaceMapping(declaredPackageName, fullyQualifiedPackage) - if (result == null) { - // if result is null that means item got added first time otherwise it has been already added to global map - val rootNode = createParserNodeInfo(parserResult.json) - val ast = astForPackage(rootNode) - Ast.storeInDiffGraph(ast, diffGraph) + if (checkIfGivenDependencyPackageCanBeProcessed()) { + cpgOpt.map { _ => + // We don't want to process this part when third party dependencies are being processed. + if (goGlobal.recordForThisNamespace(fullyQualifiedPackage)) { + // java.util.Set.Add method will return true when set already doesn't contain the same value. + val rootNode = createParserNodeInfo(parserResult.json) + val ast = astForPackage(rootNode) + Ast.storeInDiffGraph(ast, diffGraph) + } } + identifyAndRecordPackagesWithDifferentName() + findAndProcess(parserResult.json) + // NOTE: For dependencies we are just caching the global variables Types. + processPackageLevelGolbalVaraiblesAndConstants(parserResult.json) } - findAndProcess(parserResult.json) - processPackageLevelGolbalVaraiblesAndConstants(parserResult.json) } catch { case ex: Exception => logger.warn(s"Error: While processing - ${parserResult.fullPath}", ex) @@ -37,6 +40,16 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre diffGraph } + private def checkIfGivenDependencyPackageCanBeProcessed(): Boolean = + !goGlobal.processingDependencies || goGlobal.processingDependencies && goGlobal.aliasToNameSpaceMapping + .containsValue(fullyQualifiedPackage) + + private def identifyAndRecordPackagesWithDifferentName(): Unit = { + // record the package to full namespace mapping only when declared package name is not matching with containing folder name + if (declaredPackageName != fullyQualifiedPackage.split("/").last) + goGlobal.recordAliasToNamespaceMapping(declaredPackageName, fullyQualifiedPackage) + } + private def astForPackage(rootNode: ParserNodeInfo): Ast = { val pathTokens = relPathFileName.split(fileSeparateorPattern) val packageFolderPath = if (pathTokens.nonEmpty && pathTokens.size > 1) { @@ -84,17 +97,17 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.ImportSpec" && !json.obj.contains( ParserKeys.NodeReferenceId - ) + ) && !goGlobal.processingDependencies ) { - processImports(obj) + // NOTE: Dependency code is not being processed here. + processImports(obj, true) } else if ( json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.TypeSpec" && !json.obj.contains( ParserKeys.NodeReferenceId ) ) { - createParserNodeInfo(obj) - processTypeSepc(obj) + processTypeSepc(createParserNodeInfo(obj)) } else if ( json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.FuncDecl" && !json.obj.contains( @@ -107,10 +120,20 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre json.obj .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.ValueSpec" && !json.obj.contains( ParserKeys.NodeReferenceId - ) + ) && !goGlobal.processingDependencies ) { + // NOTE: Dependency code is not being processed here. createParserNodeInfo(obj) + } else if ( + json.obj + .contains(ParserKeys.NodeType) && obj(ParserKeys.NodeType).str == "ast.FuncLit" && !json.obj.contains( + ParserKeys.NodeReferenceId + ) && !goGlobal.processingDependencies + ) { + // NOTE: Dependency code is not being processed here. + processFuncLiteral(obj) } + obj.value.values.foreach(subJson => findAndProcess(subJson)) case arr: Arr => arr.value.foreach(subJson => findAndProcess(subJson)) @@ -118,14 +141,21 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre } } - protected def processTypeSepc(typeSepc: Value): (String, String, Seq[Ast]) = { - val name = typeSepc(ParserKeys.Name)(ParserKeys.Name).str - if (checkForDependencyFlags(name)) { + private def processFuncLiteral(funcLit: Value): Unit = { + val LambdaFunctionMetaData(signature, _, _, _, _) = generateLambdaSignature( + createParserNodeInfo(funcLit(ParserKeys.Type)) + ) + goGlobal.recordForThisLamdbdaSignature(signature) + } + + protected def processTypeSepc(typeSepc: ParserNodeInfo): (String, String, Seq[Ast]) = { + val name = typeSepc.json(ParserKeys.Name)(ParserKeys.Name).str + if (goGlobal.checkForDependencyFlags(name)) { // Ignoring recording the Type details when we are processing dependencies code with Type name starting with lower case letter // As the Types starting with lower case letters will only be accessible within that package. Which means // these Types are not going to get referred from main source code. val fullName = fullyQualifiedPackage + Defines.dot + name - val typeNode = createParserNodeInfo(typeSepc(ParserKeys.Type)) + val typeNode = createParserNodeInfo(typeSepc.json(ParserKeys.Type)) val ast = typeNode.node match { // As of don't see any use case where InterfaceType needs to be handled. case InterfaceType => Seq.empty @@ -140,37 +170,50 @@ trait CacheBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCre ("", "", Seq.empty) } - protected def processImports(importDecl: Value): (String, String) = { + protected def processImports(importDecl: Value, recordFindings: Boolean = false): (String, String) = { val importedEntity = importDecl(ParserKeys.Path).obj(ParserKeys.Value).str.replaceAll("\"", "") - val importedAs = + if (recordFindings) { + goMod.recordUsedDependencies(importedEntity) + } + val importedAsOption = Try(importDecl(ParserKeys.Name).obj(ParserKeys.Name).str).toOption - .getOrElse(importedEntity.split("/").last) - - aliasToNameSpaceMapping.put(importedAs, importedEntity) - (importedEntity, importedAs) + importedAsOption match { + case Some(importedAs) => + // As these alias could be different for each file. Hence we maintain the cache at file level. + if (recordFindings) + aliasToNameSpaceMapping.put(importedAs, importedEntity) + (importedEntity, importedAs) + case _ => + val derivedImportedAs = importedEntity.split("/").last + if (recordFindings) + goGlobal.recordAliasToNamespaceMapping(derivedImportedAs, importedEntity) + (importedEntity, derivedImportedAs) + } } protected def processFuncDecl(funcDeclVal: Value): MethodMetadata = { val name = funcDeclVal(ParserKeys.Name).obj(ParserKeys.Name).str - if (checkForDependencyFlags(name)) { + if (goGlobal.checkForDependencyFlags(name)) { // Ignoring recording the method details when we are processing dependencies code with functions name starting with lower case letter // As the functions starting with lower case letters will only be accessible within that package. Which means // these methods / functions are not going to get referred from main source code. val receiverInfo = getReceiverInfo(Try(funcDeclVal(ParserKeys.Recv))) - val methodFullname = receiverInfo match + val (methodFullname, recordNamespace) = receiverInfo match case Some(_, typeFullName, _, _) => - s"$typeFullName.$name" + (s"$typeFullName.$name", typeFullName) case _ => - s"$fullyQualifiedPackage.$name" + (s"$fullyQualifiedPackage.$name", fullyQualifiedPackage) // TODO: handle multiple return type or tuple (int, int) val genericTypeMethodMap = processTypeParams(funcDeclVal(ParserKeys.Type)) val (returnTypeStr, _) = getReturnType(funcDeclVal(ParserKeys.Type), genericTypeMethodMap).headOption - .getOrElse(("", null)) + .getOrElse((Defines.voidTypeName, null)) val params = funcDeclVal(ParserKeys.Type)(ParserKeys.Params)(ParserKeys.List) val signature = - s"$methodFullname(${parameterSignature(params, genericTypeMethodMap)})$returnTypeStr" - goGlobal.recordFullNameToReturnType(methodFullname, returnTypeStr, signature) + s"$methodFullname(${parameterSignature(params, genericTypeMethodMap)})${ + if returnTypeStr == Defines.voidTypeName then "" else returnTypeStr + }" + goGlobal.recordMethodMetadata(recordNamespace, name, MethodCacheMetaData(returnTypeStr, signature)) MethodMetadata(name, methodFullname, signature, params, receiverInfo, genericTypeMethodMap) } else MethodMetadata() diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala index c3b0501afb98..9a42a1bbcb65 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/datastructures/GoGlobal.scala @@ -1,12 +1,13 @@ package io.joern.gosrc2cpg.datastructures import io.joern.x2cpg.Ast +import org.slf4j.LoggerFactory -import java.util.concurrent.ConcurrentHashMap - +import java.util.concurrent.{ConcurrentHashMap, ConcurrentSkipListSet} class GoGlobal { - - var processingDependencies = false + private val logger = LoggerFactory.getLogger(getClass) + var mainModule: Option[String] = None + var processingDependencies = false /** This map will only contain the mapping for those packages whose package name is different from the enclosing * folder name @@ -23,13 +24,124 @@ class GoGlobal { */ val aliasToNameSpaceMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap() - val lambdaSignatureToLambdaTypeMap: ConcurrentHashMap[String, Set[(String, String)]] = new ConcurrentHashMap() + /** This map will record the Type FullName of Struct Type defined for Lambda Expression along with return type + * fullname against the lambda signature. + * + * This will help map the Lambda TypeFullName with the respective Struct Type as supper Type + */ + val lambdaSignatureToLambdaTypeMap: ConcurrentHashMap[String, java.util.Set[LambdaTypeInfo]] = new ConcurrentHashMap() val pkgLevelVarAndConstantAstMap: ConcurrentHashMap[String, Set[(Ast, String)]] = new ConcurrentHashMap() - // Mapping method fullname to its return type and signature - val methodFullNameReturnTypeMap: ConcurrentHashMap[String, (String, String)] = new ConcurrentHashMap() + val nameSpaceMetaDataMap: ConcurrentHashMap[String, NameSpaceMetaData] = new ConcurrentHashMap() + + def recordAliasToNamespaceMapping(alias: String, namespace: String): Unit = synchronized { + val existingVal = aliasToNameSpaceMapping.putIfAbsent(alias, namespace) + // NOTE: !namespace.startsWith(mainModule.get) this check will not add the mapping for main source code imports. + // This will make sure to add the entry in CacheBuilder, which in turn creates the required Package level TypeDecl AST structure as well. + if (existingVal == null && (mainModule == None || (mainModule != None && !namespace.startsWith(mainModule.get)))) { + recordForThisNamespace(namespace) + } else if (existingVal != namespace) { + // TODO: This might need better way of recording the information. + logger.warn(s"more than one namespaces are found for given alias `$alias` -> `$existingVal` and `$namespace`") + } + } + + def recordForThisNamespace(namespace: String): Boolean = { + val existing = nameSpaceMetaDataMap.putIfAbsent(namespace, NameSpaceMetaData()) + existing == null + } + + def getMethodMetadata(namespace: String, methodName: String): Option[MethodCacheMetaData] = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + Option(existingNamespace.methodMetaMap.get(methodName)) + case _ => + None + } + } + + def recordMethodMetadata(namespace: String, methodName: String, methodMetaData: MethodCacheMetaData): Unit = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + existingNamespace.methodMetaMap.put(methodName, methodMetaData) + case _ => + // handling for types and lambda functions defined inside methods. Wrapping method becomes the part of their namespace. + val (wrappingNamespace, membertoken) = splitNamespaceFromMember(namespace) + // now check if this namespace is present in the map. If yes then make the new entry for this sub namespace + if (nameSpaceMetaDataMap.containsKey(wrappingNamespace) && checkForDependencyFlags(membertoken)) { + recordForThisNamespace(namespace) + recordMethodMetadata(namespace, methodName, methodMetaData) + } + } + } + + def getStructTypeMemberType(namespace: String, memberName: String): Option[String] = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + Option(existingNamespace.structTypeMembers.get(memberName)) + case _ => + None + } + } + def recordStructTypeMemberTypeInfo(namespace: String, memberName: String, memberType: String): Unit = { + Option(nameSpaceMetaDataMap.get(namespace)) match { + case Some(existingNamespace) => + existingNamespace.structTypeMembers.put(memberName, memberType) + case _ => + val (wrappingNamespace, membertoken) = splitNamespaceFromMember(namespace) + if (nameSpaceMetaDataMap.containsKey(wrappingNamespace) && checkForDependencyFlags(membertoken)) { + recordForThisNamespace(namespace) + recordStructTypeMemberTypeInfo(namespace, memberName, memberType) + } + } + } + + def recordPkgLevelVarAndConstantAst(pkg: String, ast: Ast, filePath: String): Unit = synchronized { + Option(pkgLevelVarAndConstantAstMap.get(pkg)) match { + case Some(existingList) => + val t = (ast, filePath) + pkgLevelVarAndConstantAstMap.put(pkg, existingList + t) + case None => pkgLevelVarAndConstantAstMap.put(pkg, Set((ast, filePath))) + } + } + def recordForThisLamdbdaSignature(signature: String): Unit = { + lambdaSignatureToLambdaTypeMap.putIfAbsent(signature, new ConcurrentSkipListSet()) + } + + def recordLambdaSigntureToLambdaType(signature: String, lambdaTypeInfo: LambdaTypeInfo): Unit = { + Option(lambdaSignatureToLambdaTypeMap.get(signature)) match { + case Some(existingList) => + existingList.add(lambdaTypeInfo) + case _ => + } + } + + def splitNamespaceFromMember(fullName: String): (String, String) = { + if (fullName.contains('.')) { + val lastDotIndex = fullName.lastIndexOf('.') + val nameSpaceName = fullName.substring(0, lastDotIndex) + val memberName = fullName.substring(lastDotIndex + 1) + (nameSpaceName, memberName) + } else { + (fullName, "") + } + } + + /** While processing the dependencies code ignoring package level global variables, constants, types, and functions + * starting with lower case letter as those are only accessible within package. So those will not be referred from + * main source code. + * + * @param name + * @return + */ + def checkForDependencyFlags(name: String): Boolean = { + !processingDependencies || processingDependencies && name.headOption.exists(_.isUpper) + } +} + +case class NameSpaceMetaData( /** Mapping fully qualified name of the member variable of a struct type to it's type It will also maintain the type * mapping for package level global variables. e.g. * @@ -47,44 +159,21 @@ class GoGlobal { * * `joern.io/sample.HostURL` - `string` */ - val structTypeMemberTypeMapping: ConcurrentHashMap[String, String] = new ConcurrentHashMap() - - def recordAliasToNamespaceMapping(alias: String, namespace: String): String = { - aliasToNameSpaceMapping.putIfAbsent(alias, namespace) - } + structTypeMembers: ConcurrentHashMap[String, String] = new ConcurrentHashMap(), + // Mapping method fullname to its return type and signature, lambda expression return type also getting recorded under this map + methodMetaMap: ConcurrentHashMap[String, MethodCacheMetaData] = ConcurrentHashMap() +) - def recordStructTypeMemberType(memberFullName: String, memberType: String): Unit = { - structTypeMemberTypeMapping.putIfAbsent(memberFullName, memberType) - } +case class MethodCacheMetaData(returnType: String, signature: String) - def recordFullNameToReturnType(methodFullName: String, returnType: String, signature: String): Unit = { - methodFullNameReturnTypeMap.putIfAbsent(methodFullName, (returnType, signature)) - } - - def recordPkgLevelVarAndConstantAst(pkg: String, ast: Ast, filePath: String): Unit = { - synchronized { - Option(pkgLevelVarAndConstantAstMap.get(pkg)) match { - case Some(existingList) => - val t = (ast, filePath) - pkgLevelVarAndConstantAstMap.put(pkg, existingList + t) - case None => pkgLevelVarAndConstantAstMap.put(pkg, Set((ast, filePath))) - } +case class LambdaTypeInfo(lambdaStructTypeFullName: String, returnTypeFullname: String) + extends Comparable[LambdaTypeInfo] { + override def compareTo(that: LambdaTypeInfo): Int = { + val lambdaStructTypeFullNameComparison = this.lambdaStructTypeFullName.compareTo(that.lambdaStructTypeFullName) + if (lambdaStructTypeFullNameComparison != 0) { + lambdaStructTypeFullNameComparison + } else { + this.returnTypeFullname.compareTo(that.returnTypeFullname) } } - - def recordLambdaSigntureToLambdaType( - signature: String, - lambdaStructTypeFullName: String, - returnTypeFullname: String - ): Unit = { - synchronized { - Option(lambdaSignatureToLambdaTypeMap.get(signature)) match { - case Some(existingList) => - val t = (lambdaStructTypeFullName, returnTypeFullname) - lambdaSignatureToLambdaTypeMap.put(signature, existingList + t) - case None => lambdaSignatureToLambdaTypeMap.put(signature, Set((lambdaStructTypeFullName, returnTypeFullname))) - } - } - } - } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala index 83b2ec4cfda8..414db4f34c4f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala @@ -5,6 +5,10 @@ import io.circe.{Decoder, HCursor} import io.joern.gosrc2cpg.Config import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern +import java.util.Set +import java.util.concurrent.ConcurrentSkipListSet +import scala.util.control.Breaks.* + class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { def getModMetaData(): Option[GoMod] = meta @@ -41,6 +45,22 @@ class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { val tokens = meta.get.module.name +: pathTokens.dropRight(1).filterNot(x => x == null || x.trim.isEmpty) tokens.mkString("/") } + + def recordUsedDependencies(importStmt: String): Unit = { + breakable { + meta.map(mod => + // TODO: && also add a check for builtin package imports to skip those + if (!importStmt.startsWith(mod.module.name)) { + for (dependency <- mod.dependencies) { + if (importStmt.startsWith(dependency.module)) { + dependency.beingUsed = true + dependency.usedPackages.add(importStmt) + } + } + } + ) + } + } } case class GoMod(fileFullPath: String, module: GoModModule, dependencies: List[GoModDependency]) @@ -55,10 +75,12 @@ case class GoModDependency( module: String, version: String, indirect: Boolean, + var beingUsed: Boolean, lineNo: Option[Int] = None, colNo: Option[Int] = None, endLineNo: Option[Int] = None, - endColNo: Option[Int] = None + endColNo: Option[Int] = None, + usedPackages: Set[String] = new ConcurrentSkipListSet[String]() ) object CirceEnDe { @@ -94,6 +116,7 @@ object CirceEnDe { module = module.getOrElse(""), version = version.getOrElse(""), indirect = indirect.getOrElse(false), + beingUsed = false, lineNo = lineNo.toOption, colNo = colNo.toOption, endLineNo = endLineNo.toOption, diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala index 4b91bfe4cd0f..7629e04eb6cb 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala @@ -12,6 +12,9 @@ import org.slf4j.LoggerFactory import java.io.File as JFile import java.nio.file.Paths +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} import scala.util.{Failure, Success, Try} class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, config: Config) { @@ -25,23 +28,30 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, con private def setupDummyProjectAndDownload(prjDir: String): Unit = { parentGoMod .getModMetaData() - .map(mod => { - ExternalCommand.run("go mod init joern.io/temp", prjDir) match + .foreach(mod => { + ExternalCommand.run("go mod init joern.io/temp", prjDir) match { case Success(_) => - mod.dependencies - .filter(dep => config.includeIndirectDependencies || !dep.indirect) - .foreach(dependency => { - val dependencyStr = s"${dependency.module}@${dependency.version}" - val cmd = s"go get $dependencyStr" - ExternalCommand.run(cmd, prjDir) match - case Success(_) => - print(". ") - processDependency(dependencyStr) - case Failure(f) => - logger.error(s"\t- command '${cmd}' failed", f) + val futures = mod.dependencies + .filter(dep => dep.beingUsed) + .map(dependency => { + Future { + val dependencyStr = s"${dependency.module}@${dependency.version}" + val cmd = s"go get $dependencyStr" + val results = synchronized(ExternalCommand.run(cmd, prjDir)) + results match { + case Success(_) => + print(". ") + processDependency(dependencyStr) + case Failure(f) => + logger.error(s"\t- command '$cmd' failed", f) + } + } }) + val allResults: Future[List[Unit]] = Future.sequence(futures) + Await.result(allResults, Duration.Inf) case Failure(f) => logger.error("\t- command 'go mod init joern.io/temp' failed", f) + } }) } @@ -49,13 +59,18 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper, goGlobal: GoGlobal, con val gopath = Try(sys.env("GOPATH")).getOrElse(Seq(os.home, "go").mkString(JFile.separator)) val dependencyLocation = (Seq(gopath, "pkg", "mod") ++ dependencyStr.split("/")).mkString(JFile.separator) File.usingTemporaryDirectory("godep") { astLocation => - val config = Config().withInputPath(dependencyLocation) - val astGenResult = new AstGenRunner(config).execute(astLocation).asInstanceOf[GoAstGenRunnerResult] + val depConfig = Config() + .withInputPath(dependencyLocation) + .withIgnoredFilesRegex(config.ignoredFilesRegex.toString()) + .withIgnoredFiles(config.ignoredFiles.toList) + // TODO: Need to implement mechanism to filter and process only used namespaces(folders) of the dependency. + // In order to achieve this filtering, we need to add support for inclusive rule with goastgen utility first. + val astGenResult = new AstGenRunner(depConfig).execute(astLocation).asInstanceOf[GoAstGenRunnerResult] val goMod = new GoModHelper( - Some(config), + Some(depConfig), astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) ) - new MethodAndTypeCacheBuilderPass(None, astGenResult.parsedFiles, config, goMod, goGlobal).process() + new MethodAndTypeCacheBuilderPass(None, astGenResult.parsedFiles, depConfig, goMod, goGlobal).process() } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala index 2e025d9ecfdd..b2bdb95e10bf 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/MethodAndTypeCacheBuilderPass.scala @@ -23,7 +23,7 @@ class MethodAndTypeCacheBuilderPass( ) { def process(): Seq[AstCreator] = { val futures = astFiles - .map(file => { + .map(file => Future { val parserResult = GoAstJsonParser.readFile(Paths.get(file)) val relPathFileName = SourceFiles.toRelativePath(parserResult.fullPath, config.inputPath) @@ -31,7 +31,7 @@ class MethodAndTypeCacheBuilderPass( val diffGraph = astCreator.buildCache(cpgOpt) (astCreator, diffGraph) } - }) + ) val allResults: Future[List[(AstCreator, DiffGraphBuilder)]] = Future.sequence(futures) val results = Await.result(allResults, Duration.Inf) val (astCreators, diffGraphs) = results.unzip diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala index a96e97f7e396..a889fcd9a641 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala @@ -2,14 +2,17 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.joern.gosrc2cpg.Config +import io.joern.gosrc2cpg.astcreation.Defines +import io.joern.gosrc2cpg.datastructures.{GoGlobal, LambdaTypeInfo, MethodCacheMetaData, NameSpaceMetaData} import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* +import scala.jdk.CollectionConverters.* + class DownloadDependencyTest extends GoCodeToCpgSuite { - // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 - // ignoring the below uni tests, which tries to download the dependencies. + val IGNORE_TEST_FILE_REGEX = ".*_test(s)?.*" "Simple use case of third-party dependency download" should { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -34,9 +37,10 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { } } - // TODO: These tests were working, something has broken. Will fix it in next PR. + // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 + // ignoring the below uni tests, which tries to download the dependencies. "Download dependency example with different package and namespace name" ignore { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -63,7 +67,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "Check if we are able to identify the type of constants accessible out side dependencies code" in { val List(t) = cpg.local("test").l - t.typeFullName shouldBe "string" + t.typeFullName shouldBe "github.com/aerospike/aerospike-client-go/v6.privilegeCode" } } @@ -124,10 +128,10 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { } } - // Note: methodFullName of call node is not resolving as per DownloadDependency so ignoring - // the below unit tests, which tries to download the dependencies and resolve it. + // NOTE: With respect to conversation on this PR - https://github.com/joernio/joern/pull/3753 + // ignoring the below uni tests, which tries to download the dependencies. "dependency resolution having type struct" ignore { - val config = Config().withFetchDependencies(true) + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) val cpg = code( """ |module joern.io/sample @@ -149,7 +153,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { |func (c *Client) setValue() { | key := "key" | value := "value" - | err := c.rdb.Set(key, value).Err() + | err := c.rdb.Close() |} |""".stripMargin) .withConfig(config) @@ -158,13 +162,265 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { val List(typeDeclNode) = cpg.typeDecl.nameExact("Client").l typeDeclNode.fullName shouldBe "main.Client" typeDeclNode.member.size shouldBe 1 - typeDeclNode.member.head.typeFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.." + typeDeclNode.member.head.typeFullName shouldBe "github.com/redis/go-redis/v9.UniversalClient" + } + + "Test call node" ignore { + // TODO: Need to handle interface Type for caching the meta data to make this test work. + val List(callNode) = cpg.call.name("Close").l + callNode.typeFullName shouldBe "error" + callNode.methodFullName shouldBe "github.com/redis/go-redis/v9.UnversalClient.Close" + } + } + + "If the dependency is not getting used then it " should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |func main() { + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "not be downloaded " in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 1 + val List(dep) = dependencies + dep.module shouldBe "github.com/rs/zerolog" + dep.beingUsed shouldBe false } - "Test call node" in { - val List(callNode) = cpg.call.name("Set").l - callNode.typeFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.Set.." - callNode.methodFullName shouldBe "github.com/redis/go-redis/v9.redis.UnversalClient.Set" + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping in the case current sample + goGlobal.aliasToNameSpaceMapping.size() shouldBe 0 + + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 0 + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + "not create any entry in method full name to return type map" in { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + "not create any entry in struct member to type map" in { + // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 + } + } + + "The dependency is getting imported somewhere but not getting used then it" should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + | github.com/google/uuid v1.3.1 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |import "github.com/rs/zerolog" + |func main() { + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "download the dependency" in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 2 + val List(depone, deptwo) = dependencies + depone.module shouldBe "github.com/rs/zerolog" + depone.beingUsed shouldBe true + + deptwo.module shouldBe "github.com/google/uuid" + deptwo.beingUsed shouldBe false + } + + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping + goGlobal.aliasToNameSpaceMapping.size() shouldBe 1 + goGlobal.aliasToNameSpaceMapping.values().toArray shouldBe Array("github.com/rs/zerolog") + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 0 + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in method full name to return type map" ignore { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in struct member to type map" ignore { + // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 + } + } + + "The dependency is getting imported and used in the code then it" should { + val goGlobal = GoGlobal() + val config = Config().withFetchDependencies(true).withIgnoredFilesRegex(IGNORE_TEST_FILE_REGEX) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + | + |require ( + | github.com/rs/zerolog v1.31.0 + | github.com/google/uuid v1.3.1 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |import ( + | "github.com/rs/zerolog" + | "github.com/rs/zerolog/log" + |) + |func main() { + | var eventHandler = func(e *zerolog.Event, level zerolog.Level, message string){ + | } + | zerolog.SetGlobalLevel(zerolog.InfoLevel) + | log.Error().Msg("Error message") + | log.Warn().Msg("Warning message") + |} + |""".stripMargin) + .withConfig(config) + .withGoGlobal(goGlobal) + + // Dummy cpg query which will initiate CPG creation. + cpg.method.l + + "Be correct for CALL Node typeFullNames" in { + val List(a, b, c, d, e) = + cpg.call.where(_.and(_.nameNot(Operators.fieldAccess), _.nameNot(Operators.assignment))).l + a.typeFullName shouldBe "void" + b.typeFullName shouldBe "void" + c.typeFullName shouldBe "*github.com/rs/zerolog.Event" + d.typeFullName shouldBe "void" + e.typeFullName shouldBe "*github.com/rs/zerolog.Event" + } + + "download the dependency" in { + val goModHelper = cpg.getModHelper() + val dependencies = goModHelper.getModMetaData().get.dependencies + dependencies.size shouldBe 2 + val List(depone, deptwo) = dependencies + depone.module shouldBe "github.com/rs/zerolog" + depone.beingUsed shouldBe true + + deptwo.module shouldBe "github.com/google/uuid" + deptwo.beingUsed shouldBe false + } + + "not create any entry in package to namespace mapping" in { + // it should not add `main` in the mapping as well as it should not contain any dependency mapping unless the folder name and package name is different. + goGlobal.aliasToNameSpaceMapping.size() shouldBe 2 + goGlobal.aliasToNameSpaceMapping.values().toArray shouldBe Array( + "github.com/rs/zerolog", + "github.com/rs/zerolog/log" + ) + } + + "not create any entry in lambda signature to return type mapping" in { + // "github.com/rs/zerolog" dependency has lambda Struct Types declared in the code. However they should not get cached as they are not getting used anywhere. + goGlobal.lambdaSignatureToLambdaTypeMap.size() shouldBe 1 + goGlobal.lambdaSignatureToLambdaTypeMap + .values() + .toArray() + .map(_.asInstanceOf[java.util.Set[LambdaTypeInfo]].asScala) + .flatMap(_.toList) shouldBe Array(LambdaTypeInfo("github.com/rs/zerolog.HookFunc", "void")) + } + + "not create any entry in package level ctor map" in { + // This anyway should only be populated for main source code. + goGlobal.pkgLevelVarAndConstantAstMap.size() shouldBe 0 + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in method full name to return type map" ignore { + // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. + // TODO: While doing the implementation we need update this test + // Lambda expression return types are also getting recorded under this map + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.methodMetaMap.size() shouldBe 1 + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + mainfullname shouldBe "main" + val Array(returnType) = metadata.methodMetaMap.values().toArray + returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") + } + + // TODO: Need to update these tests with some more improvements + "not create any entry in struct member to type map" ignore { + // TODO: This test might require to update when we implement + // 1. Struct Type is directly being used + // 2. Struct Type is being passed as parameter or returned as value of method that is being used. + // 3. A method of Struct Type being used. + goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + metadata.structTypeMembers.size() shouldBe 0 } } } + +// TODO: Add unit tests with imports having builtin packages. diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala index 252e3117ff4d..ed85c1092310 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/GlobalVariableAndConstantTests.scala @@ -4,6 +4,7 @@ import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File @@ -292,4 +293,87 @@ class GlobalVariableAndConstantTests extends GoCodeToCpgSuite { c.code shouldBe "personName" } } + + "Multiple packages with the same name but different paths" should { + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + |""".stripMargin, + "go.mod" + ).moreCode( + """ + |package lib1 + | + |const ( + | SchemeHTTP = "http" + |) + | + |""".stripMargin, + Seq("lib1", "typelib.go").mkString(File.separator) + ).moreCode( + """ + |package lib1 + | + |const ( + | SchemeHTTP = "something" + |) + | + |""".stripMargin, + Seq("another", "lib1", "dummy.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/sample/lib1" + |import anlib1 "joern.io/sample/another/lib1" + |func main() { + | var a = lib1.SchemeHTTP.value() + | var b = anlib1.SchemeHTTP.value() + |} + |""".stripMargin, + "main.go" + ) + + "Check package Type Decl" in { + val List(x) = cpg.typeDecl("main").l + x.fullName shouldBe "main" + } + + "Traversal from package type decl to global variable member nodes" in { + val List(f) = cpg.typeDecl("joern.io/sample/another/lib1").l + val List(a) = f.member.l + a.name shouldBe "SchemeHTTP" + a.typeFullName shouldBe "string" + val List(s) = cpg.typeDecl("joern.io/sample/lib1").l + val List(b) = s.member.l + b.name shouldBe "SchemeHTTP" + b.typeFullName shouldBe "string" + } + + "Create two package level TypeDecls for each package" in { + cpg.typeDecl.fullName.l shouldBe List("joern.io/sample/another/lib1", "joern.io/sample/lib1", "main") + } + + "Create two namespace blocks for each package" in { + cpg.namespaceBlock.filenameNot(FileTraversal.UNKNOWN).namespace.name.l shouldBe List( + "joern.io/sample/another/lib1", + "joern.io/sample/lib1", + "main" + ) + } + + "Be correct for Field Access CALL Node for Global variable access" in { + val List(a, b, c, d) = cpg.call(Operators.fieldAccess).l + a.typeFullName shouldBe "string" + b.typeFullName shouldBe "string" + c.method.fullName shouldBe "joern.io/sample/another/lib1" + d.method.fullName shouldBe "joern.io/sample/lib1" + } + + "Check methodfullname of constant imported from other package " in { + val List(callNode, callNode2) = cpg.call("value").l + callNode.methodFullName shouldBe "string.value" + callNode2.methodFullName shouldBe "string.value" + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala index 4df545631ea6..5d803f4310e5 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala @@ -1,6 +1,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite +import io.joern.gosrc2cpg.datastructures.GoGlobal import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* @@ -625,6 +626,7 @@ class TypeFullNameTests extends GoCodeToCpgSuite { } "Method call return value assigned to variable type check" should { + val goGlobal = GoGlobal() val cpg = code( """ |module joern.io/sample @@ -673,7 +675,7 @@ class TypeFullNameTests extends GoCodeToCpgSuite { |} |""".stripMargin, "main.go" - ) + ).withGoGlobal(goGlobal) "Call node typeFullName check with primitive return type" in { val List(bar) = cpg.call("bar").l diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala index 112206c8868d..eec45b3f0002 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala @@ -3,31 +3,47 @@ package io.joern.go2cpg.testfixtures import better.files.File import io.joern.dataflowengineoss.semanticsloader.FlowSemantic import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} +import io.joern.gosrc2cpg.datastructures.GoGlobal +import io.joern.gosrc2cpg.model.GoModHelper import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} -import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} +import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} import org.scalatest.Inside -trait Go2CpgFrontend extends LanguageFrontend { +class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with SemanticTestCpg { + + private var goGlobal: Option[GoGlobal] = None + private var goSrc2Cpg: Option[GoSrc2Cpg] = None + override protected def applyPasses(): Unit = { + super.applyPasses() + applyOssDataFlow() + } + + def withGoGlobal(goGlobal: GoGlobal): this.type = { + setGoGlobal(goGlobal) + this + } + + private def setGoGlobal(goGlobal: GoGlobal): Unit = { + if (this.goGlobal.isDefined) { + throw new RuntimeException("Frontend GoGlobal may only be set once per test") + } + this.goGlobal = Some(goGlobal) + } + def execute(sourceCodePath: java.io.File): Cpg = { val cpgOutFile = File.newTemporaryFile("go2cpg.bin") cpgOutFile.deleteOnExit() - val go2cpg = new GoSrc2Cpg() + goSrc2Cpg = Some(new GoSrc2Cpg(this.goGlobal)) val config = getConfig() .collectFirst { case x: Config => x } .getOrElse(Config()) .withInputPath(sourceCodePath.getAbsolutePath) .withOutputPath(cpgOutFile.pathAsString) - go2cpg.createCpg(config).get + goSrc2Cpg.get.createCpg(config).get } -} -class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with Go2CpgFrontend with SemanticTestCpg { - override protected def applyPasses(): Unit = { - super.applyPasses() - applyOssDataFlow() - } + def getModHelper(): GoModHelper = goSrc2Cpg.get.getGoModHelper } class GoCodeToCpgSuite(