From 7a73cca43b5288bf62757de3fba7e4708e500cf4 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Mon, 18 Sep 2023 08:47:12 +0530 Subject: [PATCH 1/7] array access ast handling --- .../gosrc2cpg/astcreation/AstCreator.scala | 1 + .../astcreation/AstForStatementsCreator.scala | 30 +++++- .../io/joern/gosrc2cpg/parser/ParserAst.scala | 2 + .../joern/go2cpg/passes/ast/ArraysTests.scala | 95 +++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala index 476bacefbd15..086f09b57f3a 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala @@ -91,6 +91,7 @@ class AstCreator(val relPathFileName: String, val parserResult: ParserResult)(im nodeInfo.node match { case GenDecl => astForGenDecl(nodeInfo) case FuncDecl => astForFuncDecl(nodeInfo) + case IndexExpr => astForIndexStatement(nodeInfo) case _: BasePrimitive => astForPrimitive(nodeInfo) case _: BaseExpr => astsForExpression(nodeInfo) case _: BaseStmt => astsForStatement(nodeInfo) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala index bc66d650a4a0..24e812936d18 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala @@ -3,8 +3,9 @@ package io.joern.gosrc2cpg.astcreation import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.Operator +import io.joern.x2cpg.utils.NodeBuilders.{newCallNode, newOperatorCallNode} import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.ExpressionNew +import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewCall, NewIdentifier, NewLiteral} import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} import ujson.Value @@ -257,4 +258,31 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t Ast() } } + def astForIndexStatement(indexNode: ParserNodeInfo): Seq[Ast] = { + val callNode = newOperatorCallNode( + Operators.indexAccess, + code = indexNode.code, + line = indexNode.lineNumber, + column = indexNode.columnNumber + ) + Seq(callAst(callNode, processIndexStatementArgs(indexNode.json))) + } + + private def processIndexStatementArgs(node: Value): Seq[Ast] = { + if (node.obj.contains(ParserKeys.Index)) { + val indexJson = node(ParserKeys.Index) + val literalValue = + if (indexJson.obj.contains(ParserKeys.Name)) indexJson(ParserKeys.Name).str else indexJson(ParserKeys.Value).str + val indexArgsAst = NewLiteral().code(literalValue) + Seq(Ast(indexArgsAst)) ++ processIndexStatementArgs(node(ParserKeys.X)) + } else { + Seq( + Ast( + NewIdentifier() + .name(node(ParserKeys.Name).str) + .code(node(ParserKeys.Name).str) + ) + ) + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala index ff5086200b12..2c3d6e62abf8 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala @@ -67,6 +67,7 @@ object ParserAst { object ChanType extends ParserNode object Field extends ParserNode object TypeSpec extends ParserNode + object IndexExpr extends ParserNode } object ParserKeys { @@ -116,4 +117,5 @@ object ParserKeys { val TypeParams = "TypeParams" val Args = "Args" val Recv = "Recv" + val Index = "Index" } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala index 3d1fc6753861..3e22d8aac593 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala @@ -405,4 +405,99 @@ class ArraysTests extends GoCodeToCpgSuite { val List(identifierNode) = assignmentCallNode.astChildren.isIdentifier.l.l identifierNode.code shouldBe "c" } + + "be correct when array access using variable having single index" in { + val cpg = code(""" + |package main + |func main() { + | value := myArray[index] + |} + |""".stripMargin) + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[index]" + indexCall.methodFullName shouldBe ".indexAccess" + + val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l + indexIdentifiers.code shouldBe "myArray" + + cpg.literal.code("index").size shouldBe 1 + } + + "be correct when array access using string having single index" in { + val cpg = code(""" + |package main + |func main() { + | value := myArray["index"] + |} + |""".stripMargin) + + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[\"index\"]" + indexCall.methodFullName shouldBe ".indexAccess" + + val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l + indexIdentifiers.code shouldBe "myArray" + + cpg.literal.code("\"index\"").size shouldBe 1 + } + + "be correct when array access using variable having multi index" in { + val cpg = code(""" + |package main + |func main() { + | value := myArray[index2][index1] + |} + |""".stripMargin) + + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[index2][index1]" + indexCall.methodFullName shouldBe ".indexAccess" + + val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l + indexIdentifiers.code shouldBe "myArray" + + val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l + literal1.code shouldBe "index1" + literal2.code shouldBe "index2" + } + + "be correct when array access using string having multi index" in { + val cpg = code(""" + |package main + |func main() { + | value := myArray["index2"]["index1"] + |} + |""".stripMargin) + + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[\"index2\"][\"index1\"]" + indexCall.methodFullName shouldBe ".indexAccess" + + val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l + indexIdentifiers.code shouldBe "myArray" + + val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l + literal1.code shouldBe "\"index1\"" + literal2.code shouldBe "\"index2\"" + } + + "be correct when array access having mixed structure" in { + val cpg = code(""" + |package main + |func main() { + | value := myArray["index2"][index1] + |} + |""".stripMargin) + + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[\"index2\"][index1]" + indexCall.methodFullName shouldBe ".indexAccess" + + val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l + indexIdentifiers.code shouldBe "myArray" + + val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l + literal1.code shouldBe "index1" + literal2.code shouldBe "\"index2\"" + } } From 384f2817ae8c57cc517573f4dd27c294b52719bc Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Mon, 18 Sep 2023 16:44:29 +0530 Subject: [PATCH 2/7] Map access fix --- .../astcreation/AstForPrimitivesCreator.scala | 34 ++++--- .../astcreation/AstForStatementsCreator.scala | 33 ++++++- .../joern/go2cpg/passes/ast/ArraysTests.scala | 99 +++++++++---------- 3 files changed, 95 insertions(+), 71 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala index 80846507f10a..642a37a261a3 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPrimitivesCreator.scala @@ -19,21 +19,25 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t } protected def astForCompositeLiteral(primitive: ParserNodeInfo): Seq[Ast] = { - val typeNode = createParserNodeInfo(primitive.json(ParserKeys.Type)) - typeNode.node match - case ArrayType => - val elementsAsts = Try(primitive.json(ParserKeys.Elts)) match - case Success(value) if !value.isNull => value.arr.flatMap(e => astForNode(createParserNodeInfo(e))).toSeq - case _ => Seq.empty - elementsAsts ++ Seq(astForArrayInitializer(primitive)) - // Handling structure initialisation by creating a call node and arguments - case Ident => - astForConstructorCall(primitive) - // Handling structure initialisation(alias present) by creating a call node and arguments - case SelectorExpr => - astForConstructorCall(primitive) - case _ => - Seq.empty + if (primitive.json.obj.contains(ParserKeys.Type)) { + val typeNode = createParserNodeInfo(primitive.json(ParserKeys.Type)) + typeNode.node match + case ArrayType => + val elementsAsts = Try(primitive.json(ParserKeys.Elts)) match + case Success(value) if !value.isNull => value.arr.flatMap(e => astForNode(createParserNodeInfo(e))).toSeq + case _ => Seq.empty + elementsAsts ++ Seq(astForArrayInitializer(primitive)) + // Handling structure initialisation by creating a call node and arguments + case Ident => + astForConstructorCall(primitive) + // Handling structure initialisation(alias present) by creating a call node and arguments + case SelectorExpr => + astForConstructorCall(primitive) + case _ => + Seq.empty + } else { + Seq.empty + } } private def astForLiteral(stringLiteral: ParserNodeInfo): Ast = { diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala index 24e812936d18..0a14f48b1c91 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala @@ -5,7 +5,7 @@ import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.Operator import io.joern.x2cpg.utils.NodeBuilders.{newCallNode, newOperatorCallNode} import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewCall, NewIdentifier, NewLiteral} +import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewCall, NewIdentifier, NewLiteral, NewLocal} import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} import ujson.Value @@ -259,13 +259,15 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } } def astForIndexStatement(indexNode: ParserNodeInfo): Seq[Ast] = { + val (indexIdentifier, indexTypeFullName) = processIndexIdentifier(indexNode.json) val callNode = newOperatorCallNode( Operators.indexAccess, code = indexNode.code, line = indexNode.lineNumber, - column = indexNode.columnNumber + column = indexNode.columnNumber, + typeFullName = Some(extractBaseType(indexTypeFullName)) ) - Seq(callAst(callNode, processIndexStatementArgs(indexNode.json))) + Seq(callAst(callNode, processIndexStatementArgs(indexNode.json) ++ indexIdentifier)) } private def processIndexStatementArgs(node: Value): Seq[Ast] = { @@ -276,13 +278,36 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t val indexArgsAst = NewLiteral().code(literalValue) Seq(Ast(indexArgsAst)) ++ processIndexStatementArgs(node(ParserKeys.X)) } else { - Seq( + Seq(Ast()) + } + } + + private def processIndexIdentifier(node: Value): (Seq[Ast], String) = { + if (node.obj.contains(ParserKeys.Name)) { + val typeFullName = scope.lookupVariable(node(ParserKeys.Name).str) match + case Some(_, typeName) => typeName + case _ => Defines.anyTypeName + val identifier = Seq( Ast( NewIdentifier() .name(node(ParserKeys.Name).str) .code(node(ParserKeys.Name).str) + .typeFullName(typeFullName) ) ) + (identifier, typeFullName) + } else { + processIndexIdentifier(node(ParserKeys.X)) + } + } + + def extractBaseType(input: String): String = { + if (input.matches("""(\[\])*(\w|\.)+""")) { + input.substring(input.lastIndexOf("]") + 1, input.length) + } else if (input.matches("""map\[(.*?)\]""")) { + input.stripPrefix("map[").stripSuffix("]") + } else { + Defines.anyTypeName } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala index 3e22d8aac593..76529006ac45 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala @@ -410,94 +410,89 @@ class ArraysTests extends GoCodeToCpgSuite { val cpg = code(""" |package main |func main() { - | value := myArray[index] - |} - |""".stripMargin) - val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[index]" - indexCall.methodFullName shouldBe ".indexAccess" - - val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l - indexIdentifiers.code shouldBe "myArray" - - cpg.literal.code("index").size shouldBe 1 - } - - "be correct when array access using string having single index" in { - val cpg = code(""" - |package main - |func main() { - | value := myArray["index"] + | var myArray []int = []int{1, 2} + | value := myArray[0] |} |""".stripMargin) val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[\"index\"]" + indexCall.code shouldBe "myArray[0]" indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "int" - val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l - indexIdentifiers.code shouldBe "myArray" + val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l - cpg.literal.code("\"index\"").size shouldBe 1 + indexIdentifier.code shouldBe "myArray" + indexIdentifier.typeFullName shouldBe "[]int" + + indexLiteral.code shouldBe "0" } - "be correct when array access using variable having multi index" in { + "be correct when map access using string having single index" ignore { val cpg = code(""" |package main |func main() { - | value := myArray[index2][index1] + | var mymap map[string]int = make(map[string]int) + | var a = mymap["key"] |} |""".stripMargin) val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[index2][index1]" + indexCall.code shouldBe "mymap[\"key\"]" indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "string" + + val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l - val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l - indexIdentifiers.code shouldBe "myArray" + indexIdentifier.code shouldBe "mymap" + indexIdentifier.typeFullName shouldBe "map[string]int" - val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l - literal1.code shouldBe "index1" - literal2.code shouldBe "index2" + indexLiteral.code shouldBe "\"key\"" } - "be correct when array access using string having multi index" in { + "be correct when array access using variable having multi index" in { val cpg = code(""" |package main |func main() { - | value := myArray["index2"]["index1"] + | var myArray [][]string = [][]string{{"1", "2"}, {"3", "4"}} + | value := myArray[0][1] |} |""".stripMargin) val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[\"index2\"][\"index1\"]" + indexCall.code shouldBe "myArray[0][1]" indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "string" - val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l - indexIdentifiers.code shouldBe "myArray" - - val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l - literal1.code shouldBe "\"index1\"" - literal2.code shouldBe "\"index2\"" + val List(indexLiteral1: Literal, indexLiteral2: Literal, indexIdentifier: Identifier) = indexCall.argument.l + indexIdentifier.code shouldBe "myArray" + indexIdentifier.typeFullName shouldBe "[][]string" + indexLiteral1.code shouldBe "1" + indexLiteral2.code shouldBe "0" } - "be correct when array access having mixed structure" in { + "be correct when struct array access using variable having single index" in { val cpg = code(""" - |package main - |func main() { - | value := myArray["index2"][index1] - |} - |""".stripMargin) + |package main + |type Person struct { + | FirstName string + |} + | + |func main() { + | var person [3]Person + | var a = person[0] + |} + | + |""".stripMargin) val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[\"index2\"][index1]" + indexCall.code shouldBe "person[0]" indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "main.Person" - val List(indexIdentifiers) = cpg.identifier.name(".*myArray.*").l - indexIdentifiers.code shouldBe "myArray" - - val List(literal1: Literal, literal2: Literal, _) = indexCall.argument.l - literal1.code shouldBe "index1" - literal2.code shouldBe "\"index2\"" + val List(indexLiteral1: Literal, indexIdentifier: Identifier) = indexCall.argument.l + indexIdentifier.code shouldBe "person" + indexIdentifier.typeFullName shouldBe "[]main.Person" + indexLiteral1.code shouldBe "0" } } From 04e7c104dcce3d59a80af587f96ef19b2ba8abfb Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Mon, 18 Sep 2023 17:10:46 +0530 Subject: [PATCH 3/7] file details in the warning log --- .../io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala | 2 +- .../astcreation/AstForMethodCallExpressionCreator.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala index 12e7628ce28e..785992c6c486 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -105,7 +105,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th EvaluationStrategies.BY_SHARING ) case x => - logger.warn(s"Unhandled class ${x.getClass} under getReceiverInfo!") + logger.warn(s"Unhandled class ${x.getClass} under getReceiverInfo! file -> ${parserResult.fullPath}") ("", "") Some(recName, typeFullName, evaluationStrategy, recnode) case _ => None diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala index a95dfe6f4f32..3cc67a7bec8c 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala @@ -50,7 +50,7 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio val xNode = createParserNodeInfo(funcDetails.json(ParserKeys.X)) (Some(xNode), funcDetails.json(ParserKeys.Sel)(ParserKeys.Name).str) case x => - logger.warn(s"Unhandled class ${x.getClass} under astForCallExpression!") + logger.warn(s"Unhandled class ${x.getClass} under astForCallExpression! file -> ${parserResult.fullPath}") (None, "") callMethodFullNameTypeFullNameAndSignature(methodName, aliasOpt) } From f39275680ac7c8791e424732a3bb0e7f04b5bdb7 Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Mon, 18 Sep 2023 18:38:53 +0530 Subject: [PATCH 4/7] Review comment fixes from the PR #3666 Review comment fixes from the PR #3666 --- .../astcreation/AstForFunctionsCreator.scala | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala index 785992c6c486..558f497178b3 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -1,6 +1,5 @@ package io.joern.gosrc2cpg.astcreation -import io.joern.gosrc2cpg.datastructures.GoGlobal import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.datastructures.Stack.* @@ -87,27 +86,26 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th protected def getReceiverInfo(receiver: Try[Value]): Option[(String, String, String, ParserNodeInfo)] = { receiver match case Success(rec) if rec != null => - val recnode = createParserNodeInfo(rec) - val recValue = rec(ParserKeys.List).arr.head - val recName = Try(recValue(ParserKeys.Names).arr.head(ParserKeys.Name).str) match - case Success(recName) => recName - case _ => Defines.This - val typeParserNode = createParserNodeInfo(recValue(ParserKeys.Type)) - val (typeFullName, evaluationStrategy) = typeParserNode.node match - case Ident => - ( - generateTypeFullName(typeName = typeParserNode.json(ParserKeys.Name).strOpt), - EvaluationStrategies.BY_VALUE - ) - case StarExpr => - ( - generateTypeFullName(typeName = typeParserNode.json(ParserKeys.X)(ParserKeys.Name).strOpt), - EvaluationStrategies.BY_SHARING - ) - case x => - logger.warn(s"Unhandled class ${x.getClass} under getReceiverInfo! file -> ${parserResult.fullPath}") - ("", "") - Some(recName, typeFullName, evaluationStrategy, recnode) + val recnode = createParserNodeInfo(rec) + rec(ParserKeys.List).arr.headOption.map(recValue => { + val recName = Try(recValue(ParserKeys.Names).arr.head(ParserKeys.Name).str).getOrElse(Defines.This) + val typeParserNode = createParserNodeInfo(recValue(ParserKeys.Type)) + val (typeFullName, evaluationStrategy) = typeParserNode.node match + case Ident => + ( + generateTypeFullName(typeName = typeParserNode.json(ParserKeys.Name).strOpt), + EvaluationStrategies.BY_VALUE + ) + case StarExpr => + ( + generateTypeFullName(typeName = typeParserNode.json(ParserKeys.X)(ParserKeys.Name).strOpt), + EvaluationStrategies.BY_SHARING + ) + case x => + logger.warn(s"Unhandled class ${x.getClass} under getReceiverInfo! file -> ${parserResult.fullPath}") + ("", "") + (recName, typeFullName, evaluationStrategy, recnode) + }) case _ => None } From 917d84ab17749787b0e60b7fa5192c9f0287fe8a Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Tue, 19 Sep 2023 19:19:12 +0530 Subject: [PATCH 5/7] Index access call node correction and refactor 1. Refactored the existing implementation of array IndexAccess call node. 2. Changed it to handle it recursively creating those many index access call nodes. 3. Updated unit tests to cover more use cases. 4. Ignored IndexAccess unit tests for Map, as it will take more time to handle that scenario. Will be done in separate PR. TODO: Handle indexAccess call node for `Map` type. --- .../gosrc2cpg/astcreation/AstCreator.scala | 1 - .../astcreation/AstForExpressionCreator.scala | 51 ++++- .../astcreation/AstForStatementsCreator.scala | 52 ----- .../io/joern/gosrc2cpg/parser/ParserAst.scala | 3 +- .../go2cpg/dataflow/ArrayDataflowTests.scala | 27 +++ ...aysTests.scala => ArraysAndMapTests.scala} | 198 ++++++++++++++---- .../go2cpg/passes/ast/TypeFullNameTests.scala | 61 +++++- 7 files changed, 285 insertions(+), 108 deletions(-) rename joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/{ArraysTests.scala => ArraysAndMapTests.scala} (72%) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala index bdf083448d1d..cf3e67028365 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala @@ -91,7 +91,6 @@ class AstCreator(val relPathFileName: String, val parserResult: ParserResult)(im nodeInfo.node match { case GenDecl => astForGenDecl(nodeInfo) case FuncDecl => astForFuncDecl(nodeInfo) - case IndexExpr => astForIndexStatement(nodeInfo) case _: BasePrimitive => astForPrimitive(nodeInfo) case _: BaseExpr => astsForExpression(nodeInfo) case _: BaseStmt => astsForStatement(nodeInfo) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala index cf14e9edfa34..803b38b33518 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala @@ -3,13 +3,12 @@ package io.joern.gosrc2cpg.astcreation import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.Operator -import io.joern.x2cpg.utils.NodeBuilders.* import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewFieldIdentifier} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} +import ujson.Value import scala.collection.immutable.Seq - trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => def astsForExpression(expr: ParserNodeInfo): Seq[Ast] = { expr.node match { @@ -17,11 +16,11 @@ trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { t case StarExpr => astForStarExpr(expr) case UnaryExpr => astForUnaryExpr(expr) case ParenExpr => astsForExpression(createParserNodeInfo(expr.json(ParserKeys.X))) - case StructType => astForStructType(expr) case TypeAssertExpr => astForNode(expr.json(ParserKeys.X)) case CallExpr => astForCallExpression(expr) case SelectorExpr => astForFieldAccess(expr) case KeyValueExpr => astForNode(createParserNodeInfo(expr.json(ParserKeys.Value))) + case IndexExpr => astForIndexExpression(expr) case _ => Seq(Ast()) } } @@ -54,8 +53,12 @@ trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { t } private def astForStarExpr(starExpr: ParserNodeInfo): Seq[Ast] = { - val cNode = createCallNodeForOperator(starExpr, Operators.indirection) val operand = astForNode(starExpr.json(ParserKeys.X)) + val typeFullName = operand.head.root.get.properties + .get(PropertyNames.TYPE_FULL_NAME) + .getOrElse(Defines.anyTypeName) + .toString + val cNode = createCallNodeForOperator(starExpr, Operators.indirection, typeFullName = Some(typeFullName)) Seq(callAst(cNode, operand)) } @@ -75,11 +78,43 @@ trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { t Seq(callAst(cNode, operand)) } + private def astForIndexExpression(indexNode: ParserNodeInfo): Seq[Ast] = { + val indexAst = astForNode(indexNode.json(ParserKeys.Index)) + val (indexIdentifier, callNodeTypeFullName) = processIndexIdentifier(indexNode.json(ParserKeys.X)) + val callNode = + createCallNodeForOperator(indexNode, Operators.indexAccess, typeFullName = Some(callNodeTypeFullName)) + Seq(callAst(callNode, indexIdentifier ++ indexAst)) + } + + private def processIndexIdentifier(identNode: Value): (Seq[Ast], String) = { + val identifierAst = astForNode(identNode) + val identifierTypeFullName = + identifierAst.head.root.get.properties + .get(PropertyNames.TYPE_FULL_NAME) + .getOrElse(Defines.anyTypeName) + .toString + .stripPrefix("*") + .stripPrefix("[]") + (identifierAst, identifierTypeFullName) + } +// +// private def extractBaseType(input: String): String = { +// if (input.matches("""(\[\])*(\w|\.)+""")) { +// input.substring(input.lastIndexOf("]") + 1, input.length) +// } else if (input.matches("""map\[(.*?)\]""")) { +// input.stripPrefix("map[").stripSuffix("]") +// } else { +// Defines.anyTypeName +// } +// } + private def createCallNodeForOperator( node: ParserNodeInfo, operatorMethod: String, - DispatchType: String = DispatchTypes.STATIC_DISPATCH + DispatchType: String = DispatchTypes.STATIC_DISPATCH, + signature: Option[String] = None, + typeFullName: Option[String] = None ): NewCall = { - callNode(node, node.code, operatorMethod, operatorMethod, DispatchType) + callNode(node, node.code, operatorMethod, operatorMethod, DispatchType, signature, typeFullName) } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala index 0a14f48b1c91..cfcf58b3aecc 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForStatementsCreator.scala @@ -258,56 +258,4 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t Ast() } } - def astForIndexStatement(indexNode: ParserNodeInfo): Seq[Ast] = { - val (indexIdentifier, indexTypeFullName) = processIndexIdentifier(indexNode.json) - val callNode = newOperatorCallNode( - Operators.indexAccess, - code = indexNode.code, - line = indexNode.lineNumber, - column = indexNode.columnNumber, - typeFullName = Some(extractBaseType(indexTypeFullName)) - ) - Seq(callAst(callNode, processIndexStatementArgs(indexNode.json) ++ indexIdentifier)) - } - - private def processIndexStatementArgs(node: Value): Seq[Ast] = { - if (node.obj.contains(ParserKeys.Index)) { - val indexJson = node(ParserKeys.Index) - val literalValue = - if (indexJson.obj.contains(ParserKeys.Name)) indexJson(ParserKeys.Name).str else indexJson(ParserKeys.Value).str - val indexArgsAst = NewLiteral().code(literalValue) - Seq(Ast(indexArgsAst)) ++ processIndexStatementArgs(node(ParserKeys.X)) - } else { - Seq(Ast()) - } - } - - private def processIndexIdentifier(node: Value): (Seq[Ast], String) = { - if (node.obj.contains(ParserKeys.Name)) { - val typeFullName = scope.lookupVariable(node(ParserKeys.Name).str) match - case Some(_, typeName) => typeName - case _ => Defines.anyTypeName - val identifier = Seq( - Ast( - NewIdentifier() - .name(node(ParserKeys.Name).str) - .code(node(ParserKeys.Name).str) - .typeFullName(typeFullName) - ) - ) - (identifier, typeFullName) - } else { - processIndexIdentifier(node(ParserKeys.X)) - } - } - - def extractBaseType(input: String): String = { - if (input.matches("""(\[\])*(\w|\.)+""")) { - input.substring(input.lastIndexOf("]") + 1, input.length) - } else if (input.matches("""map\[(.*?)\]""")) { - input.stripPrefix("map[").stripSuffix("]") - } else { - Defines.anyTypeName - } - } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala index 2c3d6e62abf8..5f10ffe62c1b 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/ParserAst.scala @@ -33,6 +33,7 @@ object ParserAst { object SelectorExpr extends BaseExpr object CallExpr extends BaseExpr object StructType extends BaseExpr + object IndexExpr extends BaseExpr sealed trait BaseStmt extends ParserNode object BlockStmt extends BaseStmt object DeclStmt extends BaseStmt @@ -65,9 +66,7 @@ object ParserAst { object ArrayType extends ParserNode object MapType extends ParserNode object ChanType extends ParserNode - object Field extends ParserNode object TypeSpec extends ParserNode - object IndexExpr extends ParserNode } object ParserKeys { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ArrayDataflowTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ArrayDataflowTests.scala index a94aaeca2cc0..de9398abaeae 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ArrayDataflowTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ArrayDataflowTests.scala @@ -47,5 +47,32 @@ class ArrayDataflowTests extends GoCodeToCpgSuite(withOssDataflow = true) { sink.reachableByFlows(source).size shouldBe 2 } } + "Data flows around multidimentional Arrays" should { + val cpg = code(""" + |package main + |func main() { + | var myArray [][]string = [][]string{{"1", "2"}, {"3", "4"}} + | first := myArray[0][1] + | second := myArray[1][0] + | third := myArray + |} + |""".stripMargin) + "data flow check one" ignore { + val src = cpg.identifier("myArray").l + val sink = cpg.identifier("first") + sink.reachableByFlows(src).size shouldBe 2 + } + "data flow check two" ignore { + val src = cpg.literal("\"2\"").l + val sink = cpg.identifier("first") + sink.reachableByFlows(src).size shouldBe 1 + } + + "data flow check three" in { + val src = cpg.identifier("myArray").l + val sink = cpg.identifier("third") + sink.reachableByFlows(src).size shouldBe 2 + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala similarity index 72% rename from joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala rename to joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala index 76529006ac45..4fa165b21b87 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala @@ -6,13 +6,13 @@ import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Dispatch import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes -class ArraysTests extends GoCodeToCpgSuite { +class ArraysAndMapTests extends GoCodeToCpgSuite { "AST Creation for Array Initialization" should { "be correct when a int array is declared" in { val cpg = code(""" |package main |func main() { - | var a []int + | var a []int |} |""".stripMargin) cpg.local("a").size shouldBe 1 @@ -406,7 +406,7 @@ class ArraysTests extends GoCodeToCpgSuite { identifierNode.code shouldBe "c" } - "be correct when array access using variable having single index" in { + "be correct when array access using variable having single index" should { val cpg = code(""" |package main |func main() { @@ -415,42 +415,91 @@ class ArraysTests extends GoCodeToCpgSuite { |} |""".stripMargin) - val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[0]" - indexCall.methodFullName shouldBe ".indexAccess" - indexCall.typeFullName shouldBe "int" + "be correct for local and identifier node" in { + val List(myArr, _) = cpg.local.l + myArr.typeFullName shouldBe "[]int" - val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l + val List(identifier, _, _) = cpg.identifier.l + identifier.typeFullName shouldBe "[]int" + } + "be correct for indexAccess call node" in { + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[0]" + indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "int" + + val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l - indexIdentifier.code shouldBe "myArray" - indexIdentifier.typeFullName shouldBe "[]int" + indexIdentifier.code shouldBe "myArray" + indexIdentifier.typeFullName shouldBe "[]int" - indexLiteral.code shouldBe "0" + indexLiteral.code shouldBe "0" + } } - "be correct when map access using string having single index" ignore { + "be correct when array of pointer access using variable having single index" should { val cpg = code(""" |package main |func main() { - | var mymap map[string]int = make(map[string]int) - | var a = mymap["key"] + | var myArray []*int + | value := myArray[0] + |} + |""".stripMargin) + + "be correct for local and identifier node" in { + val List(myArr, _) = cpg.local.l + myArr.typeFullName shouldBe "[]*int" + + val List(identifier, _, _) = cpg.identifier.l + identifier.typeFullName shouldBe "[]*int" + } + "be correct for indexAccess call node" in { + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "myArray[0]" + indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "*int" + + val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l + + indexIdentifier.code shouldBe "myArray" + indexIdentifier.typeFullName shouldBe "[]*int" + + indexLiteral.code shouldBe "0" + } + } + + "be correct when pointer of array access using variable having single index" should { + val cpg = code(""" + |package main + |func main() { + | var myArray *[]int + | value := (*myArray)[0] |} |""".stripMargin) - val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "mymap[\"key\"]" - indexCall.methodFullName shouldBe ".indexAccess" - indexCall.typeFullName shouldBe "string" + "be correct for local and identifier node" in { + val List(myArr, _) = cpg.local.l + myArr.typeFullName shouldBe "*[]int" - val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l + val List(identifier, _, _) = cpg.identifier.l + identifier.typeFullName shouldBe "*[]int" + } + "be correct for indexAccess call node" in { + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "(*myArray)[0]" + indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "int" + + val List(indexIdentifier: Call, indexLiteral: Literal) = indexCall.argument.l - indexIdentifier.code shouldBe "mymap" - indexIdentifier.typeFullName shouldBe "map[string]int" + indexIdentifier.code shouldBe "*myArray" + indexIdentifier.typeFullName shouldBe "*[]int" - indexLiteral.code shouldBe "\"key\"" + indexLiteral.code shouldBe "0" + } } - "be correct when array access using variable having multi index" in { + "be correct when array access using variable having multi index" should { val cpg = code(""" |package main |func main() { @@ -458,41 +507,102 @@ class ArraysTests extends GoCodeToCpgSuite { | value := myArray[0][1] |} |""".stripMargin) + "Be correct for local and Identifier nodes" in { + val List(myArr, _) = cpg.local.l + myArr.typeFullName shouldBe "[][]string" - val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "myArray[0][1]" - indexCall.methodFullName shouldBe ".indexAccess" - indexCall.typeFullName shouldBe "string" + val List(identifier, _, _) = cpg.identifier.l + identifier.typeFullName shouldBe "[][]string" + } + "Be correct for IndexAccess call nodes" in { + val List(indexCallFirst, _) = cpg.call.name(Operators.indexAccess).l + indexCallFirst.code shouldBe "myArray[0][1]" + indexCallFirst.typeFullName shouldBe "string" + + val List(indexIdentifier: Call, indexLiteral1: Literal) = indexCallFirst.argument.l + indexIdentifier.code shouldBe "myArray[0]" + indexIdentifier.typeFullName shouldBe "[]string" + indexLiteral1.code shouldBe "1" + + val List(indexIdentifierTwo: Identifier, indexLiteral2: Literal) = indexIdentifier.argument.l + indexIdentifierTwo.code shouldBe "myArray" + indexIdentifierTwo.typeFullName shouldBe "[][]string" + indexLiteral2.code shouldBe "0" + } + } + + "be correct when map access using string having single index" ignore { + val cpg = code(""" + |package main + |func main() { + | var mymap map[string]int = make(map[string]int) + | var a = mymap["key"] + |} + |""".stripMargin) + "be correct for local node properties" in { + val List(x) = cpg.local("mymap").l + x.typeFullName shouldBe "map[]int" + } + + "be correct for index access" in { + val List(indexCall) = cpg.call.name(".indexAccess").l + indexCall.code shouldBe "mymap[\"key\"]" + indexCall.methodFullName shouldBe ".indexAccess" + indexCall.typeFullName shouldBe "int" + + val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l - val List(indexLiteral1: Literal, indexLiteral2: Literal, indexIdentifier: Identifier) = indexCall.argument.l - indexIdentifier.code shouldBe "myArray" - indexIdentifier.typeFullName shouldBe "[][]string" - indexLiteral1.code shouldBe "1" - indexLiteral2.code shouldBe "0" + indexIdentifier.code shouldBe "mymap" + indexIdentifier.typeFullName shouldBe "map[]int" + + indexLiteral.code shouldBe "\"key\"" + indexLiteral.typeFullName shouldBe "string" + } } - "be correct when struct array access using variable having single index" in { + "be correct when struct array access using variable having single index" should { val cpg = code(""" |package main |type Person struct { - | FirstName string + | Fname string + | Lname string + |} + |func (p Person) fullName() string { + | return p.Fname + " " + p.Lname |} - | |func main() { | var person [3]Person | var a = person[0] + | var flname = person[0].fullName() |} - | |""".stripMargin) - val List(indexCall) = cpg.call.name(".indexAccess").l - indexCall.code shouldBe "person[0]" - indexCall.methodFullName shouldBe ".indexAccess" - indexCall.typeFullName shouldBe "main.Person" + "local and identifer checks" in { + val List(person, _, _) = cpg.local.l + person.typeFullName shouldBe "[]main.Person" + } + + "index access call check" in { + val List(indexOne, indexTwo) = cpg.call.name(Operators.indexAccess).l + indexOne.code shouldBe "person[0]" + indexOne.methodFullName shouldBe Operators.indexAccess + indexOne.typeFullName shouldBe "main.Person" + + val List(indexIdentifier: Identifier, indexLiteral1: Literal) = indexOne.argument.l + indexIdentifier.code shouldBe "person" + indexIdentifier.typeFullName shouldBe "[]main.Person" + indexLiteral1.code shouldBe "0" + } + + "be correct for method call node on array index access" in { + val List(fullName) = cpg.call("fullName").l + fullName.methodFullName shouldBe "main.Person.fullName" + fullName.typeFullName shouldBe "string" - val List(indexLiteral1: Literal, indexIdentifier: Identifier) = indexCall.argument.l - indexIdentifier.code shouldBe "person" - indexIdentifier.typeFullName shouldBe "[]main.Person" - indexLiteral1.code shouldBe "0" + val List(args: Call) = fullName.argument.l + args.argumentIndex shouldBe 0 + args.name shouldBe Operators.indexAccess + args.typeFullName shouldBe "main.Person" + } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala index 8995669fbb92..38bc6e08f2bd 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeFullNameTests.scala @@ -1,7 +1,6 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.joern.gosrc2cpg.astcreation.Defines import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* @@ -554,6 +553,66 @@ class TypeFullNameTests extends GoCodeToCpgSuite { } } + "Type Check for Arrays" should { + val cpg = code(""" + |package main + |func main() { + | var a [][]string = [][]string{{"1", "2"}, {"3", "4"}} + | b := a[0][1] + | var c = [][]string{{"1", "2"}, {"3", "4"}} + | var d = c[1] + | var e = d[0] + | var f = c[0][0] + | var g []int = []int{1,2,3} + | var h = g[0] + | i := g + | j := []float32{ 1.2, 2.5} + | var k = i[0] + | var l = j[0] + |} + |""".stripMargin) + "Type Check LOCAL nodes working" in { + val List(a, b, c, d, e, f, g, h, i, j, k, l) = cpg.local.l + a.typeFullName shouldBe "[][]string" + g.typeFullName shouldBe "[]int" + } + + "Type Check LOCAL nodes non working" ignore { + val List(a, b, c, d, e, f, g, h, i, j, k, l) = cpg.local.l + b.typeFullName shouldBe "string" + c.typeFullName shouldBe "[][]string" + d.typeFullName shouldBe "[]string" + e.typeFullName shouldBe "string" + f.typeFullName shouldBe "string" + h.typeFullName shouldBe "int" + i.typeFullName shouldBe "[]int" + j.typeFullName shouldBe "[]float32" + k.typeFullName shouldBe "int" + l.typeFullName shouldBe "float32" + } + + "Type check for CALL nodes working" in { + val List(a, b, c, d, e, f, g, h, i, j, k, l, m) = cpg.call.nameNot(Operators.assignment).l + a.typeFullName shouldBe "[][]string" + b.typeFullName shouldBe "string" + c.typeFullName shouldBe "[]string" + d.typeFullName shouldBe "[][]string" + i.typeFullName shouldBe "[]int" + j.typeFullName shouldBe "int" + k.typeFullName shouldBe "[]float32" + } + + "Type check for CALL nodes non working" ignore { + val List(a, b, c, d, e, f, g, h, i, j, k, l, m) = cpg.call.nameNot(Operators.assignment).l + e.typeFullName shouldBe "[]string" + f.typeFullName shouldBe "string" + g.typeFullName shouldBe "string" + h.typeFullName shouldBe "[]string" + l.typeFullName shouldBe "int" + m.typeFullName shouldBe "float32" + } + } + "Method call return value assigned to variable type check" should { val cpg = code( """ From 787e74b8cae961d8351a50126a9dc2bfbc19e6d3 Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Tue, 19 Sep 2023 19:36:18 +0530 Subject: [PATCH 6/7] compilation warning fixes --- .../go2cpg/passes/ast/ArraysAndMapTests.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala index 4fa165b21b87..5dbef450528b 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ArraysAndMapTests.scala @@ -428,7 +428,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexCall.methodFullName shouldBe ".indexAccess" indexCall.typeFullName shouldBe "int" - val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l + val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l: @unchecked indexIdentifier.code shouldBe "myArray" indexIdentifier.typeFullName shouldBe "[]int" @@ -459,7 +459,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexCall.methodFullName shouldBe ".indexAccess" indexCall.typeFullName shouldBe "*int" - val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l + val List(indexIdentifier: Identifier, indexLiteral: Literal) = indexCall.argument.l: @unchecked indexIdentifier.code shouldBe "myArray" indexIdentifier.typeFullName shouldBe "[]*int" @@ -490,7 +490,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexCall.methodFullName shouldBe ".indexAccess" indexCall.typeFullName shouldBe "int" - val List(indexIdentifier: Call, indexLiteral: Literal) = indexCall.argument.l + val List(indexIdentifier: Call, indexLiteral: Literal) = indexCall.argument.l: @unchecked indexIdentifier.code shouldBe "*myArray" indexIdentifier.typeFullName shouldBe "*[]int" @@ -519,12 +519,12 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexCallFirst.code shouldBe "myArray[0][1]" indexCallFirst.typeFullName shouldBe "string" - val List(indexIdentifier: Call, indexLiteral1: Literal) = indexCallFirst.argument.l + val List(indexIdentifier: Call, indexLiteral1: Literal) = indexCallFirst.argument.l: @unchecked indexIdentifier.code shouldBe "myArray[0]" indexIdentifier.typeFullName shouldBe "[]string" indexLiteral1.code shouldBe "1" - val List(indexIdentifierTwo: Identifier, indexLiteral2: Literal) = indexIdentifier.argument.l + val List(indexIdentifierTwo: Identifier, indexLiteral2: Literal) = indexIdentifier.argument.l: @unchecked indexIdentifierTwo.code shouldBe "myArray" indexIdentifierTwo.typeFullName shouldBe "[][]string" indexLiteral2.code shouldBe "0" @@ -550,7 +550,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexCall.methodFullName shouldBe ".indexAccess" indexCall.typeFullName shouldBe "int" - val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l + val List(indexLiteral: Literal, indexIdentifier: Identifier) = indexCall.argument.l: @unchecked indexIdentifier.code shouldBe "mymap" indexIdentifier.typeFullName shouldBe "map[]int" @@ -588,7 +588,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { indexOne.methodFullName shouldBe Operators.indexAccess indexOne.typeFullName shouldBe "main.Person" - val List(indexIdentifier: Identifier, indexLiteral1: Literal) = indexOne.argument.l + val List(indexIdentifier: Identifier, indexLiteral1: Literal) = indexOne.argument.l: @unchecked indexIdentifier.code shouldBe "person" indexIdentifier.typeFullName shouldBe "[]main.Person" indexLiteral1.code shouldBe "0" @@ -599,7 +599,7 @@ class ArraysAndMapTests extends GoCodeToCpgSuite { fullName.methodFullName shouldBe "main.Person.fullName" fullName.typeFullName shouldBe "string" - val List(args: Call) = fullName.argument.l + val List(args: Call) = fullName.argument.l: @unchecked args.argumentIndex shouldBe 0 args.name shouldBe Operators.indexAccess args.typeFullName shouldBe "main.Person" From 9051bb8024250c25baa93ded2fe3d23f81cfea01 Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Tue, 19 Sep 2023 19:45:13 +0530 Subject: [PATCH 7/7] review comment fixes --- .../astcreation/AstForExpressionCreator.scala | 14 +++++++------- .../AstForMethodCallExpressionCreator.scala | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala index 803b38b33518..5a5d333a6ff0 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForExpressionCreator.scala @@ -4,7 +4,7 @@ import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.Operator import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier} +import io.shiftleft.codepropertygraph.generated.nodes.NewCall import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} import ujson.Value @@ -54,10 +54,10 @@ trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { t private def astForStarExpr(starExpr: ParserNodeInfo): Seq[Ast] = { val operand = astForNode(starExpr.json(ParserKeys.X)) - val typeFullName = operand.head.root.get.properties - .get(PropertyNames.TYPE_FULL_NAME) + val typeFullName = operand.headOption + .flatMap(_.root) + .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString) .getOrElse(Defines.anyTypeName) - .toString val cNode = createCallNodeForOperator(starExpr, Operators.indirection, typeFullName = Some(typeFullName)) Seq(callAst(cNode, operand)) } @@ -89,10 +89,10 @@ trait AstForExpressionCreator(implicit withSchemaValidation: ValidationMode) { t private def processIndexIdentifier(identNode: Value): (Seq[Ast], String) = { val identifierAst = astForNode(identNode) val identifierTypeFullName = - identifierAst.head.root.get.properties - .get(PropertyNames.TYPE_FULL_NAME) + identifierAst.headOption + .flatMap(_.root) + .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString) .getOrElse(Defines.anyTypeName) - .toString .stripPrefix("*") .stripPrefix("[]") (identifierAst, identifierTypeFullName) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala index 3cc67a7bec8c..b4666c4d4d37 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForMethodCallExpressionCreator.scala @@ -128,10 +128,10 @@ trait AstForMethodCallExpressionCreator(implicit withSchemaValidation: Validatio ): (String, String, String, String, Seq[Ast]) = { val receiverAst = astForNode(xnode) val receiverTypeFullName = - receiverAst.head.root.get.properties - .get(PropertyNames.TYPE_FULL_NAME) + receiverAst.headOption + .flatMap(_.root) + .map(_.properties.get(PropertyNames.TYPE_FULL_NAME).get.toString) .getOrElse(Defines.anyTypeName) - .toString .stripPrefix("*") val callMethodFullName = s"$receiverTypeFullName.$methodName" val (returnTypeFullNameCache, signatureCache) =