From 965b7695decf73eca19576004f3dd4c83a101e04 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Thu, 4 Jan 2024 16:51:45 +0530 Subject: [PATCH] handle basic operators --- .../src/main/resources/application.conf | 2 +- .../astcreation/AstCreator.scala | 1 + .../AstForDeclarationsCreator.scala | 6 +- .../AstForExpressionsCreator.scala | 76 ++++++++- .../csharpsrc2cpg/parser/DotNetJsonAst.scala | 47 +++++- .../passes/ast/OperatorsTests.scala | 148 ++++++++++++++++++ .../joern/x2cpg/utils/ExternalCommand.scala | 2 +- 7 files changed, 276 insertions(+), 6 deletions(-) create mode 100644 joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/passes/ast/OperatorsTests.scala diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf b/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf index 1434213aef66..08ef9dc66658 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf @@ -1,3 +1,3 @@ csharpsrc2cpg { - dotnetastgen_version: "0.11.0" + dotnetastgen_version: "0.12.0" } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala index f7ad140f7fd4..077ed57485c9 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala @@ -61,6 +61,7 @@ class AstCreator(val relativeFileName: String, val parserResult: ParserResult, v case EqualsValueClause => astForEqualsValueClause(nodeInfo) case UsingDirective => notHandledYet(nodeInfo) case Block => notHandledYet(nodeInfo) + case ExpressionStatement => astForExpression(nodeInfo) case _ => notHandledYet(nodeInfo) } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala index e9b9655791c4..61f8c856dca8 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala @@ -131,8 +131,10 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) { } private def astForMethodBody(body: DotNetNodeInfo): Ast = { - val block = blockNode(body) - val statements = List.empty // TODO + val block = blockNode(body) + methodAstParentStack.push(block) + val statements = body.json(ParserKeys.Statements).arr.flatMap(astForNode).toList + methodAstParentStack.pop() blockAst(block, statements) } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForExpressionsCreator.scala index 9417c0011b68..10c7c6a1865c 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -1,11 +1,75 @@ package io.joern.csharpsrc2cpg.astcreation -import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.LiteralExpr +import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.csharpsrc2cpg.parser.{DotNetNodeInfo, ParserKeys} import io.joern.x2cpg.{Ast, ValidationMode} +import io.shiftleft.codepropertygraph.generated.nodes.NewCall +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + def astForExpression(expr: DotNetNodeInfo): Seq[Ast] = { + // TODO: Handle identifiers in operators + val expressionNode = createDotNetNodeInfo(expr.json(ParserKeys.Expression)) + expressionNode.node match + case _: UnaryExpr => astForUnaryExpression(expressionNode) + case _: BinaryExpr => astForBinaryExpression(expressionNode) + case _ => notHandledYet(expressionNode) + } + + private def astForUnaryExpression(unaryExpr: DotNetNodeInfo): Seq[Ast] = { + val operatorToken = unaryExpr.json(ParserKeys.OperatorToken)(ParserKeys.Value).toString.replaceAll("\"", "") + val operatorName = operatorToken match + case "+" => Operators.plus + case "-" => Operators.minus + case "++" => + if (unaryExpr.node.getClass == PostIncrementExpression.getClass) Operators.postIncrement + else Operators.preIncrement + case "--" => + if (unaryExpr.node.getClass == PostDecrementExpression.getClass) Operators.postDecrement + else Operators.preDecrement + case "~" => Operators.not + case "!" => Operators.logicalNot + case "&" => Operators.addressOf + + Seq(callAst(createCallNodeForOperator(unaryExpr, operatorName, typeFullName = Some("")))) // TODO: typeFullName + } + private def astForBinaryExpression(binaryExpr: DotNetNodeInfo): Seq[Ast] = { + val operatorToken = binaryExpr.json(ParserKeys.OperatorToken)(ParserKeys.Value).toString.replaceAll("\"", "") + val operatorName = operatorToken match + case "+" => Operators.addition + case "-" => Operators.subtraction + case "*" => Operators.multiplication + case "/" => Operators.division + case "%" => Operators.modulo + case "==" => Operators.equals + case "!=" => Operators.notEquals + case "&&" => Operators.logicalAnd + case "||" => Operators.logicalOr + case "+=" => Operators.assignmentPlus + case "-=" => Operators.assignmentMinus + case "*=" => Operators.assignmentMultiplication + case "/=" => Operators.assignmentDivision + case "%=" => Operators.assignmentModulo + case "&=" => Operators.assignmentAnd + case "|=" => Operators.assignmentOr + case "^=" => Operators.assignmentXor + case ">>=" => Operators.assignmentLogicalShiftRight + case "<<=" => Operators.assignmentShiftLeft + case ">" => Operators.greaterThan + case "<" => Operators.lessThan + case ">=" => Operators.greaterEqualsThan + case "<=" => Operators.lessEqualsThan + case "|" => Operators.or + case "&" => Operators.and + case "^" => Operators.xor + + val args = astForNode(binaryExpr.json(ParserKeys.Left)) ++: astForNode(binaryExpr.json(ParserKeys.Right)) + val cNode = + createCallNodeForOperator(binaryExpr, operatorName, typeFullName = Some("")) // TODO: Resolve typeFullName + Seq(callAst(cNode, args)) + } + protected def astForEqualsValueClause(clause: DotNetNodeInfo): Seq[Ast] = { val rhsNode = createDotNetNodeInfo(clause.json(ParserKeys.Value)) rhsNode.node match @@ -13,4 +77,14 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case _ => notHandledYet(rhsNode) } + private def createCallNodeForOperator( + node: DotNetNodeInfo, + operatorMethod: String, + DispatchType: String = DispatchTypes.STATIC_DISPATCH, + signature: Option[String] = None, + typeFullName: Option[String] = None + ): NewCall = { + callNode(node, node.code, operatorMethod, operatorMethod, DispatchType, signature, typeFullName) + } + } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala index 97802495af58..0825448d0cf1 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala @@ -1,5 +1,6 @@ package io.joern.csharpsrc2cpg.parser +import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.BaseExpr import org.slf4j.LoggerFactory object DotNetJsonAst { @@ -26,7 +27,8 @@ object DotNetJsonAst { sealed trait BaseExpr extends DotNetParserNode - object NotHandledType extends DotNetParserNode + object ExpressionStatement extends BaseExpr + object NotHandledType extends DotNetParserNode object CompilationUnit extends BaseExpr @@ -70,6 +72,45 @@ object DotNetJsonAst { object QualifiedName extends IdentifierNode + sealed trait UnaryExpr extends BaseExpr + object PostIncrementExpression extends UnaryExpr + object PostDecrementExpression extends UnaryExpr + object PreIncrementExpression extends UnaryExpr + object PreDecrementExpression extends UnaryExpr + object UnaryPlusExpression extends UnaryExpr + object UnaryMinusExpression extends UnaryExpr + object BitwiseNotExpression extends UnaryExpr + object LogicalNotExpression extends UnaryExpr + object AddressOfExpression extends UnaryExpr + + sealed trait BinaryExpr extends BaseExpr + object AddExpression extends BinaryExpr + object SubtractExpression extends BinaryExpr + object MultiplyExpression extends BinaryExpr + object DivideExpression extends BinaryExpr + object ModuloExpression extends BinaryExpr + object EqualsExpression extends BinaryExpr + object NotEqualsExpression extends BinaryExpr + object LogicalAndExpression extends BinaryExpr + object LogicalOrExpression extends BinaryExpr + object AddAssignmentExpression extends BinaryExpr + object SubtractAssignmentExpression extends BinaryExpr + object MultiplyAssignmentExpression extends BinaryExpr + object DivideAssignmentExpression extends BinaryExpr + object ModuloAssignmentExpression extends BinaryExpr + object AndAssignmentExpression extends BinaryExpr + object OrAssignmentExpression extends BinaryExpr + object ExclusiveOrAssignmentExpression extends BinaryExpr + object RightShiftAssignmentExpression extends BinaryExpr + object LeftShiftAssignmentExpression extends BinaryExpr + + object GreaterThanExpression extends BinaryExpr + object LessThanExpression extends BinaryExpr + object GreaterThanOrEqualExpression extends BinaryExpr + object LessThanOrEqualExpression extends BinaryExpr + object BitwiseAndExpression extends BinaryExpr + object BitwiseOrExpression extends BinaryExpr + object ExclusiveOrExpression extends BinaryExpr } /** The JSON key values, in alphabetical order. @@ -103,5 +144,9 @@ object ParserKeys { val Usings = "Usings" val Value = "Value" val Variables = "Variables" + val Statements = "Statements" + val Expression = "Expression" + val OperatorToken = "OperatorToken" + val Operand = "Operand" } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/passes/ast/OperatorsTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/passes/ast/OperatorsTests.scala new file mode 100644 index 000000000000..4a7c77bd9a62 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/passes/ast/OperatorsTests.scala @@ -0,0 +1,148 @@ +package io.joern.csharpsrc2cpg.passes.ast + +import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.LiteralExpr +import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.Identifier +import io.shiftleft.semanticcpg.language.* + +class OperatorsTests extends CSharpCode2CpgFixture { + "AST nodes for operators" should { + "be created for unary operators" in { + val cpg = code( + basicBoilerplate(""" + |int i = 3; + |i++; + |i--; + |++i; + |--i; + |!i; + |~i; + |+5; + |-5; + |&i; + |""".stripMargin), + "Program.cs" + ) + + val operatorCalls = cpg.method("Main").ast.isCall.nameNot(Operators.assignment).l + operatorCalls.size shouldBe 9 + operatorCalls.name.l shouldBe List( + ".postIncrement", + ".postDecrement", + ".preIncrement", + ".preDecrement", + ".logicalNot", + ".not", + ".plus", + ".minus", + ".addressOf" + ) + operatorCalls.code.l shouldBe List("i++", "i--", "++i", "--i", "!i", "~i", "+5", "-5", "&i") + // TODO: Tests for operands + } + } + + "be created for binary operators" in { + val cpg = code( + basicBoilerplate(""" + |int a = 3; + |int b = 5; + |a+b; + |a-b; + |a/b; + |a%b; + |a==b; + |a!=b; + |a&&b; + |a||b; + |a&b; + |a|b; + |a^b; + |""".stripMargin), + fileName = "Program.cs" + ) + val operatorCalls = cpg.method("Main").ast.isCall.nameNot(Operators.assignment).l + operatorCalls.size shouldBe 11 + operatorCalls.name.l shouldBe List( + ".addition", + ".subtraction", + ".division", + ".modulo", + ".equals", + ".notEquals", + ".logicalAnd", + ".logicalOr", + ".and", + ".or", + ".xor" + ) + operatorCalls.code.l shouldBe List("a+b", "a-b", "a/b", "a%b", "a==b", "a!=b", "a&&b", "a||b", "a&b", "a|b", "a^b") + + // TODO: Tests for operands + } + + "be created for shorthand assignment operators" in { + val cpg = code( + basicBoilerplate(""" + |int a = 3; + |int b = 5; + |a+=b; + |a-=b; + |a*=b; + |a/=b; + |a%=b; + |a&=b; + |a|=b; + |a^=b; + |a>>=b; + |a<<=b; + |""".stripMargin), + fileName = "Program.cs" + ) + val operatorCalls = cpg.method("Main").ast.isCall.nameNot(Operators.assignment).l + operatorCalls.size shouldBe 10 + operatorCalls.name.l shouldBe List( + ".assignmentPlus", + ".assignmentMinus", + ".assignmentMultiplication", + ".assignmentDivision", + ".assignmentModulo", + ".assignmentAnd", + ".assignmentOr", + ".assignmentXor", + ".assignmentLogicalShiftRight", + ".assignmentShiftLeft" + ) + operatorCalls.code.l shouldBe List("a+=b", "a-=b", "a*=b", "a/=b", "a%=b", "a&=b", "a|=b", "a^=b", "a>>=b", "a<<=b") + + // TODO: Tests for operands + } + + "be created for comparison operators" in { + val cpg = code( + basicBoilerplate(""" + |int a = 3; + |int b = 5; + |a > b; + |a < b; + |a == b; + |a >= b; + |a <= b; + |""".stripMargin), + fileName = "Program.cs" + ) + val operatorCalls = cpg.method("Main").ast.isCall.nameNot(Operators.assignment).l + operatorCalls.size shouldBe 5 + operatorCalls.name.l shouldBe List( + ".greaterThan", + ".lessThan", + ".equals", + ".greaterEqualsThan", + ".lessEqualsThan" + ) + operatorCalls.code.l shouldBe List("a > b", "a < b", "a == b", "a >= b", "a <= b") + + // TODO: Tests for operands + } +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala index 9e830aad5abc..2304fee2dbff 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala @@ -10,7 +10,7 @@ object ExternalCommand { private val IS_WIN: Boolean = scala.util.Properties.isWin - private val shellPrefix: Seq[String] = if (IS_WIN) "cmd" :: "/c" :: Nil else "sh" :: "-c" :: Nil + private val shellPrefix: Seq[String] = if (IS_WIN) "cmd" :: "/c" :: Nil else "bash" :: "-c" :: Nil def run( command: String,