From abb09c4ebd276da7bc65e81be570aee931b65241 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Mon, 18 Mar 2024 16:54:02 +0200 Subject: [PATCH 01/11] Ruby yield expressions --- .../AstForExpressionsCreator.scala | 16 ++++++- .../astcreation/AstForFunctionsCreator.scala | 25 +++++++++-- .../astcreation/AstForStatementsCreator.scala | 2 +- .../astcreation/AstForTypesCreator.scala | 4 +- .../astcreation/FreshVariableCreator.scala | 4 +- .../astcreation/RubyIntermediateAst.scala | 8 ++-- .../datastructures/RubyScope.scala | 44 +++++++++++++++++++ .../datastructures/ScopeElement.scala | 2 +- .../rubysrc2cpg/parser/RubyNodeCreator.scala | 13 +++++- .../io/joern/rubysrc2cpg/passes/Defines.scala | 2 + 10 files changed, 104 insertions(+), 16 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 66b55a5d8ae5..194b0a812d79 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -26,6 +26,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case node: SimpleCall => astForSimpleCall(node) case node: RequireCall => astForRequireCall(node) case node: IncludeCall => astForIncludeCall(node) + case node: YieldExpr => astForYield(node) case node: RangeExpression => astForRange(node) case node: ArrayLiteral => astForArrayLiteral(node) case node: HashLiteral => astForHashLiteral(node) @@ -197,7 +198,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) - val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(freshVariableName)) + val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(freshVariableName())) def tmpIdentifier = { val tmpAst = astForSimpleIdentifier(tmp) tmpAst.root.collect { case x: NewIdentifier => x.typeFullName(receiverTypeFullName) } @@ -365,6 +366,17 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { astForSimpleCall(node.asSimpleCall) } + protected def astForYield(node: YieldExpr): Ast = { + scope.useProcParam match { + case Some(param) => + astForMemberCall(MemberCall(SimpleIdentifier()(node.span.spanStart(param)), ".", "call", node.arguments)(node.span)) + case None => + logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") + astForUnknown(node) + + } + } + protected def astForRange(node: RangeExpression): Ast = { val lbAst = astForExpression(node.lowerBound) val ubAst = astForExpression(node.upperBound) @@ -398,7 +410,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForHashLiteral(node: HashLiteral): Ast = { - val tmp = freshVariableName + val tmp = freshVariableName() def tmpAst(tmpNode: Option[RubyNode] = None) = astForSimpleIdentifier( SimpleIdentifier()(tmpNode.map(_.span).getOrElse(node.span).spanStart(tmp)) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 129fb8bcaa74..73839eeb1afe 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -40,7 +40,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) if (methodName == XDefines.ConstructorMethodName) scope.pushNewScope(ConstructorScope(fullName)) - else scope.pushNewScope(MethodScope(fullName)) + else scope.pushNewScope(MethodScope(fullName, Left(freshVariableName(i => s"")))) val parameterAsts = astForParameters(node.parameters) @@ -76,12 +76,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th astForMethodBody(node.body, optionalStatementList) } + val anonProcParam = scope.anonProcParam.map { param => + val paramNode = ProcParameter(param)(node.span.spanStart(param)) + val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(0) + astForParameter(paramNode, index) + } + scope.popScope() val modifiers = ModifierTypes.VIRTUAL :: (if isClosure then ModifierTypes.LAMBDA :: Nil else Nil) map newModifierNode - methodAst(method, parameterAsts, stmtBlockAst, methodReturn, modifiers) :: refs + methodAst(method, parameterAsts ++ anonProcParam, stmtBlockAst, methodReturn, modifiers) :: refs } private def transformAsClosureBody(refs: List[Ast], baseStmtBlockAst: Ast) = { @@ -141,6 +147,19 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) scope.addToScope(node.name, parameterIn) Ast(parameterIn) + case node: ProcParameter => + val parameterIn = parameterInNode( + node = node, + name = node.name, + code = code(node), + index = index, + isVariadic = false, + evaluationStrategy = EvaluationStrategies.BY_REFERENCE, + typeFullName = None + ) + scope.addToScope(node.name, parameterIn) + scope.setProcParam(node.name) + Ast(parameterIn) case node: CollectionParameter => val typeFullName = node match { case ArrayParameter(_) => prefixAsBuiltin("Array") @@ -252,7 +271,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, Left(freshVariableName(i => s"")))) val thisParameterAst = Ast( newThisParameterNode( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index ceb21ca217e9..b3fe0a617eb5 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -160,7 +160,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } def generatedNode: StatementList = node.expression .map { e => - val tmp = SimpleIdentifier(None)(e.span.spanStart(freshVariableName)) + val tmp = SimpleIdentifier(None)(e.span.spanStart(freshVariableName())) StatementList( List(SingleAssignment(tmp, "=", e)(e.span)) ++ goCase(Some(tmp)) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 42d24d33e93e..464261a87971 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -125,7 +125,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, Left(""))) val block_ = blockNode(node) scope.pushNewScope(BlockScope(block_)) // TODO: Should it be `return this.@abc`? @@ -155,7 +155,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, Left(""))) val parameter = parameterInNode(node, "x", "x", 1, false, EvaluationStrategies.BY_REFERENCE) val methodBody = { val block_ = blockNode(node) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala index 73045249f0a0..3078bfa3cea8 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala @@ -5,8 +5,8 @@ trait FreshVariableCreator { this: AstCreator => private var varCounter: Int = 0 private def tmpVariableTemplate(id: Int): String = s"" - protected def freshVariableName: String = { - val name = tmpVariableTemplate(varCounter) + protected def freshVariableName(template: Int => String = tmpVariableTemplate): String = { + val name = template(varCounter) varCounter += 1 name } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala index 315b8ed8c5d2..8d56cbd28bba 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala @@ -110,9 +110,7 @@ object RubyIntermediateAst { final case class HashParameter(name: String)(span: TextSpan) extends RubyNode(span) with CollectionParameter - final case class ProcParameter(target: RubyNode)(span: TextSpan) extends RubyNode(span) with MethodParameter { - def name: String = target.text - } + final case class ProcParameter(name: String)(span: TextSpan) extends RubyNode(span) with MethodParameter final case class SingleAssignment(lhs: RubyNode, op: String, rhs: RubyNode)(span: TextSpan) extends RubyNode(span) @@ -303,6 +301,9 @@ object RubyIntermediateAst { */ final case class ProcOrLambdaExpr(block: Block)(span: TextSpan) extends RubyNode(span) + + final case class YieldExpr(arguments: List[RubyNode])(span: TextSpan) extends RubyNode(span) + /** Represents a call with a block argument. */ sealed trait RubyCallWithBlock[C <: RubyCall] extends RubyCall { @@ -336,6 +337,7 @@ object RubyIntermediateAst { def withoutBlock: MemberCall = MemberCall(target, op, methodName, arguments)(span) } + /** Represents index accesses, e.g. `x[0]`, `self.x.y[1, 2]` */ final case class IndexAccess(target: RubyNode, indices: List[RubyNode])(span: TextSpan) extends RubyNode(span) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index 468cced554e8..7e2ed69456a4 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -4,6 +4,7 @@ import better.files.File import io.joern.rubysrc2cpg.astcreation.GlobalTypes import io.joern.rubysrc2cpg.astcreation.GlobalTypes.builtinPrefix import io.joern.x2cpg.Defines +import io.joern.rubysrc2cpg.passes.Defines as RDefines import io.joern.x2cpg.datastructures.* import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{DeclarationNew, NewLocal, NewMethodParameterIn} @@ -117,6 +118,49 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) case ScopeElement(x: MethodLikeScope, _) => x.fullName } + /** Locates a position in the stack matching a partial function, modifies it and emits a result + * @param pf + * Tests ScopeElements of the stack. If they match, return the new value and the result to emi + * @return + * the emitted result if the position was found and modifies + */ + def updateSurrounding[T](pf: PartialFunction[ScopeElement[String, DeclarationNew, TypedScopeElement], (ScopeElement[String, DeclarationNew, TypedScopeElement], T)]): Option[T] = { + stack.zipWithIndex.collectFirst { + case (pf(elem, res), i) => (elem, res, i) + }.map { case (elem, res, i) => + stack = stack.updated(i, elem) + res + } + } + + + def useProcParam: Option[String] = updateSurrounding { + case scope @ ScopeElement(MethodScope(fullName, param, _), variables) => (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) + } + + def anonProcParam: Option[String] = stack.collectFirst { + case ScopeElement(MethodScope(_, Left(param), true), _) => param + } + + def setProcParam(param: String): Unit = updateSurrounding { + case scope @ ScopeElement(MethodScope(fullName, _, _), variables) => (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) + } + + // def annonymousYieldProc: Option[NewMethodParameterIn] = stack.collectFirst { + // case ScopeElement(MethodScope(_, false), variables) => variables.get(RDefines.AnonymousProcParameter).map { case x: NewMethodParameterIn => x } + // }.flatten + // + // def getOrCreateYieldProc(anonParam: => NewMethodParameterIn): Option[NewMethodParameterIn] = updateSurrounding[NewMethodParameterIn] { + // case scope@ScopeElement(MethodScope(_, false), variables) => + // variables.collectFirst { + // case (_, param: NewMethodParameterIn) if param.code.startsWith("&") => + // (scope, param) + // }.getOrElse { + // val p = anonParam + // (scope.addVariable(p.name, p), p) + // } + // } + def surroundingTypeFullName: Option[String] = stack.collectFirst { case ScopeElement(x: TypeLikeScope, _) => x.fullName } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala index b88bbf1954ff..f2d8430bba5b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala @@ -57,7 +57,7 @@ trait MethodLikeScope extends TypedScopeElement { def fullName: String } -case class MethodScope(fullName: String) extends MethodLikeScope +case class MethodScope(fullName: String, procParam: Either[String,String], hasYield: Boolean = false) extends MethodLikeScope case class ConstructorScope(fullName: String) extends MethodLikeScope diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index 88df3121bb52..d4453973dd7b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -2,7 +2,6 @@ package io.joern.rubysrc2cpg.parser import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* import io.joern.rubysrc2cpg.parser.AntlrContextHelpers.* -import io.joern.rubysrc2cpg.parser.RubyParser.RangeOperatorContext import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType import org.antlr.v4.runtime.tree.{ParseTree, RuleNode} @@ -523,6 +522,16 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } + override def visitYieldExpression(ctx: RubyParser.YieldExpressionContext): RubyNode = { + val arguments = Option(ctx.argumentWithParentheses()).iterator.flatMap(_.arguments).map(visit).toList + YieldExpr(arguments)(ctx.toTextSpan) + } + + override def visitYieldMethodInvocationWithoutParentheses(ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext): RubyNode = { + val arguments = ctx.primaryValueList().primaryValue().asScala.map(visit).toList + YieldExpr(arguments)(ctx.toTextSpan) + } + override def visitConstantIdentifierVariable(ctx: RubyParser.ConstantIdentifierVariableContext): RubyNode = { SimpleIdentifier()(ctx.toTextSpan) } @@ -870,7 +879,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } override def visitProcParameter(ctx: RubyParser.ProcParameterContext): RubyNode = { - ProcParameter(visit(ctx.procParameterName()))(ctx.toTextSpan) + ProcParameter(Option(ctx.procParameterName).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText()))(ctx.toTextSpan) } override def visitHashParameter(ctx: RubyParser.HashParameterContext): RubyNode = { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala index e97935881e6d..0c6b3091009d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala @@ -27,6 +27,8 @@ object Defines { val Resolver: String = "" + val AnonymousProcParameter = "" + def getBuiltInType(typeInString: String) = s"${GlobalTypes.builtinPrefix}.$typeInString" object RubyOperators { From 16d3dcb1498d9cb8d1ccd3fbaa586b502e647afb Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Mon, 18 Mar 2024 16:56:37 +0200 Subject: [PATCH 02/11] Scalafmt --- .../AstForExpressionsCreator.scala | 6 ++- .../astcreation/AstForFunctionsCreator.scala | 6 +-- .../astcreation/RubyIntermediateAst.scala | 2 - .../datastructures/RubyScope.scala | 46 +++++++++++-------- .../datastructures/ScopeElement.scala | 3 +- .../rubysrc2cpg/parser/RubyNodeCreator.scala | 8 +++- 6 files changed, 42 insertions(+), 29 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 194b0a812d79..32bf2e173b9e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -368,8 +368,10 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { protected def astForYield(node: YieldExpr): Ast = { scope.useProcParam match { - case Some(param) => - astForMemberCall(MemberCall(SimpleIdentifier()(node.span.spanStart(param)), ".", "call", node.arguments)(node.span)) + case Some(param) => + astForMemberCall( + MemberCall(SimpleIdentifier()(node.span.spanStart(param)), ".", "call", node.arguments)(node.span) + ) case None => logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") astForUnknown(node) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 73839eeb1afe..f2f58d99646a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -77,9 +77,9 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } val anonProcParam = scope.anonProcParam.map { param => - val paramNode = ProcParameter(param)(node.span.spanStart(param)) - val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(0) - astForParameter(paramNode, index) + val paramNode = ProcParameter(param)(node.span.spanStart(param)) + val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(0) + astForParameter(paramNode, index) } scope.popScope() diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala index 8d56cbd28bba..8c6bd5e43527 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala @@ -301,7 +301,6 @@ object RubyIntermediateAst { */ final case class ProcOrLambdaExpr(block: Block)(span: TextSpan) extends RubyNode(span) - final case class YieldExpr(arguments: List[RubyNode])(span: TextSpan) extends RubyNode(span) /** Represents a call with a block argument. @@ -337,7 +336,6 @@ object RubyIntermediateAst { def withoutBlock: MemberCall = MemberCall(target, op, methodName, arguments)(span) } - /** Represents index accesses, e.g. `x[0]`, `self.x.y[1, 2]` */ final case class IndexAccess(target: RubyNode, indices: List[RubyNode])(span: TextSpan) extends RubyNode(span) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index 7e2ed69456a4..4bb244fd95a3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -119,31 +119,39 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) } /** Locates a position in the stack matching a partial function, modifies it and emits a result - * @param pf - * Tests ScopeElements of the stack. If they match, return the new value and the result to emi - * @return - * the emitted result if the position was found and modifies - */ - def updateSurrounding[T](pf: PartialFunction[ScopeElement[String, DeclarationNew, TypedScopeElement], (ScopeElement[String, DeclarationNew, TypedScopeElement], T)]): Option[T] = { - stack.zipWithIndex.collectFirst { - case (pf(elem, res), i) => (elem, res, i) - }.map { case (elem, res, i) => - stack = stack.updated(i, elem) - res - } + * @param pf + * Tests ScopeElements of the stack. If they match, return the new value and the result to emi + * @return + * the emitted result if the position was found and modifies + */ + def updateSurrounding[T]( + pf: PartialFunction[ + ScopeElement[String, DeclarationNew, TypedScopeElement], + (ScopeElement[String, DeclarationNew, TypedScopeElement], T) + ] + ): Option[T] = { + stack.zipWithIndex + .collectFirst { case (pf(elem, res), i) => + (elem, res, i) + } + .map { case (elem, res, i) => + stack = stack.updated(i, elem) + res + } } - def useProcParam: Option[String] = updateSurrounding { - case scope @ ScopeElement(MethodScope(fullName, param, _), variables) => (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) + case scope @ ScopeElement(MethodScope(fullName, param, _), variables) => + (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) } - def anonProcParam: Option[String] = stack.collectFirst { - case ScopeElement(MethodScope(_, Left(param), true), _) => param + def anonProcParam: Option[String] = stack.collectFirst { case ScopeElement(MethodScope(_, Left(param), true), _) => + param } def setProcParam(param: String): Unit = updateSurrounding { - case scope @ ScopeElement(MethodScope(fullName, _, _), variables) => (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) + case scope @ ScopeElement(MethodScope(fullName, _, _), variables) => + (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) } // def annonymousYieldProc: Option[NewMethodParameterIn] = stack.collectFirst { @@ -151,9 +159,9 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) // }.flatten // // def getOrCreateYieldProc(anonParam: => NewMethodParameterIn): Option[NewMethodParameterIn] = updateSurrounding[NewMethodParameterIn] { - // case scope@ScopeElement(MethodScope(_, false), variables) => + // case scope@ScopeElement(MethodScope(_, false), variables) => // variables.collectFirst { - // case (_, param: NewMethodParameterIn) if param.code.startsWith("&") => + // case (_, param: NewMethodParameterIn) if param.code.startsWith("&") => // (scope, param) // }.getOrElse { // val p = anonParam diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala index f2d8430bba5b..a6d07440f86a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala @@ -57,7 +57,8 @@ trait MethodLikeScope extends TypedScopeElement { def fullName: String } -case class MethodScope(fullName: String, procParam: Either[String,String], hasYield: Boolean = false) extends MethodLikeScope +case class MethodScope(fullName: String, procParam: Either[String, String], hasYield: Boolean = false) + extends MethodLikeScope case class ConstructorScope(fullName: String) extends MethodLikeScope diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index d4453973dd7b..c6eb978cd56f 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -527,7 +527,9 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { YieldExpr(arguments)(ctx.toTextSpan) } - override def visitYieldMethodInvocationWithoutParentheses(ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext): RubyNode = { + override def visitYieldMethodInvocationWithoutParentheses( + ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext + ): RubyNode = { val arguments = ctx.primaryValueList().primaryValue().asScala.map(visit).toList YieldExpr(arguments)(ctx.toTextSpan) } @@ -879,7 +881,9 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } override def visitProcParameter(ctx: RubyParser.ProcParameterContext): RubyNode = { - ProcParameter(Option(ctx.procParameterName).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText()))(ctx.toTextSpan) + ProcParameter( + Option(ctx.procParameterName).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText()) + )(ctx.toTextSpan) } override def visitHashParameter(ctx: RubyParser.HashParameterContext): RubyNode = { From f9e6a51692e1a769febcfa8200a45a4cc5b3a2c4 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 11:42:37 +0200 Subject: [PATCH 03/11] Create FreshNameGenerator class --- .../joern/rubysrc2cpg/astcreation/AstCreator.scala | 1 - .../astcreation/AstForExpressionsCreator.scala | 7 +++++-- .../astcreation/AstForFunctionsCreator.scala | 7 +++++-- .../astcreation/AstForStatementsCreator.scala | 2 +- .../astcreation/AstForTypesCreator.scala | 4 ++-- .../astcreation/FreshVariableCreator.scala | 13 ------------- .../joern/rubysrc2cpg/parser/RubyNodeCreator.scala | 10 +++------- .../rubysrc2cpg/utils/FreshNameGenerator.scala | 10 ++++++++++ 8 files changed, 26 insertions(+), 28 deletions(-) delete mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 04b187c62ddd..e329b69b034c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -27,7 +27,6 @@ class AstCreator( with AstForExpressionsCreator with AstForFunctionsCreator with AstForTypesCreator - with FreshVariableCreator with AstSummaryVisitor with AstNodeBuilder[RubyNode, AstCreator] { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 32bf2e173b9e..3f149581f897 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -7,9 +7,12 @@ import io.joern.rubysrc2cpg.passes.Defines.{RubyOperators, getBuiltInType} import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators, PropertyNames} +import io.joern.rubysrc2cpg.utils.FreshNameGenerator trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + val tmpGen = FreshNameGenerator(i => s"") + protected def astForExpression(node: RubyNode): Ast = node match case node: StaticLiteral => astForStaticLiteral(node) case node: HereDocNode => astForHereDoc(node) @@ -198,7 +201,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) - val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(freshVariableName())) + val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(tmpGen.fresh)) def tmpIdentifier = { val tmpAst = astForSimpleIdentifier(tmp) tmpAst.root.collect { case x: NewIdentifier => x.typeFullName(receiverTypeFullName) } @@ -412,7 +415,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForHashLiteral(node: HashLiteral): Ast = { - val tmp = freshVariableName() + val tmp = tmpGen.fresh def tmpAst(tmpNode: Option[RubyNode] = None) = astForSimpleIdentifier( SimpleIdentifier()(tmpNode.map(_.span).getOrElse(node.span).spanStart(tmp)) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index f2f58d99646a..79be42a1b013 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -7,9 +7,12 @@ import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode, n import io.joern.x2cpg.{Ast, AstEdge, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, ModifierTypes, NodeTypes} +import io.joern.rubysrc2cpg.utils.FreshNameGenerator trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + val procParamGen = FreshNameGenerator(i => Left(s"")) + /** Creates method declaration related structures. * @param node * the node to create the AST structure from. @@ -40,7 +43,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) if (methodName == XDefines.ConstructorMethodName) scope.pushNewScope(ConstructorScope(fullName)) - else scope.pushNewScope(MethodScope(fullName, Left(freshVariableName(i => s"")))) + else scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val parameterAsts = astForParameters(node.parameters) @@ -271,7 +274,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName, Left(freshVariableName(i => s"")))) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val thisParameterAst = Ast( newThisParameterNode( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index b3fe0a617eb5..098e649148ab 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -160,7 +160,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } def generatedNode: StatementList = node.expression .map { e => - val tmp = SimpleIdentifier(None)(e.span.spanStart(freshVariableName())) + val tmp = SimpleIdentifier(None)(e.span.spanStart(tmpGen.fresh)) StatementList( List(SingleAssignment(tmp, "=", e)(e.span)) ++ goCase(Some(tmp)) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 464261a87971..426128fa3bdc 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -125,7 +125,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName, Left(""))) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val block_ = blockNode(node) scope.pushNewScope(BlockScope(block_)) // TODO: Should it be `return this.@abc`? @@ -155,7 +155,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName, Left(""))) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val parameter = parameterInNode(node, "x", "x", 1, false, EvaluationStrategies.BY_REFERENCE) val methodBody = { val block_ = blockNode(node) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala deleted file mode 100644 index 3078bfa3cea8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala +++ /dev/null @@ -1,13 +0,0 @@ -package io.joern.rubysrc2cpg.astcreation - -trait FreshVariableCreator { this: AstCreator => - // This is in a single-threaded context. - private var varCounter: Int = 0 - - private def tmpVariableTemplate(id: Int): String = s"" - protected def freshVariableName(template: Int => String = tmpVariableTemplate): String = { - val name = template(varCounter) - varCounter += 1 - name - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index c6eb978cd56f..1c3d78079bfd 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -8,19 +8,15 @@ import org.antlr.v4.runtime.tree.{ParseTree, RuleNode} import io.joern.x2cpg.Defines as XDefines; import scala.jdk.CollectionConverters.* +import io.joern.rubysrc2cpg.utils.FreshNameGenerator /** Converts an ANTLR Ruby Parse Tree into the intermediate Ruby AST. */ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { - private var classCounter: Int = 0 - - private def tmpClassTemplate(id: Int): String = s"" - + private val classNameGen = FreshNameGenerator(id => s"") protected def freshClassName(span: TextSpan): SimpleIdentifier = { - val name = tmpClassTemplate(classCounter) - classCounter += 1 - SimpleIdentifier(None)(span.spanStart(name)) + SimpleIdentifier(None)(span.spanStart(classNameGen.fresh)) } private def defaultTextSpan(code: String = ""): TextSpan = TextSpan(None, None, None, None, code) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala new file mode 100644 index 000000000000..a7e77e248d8a --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala @@ -0,0 +1,10 @@ +package io.joern.rubysrc2cpg.utils + +class FreshNameGenerator[T](template: Int => T) { + private var counter: Int = 0 + def fresh: T = { + val name = template(counter) + counter += 1 + name + } +} From 238dc7329b050d0533cae7c871a14501a1d8c29a Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 14:09:53 +0200 Subject: [PATCH 04/11] Add tests and singleton methods --- .../astcreation/AstForFunctionsCreator.scala | 10 +++- .../astcreation/AstForStatementsCreator.scala | 3 +- .../datastructures/RubyScope.scala | 15 ----- .../querying/ProcParameterAndYieldTests.scala | 59 +++++++++++++++++++ 4 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 79be42a1b013..32d6afba1b15 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -80,7 +80,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } val anonProcParam = scope.anonProcParam.map { param => - val paramNode = ProcParameter(param)(node.span.spanStart(param)) + val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(0) astForParameter(paramNode, index) } @@ -290,8 +290,14 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val stmtBlockAst = astForMethodBody(node.body, optionalStatementList) + val anonProcParam = scope.anonProcParam.map { param => + val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(1) + astForParameter(paramNode, index) + } + scope.popScope() - methodAst(method, thisParameterAst +: parameterAsts, stmtBlockAst, methodReturnNode(node, Defines.Any)) + methodAst(method, (thisParameterAst +: parameterAsts) ++ anonProcParam, stmtBlockAst, methodReturnNode(node, Defines.Any)) case targetNode => logger.warn( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 098e649148ab..792860a539c7 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -252,7 +252,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MemberCallWithBlock => returnAstForRubyCall(node) case node: SimpleCallWithBlock => returnAstForRubyCall(node) case _: (LiteralExpr | BinaryExpression | UnaryExpression | SimpleIdentifier | IndexAccess | Association | - RubyCall) => + RubyCall | YieldExpr) => astForReturnStatement(ReturnExpression(List(node))(node.span)) :: Nil case node: SingleAssignment => astForSingleAssignment(node) :: List(astForReturnStatement(ReturnExpression(List(node.lhs))(node.span))) @@ -265,6 +265,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case ret: ReturnExpression => astForReturnStatement(ret) :: Nil case node: MethodDeclaration => (astForMethodDeclaration(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList + case node => logger.warn( s"Implicit return here not supported yet: ${node.text} (${node.getClass.getSimpleName}), only generating statement" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index 4bb244fd95a3..a34bfeb70211 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -154,21 +154,6 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) } - // def annonymousYieldProc: Option[NewMethodParameterIn] = stack.collectFirst { - // case ScopeElement(MethodScope(_, false), variables) => variables.get(RDefines.AnonymousProcParameter).map { case x: NewMethodParameterIn => x } - // }.flatten - // - // def getOrCreateYieldProc(anonParam: => NewMethodParameterIn): Option[NewMethodParameterIn] = updateSurrounding[NewMethodParameterIn] { - // case scope@ScopeElement(MethodScope(_, false), variables) => - // variables.collectFirst { - // case (_, param: NewMethodParameterIn) if param.code.startsWith("&") => - // (scope, param) - // }.getOrElse { - // val p = anonParam - // (scope.addVariable(p.name, p), p) - // } - // } - def surroundingTypeFullName: Option[String] = stack.collectFirst { case ScopeElement(x: TypeLikeScope, _) => x.fullName } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala new file mode 100644 index 000000000000..fa91fdd7ad96 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -0,0 +1,59 @@ +package io.joern.rubysrc2cpg.querying + +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import org.scalatest.Inspectors +import io.shiftleft.semanticcpg.language.* + +class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { + "Methods" should { + "with a yield expression" should { + "with a proc parameter" should { + val cpg1 = code( + """| def foo(&b) yield end + |""".stripMargin) + val cpg2 = code( + """| def self.foo(&b) yield end + |""".stripMargin) + val cpgs = List(cpg1, cpg2) + "have a single block argument" in { + forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) + } + "replace the yield with a call to the block parameter" in { + forAll(cpgs) (_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("b", "call")) + } + } + "without a proc parameter" should { + + val cpg1 = code( + """| def foo() yield end + |""".stripMargin) + val cpg2 = code( + """| def self.foo() yield end + |""".stripMargin) + val cpgs = List(cpg1, cpg2) + + "replace the yield with a call to a block parameter" in { + forAll(cpgs) (_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("", "call")) + } + + "add a block argument" in { + forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l shouldBe List("")) + } + } + } + "that don't have a yield nor a proc parameter" should { + val cpg1 = code( + """| def foo() end + |""".stripMargin) + val cpg2 = code( + """| def self.foo() end + |""".stripMargin) + val cpgs = List(cpg1, cpg2) + + "not add a block argument" in { + forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l should be(empty)) + } + } + } + +} From b6f33d47fc191bb56de8760f7b7b726fe9ebe0dd Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 14:12:28 +0200 Subject: [PATCH 05/11] scalafmt --- .../astcreation/AstForFunctionsCreator.scala | 10 ++++-- .../querying/ProcParameterAndYieldTests.scala | 34 ++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 32d6afba1b15..62e6f92f8cb5 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -292,12 +292,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val anonProcParam = scope.anonProcParam.map { param => val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) - val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(1) + val index = + parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(1) astForParameter(paramNode, index) } scope.popScope() - methodAst(method, (thisParameterAst +: parameterAsts) ++ anonProcParam, stmtBlockAst, methodReturnNode(node, Defines.Any)) + methodAst( + method, + (thisParameterAst +: parameterAsts) ++ anonProcParam, + stmtBlockAst, + methodReturnNode(node, Defines.Any) + ) case targetNode => logger.warn( diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index fa91fdd7ad96..d0971e43d602 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -8,51 +8,45 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { "Methods" should { "with a yield expression" should { "with a proc parameter" should { - val cpg1 = code( - """| def foo(&b) yield end + val cpg1 = code("""| def foo(&b) yield end |""".stripMargin) - val cpg2 = code( - """| def self.foo(&b) yield end + val cpg2 = code("""| def self.foo(&b) yield end |""".stripMargin) val cpgs = List(cpg1, cpg2) "have a single block argument" in { - forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) + forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) } "replace the yield with a call to the block parameter" in { - forAll(cpgs) (_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("b", "call")) + forAll(cpgs)(_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("b", "call")) } } "without a proc parameter" should { - val cpg1 = code( - """| def foo() yield end + val cpg1 = code("""| def foo() yield end |""".stripMargin) - val cpg2 = code( - """| def self.foo() yield end + val cpg2 = code("""| def self.foo() yield end |""".stripMargin) val cpgs = List(cpg1, cpg2) "replace the yield with a call to a block parameter" in { - forAll(cpgs) (_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("", "call")) + forAll(cpgs)(_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("", "call")) } "add a block argument" in { - forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l shouldBe List("")) + forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("")) } } } "that don't have a yield nor a proc parameter" should { - val cpg1 = code( - """| def foo() end + val cpg1 = code("""| def foo() end |""".stripMargin) - val cpg2 = code( - """| def self.foo() end + val cpg2 = code("""| def self.foo() end |""".stripMargin) - val cpgs = List(cpg1, cpg2) + val cpgs = List(cpg1, cpg2) - "not add a block argument" in { - forAll(cpgs) (_.method("foo").parameter.code("&.*").name.l should be(empty)) - } + "not add a block argument" in { + forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l should be(empty)) + } } } From 2db6ac28b6535146e25affeeef88f5e651fa84da Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 14:47:08 +0200 Subject: [PATCH 06/11] Address some PR comments --- .../astcreation/AstForFunctionsCreator.scala | 10 +++---- .../querying/ProcParameterAndYieldTests.scala | 28 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 62e6f92f8cb5..7aff8318f249 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -81,8 +81,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val anonProcParam = scope.anonProcParam.map { param => val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) - val index = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(0) - astForParameter(paramNode, index) + val nextIndex = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) + astForParameter(paramNode, nextIndex) } scope.popScope() @@ -292,9 +292,9 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val anonProcParam = scope.anonProcParam.map { param => val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) - val index = - parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index }.getOrElse(1) - astForParameter(paramNode, index) + val nextIndex = + parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(1) + astForParameter(paramNode, nextIndex) } scope.popScope() diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index d0971e43d602..80c433830311 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -8,24 +8,21 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { "Methods" should { "with a yield expression" should { "with a proc parameter" should { - val cpg1 = code("""| def foo(&b) yield end - |""".stripMargin) - val cpg2 = code("""| def self.foo(&b) yield end - |""".stripMargin) + val cpg1 = code("def foo(&b) yield end") + val cpg2 = code("def self.foo(&b) yield end") val cpgs = List(cpg1, cpg2) + "have a single block argument" in { forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) } + "replace the yield with a call to the block parameter" in { forAll(cpgs)(_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("b", "call")) } } "without a proc parameter" should { - - val cpg1 = code("""| def foo() yield end - |""".stripMargin) - val cpg2 = code("""| def self.foo() yield end - |""".stripMargin) + val cpg1 = code("def foo() yield end") + val cpg2 = code("def self.foo() yield end") val cpgs = List(cpg1, cpg2) "replace the yield with a call to a block parameter" in { @@ -33,15 +30,18 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { } "add a block argument" in { - forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("")) + forAll(cpgs.zipWithIndex) { (cpg, i) => + val List(param) = cpg.method("foo").parameter.code("&.*").l + param.name shouldBe "" + param.index shouldBe i + } } } } + "that don't have a yield nor a proc parameter" should { - val cpg1 = code("""| def foo() end - |""".stripMargin) - val cpg2 = code("""| def self.foo() end - |""".stripMargin) + val cpg1 = code("def foo() end") + val cpg2 = code("def self.foo() end") val cpgs = List(cpg1, cpg2) "not add a block argument" in { From ee9e8af35862b422dfc1b7f30250fc15204da991 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 15:28:44 +0200 Subject: [PATCH 07/11] Attempt 1 at return flow --- .../astcreation/AstForExpressionsCreator.scala | 6 ++++-- .../astcreation/AstForStatementsCreator.scala | 5 ++++- .../rubysrc2cpg/datastructures/RubyScope.scala | 8 ++++++-- .../querying/ProcParameterAndYieldTests.scala | 17 +++++++++++++++-- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 3f149581f897..929fe32ed439 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -372,8 +372,10 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { protected def astForYield(node: YieldExpr): Ast = { scope.useProcParam match { case Some(param) => - astForMemberCall( - MemberCall(SimpleIdentifier()(node.span.spanStart(param)), ".", "call", node.arguments)(node.span) + astForReturnStatement( + ReturnExpression(List( + SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span) + ))(node.span) ) case None => logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 792860a539c7..812f141df392 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -252,7 +252,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MemberCallWithBlock => returnAstForRubyCall(node) case node: SimpleCallWithBlock => returnAstForRubyCall(node) case _: (LiteralExpr | BinaryExpression | UnaryExpression | SimpleIdentifier | IndexAccess | Association | - RubyCall | YieldExpr) => + RubyCall) => astForReturnStatement(ReturnExpression(List(node))(node.span)) :: Nil case node: SingleAssignment => astForSingleAssignment(node) :: List(astForReturnStatement(ReturnExpression(List(node.lhs))(node.span))) @@ -266,6 +266,9 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MethodDeclaration => (astForMethodDeclaration(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList + case node: YieldExpr => // Yield is already a return expression to handle do block returns + astForExpression(node) :: Nil + case node => logger.warn( s"Implicit return here not supported yet: ${node.text} (${node.getClass.getSimpleName}), only generating statement" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index a34bfeb70211..cd716530de5d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -140,17 +140,21 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) } } + /** Get the name of the implicit or explict proc param and mark the method scope as using the proc param + */ def useProcParam: Option[String] = updateSurrounding { - case scope @ ScopeElement(MethodScope(fullName, param, _), variables) => + case ScopeElement(MethodScope(fullName, param, _), variables) => (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) } + /** Get the name of the implicit or explict proc param */ def anonProcParam: Option[String] = stack.collectFirst { case ScopeElement(MethodScope(_, Left(param), true), _) => param } + /** Set the name of explict proc param */ def setProcParam(param: String): Unit = updateSurrounding { - case scope @ ScopeElement(MethodScope(fullName, _, _), variables) => + case ScopeElement(MethodScope(fullName, _, _), variables) => (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index 80c433830311..6ba6b95d5216 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -17,7 +17,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { } "replace the yield with a call to the block parameter" in { - forAll(cpgs)(_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("b", "call")) + forAll(cpgs)(_.call.code("yield").name.l shouldBe List("b")) } } "without a proc parameter" should { @@ -26,7 +26,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { val cpgs = List(cpg1, cpg2) "replace the yield with a call to a block parameter" in { - forAll(cpgs)(_.call.code("yield").receiver.isCall.argument.code.l shouldBe List("", "call")) + forAll(cpgs)(_.call.code("yield").name.l shouldBe List("")) } "add a block argument" in { @@ -48,6 +48,19 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l should be(empty)) } } + + "with non-implicitly returned yield" should { + val cpg = code(""" + |def foo() + | yield + | 1 + |end + |""".stripMargin) + "have a return node for the yield" in { + val List(ret) = cpg.method("foo").call.code("yield").astParent.isReturn.l + ret.code shouldBe "yield" + } + } } } From 6498882ab3bbcca2908a7614e46eb397156be6b6 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 15:29:58 +0200 Subject: [PATCH 08/11] scalafmt --- .../rubysrc2cpg/astcreation/AstForExpressionsCreator.scala | 6 +++--- .../rubysrc2cpg/astcreation/AstForFunctionsCreator.scala | 3 ++- .../io/joern/rubysrc2cpg/datastructures/RubyScope.scala | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 929fe32ed439..e976fc14d260 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -373,9 +373,9 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { scope.useProcParam match { case Some(param) => astForReturnStatement( - ReturnExpression(List( - SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span) - ))(node.span) + ReturnExpression(List(SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span)))( + node.span + ) ) case None => logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 7aff8318f249..8369acf30225 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -81,7 +81,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val anonProcParam = scope.anonProcParam.map { param => val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) - val nextIndex = parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) + val nextIndex = + parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) astForParameter(paramNode, nextIndex) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index cd716530de5d..9ce8bc4bfc6d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -141,7 +141,7 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) } /** Get the name of the implicit or explict proc param and mark the method scope as using the proc param - */ + */ def useProcParam: Option[String] = updateSurrounding { case ScopeElement(MethodScope(fullName, param, _), variables) => (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) From 7bc5cc0529973b30b6559b7d98feefbd050121fe Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 17:04:15 +0200 Subject: [PATCH 09/11] Add test for yield argument --- .../querying/ProcParameterAndYieldTests.scala | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index 6ba6b95d5216..e4dc609a0198 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -20,6 +20,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { forAll(cpgs)(_.call.code("yield").name.l shouldBe List("b")) } } + "without a proc parameter" should { val cpg1 = code("def foo() yield end") val cpg2 = code("def self.foo() yield end") @@ -37,6 +38,16 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { } } } + + "with yield arguments" should { + val cpg = code("def foo(x) yield(x) end") + "replace the yield with a call to the block parameter with arguments" in { + val List(call) = cpg.call.codeExact("yield(x)").l + call.name shouldBe "b" + call.argument.code.l shouldBe List("x") + } + + } } "that don't have a yield nor a proc parameter" should { @@ -61,6 +72,19 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { ret.code shouldBe "yield" } } + + "with implicitly returned yield" should { + val cpg = code(""" + |def foo() + | yield + |end + |""".stripMargin) + "have a return node for the yield" in { + val List(ret) = cpg.method("foo").call.code("yield").astParent.isReturn.l + ret.code shouldBe "yield" + } + } + } } From a5dbb2a3031618f403d4029ba0c92b18b7c67c78 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 17:05:18 +0200 Subject: [PATCH 10/11] scalafmt --- .../joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index e4dc609a0198..2b86db295f50 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -46,7 +46,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { call.name shouldBe "b" call.argument.code.l shouldBe List("x") } - + } } From ce033bfc0e492a350ec79fc206ca503b580708c0 Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Tue, 19 Mar 2024 19:20:56 +0200 Subject: [PATCH 11/11] Make return for yield --- .../AstForExpressionsCreator.scala | 14 ++++-- .../astcreation/AstForStatementsCreator.scala | 5 +- .../querying/ProcParameterAndYieldTests.scala | 47 +++++++------------ 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index e976fc14d260..707fbb3354b3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -372,10 +372,16 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { protected def astForYield(node: YieldExpr): Ast = { scope.useProcParam match { case Some(param) => - astForReturnStatement( - ReturnExpression(List(SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span)))( - node.span - ) + val call = astForExpression( + SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span) + ) + val ret = returnAst(returnNode(node, code(node))) + val cond = astForExpression( + SimpleCall(SimpleIdentifier()(node.span.spanStart(tmpGen.fresh)), List())(node.span.spanStart("")) + ) + callAst( + callNode(node, code(node), Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH), + List(cond, call, ret) ) case None => logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 812f141df392..96bf0ead605a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -252,7 +252,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MemberCallWithBlock => returnAstForRubyCall(node) case node: SimpleCallWithBlock => returnAstForRubyCall(node) case _: (LiteralExpr | BinaryExpression | UnaryExpression | SimpleIdentifier | IndexAccess | Association | - RubyCall) => + YieldExpr | RubyCall) => astForReturnStatement(ReturnExpression(List(node))(node.span)) :: Nil case node: SingleAssignment => astForSingleAssignment(node) :: List(astForReturnStatement(ReturnExpression(List(node.lhs))(node.span))) @@ -266,9 +266,6 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MethodDeclaration => (astForMethodDeclaration(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList - case node: YieldExpr => // Yield is already a return expression to handle do block returns - astForExpression(node) :: Nil - case node => logger.warn( s"Implicit return here not supported yet: ${node.text} (${node.getClass.getSimpleName}), only generating statement" diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index 2b86db295f50..d03d7d33239b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -3,6 +3,7 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import org.scalatest.Inspectors import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.nodes.* class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { "Methods" should { @@ -16,8 +17,17 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) } - "replace the yield with a call to the block parameter" in { - forAll(cpgs)(_.call.code("yield").name.l shouldBe List("b")) + "represent the yield as a conditional with a call and return node as children" in { + forAll(cpgs) { cpg => + inside(cpg.method("foo").call(".conditional").code("yield").astChildren.l) { + case List(cond: Expression, call: Call, ret: Return) => { + cond.code shouldBe "" + call.name shouldBe "b" + call.code shouldBe "yield" + ret.code shouldBe "yield" + } + } + } } } @@ -26,8 +36,8 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { val cpg2 = code("def self.foo() yield end") val cpgs = List(cpg1, cpg2) - "replace the yield with a call to a block parameter" in { - forAll(cpgs)(_.call.code("yield").name.l shouldBe List("")) + "have a call to a block parameter" in { + forAll(cpgs)(_.call.code("yield").astChildren.isCall.code("yield").name.l shouldBe List("")) } "add a block argument" in { @@ -42,8 +52,8 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { "with yield arguments" should { val cpg = code("def foo(x) yield(x) end") "replace the yield with a call to the block parameter with arguments" in { - val List(call) = cpg.call.codeExact("yield(x)").l - call.name shouldBe "b" + val List(call) = cpg.call.codeExact("yield(x)").astChildren.isCall.codeExact("yield(x)").l + call.name shouldBe "" call.argument.code.l shouldBe List("x") } @@ -60,31 +70,6 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { } } - "with non-implicitly returned yield" should { - val cpg = code(""" - |def foo() - | yield - | 1 - |end - |""".stripMargin) - "have a return node for the yield" in { - val List(ret) = cpg.method("foo").call.code("yield").astParent.isReturn.l - ret.code shouldBe "yield" - } - } - - "with implicitly returned yield" should { - val cpg = code(""" - |def foo() - | yield - |end - |""".stripMargin) - "have a return node for the yield" in { - val List(ret) = cpg.method("foo").call.code("yield").astParent.isReturn.l - ret.code shouldBe "yield" - } - } - } }