From 4a12a276ce2ebec7735b91267f069b4db8f1c7bd Mon Sep 17 00:00:00 2001 From: Reuben Steenekamp Date: Wed, 20 Mar 2024 10:30:57 +0200 Subject: [PATCH] [Ruby] do-end Blocks (with implicit &Proc and yield handling) #3928 (#4359) * Ruby yield expressions * Scalafmt * Create FreshNameGenerator class * Add tests and singleton methods * scalafmt * Address some PR comments * Attempt 1 at return flow * scalafmt * Add test for yield argument * scalafmt * Make return for yield --- .../rubysrc2cpg/astcreation/AstCreator.scala | 1 - .../AstForExpressionsCreator.scala | 29 ++++++- .../astcreation/AstForFunctionsCreator.scala | 43 ++++++++++- .../astcreation/AstForStatementsCreator.scala | 5 +- .../astcreation/AstForTypesCreator.scala | 4 +- .../astcreation/FreshVariableCreator.scala | 13 ---- .../astcreation/RubyIntermediateAst.scala | 6 +- .../datastructures/RubyScope.scala | 41 ++++++++++ .../datastructures/ScopeElement.scala | 3 +- .../rubysrc2cpg/parser/RubyNodeCreator.scala | 27 ++++--- .../io/joern/rubysrc2cpg/passes/Defines.scala | 2 + .../utils/FreshNameGenerator.scala | 10 +++ .../querying/ProcParameterAndYieldTests.scala | 75 +++++++++++++++++++ 13 files changed, 222 insertions(+), 37 deletions(-) delete mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 04b187c62ddd..e329b69b034c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -27,7 +27,6 @@ class AstCreator( with AstForExpressionsCreator with AstForFunctionsCreator with AstForTypesCreator - with FreshVariableCreator with AstSummaryVisitor with AstNodeBuilder[RubyNode, AstCreator] { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 66b55a5d8ae5..707fbb3354b3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -7,9 +7,12 @@ import io.joern.rubysrc2cpg.passes.Defines.{RubyOperators, getBuiltInType} import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators, PropertyNames} +import io.joern.rubysrc2cpg.utils.FreshNameGenerator trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + val tmpGen = FreshNameGenerator(i => s"") + protected def astForExpression(node: RubyNode): Ast = node match case node: StaticLiteral => astForStaticLiteral(node) case node: HereDocNode => astForHereDoc(node) @@ -26,6 +29,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case node: SimpleCall => astForSimpleCall(node) case node: RequireCall => astForRequireCall(node) case node: IncludeCall => astForIncludeCall(node) + case node: YieldExpr => astForYield(node) case node: RangeExpression => astForRange(node) case node: ArrayLiteral => astForArrayLiteral(node) case node: HashLiteral => astForHashLiteral(node) @@ -197,7 +201,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) - val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(freshVariableName)) + val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(tmpGen.fresh)) def tmpIdentifier = { val tmpAst = astForSimpleIdentifier(tmp) tmpAst.root.collect { case x: NewIdentifier => x.typeFullName(receiverTypeFullName) } @@ -365,6 +369,27 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { astForSimpleCall(node.asSimpleCall) } + protected def astForYield(node: YieldExpr): Ast = { + scope.useProcParam match { + case Some(param) => + val call = astForExpression( + SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span) + ) + val ret = returnAst(returnNode(node, code(node))) + val cond = astForExpression( + SimpleCall(SimpleIdentifier()(node.span.spanStart(tmpGen.fresh)), List())(node.span.spanStart("")) + ) + callAst( + callNode(node, code(node), Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH), + List(cond, call, ret) + ) + case None => + logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") + astForUnknown(node) + + } + } + protected def astForRange(node: RangeExpression): Ast = { val lbAst = astForExpression(node.lowerBound) val ubAst = astForExpression(node.upperBound) @@ -398,7 +423,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForHashLiteral(node: HashLiteral): Ast = { - val tmp = freshVariableName + val tmp = tmpGen.fresh def tmpAst(tmpNode: Option[RubyNode] = None) = astForSimpleIdentifier( SimpleIdentifier()(tmpNode.map(_.span).getOrElse(node.span).spanStart(tmp)) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 129fb8bcaa74..8369acf30225 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -7,9 +7,12 @@ import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode, n import io.joern.x2cpg.{Ast, AstEdge, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, ModifierTypes, NodeTypes} +import io.joern.rubysrc2cpg.utils.FreshNameGenerator trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + val procParamGen = FreshNameGenerator(i => Left(s"")) + /** Creates method declaration related structures. * @param node * the node to create the AST structure from. @@ -40,7 +43,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) if (methodName == XDefines.ConstructorMethodName) scope.pushNewScope(ConstructorScope(fullName)) - else scope.pushNewScope(MethodScope(fullName)) + else scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val parameterAsts = astForParameters(node.parameters) @@ -76,12 +79,19 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th astForMethodBody(node.body, optionalStatementList) } + val anonProcParam = scope.anonProcParam.map { param => + val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + val nextIndex = + parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) + astForParameter(paramNode, nextIndex) + } + scope.popScope() val modifiers = ModifierTypes.VIRTUAL :: (if isClosure then ModifierTypes.LAMBDA :: Nil else Nil) map newModifierNode - methodAst(method, parameterAsts, stmtBlockAst, methodReturn, modifiers) :: refs + methodAst(method, parameterAsts ++ anonProcParam, stmtBlockAst, methodReturn, modifiers) :: refs } private def transformAsClosureBody(refs: List[Ast], baseStmtBlockAst: Ast) = { @@ -141,6 +151,19 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) scope.addToScope(node.name, parameterIn) Ast(parameterIn) + case node: ProcParameter => + val parameterIn = parameterInNode( + node = node, + name = node.name, + code = code(node), + index = index, + isVariadic = false, + evaluationStrategy = EvaluationStrategies.BY_REFERENCE, + typeFullName = None + ) + scope.addToScope(node.name, parameterIn) + scope.setProcParam(node.name) + Ast(parameterIn) case node: CollectionParameter => val typeFullName = node match { case ArrayParameter(_) => prefixAsBuiltin("Array") @@ -252,7 +275,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val thisParameterAst = Ast( newThisParameterNode( @@ -268,8 +291,20 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val stmtBlockAst = astForMethodBody(node.body, optionalStatementList) + val anonProcParam = scope.anonProcParam.map { param => + val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + val nextIndex = + parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(1) + astForParameter(paramNode, nextIndex) + } + scope.popScope() - methodAst(method, thisParameterAst +: parameterAsts, stmtBlockAst, methodReturnNode(node, Defines.Any)) + methodAst( + method, + (thisParameterAst +: parameterAsts) ++ anonProcParam, + stmtBlockAst, + methodReturnNode(node, Defines.Any) + ) case targetNode => logger.warn( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index ceb21ca217e9..96bf0ead605a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -160,7 +160,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } def generatedNode: StatementList = node.expression .map { e => - val tmp = SimpleIdentifier(None)(e.span.spanStart(freshVariableName)) + val tmp = SimpleIdentifier(None)(e.span.spanStart(tmpGen.fresh)) StatementList( List(SingleAssignment(tmp, "=", e)(e.span)) ++ goCase(Some(tmp)) @@ -252,7 +252,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case node: MemberCallWithBlock => returnAstForRubyCall(node) case node: SimpleCallWithBlock => returnAstForRubyCall(node) case _: (LiteralExpr | BinaryExpression | UnaryExpression | SimpleIdentifier | IndexAccess | Association | - RubyCall) => + YieldExpr | RubyCall) => astForReturnStatement(ReturnExpression(List(node))(node.span)) :: Nil case node: SingleAssignment => astForSingleAssignment(node) :: List(astForReturnStatement(ReturnExpression(List(node.lhs))(node.span))) @@ -265,6 +265,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t case ret: ReturnExpression => astForReturnStatement(ret) :: Nil case node: MethodDeclaration => (astForMethodDeclaration(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList + case node => logger.warn( s"Implicit return here not supported yet: ${node.text} (${node.getClass.getSimpleName}), only generating statement" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 42d24d33e93e..426128fa3bdc 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -125,7 +125,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val block_ = blockNode(node) scope.pushNewScope(BlockScope(block_)) // TODO: Should it be `return this.@abc`? @@ -155,7 +155,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astParentType = scope.surroundingAstLabel, astParentFullName = scope.surroundingScopeFullName ) - scope.pushNewScope(MethodScope(fullName)) + scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) val parameter = parameterInNode(node, "x", "x", 1, false, EvaluationStrategies.BY_REFERENCE) val methodBody = { val block_ = blockNode(node) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala deleted file mode 100644 index 73045249f0a0..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/FreshVariableCreator.scala +++ /dev/null @@ -1,13 +0,0 @@ -package io.joern.rubysrc2cpg.astcreation - -trait FreshVariableCreator { this: AstCreator => - // This is in a single-threaded context. - private var varCounter: Int = 0 - - private def tmpVariableTemplate(id: Int): String = s"" - protected def freshVariableName: String = { - val name = tmpVariableTemplate(varCounter) - varCounter += 1 - name - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala index 315b8ed8c5d2..8c6bd5e43527 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala @@ -110,9 +110,7 @@ object RubyIntermediateAst { final case class HashParameter(name: String)(span: TextSpan) extends RubyNode(span) with CollectionParameter - final case class ProcParameter(target: RubyNode)(span: TextSpan) extends RubyNode(span) with MethodParameter { - def name: String = target.text - } + final case class ProcParameter(name: String)(span: TextSpan) extends RubyNode(span) with MethodParameter final case class SingleAssignment(lhs: RubyNode, op: String, rhs: RubyNode)(span: TextSpan) extends RubyNode(span) @@ -303,6 +301,8 @@ object RubyIntermediateAst { */ final case class ProcOrLambdaExpr(block: Block)(span: TextSpan) extends RubyNode(span) + final case class YieldExpr(arguments: List[RubyNode])(span: TextSpan) extends RubyNode(span) + /** Represents a call with a block argument. */ sealed trait RubyCallWithBlock[C <: RubyCall] extends RubyCall { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index 468cced554e8..9ce8bc4bfc6d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -4,6 +4,7 @@ import better.files.File import io.joern.rubysrc2cpg.astcreation.GlobalTypes import io.joern.rubysrc2cpg.astcreation.GlobalTypes.builtinPrefix import io.joern.x2cpg.Defines +import io.joern.rubysrc2cpg.passes.Defines as RDefines import io.joern.x2cpg.datastructures.* import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{DeclarationNew, NewLocal, NewMethodParameterIn} @@ -117,6 +118,46 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) case ScopeElement(x: MethodLikeScope, _) => x.fullName } + /** Locates a position in the stack matching a partial function, modifies it and emits a result + * @param pf + * Tests ScopeElements of the stack. If they match, return the new value and the result to emi + * @return + * the emitted result if the position was found and modifies + */ + def updateSurrounding[T]( + pf: PartialFunction[ + ScopeElement[String, DeclarationNew, TypedScopeElement], + (ScopeElement[String, DeclarationNew, TypedScopeElement], T) + ] + ): Option[T] = { + stack.zipWithIndex + .collectFirst { case (pf(elem, res), i) => + (elem, res, i) + } + .map { case (elem, res, i) => + stack = stack.updated(i, elem) + res + } + } + + /** Get the name of the implicit or explict proc param and mark the method scope as using the proc param + */ + def useProcParam: Option[String] = updateSurrounding { + case ScopeElement(MethodScope(fullName, param, _), variables) => + (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) + } + + /** Get the name of the implicit or explict proc param */ + def anonProcParam: Option[String] = stack.collectFirst { case ScopeElement(MethodScope(_, Left(param), true), _) => + param + } + + /** Set the name of explict proc param */ + def setProcParam(param: String): Unit = updateSurrounding { + case ScopeElement(MethodScope(fullName, _, _), variables) => + (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) + } + def surroundingTypeFullName: Option[String] = stack.collectFirst { case ScopeElement(x: TypeLikeScope, _) => x.fullName } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala index b88bbf1954ff..a6d07440f86a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala @@ -57,7 +57,8 @@ trait MethodLikeScope extends TypedScopeElement { def fullName: String } -case class MethodScope(fullName: String) extends MethodLikeScope +case class MethodScope(fullName: String, procParam: Either[String, String], hasYield: Boolean = false) + extends MethodLikeScope case class ConstructorScope(fullName: String) extends MethodLikeScope diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index 88df3121bb52..1c3d78079bfd 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -2,26 +2,21 @@ package io.joern.rubysrc2cpg.parser import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* import io.joern.rubysrc2cpg.parser.AntlrContextHelpers.* -import io.joern.rubysrc2cpg.parser.RubyParser.RangeOperatorContext import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType import org.antlr.v4.runtime.tree.{ParseTree, RuleNode} import io.joern.x2cpg.Defines as XDefines; import scala.jdk.CollectionConverters.* +import io.joern.rubysrc2cpg.utils.FreshNameGenerator /** Converts an ANTLR Ruby Parse Tree into the intermediate Ruby AST. */ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { - private var classCounter: Int = 0 - - private def tmpClassTemplate(id: Int): String = s"" - + private val classNameGen = FreshNameGenerator(id => s"") protected def freshClassName(span: TextSpan): SimpleIdentifier = { - val name = tmpClassTemplate(classCounter) - classCounter += 1 - SimpleIdentifier(None)(span.spanStart(name)) + SimpleIdentifier(None)(span.spanStart(classNameGen.fresh)) } private def defaultTextSpan(code: String = ""): TextSpan = TextSpan(None, None, None, None, code) @@ -523,6 +518,18 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } + override def visitYieldExpression(ctx: RubyParser.YieldExpressionContext): RubyNode = { + val arguments = Option(ctx.argumentWithParentheses()).iterator.flatMap(_.arguments).map(visit).toList + YieldExpr(arguments)(ctx.toTextSpan) + } + + override def visitYieldMethodInvocationWithoutParentheses( + ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext + ): RubyNode = { + val arguments = ctx.primaryValueList().primaryValue().asScala.map(visit).toList + YieldExpr(arguments)(ctx.toTextSpan) + } + override def visitConstantIdentifierVariable(ctx: RubyParser.ConstantIdentifierVariableContext): RubyNode = { SimpleIdentifier()(ctx.toTextSpan) } @@ -870,7 +877,9 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } override def visitProcParameter(ctx: RubyParser.ProcParameterContext): RubyNode = { - ProcParameter(visit(ctx.procParameterName()))(ctx.toTextSpan) + ProcParameter( + Option(ctx.procParameterName).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText()) + )(ctx.toTextSpan) } override def visitHashParameter(ctx: RubyParser.HashParameterContext): RubyNode = { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala index e97935881e6d..0c6b3091009d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala @@ -27,6 +27,8 @@ object Defines { val Resolver: String = "" + val AnonymousProcParameter = "" + def getBuiltInType(typeInString: String) = s"${GlobalTypes.builtinPrefix}.$typeInString" object RubyOperators { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala new file mode 100644 index 000000000000..a7e77e248d8a --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala @@ -0,0 +1,10 @@ +package io.joern.rubysrc2cpg.utils + +class FreshNameGenerator[T](template: Int => T) { + private var counter: Int = 0 + def fresh: T = { + val name = template(counter) + counter += 1 + name + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala new file mode 100644 index 000000000000..d03d7d33239b --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -0,0 +1,75 @@ +package io.joern.rubysrc2cpg.querying + +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import org.scalatest.Inspectors +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.nodes.* + +class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { + "Methods" should { + "with a yield expression" should { + "with a proc parameter" should { + val cpg1 = code("def foo(&b) yield end") + val cpg2 = code("def self.foo(&b) yield end") + val cpgs = List(cpg1, cpg2) + + "have a single block argument" in { + forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) + } + + "represent the yield as a conditional with a call and return node as children" in { + forAll(cpgs) { cpg => + inside(cpg.method("foo").call(".conditional").code("yield").astChildren.l) { + case List(cond: Expression, call: Call, ret: Return) => { + cond.code shouldBe "" + call.name shouldBe "b" + call.code shouldBe "yield" + ret.code shouldBe "yield" + } + } + } + } + } + + "without a proc parameter" should { + val cpg1 = code("def foo() yield end") + val cpg2 = code("def self.foo() yield end") + val cpgs = List(cpg1, cpg2) + + "have a call to a block parameter" in { + forAll(cpgs)(_.call.code("yield").astChildren.isCall.code("yield").name.l shouldBe List("")) + } + + "add a block argument" in { + forAll(cpgs.zipWithIndex) { (cpg, i) => + val List(param) = cpg.method("foo").parameter.code("&.*").l + param.name shouldBe "" + param.index shouldBe i + } + } + } + + "with yield arguments" should { + val cpg = code("def foo(x) yield(x) end") + "replace the yield with a call to the block parameter with arguments" in { + val List(call) = cpg.call.codeExact("yield(x)").astChildren.isCall.codeExact("yield(x)").l + call.name shouldBe "" + call.argument.code.l shouldBe List("x") + } + + } + } + + "that don't have a yield nor a proc parameter" should { + val cpg1 = code("def foo() end") + val cpg2 = code("def self.foo() end") + val cpgs = List(cpg1, cpg2) + + "not add a block argument" in { + forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l should be(empty)) + } + } + + } + +}