Skip to content

Commit

Permalink
Extracted shortenCode to x2cpg (#3789)
Browse files Browse the repository at this point in the history 
Can now be configured using `JOERN_MAX_CODE_LENGTH` env variable.
  • Loading branch information
@max-leuthaeuser
max-leuthaeuser authored Nov 3, 2023
1 parent 34ba835 commit b9722a6
Show file tree
Hide file tree
Showing 29 changed files with 261 additions and 241 deletions.
27 changes: 13 additions & 14 deletions joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ object AstCreatorHelper {

trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator =>

import AstCreatorHelper._
import io.joern.c2cpg.astcreation.AstCreatorHelper._

private var usedVariablePostfix: Int = 0

Expand Down Expand Up @@ -88,6 +88,8 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
SourceFiles.toRelativePath(path, config.inputPath)
}

protected def code(node: IASTNode): String = shortenCode(nodeSignature(node))

protected def line(node: IASTNode): Option[Integer] = {
nullSafeFileLocation(node).map(_.getStartingLineNumber)
}
Expand Down Expand Up @@ -232,9 +234,6 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
}
}

protected def shortenCode(code: String, length: Int = MaxCodeLength): String =
StringUtils.abbreviate(code, math.max(MinCodeLength, length))

private def notHandledText(node: IASTNode): String =
s"""Node '${node.getClass.getSimpleName}' not handled yet!
| Code: '${node.getRawSignature}'
Expand All @@ -247,11 +246,11 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
val text = notHandledText(node)
logger.info(text)
}
Ast(unknownNode(node, nodeSignature(node)))
Ast(unknownNode(node, code(node)))
}

protected def nullSafeCode(node: IASTNode): String = {
Option(node).map(nodeSignature).getOrElse("")
Option(node).map(code).getOrElse("")
}

protected def nullSafeAst(node: IASTExpression, argIndex: Int): Ast = {
Expand Down Expand Up @@ -339,7 +338,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
s"${fullName(e.getParent)}.${ASTStringUtil.getSimpleName(e.getName)}"
case d: IASTIdExpression => ASTStringUtil.getSimpleName(d.getName)
case _: IASTTranslationUnit => ""
case u: IASTUnaryExpression => nodeSignature(u.getOperand)
case u: IASTUnaryExpression => code(u.getOperand)
case other if other.getParent != null => fullName(other.getParent)
case other if other != null => notHandledYet(other); ""
case null => ""
Expand Down Expand Up @@ -406,7 +405,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
allIncludes.map { include =>
val name = include.getName.toString
val _dependencyNode = newDependencyNode(name, name, IncludeKeyword)
val importNode = newImportNode(nodeSignature(include), name, name, include)
val importNode = newImportNode(code(include), name, name, include)
diffGraph.addNode(_dependencyNode)
diffGraph.addEdge(importNode, _dependencyNode, EdgeTypes.IMPORTS)
Ast(importNode)
Expand All @@ -423,7 +422,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As

private def astForDecltypeSpecifier(decl: ICPPASTDecltypeSpecifier): Ast = {
val op = "<operator>.typeOf"
val cpgUnary = callNode(decl, nodeSignature(decl), op, op, DispatchTypes.STATIC_DISPATCH)
val cpgUnary = callNode(decl, code(decl), op, op, DispatchTypes.STATIC_DISPATCH)
val operand = nullSafeAst(decl.getDecltypeExpression)
callAst(cpgUnary, List(operand))
}
Expand All @@ -434,7 +433,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
val op = Operators.assignment
val calls = withIndex(d.getDesignators) { (des, o) =>
val callNode_ =
callNode(d, nodeSignature(d), op, op, DispatchTypes.STATIC_DISPATCH)
callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH)
.argumentIndex(o)
val left = astForNode(des)
val right = astForNode(d.getOperand)
Expand All @@ -450,7 +449,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
val op = Operators.assignment
val calls = withIndex(d.getDesignators) { (des, o) =>
val callNode_ =
callNode(d, nodeSignature(d), op, op, DispatchTypes.STATIC_DISPATCH)
callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH)
.argumentIndex(o)
val left = astForNode(des)
val right = astForNode(d.getOperand)
Expand All @@ -463,22 +462,22 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
private def astForCPPASTConstructorInitializer(c: ICPPASTConstructorInitializer): Ast = {
val name = "<operator>.constructorInitializer"
val callNode_ =
callNode(c, nodeSignature(c), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH)
val args = c.getArguments.toList.map(a => astForNode(a))
callAst(callNode_, args)
}

private def astForCASTArrayRangeDesignator(des: CASTArrayRangeDesignator): Ast = {
val op = Operators.arrayInitializer
val callNode_ = callNode(des, nodeSignature(des), op, op, DispatchTypes.STATIC_DISPATCH)
val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH)
val floorAst = nullSafeAst(des.getRangeFloor)
val ceilingAst = nullSafeAst(des.getRangeCeiling)
callAst(callNode_, List(floorAst, ceilingAst))
}

private def astForCPPASTArrayRangeDesignator(des: CPPASTArrayRangeDesignator): Ast = {
val op = Operators.arrayInitializer
val callNode_ = callNode(des, nodeSignature(des), op, op, DispatchTypes.STATIC_DISPATCH)
val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH)
val floorAst = nullSafeAst(des.getRangeFloor)
val ceilingAst = nullSafeAst(des.getRangeCeiling)
callAst(callNode_, List(floorAst, ceilingAst))
Expand Down
40 changes: 17 additions & 23 deletions .../frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForExpressionsCreator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
case _ => "<operator>.unknown"
}

val callNode_ = callNode(bin, nodeSignature(bin), op, op, DispatchTypes.STATIC_DISPATCH)
val callNode_ = callNode(bin, code(bin), op, op, DispatchTypes.STATIC_DISPATCH)
val left = nullSafeAst(bin.getOperand1)
val right = nullSafeAst(bin.getOperand2)
callAst(callNode_, List(left, right))
Expand All @@ -58,7 +58,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
private def astForExpressionList(exprList: IASTExpressionList): Ast = {
val name = "<operator>.expressionList"
val callNode_ =
callNode(exprList, nodeSignature(exprList), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(exprList, code(exprList), name, name, DispatchTypes.STATIC_DISPATCH)
val childAsts = exprList.getExpressions.map(nullSafeAst)
callAst(callNode_, childAsts.toIndexedSeq)
}
Expand Down Expand Up @@ -91,12 +91,12 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
.isInstanceOf[IASTFieldReference] =>
(
DispatchTypes.STATIC_DISPATCH,
nodeSignature(call.getFunctionNameExpression.asInstanceOf[IASTFieldReference].getFieldName)
code(call.getFunctionNameExpression.asInstanceOf[IASTFieldReference].getFieldName)
)
case _ if rec.root.exists(_.isInstanceOf[NewCall]) =>
(DispatchTypes.STATIC_DISPATCH, rec.root.get.asInstanceOf[NewCall].code)
case reference: IASTIdExpression =>
(DispatchTypes.STATIC_DISPATCH, nodeSignature(reference))
(DispatchTypes.STATIC_DISPATCH, code(reference))
case _ =>
(DispatchTypes.STATIC_DISPATCH, "")
}
Expand All @@ -106,7 +106,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
case t if t != Defines.anyTypeName => s"${dereferenceTypeFullName(t)}.$shortName"
case _ => shortName
}
val cpgCall = callNode(call, nodeSignature(call), shortName, fullName, dd)
val cpgCall = callNode(call, code(call), shortName, fullName, dd)
val args = call.getArguments.toList.map(a => astForNode(a))
rec.root match {
// Optimization: do not include the receiver if the receiver is just the function name,
Expand Down Expand Up @@ -149,7 +149,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
nullSafeAst(unary.getOperand)
} else {
val cpgUnary =
callNode(unary, nodeSignature(unary), operatorMethod, operatorMethod, DispatchTypes.STATIC_DISPATCH)
callNode(unary, code(unary), operatorMethod, operatorMethod, DispatchTypes.STATIC_DISPATCH)
val operand = nullSafeAst(unary.getOperand)
callAst(cpgUnary, List(operand))
}
Expand All @@ -164,7 +164,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
op == IASTTypeIdExpression.op_alignof ||
op == IASTTypeIdExpression.op_typeof =>
val call =
callNode(typeId, nodeSignature(typeId), Operators.sizeOf, Operators.sizeOf, DispatchTypes.STATIC_DISPATCH)
callNode(typeId, code(typeId), Operators.sizeOf, Operators.sizeOf, DispatchTypes.STATIC_DISPATCH)
val arg = astForNode(typeId.getTypeId.getDeclSpecifier)
callAst(call, List(arg))
case _ => notHandledYet(typeId)
Expand All @@ -173,7 +173,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {

private def astForConditionalExpression(expr: IASTConditionalExpression): Ast = {
val name = Operators.conditional
val call = callNode(expr, nodeSignature(expr), name, name, DispatchTypes.STATIC_DISPATCH)
val call = callNode(expr, code(expr), name, name, DispatchTypes.STATIC_DISPATCH)

val condAst = nullSafeAst(expr.getLogicalConditionExpression)
val posAst = nullSafeAst(expr.getPositiveResultExpression)
Expand All @@ -186,7 +186,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
private def astForArrayIndexExpression(arrayIndexExpression: IASTArraySubscriptExpression): Ast = {
val name = Operators.indirectIndexAccess
val cpgArrayIndexing =
callNode(arrayIndexExpression, nodeSignature(arrayIndexExpression), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(arrayIndexExpression, code(arrayIndexExpression), name, name, DispatchTypes.STATIC_DISPATCH)

val expr = astForExpression(arrayIndexExpression.getArrayExpression)
val arg = astForNode(arrayIndexExpression.getArgument)
Expand All @@ -195,17 +195,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {

private def astForCastExpression(castExpression: IASTCastExpression): Ast = {
val cpgCastExpression =
callNode(
castExpression,
nodeSignature(castExpression),
Operators.cast,
Operators.cast,
DispatchTypes.STATIC_DISPATCH
)
callNode(castExpression, code(castExpression), Operators.cast, Operators.cast, DispatchTypes.STATIC_DISPATCH)

val expr = astForExpression(castExpression.getOperand)
val argNode = castExpression.getTypeId
val arg = unknownNode(argNode, nodeSignature(argNode))
val arg = unknownNode(argNode, code(argNode))

callAst(cpgCastExpression, List(Ast(arg), expr))
}
Expand All @@ -225,7 +219,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
private def astForNewExpression(newExpression: ICPPASTNewExpression): Ast = {
val name = "<operator>.new"
val cpgNewExpression =
callNode(newExpression, nodeSignature(newExpression), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(newExpression, code(newExpression), name, name, DispatchTypes.STATIC_DISPATCH)

val typeId = newExpression.getTypeId
if (newExpression.isArrayAllocation) {
Expand All @@ -242,24 +236,24 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
private def astForDeleteExpression(delExpression: ICPPASTDeleteExpression): Ast = {
val name = Operators.delete
val cpgDeleteNode =
callNode(delExpression, nodeSignature(delExpression), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(delExpression, code(delExpression), name, name, DispatchTypes.STATIC_DISPATCH)
val arg = astForExpression(delExpression.getOperand)
callAst(cpgDeleteNode, List(arg))
}

private def astForTypeIdInitExpression(typeIdInit: IASTTypeIdInitializerExpression): Ast = {
val name = Operators.cast
val cpgCastExpression =
callNode(typeIdInit, nodeSignature(typeIdInit), name, name, DispatchTypes.STATIC_DISPATCH)
callNode(typeIdInit, code(typeIdInit), name, name, DispatchTypes.STATIC_DISPATCH)

val typeAst = unknownNode(typeIdInit.getTypeId, nodeSignature(typeIdInit.getTypeId))
val typeAst = unknownNode(typeIdInit.getTypeId, code(typeIdInit.getTypeId))
val expr = astForNode(typeIdInit.getInitializer)
callAst(cpgCastExpression, List(Ast(typeAst), expr))
}

private def astForConstructorExpression(c: ICPPASTSimpleTypeConstructorExpression): Ast = {
val name = c.getDeclSpecifier.toString
val callNode_ = callNode(c, nodeSignature(c), name, name, DispatchTypes.STATIC_DISPATCH)
val callNode_ = callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH)
val arg = astForNode(c.getInitializer)
callAst(callNode_, List(arg))
}
Expand Down Expand Up @@ -302,7 +296,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {

protected def astForStaticAssert(a: ICPPASTStaticAssertDeclaration): Ast = {
val name = "static_assert"
val call = callNode(a, nodeSignature(a), name, name, DispatchTypes.STATIC_DISPATCH)
val call = callNode(a, code(a), name, name, DispatchTypes.STATIC_DISPATCH)
val cond = nullSafeAst(a.getCondition)
val messg = nullSafeAst(a.getMessage)
callAst(call, List(cond, messg))
Expand Down
34 changes: 21 additions & 13 deletions ...li/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
}
val (name, fullname) = uniqueName("lambda", "", fullName(lambdaExpression))
val signature = s"$returnType $fullname ${parameterListSignature(lambdaExpression)}"
val code = nodeSignature(lambdaExpression)
val methodNode_ = methodNode(lambdaExpression, name, code, fullname, Some(signature), filename)
val codeString = code(lambdaExpression)
val methodNode_ = methodNode(lambdaExpression, name, codeString, fullname, Some(signature), filename)

scope.pushNewScope(methodNode_)
val parameterNodes = withIndex(parameters(lambdaExpression.getDeclarator)) { (p, i) =>
Expand All @@ -123,7 +123,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
val typeDeclAst = createFunctionTypeAndTypeDecl(lambdaExpression, methodNode_, name, fullname, signature)
Ast.storeInDiffGraph(astForLambda.merge(typeDeclAst), diffGraph)

Ast(methodRefNode(lambdaExpression, code, fullname, methodNode_.astParentFullName))
Ast(methodRefNode(lambdaExpression, codeString, fullname, methodNode_.astParentFullName))
}

protected def astForFunctionDeclarator(funcDecl: IASTFunctionDeclarator): Ast = {
Expand All @@ -135,9 +135,9 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th

if (seenFunctionSignatures.add(signature)) {
val name = shortName(funcDecl)
val code = nodeSignature(funcDecl.getParent)
val codeString = code(funcDecl.getParent)
val filename = fileName(funcDecl)
val methodNode_ = methodNode(funcDecl, name, code, fullname, Some(signature), filename)
val methodNode_ = methodNode(funcDecl, name, codeString, fullname, Some(signature), filename)

scope.pushNewScope(methodNode_)

Expand Down Expand Up @@ -167,8 +167,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
s"$returnType $fullname$templateParams ${parameterListSignature(funcDef)}"
seenFunctionSignatures.add(signature)

val code = nodeSignature(funcDef)
val methodNode_ = methodNode(funcDef, name, code, fullname, Some(signature), filename)
val codeString = code(funcDef)
val methodNode_ = methodNode(funcDef, name, codeString, fullname, Some(signature), filename)

methodAstParentStack.push(methodNode_)
scope.pushNewScope(methodNode_)
Expand All @@ -193,18 +193,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
}

private def parameterNode(parameter: IASTNode, paramIndex: Int): NewMethodParameterIn = {
val (name, code, tpe, variadic) = parameter match {
val (name, codeString, tpe, variadic) = parameter match {
case p: CASTParameterDeclaration =>
(
ASTStringUtil.getSimpleName(p.getDeclarator.getName),
nodeSignature(p),
code(p),
cleanType(typeForDeclSpecifier(p.getDeclSpecifier)),
false
)
case p: CPPASTParameterDeclaration =>
(
ASTStringUtil.getSimpleName(p.getDeclarator.getName),
nodeSignature(p),
code(p),
cleanType(typeForDeclSpecifier(p.getDeclSpecifier)),
p.getDeclarator.declaresParameterPack()
)
Expand All @@ -213,16 +213,24 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
s.getDeclarators.headOption
.map(n => ASTStringUtil.getSimpleName(n.getName))
.getOrElse(uniqueName("parameter", "", "")._1),
nodeSignature(s),
code(s),
cleanType(typeForDeclSpecifier(s)),
false
)
case other =>
(nodeSignature(other), nodeSignature(other), cleanType(typeForDeclSpecifier(other)), false)
(code(other), code(other), cleanType(typeForDeclSpecifier(other)), false)
}

val parameterNode =
parameterInNode(parameter, name, code, paramIndex, variadic, EvaluationStrategies.BY_VALUE, registerType(tpe))
parameterInNode(
parameter,
name,
codeString,
paramIndex,
variadic,
EvaluationStrategies.BY_VALUE,
registerType(tpe)
)
scope.addToScope(name, (parameterNode, tpe))
parameterNode
}
Expand Down
Loading

0 comments on commit b9722a6

Please sign in to comment.