From 339a0c715c751d66bbb284a65c2e9754985b8e3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Max=20Leuth=C3=A4user?=
<1417198+max-leuthaeuser@users.noreply.github.com>
Date: Mon, 19 Feb 2024 13:27:52 +0100
Subject: [PATCH] [jssrc2cpg] Enable file content and content for method and
type decl (#4186)
Fixes: https://github.com/joernio/joern/issues/4185
---
.../jssrc2cpg/astcreation/AstCreator.scala | 19 ++++-
.../astcreation/AstCreatorHelper.scala | 16 ++--
.../io/CodeDumperFromContentTest.scala | 78 +++++++++++++++++++
...est.scala => CodeDumperFromFileTest.scala} | 2 +-
.../scala/io/joern/x2cpg/AstNodeBuilder.scala | 6 +-
5 files changed, 112 insertions(+), 9 deletions(-)
create mode 100644 joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala
rename joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/{CodeDumperTest.scala => CodeDumperFromFileTest.scala} (97%)
diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala
index 005307d290e1..82b04979505d 100644
--- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala
+++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala
@@ -62,7 +62,9 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse
positionLookupTables(parserResult.fileContent)
override def createAst(): DiffGraphBuilder = {
- val fileNode = NewFile().name(parserResult.filename).order(1)
+ val fileContent = if (!config.disableFileContent) Option(parserResult.fileContent) else None
+ val fileNode = NewFile().name(parserResult.filename).order(1)
+ fileContent.foreach(fileNode.content(_))
val namespaceBlock = globalNamespaceBlock()
methodAstParentStack.push(namespaceBlock)
val ast = Ast(fileNode).withChild(Ast(namespaceBlock).withChild(createProgramMethod()))
@@ -257,4 +259,19 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse
protected def lineEnd(node: BabelNodeInfo): Option[Integer] = node.lineNumberEnd
protected def columnEnd(node: BabelNodeInfo): Option[Integer] = node.columnNumberEnd
protected def code(node: BabelNodeInfo): String = node.code
+
+ protected def nodeOffsets(node: Value): Option[(Int, Int)] = {
+ for {
+ startOffset <- start(node)
+ endOffset <- end(node)
+ } yield (startOffset, endOffset)
+ }
+
+ override protected def offset(node: BabelNodeInfo): Option[(Int, Int)] = {
+ Option
+ .when(!config.disableFileContent) {
+ nodeOffsets(node.json)
+ }
+ .flatten
+ }
}
diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala
index c512e98a2dc3..aa33ef29256f 100644
--- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala
+++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala
@@ -8,9 +8,10 @@ import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode}
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies}
+import io.shiftleft.codepropertygraph.generated.nodes.File.PropertyDefaults
import ujson.Value
-import scala.collection.{SortedMap, mutable}
+import scala.collection.{mutable, SortedMap}
import scala.util.{Success, Try}
trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator =>
@@ -70,9 +71,12 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
}
protected def code(node: Value): String = {
- val startIndex = start(node).getOrElse(0)
- val endIndex = Math.min(end(node).getOrElse(0), parserResult.fileContent.length)
- shortenCode(parserResult.fileContent.substring(startIndex, endIndex).trim)
+ nodeOffsets(node) match {
+ case Some((startOffset, endOffset))
+ if startOffset < endOffset && startOffset >= 0 && endOffset <= parserResult.fileContent.length =>
+ shortenCode(parserResult.fileContent.substring(startOffset, endOffset).trim)
+ case _ => PropertyDefaults.Code
+ }
}
protected def hasKey(node: Value, key: String): Boolean = Try(node(key)).isSuccess
@@ -90,9 +94,9 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
case _ => None
}
- private def start(node: Value): Option[Int] = Try(node("start").num.toInt).toOption
+ protected def start(node: Value): Option[Int] = Try(node("start").num.toInt).toOption
- private def end(node: Value): Option[Int] = Try(node("end").num.toInt).toOption
+ protected def end(node: Value): Option[Int] = Try(node("end").num.toInt).toOption
protected def pos(node: Value): Option[Int] = Try(node("start").num.toInt).toOption
diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala
new file mode 100644
index 000000000000..dd9f484474ac
--- /dev/null
+++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala
@@ -0,0 +1,78 @@
+package io.joern.jssrc2cpg.io
+
+import io.joern.jssrc2cpg.testfixtures.JsSrc2CpgSuite
+import io.joern.jssrc2cpg.Config
+import io.shiftleft.semanticcpg.language.*
+
+class CodeDumperFromContentTest extends JsSrc2CpgSuite {
+
+ private implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder
+
+ "dumping code from content" should {
+ val cpg = code(
+ """
+ |// A comment
+ |function my_func(param1)
+ |{
+ | var x = foo(param1);
+ |}""".stripMargin,
+ "index.js"
+ ).withConfig(Config().withDisableFileContent(false))
+
+ "allow one to dump a method node's source code from `File.contents`" in {
+ inside(cpg.method.nameExact("my_func").dumpRaw.l) {
+ case content :: Nil =>
+ content.linesIterator.map(_.strip).l shouldBe List(
+ "function my_func(param1) /* <=== index.js::program:my_func */",
+ "{",
+ "var x = foo(param1);",
+ "}"
+ )
+ case content => fail(s"Expected exactly 1 content dump, but got: $content")
+ }
+ }
+ }
+
+ "code from method content" should {
+ val myFuncContent =
+ """function my_func(param1)
+ |{
+ | var x = foo(param1);
+ |}""".stripMargin
+
+ val cpg = code(
+ s"""
+ |// A comment
+ |$myFuncContent
+ |""".stripMargin,
+ "index.js"
+ ).withConfig(Config().withDisableFileContent(false))
+
+ "allow one to dump a method node's source code from `Method.content`" in {
+ val List(content) = cpg.method.nameExact("my_func").content.l
+ content shouldBe myFuncContent
+ }
+ }
+
+ "code from typedecl content" should {
+ val myClassContent =
+ """class Foo
+ |{
+ | x = 'foo';
+ |}""".stripMargin
+
+ val cpg = code(
+ s"""
+ |// A comment
+ |$myClassContent
+ |""".stripMargin,
+ "index.js"
+ ).withConfig(Config().withDisableFileContent(false))
+
+ "allow one to dump a method node's source code from `TypeDecl.content`" in {
+ val List(content) = cpg.typeDecl.nameExact("Foo").content.l
+ content shouldBe myClassContent
+ }
+ }
+
+}
diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTest.scala
similarity index 97%
rename from joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperTest.scala
rename to joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTest.scala
index 8c82e9458385..3e0865672ae9 100644
--- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperTest.scala
+++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTest.scala
@@ -7,7 +7,7 @@ import io.shiftleft.semanticcpg.language._
import java.util.regex.Pattern
-class CodeDumperTest extends JsSrc2CpgSuite {
+class CodeDumperFromFileTest extends JsSrc2CpgSuite {
implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala
index c37e4ab90742..109b9095d914 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala
+++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala
@@ -142,7 +142,7 @@ trait AstNodeBuilder[Node, NodeProcessor] { this: NodeProcessor =>
inherits: Seq[String] = Seq.empty,
alias: Option[String] = None
): NewTypeDecl = {
- NewTypeDecl()
+ val node_ = NewTypeDecl()
.name(name)
.fullName(fullName)
.code(code)
@@ -154,6 +154,10 @@ trait AstNodeBuilder[Node, NodeProcessor] { this: NodeProcessor =>
.aliasTypeFullName(alias)
.lineNumber(line(node))
.columnNumber(column(node))
+ offset(node).foreach { case (offset, offsetEnd) =>
+ node_.offset(offset).offsetEnd(offsetEnd)
+ }
+ node_
}
protected def parameterInNode(