Adde DYNAMIC_TYPE_HINT_FULL_NAME on literal nodes. (#3832)

* Adde DYNAMIC_TYPE_HINT_FULL_NAME on literal nodes. This is required as type anchor for type resolution. * Add support for bytes literals.
joernio · Nov 17, 2023 · 83a1f76 · 83a1f76
1 parent 9fa5315
commit 83a1f76
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 33 deletions.
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Constants.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Constants.scala
@@ -1,6 +1,11 @@
 package io.joern.pysrc2cpg
 
 object Constants {
-  val ANY              = "ANY"
-  val GLOBAL_NAMESPACE = "<global>"
+  val ANY                = "ANY"
+  val GLOBAL_NAMESPACE   = "<global>"
+  val builtinStrType     = "builtin.str"
+  val builtinBytesType   = "builtin.bytes"
+  val builtinIntType     = "builtin.int"
+  val builtinFloatType   = "builtin.float"
+  val builtinComplexType = "builtin.complex"
 }
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala
@@ -223,28 +223,35 @@ class NodeBuilder(diffGraph: DiffGraphBuilder) {
     addNodeToDiff(fieldIdentifierNode)
   }
 
-  def numberLiteralNode(number: Int, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
-    numberLiteralNode(number.toString, lineAndColumn)
-  }
-
-  def numberLiteralNode(number: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
+  def literalNode(string: String, dynamicTypeHint: Option[String], lineAndColumn: LineAndColumn): nodes.NewLiteral = {
     val literalNode = nodes
       .NewLiteral()
-      .code(number)
+      .code(string)
       .typeFullName(Constants.ANY)
+      .dynamicTypeHintFullName(dynamicTypeHint.toList)
       .lineNumber(lineAndColumn.line)
       .columnNumber(lineAndColumn.column)
     addNodeToDiff(literalNode)
   }
 
   def stringLiteralNode(string: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
-    val literalNode = nodes
-      .NewLiteral()
-      .code(string)
-      .typeFullName(Constants.ANY)
-      .lineNumber(lineAndColumn.line)
-      .columnNumber(lineAndColumn.column)
-    addNodeToDiff(literalNode)
+    literalNode(string, Some(Constants.builtinStrType), lineAndColumn)
+  }
+
+  def bytesLiteralNode(string: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
+    literalNode(string, Some(Constants.builtinBytesType), lineAndColumn)
+  }
+
+  def intLiteralNode(string: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
+    literalNode(string, Some(Constants.builtinIntType), lineAndColumn)
+  }
+
+  def floatLiteralNode(string: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
+    literalNode(string, Some(Constants.builtinFloatType), lineAndColumn)
+  }
+
+  def complexLiteralNode(string: String, lineAndColumn: LineAndColumn): nodes.NewLiteral = {
+    literalNode(string, Some(Constants.builtinComplexType), lineAndColumn)
   }
 
   def blockNode(code: String, lineAndColumn: LineAndColumn): nodes.NewBlock = {

diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala
@@ -1816,25 +1816,32 @@ class PythonAstVisitor(relFileName: String, protected val nodeToCode: NodeToCode
   def convert(constant: ast.Constant): nodes.NewNode = {
     constant.value match {
       case stringConstant: ast.StringConstant =>
-        nodeBuilder.stringLiteralNode(
-          stringConstant.prefix + stringConstant.quote + stringConstant.value + stringConstant.quote,
-          lineAndColOf(constant)
-        )
+        if (stringConstant.prefix.contains("b") || stringConstant.prefix.contains("B")) {
+          nodeBuilder.bytesLiteralNode(
+            stringConstant.prefix + stringConstant.quote + stringConstant.value + stringConstant.quote,
+            lineAndColOf(constant)
+          )
+        } else {
+          nodeBuilder.stringLiteralNode(
+            stringConstant.prefix + stringConstant.quote + stringConstant.value + stringConstant.quote,
+            lineAndColOf(constant)
+          )
+        }
       case stringConstant: ast.JoinedStringConstant =>
         nodeBuilder.stringLiteralNode(stringConstant.value, lineAndColOf(constant))
       case boolConstant: ast.BoolConstant =>
         val boolStr = if (boolConstant.value) "True" else "False"
-        nodeBuilder.stringLiteralNode(boolStr, lineAndColOf(constant))
+        nodeBuilder.intLiteralNode(boolStr, lineAndColOf(constant))
       case intConstant: ast.IntConstant =>
-        nodeBuilder.numberLiteralNode(intConstant.value, lineAndColOf(constant))
+        nodeBuilder.intLiteralNode(intConstant.value, lineAndColOf(constant))
       case floatConstant: ast.FloatConstant =>
-        nodeBuilder.numberLiteralNode(floatConstant.value, lineAndColOf(constant))
+        nodeBuilder.floatLiteralNode(floatConstant.value, lineAndColOf(constant))
       case imaginaryConstant: ast.ImaginaryConstant =>
-        nodeBuilder.numberLiteralNode(imaginaryConstant.value + "j", lineAndColOf(constant))
+        nodeBuilder.complexLiteralNode(imaginaryConstant.value + "j", lineAndColOf(constant))
       case ast.NoneConstant =>
-        nodeBuilder.numberLiteralNode("None", lineAndColOf(constant))
+        nodeBuilder.literalNode("None", None, lineAndColOf(constant))
       case ast.EllipsisConstant =>
-        nodeBuilder.numberLiteralNode("...", lineAndColOf(constant))
+        nodeBuilder.literalNode("...", None, lineAndColOf(constant))
     }
   }
 

diff --git a/...n-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitorHelpers.scala b/...n-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitorHelpers.scala
@@ -548,7 +548,7 @@ trait PythonAstVisitorHelpers { this: PythonAstVisitor =>
     accessChain match {
       case accessIndex :: tail =>
         val baseNode  = createIndexAccessChain(rootNode, tail, lineAndColumn)
-        val indexNode = nodeBuilder.numberLiteralNode(accessIndex, lineAndColumn)
+        val indexNode = nodeBuilder.intLiteralNode(accessIndex.toString, lineAndColumn)
 
         createIndexAccess(baseNode, indexNode, lineAndColumn)
       case Nil =>

diff --git a/...-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala b/...-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala
@@ -0,0 +1,19 @@
+package io.joern.pysrc2cpg.cpg
+
+import io.joern.pysrc2cpg.{Constants, Py2CpgTestContext}
+import io.shiftleft.semanticcpg.language.*
+import org.scalatest.freespec.AnyFreeSpec
+import org.scalatest.matchers.should.Matchers
+
+class BytesLiteralCpgTests extends AnyFreeSpec with Matchers {
+  lazy val cpg = Py2CpgTestContext.buildCpg("""b"123"""".stripMargin)
+
+  "test num literal node properties" in {
+    val literal = cpg.literal.head
+    literal.code shouldBe "b\"123\""
+    literal.typeFullName shouldBe Constants.ANY
+    literal.dynamicTypeHintFullName should contain only (Constants.builtinBytesType)
+    literal.lineNumber shouldBe Some(1)
+    literal.columnNumber shouldBe Some(1)
+  }
+}
diff --git a/.../io/joern/pysrc2cpg/cpg/NumCpgTests.scala → ...rn/pysrc2cpg/cpg/IntLiteralCpgTests.scala b/.../io/joern/pysrc2cpg/cpg/NumCpgTests.scala → ...rn/pysrc2cpg/cpg/IntLiteralCpgTests.scala
@@ -1,16 +1,18 @@
 package io.joern.pysrc2cpg.cpg
 
-import io.joern.pysrc2cpg.Py2CpgTestContext
-import io.shiftleft.semanticcpg.language._
+import io.joern.pysrc2cpg.{Constants, Py2CpgTestContext}
+import io.shiftleft.semanticcpg.language.*
 import org.scalatest.freespec.AnyFreeSpec
 import org.scalatest.matchers.should.Matchers
 
-class NumCpgTests extends AnyFreeSpec with Matchers {
+class IntLiteralCpgTests extends AnyFreeSpec with Matchers {
   lazy val cpg = Py2CpgTestContext.buildCpg("""1""".stripMargin)
 
-  "test num literal node properties" in {
+  "test int literal node properties" in {
     val literal = cpg.literal.head
     literal.code shouldBe "1"
+    literal.typeFullName shouldBe Constants.ANY
+    literal.dynamicTypeHintFullName should contain only (Constants.builtinIntType)
     literal.lineNumber shouldBe Some(1)
     literal.columnNumber shouldBe Some(1)
   }

diff --git a/.../io/joern/pysrc2cpg/cpg/StrCpgTests.scala → ...rn/pysrc2cpg/cpg/StrLiteralCpgTests.scala b/.../io/joern/pysrc2cpg/cpg/StrCpgTests.scala → ...rn/pysrc2cpg/cpg/StrLiteralCpgTests.scala
@@ -1,16 +1,18 @@
 package io.joern.pysrc2cpg.cpg
 
-import io.joern.pysrc2cpg.Py2CpgTestContext
-import io.shiftleft.semanticcpg.language._
+import io.joern.pysrc2cpg.{Constants, Py2CpgTestContext}
+import io.shiftleft.semanticcpg.language.*
 import org.scalatest.freespec.AnyFreeSpec
 import org.scalatest.matchers.should.Matchers
 
-class StrCpgTests extends AnyFreeSpec with Matchers {
+class StrLiteralCpgTests extends AnyFreeSpec with Matchers {
   lazy val cpg = Py2CpgTestContext.buildCpg(""""abc"""".stripMargin)
 
   "test string literal node properties" in {
     val literal = cpg.literal.head
     literal.code shouldBe "\"abc\""
+    literal.typeFullName shouldBe Constants.ANY
+    literal.dynamicTypeHintFullName should contain only (Constants.builtinStrType)
     literal.lineNumber shouldBe Some(1)
     literal.columnNumber shouldBe Some(1)
   }