diff --git a/build.sbt b/build.sbt index 13fe87cbc6b9..82fbd26eafc1 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "joern" ThisBuild / organization := "io.joern" ThisBuild / scalaVersion := "3.3.1" -val cpgVersion = "1.6.2" +val cpgVersion = "1.6.4" lazy val joerncli = Projects.joerncli lazy val querydb = Projects.querydb diff --git a/console/build.sbt b/console/build.sbt index b6e1d75bd831..a6534ce94f81 100644 --- a/console/build.sbt +++ b/console/build.sbt @@ -8,7 +8,7 @@ val CirceVersion = "0.14.6" val ZeroturnaroundVersion = "1.15" val OsLibVersion = "0.9.1" val PprintVersion = "0.7.3" -val CommonsLangVersion = "3.12.0" +val CommonsLangVersion = "3.14.0" dependsOn( Projects.semanticcpg, diff --git a/console/src/main/scala/io/joern/console/Console.scala b/console/src/main/scala/io/joern/console/Console.scala index d84f55db7b32..c6733bbb2def 100644 --- a/console/src/main/scala/io/joern/console/Console.scala +++ b/console/src/main/scala/io/joern/console/Console.scala @@ -9,17 +9,19 @@ import io.joern.console.workspacehandling.{Project, WorkspaceLoader, WorkspaceMa import io.joern.x2cpg.X2Cpg.defaultOverlayCreators import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.cpgloading.CpgLoader -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.dotextension.ImageViewer import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import overflowdb.traversal.help.Doc +import overflowdb.traversal.help.Table.AvailableWidthProvider import scala.sys.process.Process import scala.util.control.NoStackTrace import scala.util.{Failure, Success, Try} -class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory) - extends Reporting { +class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory)(implicit + availableWidthProvider: AvailableWidthProvider +) extends Reporting { import Console._ diff --git a/console/src/main/scala/io/joern/console/Help.scala b/console/src/main/scala/io/joern/console/Help.scala index 97f4cba4bdc2..07f61382d0f2 100644 --- a/console/src/main/scala/io/joern/console/Help.scala +++ b/console/src/main/scala/io/joern/console/Help.scala @@ -1,14 +1,12 @@ package io.joern.console -import org.apache.commons.lang.WordUtils -import overflowdb.traversal.help.DocFinder._ -import overflowdb.traversal.help.{Table, DocFinder} +import overflowdb.traversal.help.DocFinder.* +import overflowdb.traversal.help.Table.AvailableWidthProvider +import overflowdb.traversal.help.{DocFinder, Table} object Help { - private val width = 80 - - def overview(clazz: Class[_]): String = { + def overview(clazz: Class[_])(using AvailableWidthProvider): String = { val columnNames = List("command", "description", "example") val rows = DocFinder .findDocumentedMethodsOf(clazz) @@ -39,7 +37,6 @@ object Help { def formatNoQuotes(text: String): String = { text.stripMargin .split("\n\n") - .map(x => WordUtils.wrap(x.replace("\n", " "), width)) .mkString("\n\n") .trim } diff --git a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala index 3e3b1fee8cd0..a678202e1bda 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala @@ -6,11 +6,14 @@ import io.joern.console.{ConsoleException, FrontendConfig, Reporting} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.Languages import overflowdb.traversal.help.Table +import overflowdb.traversal.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.util.{Failure, Success, Try} -class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Reporting { +class ImportCode[T <: Project](console: io.joern.console.Console[T])(implicit + availableWidthProvider: AvailableWidthProvider +) extends Reporting { import io.joern.console.Console.* private val config = console.config @@ -86,7 +89,9 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep ) // this is only abstract to force people adding frontends to make a decision whether the frontend consumes binaries or source - abstract class Frontend(val name: String, val language: String, val description: String = "") { + abstract class Frontend(val name: String, val language: String, val description: String = "")(implicit + availableWidthProvider: AvailableWidthProvider + ) { def cpgGeneratorForLanguage( language: String, config: FrontendConfig, @@ -101,7 +106,7 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep def apply(inputPath: String, projectName: String = "", args: List[String] = List()): Cpg = { val frontend = cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args) .getOrElse(throw new ConsoleException(s"no cpg generator for language=$language available!")) - new ImportCode(console)(frontend, inputPath, projectName) + new ImportCode(console).apply(frontend, inputPath, projectName) } } diff --git a/console/src/main/scala/io/joern/console/package.scala b/console/src/main/scala/io/joern/console/package.scala index 4d50c43e7a58..a29435afba29 100644 --- a/console/src/main/scala/io/joern/console/package.scala +++ b/console/src/main/scala/io/joern/console/package.scala @@ -1,11 +1,15 @@ package io.joern +import overflowdb.traversal.help.Table.AvailableWidthProvider import replpp.Operators.* import replpp.Colors -// TODO remove any time after the end of 2023 - this is completely deprecated package object console { + implicit val defaultAvailableWidthProvider: AvailableWidthProvider = + io.shiftleft.semanticcpg.defaultAvailableWidthProvider + + // TODO remove any time after the end of 2023 - this is completely deprecated implicit class UnixUtils[A](content: Iterable[A]) { given Colors = Colors.Default diff --git a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala index 19ffbd30df2a..0d3a068d4bc2 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala @@ -1,6 +1,8 @@ package io.joern.console.workspacehandling +import io.joern.console.defaultAvailableWidthProvider import overflowdb.traversal.help.Table +import overflowdb.traversal.help.Table.AvailableWidthProvider import scala.collection.mutable.ListBuffer @@ -8,7 +10,9 @@ import scala.collection.mutable.ListBuffer * @param projects * list of projects present in this workspace */ -class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType]) { +class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType])(implicit + availableWidthProvider: AvailableWidthProvider = defaultAvailableWidthProvider +) { /** Returns total number of projects in this workspace */ @@ -35,7 +39,6 @@ class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType]) { rows = projects.map(_.toTableRow).toList ).render } - } } diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala index eba2c2c996df..c0297fb43551 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala @@ -2,6 +2,7 @@ package io.joern.console.workspacehandling import better.files.Dsl.mkdirs import better.files.File +import overflowdb.traversal.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.collection.mutable.ListBuffer @@ -9,7 +10,7 @@ import scala.util.{Failure, Success, Try} /** This component loads a workspace from disk and creates a corresponding `Workspace` object. */ -abstract class WorkspaceLoader[ProjectType <: Project] { +abstract class WorkspaceLoader[ProjectType <: Project](implicit availableWidthProvider: AvailableWidthProvider) { /** Initialize workspace from a directory * @param path diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala index 70dc02f3ae3c..26b404d5efc0 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala @@ -3,6 +3,7 @@ package io.joern.console.workspacehandling import better.files.Dsl._ import better.files._ import io.joern.console +import io.joern.console.defaultAvailableWidthProvider import io.joern.console.Reporting import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} diff --git a/console/src/test/scala/io/joern/console/ConsoleTests.scala b/console/src/test/scala/io/joern/console/ConsoleTests.scala index 5db23f15d6dd..88ca197a1d6d 100644 --- a/console/src/test/scala/io/joern/console/ConsoleTests.scala +++ b/console/src/test/scala/io/joern/console/ConsoleTests.scala @@ -46,7 +46,7 @@ class ConsoleTests extends AnyWordSpec with Matchers { } "provide overview of available language modules" in ConsoleFixture() { (console, _) => - console.importCode.toString.contains("| C") shouldBe true + console.importCode.toString should include("testCFrontend") } "allow importing code with specific module (c2cpg)" in ConsoleFixture() { (console, codeDir) => @@ -393,6 +393,7 @@ class ConsoleTests extends AnyWordSpec with Matchers { "provide .help command" in ConsoleFixture() { (console, codeDir) => // part of Predefined.shared, which makes the below work in the repl without separate import import io.shiftleft.codepropertygraph.Cpg.docSearchPackages + import io.joern.console.testing.availableWidthProvider console.importCode(codeDir.toString) val nodeStartersHelp = console.cpg.help diff --git a/console/src/test/scala/io/joern/console/HelpTests.scala b/console/src/test/scala/io/joern/console/HelpTests.scala index 299e56e819f9..6d95f78d355f 100644 --- a/console/src/test/scala/io/joern/console/HelpTests.scala +++ b/console/src/test/scala/io/joern/console/HelpTests.scala @@ -5,10 +5,11 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec class HelpTests extends AnyWordSpec with Matchers { + import testing.availableWidthProvider "Help" should { "provide overview of commands as table" in { - Help.overview(classOf[io.joern.console.Console[Project]]).contains("| CPG") shouldBe true + Help.overview(classOf[io.joern.console.Console[Project]]) should include("CPG of the active project") } } diff --git a/console/src/test/scala/io/joern/console/testing/package.scala b/console/src/test/scala/io/joern/console/testing/package.scala index b30149b4a961..897bb9e9633c 100644 --- a/console/src/test/scala/io/joern/console/testing/package.scala +++ b/console/src/test/scala/io/joern/console/testing/package.scala @@ -1,12 +1,16 @@ package io.joern.console -import better.files.Dsl._ -import better.files._ + +import better.files.Dsl.* +import better.files.* import io.joern.console.workspacehandling.Project +import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} import scala.util.Try package object testing { + implicit val availableWidthProvider: AvailableWidthProvider = ConstantWidth(120) + object WithStandaloneCpg { def apply(console: Console[Project], codeDir: File)(fun: File => Unit): Unit = { val tmpProjectName = "standalonecpg" diff --git a/console/src/test/scala/io/joern/console/workspacehandling/TestLoader.scala b/console/src/test/scala/io/joern/console/workspacehandling/TestLoader.scala index 97618a20f5dd..12e643b222b4 100644 --- a/console/src/test/scala/io/joern/console/workspacehandling/TestLoader.scala +++ b/console/src/test/scala/io/joern/console/workspacehandling/TestLoader.scala @@ -1,5 +1,6 @@ package io.joern.console.workspacehandling +import io.joern.console.testing.availableWidthProvider import java.nio.file.Path object TestLoader { diff --git a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala index a73c2441c0ab..5b1930a9455f 100644 --- a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala +++ b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala @@ -2,6 +2,7 @@ package io.joern.console.workspacehandling import better.files.Dsl._ import better.files.File +import io.joern.console.testing.availableWidthProvider import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -26,11 +27,12 @@ class WorkspaceTests extends AnyWordSpec with Matchers { val projects = ListBuffer(Project(projectFile, project.path, Some(cpg))) val workspace = new Workspace(projects) val output = workspace.toString - val lines = output.split("\n") - lines.length shouldBe 5 - lines(4).contains(project.name) shouldBe true - lines(4).contains(inputPath) - lines(4).contains("foo,bar") + + output should include(project.name) + output should include(inputPath) + // TODO reenable test - this has been broken for ages, the overlays handling via directories works in staged joern but not in our test setup + // what does work in both though is `cpg.metaData.overlays` - I'm asking around which way forward we want +// output should include("foo,bar") } } diff --git a/dataflowengineoss/build.sbt b/dataflowengineoss/build.sbt index ea00f783fa71..1820abbbfd86 100644 --- a/dataflowengineoss/build.sbt +++ b/dataflowengineoss/build.sbt @@ -3,6 +3,7 @@ name := "dataflowengineoss" dependsOn(Projects.semanticcpg, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( + "org.antlr" % "antlr4" % Versions.antlr, "org.antlr" % "antlr4-runtime" % Versions.antlr, "io.circe" %% "circe-core" % Versions.circe, "io.circe" %% "circe-generic" % Versions.circe, diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala index 38c8b11a1199..06409a6ea508 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala @@ -1,9 +1,10 @@ package io.joern.dataflowengineoss.language import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member, MethodParameterIn} -import io.shiftleft.semanticcpg.language._ -import org.apache.commons.lang.StringUtils +import io.shiftleft.semanticcpg +import io.shiftleft.semanticcpg.language.* import overflowdb.traversal.help.Table +import overflowdb.traversal.help.Table.AvailableWidthProvider case class Path(elements: List[AstNode]) { def resultPairs(): List[(String, Option[Integer])] = { @@ -21,12 +22,10 @@ case class Path(elements: List[AstNode]) { object Path { - val DefaultMaxTrackedWidth = 30 - // TODO replace with dynamic rendering based on the terminal's width, e.g. in scala-repl-pp - lazy val maxTrackedWidth = sys.env.get("JOERN_DATAFLOW_TRACKED_WIDTH").map(_.toInt).getOrElse(DefaultMaxTrackedWidth) - - implicit val show: Show[Path] = { path => - Table( + implicit def show(implicit + availableWidthProvider: AvailableWidthProvider = semanticcpg.defaultAvailableWidthProvider + ): Show[Path] = { path => + val table = Table( columnNames = Array("nodeType", "tracked", "lineNumber", "method", "file"), rows = path.elements.map { astNode => val nodeType = astNode.getClass.getSimpleName @@ -47,11 +46,12 @@ object Path { s"$methodName($paramsPretty)" case _ => cfgNode.statement.repr } - val tracked = StringUtils.normalizeSpace(StringUtils.abbreviate(statement, maxTrackedWidth)) - Array(nodeType, tracked, lineNumber, methodName, fileName) + Array(nodeType, statement, lineNumber, methodName, fileName) } } - ).render + ) + // add a line break for nicer repl rendering + "\n" + table.render } } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala index fe266da422ae..b219e62d40be 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala @@ -6,7 +6,7 @@ import io.joern.x2cpg.{Ast, SourceFiles, ValidationMode} import io.joern.x2cpg.utils.NodeBuilders.newDependencyNode import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.utils.IOUtils -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.c.{ICASTArrayDesignator, ICASTDesignatedInitializer, ICASTFieldDesignator} import org.eclipse.cdt.core.dom.ast.cpp.* diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala index 8622039731ce..7b2853357239 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala @@ -5,7 +5,7 @@ import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes} -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTLambdaExpression import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala index b5b9fb1f754f..9001ddcd0855 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ } import io.joern.x2cpg.{Ast, AstEdge, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.NewLocal -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast.{IASTMacroExpansionLocation, IASTNode, IASTPreprocessorMacroDefinition} import org.eclipse.cdt.core.dom.ast.IASTBinaryExpression import org.eclipse.cdt.internal.core.model.ASTStringUtil diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala index d02ca85034a8..5d9e0e4cd7be 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala @@ -1,11 +1,12 @@ package io.joern.c2cpg.dataflow import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import overflowdb.traversal.help.Table.AvailableWidthProvider import overflowdb.traversal.toNodeTraversal class DataFlowTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala index 5578f57ab6ad..882285ba0366 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala @@ -7,7 +7,7 @@ import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.{NewModifier, NewNode} import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes, PropertyNames} -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import ujson.Value import scala.collection.mutable diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala index 9f2cb2bde878..f06b82f37a0f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala @@ -5,7 +5,7 @@ import io.joern.gosrc2cpg.parser.ParserNodeInfo import io.joern.x2cpg.astgen.AstGenNodeBuilder import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.NodeTypes -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import overflowdb.BatchedUpdate.DiffGraphBuilder import ujson.Value diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala index e1e7e79b75f9..0502a4e36200 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala @@ -4,7 +4,7 @@ import java.nio.file.{Path, Paths} import org.slf4j.LoggerFactory import com.fasterxml.jackson.databind.ObjectMapper import io.shiftleft.utils.IOUtils -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import scala.collection.concurrent.TrieMap import scala.util.Try diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractDomPassTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractDomPassTest.scala index a3215f7df352..34cfb852fec7 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractDomPassTest.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractDomPassTest.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.Expression import io.shiftleft.codepropertygraph.generated.nodes.TemplateDom import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language._ -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils abstract class AbstractDomPassTest extends AbstractPassTest { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala index 0fea604b5d70..c37e4ab90742 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala @@ -23,7 +23,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ NewTypeRef, NewUnknown } -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import scala.util.Try trait AstNodeBuilder[Node, NodeProcessor] { this: NodeProcessor => diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala index 3d6e801d1a12..bd56e63ebf26 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala @@ -1,6 +1,7 @@ package io.joern.joerncli.console import better.files._ +import io.joern.console.defaultAvailableWidthProvider import io.joern.console.workspacehandling.{ProjectFile, WorkspaceLoader} import io.joern.console.{Console, ConsoleConfig, InstallConfig} import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} diff --git a/project/Versions.scala b/project/Versions.scala index 9730721dd11b..4f712070e005 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -11,7 +11,7 @@ object Versions { val circe = "0.14.5" val requests = "0.8.0" val upickle = "3.1.3" - val scalaReplPP = "0.1.76" + val scalaReplPP = "0.1.77" val typeSafeConfig = "1.4.2" val versionSort = "1.0.11" diff --git a/semanticcpg/build.sbt b/semanticcpg/build.sbt index 732643040445..e5b9f918c504 100644 --- a/semanticcpg/build.sbt +++ b/semanticcpg/build.sbt @@ -4,6 +4,7 @@ libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, "org.json4s" %% "json4s-native" % Versions.json4s, + "org.apache.commons" % "commons-text" % "1.11.0", "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala index 5791966ae81d..6ee27691b66f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.utils.MemberAccess -import org.apache.commons.lang.StringEscapeUtils +import org.apache.commons.text.StringEscapeUtils import java.util.Optional import scala.collection.immutable.HashMap @@ -50,7 +50,7 @@ object DotSerializer { private def namedGraphBegin(root: AstNode): mutable.StringBuilder = { val sb = new mutable.StringBuilder - val name = StringEscapeUtils.escapeHtml(root match { + val name = StringEscapeUtils.escapeHtml4(root match { case method: Method => method.name case _ => "" }) @@ -71,7 +71,7 @@ object DotSerializer { private def stringRepr(vertex: StoredNode): String = { val maybeLineNo: Optional[AnyRef] = vertex.propertyOption(PropertyNames.LINE_NUMBER) - StringEscapeUtils.escapeHtml(vertex match { + StringEscapeUtils.escapeHtml4(vertex match { case call: Call => (call.name, limit(call.code)).toString case contrl: ControlStructure => (contrl.label, contrl.controlStructureType, contrl.code).toString case expr: Expression => (expr.label, limit(expr.code), limit(toCfgNode(expr).code)).toString @@ -111,16 +111,16 @@ object DotSerializer { private def edgeToDot(edge: Edge, withEdgeTypes: Boolean): String = { val edgeLabel = if (withEdgeTypes) { - edge.edgeType + ": " + StringEscapeUtils.escapeHtml(edge.label) + edge.edgeType + ": " + StringEscapeUtils.escapeHtml4(edge.label) } else { - StringEscapeUtils.escapeHtml(edge.label) + StringEscapeUtils.escapeHtml4(edge.label) } val labelStr = Some(s""" [ label = "$edgeLabel"] """).filter(_ => edgeLabel != "").getOrElse("") s""" "${edge.src.id}" -> "${edge.dst.id}" """ + labelStr } def nodesToSubGraphs(subgraph: String, children: Seq[StoredNode], idx: Int): String = { - val escapedName = StringEscapeUtils.escapeHtml(subgraph) + val escapedName = StringEscapeUtils.escapeHtml4(subgraph) val childString = children.map { c => s" \"${c.id()}\";" }.mkString("\n") s""" subgraph cluster_$idx { |$childString diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala index 79ee059b3f23..be26a307cb1e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala @@ -1,5 +1,7 @@ package io.shiftleft +import overflowdb.traversal.help.Table.AvailableWidthProvider + /** Domain specific language for querying code property graphs * * This is the API reference for the CPG query language, a language to mine code for defects and vulnerabilities both @@ -85,4 +87,10 @@ package io.shiftleft * query.toList * }}} */ -package object semanticcpg {} + +package object semanticcpg { + + implicit val defaultAvailableWidthProvider: AvailableWidthProvider = + () => replpp.util.terminalWidth.filter(_ > 0).getOrElse(120) + +} diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala index 3561f1abd1e0..af5d052f8a27 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala @@ -9,6 +9,7 @@ import org.json4s.* import org.json4s.native.JsonMethods.parse import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} import scala.jdk.CollectionConverters.IteratorHasAsScala @@ -190,6 +191,8 @@ class StepsTest extends AnyWordSpec with Matchers { } ".help step" should { + implicit val availableWidthProvider: AvailableWidthProvider = new ConstantWidth(120) + "show domain overview" in { val domainStartersHelp = Cpg.emptyCpg.help domainStartersHelp should include(".comment") @@ -206,7 +209,7 @@ class StepsTest extends AnyWordSpec with Matchers { val methodStepsHelpVerbose = Cpg.emptyCpg.method.helpVerbose methodStepsHelpVerbose should include("traversal name") - methodStepsHelpVerbose should include("io.shiftleft.semanticcpg.language.types.structure.Method") + methodStepsHelpVerbose should include("structure.MethodTraversal") val assignmentStepsHelp = Cpg.emptyCpg.assignment.help assignmentStepsHelp should include("Left-hand sides of assignments") // from AssignmentTraversal