dynamically pass through available terminal width for help table rend…

…ering (#4143)

build.sbt

@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.3.1"
 
-val cpgVersion = "1.6.2"
+val cpgVersion = "1.6.4"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb

console/build.sbt

@@ -8,7 +8,7 @@ val CirceVersion          = "0.14.6"
 val ZeroturnaroundVersion = "1.15"
 val OsLibVersion          = "0.9.1"
 val PprintVersion         = "0.7.3"
-val CommonsLangVersion    = "3.12.0"
+val CommonsLangVersion    = "3.14.0"
 
 dependsOn(
   Projects.semanticcpg,

console/src/main/scala/io/joern/console/Console.scala

@@ -9,17 +9,19 @@ import io.joern.console.workspacehandling.{Project, WorkspaceLoader, WorkspaceMa
 import io.joern.x2cpg.X2Cpg.defaultOverlayCreators
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.dotextension.ImageViewer
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import overflowdb.traversal.help.Doc
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 import scala.sys.process.Process
 import scala.util.control.NoStackTrace
 import scala.util.{Failure, Success, Try}
 
-class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory)
-    extends Reporting {
+class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory)(implicit
+  availableWidthProvider: AvailableWidthProvider
+) extends Reporting {
 
   import Console._
 

console/src/main/scala/io/joern/console/Help.scala

@@ -1,14 +1,12 @@
 package io.joern.console
 
-import org.apache.commons.lang.WordUtils
-import overflowdb.traversal.help.DocFinder._
-import overflowdb.traversal.help.{Table, DocFinder}
+import overflowdb.traversal.help.DocFinder.*
+import overflowdb.traversal.help.Table.AvailableWidthProvider
+import overflowdb.traversal.help.{DocFinder, Table}
 
 object Help {
 
-  private val width = 80
-
-  def overview(clazz: Class[_]): String = {
+  def overview(clazz: Class[_])(using AvailableWidthProvider): String = {
     val columnNames = List("command", "description", "example")
     val rows = DocFinder
       .findDocumentedMethodsOf(clazz)
@@ -39,7 +37,6 @@ object Help {
   def formatNoQuotes(text: String): String = {
     text.stripMargin
       .split("\n\n")
-      .map(x => WordUtils.wrap(x.replace("\n", " "), width))
       .mkString("\n\n")
       .trim
   }

console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala

@@ -6,11 +6,14 @@ import io.joern.console.{ConsoleException, FrontendConfig, Reporting}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.Languages
 import overflowdb.traversal.help.Table
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 import java.nio.file.Path
 import scala.util.{Failure, Success, Try}
 
-class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Reporting {
+class ImportCode[T <: Project](console: io.joern.console.Console[T])(implicit
+  availableWidthProvider: AvailableWidthProvider
+) extends Reporting {
   import io.joern.console.Console.*
 
   private val config             = console.config
@@ -86,7 +89,9 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep
     )
 
   // this is only abstract to force people adding frontends to make a decision whether the frontend consumes binaries or source
-  abstract class Frontend(val name: String, val language: String, val description: String = "") {
+  abstract class Frontend(val name: String, val language: String, val description: String = "")(implicit
+    availableWidthProvider: AvailableWidthProvider
+  ) {
     def cpgGeneratorForLanguage(
       language: String,
       config: FrontendConfig,
@@ -101,7 +106,7 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep
     def apply(inputPath: String, projectName: String = "", args: List[String] = List()): Cpg = {
       val frontend = cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args)
         .getOrElse(throw new ConsoleException(s"no cpg generator for language=$language available!"))
-      new ImportCode(console)(frontend, inputPath, projectName)
+      new ImportCode(console).apply(frontend, inputPath, projectName)
     }
   }
 

console/src/main/scala/io/joern/console/package.scala

@@ -1,11 +1,15 @@
 package io.joern
 
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 import replpp.Operators.*
 import replpp.Colors
 
-// TODO remove any time after the end of 2023 - this is completely deprecated
 package object console {
 
+  implicit val defaultAvailableWidthProvider: AvailableWidthProvider =
+    io.shiftleft.semanticcpg.defaultAvailableWidthProvider
+
+  // TODO remove any time after the end of 2023 - this is completely deprecated
   implicit class UnixUtils[A](content: Iterable[A]) {
     given Colors = Colors.Default
 

console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala

@@ -1,14 +1,18 @@
 package io.joern.console.workspacehandling
 
+import io.joern.console.defaultAvailableWidthProvider
 import overflowdb.traversal.help.Table
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 import scala.collection.mutable.ListBuffer
 
 /** Create a workspace from a list of projects. Workspace is a passive object that is managed by WorkspaceManager
   * @param projects
   *   list of projects present in this workspace
   */
-class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType]) {
+class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType])(implicit
+  availableWidthProvider: AvailableWidthProvider = defaultAvailableWidthProvider
+) {
 
   /** Returns total number of projects in this workspace
     */
@@ -35,7 +39,6 @@ class Workspace[ProjectType <: Project](var projects: ListBuffer[ProjectType]) {
         rows = projects.map(_.toTableRow).toList
       ).render
     }
-
   }
 
 }

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala

@@ -2,14 +2,15 @@ package io.joern.console.workspacehandling
 
 import better.files.Dsl.mkdirs
 import better.files.File
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 import java.nio.file.Path
 import scala.collection.mutable.ListBuffer
 import scala.util.{Failure, Success, Try}
 
 /** This component loads a workspace from disk and creates a corresponding `Workspace` object.
   */
-abstract class WorkspaceLoader[ProjectType <: Project] {
+abstract class WorkspaceLoader[ProjectType <: Project](implicit availableWidthProvider: AvailableWidthProvider) {
 
   /** Initialize workspace from a directory
     * @param path

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala

@@ -3,6 +3,7 @@ package io.joern.console.workspacehandling
 import better.files.Dsl._
 import better.files._
 import io.joern.console
+import io.joern.console.defaultAvailableWidthProvider
 import io.joern.console.Reporting
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}

console/src/test/scala/io/joern/console/ConsoleTests.scala

@@ -46,7 +46,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
     }
 
     "provide overview of available language modules" in ConsoleFixture() { (console, _) =>
-      console.importCode.toString.contains("| C") shouldBe true
+      console.importCode.toString should include("testCFrontend")
     }
 
     "allow importing code with specific module (c2cpg)" in ConsoleFixture() { (console, codeDir) =>
@@ -393,6 +393,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
     "provide .help command" in ConsoleFixture() { (console, codeDir) =>
       // part of Predefined.shared, which makes the below work in the repl without separate import
       import io.shiftleft.codepropertygraph.Cpg.docSearchPackages
+      import io.joern.console.testing.availableWidthProvider
 
       console.importCode(codeDir.toString)
       val nodeStartersHelp = console.cpg.help

console/src/test/scala/io/joern/console/HelpTests.scala

@@ -5,10 +5,11 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
 class HelpTests extends AnyWordSpec with Matchers {
+  import testing.availableWidthProvider
 
   "Help" should {
     "provide overview of commands as table" in {
-      Help.overview(classOf[io.joern.console.Console[Project]]).contains("| CPG") shouldBe true
+      Help.overview(classOf[io.joern.console.Console[Project]]) should include("CPG of the active project")
     }
   }
 

console/src/test/scala/io/joern/console/testing/package.scala

@@ -1,12 +1,16 @@
 package io.joern.console
-import better.files.Dsl._
-import better.files._
+
+import better.files.Dsl.*
+import better.files.*
 import io.joern.console.workspacehandling.Project
+import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth}
 
 import scala.util.Try
 
 package object testing {
 
+  implicit val availableWidthProvider: AvailableWidthProvider = ConstantWidth(120)
+
   object WithStandaloneCpg {
     def apply(console: Console[Project], codeDir: File)(fun: File => Unit): Unit = {
       val tmpProjectName = "standalonecpg"

console/src/test/scala/io/joern/console/workspacehandling/TestLoader.scala

@@ -1,5 +1,6 @@
 package io.joern.console.workspacehandling
 
+import io.joern.console.testing.availableWidthProvider
 import java.nio.file.Path
 
 object TestLoader {

console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala

@@ -2,6 +2,7 @@ package io.joern.console.workspacehandling
 
 import better.files.Dsl._
 import better.files.File
+import io.joern.console.testing.availableWidthProvider
 import io.shiftleft.semanticcpg.testing.MockCpg
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
@@ -26,11 +27,12 @@ class WorkspaceTests extends AnyWordSpec with Matchers {
         val projects    = ListBuffer(Project(projectFile, project.path, Some(cpg)))
         val workspace   = new Workspace(projects)
         val output      = workspace.toString
-        val lines       = output.split("\n")
-        lines.length shouldBe 5
-        lines(4).contains(project.name) shouldBe true
-        lines(4).contains(inputPath)
-        lines(4).contains("foo,bar")
+
+        output should include(project.name)
+        output should include(inputPath)
+        // TODO reenable test - this has been broken for ages, the overlays handling via directories works in staged joern but not in our test setup
+        // what does work in both though is `cpg.metaData.overlays` - I'm asking around which way forward we want
+//        output should include("foo,bar")
       }
 
     }

dataflowengineoss/build.sbt

@@ -3,6 +3,7 @@ name := "dataflowengineoss"
 dependsOn(Projects.semanticcpg, Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
+  "org.antlr"               % "antlr4"                     % Versions.antlr,
   "org.antlr"               % "antlr4-runtime"             % Versions.antlr,
   "io.circe"               %% "circe-core"                 % Versions.circe,
   "io.circe"               %% "circe-generic"              % Versions.circe,

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala

@@ -1,9 +1,10 @@
 package io.joern.dataflowengineoss.language
 
 import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member, MethodParameterIn}
-import io.shiftleft.semanticcpg.language._
-import org.apache.commons.lang.StringUtils
+import io.shiftleft.semanticcpg
+import io.shiftleft.semanticcpg.language.*
 import overflowdb.traversal.help.Table
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 case class Path(elements: List[AstNode]) {
   def resultPairs(): List[(String, Option[Integer])] = {
@@ -21,12 +22,10 @@ case class Path(elements: List[AstNode]) {
 
 object Path {
 
-  val DefaultMaxTrackedWidth = 30
-  // TODO replace with dynamic rendering based on the terminal's width, e.g. in scala-repl-pp
-  lazy val maxTrackedWidth = sys.env.get("JOERN_DATAFLOW_TRACKED_WIDTH").map(_.toInt).getOrElse(DefaultMaxTrackedWidth)
-
-  implicit val show: Show[Path] = { path =>
-    Table(
+  implicit def show(implicit
+    availableWidthProvider: AvailableWidthProvider = semanticcpg.defaultAvailableWidthProvider
+  ): Show[Path] = { path =>
+    val table = Table(
       columnNames = Array("nodeType", "tracked", "lineNumber", "method", "file"),
       rows = path.elements.map { astNode =>
         val nodeType   = astNode.getClass.getSimpleName
@@ -47,11 +46,12 @@ object Path {
                 s"$methodName($paramsPretty)"
               case _ => cfgNode.statement.repr
             }
-            val tracked = StringUtils.normalizeSpace(StringUtils.abbreviate(statement, maxTrackedWidth))
-            Array(nodeType, tracked, lineNumber, methodName, fileName)
+            Array(nodeType, statement, lineNumber, methodName, fileName)
         }
       }
-    ).render
+    )
+    // add a line break for nicer repl rendering
+    "\n" + table.render
   }
 
 }

joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala

@@ -6,7 +6,7 @@ import io.joern.x2cpg.{Ast, SourceFiles, ValidationMode}
 import io.joern.x2cpg.utils.NodeBuilders.newDependencyNode
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.utils.IOUtils
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 import org.eclipse.cdt.core.dom.ast.*
 import org.eclipse.cdt.core.dom.ast.c.{ICASTArrayDesignator, ICASTDesignatedInitializer, ICASTFieldDesignator}
 import org.eclipse.cdt.core.dom.ast.cpp.*

joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala

@@ -5,7 +5,7 @@ import io.joern.x2cpg.datastructures.Stack.*
 import io.joern.x2cpg.utils.NodeBuilders.newModifierNode
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes}
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 import org.eclipse.cdt.core.dom.ast.*
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTLambdaExpression
 import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator

joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala

@@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{
 }
 import io.joern.x2cpg.{Ast, AstEdge, ValidationMode}
 import io.shiftleft.codepropertygraph.generated.nodes.NewLocal
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 import org.eclipse.cdt.core.dom.ast.{IASTMacroExpansionLocation, IASTNode, IASTPreprocessorMacroDefinition}
 import org.eclipse.cdt.core.dom.ast.IASTBinaryExpression
 import org.eclipse.cdt.internal.core.model.ASTStringUtil

joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala

@@ -1,11 +1,12 @@
 package io.joern.c2cpg.dataflow
 
 import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite
-import io.joern.dataflowengineoss.language._
+import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext}
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Identifier, Literal}
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
+import overflowdb.traversal.help.Table.AvailableWidthProvider
 import overflowdb.traversal.toNodeTraversal
 
 class DataFlowTests extends DataFlowCodeToCpgSuite {

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreatorHelper.scala

@@ -7,7 +7,7 @@ import io.joern.x2cpg.utils.NodeBuilders.newModifierNode
 import io.joern.x2cpg.{Ast, Defines as XDefines}
 import io.shiftleft.codepropertygraph.generated.nodes.{NewModifier, NewNode}
 import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes, PropertyNames}
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 import ujson.Value
 
 import scala.collection.mutable

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala

@@ -5,7 +5,7 @@ import io.joern.gosrc2cpg.parser.ParserNodeInfo
 import io.joern.x2cpg.astgen.AstGenNodeBuilder
 import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, Defines as XDefines}
 import io.shiftleft.codepropertygraph.generated.NodeTypes
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 import ujson.Value
 

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala

@@ -4,7 +4,7 @@ import java.nio.file.{Path, Paths}
 import org.slf4j.LoggerFactory
 import com.fasterxml.jackson.databind.ObjectMapper
 import io.shiftleft.utils.IOUtils
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 
 import scala.collection.concurrent.TrieMap
 import scala.util.Try

joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractDomPassTest.scala

@@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.Expression
 import io.shiftleft.codepropertygraph.generated.nodes.TemplateDom
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.language._
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 
 abstract class AbstractDomPassTest extends AbstractPassTest {
 

joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala

@@ -23,7 +23,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{
   NewTypeRef,
   NewUnknown
 }
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 
 import scala.util.Try
 trait AstNodeBuilder[Node, NodeProcessor] { this: NodeProcessor =>