add test domains: starting with gratefuldead

.gitignore

@@ -1,19 +1,6 @@
-/target/
-/project/target/
+target
 /project/project/
-/benchJoern/target/
-/domain-classes-generator/target/
-/domain-classes-generator-codescience/target/
-/odb-convert/target/
-/joern-generated/target/
 /codescience-generated/src/main/scala
-/codescience-generated/target/
-/benchmarks/target/
-/core/target/
-/formats/target/
-/schema/target/
-/domain-classes-generator*/target/
-/sbt-flatgraph/target/
 /cpg.bin
 /cpg.bin.tmp
 /cpg.fg

build.sbt

@@ -12,10 +12,13 @@ val scala2_12 = "2.12.18"
 lazy val root = (project in file(".")).aggregate(
     core,
     formats,
+    formatsTests,
     domainClassesGenerator_3,
     domainClassesGenerator_2_12,
     sbtPlugin,
-    odbConvert
+    odbConvert,
+    testSchemas,
+    testSchemasDomainClasses,
   )
 
 lazy val core = project
@@ -43,10 +46,18 @@ lazy val formats = project
       "org.scala-lang.modules" %% "scala-xml" % "2.1.0",
       "io.spray" %% "spray-json" % "1.3.6",
       "com.github.scopt" %% "scopt" % "4.1.0",
-      "com.github.pathikrit" %% "better-files" % "3.9.2" % Test,
     )
   )
 
+lazy val formatsTests = project
+  .in(file("formats-tests"))
+  .dependsOn(formats, testSchemasDomainClasses)
+  .settings(
+    name := "flatgraph-formats-tests",
+    publish / skip := true,
+    libraryDependencies += "com.github.pathikrit" %% "better-files" % "3.9.2" % Test
+  )
+
 lazy val domainClassesGenerator_3 = project
   .in(file("domain-classes-generator_3"))
   .settings(
@@ -100,6 +111,23 @@ lazy val odbConvert = project
     )
   )
 
+lazy val testSchemas = project
+  .in(file("test-schemas"))
+  .dependsOn(domainClassesGenerator_3)
+  .settings(
+    name := "test-schemas",
+    scalaVersion := scala3,
+    publish / skip := true,
+  )
+
+lazy val testSchemasDomainClasses = project
+  .in(file("test-schemas-domain-classes"))
+  .dependsOn(core)
+  .settings(
+    name := "test-schemas-domain-classes",
+    publish / skip := true
+  )
+
 /** temporarily we still want to keep the generated files for the cpg domain in here,
   * in order to be able to quickly see the differences in the generated files if we
   * change the codegen

domain-classes-generator/src/main/scala/flatgraph/codegen/DomainClassesGenerator.scala

@@ -465,7 +465,7 @@ class DomainClassesGenerator(schema: Schema) {
       val nodeSource = {
         s"""package $basePackage.nodes
              |
-             |import io.shiftleft.codepropertygraph.generated.Language.*
+             |import $basePackage.Language.*
              |import scala.collection.immutable.{IndexedSeq, ArraySeq}
              |
              |$erasedMarkerType
@@ -792,6 +792,7 @@ class DomainClassesGenerator(schema: Schema) {
 
     // domain object and starters: start
     // TODO: extract into separate method
+    val domainShortName = schema.domainShortName
     val sanitizeReservedNames = Map("return" -> "ret", "type" -> "typ", "import" -> "imports").withDefault(identity)
     val starters              = mutable.ArrayBuffer[String]()
     nodeTypes.zipWithIndex.collect { case (typ, idx) =>
@@ -801,7 +802,7 @@ class DomainClassesGenerator(schema: Schema) {
         starters.append(
           s"""/** $comment */
              |@flatgraph.help.Doc(info = \"\"\"$comment\"\"\")
-             |def $starterName: Iterator[nodes.${typ.className}] = wrappedCpg.graph._nodes($idx).asInstanceOf[Iterator[nodes.${typ.className}]]""".stripMargin
+             |def $starterName: Iterator[nodes.${typ.className}] = wrapped$domainShortName.graph._nodes($idx).asInstanceOf[Iterator[nodes.${typ.className}]]""".stripMargin
         )
 
         // starter for primary key property (if defined) of this concrete node type
@@ -838,7 +839,6 @@ class DomainClassesGenerator(schema: Schema) {
       }
     }
 
-    val domainShortName = schema.domainShortName
     val domainMain =
       s"""package $basePackage
          |import flatgraph.DiffGraphBuilder
@@ -896,10 +896,10 @@ class DomainClassesGenerator(schema: Schema) {
          |}
          |
          |@flatgraph.help.TraversalSource
-         |class ${domainShortName}NodeStarters(val wrappedCpg: $domainShortName) {
+         |class ${domainShortName}NodeStarters(val wrapped$domainShortName: $domainShortName) {
          |
          |  @flatgraph.help.Doc(info = "all nodes")
-         |  def all: Iterator[nodes.StoredNode] = wrappedCpg.graph.allNodes.asInstanceOf[Iterator[nodes.StoredNode]]
+         |  def all: Iterator[nodes.StoredNode] = wrapped$domainShortName.graph.allNodes.asInstanceOf[Iterator[nodes.StoredNode]]
          |
          |${starters.mkString("\n\n")}
          |}
@@ -918,7 +918,7 @@ class DomainClassesGenerator(schema: Schema) {
          |  with neighboraccessors.Conversions
          |  with flatgraph.traversal.Language
          |  with flatgraph.Implicits {
-         |    implicit def cpgToGeneratedNodeStarters(cpg: Cpg): CpgNodeStarters = CpgNodeStarters(cpg)
+         |    implicit def toGeneratedNodeStarters(domain: $domainShortName): ${domainShortName}NodeStarters = ${domainShortName}NodeStarters(domain)
          |  }
          |
          |object Language extends Language

generateDomainClasses.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-sbt domainClassesGeneratorJoern/run
+sbt testSchemas/run domainClassesGeneratorJoern/run

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/Accessors.scala

@@ -0,0 +1,40 @@
+package flatgraph.testdomains.gratefuldead.accessors
+import flatgraph.testdomains.gratefuldead.nodes
+import scala.collection.immutable.IndexedSeq
+
+object Lang extends ConcreteStoredConversions
+
+object Accessors {
+  /* accessors for concrete stored nodes start */
+  final class Access_Property_Name(val node: nodes.StoredNode) extends AnyVal {
+    def name: String = flatgraph.Accessors.getNodePropertySingle(node.graph, node.nodeKind, 0, node.seq(), "": String)
+  }
+  /* accessors for concrete stored nodes end */
+
+  /* accessors for base nodes start */
+  final class Access_ArtistBase(val node: nodes.ArtistBase) extends AnyVal {
+    def name: String = node match {
+      case stored: nodes.StoredNode => new Access_Property_Name(stored).name
+      case newNode: nodes.NewArtist => newNode.name
+    }
+  }
+  final class Access_SongBase(val node: nodes.SongBase) extends AnyVal {
+    def name: String = node match {
+      case stored: nodes.StoredNode => new Access_Property_Name(stored).name
+      case newNode: nodes.NewSong   => newNode.name
+    }
+  }
+  /* accessors for base nodes end */
+}
+
+trait ConcreteStoredConversions extends ConcreteBaseConversions {
+  import Accessors.*
+  implicit def accessPropertyName(node: nodes.StoredNode & nodes.StaticType[nodes.HasNameEMT]): Access_Property_Name =
+    new Access_Property_Name(node)
+}
+
+trait ConcreteBaseConversions {
+  import Accessors.*
+  implicit def access_ArtistBase(node: nodes.ArtistBase): Access_ArtistBase = new Access_ArtistBase(node)
+  implicit def access_SongBase(node: nodes.SongBase): Access_SongBase = new Access_SongBase(node)
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/BaseTypes.scala

@@ -0,0 +1,6 @@
+package flatgraph.testdomains.gratefuldead.nodes
+
+/** Node types with this marker trait are guaranteed to have the Name property. EMT stands for: "erased marker trait",
+  * it exists only at compile time in order to improve type safety.
+  */
+trait HasNameEMT

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/EdgeTypes.java

@@ -0,0 +1,15 @@
+package flatgraph.testdomains.gratefuldead;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class EdgeTypes {
+
+
+public static final String followedBy = "followedBy";
+
+public static Set<String> ALL = new HashSet<String>() {{
+add(followedBy);
+}};
+
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/EdgeTypes.scala

@@ -0,0 +1,6 @@
+package flatgraph.testdomains.gratefuldead.edges
+
+class Followedby(src_4762: flatgraph.GNode, dst_4762: flatgraph.GNode, subSeq_4862: Int, property_4862: Any)
+    extends flatgraph.Edge(src_4762, dst_4762, 0.toShort, subSeq_4862, property_4862) {
+  def weight: Long = this.property.asInstanceOf[Long]
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/GraphSchema.scala

@@ -0,0 +1,68 @@
+package flatgraph.testdomains.gratefuldead
+import flatgraph.testdomains.gratefuldead.nodes
+import flatgraph.testdomains.gratefuldead.edges
+import flatgraph.FormalQtyType
+
+object GraphSchema extends flatgraph.Schema {
+  val nodeLabels = Array("Artist", "Song")
+  val nodeKindByLabel = nodeLabels.zipWithIndex.toMap
+  val edgeLabels = Array("followedBy")
+  val edgeIdByLabel = edgeLabels.zipWithIndex.toMap
+  val edgePropertyAllocators: Array[Int => Array[?]] =
+    Array(size => Array.fill(size)(0: Long) /* label = followedBy, id = 0 */ )
+  val nodeFactories: Array[(flatgraph.Graph, Int) => nodes.StoredNode] =
+    Array((g, seq) => new nodes.Artist(g, seq), (g, seq) => new nodes.Song(g, seq))
+  val edgeFactories: Array[(flatgraph.GNode, flatgraph.GNode, Int, Any) => flatgraph.Edge] =
+    Array((s, d, subseq, p) => new edges.Followedby(s, d, subseq, p))
+  val nodePropertyAllocators: Array[Int => Array[?]] = Array(size => new Array[String](size))
+  val normalNodePropertyNames = Array("Name")
+  val nodePropertyByLabel = normalNodePropertyNames.zipWithIndex.toMap
+  val nodePropertyDescriptors: Array[FormalQtyType.FormalQuantity | FormalQtyType.FormalType] = {
+    val nodePropertyDescriptors = new Array[FormalQtyType.FormalQuantity | FormalQtyType.FormalType](4)
+    for (idx <- Range(0, 4)) {
+      nodePropertyDescriptors(idx) =
+        if ((idx & 1) == 0) FormalQtyType.NothingType
+        else FormalQtyType.QtyNone
+    }
+
+    nodePropertyDescriptors(0) = FormalQtyType.StringType // Artist.Name
+    nodePropertyDescriptors(1) = FormalQtyType.QtyOne
+    nodePropertyDescriptors(2) = FormalQtyType.StringType // Song.Name
+    nodePropertyDescriptors(3) = FormalQtyType.QtyOne
+    nodePropertyDescriptors
+  }
+  override def getNumberOfNodeKinds: Int = 2
+  override def getNumberOfEdgeKinds: Int = 1
+  override def getNodeLabel(nodeKind: Int): String = nodeLabels(nodeKind)
+  override def getNodeKindByLabel(label: String): Int = nodeKindByLabel.getOrElse(label, flatgraph.Schema.UndefinedKind)
+  override def getEdgeLabel(nodeKind: Int, edgeKind: Int): String = edgeLabels(edgeKind)
+  override def getEdgeKindByLabel(label: String): Int = edgeIdByLabel.getOrElse(label, flatgraph.Schema.UndefinedKind)
+  override def getPropertyLabel(nodeKind: Int, propertyKind: Int): String = {
+    if (propertyKind < 1) normalNodePropertyNames(propertyKind)
+    else null
+  }
+
+  override def getPropertyKindByName(label: String): Int =
+    nodePropertyByLabel.getOrElse(label, flatgraph.Schema.UndefinedKind)
+  override def getNumberOfPropertyKinds: Int = 1
+  override def makeNode(graph: flatgraph.Graph, nodeKind: Short, seq: Int): nodes.StoredNode =
+    nodeFactories(nodeKind)(graph, seq)
+  override def makeEdge(
+      src: flatgraph.GNode,
+      dst: flatgraph.GNode,
+      edgeKind: Short,
+      subSeq: Int,
+      property: Any
+  ): flatgraph.Edge = edgeFactories(edgeKind)(src, dst, subSeq, property)
+  override def allocateEdgeProperty(
+      nodeKind: Int,
+      direction: flatgraph.Edge.Direction,
+      edgeKind: Int,
+      size: Int
+  ): Array[?] = edgePropertyAllocators(edgeKind)(size)
+  override def getNodePropertyFormalType(nodeKind: Int, propertyKind: Int): FormalQtyType.FormalType =
+    nodePropertyDescriptors(propertyOffsetArrayIndex(nodeKind, propertyKind)).asInstanceOf[FormalQtyType.FormalType]
+  override def getNodePropertyFormalQuantity(nodeKind: Int, propertyKind: Int): FormalQtyType.FormalQuantity =
+    nodePropertyDescriptors(1 + propertyOffsetArrayIndex(nodeKind, propertyKind))
+      .asInstanceOf[FormalQtyType.FormalQuantity]
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/GratefulDead.scala

@@ -0,0 +1,68 @@
+package flatgraph.testdomains.gratefuldead
+import flatgraph.DiffGraphBuilder
+import flatgraph.help.DocSearchPackages
+import flatgraph.help.Table.AvailableWidthProvider
+import Language.*
+
+object GratefulDead {
+  val defaultDocSearchPackage = DocSearchPackages.default.withAdditionalPackage(getClass.getPackage.getName)
+
+  @scala.annotation.implicitNotFound("""If you're using flatgraph purely without a schema and associated generated domain classes, you can
+    |start with `given DocSearchPackages = DocSearchPackages.default`.
+    |If you have generated domain classes, use `given DocSearchPackages = MyDomain.defaultDocSearchPackage`.
+    |If you have additional custom extension steps that specify help texts via @Doc annotations, use `given DocSearchPackages = MyDomain.defaultDocSearchPackage.withAdditionalPackage("my.custom.package)"`
+    |""".stripMargin)
+  def help(implicit searchPackageNames: DocSearchPackages, availableWidthProvider: AvailableWidthProvider) =
+    flatgraph.help.TraversalHelp(searchPackageNames).forTraversalSources(verbose = false)
+
+  @scala.annotation.implicitNotFound("""If you're using flatgraph purely without a schema and associated generated domain classes, you can
+    |start with `given DocSearchPackages = DocSearchPackages.default`.
+    |If you have generated domain classes, use `given DocSearchPackages = MyDomain.defaultDocSearchPackage`.
+    |If you have additional custom extension steps that specify help texts via @Doc annotations, use `given DocSearchPackages = MyDomain.defaultDocSearchPackage.withAdditionalPackage("my.custom.package)"`
+    |""".stripMargin)
+  def helpVerbose(implicit searchPackageNames: DocSearchPackages, availableWidthProvider: AvailableWidthProvider) =
+    flatgraph.help.TraversalHelp(searchPackageNames).forTraversalSources(verbose = true)
+
+  def empty: GratefulDead = new GratefulDead(new flatgraph.Graph(GraphSchema))
+
+  /** Instantiate a new graph with storage. If the file already exists, this will deserialize the given file into
+    * memory. `Graph.close` will serialise graph to that given file (and override whatever was there before), unless you
+    * specify `persistOnClose = false`.
+    */
+  def withStorage(storagePath: java.nio.file.Path, persistOnClose: Boolean = true): GratefulDead = {
+    val graph = flatgraph.Graph.withStorage(GraphSchema, storagePath, persistOnClose)
+    new GratefulDead(graph)
+  }
+
+  def newDiffGraphBuilder: DiffGraphBuilder = new DiffGraphBuilder(GraphSchema)
+}
+
+class GratefulDead(private val _graph: flatgraph.Graph = new flatgraph.Graph(GraphSchema)) extends AutoCloseable {
+  def graph: flatgraph.Graph = _graph
+
+  def help(implicit searchPackageNames: DocSearchPackages, availableWidthProvider: AvailableWidthProvider) =
+    GratefulDead.help
+  def helpVerbose(implicit searchPackageNames: DocSearchPackages, availableWidthProvider: AvailableWidthProvider) =
+    GratefulDead.helpVerbose
+
+  override def close(): Unit =
+    _graph.close()
+
+  override def toString(): String =
+    String.format("GratefulDead[%s]", graph)
+}
+
+@flatgraph.help.TraversalSource
+class GratefulDeadNodeStarters(val wrappedGratefulDead: GratefulDead) {
+
+  @flatgraph.help.Doc(info = "all nodes")
+  def all: Iterator[nodes.StoredNode] = wrappedGratefulDead.graph.allNodes.asInstanceOf[Iterator[nodes.StoredNode]]
+
+  /** */
+  @flatgraph.help.Doc(info = """""")
+  def artist: Iterator[nodes.Artist] = wrappedGratefulDead.graph._nodes(0).asInstanceOf[Iterator[nodes.Artist]]
+
+  /** */
+  @flatgraph.help.Doc(info = """""")
+  def song: Iterator[nodes.Song] = wrappedGratefulDead.graph._nodes(1).asInstanceOf[Iterator[nodes.Song]]
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/Language.scala

@@ -0,0 +1,15 @@
+package flatgraph.testdomains.gratefuldead
+
+/** combining all implicits into one trait that can be mixed in further downstream */
+trait Language
+    extends accessors.ConcreteStoredConversions
+    with traversals.ConcreteStoredConversions
+    with neighboraccessors.Conversions
+    with flatgraph.traversal.Language
+    with flatgraph.Implicits {
+  implicit def toGeneratedNodeStarters(domain: GratefulDead): GratefulDeadNodeStarters = GratefulDeadNodeStarters(
+    domain
+  )
+}
+
+object Language extends Language

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/NodeTypes.java

@@ -0,0 +1,19 @@
+package flatgraph.testdomains.gratefuldead;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class NodeTypes {
+
+
+public static final String Artist = "Artist";
+
+
+public static final String Song = "Song";
+
+public static Set<String> ALL = new HashSet<String>() {{
+add(Artist);
+add(Song);
+}};
+
+}

test-schemas-domain-classes/src/main/scala/flatgraph/testdomains/gratefuldead/PropertyKeys.scala

@@ -0,0 +1,9 @@
+package flatgraph.testdomains.gratefuldead
+
+import flatgraph.PropertyKey
+
+object PropertyKeys {
+
+  val Name = flatgraph.SinglePropertyKey[String](kind = 0, name = "Name", default = "")
+
+}