Skip to content

Commit e11e088

Browse files
ml86ml86
committed
Deprecated SerializedCpg APIs.
This PR does not change behaviour. The serialization feature was internally not doing anything anymore for quite a while. Time to deprecated the API to eventually removed them.
1 parent 88d9845 commit e11e088

File tree

3 files changed

+45
-59
lines changed

3 files changed

+45
-59
lines changed

‎codepropertygraph/src/main/scala/io/shiftleft/SerializedCpg.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import java.net.{URI, URISyntaxException}
77
import java.nio.file.{FileSystem, FileSystems, Files}
88
import java.util
99

10+
@deprecated("New API functions do not use this class anymore")
1011
class SerializedCpg extends AutoCloseable {
1112

1213
/** We allow creating a dummy serialized CPG that does not do anything.

‎codepropertygraph/src/main/scala/io/shiftleft/passes/CpgPass.scala

Lines changed: 42 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,43 @@ abstract class ForkJoinParallelCpgPass[T <: AnyRef](cpg: Cpg, @nowarn outName: S
6262
// Override this to disable parallelism of passes. Useful for debugging.
6363
def isParallel: Boolean = true
6464

65-
override def createAndApply(): Unit = createApplySerializeAndStore(null)
65+
override def createAndApply(): Unit = {
66+
baseLogger.info(s"Start of pass: $name")
67+
val nanosStart = System.nanoTime()
68+
var nParts = 0
69+
var nanosBuilt = -1L
70+
var nDiff = -1
71+
var nDiffT = -1
72+
try {
73+
val diffGraph = Cpg.newDiffGraphBuilder
74+
nParts = runWithBuilder(diffGraph)
75+
nanosBuilt = System.nanoTime()
76+
nDiff = diffGraph.size
77+
78+
// TODO how about `nDiffT` which seems to count the number of modifications..
79+
// nDiffT = overflowdb.BatchedUpdate
80+
// .applyDiff(cpg.graph, diffGraph, null)
81+
// .transitiveModifications()
82+
83+
flatgraph.DiffGraphApplier.applyDiff(cpg.graph, diffGraph)
84+
} catch {
85+
case exc: Exception =>
86+
baseLogger.error(s"Pass ${name} failed", exc)
87+
throw exc
88+
} finally {
89+
try {
90+
finish()
91+
} finally {
92+
// the nested finally is somewhat ugly -- but we promised to clean up with finish(), we want to include finish()
93+
// in the reported timings, and we must have our final log message if finish() throws
94+
val nanosStop = System.nanoTime()
95+
val fracRun = if (nanosBuilt == -1) 0.0 else (nanosStop - nanosBuilt) * 100.0 / (nanosStop - nanosStart + 1)
96+
baseLogger.info(
97+
f"Pass $name completed in ${(nanosStop - nanosStart) * 1e-6}%.0f ms (${fracRun}%.0f%% on mutations). ${nDiff}%d + ${nDiffT - nDiff}%d changes committed from ${nParts}%d parts."
98+
)
99+
}
100+
}
101+
}
66102

67103
override def runWithBuilder(externalBuilder: DiffGraphBuilder): Int = {
68104
try {
@@ -105,45 +141,9 @@ abstract class ForkJoinParallelCpgPass[T <: AnyRef](cpg: Cpg, @nowarn outName: S
105141
}
106142
}
107143

144+
@deprecated("Please use createAndApply")
108145
override def createApplySerializeAndStore(serializedCpg: SerializedCpg, prefix: String = ""): Unit = {
109-
baseLogger.info(s"Start of pass: $name")
110-
val nanosStart = System.nanoTime()
111-
var nParts = 0
112-
var nanosBuilt = -1L
113-
var nDiff = -1
114-
var nDiffT = -1
115-
try {
116-
val diffGraph = Cpg.newDiffGraphBuilder
117-
nParts = runWithBuilder(diffGraph)
118-
nanosBuilt = System.nanoTime()
119-
nDiff = diffGraph.size
120-
121-
// TODO how about `nDiffT` which seems to count the number of modifications..
122-
// nDiffT = overflowdb.BatchedUpdate
123-
// .applyDiff(cpg.graph, diffGraph, null)
124-
// .transitiveModifications()
125-
126-
flatgraph.DiffGraphApplier.applyDiff(cpg.graph, diffGraph)
127-
} catch {
128-
case exc: Exception =>
129-
baseLogger.error(s"Pass ${name} failed", exc)
130-
throw exc
131-
} finally {
132-
try {
133-
finish()
134-
} finally {
135-
// the nested finally is somewhat ugly -- but we promised to clean up with finish(), we want to include finish()
136-
// in the reported timings, and we must have our final log message if finish() throws
137-
val nanosStop = System.nanoTime()
138-
val fracRun = if (nanosBuilt == -1) 0.0 else (nanosStop - nanosBuilt) * 100.0 / (nanosStop - nanosStart + 1)
139-
val serializationString = if (serializedCpg != null && !serializedCpg.isEmpty) {
140-
" Diff serialized and stored."
141-
} else ""
142-
baseLogger.info(
143-
f"Pass $name completed in ${(nanosStop - nanosStart) * 1e-6}%.0f ms (${fracRun}%.0f%% on mutations). ${nDiff}%d + ${nDiffT - nDiff}%d changes committed from ${nParts}%d parts.${serializationString}%s"
144-
)
145-
}
146-
}
146+
createAndApply()
147147
}
148148

149149
}
@@ -154,6 +154,7 @@ trait CpgPassBase {
154154

155155
def createAndApply(): Unit
156156

157+
@deprecated("Please use createAndApply")
157158
def createApplySerializeAndStore(serializedCpg: SerializedCpg, prefix: String = ""): Unit
158159

159160
/** Name of the pass. By default it is inferred from the name of the class, override if needed.
@@ -196,11 +197,8 @@ trait CpgPassBase {
196197
prefix + "_" + outputName + "_" + index
197198
}
198199

199-
protected def store(overlay: GeneratedMessageV3, name: String, serializedCpg: SerializedCpg): Unit = {
200-
if (overlay.getSerializedSize > 0) {
201-
serializedCpg.addOverlay(overlay, name)
202-
}
203-
}
200+
@deprecated
201+
protected def store(overlay: GeneratedMessageV3, name: String, serializedCpg: SerializedCpg): Unit = {}
204202

205203
protected def withStartEndTimesLogged[A](fun: => A): A = {
206204
baseLogger.info(s"Running pass: $name")

‎codepropertygraph/src/test/scala/io/shiftleft/passes/CpgPassNewTests.scala

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package io.shiftleft.passes
22

33
import better.files.File
44
import flatgraph.SchemaViolationException
5-
import io.shiftleft.SerializedCpg
65
import io.shiftleft.codepropertygraph.generated.Cpg
76
import io.shiftleft.codepropertygraph.generated.nodes.NewFile
87
import io.shiftleft.codepropertygraph.generated.language.*
@@ -16,7 +15,7 @@ class CpgPassNewTests extends AnyWordSpec with Matchers {
1615
private object Fixture {
1716
def apply(f: (Cpg, CpgPassBase) => Unit): Unit = {
1817
val cpg = Cpg.empty
19-
class MyPass(cpg: Cpg) extends SimpleCpgPass(cpg, "MyPass") {
18+
class MyPass(cpg: Cpg) extends CpgPass(cpg, "MyPass") {
2019
override def run(builder: DiffGraphBuilder): Unit = {
2120
val builder2 = Cpg.newDiffGraphBuilder
2221
builder.addNode(NewFile().name("foo"))
@@ -35,21 +34,9 @@ class CpgPassNewTests extends AnyWordSpec with Matchers {
3534
cpg.all.label.toSet shouldBe Set("FILE")
3635
}
3736

38-
"produce a serialized CPG file" in Fixture { (_, pass) =>
39-
File.usingTemporaryFile("pass", ".zip") { file =>
40-
file.delete()
41-
val filename = file.path.toString
42-
val serializedCpg = new SerializedCpg(filename)
43-
pass.createApplySerializeAndStore(serializedCpg)
44-
serializedCpg.close()
45-
file.exists shouldBe true
46-
Files.size(file.path) should not be 0
47-
}
48-
}
49-
5037
"fail for schema violations" in {
5138
val cpg = Cpg.empty
52-
val pass = new SimpleCpgPass(cpg, "pass1") {
39+
val pass = new CpgPass(cpg, "pass1") {
5340
override def run(dst: DiffGraphBuilder): Unit = {
5441
val file1 = NewFile().name("foo")
5542
val file2 = NewFile().name("bar")

0 commit comments

Comments
 (0)