Cleanups (#200)

* Light cleanups to prepare for overflowdb-only generator

* Upgrade CPG

* Use `KeyPool` from `codepropertygraph`

* Pass keyPool to all users instead of calling global `IdPool`

build.sbt

@@ -3,7 +3,7 @@ organization := "io.shiftleft"
 scalaVersion := "2.13.1"
 enablePlugins(GitVersioning)
 
-val cpgVersion = "0.11.276"
+val cpgVersion = "0.11.307"
 val antlrVersion = "4.7.2"
 
 libraryDependencies ++= Seq(

src/main/java/io/shiftleft/fuzzyc2cpg/ast/AstNode.java

@@ -1,6 +1,5 @@
 package io.shiftleft.fuzzyc2cpg.ast;
 
-import io.shiftleft.fuzzyc2cpg.ast.expressions.Expression;
 import io.shiftleft.fuzzyc2cpg.ast.walking.ASTNodeVisitor;
 
 import java.util.ArrayList;
@@ -119,10 +118,6 @@ public void setCodeStr(String aCodeStr) {
     code = aCodeStr;
   }
 
-  public String getLocationString() {
-    return this.location.toString();
-  }
-
   public CodeLocation getLocation() {
     return this.location;
   }
@@ -135,20 +130,9 @@ public String getTypeAsString() {
     return this.getClass().getSimpleName();
   }
 
-  public String getFullTypeName() {
-    return this.getClass().getName();
-  }
-
 
   /* special methods */
 
-  public String getOperatorCode() {
-    if (Expression.class.isAssignableFrom(this.getClass())) {
-      return ((Expression) this).getOperator();
-    }
-    return null;
-  }
-
   public void accept(ASTNodeVisitor visitor) {
     visitor.visit(this);
   }

src/main/java/io/shiftleft/fuzzyc2cpg/ast/declarations/ClassDefStatement.java

@@ -1,6 +1,5 @@
 package io.shiftleft.fuzzyc2cpg.ast.declarations;
 
-import io.shiftleft.fuzzyc2cpg.CommonParser;
 import io.shiftleft.fuzzyc2cpg.ast.AstNode;
 import io.shiftleft.fuzzyc2cpg.ast.DummyIdentifierNode;
 import io.shiftleft.fuzzyc2cpg.ast.expressions.Identifier;

src/main/java/io/shiftleft/fuzzyc2cpg/output/inmemory/OutputModule.java

@@ -23,9 +23,7 @@ public Cpg getInternalGraph() {
   }
 
   @Override
-  public void setOutputIdentifier(String identifier) {
-
-  }
+  public void setOutputIdentifier(String identifier) { }
 
   @Override
   public void persistCpg(CpgStruct.Builder cpg) {
@@ -35,10 +33,6 @@ public void persistCpg(CpgStruct.Builder cpg) {
   }
 
   public void persist() {
-    constructTinkerGraphFromCpg();
-  }
-
-  private void constructTinkerGraphFromCpg() {
     CpgStruct.Builder mergedBuilder = CpgStruct.newBuilder();
 
     cpgBuilders.forEach(builder -> {
@@ -48,7 +42,6 @@ private void constructTinkerGraphFromCpg() {
     List<CpgStruct> list = new LinkedList<>();
     list.add(mergedBuilder.build());
     cpg = ProtoCpgLoader.loadFromListOfProtos(list, OdbConfig.withoutOverflow());
-
   }
 
 }

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModule.java

@@ -17,12 +17,10 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 public class OutputModule implements CpgOutputModule {
-  private Logger logger = LoggerFactory.getLogger(getClass());
-  private static String ProtoSuffix = ".bin";
+  private final Logger logger = LoggerFactory.getLogger(getClass());
 
   private final Path protoTempDir;
-
-  private boolean writeToDisk;
+  private final boolean writeToDisk;
 
   private String outputIdentifier;
 
@@ -94,6 +92,7 @@ private String generateOutputFilename(int postfix) {
     hasher.putUnencodedChars(outputIdentifier);
     hasher.putInt(postfix);
 
-    return protoTempDir.toString() + File.separator + hasher.hash() + ProtoSuffix;
+    String protoSuffix = ".bin";
+    return protoTempDir.toString() + File.separator + hasher.hash() + protoSuffix;
   }
 }

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModuleFactory.java

@@ -2,7 +2,6 @@
 
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;

src/main/java/io/shiftleft/fuzzyc2cpg/parser/AntlrParserDriver.java

@@ -4,13 +4,13 @@
 import io.shiftleft.fuzzyc2cpg.ast.AstNode;
 import io.shiftleft.fuzzyc2cpg.ast.AstNodeBuilder;
 import io.shiftleft.fuzzyc2cpg.ast.logical.statements.CompoundStatement;
+import io.shiftleft.passes.KeyPool;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Stack;
 import java.util.function.Consumer;
 
-import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
 import io.shiftleft.proto.cpg.Cpg;
 import jdk.nashorn.internal.runtime.ParserException;
 import org.antlr.v4.runtime.*;
@@ -41,6 +41,7 @@ abstract public class AntlrParserDriver {
   private List<AntlrParserDriverObserver> observers = new ArrayList<>();
   private Cpg.CpgStruct.Builder cpg;
   private Cpg.CpgStruct.Node fileNode;
+  private KeyPool keyPool;
 
   public AntlrParserDriver() {
     super();
@@ -50,6 +51,10 @@ public void setCpg(Cpg.CpgStruct.Builder cpg) {
     this.cpg = cpg;
   }
 
+  public void setKeyPool(KeyPool keyPool) {
+    this.keyPool = keyPool;
+  }
+
   public void setFileNode(Cpg.CpgStruct.Node fileNode) {
     this.fileNode = fileNode;
   }
@@ -84,6 +89,7 @@ private void handleHiddenTokens(String filename) {
       // We can add to `CPG` here
 
       Cpg.CpgStruct.Node commentNode = Utils.newNode(Cpg.CpgStruct.Node.NodeType.COMMENT)
+              .setKey(keyPool.next())
               .addProperty(Cpg.CpgStruct.Node.Property.newBuilder()
                       .setName(Cpg.NodePropertyName.LINE_NUMBER)
                       .setValue(Cpg.PropertyValue.newBuilder().setIntValue(line)))

src/main/java/io/shiftleft/fuzzyc2cpg/parser/modules/CModuleParserTreeListener.java

@@ -1,10 +1,7 @@
 package io.shiftleft.fuzzyc2cpg.parser.modules;
 
-import java.util.Iterator;
 import java.util.List;
 
-import io.shiftleft.fuzzyc2cpg.ast.expressions.Identifier;
-import io.shiftleft.fuzzyc2cpg.parser.AstNodeFactory;
 import org.antlr.v4.runtime.ParserRuleContext;
 
 import io.shiftleft.fuzzyc2cpg.ModuleBaseListener;

src/main/scala/io/shiftleft/fuzzyc2cpg/AstVisitor.scala

@@ -12,11 +12,15 @@ import io.shiftleft.fuzzyc2cpg.astnew.AstToCpgConverter
 import io.shiftleft.fuzzyc2cpg.cfg.AstToCfgConverter
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory
 import io.shiftleft.fuzzyc2cpg.parser.AntlrParserDriverObserver
+import io.shiftleft.passes.KeyPool
 import io.shiftleft.proto.cpg.Cpg.CpgStruct
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node
 import org.antlr.v4.runtime.ParserRuleContext
 
-class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgStruct.Builder, astParentNode: Node)
+class AstVisitor(outputModuleFactory: CpgOutputModuleFactory,
+                 structureCpg: CpgStruct.Builder,
+                 astParentNode: Node,
+                 keyPool: KeyPool)
     extends ASTNodeVisitor
     with AntlrParserDriverObserver {
   private var fileNameOption = Option.empty[String]
@@ -31,7 +35,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
     outputModule.setOutputIdentifier(outputIdentifier)
 
     val bodyCpg = CpgStruct.newBuilder()
-    val cpgAdapter = new ProtoCpgAdapter(bodyCpg)
+    val cpgAdapter = new ProtoCpgAdapter(bodyCpg, keyPool)
     val astToCpgConverter =
       new AstToCpgConverter(astParentNode, cpgAdapter)
     astToCpgConverter.convert(functionDef)
@@ -56,7 +60,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
     * Callback triggered for every class/struct
     * */
   override def visit(classDefStatement: ClassDefStatement): Unit = {
-    val cpgAdapter = new ProtoCpgAdapter(structureCpg)
+    val cpgAdapter = new ProtoCpgAdapter(structureCpg, keyPool)
     val astToCpgConverter =
       new AstToCpgConverter(astParentNode, cpgAdapter)
     astToCpgConverter.convert(classDefStatement)
@@ -66,7 +70,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
     * Callback triggered for every global identifier declaration
     * */
   override def visit(identifierDeclStmt: IdentifierDeclStatement): Unit = {
-    val cpgAdapter = new ProtoCpgAdapter(structureCpg)
+    val cpgAdapter = new ProtoCpgAdapter(structureCpg, keyPool)
     val astToCpgConverter =
       new AstToCpgConverter(astParentNode, cpgAdapter)
     astToCpgConverter.convert(identifierDeclStmt)