Updates to Scala stuff

Author: wzorgdrager

.gitignore

@@ -8,4 +8,5 @@ data/*
 .idea
 
 project/
-.DS_Store
+.DS_Store
+target/

lamdamart.py

@@ -48,6 +48,9 @@ class CandidateGenerator(dataset: Dataset, backgroundData: BackgroundData) {
     mutable.LinkedHashMap.empty
 
   def generateCandidates(synthetic: Int = 0) = {
+    candidateHash.clear()
+    synthethicCandidateHash.clear()
+
     println("Now generating candidates: ")
     val candidateFile = new File(
       candidate_file.replace(".txt", s"_${synthetic}.txt"))
@@ -115,7 +118,6 @@ class CandidateGenerator(dataset: Dataset, backgroundData: BackgroundData) {
 
     }
 
-    println(MRR.result())
     bufferedWriter.close()
   }
 

src/main/scala/CandidateGenerator.scala

@@ -1,4 +1,10 @@
 import scala.io.Source
+object Dataset {
+  val TEST = "test.txt"
+  val BACKGROUND = "background.txt"
+  val VALIDATION = "validation.txt"
+  val TRAINING = "training.xt"
+}
 class Dataset(val filename: String) {
   def queries = Source.fromFile(filename).getLines.drop(0)
 }

src/main/scala/Dataset.scala

@@ -1,3 +1,5 @@
+import java.io.BufferedWriter
+
 case class FeatureVec(freq: Int,
                       prefix: String,
                       lenPrefix: Int,
@@ -19,6 +21,8 @@ object Feature {
   private lazy val background = new BackgroundData()
   private var currentGroup = ""
   private var currentGroupId = 0
+  private var currentGroupSize = 0
+  var groupWriter: BufferedWriter = null
 
   def computeFeatureVec(candidate: Candidate): FeatureVec = {
     val freq =
@@ -51,8 +55,12 @@ object Feature {
 
   def writeFeature(feature: FeatureVec): String = {
     if (currentGroup != feature.prefix) {
+      if (currentGroupSize != 0) {
+        groupWriter.write(s"$currentGroupSize\n")
+      }
       currentGroup = feature.prefix
       currentGroupId += 1
+      currentGroupSize = 0
     }
     val builder = new StringBuilder()
     builder.append(feature.relevant + " ")

src/main/scala/Feature.scala

@@ -5,18 +5,25 @@ import scala.io.Source
 class FeatureGenerator(filename: String) {
 
   private val writeFilename = filename.replace(".txt", "_features.txt")
+  private val writeGroupname = filename.replace(".txt", "_groups.txt")
 
   def generateAndWriteFeatures(): Unit = {
     val featureFile = new File(writeFilename)
+    val groupFile = new File(writeGroupname)
 
     if (!featureFile.exists()) featureFile.createNewFile()
+    if (!groupFile.exists()) groupFile.createNewFile()
 
     var i = 0
     val size = candidates.size
 
     val bufferedWriter: BufferedWriter = new BufferedWriter(
       new FileWriter(writeFilename))
 
+    val groupWriter: BufferedWriter = new BufferedWriter(new FileWriter(writeGroupname))
+
+    Feature.groupWriter = groupWriter
+
     for (rawC <- candidates) {
       i += 1
       if (i % 100000 == 0) {
@@ -28,6 +35,8 @@ class FeatureGenerator(filename: String) {
       bufferedWriter.write(Feature.candidate2FeatureVec(rawC))
     }
 
+    groupWriter.flush()
+    groupWriter.close()
     bufferedWriter.flush()
     bufferedWriter.close()
 

src/main/scala/FeatureGenerator.scala

@@ -3,23 +3,50 @@ import cats.implicits._
 object Main {
 
   def main(args: Array[String]): Unit = {
-//    val candidateGenerator =
-//      new CandidateGenerator(new Dataset("validation.txt"),
-//                             new BackgroundData())
-//    candidateGenerator.generateCandidates(Synthetic.NO_SYNTHETIC)
-//
-//    val mpc = new MPC("validation_candidates_0.txt")
-//    mpc.computeMPC()
+    // Generate all candidates. Note: this takes quite a while.
+    genAllCandidates(Dataset.TEST)
+//    genAllCandidates(Dataset.VALIDATION)
+//    genAllCandidates(Dataset.TRAINING)
+
+
+    // Compute MPC for the test candidates WITHOUT synthetic candidates. We use this as baseline.
+    val mpc = new MPC("test_candidates_0.txt")
+    mpc.computeMPC()
+
+    // This probably takes a bit of time.
+    genAllFeatures(Dataset.TEST)
+//    genAllFeatures(Dataset.TRAINING)
+//    genAllFeatures(Dataset.VALIDATION)
+
+
 //
-//    val featureGenerator = new FeatureGenerator("validation_candidates_0.txt")
-//    featureGenerator.generateAndWriteFeatures()
 //    val lambda = LambdaMart.trainModel("train_candidates_0_features.txt",
 //                                       "validation_candidates_0_features.txt",
 //                                       "model.txt")
+//
+//    LambdaMart.evaluateModel("./model/xgb.model",
+//                             "test_candidates_0_features.txt")
+
+  }
+
+  def genAllCandidates(file: String): Unit = {
+    val candidateGenerator =
+      new CandidateGenerator(new Dataset(file),
+        new BackgroundData())
+    candidateGenerator.generateCandidates(Synthetic.NO_SYNTHETIC)
+    candidateGenerator.generateCandidates(Synthetic.SYNTHETIC_10K)
+    candidateGenerator.generateCandidates(Synthetic.SYNTHETIC_100K)
+  }
+
+  def genAllFeatures(file: String): Unit = {
+    val featureGenerator = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_0.txt")
+    featureGenerator.generateAndWriteFeatures()
 
-    LambdaMart.evaluateModel("./model/xgb.model",
-                             "test_candidates_0_features.txt")
+    val featureGenerator2 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_1.txt")
+    featureGenerator2.generateAndWriteFeatures()
 
+    val featureGenerator3 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_2.txt")
+    featureGenerator3.generateAndWriteFeatures()
   }
 
 }