Ignoring .group files

Author: wzorgdrager

.gitignore

@@ -9,4 +9,5 @@ data/*
 
 project/
 .DS_Store
-target/
+target/
+*.group

src/main/scala/CandidateGenerator.scala

@@ -47,14 +47,20 @@ class CandidateGenerator(dataset: Dataset, backgroundData: BackgroundData) {
     : mutable.LinkedHashMap[String, List[String]] =
     mutable.LinkedHashMap.empty
 
-  def generateCandidates(synthetic: Int = 0) = {
+  def generateCandidates(synthetic: Int = 0): Unit = {
     candidateHash.clear()
     synthethicCandidateHash.clear()
 
-    println("Now generating candidates: ")
+    println(s"Now generating candidates for ${dataset.filename}, synthetic: $synthetic: ")
     val candidateFile = new File(
       candidate_file.replace(".txt", s"_${synthetic}.txt"))
-    if (!candidateFile.exists()) candidateFile.createNewFile()
+    if (!candidateFile.exists()) {
+      candidateFile.createNewFile()
+    }
+    else {
+      println(s"Candidates already exists in ${candidateFile.getName}. Exiting.")
+      return
+    }
 
     val bufferedWriter: BufferedWriter = new BufferedWriter(
       new FileWriter(candidateFile))

src/main/scala/Dataset.scala

@@ -3,7 +3,7 @@ object Dataset {
   val TEST = "test.txt"
   val BACKGROUND = "background.txt"
   val VALIDATION = "validation.txt"
-  val TRAINING = "training.xt"
+  val TRAINING = "train.txt"
 }
 class Dataset(val filename: String) {
   def queries = Source.fromFile(filename).getLines.drop(0)

src/main/scala/Feature.scala

@@ -62,6 +62,8 @@ object Feature {
       currentGroupId += 1
       currentGroupSize = 0
     }
+
+    currentGroupSize += 1
     val builder = new StringBuilder()
     builder.append(feature.relevant + " ")
     builder.append(s"qid:${currentGroupId} ")

src/main/scala/FeatureGenerator.scala

@@ -5,13 +5,19 @@ import scala.io.Source
 class FeatureGenerator(filename: String) {
 
   private val writeFilename = filename.replace(".txt", "_features.txt")
-  private val writeGroupname = filename.replace(".txt", "_groups.txt")
+  private val writeGroupname = filename.replace(".txt", "_features.txt.group")
 
   def generateAndWriteFeatures(): Unit = {
     val featureFile = new File(writeFilename)
     val groupFile = new File(writeGroupname)
 
-    if (!featureFile.exists()) featureFile.createNewFile()
+    println("Now generating features.")
+    if (!featureFile.exists()) {
+      featureFile.createNewFile()
+    } else {
+      println(s"Features already exist for ${featureFile.getName}.")
+      return
+    }
     if (!groupFile.exists()) groupFile.createNewFile()
 
     var i = 0

src/main/scala/LambdaMart.scala

@@ -3,6 +3,7 @@ import java.io.File
 import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost}
 
 import scala.collection.mutable
+import scala.io.Source
 
 object LambdaMart {
 
@@ -13,7 +14,6 @@ object LambdaMart {
     val validateM = new DMatrix(validationFile)
 
     val watches = new mutable.HashMap[String, DMatrix]
-    watches += "train" -> trainM
     watches += "validation" -> validateM
 
     val params = new mutable.HashMap[String, Any]()
@@ -26,6 +26,7 @@ object LambdaMart {
 
     val round = 4
     // train a model
+    println("Now training the model!")
     val booster = XGBoost.train(trainM, params.toMap, round, watches.toMap)
 
     // predict
@@ -34,11 +35,8 @@ object LambdaMart {
     if (!file.exists()) {
       file.mkdirs()
     }
-    booster.saveModel(file.getAbsolutePath + "/xgb.model")
-    // dump model with feature map
-    val modelInfos =
-      booster.getModelDump(file.getAbsolutePath + "/featmap.txt", false)
 
+    booster.saveModel(file.getAbsolutePath + s"/$modelName")
   }
 
   def evaluateModel(pathModel: String, pathTest: String) = {

src/main/scala/Main.scala

@@ -4,26 +4,26 @@ object Main {
 
   def main(args: Array[String]): Unit = {
     // Generate all candidates. Note: this takes quite a while.
-    genAllCandidates(Dataset.TEST)
-//    genAllCandidates(Dataset.VALIDATION)
-//    genAllCandidates(Dataset.TRAINING)
+//   genAllCandidates(Dataset.TEST)
+//     genAllCandidates(Dataset.VALIDATION)
+//     genAllCandidates(Dataset.TRAINING)
 
 
     // Compute MPC for the test candidates WITHOUT synthetic candidates. We use this as baseline.
-    val mpc = new MPC("test_candidates_0.txt")
-    mpc.computeMPC()
+//    val mpc = new MPC("test_candidates_0.txt")
+//    mpc.computeMPC()
 
     // This probably takes a bit of time.
-    genAllFeatures(Dataset.TEST)
+    // genAllFeatures(Dataset.TEST)
 //    genAllFeatures(Dataset.TRAINING)
 //    genAllFeatures(Dataset.VALIDATION)
 
 
-//
-//    val lambda = LambdaMart.trainModel("train_candidates_0_features.txt",
-//                                       "validation_candidates_0_features.txt",
-//                                       "model.txt")
-//
+
+    val lambda = LambdaMart.trainModel("train_candidates_0_features.txt",
+                                       "validation_candidates_0_features.txt",
+                                       "no_ngrams_0_synthetic")
+
 //    LambdaMart.evaluateModel("./model/xgb.model",
 //                             "test_candidates_0_features.txt")
 
@@ -42,11 +42,11 @@ object Main {
     val featureGenerator = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_0.txt")
     featureGenerator.generateAndWriteFeatures()
 
-    val featureGenerator2 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_1.txt")
-    featureGenerator2.generateAndWriteFeatures()
-
-    val featureGenerator3 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_2.txt")
-    featureGenerator3.generateAndWriteFeatures()
+//    val featureGenerator2 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_1.txt")
+//    featureGenerator2.generateAndWriteFeatures()
+//
+//    val featureGenerator3 = new FeatureGenerator(s"${file.replace(".txt", "")}_candidates_2.txt")
+//    featureGenerator3.generateAndWriteFeatures()
   }
 
 }